{"name": "AI crawler access on the top 10,000 sites", "date": "2026-10-19", "started_at": "2026-10-19T01:42:21Z", "finished_at": "2026-10-19T01:59:07Z", "list": {"name": "Majestic Million", "url": "https://downloads.majestic.com/majestic_million.csv", "license": "CC BY 3.0, Majestic-12 Ltd", "last_modified": "Sun, 18 Oct 2026 05:00:28 GMT", "top": 10000}, "crawler": {"user_agent": "ContextFetchBot/1.0 (+https://contextfetch.dev/bot)", "token": "ContextFetchBot"}, "bots": [{"token": "GPTBot", "operator": "OpenAI"}, {"token": "ChatGPT-User", "operator": "OpenAI"}, {"token": "OAI-SearchBot", "operator": "OpenAI"}, {"token": "ClaudeBot", "operator": "Anthropic"}, {"token": "Claude-User", "operator": "Anthropic"}, {"token": "Claude-SearchBot", "operator": "Anthropic"}, {"token": "anthropic-ai", "operator": "Anthropic"}, {"token": "PerplexityBot", "operator": "Perplexity"}, {"token": "Perplexity-User", "operator": "Perplexity"}, {"token": "Google-Extended", "operator": "Google"}, {"token": "Applebot-Extended", "operator": "Apple"}, {"token": "CCBot", "operator": "Common Crawl"}, {"token": "Bytespider", "operator": "ByteDance"}, {"token": "Amazonbot", "operator": "Amazon"}, {"token": "meta-externalagent", "operator": "Meta"}, {"token": "cohere-ai", "operator": "Cohere"}, {"token": "Diffbot", "operator": "Diffbot"}, {"token": "ContextFetchBot", "operator": "ContextFetch"}], "method": {"robots": "RFC 9309 (data/ai-crawlers/robots.py); 404/410 = no robots.txt, everything allowed; an HTML page with a 2xx, other 4xx (refused) and 5xx = rules not readable, no page fetched", "origins": "https://<domain>, then https://www.<domain>, then http://<domain>, each when the one before it was unreachable or timed out", "verdicts": "allowed: no path disallowed; blocked: no path allowed; partial: some paths disallowed", "llms_txt": "present: 2xx, not HTML, starts with '#'; fetched only where robots.txt allows ContextFetchBot", "tokens": "ceil(chars / 4) (engine count_tokens): html = the homepage's decoded HTML, markdown = scraper.page_to_result, pruned = neural.prune_markdown_for_llm(query, markdown, top_k=3, threshold=0.1)", "prune_query": "What is {domain}?", "timeouts_s": {"robots": 10.0, "llms_txt": 10.0, "homepage": 15.0}, "host_delay_s": 1.0, "max_crawl_delay_s": 60.0, "concurrency": 48, "max_event_loop_stall_s": 0.64}, "machine": {"platform": "macOS-27.0.1-arm64-arm-64bit-Mach-O", "cpu": "arm", "logical_cpus": 8, "python": "3.13.3", "httpx": "0.28.1", "trafilatura": "2.3.1", "selectolax": "1.0.0", "fastembed": "0.9.0", "onnxruntime": "1.31.0"}, "command": "python data/ai-crawlers/build_dataset.py --top 10000", "sites": [
{"rank":1,"domain":"google.com","origin":"https://google.com","robots":{"status":"ok","http":200,"url":"https://www.google.com/robots.txt","bytes":6624,"groups":4},"bots":{"GPTBot":["partial","*",169,64],"ChatGPT-User":["partial","*",169,64],"OAI-SearchBot":["partial","*",169,64],"ClaudeBot":["partial","*",169,64],"Claude-User":["partial","*",169,64],"Claude-SearchBot":["partial","*",169,64],"anthropic-ai":["partial","*",169,64],"PerplexityBot":["partial","*",169,64],"Perplexity-User":["partial","*",169,64],"Google-Extended":["partial","*",169,64],"Applebot-Extended":["partial","*",169,64],"CCBot":["partial","*",169,64],"Bytespider":["partial","*",169,64],"Amazonbot":["partial","*",169,64],"meta-externalagent":["partial","*",169,64],"cohere-ai":["partial","*",169,64],"Diffbot":["partial","*",169,64],"ContextFetchBot":["partial","*",169,64]},"llms":{"status":"missing","http":404,"bytes":1569},"home":{"status":"ok","http":200,"url":"https://www.google.com/","html_tokens":21449,"markdown_tokens":30,"pruned_tokens":30}},
{"rank":2,"domain":"facebook.com","origin":"https://facebook.com","robots":{"status":"ok","http":200,"url":"https://facebook.com/robots.txt","bytes":26872,"groups":36},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["blocked","*",1,0],"OAI-SearchBot":["blocked","*",1,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["blocked","*",1,0],"Claude-SearchBot":["blocked","*",1,0],"anthropic-ai":["blocked","*",1,0],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["blocked","*",1,0],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","*",1,0],"Bytespider":["blocked","*",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","*",1,0],"cohere-ai":["blocked","*",1,0],"Diffbot":["blocked","*",1,0],"ContextFetchBot":["blocked","*",1,0]},"llms":{"status":"disallowed"},"home":{"status":"disallowed"}},
{"rank":3,"domain":"youtube.com","origin":"https://youtube.com","robots":{"status":"ok","http":200,"url":"https://www.youtube.com/robots.txt","bytes":792,"groups":2},"bots":{"GPTBot":["partial","*",21,0],"ChatGPT-User":["partial","*",21,0],"OAI-SearchBot":["partial","*",21,0],"ClaudeBot":["partial","*",21,0],"Claude-User":["partial","*",21,0],"Claude-SearchBot":["partial","*",21,0],"anthropic-ai":["partial","*",21,0],"PerplexityBot":["partial","*",21,0],"Perplexity-User":["partial","*",21,0],"Google-Extended":["partial","*",21,0],"Applebot-Extended":["partial","*",21,0],"CCBot":["partial","*",21,0],"Bytespider":["partial","*",21,0],"Amazonbot":["partial","*",21,0],"meta-externalagent":["partial","*",21,0],"cohere-ai":["partial","*",21,0],"Diffbot":["partial","*",21,0],"ContextFetchBot":["partial","*",21,0]},"llms":{"status":"missing","http":404,"bytes":755},"home":{"status":"ok","http":200,"url":"https://www.youtube.com/","html_tokens":237731,"markdown_tokens":41,"pruned_tokens":41}},
{"rank":4,"domain":"instagram.com","origin":"https://instagram.com","robots":{"status":"ok","http":200,"url":"https://www.instagram.com/robots.txt","bytes":6184,"groups":36},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["blocked","*",1,0],"OAI-SearchBot":["blocked","*",1,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["blocked","*",1,0],"Claude-SearchBot":["blocked","*",1,0],"anthropic-ai":["blocked","*",1,0],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["blocked","*",1,0],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","*",1,0],"Bytespider":["blocked","*",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","*",1,0],"cohere-ai":["blocked","*",1,0],"Diffbot":["blocked","*",1,0],"ContextFetchBot":["blocked","*",1,0]},"llms":{"status":"disallowed"},"home":{"status":"disallowed"}},
{"rank":5,"domain":"twitter.com","origin":"https://twitter.com","robots":{"status":"ok","http":200,"url":"https://twitter.com/robots.txt","bytes":2678,"groups":10},"bots":{"GPTBot":["blocked","*",2,0],"ChatGPT-User":["blocked","*",2,0],"OAI-SearchBot":["blocked","*",2,0],"ClaudeBot":["blocked","*",2,0],"Claude-User":["blocked","*",2,0],"Claude-SearchBot":["blocked","*",2,0],"anthropic-ai":["blocked","*",2,0],"PerplexityBot":["blocked","*",2,0],"Perplexity-User":["blocked","*",2,0],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","*",2,0],"CCBot":["blocked","*",2,0],"Bytespider":["blocked","*",2,0],"Amazonbot":["blocked","*",2,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","*",2,0],"Diffbot":["blocked","*",2,0],"ContextFetchBot":["blocked","*",2,0]},"llms":{"status":"disallowed"},"home":{"status":"disallowed"}},
{"rank":6,"domain":"linkedin.com","origin":"https://linkedin.com","robots":{"status":"ok","http":200,"url":"https://www.linkedin.com/robots.txt","bytes":120190,"groups":77},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["blocked","named",1,0],"OAI-SearchBot":["partial","named",111,9],"ClaudeBot":["blocked","named",1,0],"Claude-User":["blocked","named",1,0],"Claude-SearchBot":["partial","named",111,9],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["blocked","named",1,0],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","*",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","*",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["blocked","*",1,0]},"llms":{"status":"disallowed"},"home":{"status":"disallowed"}},
{"rank":7,"domain":"apple.com","origin":"https://apple.com","robots":{"status":"ok","http":200,"url":"https://www.apple.com/robots.txt","bytes":1018,"groups":6},"bots":{"GPTBot":["partial","*",4,1],"ChatGPT-User":["partial","*",4,1],"OAI-SearchBot":["partial","*",4,1],"ClaudeBot":["partial","*",4,1],"Claude-User":["partial","*",4,1],"Claude-SearchBot":["partial","*",4,1],"anthropic-ai":["partial","*",4,1],"PerplexityBot":["partial","*",4,1],"Perplexity-User":["partial","*",4,1],"Google-Extended":["partial","*",4,1],"Applebot-Extended":["partial","*",4,1],"CCBot":["partial","*",4,1],"Bytespider":["partial","*",4,1],"Amazonbot":["partial","*",4,1],"meta-externalagent":["partial","*",4,1],"cohere-ai":["partial","*",4,1],"Diffbot":["partial","*",4,1],"ContextFetchBot":["partial","*",4,1]},"llms":{"status":"missing","http":404,"bytes":109400},"home":{"status":"ok","http":200,"url":"https://www.apple.com/","html_tokens":81664,"markdown_tokens":27,"pruned_tokens":27}},
{"rank":8,"domain":"googletagmanager.com","origin":"https://googletagmanager.com","robots":{"status":"missing","http":404,"url":"https://googletagmanager.com/robots.txt","bytes":1571,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":1569},"home":{"status":"http_error","http":404,"url":"https://googletagmanager.com/"}},
{"rank":9,"domain":"microsoft.com","origin":"https://microsoft.com","robots":{"status":"ok","http":200,"url":"https://www.microsoft.com/robots.txt","bytes":9126,"groups":1},"bots":{"GPTBot":["partial","*",154,47],"ChatGPT-User":["partial","*",154,47],"OAI-SearchBot":["partial","*",154,47],"ClaudeBot":["partial","*",154,47],"Claude-User":["partial","*",154,47],"Claude-SearchBot":["partial","*",154,47],"anthropic-ai":["partial","*",154,47],"PerplexityBot":["partial","*",154,47],"Perplexity-User":["partial","*",154,47],"Google-Extended":["partial","*",154,47],"Applebot-Extended":["partial","*",154,47],"CCBot":["partial","*",154,47],"Bytespider":["partial","*",154,47],"Amazonbot":["partial","*",154,47],"meta-externalagent":["partial","*",154,47],"cohere-ai":["partial","*",154,47],"Diffbot":["partial","*",154,47],"ContextFetchBot":["partial","*",154,47]},"llms":{"status":"missing","http":404,"bytes":76859},"home":{"status":"ok","http":200,"url":"https://www.microsoft.com/en-us","html_tokens":70880,"markdown_tokens":835,"pruned_tokens":48}},
{"rank":10,"domain":"github.com","origin":"https://github.com","robots":{"status":"ok","http":200,"url":"https://github.com/robots.txt","bytes":14489,"groups":5},"bots":{"GPTBot":["partial","named",162,3],"ChatGPT-User":["partial","*",162,3],"OAI-SearchBot":["partial","named",162,3],"ClaudeBot":["partial","named",162,3],"Claude-User":["partial","*",162,3],"Claude-SearchBot":["partial","*",162,3],"anthropic-ai":["partial","named",162,3],"PerplexityBot":["partial","named",162,3],"Perplexity-User":["partial","*",162,3],"Google-Extended":["partial","*",162,3],"Applebot-Extended":["partial","*",162,3],"CCBot":["partial","*",162,3],"Bytespider":["blocked","named",1,0],"Amazonbot":["partial","*",162,3],"meta-externalagent":["partial","*",162,3],"cohere-ai":["partial","*",162,3],"Diffbot":["partial","*",162,3],"ContextFetchBot":["partial","*",162,3]},"llms":{"status":"present","http":200,"bytes":28658},"home":{"status":"ok","http":200,"url":"https://github.com/","html_tokens":143069,"markdown_tokens":173,"pruned_tokens":140}},
{"rank":11,"domain":"w2rt6byl5nf1opmc-a.com","origin":"https://w2rt6byl5nf1opmc-a.com","robots":{"status":"unreachable","http":null,"url":"https://w2rt6byl5nf1opmc-a.com/robots.txt","bytes":0,"groups":0,"error":"cannot resolve w2rt6byl5nf1opmc-a.com"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":12,"domain":"wikipedia.org","origin":"https://wikipedia.org","robots":{"status":"ok","http":200,"url":"https://en.wikipedia.org/robots.txt","bytes":28275,"groups":33},"bots":{"GPTBot":["partial","*",428,4],"ChatGPT-User":["partial","*",428,4],"OAI-SearchBot":["partial","*",428,4],"ClaudeBot":["partial","*",428,4],"Claude-User":["partial","*",428,4],"Claude-SearchBot":["partial","*",428,4],"anthropic-ai":["partial","*",428,4],"PerplexityBot":["partial","*",428,4],"Perplexity-User":["partial","*",428,4],"Google-Extended":["partial","*",428,4],"Applebot-Extended":["partial","*",428,4],"CCBot":["partial","*",428,4],"Bytespider":["partial","*",428,4],"Amazonbot":["partial","*",428,4],"meta-externalagent":["partial","*",428,4],"cohere-ai":["partial","*",428,4],"Diffbot":["partial","*",428,4],"ContextFetchBot":["partial","*",428,4]},"llms":{"status":"missing","http":404,"bytes":1868},"home":{"status":"ok","http":200,"url":"https://www.wikipedia.org/","html_tokens":30020,"markdown_tokens":665,"pruned_tokens":298}},
{"rank":13,"domain":"play.google.com","origin":"https://play.google.com","robots":{"status":"ok","http":200,"url":"https://play.google.com/robots.txt","bytes":3199,"groups":1},"bots":{"GPTBot":["partial","*",97,6],"ChatGPT-User":["partial","*",97,6],"OAI-SearchBot":["partial","*",97,6],"ClaudeBot":["partial","*",97,6],"Claude-User":["partial","*",97,6],"Claude-SearchBot":["partial","*",97,6],"anthropic-ai":["partial","*",97,6],"PerplexityBot":["partial","*",97,6],"Perplexity-User":["partial","*",97,6],"Google-Extended":["partial","*",97,6],"Applebot-Extended":["partial","*",97,6],"CCBot":["partial","*",97,6],"Bytespider":["partial","*",97,6],"Amazonbot":["partial","*",97,6],"meta-externalagent":["partial","*",97,6],"cohere-ai":["partial","*",97,6],"Diffbot":["partial","*",97,6],"ContextFetchBot":["partial","*",97,6]},"llms":{"status":"missing","http":404,"bytes":1652},"home":{"status":"ok","http":200,"url":"https://play.google.com/store/games","html_tokens":622682,"markdown_tokens":77,"pruned_tokens":77}},
{"rank":14,"domain":"apps.apple.com","origin":"https://apps.apple.com","robots":{"status":"ok","http":200,"url":"https://apps.apple.com/robots.txt","bytes":506,"groups":1},"bots":{"GPTBot":["partial","*",5,0],"ChatGPT-User":["partial","*",5,0],"OAI-SearchBot":["partial","*",5,0],"ClaudeBot":["partial","*",5,0],"Claude-User":["partial","*",5,0],"Claude-SearchBot":["partial","*",5,0],"anthropic-ai":["partial","*",5,0],"PerplexityBot":["partial","*",5,0],"Perplexity-User":["partial","*",5,0],"Google-Extended":["partial","*",5,0],"Applebot-Extended":["partial","*",5,0],"CCBot":["partial","*",5,0],"Bytespider":["partial","*",5,0],"Amazonbot":["partial","*",5,0],"meta-externalagent":["partial","*",5,0],"cohere-ai":["partial","*",5,0],"Diffbot":["partial","*",5,0],"ContextFetchBot":["partial","*",5,0]},"llms":{"status":"missing","http":404,"bytes":1146},"home":{"status":"ok","http":200,"url":"https://apps.apple.com/us/iphone/today","html_tokens":455601,"markdown_tokens":50,"pruned_tokens":49}},
{"rank":15,"domain":"youtu.be","origin":"https://youtu.be","robots":{"status":"ok","http":200,"url":"https://youtu.be/robots.txt","bytes":511,"groups":1},"bots":{"GPTBot":["partial","*",18,2],"ChatGPT-User":["partial","*",18,2],"OAI-SearchBot":["partial","*",18,2],"ClaudeBot":["partial","*",18,2],"Claude-User":["partial","*",18,2],"Claude-SearchBot":["partial","*",18,2],"anthropic-ai":["partial","*",18,2],"PerplexityBot":["partial","*",18,2],"Perplexity-User":["partial","*",18,2],"Google-Extended":["partial","*",18,2],"Applebot-Extended":["partial","*",18,2],"CCBot":["partial","*",18,2],"Bytespider":["partial","*",18,2],"Amazonbot":["partial","*",18,2],"meta-externalagent":["partial","*",18,2],"cohere-ai":["partial","*",18,2],"Diffbot":["partial","*",18,2],"ContextFetchBot":["partial","*",18,2]},"llms":{"status":"html","http":200,"bytes":842915},"home":{"status":"ok","http":200,"url":"https://www.youtube.com/?feature=youtu.be","html_tokens":238172,"markdown_tokens":41,"pruned_tokens":41}},
{"rank":16,"domain":"wordpress.org","origin":"https://wordpress.org","robots":{"status":"ok","http":200,"url":"https://wordpress.org/robots.txt","bytes":976,"groups":12},"bots":{"GPTBot":["allowed","named",0,1],"ChatGPT-User":["partial","*",4,3],"OAI-SearchBot":["partial","*",4,3],"ClaudeBot":["allowed","named",0,1],"Claude-User":["partial","*",4,3],"Claude-SearchBot":["partial","*",4,3],"anthropic-ai":["allowed","named",0,1],"PerplexityBot":["allowed","named",0,1],"Perplexity-User":["partial","*",4,3],"Google-Extended":["allowed","named",0,1],"Applebot-Extended":["allowed","named",0,1],"CCBot":["allowed","named",0,1],"Bytespider":["allowed","named",0,1],"Amazonbot":["partial","*",4,3],"meta-externalagent":["partial","*",4,3],"cohere-ai":["partial","*",4,3],"Diffbot":["partial","*",4,3],"ContextFetchBot":["partial","*",4,3]},"llms":{"status":"present","http":200,"bytes":5765},"home":{"status":"ok","http":200,"url":"https://wordpress.org/","html_tokens":44397,"markdown_tokens":434,"pruned_tokens":349}},
{"rank":17,"domain":"x.com","origin":"https://x.com","robots":{"status":"ok","http":200,"url":"https://x.com/robots.txt","bytes":2678,"groups":10},"bots":{"GPTBot":["blocked","*",2,0],"ChatGPT-User":["blocked","*",2,0],"OAI-SearchBot":["blocked","*",2,0],"ClaudeBot":["blocked","*",2,0],"Claude-User":["blocked","*",2,0],"Claude-SearchBot":["blocked","*",2,0],"anthropic-ai":["blocked","*",2,0],"PerplexityBot":["blocked","*",2,0],"Perplexity-User":["blocked","*",2,0],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","*",2,0],"CCBot":["blocked","*",2,0],"Bytespider":["blocked","*",2,0],"Amazonbot":["blocked","*",2,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","*",2,0],"Diffbot":["blocked","*",2,0],"ContextFetchBot":["blocked","*",2,0]},"llms":{"status":"disallowed"},"home":{"status":"disallowed"}},
{"rank":18,"domain":"goo.gl","origin":"https://goo.gl","robots":{"status":"refused","http":400,"url":"https://goo.gl/robots.txt","bytes":9258,"groups":0,"error":"HTTP 400"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":19,"domain":"en.wikipedia.org","origin":"https://en.wikipedia.org","robots":{"status":"ok","http":200,"url":"https://en.wikipedia.org/robots.txt","bytes":28275,"groups":33},"bots":{"GPTBot":["partial","*",428,4],"ChatGPT-User":["partial","*",428,4],"OAI-SearchBot":["partial","*",428,4],"ClaudeBot":["partial","*",428,4],"Claude-User":["partial","*",428,4],"Claude-SearchBot":["partial","*",428,4],"anthropic-ai":["partial","*",428,4],"PerplexityBot":["partial","*",428,4],"Perplexity-User":["partial","*",428,4],"Google-Extended":["partial","*",428,4],"Applebot-Extended":["partial","*",428,4],"CCBot":["partial","*",428,4],"Bytespider":["partial","*",428,4],"Amazonbot":["partial","*",428,4],"meta-externalagent":["partial","*",428,4],"cohere-ai":["partial","*",428,4],"Diffbot":["partial","*",428,4],"ContextFetchBot":["partial","*",428,4]},"llms":{"status":"missing","http":404,"bytes":1868},"home":{"status":"ok","http":200,"url":"https://en.wikipedia.org/wiki/Main_Page","html_tokens":64326,"markdown_tokens":1333,"pruned_tokens":135}},
{"rank":20,"domain":"maps.google.com","origin":"https://maps.google.com","robots":{"status":"ok","http":200,"url":"https://maps.google.com/robots.txt","bytes":6624,"groups":4},"bots":{"GPTBot":["partial","*",169,64],"ChatGPT-User":["partial","*",169,64],"OAI-SearchBot":["partial","*",169,64],"ClaudeBot":["partial","*",169,64],"Claude-User":["partial","*",169,64],"Claude-SearchBot":["partial","*",169,64],"anthropic-ai":["partial","*",169,64],"PerplexityBot":["partial","*",169,64],"Perplexity-User":["partial","*",169,64],"Google-Extended":["partial","*",169,64],"Applebot-Extended":["partial","*",169,64],"CCBot":["partial","*",169,64],"Bytespider":["partial","*",169,64],"Amazonbot":["partial","*",169,64],"meta-externalagent":["partial","*",169,64],"cohere-ai":["partial","*",169,64],"Diffbot":["partial","*",169,64],"ContextFetchBot":["partial","*",169,64]},"llms":{"status":"missing","http":404,"bytes":1569},"home":{"status":"ok","http":200,"url":"https://www.google.com/maps","html_tokens":54431,"markdown_tokens":30,"pruned_tokens":30}},
{"rank":21,"domain":"wa.me","origin":"https://wa.me","robots":{"status":"ok","http":200,"url":"https://wa.me/robots.txt","bytes":1525,"groups":1},"bots":{"GPTBot":["partial","*",56,0],"ChatGPT-User":["partial","*",56,0],"OAI-SearchBot":["partial","*",56,0],"ClaudeBot":["partial","*",56,0],"Claude-User":["partial","*",56,0],"Claude-SearchBot":["partial","*",56,0],"anthropic-ai":["partial","*",56,0],"PerplexityBot":["partial","*",56,0],"Perplexity-User":["partial","*",56,0],"Google-Extended":["partial","*",56,0],"Applebot-Extended":["partial","*",56,0],"CCBot":["partial","*",56,0],"Bytespider":["partial","*",56,0],"Amazonbot":["partial","*",56,0],"meta-externalagent":["partial","*",56,0],"cohere-ai":["partial","*",56,0],"Diffbot":["partial","*",56,0],"ContextFetchBot":["partial","*",56,0]},"llms":{"status":"disallowed","error":"robots.txt of api.whatsapp.com disallows /resolve/?deeplink=%2Fllms.txt&not_found=1"},"home":{"status":"disallowed","error":"robots.txt of api.whatsapp.com disallows /"}},
{"rank":22,"domain":"pinterest.com","origin":"https://pinterest.com","robots":{"status":"ok","http":200,"url":"https://www.pinterest.com/robots.txt","bytes":43558,"groups":8},"bots":{"GPTBot":["blocked","*",1,0],"ChatGPT-User":["blocked","*",1,0],"OAI-SearchBot":["blocked","*",1,0],"ClaudeBot":["blocked","*",1,0],"Claude-User":["blocked","*",1,0],"Claude-SearchBot":["blocked","*",1,0],"anthropic-ai":["blocked","*",1,0],"PerplexityBot":["blocked","*",1,0],"Perplexity-User":["blocked","*",1,0],"Google-Extended":["blocked","*",1,0],"Applebot-Extended":["blocked","*",1,0],"CCBot":["blocked","*",1,0],"Bytespider":["blocked","*",1,0],"Amazonbot":["blocked","*",1,0],"meta-externalagent":["blocked","*",1,0],"cohere-ai":["blocked","*",1,0],"Diffbot":["blocked","*",1,0],"ContextFetchBot":["blocked","*",1,0]},"llms":{"status":"disallowed"},"home":{"status":"disallowed"}},
{"rank":23,"domain":"tiktok.com","origin":"https://tiktok.com","robots":{"status":"ok","http":200,"url":"https://www.tiktok.com/robots.txt","bytes":1567,"groups":3},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["blocked","named",1,0],"OAI-SearchBot":["blocked","named",1,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["blocked","named",1,0],"Claude-SearchBot":["blocked","named",1,0],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["blocked","named",1,0],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["partial","*",16,12],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["partial","*",16,12],"Diffbot":["partial","*",16,12],"ContextFetchBot":["partial","*",16,12]},"llms":{"status":"html","http":200,"bytes":33016},"home":{"status":"ok","http":200,"url":"https://www.tiktok.com/","html_tokens":91865,"markdown_tokens":0,"pruned_tokens":0}},
{"rank":24,"domain":"whatsapp.com","origin":"https://whatsapp.com","robots":{"status":"ok","http":200,"url":"https://www.whatsapp.com/robots.txt","bytes":1180,"groups":17},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["partial","*",11,0],"OAI-SearchBot":["partial","*",11,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",11,0],"Claude-SearchBot":["partial","*",11,0],"anthropic-ai":["partial","*",11,0],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["partial","*",11,0],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["partial","*",11,0],"Bytespider":["partial","*",11,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["partial","*",11,0],"cohere-ai":["partial","*",11,0],"Diffbot":["partial","*",11,0],"ContextFetchBot":["partial","*",11,0]},"llms":{"status":"missing","http":404,"bytes":258788},"home":{"status":"ok","http":200,"url":"https://www.whatsapp.com/","html_tokens":72882,"markdown_tokens":197,"pruned_tokens":50}},
{"rank":25,"domain":"policies.google.com","origin":"https://policies.google.com","robots":{"status":"missing","http":404,"url":"https://policies.google.com/robots.txt","bytes":1652,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":1652},"home":{"status":"ok","http":200,"url":"https://policies.google.com/","html_tokens":135426,"markdown_tokens":301,"pruned_tokens":112}},
{"rank":26,"domain":"docs.google.com","origin":"https://docs.google.com","robots":{"status":"ok","http":200,"url":"https://docs.google.com/robots.txt","bytes":597,"groups":1},"bots":{"GPTBot":["partial","*",3,33],"ChatGPT-User":["partial","*",3,33],"OAI-SearchBot":["partial","*",3,33],"ClaudeBot":["partial","*",3,33],"Claude-User":["partial","*",3,33],"Claude-SearchBot":["partial","*",3,33],"anthropic-ai":["partial","*",3,33],"PerplexityBot":["partial","*",3,33],"Perplexity-User":["partial","*",3,33],"Google-Extended":["partial","*",3,33],"Applebot-Extended":["partial","*",3,33],"CCBot":["partial","*",3,33],"Bytespider":["partial","*",3,33],"Amazonbot":["partial","*",3,33],"meta-externalagent":["partial","*",3,33],"cohere-ai":["partial","*",3,33],"Diffbot":["partial","*",3,33],"ContextFetchBot":["partial","*",3,33]},"llms":{"status":"disallowed"},"home":{"status":"ok","http":200,"url":"https://accounts.google.com/v3/signin/identifier?continue=https://docs.google.com/&emr=1&followup=https://docs.google.com/&osid=1&passive=1209600&flowName=WebLiteSignIn&flowEntry=ServiceLogin&dsh=S1933264140:1792374146694775","html_tokens":236545,"markdown_tokens":34,"pruned_tokens":10}},
{"rank":27,"domain":"nginx.org","origin":"https://nginx.org","robots":{"status":"ok","http":200,"url":"https://nginx.org/robots.txt","bytes":72,"groups":1},"bots":{"GPTBot":["partial","*",1,0],"ChatGPT-User":["partial","*",1,0],"OAI-SearchBot":["partial","*",1,0],"ClaudeBot":["partial","*",1,0],"Claude-User":["partial","*",1,0],"Claude-SearchBot":["partial","*",1,0],"anthropic-ai":["partial","*",1,0],"PerplexityBot":["partial","*",1,0],"Perplexity-User":["partial","*",1,0],"Google-Extended":["partial","*",1,0],"Applebot-Extended":["partial","*",1,0],"CCBot":["partial","*",1,0],"Bytespider":["partial","*",1,0],"Amazonbot":["partial","*",1,0],"meta-externalagent":["partial","*",1,0],"cohere-ai":["partial","*",1,0],"Diffbot":["partial","*",1,0],"ContextFetchBot":["partial","*",1,0]},"llms":{"status":"missing","http":404,"bytes":2010},"home":{"status":"ok","http":200,"url":"https://nginx.org/","html_tokens":1680,"markdown_tokens":517,"pruned_tokens":175}},
{"rank":28,"domain":"f5.com","origin":"https://f5.com","robots":{"status":"html","http":200,"url":"https://f5.com/robots.txt","bytes":1804,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":29,"domain":"vimeo.com","origin":"https://vimeo.com","robots":{"status":"ok","http":200,"url":"https://vimeo.com/robots.txt","bytes":22615,"groups":15},"bots":{"GPTBot":["partial","named",1,106],"ChatGPT-User":["partial","*",325,296],"OAI-SearchBot":["partial","*",325,296],"ClaudeBot":["partial","named",1,106],"Claude-User":["partial","*",325,296],"Claude-SearchBot":["partial","*",325,296],"anthropic-ai":["partial","named",1,106],"PerplexityBot":["partial","*",325,296],"Perplexity-User":["partial","*",325,296],"Google-Extended":["partial","named",1,106],"Applebot-Extended":["partial","*",325,296],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["partial","*",325,296],"meta-externalagent":["partial","*",325,296],"cohere-ai":["partial","*",325,296],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",325,296]},"llms":{"status":"missing","http":404,"bytes":163542},"home":{"status":"ok","http":200,"url":"https://vimeo.com/","html_tokens":404729,"markdown_tokens":476,"pruned_tokens":256}},
{"rank":30,"domain":"nginx.com","origin":"https://nginx.com","robots":{"status":"html","http":200,"url":"https://nginx.com/robots.txt","bytes":1804,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":31,"domain":"amazon.com","origin":"https://amazon.com","robots":{"status":"ok","http":200,"url":"https://www.amazon.com/robots.txt","bytes":7947,"groups":101},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["blocked","named",1,0],"OAI-SearchBot":["blocked","named",1,0],"ClaudeBot":["blocked","named",2,0],"Claude-User":["blocked","named",1,0],"Claude-SearchBot":["blocked","named",1,0],"anthropic-ai":["partial","*",120,17],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["blocked","named",1,0],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["partial","*",120,17],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["partial","*",120,17],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",120,17]},"llms":{"status":"missing","http":404,"bytes":2296},"home":{"status":"ok","http":200,"url":"https://www.amazon.com/","html_tokens":159973,"markdown_tokens":1138,"pruned_tokens":106}},
{"rank":32,"domain":"api.whatsapp.com","origin":"https://api.whatsapp.com","robots":{"status":"ok","http":200,"url":"https://api.whatsapp.com/robots.txt","bytes":160,"groups":1},"bots":{"GPTBot":["blocked","*",1,0],"ChatGPT-User":["blocked","*",1,0],"OAI-SearchBot":["blocked","*",1,0],"ClaudeBot":["blocked","*",1,0],"Claude-User":["blocked","*",1,0],"Claude-SearchBot":["blocked","*",1,0],"anthropic-ai":["blocked","*",1,0],"PerplexityBot":["blocked","*",1,0],"Perplexity-User":["blocked","*",1,0],"Google-Extended":["blocked","*",1,0],"Applebot-Extended":["blocked","*",1,0],"CCBot":["blocked","*",1,0],"Bytespider":["blocked","*",1,0],"Amazonbot":["blocked","*",1,0],"meta-externalagent":["blocked","*",1,0],"cohere-ai":["blocked","*",1,0],"Diffbot":["blocked","*",1,0],"ContextFetchBot":["blocked","*",1,0]},"llms":{"status":"disallowed"},"home":{"status":"disallowed"}},
{"rank":33,"domain":"maps.app.goo.gl","origin":"https://maps.app.goo.gl","robots":{"status":"refused","http":400,"url":"https://maps.app.goo.gl/robots.txt","bytes":9258,"groups":0,"error":"HTTP 400"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":34,"domain":"drive.google.com","origin":"https://drive.google.com","robots":{"status":"ok","http":200,"url":"https://drive.google.com/robots.txt","bytes":600,"groups":1},"bots":{"GPTBot":["partial","*",4,32],"ChatGPT-User":["partial","*",4,32],"OAI-SearchBot":["partial","*",4,32],"ClaudeBot":["partial","*",4,32],"Claude-User":["partial","*",4,32],"Claude-SearchBot":["partial","*",4,32],"anthropic-ai":["partial","*",4,32],"PerplexityBot":["partial","*",4,32],"Perplexity-User":["partial","*",4,32],"Google-Extended":["partial","*",4,32],"Applebot-Extended":["partial","*",4,32],"CCBot":["partial","*",4,32],"Bytespider":["partial","*",4,32],"Amazonbot":["partial","*",4,32],"meta-externalagent":["partial","*",4,32],"cohere-ai":["partial","*",4,32],"Diffbot":["partial","*",4,32],"ContextFetchBot":["partial","*",4,32]},"llms":{"status":"disallowed"},"home":{"status":"ok","http":200,"url":"https://accounts.google.com/v3/signin/identifier?continue=https://drive.google.com/&emr=1&followup=https://drive.google.com/&osid=1&passive=1209600&service=wise&flowName=WebLiteSignIn&flowEntry=ServiceLogin&dsh=S118168547:1792374145717575","html_tokens":236586,"markdown_tokens":7,"pruned_tokens":7}},
{"rank":35,"domain":"t.me","origin":"https://t.me","robots":{"status":"missing","http":404,"url":"https://t.me/robots.txt","bytes":153,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"html","http":200,"bytes":19928},"home":{"status":"ok","http":200,"url":"https://telegram.org/","html_tokens":4966,"markdown_tokens":57,"pruned_tokens":57}},
{"rank":36,"domain":"mozilla.org","origin":"https://mozilla.org","robots":{"status":"ok","http":200,"url":"https://www.mozilla.org/robots.txt","bytes":66,"groups":1},"bots":{"GPTBot":["partial","*",2,0],"ChatGPT-User":["partial","*",2,0],"OAI-SearchBot":["partial","*",2,0],"ClaudeBot":["partial","*",2,0],"Claude-User":["partial","*",2,0],"Claude-SearchBot":["partial","*",2,0],"anthropic-ai":["partial","*",2,0],"PerplexityBot":["partial","*",2,0],"Perplexity-User":["partial","*",2,0],"Google-Extended":["partial","*",2,0],"Applebot-Extended":["partial","*",2,0],"CCBot":["partial","*",2,0],"Bytespider":["partial","*",2,0],"Amazonbot":["partial","*",2,0],"meta-externalagent":["partial","*",2,0],"cohere-ai":["partial","*",2,0],"Diffbot":["partial","*",2,0],"ContextFetchBot":["partial","*",2,0]},"llms":{"status":"missing","http":404,"bytes":8955},"home":{"status":"ok","http":200,"url":"https://www.mozilla.org/en-US/","html_tokens":28571,"markdown_tokens":229,"pruned_tokens":166}},
{"rank":37,"domain":"europa.eu","origin":"https://europa.eu","robots":{"status":"ok","http":200,"url":"https://europa.eu/robots.txt","bytes":4930,"groups":2},"bots":{"GPTBot":["partial","*",129,0],"ChatGPT-User":["partial","*",129,0],"OAI-SearchBot":["partial","*",129,0],"ClaudeBot":["partial","*",129,0],"Claude-User":["partial","*",129,0],"Claude-SearchBot":["partial","*",129,0],"anthropic-ai":["partial","*",129,0],"PerplexityBot":["partial","*",129,0],"Perplexity-User":["partial","*",129,0],"Google-Extended":["partial","*",129,0],"Applebot-Extended":["partial","*",129,0],"CCBot":["partial","*",129,0],"Bytespider":["partial","*",129,0],"Amazonbot":["partial","*",129,0],"meta-externalagent":["partial","*",129,0],"cohere-ai":["partial","*",129,0],"Diffbot":["partial","*",129,0],"ContextFetchBot":["partial","*",129,0]},"llms":{"status":"missing","http":404,"bytes":54395},"home":{"status":"ok","http":200,"url":"https://european-union.europa.eu/select-language?destination=/node/1","html_tokens":4535,"markdown_tokens":5,"pruned_tokens":5}},
{"rank":38,"domain":"adobe.com","origin":"https://adobe.com","robots":{"status":"ok","http":200,"url":"https://www.adobe.com/robots.txt","bytes":10199,"groups":7},"bots":{"GPTBot":["partial","*",89,18],"ChatGPT-User":["partial","*",89,18],"OAI-SearchBot":["partial","*",89,18],"ClaudeBot":["partial","*",89,18],"Claude-User":["partial","*",89,18],"Claude-SearchBot":["partial","*",89,18],"anthropic-ai":["partial","*",89,18],"PerplexityBot":["partial","*",89,18],"Perplexity-User":["partial","*",89,18],"Google-Extended":["partial","*",89,18],"Applebot-Extended":["partial","*",89,18],"CCBot":["partial","*",89,18],"Bytespider":["partial","*",89,18],"Amazonbot":["partial","*",89,18],"meta-externalagent":["partial","*",89,18],"cohere-ai":["partial","*",89,18],"Diffbot":["partial","*",89,18],"ContextFetchBot":["partial","*",89,18]},"llms":{"status":"present","http":200,"bytes":10499},"home":{"status":"ok","http":200,"url":"https://www.adobe.com/","html_tokens":12494,"markdown_tokens":874,"pruned_tokens":67}},
{"rank":39,"domain":"support.google.com","origin":"https://support.google.com","robots":{"status":"ok","http":200,"url":"https://support.google.com/robots.txt","bytes":176,"groups":1},"bots":{"GPTBot":["partial","*",7,0],"ChatGPT-User":["partial","*",7,0],"OAI-SearchBot":["partial","*",7,0],"ClaudeBot":["partial","*",7,0],"Claude-User":["partial","*",7,0],"Claude-SearchBot":["partial","*",7,0],"anthropic-ai":["partial","*",7,0],"PerplexityBot":["partial","*",7,0],"Perplexity-User":["partial","*",7,0],"Google-Extended":["partial","*",7,0],"Applebot-Extended":["partial","*",7,0],"CCBot":["partial","*",7,0],"Bytespider":["partial","*",7,0],"Amazonbot":["partial","*",7,0],"meta-externalagent":["partial","*",7,0],"cohere-ai":["partial","*",7,0],"Diffbot":["partial","*",7,0],"ContextFetchBot":["partial","*",7,0]},"llms":{"status":"html","http":200,"bytes":1510823},"home":{"status":"ok","http":200,"url":"https://support.google.com/","html_tokens":377635,"markdown_tokens":74,"pruned_tokens":74}},
{"rank":40,"domain":"wordpress.com","origin":"https://wordpress.com","robots":{"status":"ok","http":200,"url":"https://wordpress.com/robots.txt","bytes":1387,"groups":1},"bots":{"GPTBot":["partial","*",32,7],"ChatGPT-User":["partial","*",32,7],"OAI-SearchBot":["partial","*",32,7],"ClaudeBot":["partial","*",32,7],"Claude-User":["partial","*",32,7],"Claude-SearchBot":["partial","*",32,7],"anthropic-ai":["partial","*",32,7],"PerplexityBot":["partial","*",32,7],"Perplexity-User":["partial","*",32,7],"Google-Extended":["partial","*",32,7],"Applebot-Extended":["partial","*",32,7],"CCBot":["partial","*",32,7],"Bytespider":["partial","*",32,7],"Amazonbot":["partial","*",32,7],"meta-externalagent":["partial","*",32,7],"cohere-ai":["partial","*",32,7],"Diffbot":["partial","*",32,7],"ContextFetchBot":["partial","*",32,7]},"llms":{"status":"present","http":200,"bytes":7355},"home":{"status":"ok","http":200,"url":"https://wordpress.com/","html_tokens":85475,"markdown_tokens":155,"pruned_tokens":126}},
{"rank":41,"domain":"bit.ly","origin":"https://bit.ly","robots":{"status":"ok","http":200,"url":"https://bit.ly/robots.txt","bytes":123,"groups":1},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"html","http":200,"bytes":158978},"home":{"status":"ok","http":200,"url":"https://bitly.com/","html_tokens":77218,"markdown_tokens":282,"pruned_tokens":173}},
{"rank":42,"domain":"blogspot.com","origin":"https://blogspot.com","robots":{"status":"ok","http":200,"url":"https://www.blogger.com/robots.txt","bytes":572,"groups":1},"bots":{"GPTBot":["partial","*",21,0],"ChatGPT-User":["partial","*",21,0],"OAI-SearchBot":["partial","*",21,0],"ClaudeBot":["partial","*",21,0],"Claude-User":["partial","*",21,0],"Claude-SearchBot":["partial","*",21,0],"anthropic-ai":["partial","*",21,0],"PerplexityBot":["partial","*",21,0],"Perplexity-User":["partial","*",21,0],"Google-Extended":["partial","*",21,0],"Applebot-Extended":["partial","*",21,0],"CCBot":["partial","*",21,0],"Bytespider":["partial","*",21,0],"Amazonbot":["partial","*",21,0],"meta-externalagent":["partial","*",21,0],"cohere-ai":["partial","*",21,0],"Diffbot":["partial","*",21,0],"ContextFetchBot":["partial","*",21,0]},"llms":{"status":"missing","http":404,"bytes":168},"home":{"status":"ok","http":200,"url":"https://www.blogger.com/about/?bpli=1","html_tokens":42097,"markdown_tokens":271,"pruned_tokens":30}},
{"rank":43,"domain":"player.vimeo.com","origin":"https://player.vimeo.com","robots":{"status":"ok","http":200,"url":"https://player.vimeo.com/robots.txt","bytes":112,"groups":2},"bots":{"GPTBot":["partial","*",1,1],"ChatGPT-User":["partial","*",1,1],"OAI-SearchBot":["partial","*",1,1],"ClaudeBot":["partial","*",1,1],"Claude-User":["partial","*",1,1],"Claude-SearchBot":["partial","*",1,1],"anthropic-ai":["partial","*",1,1],"PerplexityBot":["partial","*",1,1],"Perplexity-User":["partial","*",1,1],"Google-Extended":["partial","*",1,1],"Applebot-Extended":["partial","*",1,1],"CCBot":["partial","*",1,1],"Bytespider":["blocked","named",1,0],"Amazonbot":["partial","*",1,1],"meta-externalagent":["partial","*",1,1],"cohere-ai":["partial","*",1,1],"Diffbot":["partial","*",1,1],"ContextFetchBot":["partial","*",1,1]},"llms":{"status":"missing","http":404,"bytes":3006},"home":{"status":"ok","http":200,"url":"https://vimeo.com/features","html_tokens":301708,"markdown_tokens":63,"pruned_tokens":63}},
{"rank":44,"domain":"apache.org","origin":"https://apache.org","robots":{"status":"ok","http":200,"url":"https://apache.org/robots.txt","bytes":47,"groups":1},"bots":{"GPTBot":["partial","*",1,0],"ChatGPT-User":["partial","*",1,0],"OAI-SearchBot":["partial","*",1,0],"ClaudeBot":["partial","*",1,0],"Claude-User":["partial","*",1,0],"Claude-SearchBot":["partial","*",1,0],"anthropic-ai":["partial","*",1,0],"PerplexityBot":["partial","*",1,0],"Perplexity-User":["partial","*",1,0],"Google-Extended":["partial","*",1,0],"Applebot-Extended":["partial","*",1,0],"CCBot":["partial","*",1,0],"Bytespider":["partial","*",1,0],"Amazonbot":["partial","*",1,0],"meta-externalagent":["partial","*",1,0],"cohere-ai":["partial","*",1,0],"Diffbot":["partial","*",1,0],"ContextFetchBot":["partial","*",1,0]},"llms":{"status":"missing","http":404,"bytes":24879},"home":{"status":"ok","http":200,"url":"https://apache.org/","html_tokens":13212,"markdown_tokens":174,"pruned_tokens":85}},
{"rank":45,"domain":"plus.google.com","origin":"https://plus.google.com","robots":{"status":"html","http":200,"url":"https://workspaceupdates.googleblog.com/2023/04/new-community-features-for-google-chat-and-an-update-currents%20.html","bytes":229987,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":46,"domain":"github.io","origin":"https://github.io","robots":{"status":"html","http":200,"url":"https://pages.github.com/","bytes":14446,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":47,"domain":"support.microsoft.com","origin":"https://support.microsoft.com","robots":{"status":"ok","http":200,"url":"https://support.microsoft.com/robots.txt","bytes":1080,"groups":1},"bots":{"GPTBot":["partial","*",34,0],"ChatGPT-User":["partial","*",34,0],"OAI-SearchBot":["partial","*",34,0],"ClaudeBot":["partial","*",34,0],"Claude-User":["partial","*",34,0],"Claude-SearchBot":["partial","*",34,0],"anthropic-ai":["partial","*",34,0],"PerplexityBot":["partial","*",34,0],"Perplexity-User":["partial","*",34,0],"Google-Extended":["partial","*",34,0],"Applebot-Extended":["partial","*",34,0],"CCBot":["partial","*",34,0],"Bytespider":["partial","*",34,0],"Amazonbot":["partial","*",34,0],"meta-externalagent":["partial","*",34,0],"cohere-ai":["partial","*",34,0],"Diffbot":["partial","*",34,0],"ContextFetchBot":["partial","*",34,0]},"llms":{"status":"missing","http":404,"bytes":91231},"home":{"status":"ok","http":200,"url":"https://support.microsoft.com/en-us/","html_tokens":27306,"markdown_tokens":31,"pruned_tokens":31}},
{"rank":48,"domain":"reddit.com","origin":"https://reddit.com","robots":{"status":"ok","http":200,"url":"https://www.reddit.com/robots.txt","bytes":538,"groups":1},"bots":{"GPTBot":["blocked","*",1,0],"ChatGPT-User":["blocked","*",1,0],"OAI-SearchBot":["blocked","*",1,0],"ClaudeBot":["blocked","*",1,0],"Claude-User":["blocked","*",1,0],"Claude-SearchBot":["blocked","*",1,0],"anthropic-ai":["blocked","*",1,0],"PerplexityBot":["blocked","*",1,0],"Perplexity-User":["blocked","*",1,0],"Google-Extended":["blocked","*",1,0],"Applebot-Extended":["blocked","*",1,0],"CCBot":["blocked","*",1,0],"Bytespider":["blocked","*",1,0],"Amazonbot":["blocked","*",1,0],"meta-externalagent":["blocked","*",1,0],"cohere-ai":["blocked","*",1,0],"Diffbot":["blocked","*",1,0],"ContextFetchBot":["blocked","*",1,0]},"llms":{"status":"disallowed"},"home":{"status":"disallowed"}},
{"rank":49,"domain":"forms.gle","origin":"https://forms.gle","robots":{"status":"refused","http":400,"url":"https://forms.gle/robots.txt","bytes":9258,"groups":0,"error":"HTTP 400"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":50,"domain":"nih.gov","origin":"https://nih.gov","robots":{"status":"refused","http":403,"url":"https://www.nih.gov/robots.txt","bytes":4901,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":51,"domain":"yahoo.com","origin":"https://yahoo.com","robots":{"status":"ok","http":200,"url":"https://www.yahoo.com/robots.txt","bytes":1464,"groups":3},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["blocked","named",1,0],"OAI-SearchBot":["partial","named",1,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",21,0],"Claude-SearchBot":["partial","named",1,0],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["partial","*",21,0],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["partial","*",21,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["partial","*",21,0],"meta-externalagent":["partial","*",21,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",21,0]},"llms":{"status":"http_error","http":429,"bytes":23},"home":{"status":"http_error","http":429,"url":"https://www.yahoo.com/"}},
{"rank":52,"domain":"spotify.com","origin":"https://spotify.com","robots":{"status":"ok","http":200,"url":"https://www.spotify.com/robots.txt","bytes":635,"groups":1},"bots":{"GPTBot":["partial","*",16,0],"ChatGPT-User":["partial","*",16,0],"OAI-SearchBot":["partial","*",16,0],"ClaudeBot":["partial","*",16,0],"Claude-User":["partial","*",16,0],"Claude-SearchBot":["partial","*",16,0],"anthropic-ai":["partial","*",16,0],"PerplexityBot":["partial","*",16,0],"Perplexity-User":["partial","*",16,0],"Google-Extended":["partial","*",16,0],"Applebot-Extended":["partial","*",16,0],"CCBot":["partial","*",16,0],"Bytespider":["partial","*",16,0],"Amazonbot":["partial","*",16,0],"meta-externalagent":["partial","*",16,0],"cohere-ai":["partial","*",16,0],"Diffbot":["partial","*",16,0],"ContextFetchBot":["partial","*",16,0]},"llms":{"status":"html","http":200,"bytes":50523},"home":{"status":"ok","http":200,"url":"https://open.spotify.com/","html_tokens":42345,"markdown_tokens":0,"pruned_tokens":0}},
{"rank":53,"domain":"qq.com","origin":"https://qq.com","robots":{"status":"ok","http":200,"url":"https://www.qq.com/robots.txt","bytes":71,"groups":1},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"html","http":200,"bytes":4183},"home":{"status":"ok","http":200,"url":"https://www.qq.com/","html_tokens":27864,"markdown_tokens":212,"pruned_tokens":171}},
{"rank":54,"domain":"support.apple.com","origin":"https://support.apple.com","robots":{"status":"ok","http":200,"url":"https://support.apple.com/robots.txt","bytes":9987,"groups":2},"bots":{"GPTBot":["partial","*",5,0],"ChatGPT-User":["partial","*",5,0],"OAI-SearchBot":["partial","*",5,0],"ClaudeBot":["partial","*",5,0],"Claude-User":["partial","*",5,0],"Claude-SearchBot":["partial","*",5,0],"anthropic-ai":["partial","*",5,0],"PerplexityBot":["partial","*",5,0],"Perplexity-User":["partial","*",5,0],"Google-Extended":["partial","*",5,0],"Applebot-Extended":["partial","*",5,0],"CCBot":["partial","*",5,0],"Bytespider":["partial","*",5,0],"Amazonbot":["partial","*",5,0],"meta-externalagent":["partial","*",5,0],"cohere-ai":["partial","*",5,0],"Diffbot":["partial","*",5,0],"ContextFetchBot":["partial","*",5,0]},"llms":{"status":"missing","http":404,"bytes":109933},"home":{"status":"ok","http":200,"url":"https://support.apple.com/","html_tokens":32224,"markdown_tokens":385,"pruned_tokens":166}},
{"rank":55,"domain":"itunes.apple.com","origin":"https://itunes.apple.com","robots":{"status":"ok","http":200,"url":"https://itunes.apple.com/robots.txt","bytes":447,"groups":2},"bots":{"GPTBot":["partial","*",9,1],"ChatGPT-User":["partial","*",9,1],"OAI-SearchBot":["partial","*",9,1],"ClaudeBot":["partial","*",9,1],"Claude-User":["partial","*",9,1],"Claude-SearchBot":["partial","*",9,1],"anthropic-ai":["partial","*",9,1],"PerplexityBot":["partial","*",9,1],"Perplexity-User":["partial","*",9,1],"Google-Extended":["partial","*",9,1],"Applebot-Extended":["partial","*",9,1],"CCBot":["partial","*",9,1],"Bytespider":["partial","*",9,1],"Amazonbot":["partial","*",9,1],"meta-externalagent":["partial","*",9,1],"cohere-ai":["partial","*",9,1],"Diffbot":["partial","*",9,1],"ContextFetchBot":["partial","*",9,1]},"llms":{"status":"missing","http":404,"bytes":236},"home":{"status":"ok","http":200,"url":"https://www.apple.com/itunes/","html_tokens":33623,"markdown_tokens":1177,"pruned_tokens":418}},
{"rank":56,"domain":"httpd.apache.org","origin":"https://httpd.apache.org","robots":{"status":"ok","http":200,"url":"https://httpd.apache.org/robots.txt","bytes":33,"groups":1},"bots":{"GPTBot":["partial","*",1,0],"ChatGPT-User":["partial","*",1,0],"OAI-SearchBot":["partial","*",1,0],"ClaudeBot":["partial","*",1,0],"Claude-User":["partial","*",1,0],"Claude-SearchBot":["partial","*",1,0],"anthropic-ai":["partial","*",1,0],"PerplexityBot":["partial","*",1,0],"Perplexity-User":["partial","*",1,0],"Google-Extended":["partial","*",1,0],"Applebot-Extended":["partial","*",1,0],"CCBot":["partial","*",1,0],"Bytespider":["partial","*",1,0],"Amazonbot":["partial","*",1,0],"meta-externalagent":["partial","*",1,0],"cohere-ai":["partial","*",1,0],"Diffbot":["partial","*",1,0],"ContextFetchBot":["partial","*",1,0]},"llms":{"status":"missing","http":404,"bytes":196},"home":{"status":"ok","http":200,"url":"https://httpd.apache.org/","html_tokens":2177,"markdown_tokens":279,"pruned_tokens":133}},
{"rank":57,"domain":"open.spotify.com","origin":"https://open.spotify.com","robots":{"status":"ok","http":200,"url":"https://open.spotify.com/robots.txt","bytes":1859,"groups":27},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["allowed","named",0,2],"OAI-SearchBot":["allowed","named",0,2],"ClaudeBot":["blocked","named",1,0],"Claude-User":["allowed","named",0,2],"Claude-SearchBot":["allowed","named",0,2],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["allowed","named",0,2],"Perplexity-User":["allowed","named",0,2],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["allowed","named",0,2],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["partial","*",3,1],"ContextFetchBot":["partial","*",3,1]},"llms":{"status":"missing","http":404,"bytes":5561},"home":{"status":"ok","http":200,"url":"https://open.spotify.com/","html_tokens":42345,"markdown_tokens":0,"pruned_tokens":0}},
{"rank":58,"domain":"godaddy.com","origin":"https://godaddy.com","robots":{"status":"refused","http":403,"url":"https://www.godaddy.com/robots.txt","bytes":383,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":59,"domain":"gravatar.com","origin":"https://gravatar.com","robots":{"status":"ok","http":200,"url":"https://gravatar.com/robots.txt","bytes":1022,"groups":11},"bots":{"GPTBot":["partial","named",1,2],"ChatGPT-User":["partial","*",12,3],"OAI-SearchBot":["partial","*",12,3],"ClaudeBot":["partial","*",12,3],"Claude-User":["partial","*",12,3],"Claude-SearchBot":["partial","*",12,3],"anthropic-ai":["partial","*",12,3],"PerplexityBot":["partial","*",12,3],"Perplexity-User":["partial","*",12,3],"Google-Extended":["partial","named",1,2],"Applebot-Extended":["partial","*",12,3],"CCBot":["partial","named",1,2],"Bytespider":["partial","named",1,2],"Amazonbot":["partial","named",1,2],"meta-externalagent":["partial","*",12,3],"cohere-ai":["partial","*",12,3],"Diffbot":["partial","*",12,3],"ContextFetchBot":["partial","*",12,3]},"llms":{"status":"present","http":200,"bytes":21927},"home":{"status":"ok","http":200,"url":"https://gravatar.com/","html_tokens":22058,"markdown_tokens":499,"pruned_tokens":403}},
{"rank":60,"domain":"tumblr.com","origin":"https://tumblr.com","robots":{"status":"unreachable","http":null,"url":"https://tumblr.com/robots.txt","bytes":0,"groups":0,"error":"<StreamReset stream_id:1, error_code:1, remote_reset:True>"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":61,"domain":"archive.org","origin":"https://archive.org","robots":{"status":"ok","http":200,"url":"https://archive.org/robots.txt","bytes":238,"groups":1},"bots":{"GPTBot":["partial","*",2,0],"ChatGPT-User":["partial","*",2,0],"OAI-SearchBot":["partial","*",2,0],"ClaudeBot":["partial","*",2,0],"Claude-User":["partial","*",2,0],"Claude-SearchBot":["partial","*",2,0],"anthropic-ai":["partial","*",2,0],"PerplexityBot":["partial","*",2,0],"Perplexity-User":["partial","*",2,0],"Google-Extended":["partial","*",2,0],"Applebot-Extended":["partial","*",2,0],"CCBot":["partial","*",2,0],"Bytespider":["partial","*",2,0],"Amazonbot":["partial","*",2,0],"meta-externalagent":["partial","*",2,0],"cohere-ai":["partial","*",2,0],"Diffbot":["partial","*",2,0],"ContextFetchBot":["partial","*",2,0]},"llms":{"status":"missing","http":404,"bytes":4963},"home":{"status":"ok","http":200,"url":"https://archive.org/","html_tokens":488,"markdown_tokens":25,"pruned_tokens":25}},
{"rank":62,"domain":"macromedia.com","origin":"https://www.macromedia.com","robots":{"status":"ok","http":200,"url":"https://www.adobe.com/robots.txt","bytes":10199,"groups":7},"bots":{"GPTBot":["partial","*",89,18],"ChatGPT-User":["partial","*",89,18],"OAI-SearchBot":["partial","*",89,18],"ClaudeBot":["partial","*",89,18],"Claude-User":["partial","*",89,18],"Claude-SearchBot":["partial","*",89,18],"anthropic-ai":["partial","*",89,18],"PerplexityBot":["partial","*",89,18],"Perplexity-User":["partial","*",89,18],"Google-Extended":["partial","*",89,18],"Applebot-Extended":["partial","*",89,18],"CCBot":["partial","*",89,18],"Bytespider":["partial","*",89,18],"Amazonbot":["partial","*",89,18],"meta-externalagent":["partial","*",89,18],"cohere-ai":["partial","*",89,18],"Diffbot":["partial","*",89,18],"ContextFetchBot":["partial","*",89,18]},"llms":{"status":"present","http":200,"bytes":10499},"home":{"status":"ok","http":200,"url":"https://www.adobe.com/","html_tokens":12494,"markdown_tokens":874,"pruned_tokens":104}},
{"rank":63,"domain":"sites.google.com","origin":"https://sites.google.com","robots":{"status":"ok","http":200,"url":"https://sites.google.com/robots.txt","bytes":84,"groups":1},"bots":{"GPTBot":["partial","*",2,2],"ChatGPT-User":["partial","*",2,2],"OAI-SearchBot":["partial","*",2,2],"ClaudeBot":["partial","*",2,2],"Claude-User":["partial","*",2,2],"Claude-SearchBot":["partial","*",2,2],"anthropic-ai":["partial","*",2,2],"PerplexityBot":["partial","*",2,2],"Perplexity-User":["partial","*",2,2],"Google-Extended":["partial","*",2,2],"Applebot-Extended":["partial","*",2,2],"CCBot":["partial","*",2,2],"Bytespider":["partial","*",2,2],"Amazonbot":["partial","*",2,2],"meta-externalagent":["partial","*",2,2],"cohere-ai":["partial","*",2,2],"Diffbot":["partial","*",2,2],"ContextFetchBot":["partial","*",2,2]},"llms":{"status":"missing","http":404,"bytes":1652},"home":{"status":"ok","http":200,"url":"https://accounts.google.com/v3/signin/identifier?continue=https://sites.google.com/&followup=https://sites.google.com/&osid=1&passive=1209600&service=wise&flowName=WebLiteSignIn&flowEntry=ServiceLogin&dsh=S1427494132:1792374148678083","html_tokens":236621,"markdown_tokens":7,"pruned_tokens":7}},
{"rank":64,"domain":"creativecommons.org","origin":"https://creativecommons.org","robots":{"status":"ok","http":200,"url":"https://creativecommons.org/robots.txt","bytes":177,"groups":1},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"missing","http":404,"bytes":32134},"home":{"status":"ok","http":200,"url":"https://creativecommons.org/","html_tokens":12321,"markdown_tokens":345,"pruned_tokens":58}},
{"rank":65,"domain":"office.com","origin":"https://office.com","robots":{"status":"html","http":200,"url":"https://login.microsoftonline.com/common/oauth2/v2.0/authorize?client_id=4765445b-32c6-49b0-83e6-1d93765276ca&redirect_uri=https%3A%2F%2Fwww.office.com%2Flandingv2&response_type=code%20id_token&scope=openid%20profile%20https%3A%2F%2Fwww.office.com%2Fv2%2FOfficeHome.All&response_mode=form_post&nonce=639279709444467556.NmE5ZDE0OGEtNjBkOS00NzcwLWJiYTctMmNiZjQ3ZTFhMmQ0NDVhNGJhOTItNGY1MS00Mzk3LThhNDYtOWZiOWM4Zjk4MTk4&ui_locales=en-US&mkt=en-US&client-request-id=f07c2b97-cfd2-4bcd-a2be-011351cea511&siwa=1&siwg=1&state=9Sm9dbOY8Z8M53RwmvYI8oGGndCUNFKXFnsq66DESCST4Zv5n4pV6aWJXxJhl2hMB5AlFHhgo4XWv71S9qC114Ao7-ceJZ0e8wsFlChiuTxy3WHpM5rFOlLhEEo9S25JN0ewlij0t7lUfahoXdXoeDYiLZTxC78lDUv-88kNVFvTThy70ulFY__xlUVgljIStMkMMF8-iwEjyjDyVfOnBOUwEeSjf_Nmgbt8tb4h4NjkFGNW5xoxq-aKKlceE9AoHjZAMeNYapHVeRc2Q6Qs3jyYxFx7LX22J4buwCVd1-7JHq8YbXS2cjwKZkr5CR0skLhgVxla2kFcZOgUGJOKvP_gEka9yGvCk9X5c9J_QNLQkn5oeE6OG6M6fFAP_Sl2ZfolZyhM_O_3Rgrb3x3cSpDyCUWqWdQFfnoY5fvrGiPokokMj2ANT142O_HpWggmCBE2bw8ixn5R3DP5duAEE7_hPhhGgUxf9ablZ4l6KN_aehQTZRwxmnSdVI73_mUGGOk87DJ-0GKYMs-opjQww-A3J6FxOoX5pI-YGoByo34&x-client-SKU=ID_NET8_0&x-client-ver=8.16.0.0","bytes":60415,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":66,"domain":"amazonaws.com","origin":"https://amazonaws.com","robots":{"status":"ok","http":200,"url":"https://aws.amazon.com/robots.txt","bytes":13822,"groups":2},"bots":{"GPTBot":["partial","*",266,6],"ChatGPT-User":["partial","*",266,6],"OAI-SearchBot":["partial","*",266,6],"ClaudeBot":["partial","*",266,6],"Claude-User":["partial","*",266,6],"Claude-SearchBot":["partial","*",266,6],"anthropic-ai":["partial","*",266,6],"PerplexityBot":["partial","*",266,6],"Perplexity-User":["partial","*",266,6],"Google-Extended":["partial","*",266,6],"Applebot-Extended":["partial","*",266,6],"CCBot":["partial","*",266,6],"Bytespider":["partial","*",266,6],"Amazonbot":["partial","*",266,6],"meta-externalagent":["partial","*",266,6],"cohere-ai":["partial","*",266,6],"Diffbot":["partial","*",266,6],"ContextFetchBot":["partial","*",266,6]},"llms":{"status":"missing","http":404,"bytes":302410},"home":{"status":"ok","http":200,"url":"https://aws.amazon.com/","html_tokens":125427,"markdown_tokens":988,"pruned_tokens":141}},
{"rank":67,"domain":"w3.org","origin":"https://w3.org","robots":{"status":"ok","http":200,"url":"https://www.w3.org/robots.txt","bytes":3844,"groups":2},"bots":{"GPTBot":["partial","*",112,0],"ChatGPT-User":["partial","*",112,0],"OAI-SearchBot":["partial","*",112,0],"ClaudeBot":["partial","*",112,0],"Claude-User":["partial","*",112,0],"Claude-SearchBot":["partial","*",112,0],"anthropic-ai":["partial","*",112,0],"PerplexityBot":["partial","*",112,0],"Perplexity-User":["partial","*",112,0],"Google-Extended":["partial","*",112,0],"Applebot-Extended":["partial","*",112,0],"CCBot":["partial","*",112,0],"Bytespider":["partial","*",112,0],"Amazonbot":["partial","*",112,0],"meta-externalagent":["partial","*",112,0],"cohere-ai":["partial","*",112,0],"Diffbot":["partial","*",112,0],"ContextFetchBot":["partial","*",112,0]},"llms":{"status":"missing","http":404,"bytes":20893},"home":{"status":"ok","http":200,"url":"https://www.w3.org/","html_tokens":12603,"markdown_tokens":378,"pruned_tokens":97}},
{"rank":68,"domain":"flickr.com","origin":"https://flickr.com","robots":{"status":"ok","http":200,"url":"https://flickr.com/robots.txt","bytes":2210,"groups":3},"bots":{"GPTBot":["blocked","*",1,0],"ChatGPT-User":["partial","named",19,0],"OAI-SearchBot":["partial","named",19,0],"ClaudeBot":["blocked","*",1,0],"Claude-User":["partial","named",19,0],"Claude-SearchBot":["partial","named",19,0],"anthropic-ai":["blocked","*",1,0],"PerplexityBot":["partial","named",19,0],"Perplexity-User":["blocked","*",1,0],"Google-Extended":["blocked","*",1,0],"Applebot-Extended":["blocked","*",1,0],"CCBot":["blocked","*",1,0],"Bytespider":["blocked","*",1,0],"Amazonbot":["blocked","*",1,0],"meta-externalagent":["blocked","*",1,0],"cohere-ai":["blocked","*",1,0],"Diffbot":["blocked","*",1,0],"ContextFetchBot":["blocked","*",1,0]},"llms":{"status":"disallowed"},"home":{"status":"disallowed"}},
{"rank":69,"domain":"nytimes.com","origin":"https://nytimes.com","robots":{"status":"ok","http":200,"url":"https://www.nytimes.com/robots.txt","bytes":8547,"groups":63},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["blocked","named",1,0],"OAI-SearchBot":["blocked","named",1,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["blocked","named",1,0],"Claude-SearchBot":["blocked","named",1,0],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["blocked","named",1,0],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["partial","named",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",86,23]},"llms":{"status":"missing","http":404,"bytes":68678},"home":{"status":"http_error","http":403,"url":"https://www.nytimes.com/"}},
{"rank":70,"domain":"baidu.com","origin":"https://baidu.com","robots":{"status":"ok","http":200,"url":"https://www.baidu.com/robots.txt","bytes":2814,"groups":16},"bots":{"GPTBot":["blocked","*",1,0],"ChatGPT-User":["blocked","*",1,0],"OAI-SearchBot":["blocked","*",1,0],"ClaudeBot":["blocked","*",1,0],"Claude-User":["blocked","*",1,0],"Claude-SearchBot":["blocked","*",1,0],"anthropic-ai":["blocked","*",1,0],"PerplexityBot":["blocked","*",1,0],"Perplexity-User":["blocked","*",1,0],"Google-Extended":["blocked","*",1,0],"Applebot-Extended":["blocked","*",1,0],"CCBot":["blocked","*",1,0],"Bytespider":["blocked","*",1,0],"Amazonbot":["blocked","*",1,0],"meta-externalagent":["blocked","*",1,0],"cohere-ai":["blocked","*",1,0],"Diffbot":["blocked","*",1,0],"ContextFetchBot":["blocked","*",1,0]},"llms":{"status":"disallowed"},"home":{"status":"disallowed"}},
{"rank":71,"domain":"ailawandorder.com","origin":"https://ailawandorder.com","robots":{"status":"unreachable","http":null,"url":"https://ailawandorder.com/robots.txt","bytes":0,"groups":0,"error":"[SSL: CERTIFICATE_VERIFY_FAILED] certificate verify failed: Hostname mismatch, certificate is not valid for 'ailawandorder.com'. (_ssl.c:1028)"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":72,"domain":"doi.org","origin":"https://doi.org","robots":{"status":"missing","http":404,"url":"https://www.doi.org/robots.txt","bytes":7625,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":7625},"home":{"status":"ok","http":200,"url":"https://www.doi.org/","html_tokens":6050,"markdown_tokens":526,"pruned_tokens":63}},
{"rank":73,"domain":"medium.com","origin":"https://medium.com","robots":{"status":"ok","http":200,"url":"https://medium.com/robots.txt","bytes":884,"groups":2},"bots":{"GPTBot":["partial","named",1,7],"ChatGPT-User":["partial","*",15,5],"OAI-SearchBot":["partial","*",15,5],"ClaudeBot":["partial","named",1,7],"Claude-User":["partial","*",15,5],"Claude-SearchBot":["partial","*",15,5],"anthropic-ai":["partial","*",15,5],"PerplexityBot":["partial","*",15,5],"Perplexity-User":["partial","*",15,5],"Google-Extended":["partial","*",15,5],"Applebot-Extended":["partial","named",1,7],"CCBot":["partial","*",15,5],"Bytespider":["partial","named",1,7],"Amazonbot":["partial","named",1,7],"meta-externalagent":["partial","named",1,7],"cohere-ai":["partial","*",15,5],"Diffbot":["partial","*",15,5],"ContextFetchBot":["partial","*",15,5]},"llms":{"status":"http_error","http":403,"bytes":4904},"home":{"status":"http_error","http":403,"url":"https://medium.com/"}},
{"rank":74,"domain":"youtube-nocookie.com","origin":"https://www.youtube-nocookie.com","robots":{"status":"ok","http":200,"url":"https://www.youtube-nocookie.com/robots.txt","bytes":548,"groups":2},"bots":{"GPTBot":["partial","*",16,0],"ChatGPT-User":["partial","*",16,0],"OAI-SearchBot":["partial","*",16,0],"ClaudeBot":["partial","*",16,0],"Claude-User":["partial","*",16,0],"Claude-SearchBot":["partial","*",16,0],"anthropic-ai":["partial","*",16,0],"PerplexityBot":["partial","*",16,0],"Perplexity-User":["partial","*",16,0],"Google-Extended":["partial","*",16,0],"Applebot-Extended":["partial","*",16,0],"CCBot":["partial","*",16,0],"Bytespider":["partial","*",16,0],"Amazonbot":["partial","*",16,0],"meta-externalagent":["partial","*",16,0],"cohere-ai":["partial","*",16,0],"Diffbot":["partial","*",16,0],"ContextFetchBot":["partial","*",16,0]},"llms":{"status":"missing","http":404,"bytes":7046},"home":{"status":"http_error","http":404,"url":"https://www.youtube-nocookie.com/"}},
{"rank":75,"domain":"soundcloud.com","origin":"https://soundcloud.com","robots":{"status":"ok","http":200,"url":"https://soundcloud.com/robots.txt","bytes":1441,"groups":2},"bots":{"GPTBot":["partial","named",1,17],"ChatGPT-User":["partial","named",1,17],"OAI-SearchBot":["partial","named",1,17],"ClaudeBot":["partial","named",1,17],"Claude-User":["partial","*",7,0],"Claude-SearchBot":["partial","*",7,0],"anthropic-ai":["partial","named",1,17],"PerplexityBot":["partial","named",1,17],"Perplexity-User":["partial","*",7,0],"Google-Extended":["partial","named",1,17],"Applebot-Extended":["partial","named",1,17],"CCBot":["partial","named",1,17],"Bytespider":["partial","named",1,17],"Amazonbot":["partial","named",1,17],"meta-externalagent":["partial","named",1,17],"cohere-ai":["partial","named",1,17],"Diffbot":["partial","*",7,0],"ContextFetchBot":["partial","*",7,0]},"llms":{"status":"missing","http":404,"bytes":0},"home":{"status":"ok","http":200,"url":"https://soundcloud.com/","html_tokens":18470,"markdown_tokens":14,"pruned_tokens":14}},
{"rank":76,"domain":"namecheap.com","origin":"https://namecheap.com","robots":{"status":"ok","http":200,"url":"https://www.namecheap.com/robots.txt","bytes":1359,"groups":1},"bots":{"GPTBot":["partial","*",42,1],"ChatGPT-User":["partial","*",42,1],"OAI-SearchBot":["partial","*",42,1],"ClaudeBot":["partial","*",42,1],"Claude-User":["partial","*",42,1],"Claude-SearchBot":["partial","*",42,1],"anthropic-ai":["partial","*",42,1],"PerplexityBot":["partial","*",42,1],"Perplexity-User":["partial","*",42,1],"Google-Extended":["partial","*",42,1],"Applebot-Extended":["partial","*",42,1],"CCBot":["partial","*",42,1],"Bytespider":["partial","*",42,1],"Amazonbot":["partial","*",42,1],"meta-externalagent":["partial","*",42,1],"cohere-ai":["partial","*",42,1],"Diffbot":["partial","*",42,1],"ContextFetchBot":["partial","*",42,1]},"llms":{"status":"http_error","http":403,"bytes":5613},"home":{"status":"http_error","http":403,"url":"https://www.namecheap.com/"}},
{"rank":77,"domain":"forbes.com","origin":"https://forbes.com","robots":{"status":"ok","http":200,"url":"https://www.forbes.com/robots.txt","bytes":33649,"groups":22},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["partial","*",20,2],"OAI-SearchBot":["allowed","named",0,1],"ClaudeBot":["blocked","named",1,0],"Claude-User":["allowed","named",0,1],"Claude-SearchBot":["allowed","named",0,1],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["partial","*",20,2],"Google-Extended":["partial","*",20,2],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",20,2]},"llms":{"status":"missing","http":404,"bytes":768782},"home":{"status":"ok","http":200,"url":"https://www.forbes.com/","html_tokens":181413,"markdown_tokens":166,"pruned_tokens":44}},
{"rank":78,"domain":"accounts.google.com","origin":"https://accounts.google.com","robots":{"status":"ok","http":200,"url":"https://accounts.google.com/robots.txt","bytes":187,"groups":1},"bots":{"GPTBot":["partial","*",6,1],"ChatGPT-User":["partial","*",6,1],"OAI-SearchBot":["partial","*",6,1],"ClaudeBot":["partial","*",6,1],"Claude-User":["partial","*",6,1],"Claude-SearchBot":["partial","*",6,1],"anthropic-ai":["partial","*",6,1],"PerplexityBot":["partial","*",6,1],"Perplexity-User":["partial","*",6,1],"Google-Extended":["partial","*",6,1],"Applebot-Extended":["partial","*",6,1],"CCBot":["partial","*",6,1],"Bytespider":["partial","*",6,1],"Amazonbot":["partial","*",6,1],"meta-externalagent":["partial","*",6,1],"cohere-ai":["partial","*",6,1],"Diffbot":["partial","*",6,1],"ContextFetchBot":["partial","*",6,1]},"llms":{"status":"missing","http":404,"bytes":1652},"home":{"status":"ok","http":200,"url":"https://accounts.google.com/v3/signin/identifier?continue=https://accounts.google.com/&followup=https://accounts.google.com/&passive=1209600&flowName=WebLiteSignIn&flowEntry=ServiceLogin&dsh=S1057623980:1792374154670923","html_tokens":236605,"markdown_tokens":34,"pruned_tokens":10}},
{"rank":79,"domain":"web.archive.org","origin":"https://web.archive.org","robots":{"status":"missing","http":404,"url":"https://web.archive.org/robots.txt","bytes":146,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":142489},"home":{"status":"ok","http":200,"url":"https://web.archive.org/","html_tokens":36374,"markdown_tokens":62,"pruned_tokens":43}},
{"rank":80,"domain":"sciencedirect.com","origin":"https://sciencedirect.com","robots":{"status":"refused","http":403,"url":"https://www.sciencedirect.com/robots.txt","bytes":831864,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":81,"domain":"live.com","origin":"https://live.com","robots":{"status":"html","http":200,"url":"https://outlook.live.com/mail/","bytes":14277,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":82,"domain":"zoom.us","origin":"https://zoom.us","robots":{"status":"ok","http":200,"url":"https://zoom.us/robots.txt","bytes":2005,"groups":13},"bots":{"GPTBot":["partial","*",28,0],"ChatGPT-User":["partial","*",28,0],"OAI-SearchBot":["partial","*",28,0],"ClaudeBot":["partial","*",28,0],"Claude-User":["partial","*",28,0],"Claude-SearchBot":["partial","*",28,0],"anthropic-ai":["partial","*",28,0],"PerplexityBot":["partial","*",28,0],"Perplexity-User":["partial","*",28,0],"Google-Extended":["partial","*",28,0],"Applebot-Extended":["partial","*",28,0],"CCBot":["partial","*",28,0],"Bytespider":["partial","*",28,0],"Amazonbot":["partial","*",28,0],"meta-externalagent":["partial","*",28,0],"cohere-ai":["partial","*",28,0],"Diffbot":["partial","*",28,0],"ContextFetchBot":["partial","*",28,0]},"llms":{"status":"missing","http":404,"bytes":526015},"home":{"status":"ok","http":200,"url":"https://www.zoom.com/","html_tokens":79701,"markdown_tokens":938,"pruned_tokens":156}},
{"rank":83,"domain":"theguardian.com","origin":"https://theguardian.com","robots":{"status":"ok","http":200,"url":"https://www.theguardian.com/robots.txt","bytes":2873,"groups":3},"bots":{"GPTBot":["partial","*",48,0],"ChatGPT-User":["partial","*",48,0],"OAI-SearchBot":["partial","*",48,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["blocked","named",1,0],"Claude-SearchBot":["blocked","named",1,0],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["partial","*",48,0],"Google-Extended":["partial","*",48,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["partial","*",48,0],"Diffbot":["partial","*",48,0],"ContextFetchBot":["partial","*",48,0]},"llms":{"status":"missing","http":404,"bytes":24345},"home":{"status":"ok","http":200,"url":"https://www.theguardian.com/us","html_tokens":349444,"markdown_tokens":102,"pruned_tokens":102}},
{"rank":84,"domain":"www.ncbi.nlm.nih.gov","origin":"https://www.ncbi.nlm.nih.gov","robots":{"status":"ok","http":200,"url":"https://www.ncbi.nlm.nih.gov/robots.txt","bytes":10819,"groups":11},"bots":{"GPTBot":["partial","*",74,2],"ChatGPT-User":["partial","*",74,2],"OAI-SearchBot":["partial","*",74,2],"ClaudeBot":["partial","*",74,2],"Claude-User":["partial","*",74,2],"Claude-SearchBot":["partial","*",74,2],"anthropic-ai":["partial","*",74,2],"PerplexityBot":["partial","*",74,2],"Perplexity-User":["partial","*",74,2],"Google-Extended":["partial","*",74,2],"Applebot-Extended":["partial","*",74,2],"CCBot":["partial","*",74,2],"Bytespider":["partial","*",74,2],"Amazonbot":["partial","*",74,2],"meta-externalagent":["partial","*",74,2],"cohere-ai":["partial","*",74,2],"Diffbot":["partial","*",74,2],"ContextFetchBot":["partial","*",74,2]},"llms":{"status":"missing","http":404,"bytes":135070},"home":{"status":"ok","http":200,"url":"https://www.ncbi.nlm.nih.gov/","html_tokens":9806,"markdown_tokens":97,"pruned_tokens":84}},
{"rank":85,"domain":"sourceforge.net","origin":"https://sourceforge.net","robots":{"status":"ok","http":200,"url":"https://sourceforge.net/robots.txt","bytes":5837,"groups":3},"bots":{"GPTBot":["partial","*",164,12],"ChatGPT-User":["partial","*",164,12],"OAI-SearchBot":["partial","*",164,12],"ClaudeBot":["partial","*",164,12],"Claude-User":["partial","*",164,12],"Claude-SearchBot":["partial","*",164,12],"anthropic-ai":["partial","*",164,12],"PerplexityBot":["partial","*",164,12],"Perplexity-User":["partial","*",164,12],"Google-Extended":["partial","*",164,12],"Applebot-Extended":["partial","*",164,12],"CCBot":["partial","*",164,12],"Bytespider":["partial","*",164,12],"Amazonbot":["partial","*",164,12],"meta-externalagent":["partial","*",164,12],"cohere-ai":["partial","*",164,12],"Diffbot":["partial","*",164,12],"ContextFetchBot":["partial","*",164,12]},"llms":{"status":"present","http":200,"bytes":3334905},"home":{"status":"ok","http":200,"url":"https://sourceforge.net/","html_tokens":31001,"markdown_tokens":166,"pruned_tokens":166}},
{"rank":86,"domain":"paypal.com","origin":"https://paypal.com","robots":{"status":"ok","http":200,"url":"https://www.paypal.com/robots.txt","bytes":2628,"groups":2},"bots":{"GPTBot":["partial","*",91,0],"ChatGPT-User":["partial","*",91,0],"OAI-SearchBot":["partial","*",91,0],"ClaudeBot":["partial","*",91,0],"Claude-User":["partial","*",91,0],"Claude-SearchBot":["partial","*",91,0],"anthropic-ai":["partial","*",91,0],"PerplexityBot":["partial","*",91,0],"Perplexity-User":["partial","*",91,0],"Google-Extended":["partial","*",91,0],"Applebot-Extended":["partial","*",91,0],"CCBot":["partial","*",91,0],"Bytespider":["partial","*",91,0],"Amazonbot":["partial","*",91,0],"meta-externalagent":["partial","*",91,0],"cohere-ai":["partial","*",91,0],"Diffbot":["partial","*",91,0],"ContextFetchBot":["partial","*",91,0]},"llms":{"status":"other_text","http":200,"bytes":40103},"home":{"status":"ok","http":200,"url":"https://www.paypal.com/us/home","html_tokens":92153,"markdown_tokens":358,"pruned_tokens":108}},
{"rank":87,"domain":"googleblog.com","origin":"https://googleblog.com","robots":{"status":"html","http":200,"url":"https://blog.google/","bytes":363549,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":88,"domain":"cpanel.net","origin":"https://cpanel.net","robots":{"status":"ok","http":200,"url":"https://www.cpanel.net/robots.txt","bytes":297,"groups":1},"bots":{"GPTBot":["partial","*",7,0],"ChatGPT-User":["partial","*",7,0],"OAI-SearchBot":["partial","*",7,0],"ClaudeBot":["partial","*",7,0],"Claude-User":["partial","*",7,0],"Claude-SearchBot":["partial","*",7,0],"anthropic-ai":["partial","*",7,0],"PerplexityBot":["partial","*",7,0],"Perplexity-User":["partial","*",7,0],"Google-Extended":["partial","*",7,0],"Applebot-Extended":["partial","*",7,0],"CCBot":["partial","*",7,0],"Bytespider":["partial","*",7,0],"Amazonbot":["partial","*",7,0],"meta-externalagent":["partial","*",7,0],"cohere-ai":["partial","*",7,0],"Diffbot":["partial","*",7,0],"ContextFetchBot":["partial","*",7,0]},"llms":{"status":"missing","http":404,"bytes":107830},"home":{"status":"ok","http":200,"url":"https://www.cpanel.net/","html_tokens":56509,"markdown_tokens":272,"pruned_tokens":128}},
{"rank":89,"domain":"who.int","origin":"https://who.int","robots":{"status":"ok","http":200,"url":"https://www.who.int/robots.txt","bytes":18983,"groups":527},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["blocked","named",1,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":72505},"home":{"status":"ok","http":200,"url":"https://www.who.int/","html_tokens":34705,"markdown_tokens":46,"pruned_tokens":41}},
{"rank":90,"domain":"oracle.com","origin":"https://oracle.com","robots":{"status":"refused","http":403,"url":"https://www.oracle.com/robots.txt","bytes":1448,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":91,"domain":"shopify.com","origin":"https://shopify.com","robots":{"status":"ok","http":200,"url":"https://shopify.com/robots.txt","bytes":62,"groups":1},"bots":{"GPTBot":["partial","*",2,0],"ChatGPT-User":["partial","*",2,0],"OAI-SearchBot":["partial","*",2,0],"ClaudeBot":["partial","*",2,0],"Claude-User":["partial","*",2,0],"Claude-SearchBot":["partial","*",2,0],"anthropic-ai":["partial","*",2,0],"PerplexityBot":["partial","*",2,0],"Perplexity-User":["partial","*",2,0],"Google-Extended":["partial","*",2,0],"Applebot-Extended":["partial","*",2,0],"CCBot":["partial","*",2,0],"Bytespider":["partial","*",2,0],"Amazonbot":["partial","*",2,0],"meta-externalagent":["partial","*",2,0],"cohere-ai":["partial","*",2,0],"Diffbot":["partial","*",2,0],"ContextFetchBot":["partial","*",2,0]},"llms":{"status":"present","http":200,"bytes":12520},"home":{"status":"ok","http":200,"url":"https://www.shopify.com/","html_tokens":166997,"markdown_tokens":134,"pruned_tokens":90}},
{"rank":92,"domain":"wixsite.com","origin":"https://www.wixsite.com","robots":{"status":"ok","http":200,"url":"https://www.wixsite.com/robots.txt","bytes":443,"groups":4},"bots":{"GPTBot":["partial","*",1,1],"ChatGPT-User":["partial","*",1,1],"OAI-SearchBot":["partial","*",1,1],"ClaudeBot":["partial","*",1,1],"Claude-User":["partial","*",1,1],"Claude-SearchBot":["partial","*",1,1],"anthropic-ai":["partial","*",1,1],"PerplexityBot":["partial","*",1,1],"Perplexity-User":["partial","*",1,1],"Google-Extended":["partial","*",1,1],"Applebot-Extended":["partial","*",1,1],"CCBot":["partial","*",1,1],"Bytespider":["partial","*",1,1],"Amazonbot":["partial","*",1,1],"meta-externalagent":["partial","*",1,1],"cohere-ai":["partial","*",1,1],"Diffbot":["partial","*",1,1],"ContextFetchBot":["partial","*",1,1]},"llms":{"status":"missing","http":404,"bytes":2243},"home":{"status":"ok","http":200,"url":"https://www.wix.com/","html_tokens":751810,"markdown_tokens":972,"pruned_tokens":156}},
{"rank":93,"domain":"opera.com","origin":"https://opera.com","robots":{"status":"ok","http":200,"url":"https://www.opera.com/robots.txt","bytes":994,"groups":1},"bots":{"GPTBot":["partial","*",34,7],"ChatGPT-User":["partial","*",34,7],"OAI-SearchBot":["partial","*",34,7],"ClaudeBot":["partial","*",34,7],"Claude-User":["partial","*",34,7],"Claude-SearchBot":["partial","*",34,7],"anthropic-ai":["partial","*",34,7],"PerplexityBot":["partial","*",34,7],"Perplexity-User":["partial","*",34,7],"Google-Extended":["partial","*",34,7],"Applebot-Extended":["partial","*",34,7],"CCBot":["partial","*",34,7],"Bytespider":["partial","*",34,7],"Amazonbot":["partial","*",34,7],"meta-externalagent":["partial","*",34,7],"cohere-ai":["partial","*",34,7],"Diffbot":["partial","*",34,7],"ContextFetchBot":["partial","*",34,7]},"llms":{"status":"present","http":200,"bytes":11613},"home":{"status":"ok","http":200,"url":"https://www.opera.com/","html_tokens":46531,"markdown_tokens":105,"pruned_tokens":72}},
{"rank":94,"domain":"dropbox.com","origin":"https://dropbox.com","robots":{"status":"ok","http":200,"url":"https://www.dropbox.com/robots.txt","bytes":801,"groups":3},"bots":{"GPTBot":["partial","*",23,4],"ChatGPT-User":["partial","*",23,4],"OAI-SearchBot":["partial","*",23,4],"ClaudeBot":["partial","*",23,4],"Claude-User":["partial","*",23,4],"Claude-SearchBot":["partial","*",23,4],"anthropic-ai":["partial","*",23,4],"PerplexityBot":["partial","*",23,4],"Perplexity-User":["partial","*",23,4],"Google-Extended":["partial","*",23,4],"Applebot-Extended":["partial","*",23,4],"CCBot":["partial","*",23,4],"Bytespider":["partial","*",23,4],"Amazonbot":["partial","*",23,4],"meta-externalagent":["partial","*",23,4],"cohere-ai":["partial","*",23,4],"Diffbot":["partial","*",23,4],"ContextFetchBot":["partial","*",23,4]},"llms":{"status":"present","http":200,"bytes":209330},"home":{"status":"ok","http":200,"url":"https://www.dropbox.com/","html_tokens":158244,"markdown_tokens":221,"pruned_tokens":221}},
{"rank":95,"domain":"bbc.com","origin":"https://bbc.com","robots":{"status":"ok","http":200,"url":"https://www.bbc.com/robots.txt","bytes":6169,"groups":40},"bots":{"GPTBot":["partial","named",1,2],"ChatGPT-User":["partial","named",1,2],"OAI-SearchBot":["partial","named",1,2],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",49,0],"Claude-SearchBot":["partial","*",49,0],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["blocked","named",1,0],"Google-Extended":["partial","named",1,2],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",49,0]},"llms":{"status":"missing","http":404,"bytes":32428},"home":{"status":"ok","http":200,"url":"https://www.bbc.com/","html_tokens":175603,"markdown_tokens":47,"pruned_tokens":47}},
{"rank":96,"domain":"t.co","origin":"https://t.co","robots":{"status":"ok","http":200,"url":"https://t.co/robots.txt","bytes":150,"groups":3},"bots":{"GPTBot":["blocked","*",1,0],"ChatGPT-User":["blocked","*",1,0],"OAI-SearchBot":["blocked","*",1,0],"ClaudeBot":["blocked","*",1,0],"Claude-User":["blocked","*",1,0],"Claude-SearchBot":["blocked","*",1,0],"anthropic-ai":["blocked","*",1,0],"PerplexityBot":["blocked","*",1,0],"Perplexity-User":["blocked","*",1,0],"Google-Extended":["blocked","*",1,0],"Applebot-Extended":["blocked","*",1,0],"CCBot":["blocked","*",1,0],"Bytespider":["blocked","*",1,0],"Amazonbot":["blocked","*",1,0],"meta-externalagent":["blocked","*",1,0],"cohere-ai":["blocked","*",1,0],"Diffbot":["blocked","*",1,0],"ContextFetchBot":["blocked","*",1,0]},"llms":{"status":"disallowed"},"home":{"status":"disallowed"}},
{"rank":97,"domain":"cnn.com","origin":"https://cnn.com","robots":{"status":"ok","http":200,"url":"https://www.cnn.com/robots.txt","bytes":3515,"groups":3},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["blocked","named",1,0],"OAI-SearchBot":["blocked","named",1,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["blocked","named",1,0],"Claude-SearchBot":["blocked","named",1,0],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["blocked","named",1,0],"Google-Extended":["partial","*",36,1],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["partial","*",36,1],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",36,1]},"llms":{"status":"missing","http":404,"bytes":1093875},"home":{"status":"too_large","error":"response larger than 5 MB"}},
{"rank":98,"domain":"vk.com","origin":"https://vk.com","robots":{"status":"ok","http":200,"url":"https://vk.com/robots.txt","bytes":6175,"groups":3},"bots":{"GPTBot":["partial","*",79,11],"ChatGPT-User":["partial","*",79,11],"OAI-SearchBot":["partial","*",79,11],"ClaudeBot":["partial","*",79,11],"Claude-User":["partial","*",79,11],"Claude-SearchBot":["partial","*",79,11],"anthropic-ai":["partial","*",79,11],"PerplexityBot":["partial","*",79,11],"Perplexity-User":["partial","*",79,11],"Google-Extended":["partial","*",79,11],"Applebot-Extended":["partial","*",79,11],"CCBot":["partial","*",79,11],"Bytespider":["partial","*",79,11],"Amazonbot":["partial","*",79,11],"meta-externalagent":["partial","*",79,11],"cohere-ai":["partial","*",79,11],"Diffbot":["partial","*",79,11],"ContextFetchBot":["partial","*",79,11]},"llms":{"status":"missing","http":404,"bytes":13067},"home":{"status":"ok","http":200,"url":"https://m.vk.com/","html_tokens":27058,"markdown_tokens":27,"pruned_tokens":27}},
{"rank":99,"domain":"researchgate.net","origin":"https://researchgate.net","robots":{"status":"ok","http":200,"url":"https://www.researchgate.net/robots.txt","bytes":293,"groups":1},"bots":{"GPTBot":["partial","*",7,2],"ChatGPT-User":["partial","*",7,2],"OAI-SearchBot":["partial","*",7,2],"ClaudeBot":["partial","*",7,2],"Claude-User":["partial","*",7,2],"Claude-SearchBot":["partial","*",7,2],"anthropic-ai":["partial","*",7,2],"PerplexityBot":["partial","*",7,2],"Perplexity-User":["partial","*",7,2],"Google-Extended":["partial","*",7,2],"Applebot-Extended":["partial","*",7,2],"CCBot":["partial","*",7,2],"Bytespider":["partial","*",7,2],"Amazonbot":["partial","*",7,2],"meta-externalagent":["partial","*",7,2],"cohere-ai":["partial","*",7,2],"Diffbot":["partial","*",7,2],"ContextFetchBot":["partial","*",7,2]},"llms":{"status":"http_error","http":403,"bytes":20175},"home":{"status":"http_error","http":403,"url":"https://www.researchgate.net/"}},
{"rank":100,"domain":"tinyurl.com","origin":"https://tinyurl.com","robots":{"status":"ok","http":200,"url":"https://tinyurl.com/robots.txt","bytes":154,"groups":1},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"missing","http":404,"bytes":28433},"home":{"status":"ok","http":200,"url":"https://tinyurl.com/","html_tokens":3467,"markdown_tokens":0,"pruned_tokens":0}},
{"rank":101,"domain":"reuters.com","origin":"https://reuters.com","robots":{"status":"ok","http":200,"url":"https://www.reuters.com/robots.txt","bytes":4627,"groups":3},"bots":{"GPTBot":["partial","*",1,1],"ChatGPT-User":["partial","named",26,0],"OAI-SearchBot":["partial","named",26,0],"ClaudeBot":["partial","*",1,1],"Claude-User":["partial","*",1,1],"Claude-SearchBot":["partial","*",1,1],"anthropic-ai":["partial","*",1,1],"PerplexityBot":["partial","*",1,1],"Perplexity-User":["partial","*",1,1],"Google-Extended":["partial","*",1,1],"Applebot-Extended":["blocked","named",1,0],"CCBot":["partial","*",1,1],"Bytespider":["partial","*",1,1],"Amazonbot":["partial","*",1,1],"meta-externalagent":["partial","*",1,1],"cohere-ai":["partial","*",1,1],"Diffbot":["partial","*",1,1],"ContextFetchBot":["partial","*",1,1]},"llms":{"status":"disallowed"},"home":{"status":"disallowed"}},
{"rank":102,"domain":"launchpad.net","origin":"https://launchpad.net","robots":{"status":"ok","http":200,"url":"https://launchpad.net/robots.txt","bytes":1881,"groups":2},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["partial","*",4,1],"OAI-SearchBot":["blocked","named",1,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["blocked","named",1,0],"Claude-SearchBot":["blocked","named",1,0],"anthropic-ai":["partial","*",4,1],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["partial","*",4,1],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",4,1]},"llms":{"status":"missing","http":404,"bytes":9665},"home":{"status":"ok","http":200,"url":"https://launchpad.net/","html_tokens":5082,"markdown_tokens":109,"pruned_tokens":109}},
{"rank":103,"domain":"nature.com","origin":"https://nature.com","robots":{"status":"ok","http":200,"url":"https://www.nature.com/robots.txt","bytes":2780,"groups":23},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["blocked","named",1,0],"OAI-SearchBot":["partial","*",48,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",48,0],"Claude-SearchBot":["partial","*",48,0],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["partial","*",48,0],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",48,0]},"llms":{"status":"missing","http":404,"bytes":99493},"home":{"status":"ok","http":200,"url":"https://www.nature.com","html_tokens":58444,"markdown_tokens":159,"pruned_tokens":159}},
{"rank":104,"domain":"mit.edu","origin":"https://mit.edu","robots":{"status":"ok","http":200,"url":"https://web.mit.edu/robots.txt","bytes":199,"groups":1},"bots":{"GPTBot":["partial","*",10,0],"ChatGPT-User":["partial","*",10,0],"OAI-SearchBot":["partial","*",10,0],"ClaudeBot":["partial","*",10,0],"Claude-User":["partial","*",10,0],"Claude-SearchBot":["partial","*",10,0],"anthropic-ai":["partial","*",10,0],"PerplexityBot":["partial","*",10,0],"Perplexity-User":["partial","*",10,0],"Google-Extended":["partial","*",10,0],"Applebot-Extended":["partial","*",10,0],"CCBot":["partial","*",10,0],"Bytespider":["partial","*",10,0],"Amazonbot":["partial","*",10,0],"meta-externalagent":["partial","*",10,0],"cohere-ai":["partial","*",10,0],"Diffbot":["partial","*",10,0],"ContextFetchBot":["partial","*",10,0]},"llms":{"status":"missing","http":404,"bytes":25731},"home":{"status":"ok","http":200,"url":"https://web.mit.edu/","html_tokens":11988,"markdown_tokens":69,"pruned_tokens":69}},
{"rank":105,"domain":"wikimedia.org","origin":"https://wikimedia.org","robots":{"status":"missing","http":404,"url":"https://www.wikimedia.org/robots.txt","bytes":1874,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":1868},"home":{"status":"ok","http":200,"url":"https://www.wikimedia.org/","html_tokens":12341,"markdown_tokens":178,"pruned_tokens":158}},
{"rank":106,"domain":"discord.com","origin":"https://discord.com","robots":{"status":"ok","http":200,"url":"https://discord.com/robots.txt","bytes":1774,"groups":3},"bots":{"GPTBot":["partial","*",24,10],"ChatGPT-User":["partial","*",24,10],"OAI-SearchBot":["partial","*",24,10],"ClaudeBot":["partial","*",24,10],"Claude-User":["partial","*",24,10],"Claude-SearchBot":["partial","*",24,10],"anthropic-ai":["partial","*",24,10],"PerplexityBot":["partial","*",24,10],"Perplexity-User":["partial","*",24,10],"Google-Extended":["partial","*",24,10],"Applebot-Extended":["partial","*",24,10],"CCBot":["partial","*",24,10],"Bytespider":["partial","*",24,10],"Amazonbot":["partial","*",24,10],"meta-externalagent":["partial","*",24,10],"cohere-ai":["partial","*",24,10],"Diffbot":["partial","*",24,10],"ContextFetchBot":["partial","*",24,10]},"llms":{"status":"missing","http":404,"bytes":44839},"home":{"status":"ok","http":200,"url":"https://discord.com/","html_tokens":42531,"markdown_tokens":367,"pruned_tokens":48}},
{"rank":107,"domain":"workspaceupdates.googleblog.com","origin":"https://workspaceupdates.googleblog.com","robots":{"status":"ok","http":200,"url":"https://workspaceupdates.googleblog.com/robots.txt","bytes":172,"groups":2},"bots":{"GPTBot":["partial","*",2,1],"ChatGPT-User":["partial","*",2,1],"OAI-SearchBot":["partial","*",2,1],"ClaudeBot":["partial","*",2,1],"Claude-User":["partial","*",2,1],"Claude-SearchBot":["partial","*",2,1],"anthropic-ai":["partial","*",2,1],"PerplexityBot":["partial","*",2,1],"Perplexity-User":["partial","*",2,1],"Google-Extended":["partial","*",2,1],"Applebot-Extended":["partial","*",2,1],"CCBot":["partial","*",2,1],"Bytespider":["partial","*",2,1],"Amazonbot":["partial","*",2,1],"meta-externalagent":["partial","*",2,1],"cohere-ai":["partial","*",2,1],"Diffbot":["partial","*",2,1],"ContextFetchBot":["partial","*",2,1]},"llms":{"status":"missing","http":404,"bytes":193596},"home":{"status":"ok","http":200,"url":"https://workspaceupdates.googleblog.com/","html_tokens":55936,"markdown_tokens":850,"pruned_tokens":327}},
{"rank":108,"domain":"ibm.com","origin":"https://ibm.com","robots":{"status":"ok","http":200,"url":"https://www.ibm.com/robots.txt","bytes":4616,"groups":10},"bots":{"GPTBot":["partial","*",63,0],"ChatGPT-User":["partial","*",63,0],"OAI-SearchBot":["partial","*",63,0],"ClaudeBot":["partial","*",63,0],"Claude-User":["partial","*",63,0],"Claude-SearchBot":["partial","*",63,0],"anthropic-ai":["partial","*",63,0],"PerplexityBot":["partial","*",63,0],"Perplexity-User":["partial","*",63,0],"Google-Extended":["partial","*",63,0],"Applebot-Extended":["partial","*",63,0],"CCBot":["partial","*",63,0],"Bytespider":["partial","*",63,0],"Amazonbot":["partial","*",63,0],"meta-externalagent":["partial","*",63,0],"cohere-ai":["partial","*",63,0],"Diffbot":["partial","*",63,0],"ContextFetchBot":["partial","*",63,0]},"llms":{"status":"missing","http":404,"bytes":79283},"home":{"status":"ok","http":200,"url":"https://www.ibm.com/us-en","html_tokens":53257,"markdown_tokens":319,"pruned_tokens":111}},
{"rank":109,"domain":"bbc.co.uk","origin":"https://bbc.co.uk","robots":{"status":"ok","http":200,"url":"https://www.bbc.co.uk/robots.txt","bytes":4907,"groups":40},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["blocked","named",1,0],"OAI-SearchBot":["blocked","named",1,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",57,0],"Claude-SearchBot":["partial","*",57,0],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["blocked","named",1,0],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",57,0]},"llms":{"status":"missing","http":404,"bytes":32428},"home":{"status":"ok","http":200,"url":"https://www.bbc.co.uk/","html_tokens":179216,"markdown_tokens":56,"pruned_tokens":26}},
{"rank":110,"domain":"springer.com","origin":"https://springer.com","robots":{"status":"ok","http":200,"url":"https://www.springer.com/robots.txt","bytes":286,"groups":1},"bots":{"GPTBot":["partial","*",6,2],"ChatGPT-User":["partial","*",6,2],"OAI-SearchBot":["partial","*",6,2],"ClaudeBot":["partial","*",6,2],"Claude-User":["partial","*",6,2],"Claude-SearchBot":["partial","*",6,2],"anthropic-ai":["partial","*",6,2],"PerplexityBot":["partial","*",6,2],"Perplexity-User":["partial","*",6,2],"Google-Extended":["partial","*",6,2],"Applebot-Extended":["partial","*",6,2],"CCBot":["partial","*",6,2],"Bytespider":["partial","*",6,2],"Amazonbot":["partial","*",6,2],"meta-externalagent":["partial","*",6,2],"cohere-ai":["partial","*",6,2],"Diffbot":["partial","*",6,2],"ContextFetchBot":["partial","*",6,2]},"llms":{"status":"html","http":200,"bytes":265322},"home":{"status":"ok","http":200,"url":"https://link.springer.com/brands/springer","html_tokens":66326,"markdown_tokens":673,"pruned_tokens":128}},
{"rank":111,"domain":"example.com","origin":"https://example.com","robots":{"status":"missing","http":404,"url":"https://example.com/robots.txt","bytes":577,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":577},"home":{"status":"ok","http":200,"url":"https://example.com/","html_tokens":145,"markdown_tokens":39,"pruned_tokens":39}},
{"rank":112,"domain":"bing.com","origin":"https://bing.com","robots":{"status":"ok","http":200,"url":"https://www.bing.com/robots.txt?toWww=1&redig=FC80C4B08A924E56BAC858D54860852D","bytes":2676,"groups":3},"bots":{"GPTBot":["partial","*",95,3],"ChatGPT-User":["partial","*",95,3],"OAI-SearchBot":["partial","*",95,3],"ClaudeBot":["partial","*",95,3],"Claude-User":["partial","*",95,3],"Claude-SearchBot":["partial","*",95,3],"anthropic-ai":["partial","*",95,3],"PerplexityBot":["partial","*",95,3],"Perplexity-User":["partial","*",95,3],"Google-Extended":["partial","*",95,3],"Applebot-Extended":["partial","*",95,3],"CCBot":["partial","*",95,3],"Bytespider":["partial","*",95,3],"Amazonbot":["partial","*",95,3],"meta-externalagent":["partial","*",95,3],"cohere-ai":["partial","*",95,3],"Diffbot":["partial","*",95,3],"ContextFetchBot":["partial","*",95,3]},"llms":{"status":"missing","http":404,"bytes":0},"home":{"status":"ok","http":200,"url":"https://www.bing.com/?toWww=1&redig=AD3CDBC426514EC49076C396D5B421CB","html_tokens":19486,"markdown_tokens":49,"pruned_tokens":49}},
{"rank":113,"domain":"php.net","origin":"https://php.net","robots":{"status":"ok","http":200,"url":"https://www.php.net/robots.txt","bytes":292,"groups":1},"bots":{"GPTBot":["partial","*",11,0],"ChatGPT-User":["partial","*",11,0],"OAI-SearchBot":["partial","*",11,0],"ClaudeBot":["partial","*",11,0],"Claude-User":["partial","*",11,0],"Claude-SearchBot":["partial","*",11,0],"anthropic-ai":["partial","*",11,0],"PerplexityBot":["partial","*",11,0],"Perplexity-User":["partial","*",11,0],"Google-Extended":["partial","*",11,0],"Applebot-Extended":["partial","*",11,0],"CCBot":["partial","*",11,0],"Bytespider":["partial","*",11,0],"Amazonbot":["partial","*",11,0],"meta-externalagent":["partial","*",11,0],"cohere-ai":["partial","*",11,0],"Diffbot":["partial","*",11,0],"ContextFetchBot":["partial","*",11,0]},"llms":{"status":"disallowed","error":"robots.txt of www.php.net disallows /search.php?show=404quickref&lang=en&pattern=llms.txt"},"home":{"status":"ok","http":200,"url":"https://www.php.net/","html_tokens":13478,"markdown_tokens":204,"pruned_tokens":83}},
{"rank":114,"domain":"discord.gg","origin":"https://discord.gg","robots":{"status":"ok","http":200,"url":"https://discord.gg/robots.txt","bytes":23,"groups":1},"bots":{"GPTBot":["allowed","*",0,1],"ChatGPT-User":["allowed","*",0,1],"OAI-SearchBot":["allowed","*",0,1],"ClaudeBot":["allowed","*",0,1],"Claude-User":["allowed","*",0,1],"Claude-SearchBot":["allowed","*",0,1],"anthropic-ai":["allowed","*",0,1],"PerplexityBot":["allowed","*",0,1],"Perplexity-User":["allowed","*",0,1],"Google-Extended":["allowed","*",0,1],"Applebot-Extended":["allowed","*",0,1],"CCBot":["allowed","*",0,1],"Bytespider":["allowed","*",0,1],"Amazonbot":["allowed","*",0,1],"meta-externalagent":["allowed","*",0,1],"cohere-ai":["allowed","*",0,1],"Diffbot":["allowed","*",0,1],"ContextFetchBot":["allowed","*",0,1]},"llms":{"status":"html","http":200,"bytes":170242},"home":{"status":"ok","http":200,"url":"https://discord.com","html_tokens":42531,"markdown_tokens":367,"pruned_tokens":48}},
{"rank":115,"domain":"harvard.edu","origin":"https://harvard.edu","robots":{"status":"ok","http":200,"url":"https://www.harvard.edu/robots.txt","bytes":748,"groups":1},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"missing","http":404,"bytes":111998},"home":{"status":"ok","http":200,"url":"https://www.harvard.edu/","html_tokens":46644,"markdown_tokens":1832,"pruned_tokens":134}},
{"rank":116,"domain":"wiley.com","origin":"https://wiley.com","robots":{"status":"ok","http":200,"url":"https://www.wiley.com/robots.txt","bytes":1014,"groups":9},"bots":{"GPTBot":["partial","*",7,1],"ChatGPT-User":["partial","*",7,1],"OAI-SearchBot":["partial","*",7,1],"ClaudeBot":["partial","*",7,1],"Claude-User":["partial","*",7,1],"Claude-SearchBot":["partial","*",7,1],"anthropic-ai":["partial","*",7,1],"PerplexityBot":["partial","*",7,1],"Perplexity-User":["partial","*",7,1],"Google-Extended":["partial","*",7,1],"Applebot-Extended":["partial","*",7,1],"CCBot":["partial","*",7,1],"Bytespider":["partial","*",7,1],"Amazonbot":["partial","*",7,1],"meta-externalagent":["partial","*",7,1],"cohere-ai":["partial","*",7,1],"Diffbot":["partial","*",7,1],"ContextFetchBot":["partial","*",7,1]},"llms":{"status":"missing","http":404,"bytes":352023},"home":{"status":"ok","http":200,"url":"https://www.wiley.com/en-us","html_tokens":128959,"markdown_tokens":1536,"pruned_tokens":643}},
{"rank":117,"domain":"weibo.com","origin":"https://weibo.com","robots":{"status":"ok","http":200,"url":"https://weibo.com/robots.txt","bytes":968,"groups":4},"bots":{"GPTBot":["partial","*",1,1],"ChatGPT-User":["partial","named",1,4],"OAI-SearchBot":["partial","named",1,4],"ClaudeBot":["partial","*",1,1],"Claude-User":["partial","*",1,1],"Claude-SearchBot":["partial","*",1,1],"anthropic-ai":["partial","*",1,1],"PerplexityBot":["partial","*",1,1],"Perplexity-User":["partial","named",1,4],"Google-Extended":["partial","*",1,1],"Applebot-Extended":["partial","*",1,1],"CCBot":["partial","*",1,1],"Bytespider":["partial","named",1,4],"Amazonbot":["partial","*",1,1],"meta-externalagent":["partial","named",1,4],"cohere-ai":["partial","*",1,1],"Diffbot":["partial","*",1,1],"ContextFetchBot":["partial","*",1,1]},"llms":{"status":"disallowed"},"home":{"status":"disallowed"}},
{"rank":118,"domain":"pubmed.ncbi.nlm.nih.gov","origin":"https://pubmed.ncbi.nlm.nih.gov","robots":{"status":"refused","http":403,"url":"https://pubmed.ncbi.nlm.nih.gov/robots.txt","bytes":134,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":119,"domain":"gnu.org","origin":"https://gnu.org","robots":{"status":"ok","http":200,"url":"https://www.gnu.org/robots.txt","bytes":2862,"groups":27},"bots":{"GPTBot":["partial","*",38,0],"ChatGPT-User":["partial","*",38,0],"OAI-SearchBot":["partial","*",38,0],"ClaudeBot":["partial","*",38,0],"Claude-User":["partial","*",38,0],"Claude-SearchBot":["partial","*",38,0],"anthropic-ai":["partial","*",38,0],"PerplexityBot":["partial","*",38,0],"Perplexity-User":["partial","*",38,0],"Google-Extended":["partial","*",38,0],"Applebot-Extended":["partial","*",38,0],"CCBot":["partial","*",38,0],"Bytespider":["partial","*",38,0],"Amazonbot":["partial","*",38,0],"meta-externalagent":["partial","*",38,0],"cohere-ai":["partial","*",38,0],"Diffbot":["partial","*",38,0],"ContextFetchBot":["partial","*",38,0]},"llms":{"status":"missing","http":404,"bytes":11549},"home":{"status":"ok","http":200,"url":"https://www.gnu.org/","html_tokens":7783,"markdown_tokens":761,"pruned_tokens":421}},
{"rank":120,"domain":"cookiedatabase.org","origin":"https://cookiedatabase.org","robots":{"status":"refused","http":403,"url":"https://cookiedatabase.org/robots.txt","bytes":5513,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":121,"domain":"cdc.gov","origin":"https://cdc.gov","robots":{"status":"ok","http":200,"url":"https://www.cdc.gov/robots.txt","bytes":1699,"groups":4},"bots":{"GPTBot":["partial","*",39,0],"ChatGPT-User":["partial","*",39,0],"OAI-SearchBot":["partial","*",39,0],"ClaudeBot":["partial","*",39,0],"Claude-User":["partial","*",39,0],"Claude-SearchBot":["partial","*",39,0],"anthropic-ai":["partial","*",39,0],"PerplexityBot":["partial","*",39,0],"Perplexity-User":["partial","*",39,0],"Google-Extended":["partial","*",39,0],"Applebot-Extended":["partial","*",39,0],"CCBot":["partial","*",39,0],"Bytespider":["partial","*",39,0],"Amazonbot":["partial","*",39,0],"meta-externalagent":["partial","*",39,0],"cohere-ai":["partial","*",39,0],"Diffbot":["partial","*",39,0],"ContextFetchBot":["partial","*",39,0]},"llms":{"status":"missing","http":404,"bytes":20905},"home":{"status":"ok","http":200,"url":"https://www.cdc.gov/index.html","html_tokens":15150,"markdown_tokens":121,"pruned_tokens":87}},
{"rank":122,"domain":"un.org","origin":"https://un.org","robots":{"status":"missing","http":404,"url":"https://maintenance.un.org/robots.txt","bytes":357,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":188},"home":{"status":"ok","http":200,"url":"https://www.un.org/","html_tokens":2239,"markdown_tokens":58,"pruned_tokens":58}},
{"rank":123,"domain":"debian.org","origin":"https://debian.org","robots":{"status":"missing","http":404,"url":"https://www.debian.org/robots.txt","bytes":7468,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":7468},"home":{"status":"ok","http":200,"url":"https://www.debian.org/","html_tokens":3966,"markdown_tokens":13,"pruned_tokens":13}},
{"rank":124,"domain":"linktr.ee","origin":"https://linktr.ee","robots":{"status":"ok","http":200,"url":"https://linktr.ee/robots.txt","bytes":11029,"groups":60},"bots":{"GPTBot":["partial","named",4,0],"ChatGPT-User":["partial","named",8,0],"OAI-SearchBot":["partial","named",4,0],"ClaudeBot":["partial","named",4,0],"Claude-User":["partial","named",4,0],"Claude-SearchBot":["partial","named",4,0],"anthropic-ai":["blocked","*",1,0],"PerplexityBot":["partial","named",4,0],"Perplexity-User":["partial","named",4,0],"Google-Extended":["allowed","named",0,1],"Applebot-Extended":["allowed","named",0,1],"CCBot":["partial","named",4,0],"Bytespider":["blocked","*",1,0],"Amazonbot":["partial","named",4,0],"meta-externalagent":["blocked","*",1,0],"cohere-ai":["blocked","*",1,0],"Diffbot":["blocked","*",1,0],"ContextFetchBot":["blocked","*",1,0]},"llms":{"status":"disallowed"},"home":{"status":"disallowed"}},
{"rank":125,"domain":"issuu.com","origin":"https://issuu.com","robots":{"status":"ok","http":200,"url":"https://issuu.com/robots.txt","bytes":4280,"groups":47},"bots":{"GPTBot":["partial","*",67,10],"ChatGPT-User":["partial","*",67,10],"OAI-SearchBot":["partial","*",67,10],"ClaudeBot":["partial","*",67,10],"Claude-User":["partial","*",67,10],"Claude-SearchBot":["partial","*",67,10],"anthropic-ai":["partial","*",67,10],"PerplexityBot":["partial","*",67,10],"Perplexity-User":["partial","*",67,10],"Google-Extended":["partial","*",67,10],"Applebot-Extended":["partial","*",67,10],"CCBot":["partial","*",67,10],"Bytespider":["partial","*",67,10],"Amazonbot":["partial","*",67,10],"meta-externalagent":["partial","*",67,10],"cohere-ai":["partial","*",67,10],"Diffbot":["partial","*",67,10],"ContextFetchBot":["partial","*",67,10]},"llms":{"status":"missing","http":404,"bytes":52898},"home":{"status":"ok","http":200,"url":"https://issuu.com/","html_tokens":23055,"markdown_tokens":80,"pruned_tokens":80}},
{"rank":126,"domain":"mp.weixin.qq.com","origin":"https://mp.weixin.qq.com","robots":{"status":"ok","http":200,"url":"https://mp.weixin.qq.com/robots.txt","bytes":251,"groups":1},"bots":{"GPTBot":["partial","*",1,8],"ChatGPT-User":["partial","*",1,8],"OAI-SearchBot":["partial","*",1,8],"ClaudeBot":["partial","*",1,8],"Claude-User":["partial","*",1,8],"Claude-SearchBot":["partial","*",1,8],"anthropic-ai":["partial","*",1,8],"PerplexityBot":["partial","*",1,8],"Perplexity-User":["partial","*",1,8],"Google-Extended":["partial","*",1,8],"Applebot-Extended":["partial","*",1,8],"CCBot":["partial","*",1,8],"Bytespider":["partial","*",1,8],"Amazonbot":["partial","*",1,8],"meta-externalagent":["partial","*",1,8],"cohere-ai":["partial","*",1,8],"Diffbot":["partial","*",1,8],"ContextFetchBot":["partial","*",1,8]},"llms":{"status":"disallowed"},"home":{"status":"ok","http":200,"url":"https://mp.weixin.qq.com/","html_tokens":13992,"markdown_tokens":10,"pruned_tokens":10}},
{"rank":127,"domain":"plesk.com","origin":"https://plesk.com","robots":{"status":"ok","http":200,"url":"https://www.plesk.com/robots.txt","bytes":101,"groups":1},"bots":{"GPTBot":["partial","*",2,0],"ChatGPT-User":["partial","*",2,0],"OAI-SearchBot":["partial","*",2,0],"ClaudeBot":["partial","*",2,0],"Claude-User":["partial","*",2,0],"Claude-SearchBot":["partial","*",2,0],"anthropic-ai":["partial","*",2,0],"PerplexityBot":["partial","*",2,0],"Perplexity-User":["partial","*",2,0],"Google-Extended":["partial","*",2,0],"Applebot-Extended":["partial","*",2,0],"CCBot":["partial","*",2,0],"Bytespider":["partial","*",2,0],"Amazonbot":["partial","*",2,0],"meta-externalagent":["partial","*",2,0],"cohere-ai":["partial","*",2,0],"Diffbot":["partial","*",2,0],"ContextFetchBot":["partial","*",2,0]},"llms":{"status":"other_text","http":200,"bytes":8759},"home":{"status":"ok","http":200,"url":"https://www.plesk.com/","html_tokens":164240,"markdown_tokens":347,"pruned_tokens":101}},
{"rank":128,"domain":"imdb.com","origin":"https://imdb.com","robots":{"status":"ok","http":200,"url":"https://www.imdb.com/robots.txt","bytes":1947,"groups":3},"bots":{"GPTBot":["blocked","*",1,0],"ChatGPT-User":["blocked","*",1,0],"OAI-SearchBot":["blocked","*",1,0],"ClaudeBot":["blocked","*",1,0],"Claude-User":["blocked","*",1,0],"Claude-SearchBot":["blocked","*",1,0],"anthropic-ai":["blocked","*",1,0],"PerplexityBot":["blocked","*",1,0],"Perplexity-User":["blocked","*",1,0],"Google-Extended":["blocked","*",1,0],"Applebot-Extended":["blocked","*",1,0],"CCBot":["blocked","*",1,0],"Bytespider":["blocked","*",1,0],"Amazonbot":["partial","named",20,1],"meta-externalagent":["blocked","*",1,0],"cohere-ai":["blocked","*",1,0],"Diffbot":["blocked","*",1,0],"ContextFetchBot":["blocked","*",1,0]},"llms":{"status":"disallowed"},"home":{"status":"disallowed"}},
{"rank":129,"domain":"weebly.com","origin":"https://weebly.com","robots":{"status":"ok","http":200,"url":"https://www.weebly.com/robots.txt","bytes":777,"groups":2},"bots":{"GPTBot":["partial","*",20,4],"ChatGPT-User":["partial","*",20,4],"OAI-SearchBot":["partial","*",20,4],"ClaudeBot":["partial","*",20,4],"Claude-User":["partial","*",20,4],"Claude-SearchBot":["partial","*",20,4],"anthropic-ai":["partial","*",20,4],"PerplexityBot":["partial","*",20,4],"Perplexity-User":["partial","*",20,4],"Google-Extended":["partial","*",20,4],"Applebot-Extended":["partial","*",20,4],"CCBot":["partial","*",20,4],"Bytespider":["partial","*",20,4],"Amazonbot":["partial","*",20,4],"meta-externalagent":["partial","*",20,4],"cohere-ai":["partial","*",20,4],"Diffbot":["partial","*",20,4],"ContextFetchBot":["partial","*",20,4]},"llms":{"status":"missing","http":404,"bytes":4133},"home":{"status":"ok","http":200,"url":"https://www.weebly.com/","html_tokens":30211,"markdown_tokens":195,"pruned_tokens":131}},
{"rank":130,"domain":"expireddomains.com","origin":"https://expireddomains.com","robots":{"status":"missing","http":404,"url":"https://expireddomains.com/robots.txt","bytes":35868,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":35868},"home":{"status":"http_error","http":403,"url":"https://expireddomains.com/"}},
{"rank":131,"domain":"stanford.edu","origin":"https://stanford.edu","robots":{"status":"ok","http":200,"url":"https://www.stanford.edu/robots.txt","bytes":197,"groups":1},"bots":{"GPTBot":["partial","*",6,1],"ChatGPT-User":["partial","*",6,1],"OAI-SearchBot":["partial","*",6,1],"ClaudeBot":["partial","*",6,1],"Claude-User":["partial","*",6,1],"Claude-SearchBot":["partial","*",6,1],"anthropic-ai":["partial","*",6,1],"PerplexityBot":["partial","*",6,1],"Perplexity-User":["partial","*",6,1],"Google-Extended":["partial","*",6,1],"Applebot-Extended":["partial","*",6,1],"CCBot":["partial","*",6,1],"Bytespider":["partial","*",6,1],"Amazonbot":["partial","*",6,1],"meta-externalagent":["partial","*",6,1],"cohere-ai":["partial","*",6,1],"Diffbot":["partial","*",6,1],"ContextFetchBot":["partial","*",6,1]},"llms":{"status":"missing","http":404,"bytes":240051},"home":{"status":"ok","http":200,"url":"https://www.stanford.edu/","html_tokens":196432,"markdown_tokens":134,"pruned_tokens":134}},
{"rank":132,"domain":"m.facebook.com","origin":"https://m.facebook.com","robots":{"status":"ok","http":200,"url":"https://m.facebook.com/robots.txt","bytes":26872,"groups":36},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["blocked","*",1,0],"OAI-SearchBot":["blocked","*",1,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["blocked","*",1,0],"Claude-SearchBot":["blocked","*",1,0],"anthropic-ai":["blocked","*",1,0],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["blocked","*",1,0],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","*",1,0],"Bytespider":["blocked","*",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","*",1,0],"cohere-ai":["blocked","*",1,0],"Diffbot":["blocked","*",1,0],"ContextFetchBot":["blocked","*",1,0]},"llms":{"status":"disallowed"},"home":{"status":"disallowed"}},
{"rank":133,"domain":"openai.com","origin":"https://openai.com","robots":{"status":"ok","http":200,"url":"https://openai.com/robots.txt","bytes":98,"groups":1},"bots":{"GPTBot":["partial","*",1,1],"ChatGPT-User":["partial","*",1,1],"OAI-SearchBot":["partial","*",1,1],"ClaudeBot":["partial","*",1,1],"Claude-User":["partial","*",1,1],"Claude-SearchBot":["partial","*",1,1],"anthropic-ai":["partial","*",1,1],"PerplexityBot":["partial","*",1,1],"Perplexity-User":["partial","*",1,1],"Google-Extended":["partial","*",1,1],"Applebot-Extended":["partial","*",1,1],"CCBot":["partial","*",1,1],"Bytespider":["partial","*",1,1],"Amazonbot":["partial","*",1,1],"meta-externalagent":["partial","*",1,1],"cohere-ai":["partial","*",1,1],"Diffbot":["partial","*",1,1],"ContextFetchBot":["partial","*",1,1]},"llms":{"status":"http_error","http":403,"bytes":9799},"home":{"status":"http_error","http":403,"url":"https://openai.com/"}},
{"rank":134,"domain":"beian.gov.cn","origin":"https://beian.gov.cn","robots":{"status":"unreachable","http":null,"url":"https://beian.gov.cn/robots.txt","bytes":0,"groups":0,"error":"cannot resolve beian.gov.cn"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":135,"domain":"outlook.com","origin":"https://outlook.com","robots":{"status":"html","http":200,"url":"https://outlook.live.com/mail/","bytes":14277,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":136,"domain":"www.gov.uk","origin":"https://www.gov.uk","robots":{"status":"ok","http":200,"url":"https://www.gov.uk/robots.txt","bytes":1097,"groups":4},"bots":{"GPTBot":["partial","*",2,0],"ChatGPT-User":["partial","*",2,0],"OAI-SearchBot":["partial","*",2,0],"ClaudeBot":["partial","*",2,0],"Claude-User":["partial","*",2,0],"Claude-SearchBot":["partial","*",2,0],"anthropic-ai":["partial","*",2,0],"PerplexityBot":["partial","*",2,0],"Perplexity-User":["partial","*",2,0],"Google-Extended":["partial","*",2,0],"Applebot-Extended":["partial","*",2,0],"CCBot":["partial","*",2,0],"Bytespider":["partial","*",2,0],"Amazonbot":["partial","*",2,0],"meta-externalagent":["partial","named",1,0],"cohere-ai":["partial","*",2,0],"Diffbot":["partial","*",2,0],"ContextFetchBot":["partial","*",2,0]},"llms":{"status":"missing","http":404,"bytes":52491},"home":{"status":"ok","http":200,"url":"https://www.gov.uk/","html_tokens":21379,"markdown_tokens":26,"pruned_tokens":26}},
{"rank":137,"domain":"bloomberg.com","origin":"https://bloomberg.com","robots":{"status":"ok","http":200,"url":"https://www.bloomberg.com/robots.txt","bytes":10640,"groups":98},"bots":{"GPTBot":["partial","named",1,5],"ChatGPT-User":["partial","named",1,44],"OAI-SearchBot":["partial","named",1,44],"ClaudeBot":["partial","named",1,5],"Claude-User":["partial","named",1,44],"Claude-SearchBot":["partial","named",1,44],"anthropic-ai":["partial","named",1,5],"PerplexityBot":["partial","named",1,5],"Perplexity-User":["partial","*",25,1],"Google-Extended":["partial","named",1,5],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",2,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",25,1]},"llms":{"status":"present","http":200,"bytes":45782},"home":{"status":"too_large","error":"response larger than 5 MB"}},
{"rank":138,"domain":"dropcatch.com","origin":"https://dropcatch.com","robots":{"status":"html","http":200,"url":"https://www.dropcatch.com/robots.txt","bytes":104198,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":139,"domain":"miit.gov.cn","origin":"https://www.miit.gov.cn","robots":{"status":"missing","http":404,"url":"https://www.miit.gov.cn/robots.txt","bytes":146,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":146},"home":{"status":"ok","http":200,"url":"https://www.miit.gov.cn/","html_tokens":13253,"markdown_tokens":393,"pruned_tokens":135}},
{"rank":140,"domain":"wpa.qq.com","origin":"https://wpa.qq.com","robots":{"status":"ok","http":200,"url":"https://wpa.qq.com/robots.txt","bytes":53,"groups":2},"bots":{"GPTBot":["blocked","*",2,0],"ChatGPT-User":["blocked","*",2,0],"OAI-SearchBot":["blocked","*",2,0],"ClaudeBot":["blocked","*",2,0],"Claude-User":["blocked","*",2,0],"Claude-SearchBot":["blocked","*",2,0],"anthropic-ai":["blocked","*",2,0],"PerplexityBot":["blocked","*",2,0],"Perplexity-User":["blocked","*",2,0],"Google-Extended":["blocked","*",2,0],"Applebot-Extended":["blocked","*",2,0],"CCBot":["blocked","*",2,0],"Bytespider":["blocked","*",2,0],"Amazonbot":["blocked","*",2,0],"meta-externalagent":["blocked","*",2,0],"cohere-ai":["blocked","*",2,0],"Diffbot":["blocked","*",2,0],"ContextFetchBot":["blocked","*",2,0]},"llms":{"status":"disallowed"},"home":{"status":"disallowed"}},
{"rank":141,"domain":"washingtonpost.com","origin":"https://washingtonpost.com","robots":{"status":"ok","http":200,"url":"https://www.washingtonpost.com/robots.txt","bytes":5804,"groups":35},"bots":{"GPTBot":["partial","*",113,3],"ChatGPT-User":["partial","*",113,3],"OAI-SearchBot":["partial","*",113,3],"ClaudeBot":["partial","named",1,2],"Claude-User":["partial","*",113,3],"Claude-SearchBot":["partial","*",113,3],"anthropic-ai":["partial","named",1,2],"PerplexityBot":["partial","named",1,2],"Perplexity-User":["partial","*",113,3],"Google-Extended":["partial","*",113,3],"Applebot-Extended":["partial","named",1,2],"CCBot":["partial","named",1,2],"Bytespider":["partial","named",1,2],"Amazonbot":["partial","named",1,1],"meta-externalagent":["partial","named",1,2],"cohere-ai":["partial","*",113,3],"Diffbot":["partial","named",1,2],"ContextFetchBot":["partial","*",113,3]},"llms":{"status":"unreachable","error":"<StreamReset stream_id:5, error_code:2, remote_reset:True>"},"home":{"status":"unreachable","error":"<StreamReset stream_id:7, error_code:2, remote_reset:True>"}},
{"rank":142,"domain":"g.page","origin":"https://g.page","robots":{"status":"html","http":200,"url":"https://www.google.com/search?q=robots.txt","bytes":92252,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":143,"domain":"telegram.me","origin":"https://telegram.me","robots":{"status":"html","http":200,"url":"https://telegram.org/","bytes":19927,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":144,"domain":"calendly.com","origin":"https://calendly.com","robots":{"status":"ok","http":200,"url":"https://calendly.com/robots.txt","bytes":335,"groups":3},"bots":{"GPTBot":["partial","*",3,2],"ChatGPT-User":["partial","*",3,2],"OAI-SearchBot":["partial","*",3,2],"ClaudeBot":["partial","*",3,2],"Claude-User":["partial","*",3,2],"Claude-SearchBot":["partial","*",3,2],"anthropic-ai":["partial","*",3,2],"PerplexityBot":["partial","*",3,2],"Perplexity-User":["partial","*",3,2],"Google-Extended":["partial","*",3,2],"Applebot-Extended":["partial","*",3,2],"CCBot":["partial","named",1,2],"Bytespider":["partial","*",3,2],"Amazonbot":["partial","*",3,2],"meta-externalagent":["partial","*",3,2],"cohere-ai":["partial","*",3,2],"Diffbot":["partial","*",3,2],"ContextFetchBot":["partial","*",3,2]},"llms":{"status":"present","http":200,"bytes":11913},"home":{"status":"ok","http":200,"url":"https://calendly.com/","html_tokens":218704,"markdown_tokens":614,"pruned_tokens":101}},
{"rank":145,"domain":"stripe.com","origin":"https://stripe.com","robots":{"status":"ok","http":200,"url":"https://stripe.com/robots.txt","bytes":643,"groups":3},"bots":{"GPTBot":["partial","*",8,2],"ChatGPT-User":["partial","*",8,2],"OAI-SearchBot":["partial","*",8,2],"ClaudeBot":["partial","*",8,2],"Claude-User":["partial","*",8,2],"Claude-SearchBot":["partial","*",8,2],"anthropic-ai":["partial","*",8,2],"PerplexityBot":["partial","*",8,2],"Perplexity-User":["partial","*",8,2],"Google-Extended":["partial","*",8,2],"Applebot-Extended":["partial","*",8,2],"CCBot":["partial","*",8,2],"Bytespider":["partial","*",8,2],"Amazonbot":["partial","*",8,2],"meta-externalagent":["partial","*",8,2],"cohere-ai":["partial","*",8,2],"Diffbot":["partial","*",8,2],"ContextFetchBot":["partial","*",8,2]},"llms":{"status":"present","http":200,"bytes":69923},"home":{"status":"ok","http":200,"url":"https://stripe.com/","html_tokens":166841,"markdown_tokens":1400,"pruned_tokens":508}},
{"rank":146,"domain":"nasa.gov","origin":"https://nasa.gov","robots":{"status":"ok","http":200,"url":"https://www.nasa.gov/robots.txt","bytes":69,"groups":2},"bots":{"GPTBot":["allowed","*",0,1],"ChatGPT-User":["allowed","*",0,1],"OAI-SearchBot":["allowed","*",0,1],"ClaudeBot":["allowed","*",0,1],"Claude-User":["allowed","*",0,1],"Claude-SearchBot":["allowed","*",0,1],"anthropic-ai":["allowed","*",0,1],"PerplexityBot":["allowed","*",0,1],"Perplexity-User":["allowed","*",0,1],"Google-Extended":["allowed","*",0,1],"Applebot-Extended":["allowed","*",0,1],"CCBot":["allowed","*",0,1],"Bytespider":["allowed","*",0,1],"Amazonbot":["allowed","*",0,1],"meta-externalagent":["allowed","*",0,1],"cohere-ai":["allowed","*",0,1],"Diffbot":["allowed","*",0,1],"ContextFetchBot":["allowed","*",0,1]},"llms":{"status":"missing","http":404,"bytes":191712},"home":{"status":"ok","http":200,"url":"https://www.nasa.gov/","html_tokens":86443,"markdown_tokens":302,"pruned_tokens":87}},
{"rank":147,"domain":"wsj.com","origin":"https://wsj.com","robots":{"status":"ok","http":200,"url":"https://www.wsj.com/robots.txt","bytes":4298,"groups":2},"bots":{"GPTBot":["partial","named",49,1],"ChatGPT-User":["partial","named",49,1],"OAI-SearchBot":["partial","named",49,1],"ClaudeBot":["blocked","*",1,0],"Claude-User":["blocked","*",1,0],"Claude-SearchBot":["blocked","*",1,0],"anthropic-ai":["blocked","*",1,0],"PerplexityBot":["blocked","*",1,0],"Perplexity-User":["blocked","*",1,0],"Google-Extended":["blocked","*",1,0],"Applebot-Extended":["blocked","*",1,0],"CCBot":["blocked","*",1,0],"Bytespider":["blocked","*",1,0],"Amazonbot":["blocked","*",1,0],"meta-externalagent":["blocked","*",1,0],"cohere-ai":["blocked","*",1,0],"Diffbot":["blocked","*",1,0],"ContextFetchBot":["blocked","*",1,0]},"llms":{"status":"disallowed"},"home":{"status":"disallowed"}},
{"rank":148,"domain":"msn.com","origin":"https://msn.com","robots":{"status":"ok","http":200,"url":"https://www.msn.com/robots.txt","bytes":5245,"groups":11},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["blocked","named",1,0],"OAI-SearchBot":["blocked","named",1,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["blocked","named",1,0],"Claude-SearchBot":["blocked","named",1,0],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["blocked","named",1,0],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",14,3]},"llms":{"status":"missing","http":404,"bytes":19143},"home":{"status":"ok","http":200,"url":"https://www.msn.com/","html_tokens":11889,"markdown_tokens":0,"pruned_tokens":0}},
{"rank":149,"domain":"arxiv.org","origin":"https://arxiv.org","robots":{"status":"ok","http":200,"url":"https://arxiv.org/robots.txt","bytes":5854,"groups":10},"bots":{"GPTBot":["partial","*",28,7],"ChatGPT-User":["partial","*",28,7],"OAI-SearchBot":["partial","*",28,7],"ClaudeBot":["partial","*",28,7],"Claude-User":["partial","*",28,7],"Claude-SearchBot":["partial","*",28,7],"anthropic-ai":["partial","*",28,7],"PerplexityBot":["partial","*",28,7],"Perplexity-User":["partial","*",28,7],"Google-Extended":["partial","*",28,7],"Applebot-Extended":["partial","*",28,7],"CCBot":["partial","*",28,7],"Bytespider":["partial","*",28,7],"Amazonbot":["partial","*",28,7],"meta-externalagent":["partial","*",28,7],"cohere-ai":["partial","*",28,7],"Diffbot":["partial","*",28,7],"ContextFetchBot":["partial","*",28,7]},"llms":{"status":"missing","http":404,"bytes":7221},"home":{"status":"ok","http":200,"url":"https://arxiv.org/","html_tokens":9509,"markdown_tokens":1435,"pruned_tokens":83}},
{"rank":150,"domain":"unsplash.com","origin":"https://unsplash.com","robots":{"status":"refused","http":403,"url":"https://unsplash.com/robots.txt","bytes":0,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":151,"domain":"commons.wikimedia.org","origin":"https://commons.wikimedia.org","robots":{"status":"ok","http":200,"url":"https://commons.wikimedia.org/robots.txt","bytes":17713,"groups":33},"bots":{"GPTBot":["partial","*",239,4],"ChatGPT-User":["partial","*",239,4],"OAI-SearchBot":["partial","*",239,4],"ClaudeBot":["partial","*",239,4],"Claude-User":["partial","*",239,4],"Claude-SearchBot":["partial","*",239,4],"anthropic-ai":["partial","*",239,4],"PerplexityBot":["partial","*",239,4],"Perplexity-User":["partial","*",239,4],"Google-Extended":["partial","*",239,4],"Applebot-Extended":["partial","*",239,4],"CCBot":["partial","*",239,4],"Bytespider":["partial","*",239,4],"Amazonbot":["partial","*",239,4],"meta-externalagent":["partial","*",239,4],"cohere-ai":["partial","*",239,4],"Diffbot":["partial","*",239,4],"ContextFetchBot":["partial","*",239,4]},"llms":{"status":"missing","http":404,"bytes":1868},"home":{"status":"ok","http":200,"url":"https://commons.wikimedia.org/wiki/Main_Page","html_tokens":61794,"markdown_tokens":296,"pruned_tokens":87}},
{"rank":152,"domain":"cloudfront.net","origin":"https://cloudfront.net","robots":{"status":"unreachable","http":null,"url":"https://cloudfront.net/robots.txt","bytes":0,"groups":0,"error":"cannot resolve cloudfront.net"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":153,"domain":"trustpilot.com","origin":"https://trustpilot.com","robots":{"status":"ok","http":200,"url":"https://www.trustpilot.com/robots.txt","bytes":25101,"groups":46},"bots":{"GPTBot":["partial","named",33,1],"ChatGPT-User":["partial","named",33,1],"OAI-SearchBot":["partial","named",33,1],"ClaudeBot":["partial","named",33,1],"Claude-User":["partial","named",33,1],"Claude-SearchBot":["partial","named",33,1],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["partial","named",33,1],"Perplexity-User":["blocked","*",1,0],"Google-Extended":["partial","named",33,1],"Applebot-Extended":["blocked","named",1,0],"CCBot":["partial","named",33,1],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","*",1,0],"meta-externalagent":["partial","named",33,1],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["blocked","*",1,0]},"llms":{"status":"disallowed"},"home":{"status":"disallowed"}},
{"rank":154,"domain":"statista.com","origin":"https://statista.com","robots":{"status":"ok","http":200,"url":"https://www.statista.com/robots.txt","bytes":11030,"groups":5},"bots":{"GPTBot":["partial","named",1,30],"ChatGPT-User":["partial","*",68,1],"OAI-SearchBot":["partial","*",68,1],"ClaudeBot":["partial","named",1,30],"Claude-User":["partial","*",68,1],"Claude-SearchBot":["partial","*",68,1],"anthropic-ai":["partial","named",1,30],"PerplexityBot":["partial","named",1,30],"Perplexity-User":["partial","named",1,30],"Google-Extended":["partial","named",1,30],"Applebot-Extended":["partial","named",1,30],"CCBot":["partial","named",1,30],"Bytespider":["partial","named",1,30],"Amazonbot":["partial","named",1,30],"meta-externalagent":["partial","named",1,30],"cohere-ai":["partial","named",1,30],"Diffbot":["partial","named",1,30],"ContextFetchBot":["partial","*",68,1]},"llms":{"status":"missing","http":404,"bytes":259853},"home":{"status":"ok","http":200,"url":"https://www.statista.com/","html_tokens":91592,"markdown_tokens":247,"pruned_tokens":40}},
{"rank":155,"domain":"businessinsider.com","origin":"https://businessinsider.com","robots":{"status":"ok","http":200,"url":"https://www.businessinsider.com/robots.txt","bytes":2231,"groups":13},"bots":{"GPTBot":["partial","*",31,0],"ChatGPT-User":["partial","*",31,0],"OAI-SearchBot":["partial","*",31,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",31,0],"Claude-SearchBot":["partial","*",31,0],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["partial","*",31,0],"Perplexity-User":["partial","*",31,0],"Google-Extended":["partial","*",31,0],"Applebot-Extended":["partial","*",31,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["partial","*",31,0],"meta-externalagent":["partial","*",31,0],"cohere-ai":["partial","*",31,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",31,0]},"llms":{"status":"present","http":200,"bytes":1066},"home":{"status":"ok","http":200,"url":"https://www.businessinsider.com/","html_tokens":151111,"markdown_tokens":157,"pruned_tokens":55}},
{"rank":156,"domain":"gitlab.com","origin":"https://gitlab.com","robots":{"status":"ok","http":200,"url":"https://gitlab.com/robots.txt","bytes":3230,"groups":4},"bots":{"GPTBot":["partial","*",86,3],"ChatGPT-User":["partial","*",86,3],"OAI-SearchBot":["partial","*",86,3],"ClaudeBot":["partial","*",86,3],"Claude-User":["partial","*",86,3],"Claude-SearchBot":["partial","*",86,3],"anthropic-ai":["partial","*",86,3],"PerplexityBot":["partial","*",86,3],"Perplexity-User":["partial","*",86,3],"Google-Extended":["partial","*",86,3],"Applebot-Extended":["partial","*",86,3],"CCBot":["partial","*",86,3],"Bytespider":["partial","*",86,3],"Amazonbot":["partial","*",86,3],"meta-externalagent":["partial","*",86,3],"cohere-ai":["partial","*",86,3],"Diffbot":["partial","*",86,3],"ContextFetchBot":["partial","*",86,3]},"llms":{"status":"http_error","http":403,"bytes":5535},"home":{"status":"ok","http":200,"url":"https://about.gitlab.com/","html_tokens":54269,"markdown_tokens":502,"pruned_tokens":263}},
{"rank":157,"domain":"ietf.org","origin":"https://ietf.org","robots":{"status":"ok","http":200,"url":"https://www.ietf.org/robots.txt","bytes":51,"groups":1},"bots":{"GPTBot":["partial","*",2,0],"ChatGPT-User":["partial","*",2,0],"OAI-SearchBot":["partial","*",2,0],"ClaudeBot":["partial","*",2,0],"Claude-User":["partial","*",2,0],"Claude-SearchBot":["partial","*",2,0],"anthropic-ai":["partial","*",2,0],"PerplexityBot":["partial","*",2,0],"Perplexity-User":["partial","*",2,0],"Google-Extended":["partial","*",2,0],"Applebot-Extended":["partial","*",2,0],"CCBot":["partial","*",2,0],"Bytespider":["partial","*",2,0],"Amazonbot":["partial","*",2,0],"meta-externalagent":["partial","*",2,0],"cohere-ai":["partial","*",2,0],"Diffbot":["partial","*",2,0],"ContextFetchBot":["partial","*",2,0]},"llms":{"status":"missing","http":404,"bytes":77200},"home":{"status":"ok","http":200,"url":"https://www.ietf.org/","html_tokens":21198,"markdown_tokens":211,"pruned_tokens":151}},
{"rank":158,"domain":"line.me","origin":"https://line.me","robots":{"status":"ok","http":200,"url":"https://line.me/robots.txt","bytes":454,"groups":3},"bots":{"GPTBot":["partial","*",7,4],"ChatGPT-User":["partial","*",7,4],"OAI-SearchBot":["partial","*",7,4],"ClaudeBot":["partial","*",7,4],"Claude-User":["partial","*",7,4],"Claude-SearchBot":["partial","*",7,4],"anthropic-ai":["partial","*",7,4],"PerplexityBot":["partial","*",7,4],"Perplexity-User":["partial","*",7,4],"Google-Extended":["partial","*",7,4],"Applebot-Extended":["partial","*",7,4],"CCBot":["partial","*",7,4],"Bytespider":["partial","*",7,4],"Amazonbot":["partial","*",7,4],"meta-externalagent":["partial","*",7,4],"cohere-ai":["partial","*",7,4],"Diffbot":["partial","*",7,4],"ContextFetchBot":["partial","*",7,4]},"llms":{"status":"missing","http":404,"bytes":7835},"home":{"status":"disallowed","error":"robots.txt of www.line.me not readable (refused)"}},
{"rank":159,"domain":"amzn.to","origin":"https://amzn.to","robots":{"status":"ok","http":200,"url":"https://amzn.to/robots.txt","bytes":123,"groups":1},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"html","http":200,"bytes":709315},"home":{"status":"ok","http":200,"url":"https://www.amazon.com/","html_tokens":155264,"markdown_tokens":1138,"pruned_tokens":163}},
{"rank":160,"domain":"list-manage.com","origin":"https://list-manage.com","robots":{"status":"timeout","http":null,"url":"https://list-manage.com/robots.txt","bytes":0,"groups":0,"error":"ConnectTimeout"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":161,"domain":"bsky.app","origin":"https://bsky.app","robots":{"status":"ok","http":200,"url":"https://bsky.app/robots.txt","bytes":441,"groups":1},"bots":{"GPTBot":["partial","*",1,1],"ChatGPT-User":["partial","*",1,1],"OAI-SearchBot":["partial","*",1,1],"ClaudeBot":["partial","*",1,1],"Claude-User":["partial","*",1,1],"Claude-SearchBot":["partial","*",1,1],"anthropic-ai":["partial","*",1,1],"PerplexityBot":["partial","*",1,1],"Perplexity-User":["partial","*",1,1],"Google-Extended":["partial","*",1,1],"Applebot-Extended":["partial","*",1,1],"CCBot":["partial","*",1,1],"Bytespider":["partial","*",1,1],"Amazonbot":["partial","*",1,1],"meta-externalagent":["partial","*",1,1],"cohere-ai":["partial","*",1,1],"Diffbot":["partial","*",1,1],"ContextFetchBot":["partial","*",1,1]},"llms":{"status":"missing","http":404,"bytes":0},"home":{"status":"ok","http":200,"url":"https://bsky.app/","html_tokens":6530,"markdown_tokens":31,"pruned_tokens":31}},
{"rank":162,"domain":"share.google","origin":"https://share.google","robots":{"status":"html","http":200,"url":"https://share.google/error","bytes":1675,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":163,"domain":"substack.com","origin":"https://substack.com","robots":{"status":"ok","http":200,"url":"https://substack.com/robots.txt","bytes":585,"groups":4},"bots":{"GPTBot":["partial","*",14,0],"ChatGPT-User":["partial","*",14,0],"OAI-SearchBot":["partial","*",14,0],"ClaudeBot":["partial","*",14,0],"Claude-User":["partial","*",14,0],"Claude-SearchBot":["partial","*",14,0],"anthropic-ai":["partial","*",14,0],"PerplexityBot":["partial","*",14,0],"Perplexity-User":["partial","*",14,0],"Google-Extended":["partial","*",14,0],"Applebot-Extended":["partial","*",14,0],"CCBot":["partial","*",14,0],"Bytespider":["partial","*",14,0],"Amazonbot":["partial","*",14,0],"meta-externalagent":["partial","*",14,0],"cohere-ai":["partial","*",14,0],"Diffbot":["partial","*",14,0],"ContextFetchBot":["partial","*",14,0]},"llms":{"status":"missing","http":404,"bytes":57874},"home":{"status":"ok","http":200,"url":"https://substack.com/","html_tokens":54779,"markdown_tokens":101,"pruned_tokens":101}},
{"rank":164,"domain":"canva.com","origin":"https://canva.com","robots":{"status":"ok","http":200,"url":"https://www.canva.com/robots.txt","bytes":6527,"groups":5},"bots":{"GPTBot":["partial","named",1,91],"ChatGPT-User":["partial","*",106,2],"OAI-SearchBot":["partial","*",106,2],"ClaudeBot":["partial","named",1,91],"Claude-User":["partial","*",106,2],"Claude-SearchBot":["partial","*",106,2],"anthropic-ai":["partial","named",1,91],"PerplexityBot":["allowed","named",0,1],"Perplexity-User":["allowed","named",0,1],"Google-Extended":["partial","*",106,2],"Applebot-Extended":["partial","named",1,91],"CCBot":["partial","named",1,91],"Bytespider":["blocked","named",1,0],"Amazonbot":["partial","*",106,2],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["partial","*",106,2],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",106,2]},"llms":{"status":"http_error","http":403,"bytes":754845},"home":{"status":"http_error","http":403,"url":"https://www.canva.com/"}},
{"rank":165,"domain":"mysql.com","origin":"https://mysql.com","robots":{"status":"ok","http":200,"url":"https://www.mysql.com/robots.txt","bytes":184,"groups":2},"bots":{"GPTBot":["partial","*",3,0],"ChatGPT-User":["partial","*",3,0],"OAI-SearchBot":["partial","*",3,0],"ClaudeBot":["partial","*",3,0],"Claude-User":["partial","*",3,0],"Claude-SearchBot":["partial","*",3,0],"anthropic-ai":["partial","*",3,0],"PerplexityBot":["partial","*",3,0],"Perplexity-User":["partial","*",3,0],"Google-Extended":["partial","*",3,0],"Applebot-Extended":["partial","*",3,0],"CCBot":["partial","*",3,0],"Bytespider":["partial","*",3,0],"Amazonbot":["partial","*",3,0],"meta-externalagent":["partial","*",3,0],"cohere-ai":["partial","*",3,0],"Diffbot":["partial","*",3,0],"ContextFetchBot":["partial","*",3,0]},"llms":{"status":"missing","http":404,"bytes":19079},"home":{"status":"ok","http":200,"url":"https://www.mysql.com/","html_tokens":10926,"markdown_tokens":339,"pruned_tokens":228}},
{"rank":166,"domain":"twitch.tv","origin":"https://twitch.tv","robots":{"status":"ok","http":200,"url":"https://www.twitch.tv/robots.txt","bytes":1646,"groups":5},"bots":{"GPTBot":["partial","*",67,2],"ChatGPT-User":["partial","*",67,2],"OAI-SearchBot":["partial","*",67,2],"ClaudeBot":["partial","*",67,2],"Claude-User":["partial","*",67,2],"Claude-SearchBot":["partial","*",67,2],"anthropic-ai":["partial","*",67,2],"PerplexityBot":["partial","*",67,2],"Perplexity-User":["partial","*",67,2],"Google-Extended":["partial","*",67,2],"Applebot-Extended":["partial","*",67,2],"CCBot":["partial","*",67,2],"Bytespider":["partial","*",67,2],"Amazonbot":["allowed","named",0,0],"meta-externalagent":["partial","*",67,2],"cohere-ai":["partial","*",67,2],"Diffbot":["partial","*",67,2],"ContextFetchBot":["partial","*",67,2]},"llms":{"status":"html","http":200,"bytes":217900},"home":{"status":"ok","http":200,"url":"https://www.twitch.tv/","html_tokens":54759,"markdown_tokens":0,"pruned_tokens":0}},
{"rank":167,"domain":"sharepoint.com","origin":"https://sharepoint.com","robots":{"status":"error","http":null,"url":"https://sharepoint.com/robots.txt","bytes":0,"groups":0,"error":"more than 5 redirects"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":168,"domain":"cnbc.com","origin":"https://cnbc.com","robots":{"status":"ok","http":200,"url":"https://www.cnbc.com/robots.txt","bytes":2516,"groups":3},"bots":{"GPTBot":["partial","named",1,1],"ChatGPT-User":["partial","named",1,1],"OAI-SearchBot":["partial","named",1,1],"ClaudeBot":["partial","named",1,1],"Claude-User":["partial","named",1,1],"Claude-SearchBot":["partial","*",8,0],"anthropic-ai":["partial","named",1,1],"PerplexityBot":["partial","named",1,1],"Perplexity-User":["partial","*",8,0],"Google-Extended":["partial","named",1,1],"Applebot-Extended":["partial","named",1,1],"CCBot":["partial","named",1,1],"Bytespider":["partial","named",1,1],"Amazonbot":["partial","named",1,1],"meta-externalagent":["partial","named",1,1],"cohere-ai":["partial","named",1,1],"Diffbot":["partial","named",1,1],"ContextFetchBot":["partial","*",8,0]},"llms":{"status":"missing","http":404,"bytes":565868},"home":{"status":"ok","http":200,"url":"https://www.cnbc.com","html_tokens":747211,"markdown_tokens":40,"pruned_tokens":40}},
{"rank":169,"domain":"cloudflare.com","origin":"https://cloudflare.com","robots":{"status":"ok","http":200,"url":"https://www.cloudflare.com/robots.txt","bytes":1099,"groups":9},"bots":{"GPTBot":["allowed","named",0,1],"ChatGPT-User":["allowed","named",0,1],"OAI-SearchBot":["allowed","*",0,1],"ClaudeBot":["allowed","*",0,1],"Claude-User":["allowed","*",0,1],"Claude-SearchBot":["allowed","*",0,1],"anthropic-ai":["allowed","named",0,1],"PerplexityBot":["allowed","named",0,1],"Perplexity-User":["allowed","*",0,1],"Google-Extended":["allowed","named",0,1],"Applebot-Extended":["allowed","*",0,1],"CCBot":["allowed","named",0,1],"Bytespider":["allowed","*",0,1],"Amazonbot":["allowed","*",0,1],"meta-externalagent":["allowed","*",0,1],"cohere-ai":["allowed","named",0,1],"Diffbot":["allowed","*",0,1],"ContextFetchBot":["allowed","*",0,1]},"llms":{"status":"present","http":200,"bytes":17173},"home":{"status":"ok","http":200,"url":"https://www.cloudflare.com/","html_tokens":333579,"markdown_tokens":615,"pruned_tokens":167}},
{"rank":170,"domain":"eventbrite.com","origin":"https://eventbrite.com","robots":{"status":"ok","http":200,"url":"https://eventbrite.com/robots.txt","bytes":7559,"groups":135},"bots":{"GPTBot":["partial","*",51,2],"ChatGPT-User":["partial","*",51,2],"OAI-SearchBot":["partial","*",51,2],"ClaudeBot":["partial","*",51,2],"Claude-User":["partial","*",51,2],"Claude-SearchBot":["partial","*",51,2],"anthropic-ai":["partial","*",51,2],"PerplexityBot":["partial","*",51,2],"Perplexity-User":["partial","*",51,2],"Google-Extended":["partial","*",51,2],"Applebot-Extended":["partial","*",51,2],"CCBot":["blocked","named",1,0],"Bytespider":["partial","*",51,2],"Amazonbot":["partial","*",51,2],"meta-externalagent":["partial","*",51,2],"cohere-ai":["partial","*",51,2],"Diffbot":["partial","*",51,2],"ContextFetchBot":["partial","*",51,2]},"llms":{"status":"missing","http":404,"bytes":79245},"home":{"status":"ok","http":200,"url":"https://www.eventbrite.com","html_tokens":60526,"markdown_tokens":103,"pruned_tokens":103}},
{"rank":171,"domain":"oup.com","origin":"https://oup.com","robots":{"status":"html","http":202,"url":"http://global.oup.com/robots.txt","bytes":2003,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":172,"domain":"mail.google.com","origin":"https://mail.google.com","robots":{"status":"ok","http":200,"url":"https://mail.google.com/robots.txt","bytes":130,"groups":1},"bots":{"GPTBot":["partial","*",1,3],"ChatGPT-User":["partial","*",1,3],"OAI-SearchBot":["partial","*",1,3],"ClaudeBot":["partial","*",1,3],"Claude-User":["partial","*",1,3],"Claude-SearchBot":["partial","*",1,3],"anthropic-ai":["partial","*",1,3],"PerplexityBot":["partial","*",1,3],"Perplexity-User":["partial","*",1,3],"Google-Extended":["partial","*",1,3],"Applebot-Extended":["partial","*",1,3],"CCBot":["partial","*",1,3],"Bytespider":["partial","*",1,3],"Amazonbot":["partial","*",1,3],"meta-externalagent":["partial","*",1,3],"cohere-ai":["partial","*",1,3],"Diffbot":["partial","*",1,3],"ContextFetchBot":["partial","*",1,3]},"llms":{"status":"disallowed"},"home":{"status":"ok","http":200,"url":"https://accounts.google.com/v3/signin/identifier?continue=https://mail.google.com/mail/u/0/&emr=1&followup=https://mail.google.com/mail/u/0/&osid=1&passive=1209600&service=mail&flowName=WebLiteSignIn&flowEntry=ServiceLogin&dsh=S1595798252:1792374156715315","html_tokens":236625,"markdown_tokens":33,"pruned_tokens":33}},
{"rank":173,"domain":"get.adobe.com","origin":"https://get.adobe.com","robots":{"status":"ok","http":200,"url":"https://get.adobe.com/robots.txt","bytes":81,"groups":1},"bots":{"GPTBot":["allowed","*",0,1],"ChatGPT-User":["allowed","*",0,1],"OAI-SearchBot":["allowed","*",0,1],"ClaudeBot":["allowed","*",0,1],"Claude-User":["allowed","*",0,1],"Claude-SearchBot":["allowed","*",0,1],"anthropic-ai":["allowed","*",0,1],"PerplexityBot":["allowed","*",0,1],"Perplexity-User":["allowed","*",0,1],"Google-Extended":["allowed","*",0,1],"Applebot-Extended":["allowed","*",0,1],"CCBot":["allowed","*",0,1],"Bytespider":["allowed","*",0,1],"Amazonbot":["allowed","*",0,1],"meta-externalagent":["allowed","*",0,1],"cohere-ai":["allowed","*",0,1],"Diffbot":["allowed","*",0,1],"ContextFetchBot":["allowed","*",0,1]},"llms":{"status":"http_error","http":403,"bytes":203},"home":{"status":"ok","http":200,"url":"https://www.adobe.com/","html_tokens":12494,"markdown_tokens":874,"pruned_tokens":67}},
{"rank":174,"domain":"etsy.com","origin":"https://etsy.com","robots":{"status":"ok","http":200,"url":"https://www.etsy.com/robots.txt","bytes":52821,"groups":3},"bots":{"GPTBot":["partial","*",1650,120],"ChatGPT-User":["partial","*",1650,120],"OAI-SearchBot":["partial","*",1650,120],"ClaudeBot":["partial","*",1650,120],"Claude-User":["partial","*",1650,120],"Claude-SearchBot":["partial","*",1650,120],"anthropic-ai":["partial","*",1650,120],"PerplexityBot":["partial","*",1650,120],"Perplexity-User":["partial","*",1650,120],"Google-Extended":["partial","*",1650,120],"Applebot-Extended":["partial","*",1650,120],"CCBot":["partial","*",1650,120],"Bytespider":["partial","*",1650,120],"Amazonbot":["partial","*",1650,120],"meta-externalagent":["partial","*",1650,120],"cohere-ai":["partial","*",1650,120],"Diffbot":["partial","*",1650,120],"ContextFetchBot":["partial","*",1650,120]},"llms":{"status":"present","http":200,"bytes":26713},"home":{"status":"http_error","http":403,"url":"https://www.etsy.com/"}},
{"rank":175,"domain":"addtoany.com","origin":"https://addtoany.com","robots":{"status":"ok","http":200,"url":"https://www.addtoany.com/robots.txt","bytes":308,"groups":2},"bots":{"GPTBot":["partial","*",6,0],"ChatGPT-User":["partial","*",6,0],"OAI-SearchBot":["partial","*",6,0],"ClaudeBot":["partial","*",6,0],"Claude-User":["partial","*",6,0],"Claude-SearchBot":["partial","*",6,0],"anthropic-ai":["partial","*",6,0],"PerplexityBot":["partial","*",6,0],"Perplexity-User":["partial","*",6,0],"Google-Extended":["partial","*",6,0],"Applebot-Extended":["partial","*",6,0],"CCBot":["partial","*",6,0],"Bytespider":["partial","*",6,0],"Amazonbot":["partial","*",6,0],"meta-externalagent":["partial","*",6,0],"cohere-ai":["partial","*",6,0],"Diffbot":["partial","*",6,0],"ContextFetchBot":["partial","*",6,0]},"llms":{"status":"missing","http":404,"bytes":2021},"home":{"status":"ok","http":200,"url":"https://www.addtoany.com/","html_tokens":6465,"markdown_tokens":385,"pruned_tokens":385}},
{"rank":176,"domain":"wired.com","origin":"https://wired.com","robots":{"status":"ok","http":200,"url":"https://www.wired.com/robots.txt","bytes":2245,"groups":3},"bots":{"GPTBot":["partial","*",14,17],"ChatGPT-User":["partial","*",14,17],"OAI-SearchBot":["partial","*",14,17],"ClaudeBot":["blocked","named",1,0],"Claude-User":["blocked","named",1,0],"Claude-SearchBot":["blocked","named",1,0],"anthropic-ai":["partial","*",14,17],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["blocked","named",1,0],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",14,17]},"llms":{"status":"missing","http":404,"bytes":866157},"home":{"status":"ok","http":200,"url":"https://www.wired.com/","html_tokens":390211,"markdown_tokens":57,"pruned_tokens":57}},
{"rank":177,"domain":"aboutads.info","origin":"https://aboutads.info","robots":{"status":"html","http":200,"url":"https://youradchoices.com/","bytes":34950,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":178,"domain":"openstreetmap.org","origin":"https://openstreetmap.org","robots":{"status":"ok","http":200,"url":"https://www.openstreetmap.org/robots.txt","bytes":891,"groups":1},"bots":{"GPTBot":["partial","*",21,1],"ChatGPT-User":["partial","*",21,1],"OAI-SearchBot":["partial","*",21,1],"ClaudeBot":["partial","*",21,1],"Claude-User":["partial","*",21,1],"Claude-SearchBot":["partial","*",21,1],"anthropic-ai":["partial","*",21,1],"PerplexityBot":["partial","*",21,1],"Perplexity-User":["partial","*",21,1],"Google-Extended":["partial","*",21,1],"Applebot-Extended":["partial","*",21,1],"CCBot":["partial","*",21,1],"Bytespider":["partial","*",21,1],"Amazonbot":["partial","*",21,1],"meta-externalagent":["partial","*",21,1],"cohere-ai":["partial","*",21,1],"Diffbot":["partial","*",21,1],"ContextFetchBot":["partial","*",21,1]},"llms":{"status":"missing","http":404,"bytes":0},"home":{"status":"ok","http":200,"url":"https://www.openstreetmap.org/","html_tokens":8084,"markdown_tokens":111,"pruned_tokens":61}},
{"rank":179,"domain":"gouv.fr","origin":"https://gouv.fr","robots":{"status":"unreachable","http":null,"url":"https://gouv.fr/robots.txt","bytes":0,"groups":0,"error":"cannot resolve gouv.fr"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":180,"domain":"time.com","origin":"https://time.com","robots":{"status":"ok","http":200,"url":"https://time.com/robots.txt","bytes":2190,"groups":3},"bots":{"GPTBot":["partial","*",1,1],"ChatGPT-User":["partial","*",1,1],"OAI-SearchBot":["partial","*",1,1],"ClaudeBot":["partial","*",1,1],"Claude-User":["partial","*",1,1],"Claude-SearchBot":["partial","*",1,1],"anthropic-ai":["partial","*",1,1],"PerplexityBot":["partial","*",1,1],"Perplexity-User":["partial","*",1,1],"Google-Extended":["partial","*",1,1],"Applebot-Extended":["partial","*",1,1],"CCBot":["partial","*",1,1],"Bytespider":["partial","*",1,1],"Amazonbot":["partial","*",1,1],"meta-externalagent":["partial","*",1,1],"cohere-ai":["partial","*",1,1],"Diffbot":["partial","*",1,1],"ContextFetchBot":["partial","*",1,1]},"llms":{"status":"http_error","http":406,"bytes":0},"home":{"status":"ok","http":200,"url":"https://time.com/","html_tokens":332154,"markdown_tokens":205,"pruned_tokens":125}},
{"rank":181,"domain":"npr.org","origin":"https://npr.org","robots":{"status":"unreachable","http":null,"url":"https://npr.org/robots.txt","bytes":0,"groups":0,"error":"<StreamReset stream_id:1, error_code:2, remote_reset:True>"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":182,"domain":"googleusercontent.com","origin":"https://googleusercontent.com","robots":{"status":"missing","http":404,"url":"https://googleusercontent.com/robots.txt","bytes":1571,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":1569},"home":{"status":"http_error","http":404,"url":"https://googleusercontent.com/"}},
{"rank":183,"domain":"samsung.com","origin":"https://samsung.com","robots":{"status":"ok","http":200,"url":"https://www.samsung.com/robots.txt","bytes":10085,"groups":4},"bots":{"GPTBot":["partial","named",49,16],"ChatGPT-User":["partial","named",49,16],"OAI-SearchBot":["partial","named",49,16],"ClaudeBot":["partial","*",49,16],"Claude-User":["partial","*",49,16],"Claude-SearchBot":["partial","*",49,16],"anthropic-ai":["partial","*",49,16],"PerplexityBot":["partial","named",49,16],"Perplexity-User":["partial","*",49,16],"Google-Extended":["partial","named",49,16],"Applebot-Extended":["partial","*",49,16],"CCBot":["partial","*",49,16],"Bytespider":["partial","*",49,16],"Amazonbot":["partial","*",49,16],"meta-externalagent":["partial","*",49,16],"cohere-ai":["partial","*",49,16],"Diffbot":["partial","*",49,16],"ContextFetchBot":["partial","*",49,16]},"llms":{"status":"present","http":200,"bytes":4597},"home":{"status":"ok","http":200,"url":"https://www.samsung.com/us/","html_tokens":99078,"markdown_tokens":173,"pruned_tokens":120}},
{"rank":184,"domain":"stackoverflow.com","origin":"https://stackoverflow.com","robots":{"status":"refused","http":403,"url":"https://stackoverflow.com/robots.txt","bytes":5534,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":185,"domain":"behance.net","origin":"https://behance.net","robots":{"status":"ok","http":200,"url":"https://www.behance.net/robots.txt","bytes":1317,"groups":2},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["partial","*",24,0],"OAI-SearchBot":["partial","*",24,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",24,0],"Claude-SearchBot":["partial","*",24,0],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["partial","*",24,0],"Perplexity-User":["partial","*",24,0],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["partial","*",24,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["partial","*",24,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",24,0]},"llms":{"status":"missing","http":404,"bytes":385085},"home":{"status":"http_error","http":403,"url":"https://www.behance.net/"}},
{"rank":186,"domain":"slideshare.net","origin":"https://slideshare.net","robots":{"status":"ok","http":200,"url":"https://www.slideshare.net/robots.txt","bytes":1443,"groups":11},"bots":{"GPTBot":["partial","*",40,0],"ChatGPT-User":["partial","*",40,0],"OAI-SearchBot":["partial","*",40,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",40,0],"Claude-SearchBot":["partial","*",40,0],"anthropic-ai":["partial","*",40,0],"PerplexityBot":["partial","*",40,0],"Perplexity-User":["partial","*",40,0],"Google-Extended":["partial","*",40,0],"Applebot-Extended":["partial","*",40,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["partial","*",40,0],"meta-externalagent":["partial","*",40,0],"cohere-ai":["partial","*",40,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",40,0]},"llms":{"status":"html","http":200,"bytes":373094},"home":{"status":"ok","http":200,"url":"https://www.slideshare.net/","html_tokens":136113,"markdown_tokens":755,"pruned_tokens":217}},
{"rank":187,"domain":"tandfonline.com","origin":"https://tandfonline.com","robots":{"status":"ok","http":200,"url":"https://www.tandfonline.com/robots.txt","bytes":1379,"groups":3},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["partial","*",24,16],"OAI-SearchBot":["partial","*",24,16],"ClaudeBot":["partial","*",24,16],"Claude-User":["partial","*",24,16],"Claude-SearchBot":["partial","*",24,16],"anthropic-ai":["partial","*",24,16],"PerplexityBot":["partial","*",24,16],"Perplexity-User":["partial","*",24,16],"Google-Extended":["partial","*",24,16],"Applebot-Extended":["partial","*",24,16],"CCBot":["partial","*",24,16],"Bytespider":["partial","*",24,16],"Amazonbot":["partial","*",24,16],"meta-externalagent":["partial","*",24,16],"cohere-ai":["partial","*",24,16],"Diffbot":["partial","*",24,16],"ContextFetchBot":["partial","*",24,16]},"llms":{"status":"http_error","http":403,"bytes":5568},"home":{"status":"http_error","http":403,"url":"https://tandfonline.com/"}},
{"rank":188,"domain":"ebay.com","origin":"https://ebay.com","robots":{"status":"ok","http":200,"url":"https://www.ebay.com/robots.txt","bytes":24164,"groups":9},"bots":{"GPTBot":["partial","named",1,11],"ChatGPT-User":["partial","named",210,13],"OAI-SearchBot":["partial","named",210,13],"ClaudeBot":["partial","named",1,11],"Claude-User":["partial","named",210,13],"Claude-SearchBot":["partial","named",210,13],"anthropic-ai":["partial","named",1,11],"PerplexityBot":["partial","named",1,11],"Perplexity-User":["partial","*",216,18],"Google-Extended":["partial","*",216,18],"Applebot-Extended":["partial","named",1,11],"CCBot":["partial","named",1,11],"Bytespider":["partial","named",1,11],"Amazonbot":["partial","named",1,11],"meta-externalagent":["partial","named",1,11],"cohere-ai":["partial","*",216,18],"Diffbot":["partial","*",216,18],"ContextFetchBot":["partial","*",216,18]},"llms":{"status":"missing","http":404,"bytes":73840},"home":{"status":"http_error","http":403,"url":"https://www.ebay.com/"}},
{"rank":189,"domain":"hp.com","origin":"https://hp.com","robots":{"status":"ok","http":200,"url":"https://www.hp.com/robots.txt","bytes":17151,"groups":14},"bots":{"GPTBot":["allowed","named",0,1],"ChatGPT-User":["allowed","named",0,1],"OAI-SearchBot":["allowed","named",0,1],"ClaudeBot":["allowed","named",0,1],"Claude-User":["allowed","named",0,1],"Claude-SearchBot":["allowed","named",0,1],"anthropic-ai":["partial","*",170,10],"PerplexityBot":["allowed","named",0,1],"Perplexity-User":["allowed","named",0,1],"Google-Extended":["allowed","named",0,1],"Applebot-Extended":["partial","*",170,10],"CCBot":["partial","*",170,10],"Bytespider":["partial","*",170,10],"Amazonbot":["partial","*",170,10],"meta-externalagent":["partial","*",170,10],"cohere-ai":["partial","*",170,10],"Diffbot":["partial","*",170,10],"ContextFetchBot":["partial","*",170,10]},"llms":{"status":"missing","http":404,"bytes":317803},"home":{"status":"disallowed","error":"robots.txt of www-redirect.ext.hp.com not readable (html)"}},
{"rank":190,"domain":"unesco.org","origin":"https://unesco.org","robots":{"status":"html","http":200,"url":"https://www.unesco.org/robots.txt","bytes":48362,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":191,"domain":"fr.wikipedia.org","origin":"https://fr.wikipedia.org","robots":{"status":"ok","http":200,"url":"https://fr.wikipedia.org/robots.txt","bytes":17017,"groups":33},"bots":{"GPTBot":["partial","*",224,4],"ChatGPT-User":["partial","*",224,4],"OAI-SearchBot":["partial","*",224,4],"ClaudeBot":["partial","*",224,4],"Claude-User":["partial","*",224,4],"Claude-SearchBot":["partial","*",224,4],"anthropic-ai":["partial","*",224,4],"PerplexityBot":["partial","*",224,4],"Perplexity-User":["partial","*",224,4],"Google-Extended":["partial","*",224,4],"Applebot-Extended":["partial","*",224,4],"CCBot":["partial","*",224,4],"Bytespider":["partial","*",224,4],"Amazonbot":["partial","*",224,4],"meta-externalagent":["partial","*",224,4],"cohere-ai":["partial","*",224,4],"Diffbot":["partial","*",224,4],"ContextFetchBot":["partial","*",224,4]},"llms":{"status":"missing","http":404,"bytes":1868},"home":{"status":"ok","http":200,"url":"https://fr.wikipedia.org/wiki/Wikip%C3%A9dia:Accueil_principal","html_tokens":59459,"markdown_tokens":1287,"pruned_tokens":62}},
{"rank":192,"domain":"gc.ca","origin":"https://www.gc.ca","robots":{"status":"ok","http":200,"url":"https://www.canada.ca/robots.txt","bytes":2730,"groups":1},"bots":{"GPTBot":["partial","*",47,0],"ChatGPT-User":["partial","*",47,0],"OAI-SearchBot":["partial","*",47,0],"ClaudeBot":["partial","*",47,0],"Claude-User":["partial","*",47,0],"Claude-SearchBot":["partial","*",47,0],"anthropic-ai":["partial","*",47,0],"PerplexityBot":["partial","*",47,0],"Perplexity-User":["partial","*",47,0],"Google-Extended":["partial","*",47,0],"Applebot-Extended":["partial","*",47,0],"CCBot":["partial","*",47,0],"Bytespider":["partial","*",47,0],"Amazonbot":["partial","*",47,0],"meta-externalagent":["partial","*",47,0],"cohere-ai":["partial","*",47,0],"Diffbot":["partial","*",47,0],"ContextFetchBot":["partial","*",47,0]},"llms":{"status":"html","http":200,"bytes":13218},"home":{"status":"ok","http":200,"url":"https://www.canada.ca/","html_tokens":2258,"markdown_tokens":49,"pruned_tokens":22}},
{"rank":193,"domain":"chatgpt.com","origin":"https://chatgpt.com","robots":{"status":"ok","http":200,"url":"https://chatgpt.com/robots.txt","bytes":4390,"groups":14},"bots":{"GPTBot":["partial","*",6,165],"ChatGPT-User":["partial","*",6,165],"OAI-SearchBot":["partial","*",6,165],"ClaudeBot":["partial","*",6,165],"Claude-User":["partial","*",6,165],"Claude-SearchBot":["partial","*",6,165],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["partial","*",6,165],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["partial","*",6,165],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["partial","*",6,165],"meta-externalagent":["partial","*",6,165],"cohere-ai":["partial","*",6,165],"Diffbot":["partial","*",6,165],"ContextFetchBot":["partial","*",6,165]},"llms":{"status":"disallowed"},"home":{"status":"http_error","http":403,"url":"https://chatgpt.com/"}},
{"rank":194,"domain":"techcrunch.com","origin":"https://techcrunch.com","robots":{"status":"ok","http":200,"url":"https://techcrunch.com/robots.txt","bytes":710,"groups":13},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["blocked","named",1,0],"OAI-SearchBot":["partial","*",5,1],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",5,1],"Claude-SearchBot":["partial","*",5,1],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["partial","*",5,1],"Perplexity-User":["partial","*",5,1],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["partial","*",5,1],"meta-externalagent":["partial","*",5,1],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",5,1]},"llms":{"status":"missing","http":404,"bytes":208392},"home":{"status":"ok","http":200,"url":"https://techcrunch.com/","html_tokens":112394,"markdown_tokens":243,"pruned_tokens":105}},
{"rank":195,"domain":"hubspot.com","origin":"https://hubspot.com","robots":{"status":"ok","http":200,"url":"https://www.hubspot.com/robots.txt","bytes":12606,"groups":1},"bots":{"GPTBot":["partial","*",356,1],"ChatGPT-User":["partial","*",356,1],"OAI-SearchBot":["partial","*",356,1],"ClaudeBot":["partial","*",356,1],"Claude-User":["partial","*",356,1],"Claude-SearchBot":["partial","*",356,1],"anthropic-ai":["partial","*",356,1],"PerplexityBot":["partial","*",356,1],"Perplexity-User":["partial","*",356,1],"Google-Extended":["partial","*",356,1],"Applebot-Extended":["partial","*",356,1],"CCBot":["partial","*",356,1],"Bytespider":["partial","*",356,1],"Amazonbot":["partial","*",356,1],"meta-externalagent":["partial","*",356,1],"cohere-ai":["partial","*",356,1],"Diffbot":["partial","*",356,1],"ContextFetchBot":["partial","*",356,1]},"llms":{"status":"present","http":200,"bytes":35564},"home":{"status":"ok","http":200,"url":"https://www.hubspot.com/","html_tokens":160793,"markdown_tokens":688,"pruned_tokens":226}},
{"rank":196,"domain":"validator.w3.org","origin":"https://validator.w3.org","robots":{"status":"ok","http":200,"url":"https://validator.w3.org/robots.txt","bytes":280,"groups":1},"bots":{"GPTBot":["partial","*",10,0],"ChatGPT-User":["partial","*",10,0],"OAI-SearchBot":["partial","*",10,0],"ClaudeBot":["partial","*",10,0],"Claude-User":["partial","*",10,0],"Claude-SearchBot":["partial","*",10,0],"anthropic-ai":["partial","*",10,0],"PerplexityBot":["partial","*",10,0],"Perplexity-User":["partial","*",10,0],"Google-Extended":["partial","*",10,0],"Applebot-Extended":["partial","*",10,0],"CCBot":["partial","*",10,0],"Bytespider":["partial","*",10,0],"Amazonbot":["partial","*",10,0],"meta-externalagent":["partial","*",10,0],"cohere-ai":["partial","*",10,0],"Diffbot":["partial","*",10,0],"ContextFetchBot":["partial","*",10,0]},"llms":{"status":"missing","http":404,"bytes":236},"home":{"status":"ok","http":200,"url":"https://validator.w3.org/","html_tokens":6553,"markdown_tokens":83,"pruned_tokens":83}},
{"rank":197,"domain":"ft.com","origin":"https://ft.com","robots":{"status":"ok","http":200,"url":"https://www.ft.com/robots.txt","bytes":1556,"groups":20},"bots":{"GPTBot":["partial","*",14,2],"ChatGPT-User":["partial","*",14,2],"OAI-SearchBot":["partial","*",14,2],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",14,2],"Claude-SearchBot":["partial","*",14,2],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["blocked","named",1,0],"Google-Extended":["partial","named",1,1],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["partial","*",14,2],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",14,2]},"llms":{"status":"missing","http":404,"bytes":21535},"home":{"status":"http_error","http":403,"url":"https://www.ft.com/"}},
{"rank":198,"domain":"iso.org","origin":"https://iso.org","robots":{"status":"refused","http":403,"url":"https://iso.org/robots.txt","bytes":5630,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":199,"domain":"sagepub.com","origin":"https://sagepub.com","robots":{"status":"ok","http":200,"url":"https://www.sagepub.com/robots.txt","bytes":5472,"groups":43},"bots":{"GPTBot":["allowed","named",0,0],"ChatGPT-User":["allowed","named",0,0],"OAI-SearchBot":["partial","*",37,32],"ClaudeBot":["allowed","named",0,0],"Claude-User":["partial","*",37,32],"Claude-SearchBot":["partial","*",37,32],"anthropic-ai":["allowed","named",0,0],"PerplexityBot":["partial","*",37,32],"Perplexity-User":["partial","*",37,32],"Google-Extended":["allowed","named",0,0],"Applebot-Extended":["allowed","named",0,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",37,32]},"llms":{"status":"missing","http":404,"bytes":21880},"home":{"status":"ok","http":200,"url":"https://www.sagepub.com/","html_tokens":18383,"markdown_tokens":476,"pruned_tokens":163}},
{"rank":200,"domain":"mdpi.com","origin":"https://mdpi.com","robots":{"status":"refused","http":403,"url":"https://mdpi.com/robots.txt","bytes":372,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":201,"domain":"cloud.microsoft","origin":"https://cloud.microsoft","robots":{"status":"html","http":200,"url":"https://m365.cloud.microsoft/","bytes":348341,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":202,"domain":"wp.com","origin":"https://wp.com","robots":{"status":"ok","http":200,"url":"https://wordpress.com/robots.txt","bytes":1387,"groups":1},"bots":{"GPTBot":["partial","*",32,7],"ChatGPT-User":["partial","*",32,7],"OAI-SearchBot":["partial","*",32,7],"ClaudeBot":["partial","*",32,7],"Claude-User":["partial","*",32,7],"Claude-SearchBot":["partial","*",32,7],"anthropic-ai":["partial","*",32,7],"PerplexityBot":["partial","*",32,7],"Perplexity-User":["partial","*",32,7],"Google-Extended":["partial","*",32,7],"Applebot-Extended":["partial","*",32,7],"CCBot":["partial","*",32,7],"Bytespider":["partial","*",32,7],"Amazonbot":["partial","*",32,7],"meta-externalagent":["partial","*",32,7],"cohere-ai":["partial","*",32,7],"Diffbot":["partial","*",32,7],"ContextFetchBot":["partial","*",32,7]},"llms":{"status":"present","http":200,"bytes":7355},"home":{"status":"ok","http":200,"url":"https://wordpress.com/","html_tokens":85006,"markdown_tokens":155,"pruned_tokens":126}},
{"rank":203,"domain":"loc.gov","origin":"https://loc.gov","robots":{"status":"ok","http":200,"url":"https://www.loc.gov/robots.txt","bytes":482,"groups":4},"bots":{"GPTBot":["partial","*",12,0],"ChatGPT-User":["partial","*",12,0],"OAI-SearchBot":["partial","*",12,0],"ClaudeBot":["partial","*",12,0],"Claude-User":["partial","*",12,0],"Claude-SearchBot":["partial","*",12,0],"anthropic-ai":["partial","*",12,0],"PerplexityBot":["partial","*",12,0],"Perplexity-User":["partial","*",12,0],"Google-Extended":["partial","*",12,0],"Applebot-Extended":["partial","*",12,0],"CCBot":["partial","*",12,0],"Bytespider":["partial","*",12,0],"Amazonbot":["partial","*",12,0],"meta-externalagent":["partial","*",12,0],"cohere-ai":["partial","*",12,0],"Diffbot":["partial","*",12,0],"ContextFetchBot":["partial","*",12,0]},"llms":{"status":"missing","http":404,"bytes":206},"home":{"status":"ok","http":200,"url":"https://www.loc.gov/","html_tokens":19278,"markdown_tokens":134,"pruned_tokens":134}},
{"rank":204,"domain":"go.com","origin":"https://go.com","robots":{"status":"ok","http":200,"url":"https://www.go.com/robots.txt","bytes":212,"groups":3},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["partial","*",5,0],"OAI-SearchBot":["partial","*",5,0],"ClaudeBot":["partial","*",5,0],"Claude-User":["partial","*",5,0],"Claude-SearchBot":["partial","*",5,0],"anthropic-ai":["partial","*",5,0],"PerplexityBot":["partial","*",5,0],"Perplexity-User":["partial","*",5,0],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["partial","*",5,0],"CCBot":["partial","*",5,0],"Bytespider":["partial","*",5,0],"Amazonbot":["partial","*",5,0],"meta-externalagent":["partial","*",5,0],"cohere-ai":["partial","*",5,0],"Diffbot":["partial","*",5,0],"ContextFetchBot":["partial","*",5,0]},"llms":{"status":"html","http":200,"bytes":359619},"home":{"status":"ok","http":200,"url":"https://www.go.com/","html_tokens":89894,"markdown_tokens":1837,"pruned_tokens":81}},
{"rank":205,"domain":"aol.com","origin":"https://aol.com","robots":{"status":"ok","http":200,"url":"https://www.aol.com/robots.txt","bytes":2827,"groups":44},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["partial","named",17,1],"OAI-SearchBot":["partial","*",17,1],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",17,1],"Claude-SearchBot":["partial","*",17,1],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["partial","named",17,1],"Perplexity-User":["partial","*",17,1],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["partial","*",17,1],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["partial","*",17,1],"meta-externalagent":["partial","*",17,1],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",17,1]},"llms":{"status":"missing","http":404,"bytes":859},"home":{"status":"ok","http":200,"url":"https://www.aol.com/","html_tokens":486660,"markdown_tokens":202,"pruned_tokens":164}},
{"rank":206,"domain":"berkeley.edu","origin":"https://berkeley.edu","robots":{"status":"ok","http":200,"url":"https://www.berkeley.edu/robots.txt","bytes":190,"groups":1},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"http_error","http":403,"bytes":5543},"home":{"status":"http_error","http":403,"url":"https://berkeley.edu/"}},
{"rank":207,"domain":"wpguardian.com","origin":"https://wpguardian.com","robots":{"status":"ok","http":200,"url":"https://wpguardian.com/robots.txt","bytes":172,"groups":1},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"missing","http":404,"bytes":163742},"home":{"status":"ok","http":200,"url":"https://wpguardian.com/","html_tokens":60148,"markdown_tokens":790,"pruned_tokens":144}},
{"rank":208,"domain":"wpguardian.io","origin":"https://wpguardian.io","robots":{"status":"ok","http":200,"url":"https://wpguardian.com/robots.txt","bytes":172,"groups":1},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"missing","http":404,"bytes":163742},"home":{"status":"ok","http":200,"url":"https://wpguardian.com/","html_tokens":60148,"markdown_tokens":790,"pruned_tokens":144}},
{"rank":209,"domain":"outlook.live.com","origin":"https://outlook.live.com","robots":{"status":"html","http":200,"url":"https://outlook.live.com/mail/","bytes":14277,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":210,"domain":"google.de","origin":"https://google.de","robots":{"status":"ok","http":200,"url":"https://www.google.de/robots.txt","bytes":6624,"groups":4},"bots":{"GPTBot":["partial","*",169,64],"ChatGPT-User":["partial","*",169,64],"OAI-SearchBot":["partial","*",169,64],"ClaudeBot":["partial","*",169,64],"Claude-User":["partial","*",169,64],"Claude-SearchBot":["partial","*",169,64],"anthropic-ai":["partial","*",169,64],"PerplexityBot":["partial","*",169,64],"Perplexity-User":["partial","*",169,64],"Google-Extended":["partial","*",169,64],"Applebot-Extended":["partial","*",169,64],"CCBot":["partial","*",169,64],"Bytespider":["partial","*",169,64],"Amazonbot":["partial","*",169,64],"meta-externalagent":["partial","*",169,64],"cohere-ai":["partial","*",169,64],"Diffbot":["partial","*",169,64],"ContextFetchBot":["partial","*",169,64]},"llms":{"status":"missing","http":404,"bytes":1569},"home":{"status":"ok","http":200,"url":"https://www.google.com/","html_tokens":21437,"markdown_tokens":30,"pruned_tokens":30}},
{"rank":211,"domain":"outlook.office.com","origin":"https://outlook.office.com","robots":{"status":"html","http":200,"url":"https://outlook.office.com/mail/","bytes":14245,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":212,"domain":"cornell.edu","origin":"https://cornell.edu","robots":{"status":"ok","http":200,"url":"https://www.cornell.edu/robots.txt","bytes":1046,"groups":5},"bots":{"GPTBot":["partial","*",10,0],"ChatGPT-User":["partial","*",10,0],"OAI-SearchBot":["partial","*",10,0],"ClaudeBot":["partial","*",10,0],"Claude-User":["partial","*",10,0],"Claude-SearchBot":["partial","*",10,0],"anthropic-ai":["partial","*",10,0],"PerplexityBot":["partial","*",10,0],"Perplexity-User":["partial","*",10,0],"Google-Extended":["partial","*",10,0],"Applebot-Extended":["partial","*",10,0],"CCBot":["partial","*",10,0],"Bytespider":["partial","*",10,0],"Amazonbot":["partial","*",10,0],"meta-externalagent":["partial","*",10,0],"cohere-ai":["partial","*",10,0],"Diffbot":["partial","*",10,0],"ContextFetchBot":["partial","*",10,0]},"llms":{"status":"missing","http":404,"bytes":41182},"home":{"status":"ok","http":200,"url":"https://www.cornell.edu/","html_tokens":20191,"markdown_tokens":23,"pruned_tokens":23}},
{"rank":213,"domain":"web.facebook.com","origin":"https://web.facebook.com","robots":{"status":"ok","http":200,"url":"https://web.facebook.com/robots.txt","bytes":48799,"groups":36},"bots":{"GPTBot":["partial","named",41,4],"ChatGPT-User":["blocked","*",1,0],"OAI-SearchBot":["blocked","*",1,0],"ClaudeBot":["partial","named",41,4],"Claude-User":["blocked","*",1,0],"Claude-SearchBot":["blocked","*",1,0],"anthropic-ai":["blocked","*",1,0],"PerplexityBot":["partial","named",41,4],"Perplexity-User":["blocked","*",1,0],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["partial","named",41,4],"CCBot":["blocked","*",1,0],"Bytespider":["blocked","*",1,0],"Amazonbot":["partial","named",41,4],"meta-externalagent":["blocked","*",1,0],"cohere-ai":["blocked","*",1,0],"Diffbot":["blocked","*",1,0],"ContextFetchBot":["blocked","*",1,0]},"llms":{"status":"disallowed"},"home":{"status":"disallowed"}},
{"rank":214,"domain":"ca.gov","origin":"https://ca.gov","robots":{"status":"ok","http":200,"url":"https://www.ca.gov/robots.txt","bytes":379,"groups":1},"bots":{"GPTBot":["partial","*",7,1],"ChatGPT-User":["partial","*",7,1],"OAI-SearchBot":["partial","*",7,1],"ClaudeBot":["partial","*",7,1],"Claude-User":["partial","*",7,1],"Claude-SearchBot":["partial","*",7,1],"anthropic-ai":["partial","*",7,1],"PerplexityBot":["partial","*",7,1],"Perplexity-User":["partial","*",7,1],"Google-Extended":["partial","*",7,1],"Applebot-Extended":["partial","*",7,1],"CCBot":["partial","*",7,1],"Bytespider":["partial","*",7,1],"Amazonbot":["partial","*",7,1],"meta-externalagent":["partial","*",7,1],"cohere-ai":["partial","*",7,1],"Diffbot":["partial","*",7,1],"ContextFetchBot":["partial","*",7,1]},"llms":{"status":"missing","http":404,"bytes":18049},"home":{"status":"ok","http":200,"url":"https://www.ca.gov/","html_tokens":7588,"markdown_tokens":122,"pruned_tokens":122}},
{"rank":215,"domain":"usatoday.com","origin":"https://usatoday.com","robots":{"status":"ok","http":200,"url":"https://www.usatoday.com/robots.txt","bytes":16350,"groups":288},"bots":{"GPTBot":["partial","named",1,10],"ChatGPT-User":["partial","named",1,10],"OAI-SearchBot":["partial","named",1,10],"ClaudeBot":["blocked","named",1,0],"Claude-User":["blocked","named",1,0],"Claude-SearchBot":["blocked","named",1,0],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["partial","named",1,10],"Perplexity-User":["blocked","named",1,0],"Google-Extended":["partial","named",1,10],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",2,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",47,0]},"llms":{"status":"http_error","http":402,"bytes":254},"home":{"status":"http_error","http":402,"url":"https://www.usatoday.com/"}},
{"rank":216,"domain":"telegraph.co.uk","origin":"https://telegraph.co.uk","robots":{"status":"ok","http":200,"url":"https://www.telegraph.co.uk/robots.txt","bytes":3228,"groups":3},"bots":{"GPTBot":["partial","named",1,4],"ChatGPT-User":["partial","named",1,4],"OAI-SearchBot":["partial","named",1,4],"ClaudeBot":["partial","named",1,4],"Claude-User":["partial","named",1,4],"Claude-SearchBot":["partial","named",1,4],"anthropic-ai":["partial","named",1,4],"PerplexityBot":["partial","named",1,4],"Perplexity-User":["partial","named",1,4],"Google-Extended":["partial","named",35,2],"Applebot-Extended":["partial","named",1,4],"CCBot":["partial","named",1,4],"Bytespider":["partial","named",1,4],"Amazonbot":["partial","named",1,4],"meta-externalagent":["partial","named",1,4],"cohere-ai":["partial","named",1,4],"Diffbot":["partial","named",1,4],"ContextFetchBot":["partial","*",35,2]},"llms":{"status":"missing","http":404,"bytes":144534},"home":{"status":"http_error","http":402,"url":"https://www.telegraph.co.uk/"}},
{"rank":217,"domain":"booking.com","origin":"https://booking.com","robots":{"status":"ok","http":200,"url":"https://booking.com/robots.txt","bytes":39491,"groups":13},"bots":{"GPTBot":["partial","*",77,6],"ChatGPT-User":["partial","*",77,6],"OAI-SearchBot":["partial","*",77,6],"ClaudeBot":["partial","*",77,6],"Claude-User":["partial","*",77,6],"Claude-SearchBot":["partial","*",77,6],"anthropic-ai":["partial","*",77,6],"PerplexityBot":["partial","*",77,6],"Perplexity-User":["partial","*",77,6],"Google-Extended":["partial","*",77,6],"Applebot-Extended":["partial","*",77,6],"CCBot":["partial","*",77,6],"Bytespider":["partial","*",77,6],"Amazonbot":["partial","*",77,6],"meta-externalagent":["partial","*",77,6],"cohere-ai":["partial","*",77,6],"Diffbot":["partial","*",77,6],"ContextFetchBot":["partial","*",77,6]},"llms":{"status":"missing","http":404,"bytes":429689},"home":{"status":"ok","http":202,"url":"https://www.booking.com/","html_tokens":991,"markdown_tokens":40,"pruned_tokens":40}},
{"rank":218,"domain":"aliyun.com","origin":"https://aliyun.com","robots":{"status":"ok","http":200,"url":"https://www.aliyun.com/robots.txt","bytes":316,"groups":1},"bots":{"GPTBot":["partial","*",12,1],"ChatGPT-User":["partial","*",12,1],"OAI-SearchBot":["partial","*",12,1],"ClaudeBot":["partial","*",12,1],"Claude-User":["partial","*",12,1],"Claude-SearchBot":["partial","*",12,1],"anthropic-ai":["partial","*",12,1],"PerplexityBot":["partial","*",12,1],"Perplexity-User":["partial","*",12,1],"Google-Extended":["partial","*",12,1],"Applebot-Extended":["partial","*",12,1],"CCBot":["partial","*",12,1],"Bytespider":["partial","*",12,1],"Amazonbot":["partial","*",12,1],"meta-externalagent":["partial","*",12,1],"cohere-ai":["partial","*",12,1],"Diffbot":["partial","*",12,1],"ContextFetchBot":["partial","*",12,1]},"llms":{"status":"missing","http":404,"bytes":42405},"home":{"status":"ok","http":200,"url":"https://www.alibabacloud.com/en?_p_lc=1&f-8DC992C756BA=","html_tokens":52701,"markdown_tokens":20,"pruned_tokens":20}},
{"rank":219,"domain":"zendesk.com","origin":"https://zendesk.com","robots":{"status":"ok","http":200,"url":"https://www.zendesk.com/robots.txt","bytes":1028,"groups":1},"bots":{"GPTBot":["partial","*",27,1],"ChatGPT-User":["partial","*",27,1],"OAI-SearchBot":["partial","*",27,1],"ClaudeBot":["partial","*",27,1],"Claude-User":["partial","*",27,1],"Claude-SearchBot":["partial","*",27,1],"anthropic-ai":["partial","*",27,1],"PerplexityBot":["partial","*",27,1],"Perplexity-User":["partial","*",27,1],"Google-Extended":["partial","*",27,1],"Applebot-Extended":["partial","*",27,1],"CCBot":["partial","*",27,1],"Bytespider":["partial","*",27,1],"Amazonbot":["partial","*",27,1],"meta-externalagent":["partial","*",27,1],"cohere-ai":["partial","*",27,1],"Diffbot":["partial","*",27,1],"ContextFetchBot":["partial","*",27,1]},"llms":{"status":"present","http":200,"bytes":741350},"home":{"status":"ok","http":200,"url":"https://www.zendesk.com/","html_tokens":222354,"markdown_tokens":454,"pruned_tokens":241}},
{"rank":220,"domain":"bilibili.com","origin":"https://bilibili.com","robots":{"status":"ok","http":200,"url":"https://www.bilibili.com/robots.txt","bytes":65,"groups":1},"bots":{"GPTBot":["partial","*",2,0],"ChatGPT-User":["partial","*",2,0],"OAI-SearchBot":["partial","*",2,0],"ClaudeBot":["partial","*",2,0],"Claude-User":["partial","*",2,0],"Claude-SearchBot":["partial","*",2,0],"anthropic-ai":["partial","*",2,0],"PerplexityBot":["partial","*",2,0],"Perplexity-User":["partial","*",2,0],"Google-Extended":["partial","*",2,0],"Applebot-Extended":["partial","*",2,0],"CCBot":["partial","*",2,0],"Bytespider":["partial","*",2,0],"Amazonbot":["partial","*",2,0],"meta-externalagent":["partial","*",2,0],"cohere-ai":["partial","*",2,0],"Diffbot":["partial","*",2,0],"ContextFetchBot":["partial","*",2,0]},"llms":{"status":"present","http":200,"bytes":10994},"home":{"status":"ok","http":200,"url":"https://www.bilibili.com/","html_tokens":41110,"markdown_tokens":177,"pruned_tokens":175}},
{"rank":221,"domain":"squarespace.com","origin":"https://squarespace.com","robots":{"status":"ok","http":200,"url":"https://www.squarespace.com/robots.txt","bytes":2379,"groups":1},"bots":{"GPTBot":["partial","named",43,17],"ChatGPT-User":["partial","*",43,17],"OAI-SearchBot":["partial","*",43,17],"ClaudeBot":["partial","named",43,17],"Claude-User":["partial","*",43,17],"Claude-SearchBot":["partial","*",43,17],"anthropic-ai":["partial","named",43,17],"PerplexityBot":["partial","*",43,17],"Perplexity-User":["partial","*",43,17],"Google-Extended":["partial","named",43,17],"Applebot-Extended":["partial","named",43,17],"CCBot":["partial","named",43,17],"Bytespider":["partial","named",43,17],"Amazonbot":["partial","named",43,17],"meta-externalagent":["partial","named",43,17],"cohere-ai":["partial","named",43,17],"Diffbot":["partial","*",43,17],"ContextFetchBot":["partial","*",43,17]},"llms":{"status":"present","http":200,"bytes":2409},"home":{"status":"ok","http":200,"url":"https://www.squarespace.com/","html_tokens":154559,"markdown_tokens":276,"pruned_tokens":104}},
{"rank":222,"domain":"tripadvisor.com","origin":"https://tripadvisor.com","robots":{"status":"ok","http":200,"url":"https://www.tripadvisor.com/robots.txt","bytes":22870,"groups":8},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["partial","named",623,8],"OAI-SearchBot":["partial","named",623,8],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",621,8],"Claude-SearchBot":["partial","*",621,8],"anthropic-ai":["partial","*",621,8],"PerplexityBot":["partial","named",625,8],"Perplexity-User":["partial","*",621,8],"Google-Extended":["partial","named",622,9],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["partial","*",621,8],"ContextFetchBot":["partial","*",621,8]},"llms":{"status":"http_error","http":403,"bytes":775},"home":{"status":"http_error","http":403,"url":"https://www.tripadvisor.com/"}},
{"rank":223,"domain":"jotform.com","origin":"https://jotform.com","robots":{"status":"ok","http":200,"url":"https://www.jotform.com/robots.txt","bytes":12312,"groups":7},"bots":{"GPTBot":["partial","named",146,1],"ChatGPT-User":["partial","*",141,2],"OAI-SearchBot":["allowed","named",0,1],"ClaudeBot":["partial","named",146,1],"Claude-User":["partial","*",141,2],"Claude-SearchBot":["partial","*",141,2],"anthropic-ai":["partial","named",146,1],"PerplexityBot":["partial","named",146,1],"Perplexity-User":["partial","*",141,2],"Google-Extended":["partial","*",141,2],"Applebot-Extended":["partial","named",146,1],"CCBot":["partial","named",146,1],"Bytespider":["partial","named",146,1],"Amazonbot":["partial","*",141,2],"meta-externalagent":["partial","named",146,4],"cohere-ai":["partial","named",146,1],"Diffbot":["partial","named",146,1],"ContextFetchBot":["partial","*",141,2]},"llms":{"status":"missing","http":404,"bytes":145879},"home":{"status":"ok","http":200,"url":"https://www.jotform.com/","html_tokens":139955,"markdown_tokens":193,"pruned_tokens":44}},
{"rank":224,"domain":"fda.gov","origin":"https://fda.gov","robots":{"status":"missing","http":404,"url":"https://www.fda.gov/apology_objects/abuse-detection-apology.html","bytes":10,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":10},"home":{"status":"timeout","error":"timed out after 15 s"}},
{"rank":225,"domain":"yandex.ru","origin":"https://yandex.ru","robots":{"status":"ok","http":200,"url":"https://yandex.ru/robots.txt","bytes":40657,"groups":5},"bots":{"GPTBot":["partial","*",228,39],"ChatGPT-User":["partial","*",228,39],"OAI-SearchBot":["partial","named",228,39],"ClaudeBot":["partial","*",228,39],"Claude-User":["partial","*",228,39],"Claude-SearchBot":["partial","*",228,39],"anthropic-ai":["partial","*",228,39],"PerplexityBot":["partial","*",228,39],"Perplexity-User":["partial","*",228,39],"Google-Extended":["partial","*",228,39],"Applebot-Extended":["partial","*",228,39],"CCBot":["partial","*",228,39],"Bytespider":["partial","*",228,39],"Amazonbot":["partial","*",228,39],"meta-externalagent":["partial","*",228,39],"cohere-ai":["partial","*",228,39],"Diffbot":["partial","*",228,39],"ContextFetchBot":["partial","*",228,39]},"llms":{"status":"missing","http":404,"bytes":11773},"home":{"status":"disallowed","error":"robots.txt of dzen.ru not readable (error)"}},
{"rank":226,"domain":"rfc-editor.org","origin":"https://rfc-editor.org","robots":{"status":"ok","http":200,"url":"https://www.rfc-editor.org/robots.txt","bytes":135,"groups":1},"bots":{"GPTBot":["partial","*",3,0],"ChatGPT-User":["partial","*",3,0],"OAI-SearchBot":["partial","*",3,0],"ClaudeBot":["partial","*",3,0],"Claude-User":["partial","*",3,0],"Claude-SearchBot":["partial","*",3,0],"anthropic-ai":["partial","*",3,0],"PerplexityBot":["partial","*",3,0],"Perplexity-User":["partial","*",3,0],"Google-Extended":["partial","*",3,0],"Applebot-Extended":["partial","*",3,0],"CCBot":["partial","*",3,0],"Bytespider":["partial","*",3,0],"Amazonbot":["partial","*",3,0],"meta-externalagent":["partial","*",3,0],"cohere-ai":["partial","*",3,0],"Diffbot":["partial","*",3,0],"ContextFetchBot":["partial","*",3,0]},"llms":{"status":"missing","http":404,"bytes":15},"home":{"status":"ok","http":200,"url":"https://www.rfc-editor.org/","html_tokens":45267,"markdown_tokens":64,"pruned_tokens":64}},
{"rank":227,"domain":"baike.baidu.com","origin":"https://baike.baidu.com","robots":{"status":"ok","http":200,"url":"https://baike.baidu.com/robots.txt","bytes":3446,"groups":18},"bots":{"GPTBot":["blocked","*",1,0],"ChatGPT-User":["blocked","*",1,0],"OAI-SearchBot":["blocked","*",1,0],"ClaudeBot":["blocked","*",1,0],"Claude-User":["blocked","*",1,0],"Claude-SearchBot":["blocked","*",1,0],"anthropic-ai":["blocked","*",1,0],"PerplexityBot":["blocked","*",1,0],"Perplexity-User":["blocked","*",1,0],"Google-Extended":["blocked","*",1,0],"Applebot-Extended":["blocked","*",1,0],"CCBot":["blocked","*",1,0],"Bytespider":["blocked","*",1,0],"Amazonbot":["blocked","*",1,0],"meta-externalagent":["blocked","*",1,0],"cohere-ai":["blocked","*",1,0],"Diffbot":["blocked","*",1,0],"ContextFetchBot":["blocked","*",1,0]},"llms":{"status":"disallowed"},"home":{"status":"disallowed"}},
{"rank":228,"domain":"cnet.com","origin":"https://cnet.com","robots":{"status":"ok","http":200,"url":"https://www.cnet.com/robots.txt","bytes":22989,"groups":3},"bots":{"GPTBot":["partial","named",1,182],"ChatGPT-User":["partial","named",1,182],"OAI-SearchBot":["partial","named",1,182],"ClaudeBot":["partial","named",1,182],"Claude-User":["partial","named",1,182],"Claude-SearchBot":["partial","named",1,182],"anthropic-ai":["partial","named",1,182],"PerplexityBot":["partial","named",1,182],"Perplexity-User":["partial","named",1,182],"Google-Extended":["partial","*",47,0],"Applebot-Extended":["partial","named",1,182],"CCBot":["partial","named",1,182],"Bytespider":["partial","named",1,182],"Amazonbot":["partial","named",1,182],"meta-externalagent":["partial","named",1,182],"cohere-ai":["partial","named",1,182],"Diffbot":["partial","named",1,182],"ContextFetchBot":["partial","*",47,0]},"llms":{"status":"missing","http":404,"bytes":216615},"home":{"status":"ok","http":200,"url":"https://www.cnet.com/","html_tokens":108198,"markdown_tokens":537,"pruned_tokens":119}},
{"rank":229,"domain":"bandcamp.com","origin":"https://bandcamp.com","robots":{"status":"ok","http":200,"url":"https://bandcamp.com/robots.txt","bytes":1053,"groups":19},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["partial","*",10,4],"OAI-SearchBot":["partial","*",10,4],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",10,4],"Claude-SearchBot":["partial","*",10,4],"anthropic-ai":["partial","*",10,4],"PerplexityBot":["partial","*",10,4],"Perplexity-User":["partial","*",10,4],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["partial","*",10,4],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["partial","*",10,4],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",10,4]},"llms":{"status":"missing","http":404,"bytes":47821},"home":{"status":"ok","http":200,"url":"https://bandcamp.com/","html_tokens":23686,"markdown_tokens":58,"pruned_tokens":58}},
{"rank":230,"domain":"g.co","origin":"https://g.co","robots":{"status":"refused","http":400,"url":"https://g.co/robots.txt","bytes":9258,"groups":0,"error":"HTTP 400"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":231,"domain":"de.wikipedia.org","origin":"https://de.wikipedia.org","robots":{"status":"ok","http":200,"url":"https://de.wikipedia.org/robots.txt","bytes":23060,"groups":33},"bots":{"GPTBot":["partial","*",338,4],"ChatGPT-User":["partial","*",338,4],"OAI-SearchBot":["partial","*",338,4],"ClaudeBot":["partial","*",338,4],"Claude-User":["partial","*",338,4],"Claude-SearchBot":["partial","*",338,4],"anthropic-ai":["partial","*",338,4],"PerplexityBot":["partial","*",338,4],"Perplexity-User":["partial","*",338,4],"Google-Extended":["partial","*",338,4],"Applebot-Extended":["partial","*",338,4],"CCBot":["partial","*",338,4],"Bytespider":["partial","*",338,4],"Amazonbot":["partial","*",338,4],"meta-externalagent":["partial","*",338,4],"cohere-ai":["partial","*",338,4],"Diffbot":["partial","*",338,4],"ContextFetchBot":["partial","*",338,4]},"llms":{"status":"missing","http":404,"bytes":1868},"home":{"status":"ok","http":200,"url":"https://de.wikipedia.org/wiki/Wikipedia:Hauptseite","html_tokens":59513,"markdown_tokens":772,"pruned_tokens":157}},
{"rank":232,"domain":"dailymail.co.uk","origin":"https://dailymail.co.uk","robots":{"status":"ok","http":200,"url":"https://www.dailymail.com/robots.txt","bytes":5435,"groups":7},"bots":{"GPTBot":["partial","named",1,1],"ChatGPT-User":["partial","named",1,1],"OAI-SearchBot":["partial","named",1,1],"ClaudeBot":["partial","named",1,1],"Claude-User":["partial","named",1,1],"Claude-SearchBot":["partial","named",1,1],"anthropic-ai":["partial","named",1,1],"PerplexityBot":["partial","named",1,1],"Perplexity-User":["partial","named",1,1],"Google-Extended":["partial","named",1,1],"Applebot-Extended":["partial","named",1,1],"CCBot":["partial","named",1,1],"Bytespider":["partial","named",1,1],"Amazonbot":["partial","named",1,1],"meta-externalagent":["partial","named",1,1],"cohere-ai":["partial","named",1,1],"Diffbot":["partial","named",1,1],"ContextFetchBot":["partial","*",21,0]},"llms":{"status":"html","http":200,"bytes":1406681},"home":{"status":"ok","http":200,"url":"https://www.dailymail.com/ushome/index.html","html_tokens":241833,"markdown_tokens":1486,"pruned_tokens":122}},
{"rank":233,"domain":"blogger.com","origin":"https://blogger.com","robots":{"status":"ok","http":200,"url":"https://www.blogger.com/robots.txt","bytes":572,"groups":1},"bots":{"GPTBot":["partial","*",21,0],"ChatGPT-User":["partial","*",21,0],"OAI-SearchBot":["partial","*",21,0],"ClaudeBot":["partial","*",21,0],"Claude-User":["partial","*",21,0],"Claude-SearchBot":["partial","*",21,0],"anthropic-ai":["partial","*",21,0],"PerplexityBot":["partial","*",21,0],"Perplexity-User":["partial","*",21,0],"Google-Extended":["partial","*",21,0],"Applebot-Extended":["partial","*",21,0],"CCBot":["partial","*",21,0],"Bytespider":["partial","*",21,0],"Amazonbot":["partial","*",21,0],"meta-externalagent":["partial","*",21,0],"cohere-ai":["partial","*",21,0],"Diffbot":["partial","*",21,0],"ContextFetchBot":["partial","*",21,0]},"llms":{"status":"missing","http":404,"bytes":168},"home":{"status":"ok","http":200,"url":"https://www.blogger.com/about/?bpli=1","html_tokens":42097,"markdown_tokens":271,"pruned_tokens":145}},
{"rank":234,"domain":"scholar.google.com","origin":"https://scholar.google.com","robots":{"status":"ok","http":200,"url":"https://scholar.google.com/robots.txt","bytes":564,"groups":4},"bots":{"GPTBot":["partial","*",7,7],"ChatGPT-User":["partial","*",7,7],"OAI-SearchBot":["partial","*",7,7],"ClaudeBot":["partial","*",7,7],"Claude-User":["partial","*",7,7],"Claude-SearchBot":["partial","*",7,7],"anthropic-ai":["partial","*",7,7],"PerplexityBot":["partial","*",7,7],"Perplexity-User":["partial","*",7,7],"Google-Extended":["partial","*",7,7],"Applebot-Extended":["partial","*",7,7],"CCBot":["partial","*",7,7],"Bytespider":["partial","*",7,7],"Amazonbot":["partial","*",7,7],"meta-externalagent":["partial","*",7,7],"cohere-ai":["partial","*",7,7],"Diffbot":["partial","*",7,7],"ContextFetchBot":["partial","*",7,7]},"llms":{"status":"missing","http":404,"bytes":1569},"home":{"status":"ok","http":200,"url":"https://scholar.google.com/","html_tokens":33312,"markdown_tokens":33,"pruned_tokens":33}},
{"rank":235,"domain":"amazon.co.uk","origin":"https://amazon.co.uk","robots":{"status":"ok","http":200,"url":"https://www.amazon.co.uk/robots.txt","bytes":7258,"groups":101},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["blocked","named",1,0],"OAI-SearchBot":["blocked","named",1,0],"ClaudeBot":["blocked","named",2,0],"Claude-User":["blocked","named",1,0],"Claude-SearchBot":["blocked","named",1,0],"anthropic-ai":["partial","*",93,8],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["blocked","named",1,0],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["partial","*",93,8],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["partial","*",93,8],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",93,8]},"llms":{"status":"missing","http":404,"bytes":2165},"home":{"status":"ok","http":200,"url":"https://www.amazon.co.uk/","html_tokens":182728,"markdown_tokens":1212,"pruned_tokens":175}},
{"rank":236,"domain":"britannica.com","origin":"https://britannica.com","robots":{"status":"ok","http":200,"url":"https://www.britannica.com/robots.txt","bytes":834,"groups":21},"bots":{"GPTBot":["partial","*",2,0],"ChatGPT-User":["partial","*",2,0],"OAI-SearchBot":["partial","*",2,0],"ClaudeBot":["partial","*",2,0],"Claude-User":["partial","*",2,0],"Claude-SearchBot":["partial","*",2,0],"anthropic-ai":["partial","*",2,0],"PerplexityBot":["partial","*",2,0],"Perplexity-User":["partial","*",2,0],"Google-Extended":["partial","*",2,0],"Applebot-Extended":["partial","*",2,0],"CCBot":["partial","*",2,0],"Bytespider":["partial","*",2,0],"Amazonbot":["partial","*",2,0],"meta-externalagent":["partial","*",2,0],"cohere-ai":["partial","*",2,0],"Diffbot":["partial","*",2,0],"ContextFetchBot":["partial","*",2,0]},"llms":{"status":"missing","http":404,"bytes":53443},"home":{"status":"http_error","http":403,"url":"https://www.britannica.com/"}},
{"rank":237,"domain":"sina.com.cn","origin":"https://sina.com.cn","robots":{"status":"ok","http":200,"url":"https://www.sina.com.cn/robots.txt","bytes":528,"groups":1},"bots":{"GPTBot":["allowed","*",0,1],"ChatGPT-User":["allowed","*",0,1],"OAI-SearchBot":["allowed","*",0,1],"ClaudeBot":["allowed","*",0,1],"Claude-User":["allowed","*",0,1],"Claude-SearchBot":["allowed","*",0,1],"anthropic-ai":["allowed","*",0,1],"PerplexityBot":["allowed","*",0,1],"Perplexity-User":["allowed","*",0,1],"Google-Extended":["allowed","*",0,1],"Applebot-Extended":["allowed","*",0,1],"CCBot":["allowed","*",0,1],"Bytespider":["allowed","*",0,1],"Amazonbot":["allowed","*",0,1],"meta-externalagent":["allowed","*",0,1],"cohere-ai":["allowed","*",0,1],"Diffbot":["allowed","*",0,1],"ContextFetchBot":["allowed","*",0,1]},"llms":{"status":"missing","http":404,"bytes":1010},"home":{"status":"ok","http":200,"url":"https://www.sina.com.cn/","html_tokens":93893,"markdown_tokens":65,"pruned_tokens":65}},
{"rank":238,"domain":"redhat.com","origin":"https://redhat.com","robots":{"status":"ok","http":200,"url":"https://www.redhat.com/robots.txt","bytes":3368,"groups":1},"bots":{"GPTBot":["partial","*",59,18],"ChatGPT-User":["partial","*",59,18],"OAI-SearchBot":["partial","*",59,18],"ClaudeBot":["partial","*",59,18],"Claude-User":["partial","*",59,18],"Claude-SearchBot":["partial","*",59,18],"anthropic-ai":["partial","*",59,18],"PerplexityBot":["partial","*",59,18],"Perplexity-User":["partial","*",59,18],"Google-Extended":["partial","*",59,18],"Applebot-Extended":["partial","*",59,18],"CCBot":["partial","*",59,18],"Bytespider":["partial","*",59,18],"Amazonbot":["partial","*",59,18],"meta-externalagent":["partial","*",59,18],"cohere-ai":["partial","*",59,18],"Diffbot":["partial","*",59,18],"ContextFetchBot":["partial","*",59,18]},"llms":{"status":"present","http":200,"bytes":4366},"home":{"status":"ok","http":200,"url":"https://www.redhat.com/en","html_tokens":277034,"markdown_tokens":1383,"pruned_tokens":108}},
{"rank":239,"domain":"academic.oup.com","origin":"https://academic.oup.com","robots":{"status":"ok","http":200,"url":"https://academic.oup.com/robots.txt","bytes":4012,"groups":2},"bots":{"GPTBot":["partial","*",48,3],"ChatGPT-User":["partial","*",48,3],"OAI-SearchBot":["partial","*",48,3],"ClaudeBot":["partial","*",48,3],"Claude-User":["partial","*",48,3],"Claude-SearchBot":["partial","*",48,3],"anthropic-ai":["partial","*",48,3],"PerplexityBot":["partial","*",48,3],"Perplexity-User":["partial","*",48,3],"Google-Extended":["partial","*",48,3],"Applebot-Extended":["partial","*",48,3],"CCBot":["partial","*",48,3],"Bytespider":["partial","*",48,3],"Amazonbot":["partial","*",48,3],"meta-externalagent":["partial","*",48,3],"cohere-ai":["partial","*",48,3],"Diffbot":["partial","*",48,3],"ContextFetchBot":["partial","*",48,3]},"llms":{"status":"http_error","http":403,"bytes":5527},"home":{"status":"http_error","http":403,"url":"https://academic.oup.com/"}},
{"rank":240,"domain":"surveymonkey.com","origin":"https://surveymonkey.com","robots":{"status":"ok","http":200,"url":"https://www.surveymonkey.com/robots.txt","bytes":713,"groups":1},"bots":{"GPTBot":["partial","*",21,6],"ChatGPT-User":["partial","*",21,6],"OAI-SearchBot":["partial","*",21,6],"ClaudeBot":["partial","*",21,6],"Claude-User":["partial","*",21,6],"Claude-SearchBot":["partial","*",21,6],"anthropic-ai":["partial","*",21,6],"PerplexityBot":["partial","*",21,6],"Perplexity-User":["partial","*",21,6],"Google-Extended":["partial","*",21,6],"Applebot-Extended":["partial","*",21,6],"CCBot":["partial","*",21,6],"Bytespider":["partial","*",21,6],"Amazonbot":["partial","*",21,6],"meta-externalagent":["partial","*",21,6],"cohere-ai":["partial","*",21,6],"Diffbot":["partial","*",21,6],"ContextFetchBot":["partial","*",21,6]},"llms":{"status":"present","http":200,"bytes":13693},"home":{"status":"ok","http":200,"url":"https://www.surveymonkey.com/","html_tokens":165024,"markdown_tokens":72,"pruned_tokens":72}},
{"rank":241,"domain":"cbsnews.com","origin":"https://cbsnews.com","robots":{"status":"ok","http":200,"url":"https://www.cbsnews.com/robots.txt","bytes":950,"groups":5},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["partial","*",35,0],"OAI-SearchBot":["partial","*",35,0],"ClaudeBot":["partial","*",35,0],"Claude-User":["partial","*",35,0],"Claude-SearchBot":["partial","*",35,0],"anthropic-ai":["partial","*",35,0],"PerplexityBot":["partial","*",35,0],"Perplexity-User":["partial","*",35,0],"Google-Extended":["partial","*",35,0],"Applebot-Extended":["partial","*",35,0],"CCBot":["partial","*",35,0],"Bytespider":["partial","*",35,0],"Amazonbot":["partial","*",35,0],"meta-externalagent":["partial","*",35,0],"cohere-ai":["partial","*",35,0],"Diffbot":["partial","*",35,0],"ContextFetchBot":["partial","*",35,0]},"llms":{"status":"missing","http":404,"bytes":576251},"home":{"status":"ok","http":200,"url":"https://www.cbsnews.com/","html_tokens":171261,"markdown_tokens":75,"pruned_tokens":75}},
{"rank":242,"domain":"uk.com","origin":"https://uk.com","robots":{"status":"ok","http":200,"url":"https://uk.com/robots.txt","bytes":0,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":7990},"home":{"status":"ok","http":200,"url":"https://uk.com/","html_tokens":11624,"markdown_tokens":777,"pruned_tokens":761}},
{"rank":243,"domain":"mckinsey.com","origin":"https://mckinsey.com","robots":{"status":"unreachable","http":null,"url":"https://mckinsey.com/robots.txt","bytes":0,"groups":0,"error":"<StreamReset stream_id:1, error_code:2, remote_reset:True>"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":244,"domain":"ted.com","origin":"https://ted.com","robots":{"status":"ok","http":200,"url":"https://www.ted.com/robots.txt","bytes":2552,"groups":31},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["partial","named",16,0],"OAI-SearchBot":["partial","named",16,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","named",16,0],"Claude-SearchBot":["partial","named",16,0],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["partial","named",16,0],"Perplexity-User":["partial","named",16,0],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",2,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",14,0]},"llms":{"status":"missing","http":404,"bytes":32823},"home":{"status":"ok","http":200,"url":"https://www.ted.com/","html_tokens":145883,"markdown_tokens":137,"pruned_tokens":62}},
{"rank":245,"domain":"pixabay.com","origin":"https://pixabay.com","robots":{"status":"ok","http":200,"url":"https://pixabay.com/robots.txt","bytes":3248,"groups":2},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["partial","*",125,7],"OAI-SearchBot":["partial","*",125,7],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",125,7],"Claude-SearchBot":["partial","*",125,7],"anthropic-ai":["partial","*",125,7],"PerplexityBot":["partial","*",125,7],"Perplexity-User":["partial","*",125,7],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["partial","*",125,7],"meta-externalagent":["partial","*",125,7],"cohere-ai":["partial","*",125,7],"Diffbot":["partial","*",125,7],"ContextFetchBot":["partial","*",125,7]},"llms":{"status":"http_error","http":403,"bytes":5521},"home":{"status":"http_error","http":403,"url":"https://pixabay.com/"}},
{"rank":246,"domain":"free.fr","origin":"https://free.fr","robots":{"status":"ok","http":200,"url":"https://www.free.fr/robots.txt","bytes":1051,"groups":15},"bots":{"GPTBot":["partial","*",3,1],"ChatGPT-User":["partial","*",3,1],"OAI-SearchBot":["partial","*",3,1],"ClaudeBot":["partial","*",3,1],"Claude-User":["partial","*",3,1],"Claude-SearchBot":["partial","*",3,1],"anthropic-ai":["partial","*",3,1],"PerplexityBot":["partial","*",3,1],"Perplexity-User":["partial","*",3,1],"Google-Extended":["partial","*",3,1],"Applebot-Extended":["partial","*",3,1],"CCBot":["partial","*",3,1],"Bytespider":["partial","*",3,1],"Amazonbot":["partial","*",3,1],"meta-externalagent":["partial","*",3,1],"cohere-ai":["partial","*",3,1],"Diffbot":["partial","*",3,1],"ContextFetchBot":["partial","*",3,1]},"llms":{"status":"missing","http":404,"bytes":571658},"home":{"status":"ok","http":200,"url":"https://www.free.fr/","html_tokens":170975,"markdown_tokens":688,"pruned_tokens":130}},
{"rank":247,"domain":"cambridge.org","origin":"https://cambridge.org","robots":{"status":"ok","http":200,"url":"https://www.cambridge.org/robots.txt","bytes":2785,"groups":5},"bots":{"GPTBot":["partial","*",63,35],"ChatGPT-User":["blocked","named",1,0],"OAI-SearchBot":["partial","*",63,35],"ClaudeBot":["partial","*",63,35],"Claude-User":["partial","*",63,35],"Claude-SearchBot":["partial","*",63,35],"anthropic-ai":["partial","*",63,35],"PerplexityBot":["partial","*",63,35],"Perplexity-User":["partial","*",63,35],"Google-Extended":["partial","*",63,35],"Applebot-Extended":["partial","*",63,35],"CCBot":["partial","*",63,35],"Bytespider":["partial","*",63,35],"Amazonbot":["partial","*",63,35],"meta-externalagent":["partial","*",63,35],"cohere-ai":["partial","*",63,35],"Diffbot":["partial","*",63,35],"ContextFetchBot":["partial","*",63,35]},"llms":{"status":"missing","http":404,"bytes":204775},"home":{"status":"http_error","http":403,"url":"https://www.cambridge.org/"}},
{"rank":248,"domain":"allaboutcookies.org","origin":"https://allaboutcookies.org","robots":{"status":"refused","http":403,"url":"https://allaboutcookies.org/robots.txt","bytes":779,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":249,"domain":"wix.com","origin":"https://wix.com","robots":{"status":"ok","http":200,"url":"https://www.wix.com/robots.txt","bytes":2797,"groups":1},"bots":{"GPTBot":["partial","*",87,0],"ChatGPT-User":["partial","*",87,0],"OAI-SearchBot":["partial","*",87,0],"ClaudeBot":["partial","*",87,0],"Claude-User":["partial","*",87,0],"Claude-SearchBot":["partial","*",87,0],"anthropic-ai":["partial","*",87,0],"PerplexityBot":["partial","*",87,0],"Perplexity-User":["partial","*",87,0],"Google-Extended":["partial","*",87,0],"Applebot-Extended":["partial","*",87,0],"CCBot":["partial","*",87,0],"Bytespider":["partial","*",87,0],"Amazonbot":["partial","*",87,0],"meta-externalagent":["partial","*",87,0],"cohere-ai":["partial","*",87,0],"Diffbot":["partial","*",87,0],"ContextFetchBot":["partial","*",87,0]},"llms":{"status":"present","http":200,"bytes":36670},"home":{"status":"ok","http":200,"url":"https://www.wix.com/","html_tokens":627415,"markdown_tokens":959,"pruned_tokens":156}},
{"rank":250,"domain":"sedo.com","origin":"https://sedo.com","robots":{"status":"ok","http":200,"url":"https://sedo.com/robots.txt","bytes":270,"groups":2},"bots":{"GPTBot":["partial","*",8,1],"ChatGPT-User":["partial","*",8,1],"OAI-SearchBot":["partial","*",8,1],"ClaudeBot":["partial","*",8,1],"Claude-User":["partial","*",8,1],"Claude-SearchBot":["partial","*",8,1],"anthropic-ai":["partial","*",8,1],"PerplexityBot":["partial","*",8,1],"Perplexity-User":["partial","*",8,1],"Google-Extended":["partial","*",8,1],"Applebot-Extended":["partial","*",8,1],"CCBot":["partial","*",8,1],"Bytespider":["partial","*",8,1],"Amazonbot":["partial","*",8,1],"meta-externalagent":["partial","*",8,1],"cohere-ai":["partial","*",8,1],"Diffbot":["partial","*",8,1],"ContextFetchBot":["partial","*",8,1]},"llms":{"status":"missing","http":404,"bytes":64389},"home":{"status":"ok","http":200,"url":"https://sedo.com/us/","html_tokens":21511,"markdown_tokens":572,"pruned_tokens":248}},
{"rank":251,"domain":"prnewswire.com","origin":"https://prnewswire.com","robots":{"status":"ok","http":200,"url":"https://www.prnewswire.com/robots.txt","bytes":3353,"groups":1},"bots":{"GPTBot":["partial","*",8,0],"ChatGPT-User":["partial","*",8,0],"OAI-SearchBot":["partial","*",8,0],"ClaudeBot":["partial","*",8,0],"Claude-User":["partial","*",8,0],"Claude-SearchBot":["partial","*",8,0],"anthropic-ai":["partial","*",8,0],"PerplexityBot":["partial","*",8,0],"Perplexity-User":["partial","*",8,0],"Google-Extended":["partial","*",8,0],"Applebot-Extended":["partial","*",8,0],"CCBot":["partial","*",8,0],"Bytespider":["partial","*",8,0],"Amazonbot":["partial","*",8,0],"meta-externalagent":["partial","*",8,0],"cohere-ai":["partial","*",8,0],"Diffbot":["partial","*",8,0],"ContextFetchBot":["partial","*",8,0]},"llms":{"status":"present","http":200,"bytes":2598},"home":{"status":"ok","http":200,"url":"https://www.prnewswire.com/","html_tokens":59362,"markdown_tokens":137,"pruned_tokens":137}},
{"rank":252,"domain":"hugedomains.com","origin":"https://hugedomains.com","robots":{"status":"ok","http":200,"url":"https://www.hugedomains.com/robots.txt","bytes":606,"groups":3},"bots":{"GPTBot":["partial","*",17,0],"ChatGPT-User":["partial","*",17,0],"OAI-SearchBot":["partial","*",17,0],"ClaudeBot":["partial","*",17,0],"Claude-User":["partial","*",17,0],"Claude-SearchBot":["partial","*",17,0],"anthropic-ai":["partial","*",17,0],"PerplexityBot":["partial","*",17,0],"Perplexity-User":["partial","*",17,0],"Google-Extended":["partial","*",17,0],"Applebot-Extended":["partial","*",17,0],"CCBot":["partial","*",17,0],"Bytespider":["partial","*",17,0],"Amazonbot":["partial","*",17,0],"meta-externalagent":["partial","*",17,0],"cohere-ai":["partial","*",17,0],"Diffbot":["partial","*",17,0],"ContextFetchBot":["partial","*",17,0]},"llms":{"status":"http_error","http":403,"bytes":5486},"home":{"status":"http_error","http":403,"url":"https://hugedomains.com/"}},
{"rank":253,"domain":"networkadvertising.org","origin":"https://networkadvertising.org","robots":{"status":"ok","http":200,"url":"https://thenai.org/robots.txt","bytes":184,"groups":1},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"http_error","http":403,"bytes":5541},"home":{"status":"http_error","http":403,"url":"https://thenai.org/"}},
{"rank":254,"domain":"independent.co.uk","origin":"https://independent.co.uk","robots":{"status":"ok","http":200,"url":"https://www.independent.co.uk/robots.txt","bytes":1305,"groups":2},"bots":{"GPTBot":["partial","*",43,0],"ChatGPT-User":["partial","*",43,0],"OAI-SearchBot":["partial","*",43,0],"ClaudeBot":["partial","*",43,0],"Claude-User":["partial","*",43,0],"Claude-SearchBot":["partial","*",43,0],"anthropic-ai":["partial","*",43,0],"PerplexityBot":["partial","*",43,0],"Perplexity-User":["partial","*",43,0],"Google-Extended":["partial","*",43,0],"Applebot-Extended":["partial","*",43,0],"CCBot":["partial","*",43,0],"Bytespider":["partial","*",43,0],"Amazonbot":["partial","*",43,0],"meta-externalagent":["partial","*",43,0],"cohere-ai":["partial","*",43,0],"Diffbot":["partial","*",43,0],"ContextFetchBot":["partial","*",43,0]},"llms":{"status":"missing","http":410,"bytes":410},"home":{"status":"ok","http":200,"url":"https://www.independent.co.uk/us","html_tokens":86801,"markdown_tokens":34,"pruned_tokens":34}},
{"rank":255,"domain":"ieee.org","origin":"https://ieee.org","robots":{"status":"html","http":202,"url":"https://www.ieee.org/robots.txt","bytes":2047,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":256,"domain":"mailchimp.com","origin":"https://mailchimp.com","robots":{"status":"ok","http":200,"url":"https://mailchimp.com/robots.txt","bytes":101,"groups":1},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"present","http":200,"bytes":526024},"home":{"status":"ok","http":200,"url":"https://mailchimp.com/","html_tokens":122274,"markdown_tokens":1112,"pruned_tokens":204}},
{"rank":257,"domain":"163.com","origin":"https://163.com","robots":{"status":"ok","http":200,"url":"http://www.163.com/robots.txt","bytes":225,"groups":1},"bots":{"GPTBot":["partial","*",7,0],"ChatGPT-User":["partial","*",7,0],"OAI-SearchBot":["partial","*",7,0],"ClaudeBot":["partial","*",7,0],"Claude-User":["partial","*",7,0],"Claude-SearchBot":["partial","*",7,0],"anthropic-ai":["partial","*",7,0],"PerplexityBot":["partial","*",7,0],"Perplexity-User":["partial","*",7,0],"Google-Extended":["partial","*",7,0],"Applebot-Extended":["partial","*",7,0],"CCBot":["partial","*",7,0],"Bytespider":["partial","*",7,0],"Amazonbot":["partial","*",7,0],"meta-externalagent":["partial","*",7,0],"cohere-ai":["partial","*",7,0],"Diffbot":["partial","*",7,0],"ContextFetchBot":["partial","*",7,0]},"llms":{"status":"html","http":200,"bytes":274180},"home":{"status":"ok","http":200,"url":"http://www.163.com/","html_tokens":61305,"markdown_tokens":457,"pruned_tokens":109}},
{"rank":258,"domain":"nist.gov","origin":"https://nist.gov","robots":{"status":"ok","http":200,"url":"https://www.nist.gov/robots.txt","bytes":2425,"groups":1},"bots":{"GPTBot":["partial","*",44,18],"ChatGPT-User":["partial","*",44,18],"OAI-SearchBot":["partial","*",44,18],"ClaudeBot":["partial","*",44,18],"Claude-User":["partial","*",44,18],"Claude-SearchBot":["partial","*",44,18],"anthropic-ai":["partial","*",44,18],"PerplexityBot":["partial","*",44,18],"Perplexity-User":["partial","*",44,18],"Google-Extended":["partial","*",44,18],"Applebot-Extended":["partial","*",44,18],"CCBot":["partial","*",44,18],"Bytespider":["partial","*",44,18],"Amazonbot":["partial","*",44,18],"meta-externalagent":["partial","*",44,18],"cohere-ai":["partial","*",44,18],"Diffbot":["partial","*",44,18],"ContextFetchBot":["partial","*",44,18]},"llms":{"status":"missing","http":404,"bytes":547},"home":{"status":"ok","http":200,"url":"https://www.nist.gov/","html_tokens":24303,"markdown_tokens":239,"pruned_tokens":102}},
{"rank":259,"domain":"hbr.org","origin":"https://hbr.org","robots":{"status":"ok","http":200,"url":"https://hbr.org/robots.txt","bytes":1016,"groups":6},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["partial","*",22,5],"OAI-SearchBot":["partial","*",22,5],"ClaudeBot":["partial","*",22,5],"Claude-User":["partial","*",22,5],"Claude-SearchBot":["partial","*",22,5],"anthropic-ai":["partial","*",22,5],"PerplexityBot":["partial","*",22,5],"Perplexity-User":["partial","*",22,5],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["partial","*",22,5],"CCBot":["blocked","named",1,0],"Bytespider":["partial","*",22,5],"Amazonbot":["partial","*",22,5],"meta-externalagent":["partial","*",22,5],"cohere-ai":["partial","*",22,5],"Diffbot":["partial","*",22,5],"ContextFetchBot":["partial","*",22,5]},"llms":{"status":"missing","http":404,"bytes":59913},"home":{"status":"ok","http":200,"url":"https://hbr.org/","html_tokens":90365,"markdown_tokens":36,"pruned_tokens":36}},
{"rank":260,"domain":"theverge.com","origin":"https://theverge.com","robots":{"status":"ok","http":200,"url":"https://theverge.com/robots.txt","bytes":5650,"groups":106},"bots":{"GPTBot":["allowed","named",0,1],"ChatGPT-User":["partial","named",1,1],"OAI-SearchBot":["partial","*",15,2],"ClaudeBot":["partial","named",1,1],"Claude-User":["partial","named",1,1],"Claude-SearchBot":["partial","named",1,1],"anthropic-ai":["partial","named",1,1],"PerplexityBot":["partial","named",1,1],"Perplexity-User":["partial","named",1,1],"Google-Extended":["partial","named",1,1],"Applebot-Extended":["partial","named",1,1],"CCBot":["partial","named",1,1],"Bytespider":["partial","named",1,1],"Amazonbot":["allowed","named",0,1],"meta-externalagent":["partial","named",1,1],"cohere-ai":["partial","named",1,1],"Diffbot":["partial","named",1,1],"ContextFetchBot":["partial","*",15,2]},"llms":{"status":"missing","http":404,"bytes":99275},"home":{"status":"ok","http":200,"url":"https://www.theverge.com/","html_tokens":236701,"markdown_tokens":174,"pruned_tokens":174}},
{"rank":261,"domain":"dailymotion.com","origin":"https://dailymotion.com","robots":{"status":"ok","http":200,"url":"https://www.dailymotion.com/robots.txt","bytes":18276,"groups":7},"bots":{"GPTBot":["partial","*",107,0],"ChatGPT-User":["partial","*",107,0],"OAI-SearchBot":["partial","*",107,0],"ClaudeBot":["partial","*",107,0],"Claude-User":["partial","*",107,0],"Claude-SearchBot":["partial","*",107,0],"anthropic-ai":["partial","*",107,0],"PerplexityBot":["partial","*",107,0],"Perplexity-User":["partial","*",107,0],"Google-Extended":["partial","*",107,0],"Applebot-Extended":["partial","*",107,0],"CCBot":["partial","*",107,0],"Bytespider":["partial","*",107,0],"Amazonbot":["partial","*",107,0],"meta-externalagent":["partial","*",107,0],"cohere-ai":["partial","*",107,0],"Diffbot":["partial","*",107,0],"ContextFetchBot":["partial","*",107,0]},"llms":{"status":"present","http":200,"bytes":6153},"home":{"status":"ok","http":200,"url":"https://www.dailymotion.com/","html_tokens":15110,"markdown_tokens":0,"pruned_tokens":0}},
{"rank":262,"domain":"frontiersin.org","origin":"https://frontiersin.org","robots":{"status":"ok","http":200,"url":"https://www.frontiersin.org/robots.txt","bytes":862,"groups":1},"bots":{"GPTBot":["partial","*",11,1],"ChatGPT-User":["partial","*",11,1],"OAI-SearchBot":["partial","*",11,1],"ClaudeBot":["partial","*",11,1],"Claude-User":["partial","*",11,1],"Claude-SearchBot":["partial","*",11,1],"anthropic-ai":["partial","*",11,1],"PerplexityBot":["partial","*",11,1],"Perplexity-User":["partial","*",11,1],"Google-Extended":["partial","*",11,1],"Applebot-Extended":["partial","*",11,1],"CCBot":["partial","*",11,1],"Bytespider":["partial","*",11,1],"Amazonbot":["partial","*",11,1],"meta-externalagent":["partial","*",11,1],"cohere-ai":["partial","*",11,1],"Diffbot":["partial","*",11,1],"ContextFetchBot":["partial","*",11,1]},"llms":{"status":"present","http":200,"bytes":5255},"home":{"status":"ok","http":200,"url":"https://www.frontiersin.org/","html_tokens":62393,"markdown_tokens":651,"pruned_tokens":35}},
{"rank":263,"domain":"readthedocs.io","origin":"https://readthedocs.io","robots":{"status":"html","http":200,"url":"https://about.readthedocs.com/?ref=app.readthedocs.org","bytes":33498,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":264,"domain":"myshopify.com","origin":"https://myshopify.com","robots":{"status":"missing","http":404,"url":"https://myshopify.com/robots.txt","bytes":22,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":22},"home":{"status":"http_error","http":404,"url":"https://myshopify.com/"}},
{"rank":265,"domain":"steampowered.com","origin":"https://steampowered.com","robots":{"status":"ok","http":200,"url":"https://store.steampowered.com/robots.txt","bytes":303,"groups":1},"bots":{"GPTBot":["partial","*",9,0],"ChatGPT-User":["partial","*",9,0],"OAI-SearchBot":["partial","*",9,0],"ClaudeBot":["partial","*",9,0],"Claude-User":["partial","*",9,0],"Claude-SearchBot":["partial","*",9,0],"anthropic-ai":["partial","*",9,0],"PerplexityBot":["partial","*",9,0],"Perplexity-User":["partial","*",9,0],"Google-Extended":["partial","*",9,0],"Applebot-Extended":["partial","*",9,0],"CCBot":["partial","*",9,0],"Bytespider":["partial","*",9,0],"Amazonbot":["partial","*",9,0],"meta-externalagent":["partial","*",9,0],"cohere-ai":["partial","*",9,0],"Diffbot":["partial","*",9,0],"ContextFetchBot":["partial","*",9,0]},"llms":{"status":"html","http":200,"bytes":992021},"home":{"status":"ok","http":200,"url":"https://store.steampowered.com/","html_tokens":248009,"markdown_tokens":72,"pruned_tokens":72}},
{"rank":266,"domain":"princeton.edu","origin":"https://princeton.edu","robots":{"status":"ok","http":200,"url":"https://www.princeton.edu/robots.txt","bytes":2275,"groups":1},"bots":{"GPTBot":["partial","*",37,18],"ChatGPT-User":["partial","*",37,18],"OAI-SearchBot":["partial","*",37,18],"ClaudeBot":["partial","*",37,18],"Claude-User":["partial","*",37,18],"Claude-SearchBot":["partial","*",37,18],"anthropic-ai":["partial","*",37,18],"PerplexityBot":["partial","*",37,18],"Perplexity-User":["partial","*",37,18],"Google-Extended":["partial","*",37,18],"Applebot-Extended":["partial","*",37,18],"CCBot":["partial","*",37,18],"Bytespider":["partial","*",37,18],"Amazonbot":["partial","*",37,18],"meta-externalagent":["partial","*",37,18],"cohere-ai":["partial","*",37,18],"Diffbot":["partial","*",37,18],"ContextFetchBot":["partial","*",37,18]},"llms":{"status":"missing","http":404,"bytes":191},"home":{"status":"http_error","http":403,"url":"https://www.princeton.edu/"}},
{"rank":267,"domain":"yelp.com","origin":"https://yelp.com","robots":{"status":"ok","http":200,"url":"https://www.yelp.com/robots.txt","bytes":12697,"groups":9},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["partial","named",1,2],"OAI-SearchBot":["partial","named",1,3],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","named",1,2],"Claude-SearchBot":["partial","named",1,2],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["partial","named",1,2],"Perplexity-User":["partial","named",1,2],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","*",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","*",1,0],"Amazonbot":["blocked","*",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","*",1,0],"Diffbot":["blocked","*",1,0],"ContextFetchBot":["blocked","*",1,0]},"llms":{"status":"disallowed"},"home":{"status":"disallowed"}},
{"rank":268,"domain":"huawei.com","origin":"https://www.huawei.com","robots":{"status":"ok","http":200,"url":"https://www.huawei.com/robots.txt","bytes":2998,"groups":1},"bots":{"GPTBot":["partial","*",36,0],"ChatGPT-User":["partial","*",36,0],"OAI-SearchBot":["partial","*",36,0],"ClaudeBot":["partial","*",36,0],"Claude-User":["partial","*",36,0],"Claude-SearchBot":["partial","*",36,0],"anthropic-ai":["partial","*",36,0],"PerplexityBot":["partial","*",36,0],"Perplexity-User":["partial","*",36,0],"Google-Extended":["partial","*",36,0],"Applebot-Extended":["partial","*",36,0],"CCBot":["partial","*",36,0],"Bytespider":["partial","*",36,0],"Amazonbot":["partial","*",36,0],"meta-externalagent":["partial","*",36,0],"cohere-ai":["partial","*",36,0],"Diffbot":["partial","*",36,0],"ContextFetchBot":["partial","*",36,0]},"llms":{"status":"missing","http":404,"bytes":89150},"home":{"status":"ok","http":200,"url":"https://www.huawei.com/en/","html_tokens":30091,"markdown_tokens":90,"pruned_tokens":68}},
{"rank":269,"domain":"docker.com","origin":"https://docker.com","robots":{"status":"refused","http":403,"url":"https://docker.com/robots.txt","bytes":560,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":270,"domain":"worldbank.org","origin":"https://worldbank.org","robots":{"status":"ok","http":200,"url":"https://www.worldbank.org/robots.txt","bytes":1505,"groups":2},"bots":{"GPTBot":["allowed","*",0,1],"ChatGPT-User":["allowed","*",0,1],"OAI-SearchBot":["allowed","*",0,1],"ClaudeBot":["allowed","*",0,1],"Claude-User":["allowed","*",0,1],"Claude-SearchBot":["allowed","*",0,1],"anthropic-ai":["allowed","*",0,1],"PerplexityBot":["allowed","*",0,1],"Perplexity-User":["allowed","*",0,1],"Google-Extended":["allowed","*",0,1],"Applebot-Extended":["allowed","*",0,1],"CCBot":["allowed","*",0,1],"Bytespider":["allowed","*",0,1],"Amazonbot":["allowed","*",0,1],"meta-externalagent":["allowed","*",0,1],"cohere-ai":["allowed","*",0,1],"Diffbot":["allowed","*",0,1],"ContextFetchBot":["allowed","*",0,1]},"llms":{"status":"missing","http":404,"bytes":5128},"home":{"status":"ok","http":200,"url":"https://www.worldbank.org/ext/en/home","html_tokens":15323,"markdown_tokens":733,"pruned_tokens":140}},
{"rank":271,"domain":"epa.gov","origin":"https://epa.gov","robots":{"status":"ok","http":200,"url":"https://www.epa.gov/robots.txt","bytes":2035,"groups":1},"bots":{"GPTBot":["partial","*",28,27],"ChatGPT-User":["partial","*",28,27],"OAI-SearchBot":["partial","*",28,27],"ClaudeBot":["partial","*",28,27],"Claude-User":["partial","*",28,27],"Claude-SearchBot":["partial","*",28,27],"anthropic-ai":["partial","*",28,27],"PerplexityBot":["partial","*",28,27],"Perplexity-User":["partial","*",28,27],"Google-Extended":["partial","*",28,27],"Applebot-Extended":["partial","*",28,27],"CCBot":["partial","*",28,27],"Bytespider":["partial","*",28,27],"Amazonbot":["partial","*",28,27],"meta-externalagent":["partial","*",28,27],"cohere-ai":["partial","*",28,27],"Diffbot":["partial","*",28,27],"ContextFetchBot":["partial","*",28,27]},"llms":{"status":"missing","http":404,"bytes":6518},"home":{"status":"ok","http":200,"url":"https://www.epa.gov/","html_tokens":16241,"markdown_tokens":123,"pruned_tokens":123}},
{"rank":272,"domain":"dx.doi.org","origin":"https://dx.doi.org","robots":{"status":"ok","http":200,"url":"https://dx.doi.org/robots.txt","bytes":47,"groups":1},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"http_error","http":400,"bytes":9168},"home":{"status":"ok","http":200,"url":"https://dx.doi.org/","html_tokens":1495,"markdown_tokens":129,"pruned_tokens":129}},
{"rank":273,"domain":"web.whatsapp.com","origin":"https://web.whatsapp.com","robots":{"status":"ok","http":200,"url":"https://web.whatsapp.com/robots.txt","bytes":121,"groups":1},"bots":{"GPTBot":["partial","*",1,6],"ChatGPT-User":["partial","*",1,6],"OAI-SearchBot":["partial","*",1,6],"ClaudeBot":["partial","*",1,6],"Claude-User":["partial","*",1,6],"Claude-SearchBot":["partial","*",1,6],"anthropic-ai":["partial","*",1,6],"PerplexityBot":["partial","*",1,6],"Perplexity-User":["partial","*",1,6],"Google-Extended":["partial","*",1,6],"Applebot-Extended":["partial","*",1,6],"CCBot":["partial","*",1,6],"Bytespider":["partial","*",1,6],"Amazonbot":["partial","*",1,6],"meta-externalagent":["partial","*",1,6],"cohere-ai":["partial","*",1,6],"Diffbot":["partial","*",1,6],"ContextFetchBot":["partial","*",1,6]},"llms":{"status":"disallowed"},"home":{"status":"ok","http":200,"url":"https://web.whatsapp.com/","html_tokens":10173,"markdown_tokens":29,"pruned_tokens":29}},
{"rank":274,"domain":"taobao.com","origin":"https://taobao.com","robots":{"status":"ok","http":200,"url":"https://www.taobao.com/robots.txt","bytes":71,"groups":1},"bots":{"GPTBot":["partial","*",1,3],"ChatGPT-User":["partial","*",1,3],"OAI-SearchBot":["partial","*",1,3],"ClaudeBot":["partial","*",1,3],"Claude-User":["partial","*",1,3],"Claude-SearchBot":["partial","*",1,3],"anthropic-ai":["partial","*",1,3],"PerplexityBot":["partial","*",1,3],"Perplexity-User":["partial","*",1,3],"Google-Extended":["partial","*",1,3],"Applebot-Extended":["partial","*",1,3],"CCBot":["partial","*",1,3],"Bytespider":["partial","*",1,3],"Amazonbot":["partial","*",1,3],"meta-externalagent":["partial","*",1,3],"cohere-ai":["partial","*",1,3],"Diffbot":["partial","*",1,3],"ContextFetchBot":["partial","*",1,3]},"llms":{"status":"disallowed","error":"robots.txt of error.taobao.com not readable (html)"},"home":{"status":"ok","http":200,"url":"https://www.taobao.com/","html_tokens":22400,"markdown_tokens":128,"pruned_tokens":34}},
{"rank":275,"domain":"usda.gov","origin":"https://www.usda.gov","robots":{"status":"refused","http":403,"url":"https://www.usda.gov/robots.txt","bytes":380,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":276,"domain":"python.org","origin":"https://python.org","robots":{"status":"ok","http":200,"url":"https://www.python.org/robots.txt","bytes":537,"groups":4},"bots":{"GPTBot":["partial","*",2,0],"ChatGPT-User":["partial","*",2,0],"OAI-SearchBot":["partial","*",2,0],"ClaudeBot":["partial","*",2,0],"Claude-User":["partial","*",2,0],"Claude-SearchBot":["partial","*",2,0],"anthropic-ai":["partial","*",2,0],"PerplexityBot":["partial","*",2,0],"Perplexity-User":["partial","*",2,0],"Google-Extended":["partial","*",2,0],"Applebot-Extended":["partial","*",2,0],"CCBot":["partial","*",2,0],"Bytespider":["partial","*",2,0],"Amazonbot":["partial","*",2,0],"meta-externalagent":["partial","*",2,0],"cohere-ai":["partial","*",2,0],"Diffbot":["partial","*",2,0],"ContextFetchBot":["partial","*",2,0]},"llms":{"status":"missing","http":404,"bytes":32512},"home":{"status":"ok","http":200,"url":"https://www.python.org/","html_tokens":12940,"markdown_tokens":254,"pruned_tokens":229}},
{"rank":277,"domain":"myspace.com","origin":"https://myspace.com","robots":{"status":"ok","http":200,"url":"https://myspace.com/robots.txt","bytes":2489,"groups":1},"bots":{"GPTBot":["partial","*",38,0],"ChatGPT-User":["partial","*",38,0],"OAI-SearchBot":["partial","*",38,0],"ClaudeBot":["partial","*",38,0],"Claude-User":["partial","*",38,0],"Claude-SearchBot":["partial","*",38,0],"anthropic-ai":["partial","*",38,0],"PerplexityBot":["partial","*",38,0],"Perplexity-User":["partial","*",38,0],"Google-Extended":["partial","*",38,0],"Applebot-Extended":["partial","*",38,0],"CCBot":["partial","*",38,0],"Bytespider":["partial","*",38,0],"Amazonbot":["partial","*",38,0],"meta-externalagent":["partial","*",38,0],"cohere-ai":["partial","*",38,0],"Diffbot":["partial","*",38,0],"ContextFetchBot":["partial","*",38,0]},"llms":{"status":"missing","http":404,"bytes":87282},"home":{"status":"ok","http":200,"url":"https://myspace.com/","html_tokens":62109,"markdown_tokens":2921,"pruned_tokens":14}},
{"rank":278,"domain":"mailchi.mp","origin":"https://mailchi.mp","robots":{"status":"ok","http":200,"url":"https://mailchi.mp/robots.txt","bytes":138,"groups":4},"bots":{"GPTBot":["blocked","*",1,0],"ChatGPT-User":["blocked","*",1,0],"OAI-SearchBot":["blocked","*",1,0],"ClaudeBot":["blocked","*",1,0],"Claude-User":["blocked","*",1,0],"Claude-SearchBot":["blocked","*",1,0],"anthropic-ai":["blocked","*",1,0],"PerplexityBot":["blocked","*",1,0],"Perplexity-User":["blocked","*",1,0],"Google-Extended":["blocked","*",1,0],"Applebot-Extended":["blocked","*",1,0],"CCBot":["blocked","*",1,0],"Bytespider":["blocked","*",1,0],"Amazonbot":["blocked","*",1,0],"meta-externalagent":["blocked","*",1,0],"cohere-ai":["blocked","*",1,0],"Diffbot":["blocked","*",1,0],"ContextFetchBot":["blocked","*",1,0]},"llms":{"status":"disallowed"},"home":{"status":"disallowed"}},
{"rank":279,"domain":"telegram.org","origin":"https://telegram.org","robots":{"status":"missing","http":404,"url":"https://telegram.org/robots.txt","bytes":153,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"html","http":200,"bytes":19921},"home":{"status":"ok","http":200,"url":"https://telegram.org/","html_tokens":4966,"markdown_tokens":57,"pruned_tokens":57}},
{"rank":280,"domain":"store.steampowered.com","origin":"https://store.steampowered.com","robots":{"status":"ok","http":200,"url":"https://store.steampowered.com/robots.txt","bytes":303,"groups":1},"bots":{"GPTBot":["partial","*",9,0],"ChatGPT-User":["partial","*",9,0],"OAI-SearchBot":["partial","*",9,0],"ClaudeBot":["partial","*",9,0],"Claude-User":["partial","*",9,0],"Claude-SearchBot":["partial","*",9,0],"anthropic-ai":["partial","*",9,0],"PerplexityBot":["partial","*",9,0],"Perplexity-User":["partial","*",9,0],"Google-Extended":["partial","*",9,0],"Applebot-Extended":["partial","*",9,0],"CCBot":["partial","*",9,0],"Bytespider":["partial","*",9,0],"Amazonbot":["partial","*",9,0],"meta-externalagent":["partial","*",9,0],"cohere-ai":["partial","*",9,0],"Diffbot":["partial","*",9,0],"ContextFetchBot":["partial","*",9,0]},"llms":{"status":"html","http":200,"bytes":992021},"home":{"status":"ok","http":200,"url":"https://store.steampowered.com/","html_tokens":247969,"markdown_tokens":72,"pruned_tokens":72}},
{"rank":281,"domain":"netflix.com","origin":"https://netflix.com","robots":{"status":"ok","http":200,"url":"https://www.netflix.com/robots.txt","bytes":3790,"groups":4},"bots":{"GPTBot":["partial","named",131,2],"ChatGPT-User":["partial","named",131,2],"OAI-SearchBot":["partial","named",131,2],"ClaudeBot":["partial","named",131,2],"Claude-User":["partial","named",131,2],"Claude-SearchBot":["partial","named",131,2],"anthropic-ai":["blocked","*",1,0],"PerplexityBot":["partial","named",131,2],"Perplexity-User":["blocked","*",1,0],"Google-Extended":["partial","named",131,2],"Applebot-Extended":["blocked","*",1,0],"CCBot":["blocked","*",1,0],"Bytespider":["blocked","*",1,0],"Amazonbot":["blocked","*",1,0],"meta-externalagent":["blocked","*",1,0],"cohere-ai":["blocked","*",1,0],"Diffbot":["blocked","*",1,0],"ContextFetchBot":["blocked","*",1,0]},"llms":{"status":"disallowed"},"home":{"status":"disallowed"}},
{"rank":282,"domain":"imgur.com","origin":"https://imgur.com","robots":{"status":"ok","http":200,"url":"https://imgur.com/robots.txt","bytes":352,"groups":2},"bots":{"GPTBot":["partial","*",9,5],"ChatGPT-User":["partial","*",9,5],"OAI-SearchBot":["partial","*",9,5],"ClaudeBot":["partial","*",9,5],"Claude-User":["partial","*",9,5],"Claude-SearchBot":["partial","*",9,5],"anthropic-ai":["partial","*",9,5],"PerplexityBot":["partial","*",9,5],"Perplexity-User":["partial","*",9,5],"Google-Extended":["partial","*",9,5],"Applebot-Extended":["partial","*",9,5],"CCBot":["partial","*",9,5],"Bytespider":["partial","*",9,5],"Amazonbot":["partial","*",9,5],"meta-externalagent":["partial","*",9,5],"cohere-ai":["partial","*",9,5],"Diffbot":["partial","*",9,5],"ContextFetchBot":["partial","*",9,5]},"llms":{"status":"html","http":200,"bytes":6675},"home":{"status":"ok","http":200,"url":"https://imgur.com/","html_tokens":1669,"markdown_tokens":32,"pruned_tokens":32}},
{"rank":283,"domain":"goodreads.com","origin":"https://goodreads.com","robots":{"status":"ok","http":200,"url":"https://www.goodreads.com/robots.txt","bytes":2647,"groups":7},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["partial","*",27,2],"OAI-SearchBot":["partial","*",27,2],"ClaudeBot":["partial","*",27,2],"Claude-User":["partial","*",27,2],"Claude-SearchBot":["partial","*",27,2],"anthropic-ai":["partial","*",27,2],"PerplexityBot":["partial","*",27,2],"Perplexity-User":["partial","*",27,2],"Google-Extended":["partial","*",27,2],"Applebot-Extended":["partial","*",27,2],"CCBot":["blocked","named",1,0],"Bytespider":["partial","*",27,2],"Amazonbot":["partial","*",27,2],"meta-externalagent":["partial","*",27,2],"cohere-ai":["partial","*",27,2],"Diffbot":["partial","*",27,2],"ContextFetchBot":["partial","*",27,2]},"llms":{"status":"missing","http":404,"bytes":57378},"home":{"status":"ok","http":200,"url":"https://www.goodreads.com/","html_tokens":14108,"markdown_tokens":77,"pruned_tokens":33}},
{"rank":284,"domain":"latimes.com","origin":"https://latimes.com","robots":{"status":"ok","http":200,"url":"https://www.latimes.com/robots.txt","bytes":1768,"groups":13},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["partial","*",20,0],"OAI-SearchBot":["partial","*",20,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",20,0],"Claude-SearchBot":["partial","*",20,0],"anthropic-ai":["partial","*",20,0],"PerplexityBot":["partial","*",20,0],"Perplexity-User":["partial","*",20,0],"Google-Extended":["partial","*",20,0],"Applebot-Extended":["partial","*",20,0],"CCBot":["partial","*",20,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["partial","*",20,0],"meta-externalagent":["partial","*",20,0],"cohere-ai":["partial","*",20,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",20,0]},"llms":{"status":"missing","http":404,"bytes":338163},"home":{"status":"ok","http":200,"url":"https://www.latimes.com/","html_tokens":425544,"markdown_tokens":271,"pruned_tokens":140}},
{"rank":285,"domain":"oecd.org","origin":"https://oecd.org","robots":{"status":"refused","http":403,"url":"https://oecd.org/robots.txt","bytes":0,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":286,"domain":"ubuntu.com","origin":"https://ubuntu.com","robots":{"status":"ok","http":200,"url":"https://ubuntu.com/robots.txt","bytes":5511,"groups":2},"bots":{"GPTBot":["partial","named",71,46],"ChatGPT-User":["partial","named",71,46],"OAI-SearchBot":["partial","named",71,46],"ClaudeBot":["partial","named",71,46],"Claude-User":["partial","*",46,0],"Claude-SearchBot":["partial","named",71,46],"anthropic-ai":["partial","named",71,46],"PerplexityBot":["partial","named",71,46],"Perplexity-User":["partial","named",71,46],"Google-Extended":["partial","named",71,46],"Applebot-Extended":["partial","named",71,46],"CCBot":["partial","*",46,0],"Bytespider":["partial","named",71,46],"Amazonbot":["partial","*",46,0],"meta-externalagent":["partial","named",71,46],"cohere-ai":["partial","named",71,46],"Diffbot":["partial","*",46,0],"ContextFetchBot":["partial","*",46,0]},"llms":{"status":"present","http":200,"bytes":12879},"home":{"status":"ok","http":200,"url":"https://ubuntu.com/","html_tokens":52429,"markdown_tokens":614,"pruned_tokens":109}},
{"rank":287,"domain":"weforum.org","origin":"https://weforum.org","robots":{"status":"refused","http":403,"url":"https://www.weforum.org/robots.txt","bytes":383,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":288,"domain":"nvidia.com","origin":"https://nvidia.com","robots":{"status":"ok","http":200,"url":"https://www.nvidia.com/robots.txt","bytes":3725,"groups":2},"bots":{"GPTBot":["partial","*",55,1],"ChatGPT-User":["partial","*",55,1],"OAI-SearchBot":["partial","*",55,1],"ClaudeBot":["partial","*",55,1],"Claude-User":["partial","*",55,1],"Claude-SearchBot":["partial","*",55,1],"anthropic-ai":["partial","*",55,1],"PerplexityBot":["partial","*",55,1],"Perplexity-User":["partial","*",55,1],"Google-Extended":["partial","*",55,1],"Applebot-Extended":["partial","*",55,1],"CCBot":["partial","*",55,1],"Bytespider":["partial","*",55,1],"Amazonbot":["partial","*",55,1],"meta-externalagent":["partial","*",55,1],"cohere-ai":["partial","*",55,1],"Diffbot":["partial","*",55,1],"ContextFetchBot":["partial","*",55,1]},"llms":{"status":"present","http":200,"bytes":2610},"home":{"status":"ok","http":200,"url":"https://www.nvidia.com/en-us/","html_tokens":190195,"markdown_tokens":253,"pruned_tokens":91}},
{"rank":289,"domain":"youronlinechoices.com","origin":"https://youronlinechoices.com","robots":{"status":"ok","http":200,"url":"https://www.youronlinechoices.com/robots.txt","bytes":63,"groups":1},"bots":{"GPTBot":["partial","*",2,0],"ChatGPT-User":["partial","*",2,0],"OAI-SearchBot":["partial","*",2,0],"ClaudeBot":["partial","*",2,0],"Claude-User":["partial","*",2,0],"Claude-SearchBot":["partial","*",2,0],"anthropic-ai":["partial","*",2,0],"PerplexityBot":["partial","*",2,0],"Perplexity-User":["partial","*",2,0],"Google-Extended":["partial","*",2,0],"Applebot-Extended":["partial","*",2,0],"CCBot":["partial","*",2,0],"Bytespider":["partial","*",2,0],"Amazonbot":["partial","*",2,0],"meta-externalagent":["partial","*",2,0],"cohere-ai":["partial","*",2,0],"Diffbot":["partial","*",2,0],"ContextFetchBot":["partial","*",2,0]},"llms":{"status":"missing","http":404,"bytes":1291},"home":{"status":"ok","http":200,"url":"https://www.youronlinechoices.com/","html_tokens":1855,"markdown_tokens":447,"pruned_tokens":165}},
{"rank":290,"domain":"whitehouse.gov","origin":"https://whitehouse.gov","robots":{"status":"ok","http":200,"url":"https://www.whitehouse.gov/robots.txt","bytes":111,"groups":1},"bots":{"GPTBot":["partial","*",2,0],"ChatGPT-User":["partial","*",2,0],"OAI-SearchBot":["partial","*",2,0],"ClaudeBot":["partial","*",2,0],"Claude-User":["partial","*",2,0],"Claude-SearchBot":["partial","*",2,0],"anthropic-ai":["partial","*",2,0],"PerplexityBot":["partial","*",2,0],"Perplexity-User":["partial","*",2,0],"Google-Extended":["partial","*",2,0],"Applebot-Extended":["partial","*",2,0],"CCBot":["partial","*",2,0],"Bytespider":["partial","*",2,0],"Amazonbot":["partial","*",2,0],"meta-externalagent":["partial","*",2,0],"cohere-ai":["partial","*",2,0],"Diffbot":["partial","*",2,0],"ContextFetchBot":["partial","*",2,0]},"llms":{"status":"missing","http":404,"bytes":234633},"home":{"status":"ok","http":200,"url":"https://www.whitehouse.gov/","html_tokens":74617,"markdown_tokens":276,"pruned_tokens":116}},
{"rank":291,"domain":"nationalgeographic.com","origin":"https://nationalgeographic.com","robots":{"status":"ok","http":200,"url":"https://www.nationalgeographic.com/robots.txt","bytes":1396,"groups":1},"bots":{"GPTBot":["partial","*",38,0],"ChatGPT-User":["partial","*",38,0],"OAI-SearchBot":["partial","*",38,0],"ClaudeBot":["partial","*",38,0],"Claude-User":["partial","*",38,0],"Claude-SearchBot":["partial","*",38,0],"anthropic-ai":["partial","*",38,0],"PerplexityBot":["partial","*",38,0],"Perplexity-User":["partial","*",38,0],"Google-Extended":["partial","*",38,0],"Applebot-Extended":["partial","*",38,0],"CCBot":["partial","*",38,0],"Bytespider":["partial","*",38,0],"Amazonbot":["partial","*",38,0],"meta-externalagent":["partial","*",38,0],"cohere-ai":["partial","*",38,0],"Diffbot":["partial","*",38,0],"ContextFetchBot":["partial","*",38,0]},"llms":{"status":"missing","http":404,"bytes":1},"home":{"status":"ok","http":200,"url":"https://www.nationalgeographic.com/","html_tokens":143890,"markdown_tokens":336,"pruned_tokens":126}},
{"rank":292,"domain":"washington.edu","origin":"https://washington.edu","robots":{"status":"ok","http":200,"url":"https://www.washington.edu/robots.txt","bytes":76,"groups":1},"bots":{"GPTBot":["partial","*",3,0],"ChatGPT-User":["partial","*",3,0],"OAI-SearchBot":["partial","*",3,0],"ClaudeBot":["partial","*",3,0],"Claude-User":["partial","*",3,0],"Claude-SearchBot":["partial","*",3,0],"anthropic-ai":["partial","*",3,0],"PerplexityBot":["partial","*",3,0],"Perplexity-User":["partial","*",3,0],"Google-Extended":["partial","*",3,0],"Applebot-Extended":["partial","*",3,0],"CCBot":["partial","*",3,0],"Bytespider":["partial","*",3,0],"Amazonbot":["partial","*",3,0],"meta-externalagent":["partial","*",3,0],"cohere-ai":["partial","*",3,0],"Diffbot":["partial","*",3,0],"ContextFetchBot":["partial","*",3,0]},"llms":{"status":"missing","http":404,"bytes":19665},"home":{"status":"ok","http":200,"url":"https://www.washington.edu/","html_tokens":15928,"markdown_tokens":367,"pruned_tokens":126}},
{"rank":293,"domain":"mps.gov.cn","origin":"https://www.mps.gov.cn","robots":{"status":"server_error","http":521,"url":"https://www.mps.gov.cn/robots.txt","bytes":858,"groups":0,"error":"HTTP 521"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":294,"domain":"apnews.com","origin":"https://apnews.com","robots":{"status":"ok","http":200,"url":"https://apnews.com/robots.txt","bytes":1348,"groups":14},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["partial","*",20,0],"OAI-SearchBot":["partial","*",20,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["blocked","named",1,0],"Claude-SearchBot":["blocked","named",1,0],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["partial","*",20,0],"Google-Extended":["partial","*",20,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["partial","*",20,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["partial","*",20,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["partial","*",20,0],"ContextFetchBot":["partial","*",20,0]},"llms":{"status":"missing","http":404,"bytes":882705},"home":{"status":"http_error","http":403,"url":"https://apnews.com/"}},
{"rank":295,"domain":"thenai.org","origin":"https://thenai.org","robots":{"status":"ok","http":200,"url":"https://thenai.org/robots.txt","bytes":184,"groups":1},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"http_error","http":403,"bytes":5541},"home":{"status":"http_error","http":403,"url":"https://thenai.org/"}},
{"rank":296,"domain":"investopedia.com","origin":"https://investopedia.com","robots":{"status":"ok","http":200,"url":"https://www.investopedia.com/robots.txt","bytes":3477,"groups":6},"bots":{"GPTBot":["partial","named",1,0],"ChatGPT-User":["partial","named",1,0],"OAI-SearchBot":["partial","named",1,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["blocked","named",1,0],"Claude-SearchBot":["partial","named",1,2],"anthropic-ai":["partial","named",1,2],"PerplexityBot":["partial","named",1,2],"Perplexity-User":["blocked","named",1,0],"Google-Extended":["partial","named",1,2],"Applebot-Extended":["blocked","named",1,0],"CCBot":["partial","named",1,2],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["partial","named",1,2],"cohere-ai":["partial","named",1,2],"Diffbot":["partial","*",2,0],"ContextFetchBot":["partial","*",2,0]},"llms":{"status":"http_error","http":402,"bytes":612},"home":{"status":"http_error","http":402,"url":"https://www.investopedia.com/"}},
{"rank":297,"domain":"i0.wp.com","origin":"https://i0.wp.com","robots":{"status":"ok","http":200,"url":"https://i0.wp.com/robots.txt","bytes":343,"groups":10},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["blocked","named",1,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"http_error","http":400,"bytes":49},"home":{"status":"http_error","http":400,"url":"https://i0.wp.com/"}},
{"rank":298,"domain":"pages.dev","origin":"https://pages.dev","robots":{"status":"html","http":200,"url":"https://www.cloudflare.com/products/pages/","bytes":601759,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":299,"domain":"deloitte.com","origin":"https://deloitte.com","robots":{"status":"ok","http":200,"url":"https://www.deloitte.com/robots.txt","bytes":503,"groups":1},"bots":{"GPTBot":["partial","*",10,0],"ChatGPT-User":["partial","*",10,0],"OAI-SearchBot":["partial","*",10,0],"ClaudeBot":["partial","*",10,0],"Claude-User":["partial","*",10,0],"Claude-SearchBot":["partial","*",10,0],"anthropic-ai":["partial","*",10,0],"PerplexityBot":["partial","*",10,0],"Perplexity-User":["partial","*",10,0],"Google-Extended":["partial","*",10,0],"Applebot-Extended":["partial","*",10,0],"CCBot":["partial","*",10,0],"Bytespider":["partial","*",10,0],"Amazonbot":["partial","*",10,0],"meta-externalagent":["partial","*",10,0],"cohere-ai":["partial","*",10,0],"Diffbot":["partial","*",10,0],"ContextFetchBot":["partial","*",10,0]},"llms":{"status":"missing","http":404,"bytes":236},"home":{"status":"ok","http":200,"url":"https://www.deloitte.com/us/en.html","html_tokens":72275,"markdown_tokens":101,"pruned_tokens":51}},
{"rank":300,"domain":"scribd.com","origin":"https://scribd.com","robots":{"status":"ok","http":200,"url":"https://www.scribd.com/robots.txt","bytes":3518,"groups":9},"bots":{"GPTBot":["partial","*",95,0],"ChatGPT-User":["partial","*",95,0],"OAI-SearchBot":["partial","*",95,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",95,0],"Claude-SearchBot":["partial","*",95,0],"anthropic-ai":["partial","*",95,0],"PerplexityBot":["partial","*",95,0],"Perplexity-User":["partial","*",95,0],"Google-Extended":["partial","*",95,0],"Applebot-Extended":["partial","*",95,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["partial","*",95,0],"meta-externalagent":["partial","*",95,0],"cohere-ai":["partial","*",95,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",95,0]},"llms":{"status":"missing","http":404,"bytes":41889},"home":{"status":"ok","http":200,"url":"https://www.scribd.com/","html_tokens":99295,"markdown_tokens":396,"pruned_tokens":268}},
{"rank":301,"domain":"youku.com","origin":"https://youku.com","robots":{"status":"html","http":200,"url":"https://www.youku.com/index/y404","bytes":15349,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":302,"domain":"academia.edu","origin":"https://academia.edu","robots":{"status":"ok","http":200,"url":"https://www.academia.edu/robots.txt","bytes":59291,"groups":63},"bots":{"GPTBot":["partial","named",2,5],"ChatGPT-User":["partial","named",39,4],"OAI-SearchBot":["partial","named",39,4],"ClaudeBot":["blocked","*",1,0],"Claude-User":["partial","named",38,4],"Claude-SearchBot":["partial","named",38,4],"anthropic-ai":["blocked","*",1,0],"PerplexityBot":["partial","named",38,4],"Perplexity-User":["blocked","*",1,0],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","*",1,0],"Bytespider":["blocked","*",1,0],"Amazonbot":["blocked","*",1,0],"meta-externalagent":["blocked","*",1,0],"cohere-ai":["blocked","*",1,0],"Diffbot":["blocked","*",1,0],"ContextFetchBot":["blocked","*",1,0]},"llms":{"status":"disallowed"},"home":{"status":"disallowed"}},
{"rank":303,"domain":"vercel.app","origin":"https://vercel.app","robots":{"status":"html","http":200,"url":"https://vercel.com","bytes":624381,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":304,"domain":"safety.google","origin":"https://safety.google","robots":{"status":"ok","http":200,"url":"https://safety.google/robots.txt","bytes":66,"groups":1},"bots":{"GPTBot":["allowed","*",0,1],"ChatGPT-User":["allowed","*",0,1],"OAI-SearchBot":["allowed","*",0,1],"ClaudeBot":["allowed","*",0,1],"Claude-User":["allowed","*",0,1],"Claude-SearchBot":["allowed","*",0,1],"anthropic-ai":["allowed","*",0,1],"PerplexityBot":["allowed","*",0,1],"Perplexity-User":["allowed","*",0,1],"Google-Extended":["allowed","*",0,1],"Applebot-Extended":["allowed","*",0,1],"CCBot":["allowed","*",0,1],"Bytespider":["allowed","*",0,1],"Amazonbot":["allowed","*",0,1],"meta-externalagent":["allowed","*",0,1],"cohere-ai":["allowed","*",0,1],"Diffbot":["allowed","*",0,1],"ContextFetchBot":["allowed","*",0,1]},"llms":{"status":"missing","http":404,"bytes":1569},"home":{"status":"ok","http":200,"url":"https://safety.google/","html_tokens":51992,"markdown_tokens":66,"pruned_tokens":66}},
{"rank":305,"domain":"ovhcloud.com","origin":"https://ovhcloud.com","robots":{"status":"ok","http":200,"url":"https://www.ovhcloud.com/robots.txt","bytes":1604,"groups":1},"bots":{"GPTBot":["partial","*",3,0],"ChatGPT-User":["partial","*",3,0],"OAI-SearchBot":["partial","*",3,0],"ClaudeBot":["partial","*",3,0],"Claude-User":["partial","*",3,0],"Claude-SearchBot":["partial","*",3,0],"anthropic-ai":["partial","*",3,0],"PerplexityBot":["partial","*",3,0],"Perplexity-User":["partial","*",3,0],"Google-Extended":["partial","*",3,0],"Applebot-Extended":["partial","*",3,0],"CCBot":["partial","*",3,0],"Bytespider":["partial","*",3,0],"Amazonbot":["partial","*",3,0],"meta-externalagent":["partial","*",3,0],"cohere-ai":["partial","*",3,0],"Diffbot":["partial","*",3,0],"ContextFetchBot":["partial","*",3,0]},"llms":{"status":"missing","http":404,"bytes":977396},"home":{"status":"ok","http":200,"url":"https://us.ovhcloud.com/","html_tokens":93860,"markdown_tokens":891,"pruned_tokens":618}},
{"rank":306,"domain":"indiatimes.com","origin":"https://indiatimes.com","robots":{"status":"ok","http":200,"url":"https://www.indiatimes.com/robots.txt","bytes":1143,"groups":3},"bots":{"GPTBot":["partial","*",42,1],"ChatGPT-User":["partial","*",42,1],"OAI-SearchBot":["partial","*",42,1],"ClaudeBot":["partial","*",42,1],"Claude-User":["partial","*",42,1],"Claude-SearchBot":["partial","*",42,1],"anthropic-ai":["partial","*",42,1],"PerplexityBot":["partial","*",42,1],"Perplexity-User":["partial","*",42,1],"Google-Extended":["partial","*",42,1],"Applebot-Extended":["partial","*",42,1],"CCBot":["partial","*",42,1],"Bytespider":["partial","*",42,1],"Amazonbot":["partial","*",42,1],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["partial","*",42,1],"Diffbot":["partial","*",42,1],"ContextFetchBot":["partial","*",42,1]},"llms":{"status":"missing","http":404,"bytes":196902},"home":{"status":"ok","http":200,"url":"https://www.indiatimes.com/","html_tokens":32100,"markdown_tokens":24,"pruned_tokens":24}},
{"rank":307,"domain":"themeforest.net","origin":"https://themeforest.net","robots":{"status":"refused","http":403,"url":"https://themeforest.net/robots.txt","bytes":5574,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":308,"domain":"fandom.com","origin":"https://fandom.com","robots":{"status":"refused","http":403,"url":"https://www.fandom.com/robots.txt","bytes":5551,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":309,"domain":"noaa.gov","origin":"https://noaa.gov","robots":{"status":"refused","http":403,"url":"https://www.noaa.gov/robots.txt","bytes":919,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":310,"domain":"pexels.com","origin":"https://pexels.com","robots":{"status":"ok","http":200,"url":"https://www.pexels.com/robots.txt","bytes":11790,"groups":4},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["partial","*",110,1],"OAI-SearchBot":["partial","*",110,1],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",110,1],"Claude-SearchBot":["partial","*",110,1],"anthropic-ai":["partial","*",110,1],"PerplexityBot":["partial","*",110,1],"Perplexity-User":["partial","*",110,1],"Google-Extended":["partial","*",110,1],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["partial","*",110,1],"meta-externalagent":["partial","*",110,1],"cohere-ai":["partial","*",110,1],"Diffbot":["partial","*",110,1],"ContextFetchBot":["partial","*",110,1]},"llms":{"status":"http_error","http":403,"bytes":5567},"home":{"status":"http_error","http":403,"url":"https://www.pexels.com/"}},
{"rank":311,"domain":"mayoclinic.org","origin":"https://mayoclinic.org","robots":{"status":"refused","http":403,"url":"https://www.mayoclinic.org/robots.txt","bytes":384,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":312,"domain":"state.gov","origin":"https://state.gov","robots":{"status":"refused","http":403,"url":"https://www.state.gov/robots.txt","bytes":659508,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":313,"domain":"ox.ac.uk","origin":"https://ox.ac.uk","robots":{"status":"refused","http":403,"url":"https://www.ox.ac.uk/robots.txt","bytes":5528,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":314,"domain":"theatlantic.com","origin":"https://theatlantic.com","robots":{"status":"ok","http":200,"url":"https://www.theatlantic.com/robots.txt","bytes":3722,"groups":44},"bots":{"GPTBot":["allowed","named",0,1],"ChatGPT-User":["allowed","named",0,1],"OAI-SearchBot":["allowed","named",0,1],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","named",1,1],"Claude-SearchBot":["partial","named",1,1],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["partial","named",1,1],"Perplexity-User":["partial","named",1,1],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["partial","named",1,1],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",9,2]},"llms":{"status":"http_error","http":403,"bytes":449911},"home":{"status":"http_error","http":403,"url":"https://www.theatlantic.com/"}},
{"rank":315,"domain":"microsoftonline.com","origin":"https://microsoftonline.com","robots":{"status":"unreachable","http":null,"url":"https://microsoftonline.com/robots.txt","bytes":0,"groups":0,"error":"cannot resolve microsoftonline.com"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":316,"domain":"buydomains.com","origin":"https://buydomains.com","robots":{"status":"refused","http":403,"url":"https://buydomains.com/robots.txt","bytes":5573,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":317,"domain":"nbcnews.com","origin":"https://nbcnews.com","robots":{"status":"ok","http":200,"url":"https://www.nbcnews.com/robots.txt","bytes":13381,"groups":39},"bots":{"GPTBot":["partial","named",1,9],"ChatGPT-User":["partial","named",1,9],"OAI-SearchBot":["partial","named",1,9],"ClaudeBot":["partial","named",1,9],"Claude-User":["partial","named",1,9],"Claude-SearchBot":["partial","named",1,9],"anthropic-ai":["partial","named",1,9],"PerplexityBot":["partial","named",1,9],"Perplexity-User":["partial","named",1,9],"Google-Extended":["partial","named",1,9],"Applebot-Extended":["partial","named",1,9],"CCBot":["partial","named",1,9],"Bytespider":["partial","named",1,9],"Amazonbot":["partial","named",1,9],"meta-externalagent":["partial","named",1,9],"cohere-ai":["partial","named",1,9],"Diffbot":["partial","named",1,9],"ContextFetchBot":["partial","*",12,0]},"llms":{"status":"missing","http":404,"bytes":173729},"home":{"status":"ok","http":200,"url":"https://www.nbcnews.com","html_tokens":318795,"markdown_tokens":234,"pruned_tokens":155}},
{"rank":318,"domain":"marriott.com","origin":"https://marriott.com","robots":{"status":"ok","http":200,"url":"https://www.marriott.com/robots.txt","bytes":2643,"groups":2},"bots":{"GPTBot":["partial","*",31,25],"ChatGPT-User":["partial","*",31,25],"OAI-SearchBot":["partial","*",31,25],"ClaudeBot":["partial","*",31,25],"Claude-User":["partial","*",31,25],"Claude-SearchBot":["partial","*",31,25],"anthropic-ai":["partial","*",31,25],"PerplexityBot":["partial","*",31,25],"Perplexity-User":["partial","*",31,25],"Google-Extended":["partial","*",31,25],"Applebot-Extended":["partial","*",31,25],"CCBot":["partial","*",31,25],"Bytespider":["partial","*",31,25],"Amazonbot":["partial","*",31,25],"meta-externalagent":["partial","*",31,25],"cohere-ai":["partial","*",31,25],"Diffbot":["partial","*",31,25],"ContextFetchBot":["partial","*",31,25]},"llms":{"status":"missing","http":404,"bytes":273741},"home":{"status":"http_error","http":403,"url":"https://www.marriott.com/default.mi"}},
{"rank":319,"domain":"cisco.com","origin":"https://cisco.com","robots":{"status":"ok","http":200,"url":"https://www.cisco.com/robots.txt","bytes":15619,"groups":3},"bots":{"GPTBot":["partial","*",175,0],"ChatGPT-User":["partial","*",175,0],"OAI-SearchBot":["partial","*",175,0],"ClaudeBot":["partial","*",175,0],"Claude-User":["partial","*",175,0],"Claude-SearchBot":["partial","*",175,0],"anthropic-ai":["partial","*",175,0],"PerplexityBot":["partial","*",175,0],"Perplexity-User":["partial","*",175,0],"Google-Extended":["partial","*",175,0],"Applebot-Extended":["partial","*",175,0],"CCBot":["partial","*",175,0],"Bytespider":["partial","*",175,0],"Amazonbot":["partial","*",175,0],"meta-externalagent":["partial","*",175,0],"cohere-ai":["partial","*",175,0],"Diffbot":["partial","*",175,0],"ContextFetchBot":["partial","*",175,0]},"llms":{"status":"other_text","http":200,"bytes":1196},"home":{"status":"ok","http":200,"url":"https://www.cisco.com/","html_tokens":47479,"markdown_tokens":134,"pruned_tokens":134}},
{"rank":320,"domain":"tencent.com","origin":"https://www.tencent.com","robots":{"status":"ok","http":200,"url":"https://www.tencent.com/robots.txt","bytes":24,"groups":1},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"missing","http":404,"bytes":129705},"home":{"status":"ok","http":200,"url":"https://www.tencent.com/","html_tokens":44420,"markdown_tokens":79,"pruned_tokens":79}},
{"rank":321,"domain":"plos.org","origin":"https://plos.org","robots":{"status":"ok","http":200,"url":"https://plos.org/robots.txt","bytes":166,"groups":1},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"missing","http":404,"bytes":226501},"home":{"status":"ok","http":200,"url":"https://plos.org/","html_tokens":63830,"markdown_tokens":621,"pruned_tokens":312}},
{"rank":322,"domain":"statcounter.com","origin":"https://statcounter.com","robots":{"status":"missing","http":404,"url":"https://statcounter.com/robots.txt","bytes":162,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":15719},"home":{"status":"ok","http":200,"url":"https://statcounter.com/","html_tokens":6195,"markdown_tokens":184,"pruned_tokens":108}},
{"rank":323,"domain":"fb.com","origin":"https://fb.com","robots":{"status":"ok","http":200,"url":"https://fb.com/robots.txt","bytes":26872,"groups":36},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["blocked","*",1,0],"OAI-SearchBot":["blocked","*",1,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["blocked","*",1,0],"Claude-SearchBot":["blocked","*",1,0],"anthropic-ai":["blocked","*",1,0],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["blocked","*",1,0],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","*",1,0],"Bytespider":["blocked","*",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","*",1,0],"cohere-ai":["blocked","*",1,0],"Diffbot":["blocked","*",1,0],"ContextFetchBot":["blocked","*",1,0]},"llms":{"status":"disallowed"},"home":{"status":"disallowed"}},
{"rank":324,"domain":"huffpost.com","origin":"https://huffpost.com","robots":{"status":"ok","http":200,"url":"https://www.huffpost.com/robots.txt","bytes":2562,"groups":33},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["blocked","named",1,0],"OAI-SearchBot":["blocked","named",1,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["blocked","named",1,0],"Claude-SearchBot":["blocked","named",1,0],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["blocked","named",1,0],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",9,0]},"llms":{"status":"missing","http":404,"bytes":115144},"home":{"status":"ok","http":200,"url":"https://www.huffpost.com/","html_tokens":300629,"markdown_tokens":10,"pruned_tokens":10}},
{"rank":325,"domain":"columbia.edu","origin":"https://columbia.edu","robots":{"status":"missing","http":404,"url":"https://columbia.edu/robots.txt","bytes":1174,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":1174},"home":{"status":"http_error","http":403,"url":"https://columbia.edu/"}},
{"rank":326,"domain":"apa.org","origin":"https://apa.org","robots":{"status":"ok","http":200,"url":"https://www.apa.org/robots.txt","bytes":1066,"groups":3},"bots":{"GPTBot":["partial","*",10,1],"ChatGPT-User":["partial","*",10,1],"OAI-SearchBot":["partial","*",10,1],"ClaudeBot":["partial","*",10,1],"Claude-User":["partial","*",10,1],"Claude-SearchBot":["partial","*",10,1],"anthropic-ai":["partial","*",10,1],"PerplexityBot":["partial","*",10,1],"Perplexity-User":["partial","*",10,1],"Google-Extended":["partial","*",10,1],"Applebot-Extended":["partial","*",10,1],"CCBot":["partial","*",10,1],"Bytespider":["partial","*",10,1],"Amazonbot":["partial","*",10,1],"meta-externalagent":["partial","*",10,1],"cohere-ai":["partial","*",10,1],"Diffbot":["partial","*",10,1],"ContextFetchBot":["partial","*",10,1]},"llms":{"status":"missing","http":404,"bytes":5849},"home":{"status":"ok","http":200,"url":"https://www.apa.org/","html_tokens":233,"markdown_tokens":21,"pruned_tokens":21}},
{"rank":327,"domain":"it.com","origin":"https://it.com","robots":{"status":"missing","http":404,"url":"https://it.com/robots.txt","bytes":275,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":275},"home":{"status":"disallowed","error":"robots.txt of get.it.com not readable (refused)"}},
{"rank":328,"domain":"sohu.com","origin":"https://sohu.com","robots":{"status":"ok","http":200,"url":"https://www.sohu.com/robots.txt","bytes":3097,"groups":4},"bots":{"GPTBot":["partial","named",1,38],"ChatGPT-User":["partial","named",1,38],"OAI-SearchBot":["partial","named",1,38],"ClaudeBot":["partial","named",1,38],"Claude-User":["partial","named",1,38],"Claude-SearchBot":["partial","named",1,38],"anthropic-ai":["partial","named",1,38],"PerplexityBot":["partial","named",1,38],"Perplexity-User":["partial","named",1,38],"Google-Extended":["partial","named",1,38],"Applebot-Extended":["partial","named",1,38],"CCBot":["partial","named",1,38],"Bytespider":["partial","named",1,38],"Amazonbot":["partial","named",1,38],"meta-externalagent":["partial","named",1,38],"cohere-ai":["partial","named",1,38],"Diffbot":["partial","named",1,38],"ContextFetchBot":["blocked","*",1,0]},"llms":{"status":"disallowed"},"home":{"status":"disallowed"}},
{"rank":329,"domain":"livejournal.com","origin":"https://livejournal.com","robots":{"status":"timeout","http":null,"url":"https://livejournal.com/robots.txt","bytes":0,"groups":0,"error":"ConnectTimeout"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":330,"domain":"arcgis.com","origin":"https://arcgis.com","robots":{"status":"ok","http":200,"url":"https://www.arcgis.com/robots.txt","bytes":61,"groups":1},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"missing","http":404,"bytes":1190},"home":{"status":"ok","http":200,"url":"https://www.arcgis.com/index.html","html_tokens":206,"markdown_tokens":0,"pruned_tokens":0}},
{"rank":331,"domain":"intel.com","origin":"https://intel.com","robots":{"status":"ok","http":200,"url":"https://www.intel.com/robots.txt","bytes":4211,"groups":9},"bots":{"GPTBot":["allowed","named",0,1],"ChatGPT-User":["allowed","named",0,1],"OAI-SearchBot":["partial","*",49,4],"ClaudeBot":["allowed","named",0,1],"Claude-User":["partial","*",49,4],"Claude-SearchBot":["partial","*",49,4],"anthropic-ai":["allowed","named",0,1],"PerplexityBot":["allowed","named",0,1],"Perplexity-User":["partial","*",49,4],"Google-Extended":["allowed","named",0,1],"Applebot-Extended":["partial","*",49,4],"CCBot":["partial","*",49,4],"Bytespider":["partial","*",49,4],"Amazonbot":["partial","*",49,4],"meta-externalagent":["partial","*",49,4],"cohere-ai":["partial","*",49,4],"Diffbot":["partial","*",49,4],"ContextFetchBot":["partial","*",49,4]},"llms":{"status":"present","http":200,"bytes":12892},"home":{"status":"ok","http":200,"url":"https://www.intel.com/content/www/us/en/homepage.html","html_tokens":36128,"markdown_tokens":40,"pruned_tokens":37}},
{"rank":332,"domain":"xing.com","origin":"https://xing.com","robots":{"status":"ok","http":200,"url":"https://www.xing.com/robots.txt","bytes":5576,"groups":4},"bots":{"GPTBot":["partial","named",11,2],"ChatGPT-User":["partial","*",222,1],"OAI-SearchBot":["partial","*",222,1],"ClaudeBot":["partial","named",11,2],"Claude-User":["partial","named",11,2],"Claude-SearchBot":["partial","named",11,2],"anthropic-ai":["partial","*",222,1],"PerplexityBot":["partial","named",11,2],"Perplexity-User":["partial","named",11,2],"Google-Extended":["partial","*",222,1],"Applebot-Extended":["partial","*",222,1],"CCBot":["partial","*",222,1],"Bytespider":["partial","*",222,1],"Amazonbot":["partial","*",222,1],"meta-externalagent":["partial","*",222,1],"cohere-ai":["partial","*",222,1],"Diffbot":["partial","*",222,1],"ContextFetchBot":["partial","*",222,1]},"llms":{"status":"present","http":200,"bytes":5157},"home":{"status":"ok","http":200,"url":"https://www.xing.com/","html_tokens":107140,"markdown_tokens":27,"pruned_tokens":27}},
{"rank":333,"domain":"indeed.com","origin":"https://indeed.com","robots":{"status":"ok","http":200,"url":"https://www.indeed.com/robots.txt","bytes":13731,"groups":9},"bots":{"GPTBot":["partial","named",151,10],"ChatGPT-User":["partial","named",159,11],"OAI-SearchBot":["partial","named",159,11],"ClaudeBot":["partial","named",151,10],"Claude-User":["partial","named",159,11],"Claude-SearchBot":["partial","named",159,11],"anthropic-ai":["partial","named",151,10],"PerplexityBot":["partial","named",159,11],"Perplexity-User":["partial","named",159,11],"Google-Extended":["partial","named",159,11],"Applebot-Extended":["partial","named",151,10],"CCBot":["partial","named",151,10],"Bytespider":["partial","named",151,10],"Amazonbot":["partial","named",151,10],"meta-externalagent":["partial","named",151,10],"cohere-ai":["partial","*",169,11],"Diffbot":["partial","named",151,10],"ContextFetchBot":["partial","*",169,11]},"llms":{"status":"present","http":200,"bytes":41768},"home":{"status":"http_error","http":403,"url":"https://indeed.com/"}},
{"rank":334,"domain":"translate.google.com","origin":"https://translate.google.com","robots":{"status":"ok","http":200,"url":"https://translate.google.com/robots.txt","bytes":372,"groups":2},"bots":{"GPTBot":["partial","*",9,2],"ChatGPT-User":["partial","*",9,2],"OAI-SearchBot":["partial","*",9,2],"ClaudeBot":["partial","*",9,2],"Claude-User":["partial","*",9,2],"Claude-SearchBot":["partial","*",9,2],"anthropic-ai":["partial","*",9,2],"PerplexityBot":["partial","*",9,2],"Perplexity-User":["partial","*",9,2],"Google-Extended":["partial","*",9,2],"Applebot-Extended":["partial","*",9,2],"CCBot":["partial","*",9,2],"Bytespider":["partial","*",9,2],"Amazonbot":["partial","*",9,2],"meta-externalagent":["partial","*",9,2],"cohere-ai":["partial","*",9,2],"Diffbot":["partial","*",9,2],"ContextFetchBot":["partial","*",9,2]},"llms":{"status":"missing","http":404,"bytes":1569},"home":{"status":"ok","http":200,"url":"https://translate.google.com/","html_tokens":762707,"markdown_tokens":20,"pruned_tokens":20}},
{"rank":335,"domain":"amazon.de","origin":"https://amazon.de","robots":{"status":"ok","http":200,"url":"https://www.amazon.de/robots.txt","bytes":7508,"groups":102},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["blocked","named",1,0],"OAI-SearchBot":["blocked","named",1,0],"ClaudeBot":["blocked","named",2,0],"Claude-User":["blocked","named",1,0],"Claude-SearchBot":["blocked","named",1,0],"anthropic-ai":["partial","*",102,11],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["blocked","named",1,0],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["partial","*",102,11],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["partial","*",102,11],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",102,11]},"llms":{"status":"missing","http":404,"bytes":2165},"home":{"status":"ok","http":200,"url":"https://www.amazon.de/-/en/","html_tokens":169740,"markdown_tokens":65,"pruned_tokens":22}},
{"rank":336,"domain":"webmd.com","origin":"https://webmd.com","robots":{"status":"ok","http":200,"url":"https://www.webmd.com/robots.txt","bytes":886,"groups":6},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["partial","*",22,0],"OAI-SearchBot":["partial","*",22,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",22,0],"Claude-SearchBot":["partial","*",22,0],"anthropic-ai":["partial","*",22,0],"PerplexityBot":["partial","*",22,0],"Perplexity-User":["partial","*",22,0],"Google-Extended":["partial","*",22,0],"Applebot-Extended":["partial","*",22,0],"CCBot":["blocked","named",1,0],"Bytespider":["partial","*",22,0],"Amazonbot":["partial","*",22,0],"meta-externalagent":["partial","*",22,0],"cohere-ai":["partial","*",22,0],"Diffbot":["partial","*",22,0],"ContextFetchBot":["partial","*",22,0]},"llms":{"status":"missing","http":404,"bytes":298395},"home":{"status":"ok","http":200,"url":"https://www.webmd.com/","html_tokens":109696,"markdown_tokens":474,"pruned_tokens":130}},
{"rank":337,"domain":"eu.com","origin":"https://eu.com","robots":{"status":"ok","http":200,"url":"https://eu.com/robots.txt","bytes":0,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":7806},"home":{"status":"ok","http":200,"url":"https://eu.com/","html_tokens":10994,"markdown_tokens":820,"pruned_tokens":807}},
{"rank":338,"domain":"odoo.com","origin":"https://odoo.com","robots":{"status":"timeout","http":null,"url":"https://odoo.com/robots.txt","bytes":0,"groups":0,"error":"timed out after 10 s"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":339,"domain":"freepik.com","origin":"https://freepik.com","robots":{"status":"ok","http":200,"url":"https://www.magnific.com/robots.txt","bytes":5416,"groups":4},"bots":{"GPTBot":["partial","*",111,0],"ChatGPT-User":["partial","*",111,0],"OAI-SearchBot":["partial","*",111,0],"ClaudeBot":["partial","*",111,0],"Claude-User":["partial","*",111,0],"Claude-SearchBot":["partial","*",111,0],"anthropic-ai":["partial","*",111,0],"PerplexityBot":["partial","*",111,0],"Perplexity-User":["partial","*",111,0],"Google-Extended":["partial","*",111,0],"Applebot-Extended":["partial","*",111,0],"CCBot":["partial","*",111,0],"Bytespider":["partial","*",111,0],"Amazonbot":["partial","*",111,0],"meta-externalagent":["partial","*",111,0],"cohere-ai":["partial","*",111,0],"Diffbot":["partial","*",111,0],"ContextFetchBot":["partial","*",111,0]},"llms":{"status":"http_error","http":403,"bytes":7530},"home":{"status":"http_error","http":403,"url":"https://www.freepik.com/"}},
{"rank":340,"domain":"or.kr","origin":"https://or.kr","robots":{"status":"unreachable","http":null,"url":"https://or.kr/robots.txt","bytes":0,"groups":0,"error":"cannot resolve or.kr"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":341,"domain":"patreon.com","origin":"https://patreon.com","robots":{"status":"ok","http":200,"url":"https://patreon.com/robots.txt","bytes":759,"groups":7},"bots":{"GPTBot":["partial","*",17,0],"ChatGPT-User":["partial","*",17,0],"OAI-SearchBot":["partial","*",17,0],"ClaudeBot":["partial","*",17,0],"Claude-User":["partial","*",17,0],"Claude-SearchBot":["partial","*",17,0],"anthropic-ai":["partial","*",17,0],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["partial","*",17,0],"Google-Extended":["partial","*",17,0],"Applebot-Extended":["partial","*",17,0],"CCBot":["partial","*",17,0],"Bytespider":["partial","*",17,0],"Amazonbot":["partial","*",17,0],"meta-externalagent":["partial","*",17,0],"cohere-ai":["partial","*",17,0],"Diffbot":["partial","*",17,0],"ContextFetchBot":["partial","*",17,0]},"llms":{"status":"missing","http":404,"bytes":349570},"home":{"status":"ok","http":200,"url":"https://www.patreon.com/","html_tokens":83802,"markdown_tokens":350,"pruned_tokens":293}},
{"rank":342,"domain":"salesforce.com","origin":"https://salesforce.com","robots":{"status":"ok","http":200,"url":"https://www.salesforce.com/robots.txt","bytes":14256,"groups":9},"bots":{"GPTBot":["partial","*",181,1],"ChatGPT-User":["partial","*",181,1],"OAI-SearchBot":["partial","*",181,1],"ClaudeBot":["partial","*",181,1],"Claude-User":["partial","*",181,1],"Claude-SearchBot":["partial","*",181,1],"anthropic-ai":["partial","*",181,1],"PerplexityBot":["partial","*",181,1],"Perplexity-User":["partial","*",181,1],"Google-Extended":["partial","*",181,1],"Applebot-Extended":["partial","*",181,1],"CCBot":["partial","*",181,1],"Bytespider":["partial","*",181,1],"Amazonbot":["partial","*",181,1],"meta-externalagent":["partial","*",181,1],"cohere-ai":["partial","*",181,1],"Diffbot":["partial","*",181,1],"ContextFetchBot":["partial","*",181,1]},"llms":{"status":"other_text","http":200,"bytes":1021703},"home":{"status":"ok","http":200,"url":"https://www.salesforce.com/?bc=DB","html_tokens":307405,"markdown_tokens":3588,"pruned_tokens":611}},
{"rank":343,"domain":"yale.edu","origin":"https://www.yale.edu","robots":{"status":"ok","http":200,"url":"https://www.yale.edu/robots.txt","bytes":2491,"groups":1},"bots":{"GPTBot":["partial","*",43,34],"ChatGPT-User":["partial","*",43,34],"OAI-SearchBot":["partial","*",43,34],"ClaudeBot":["partial","*",43,34],"Claude-User":["partial","*",43,34],"Claude-SearchBot":["partial","*",43,34],"anthropic-ai":["partial","*",43,34],"PerplexityBot":["partial","*",43,34],"Perplexity-User":["partial","*",43,34],"Google-Extended":["partial","*",43,34],"Applebot-Extended":["partial","*",43,34],"CCBot":["partial","*",43,34],"Bytespider":["partial","*",43,34],"Amazonbot":["partial","*",43,34],"meta-externalagent":["partial","*",43,34],"cohere-ai":["partial","*",43,34],"Diffbot":["partial","*",43,34],"ContextFetchBot":["partial","*",43,34]},"llms":{"status":"missing","http":404,"bytes":29508},"home":{"status":"ok","http":200,"url":"https://www.yale.edu/","html_tokens":25681,"markdown_tokens":97,"pruned_tokens":97}},
{"rank":344,"domain":"v.qq.com","origin":"https://v.qq.com","robots":{"status":"ok","http":200,"url":"https://v.qq.com/robots.txt","bytes":1638,"groups":14},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["blocked","named",1,0],"OAI-SearchBot":["partial","*",36,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",36,0],"Claude-SearchBot":["partial","*",36,0],"anthropic-ai":["partial","*",36,0],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["partial","*",36,0],"Google-Extended":["partial","*",36,0],"Applebot-Extended":["partial","*",36,0],"CCBot":["partial","*",36,0],"Bytespider":["partial","*",36,0],"Amazonbot":["partial","*",36,0],"meta-externalagent":["partial","*",36,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["partial","*",36,0],"ContextFetchBot":["partial","*",36,0]},"llms":{"status":"missing","http":404,"bytes":98820},"home":{"status":"ok","http":200,"url":"https://v.qq.com/","html_tokens":34282,"markdown_tokens":40,"pruned_tokens":40}},
{"rank":345,"domain":"theconversation.com","origin":"https://theconversation.com","robots":{"status":"ok","http":200,"url":"https://theconversation.com/robots.txt","bytes":5692,"groups":3},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["blocked","named",1,0],"OAI-SearchBot":["blocked","named",1,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["blocked","named",1,0],"Claude-SearchBot":["blocked","named",1,0],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["blocked","named",1,0],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",13,0]},"llms":{"status":"missing","http":404,"bytes":145437},"home":{"status":"ok","http":200,"url":"https://theconversation.com/us","html_tokens":263546,"markdown_tokens":199,"pruned_tokens":74}},
{"rank":346,"domain":"umich.edu","origin":"https://umich.edu","robots":{"status":"ok","http":200,"url":"https://umich.edu/robots.txt","bytes":97,"groups":1},"bots":{"GPTBot":["partial","*",4,0],"ChatGPT-User":["partial","*",4,0],"OAI-SearchBot":["partial","*",4,0],"ClaudeBot":["partial","*",4,0],"Claude-User":["partial","*",4,0],"Claude-SearchBot":["partial","*",4,0],"anthropic-ai":["partial","*",4,0],"PerplexityBot":["partial","*",4,0],"Perplexity-User":["partial","*",4,0],"Google-Extended":["partial","*",4,0],"Applebot-Extended":["partial","*",4,0],"CCBot":["partial","*",4,0],"Bytespider":["partial","*",4,0],"Amazonbot":["partial","*",4,0],"meta-externalagent":["partial","*",4,0],"cohere-ai":["partial","*",4,0],"Diffbot":["partial","*",4,0],"ContextFetchBot":["partial","*",4,0]},"llms":{"status":"http_error","http":403,"bytes":5522},"home":{"status":"http_error","http":403,"url":"https://umich.edu/"}},
{"rank":347,"domain":"kickstarter.com","origin":"https://kickstarter.com","robots":{"status":"refused","http":403,"url":"https://www.kickstarter.com/robots.txt","bytes":5557,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":348,"domain":"pbs.org","origin":"https://pbs.org","robots":{"status":"ok","http":200,"url":"https://www.pbs.org/robots.txt","bytes":3502,"groups":7},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["partial","*",91,0],"OAI-SearchBot":["partial","*",91,0],"ClaudeBot":["partial","*",91,0],"Claude-User":["partial","*",91,0],"Claude-SearchBot":["partial","*",91,0],"anthropic-ai":["partial","*",91,0],"PerplexityBot":["partial","named",1,1],"Perplexity-User":["partial","*",91,0],"Google-Extended":["partial","*",91,0],"Applebot-Extended":["partial","*",91,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["partial","*",91,0],"meta-externalagent":["partial","*",91,0],"cohere-ai":["partial","*",91,0],"Diffbot":["partial","*",91,0],"ContextFetchBot":["partial","*",91,0]},"llms":{"status":"http_error","http":502,"bytes":932},"home":{"status":"ok","http":200,"url":"https://www.pbs.org/","html_tokens":309469,"markdown_tokens":68,"pruned_tokens":68}},
{"rank":349,"domain":"ftc.gov","origin":"https://ftc.gov","robots":{"status":"ok","http":200,"url":"https://www.ftc.gov/robots.txt","bytes":3056,"groups":1},"bots":{"GPTBot":["partial","*",37,18],"ChatGPT-User":["partial","*",37,18],"OAI-SearchBot":["partial","*",37,18],"ClaudeBot":["partial","*",37,18],"Claude-User":["partial","*",37,18],"Claude-SearchBot":["partial","*",37,18],"anthropic-ai":["partial","*",37,18],"PerplexityBot":["partial","*",37,18],"Perplexity-User":["partial","*",37,18],"Google-Extended":["partial","*",37,18],"Applebot-Extended":["partial","*",37,18],"CCBot":["partial","*",37,18],"Bytespider":["partial","*",37,18],"Amazonbot":["partial","*",37,18],"meta-externalagent":["partial","*",37,18],"cohere-ai":["partial","*",37,18],"Diffbot":["partial","*",37,18],"ContextFetchBot":["partial","*",37,18]},"llms":{"status":"missing","http":404,"bytes":185},"home":{"status":"ok","http":200,"url":"https://www.ftc.gov/","html_tokens":250491,"markdown_tokens":239,"pruned_tokens":131}},
{"rank":350,"domain":"quora.com","origin":"https://quora.com","robots":{"status":"ok","http":200,"url":"https://www.quora.com/robots.txt","bytes":18962,"groups":29},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["blocked","named",1,0],"OAI-SearchBot":["partial","named",1,4],"ClaudeBot":["blocked","named",1,0],"Claude-User":["blocked","named",1,0],"Claude-SearchBot":["blocked","named",1,0],"anthropic-ai":["partial","*",63,7],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["partial","*",63,7],"Google-Extended":["partial","named",62,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["partial","*",63,7],"Bytespider":["blocked","named",1,0],"Amazonbot":["partial","named",62,0],"meta-externalagent":["partial","*",63,7],"cohere-ai":["partial","*",63,7],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",63,7]},"llms":{"status":"disallowed"},"home":{"status":"http_error","http":403,"url":"https://www.quora.com/"}},
{"rank":351,"domain":"biomedcentral.com","origin":"https://biomedcentral.com","robots":{"status":"missing","http":404,"url":"https://www.biomedcentral.com/robots.txt","bytes":8190,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"html","http":200,"bytes":216549},"home":{"status":"ok","http":200,"url":"https://link.springer.com/brands/bmc","html_tokens":54136,"markdown_tokens":83,"pruned_tokens":83}},
{"rank":352,"domain":"economist.com","origin":"https://economist.com","robots":{"status":"ok","http":200,"url":"https://www.economist.com/robots.txt","bytes":2664,"groups":22},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["partial","named",1,4],"OAI-SearchBot":["partial","named",1,4],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","named",1,4],"Claude-SearchBot":["partial","named",1,4],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["partial","named",1,4],"Perplexity-User":["partial","named",1,4],"Google-Extended":["partial","named",1,4],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["partial","*",11,0],"meta-externalagent":["partial","*",11,0],"cohere-ai":["partial","*",11,0],"Diffbot":["partial","*",11,0],"ContextFetchBot":["partial","*",11,0]},"llms":{"status":"http_error","http":403,"bytes":1134},"home":{"status":"http_error","http":403,"url":"https://www.economist.com/"}},
{"rank":353,"domain":"ico.org.uk","origin":"https://ico.org.uk","robots":{"status":"ok","http":200,"url":"https://ico.org.uk/robots.txt","bytes":206,"groups":2},"bots":{"GPTBot":["partial","*",2,0],"ChatGPT-User":["partial","*",2,0],"OAI-SearchBot":["partial","*",2,0],"ClaudeBot":["partial","*",2,0],"Claude-User":["partial","*",2,0],"Claude-SearchBot":["partial","*",2,0],"anthropic-ai":["partial","*",2,0],"PerplexityBot":["partial","*",2,0],"Perplexity-User":["partial","*",2,0],"Google-Extended":["partial","*",2,0],"Applebot-Extended":["partial","*",2,0],"CCBot":["partial","*",2,0],"Bytespider":["partial","*",2,0],"Amazonbot":["partial","*",2,0],"meta-externalagent":["partial","*",2,0],"cohere-ai":["partial","*",2,0],"Diffbot":["partial","*",2,0],"ContextFetchBot":["partial","*",2,0]},"llms":{"status":"missing","http":404,"bytes":0},"home":{"status":"ok","http":200,"url":"https://ico.org.uk/","html_tokens":13096,"markdown_tokens":13,"pruned_tokens":13}},
{"rank":354,"domain":"healthline.com","origin":"https://healthline.com","robots":{"status":"ok","http":200,"url":"https://www.healthline.com/robots.txt","bytes":8864,"groups":165},"bots":{"GPTBot":["partial","named",1,2],"ChatGPT-User":["partial","*",36,11],"OAI-SearchBot":["partial","*",36,11],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",36,11],"Claude-SearchBot":["partial","*",36,11],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["partial","*",36,11],"Perplexity-User":["partial","*",36,11],"Google-Extended":["partial","*",36,11],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["partial","*",36,11],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",36,11]},"llms":{"status":"missing","http":404,"bytes":159427},"home":{"status":"ok","http":200,"url":"https://www.healthline.com/","html_tokens":106619,"markdown_tokens":151,"pruned_tokens":119}},
{"rank":355,"domain":"dribbble.com","origin":"https://dribbble.com","robots":{"status":"ok","http":200,"url":"https://dribbble.com/robots.txt","bytes":998,"groups":2},"bots":{"GPTBot":["partial","*",38,0],"ChatGPT-User":["partial","*",38,0],"OAI-SearchBot":["partial","*",38,0],"ClaudeBot":["partial","*",38,0],"Claude-User":["partial","*",38,0],"Claude-SearchBot":["partial","*",38,0],"anthropic-ai":["partial","*",38,0],"PerplexityBot":["partial","*",38,0],"Perplexity-User":["partial","*",38,0],"Google-Extended":["partial","*",38,0],"Applebot-Extended":["partial","*",38,0],"CCBot":["partial","*",38,0],"Bytespider":["partial","*",38,0],"Amazonbot":["partial","*",38,0],"meta-externalagent":["partial","*",38,0],"cohere-ai":["partial","*",38,0],"Diffbot":["partial","*",38,0],"ContextFetchBot":["partial","*",38,0]},"llms":{"status":"missing","http":404,"bytes":21145},"home":{"status":"ok","http":202,"url":"https://dribbble.com/","html_tokens":504,"markdown_tokens":40,"pruned_tokens":40}},
{"rank":356,"domain":"change.org","origin":"https://change.org","robots":{"status":"ok","http":200,"url":"https://www.change.org/robots.txt","bytes":5034,"groups":1},"bots":{"GPTBot":["partial","*",157,0],"ChatGPT-User":["partial","*",157,0],"OAI-SearchBot":["partial","*",157,0],"ClaudeBot":["partial","*",157,0],"Claude-User":["partial","*",157,0],"Claude-SearchBot":["partial","*",157,0],"anthropic-ai":["partial","*",157,0],"PerplexityBot":["partial","*",157,0],"Perplexity-User":["partial","*",157,0],"Google-Extended":["partial","*",157,0],"Applebot-Extended":["partial","*",157,0],"CCBot":["partial","*",157,0],"Bytespider":["partial","*",157,0],"Amazonbot":["partial","*",157,0],"meta-externalagent":["partial","*",157,0],"cohere-ai":["partial","*",157,0],"Diffbot":["partial","*",157,0],"ContextFetchBot":["partial","*",157,0]},"llms":{"status":"missing","http":404,"bytes":154448},"home":{"status":"ok","http":200,"url":"https://www.change.org/","html_tokens":77114,"markdown_tokens":82,"pruned_tokens":17}},
{"rank":357,"domain":"fb.me","origin":"https://fb.me","robots":{"status":"html","http":200,"url":"https://www.facebook.com/robotstxt","bytes":307321,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":358,"domain":"canada.ca","origin":"https://canada.ca","robots":{"status":"ok","http":200,"url":"https://www.canada.ca/robots.txt","bytes":2730,"groups":1},"bots":{"GPTBot":["partial","*",47,0],"ChatGPT-User":["partial","*",47,0],"OAI-SearchBot":["partial","*",47,0],"ClaudeBot":["partial","*",47,0],"Claude-User":["partial","*",47,0],"Claude-SearchBot":["partial","*",47,0],"anthropic-ai":["partial","*",47,0],"PerplexityBot":["partial","*",47,0],"Perplexity-User":["partial","*",47,0],"Google-Extended":["partial","*",47,0],"Applebot-Extended":["partial","*",47,0],"CCBot":["partial","*",47,0],"Bytespider":["partial","*",47,0],"Amazonbot":["partial","*",47,0],"meta-externalagent":["partial","*",47,0],"cohere-ai":["partial","*",47,0],"Diffbot":["partial","*",47,0],"ContextFetchBot":["partial","*",47,0]},"llms":{"status":"html","http":200,"bytes":13218},"home":{"status":"ok","http":200,"url":"https://www.canada.ca/","html_tokens":2258,"markdown_tokens":49,"pruned_tokens":21}},
{"rank":359,"domain":"businesswire.com","origin":"https://businesswire.com","robots":{"status":"refused","http":403,"url":"https://businesswire.com/robots.txt","bytes":378,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":360,"domain":"huffingtonpost.com","origin":"https://huffingtonpost.com","robots":{"status":"ok","http":200,"url":"https://www.huffpost.com/robots.txt","bytes":2562,"groups":33},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["blocked","named",1,0],"OAI-SearchBot":["blocked","named",1,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["blocked","named",1,0],"Claude-SearchBot":["blocked","named",1,0],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["blocked","named",1,0],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",9,0]},"llms":{"status":"missing","http":404,"bytes":115195},"home":{"status":"ok","http":200,"url":"https://www.huffpost.com","html_tokens":300629,"markdown_tokens":10,"pruned_tokens":10}},
{"rank":361,"domain":"android.com","origin":"https://android.com","robots":{"status":"ok","http":200,"url":"https://www.android.com/robots.txt","bytes":1890,"groups":15},"bots":{"GPTBot":["partial","named",5,0],"ChatGPT-User":["partial","*",5,0],"OAI-SearchBot":["partial","named",5,0],"ClaudeBot":["partial","named",5,0],"Claude-User":["partial","*",5,0],"Claude-SearchBot":["partial","named",5,0],"anthropic-ai":["partial","named",5,0],"PerplexityBot":["partial","named",5,0],"Perplexity-User":["partial","*",5,0],"Google-Extended":["partial","named",5,0],"Applebot-Extended":["partial","named",5,0],"CCBot":["partial","named",5,0],"Bytespider":["partial","*",5,0],"Amazonbot":["partial","*",5,0],"meta-externalagent":["partial","*",5,0],"cohere-ai":["partial","named",5,0],"Diffbot":["partial","named",5,0],"ContextFetchBot":["partial","*",5,0]},"llms":{"status":"missing","http":404,"bytes":232037},"home":{"status":"ok","http":200,"url":"https://www.android.com/","html_tokens":93247,"markdown_tokens":143,"pruned_tokens":53}},
{"rank":362,"domain":"google.co.uk","origin":"https://google.co.uk","robots":{"status":"ok","http":200,"url":"https://www.google.co.uk/robots.txt","bytes":6624,"groups":4},"bots":{"GPTBot":["partial","*",169,64],"ChatGPT-User":["partial","*",169,64],"OAI-SearchBot":["partial","*",169,64],"ClaudeBot":["partial","*",169,64],"Claude-User":["partial","*",169,64],"Claude-SearchBot":["partial","*",169,64],"anthropic-ai":["partial","*",169,64],"PerplexityBot":["partial","*",169,64],"Perplexity-User":["partial","*",169,64],"Google-Extended":["partial","*",169,64],"Applebot-Extended":["partial","*",169,64],"CCBot":["partial","*",169,64],"Bytespider":["partial","*",169,64],"Amazonbot":["partial","*",169,64],"meta-externalagent":["partial","*",169,64],"cohere-ai":["partial","*",169,64],"Diffbot":["partial","*",169,64],"ContextFetchBot":["partial","*",169,64]},"llms":{"status":"missing","http":404,"bytes":1569},"home":{"status":"ok","http":200,"url":"https://www.google.com/","html_tokens":21452,"markdown_tokens":30,"pruned_tokens":30}},
{"rank":363,"domain":"eepurl.com","origin":"https://eepurl.com","robots":{"status":"refused","http":400,"url":"https://eepurl.com/robots.txt","bytes":99,"groups":0,"error":"HTTP 400"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":364,"domain":"bp.blogspot.com","origin":"https://bp.blogspot.com","robots":{"status":"unreachable","http":null,"url":"https://bp.blogspot.com/robots.txt","bytes":0,"groups":0,"error":"cannot resolve bp.blogspot.com"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":365,"domain":"postgresql.org","origin":"https://postgresql.org","robots":{"status":"ok","http":200,"url":"https://www.postgresql.org/robots.txt","bytes":273,"groups":1},"bots":{"GPTBot":["partial","*",9,0],"ChatGPT-User":["partial","*",9,0],"OAI-SearchBot":["partial","*",9,0],"ClaudeBot":["partial","*",9,0],"Claude-User":["partial","*",9,0],"Claude-SearchBot":["partial","*",9,0],"anthropic-ai":["partial","*",9,0],"PerplexityBot":["partial","*",9,0],"Perplexity-User":["partial","*",9,0],"Google-Extended":["partial","*",9,0],"Applebot-Extended":["partial","*",9,0],"CCBot":["partial","*",9,0],"Bytespider":["partial","*",9,0],"Amazonbot":["partial","*",9,0],"meta-externalagent":["partial","*",9,0],"cohere-ai":["partial","*",9,0],"Diffbot":["partial","*",9,0],"ContextFetchBot":["partial","*",9,0]},"llms":{"status":"missing","http":404,"bytes":4217},"home":{"status":"ok","http":200,"url":"https://www.postgresql.org/","html_tokens":6189,"markdown_tokens":406,"pruned_tokens":364}},
{"rank":366,"domain":"eff.org","origin":"https://eff.org","robots":{"status":"ok","http":200,"url":"https://www.eff.org/robots.txt","bytes":3445,"groups":1},"bots":{"GPTBot":["partial","named",68,34],"ChatGPT-User":["partial","*",68,34],"OAI-SearchBot":["partial","*",68,34],"ClaudeBot":["partial","named",68,34],"Claude-User":["partial","*",68,34],"Claude-SearchBot":["partial","*",68,34],"anthropic-ai":["partial","named",68,34],"PerplexityBot":["partial","*",68,34],"Perplexity-User":["partial","*",68,34],"Google-Extended":["partial","named",68,34],"Applebot-Extended":["partial","*",68,34],"CCBot":["partial","named",68,34],"Bytespider":["partial","*",68,34],"Amazonbot":["partial","*",68,34],"meta-externalagent":["partial","*",68,34],"cohere-ai":["partial","*",68,34],"Diffbot":["partial","*",68,34],"ContextFetchBot":["partial","*",68,34]},"llms":{"status":"missing","http":404,"bytes":18900},"home":{"status":"ok","http":200,"url":"https://www.eff.org/","html_tokens":13830,"markdown_tokens":273,"pruned_tokens":130}},
{"rank":367,"domain":"alibaba.com","origin":"https://alibaba.com","robots":{"status":"ok","http":200,"url":"https://www.alibaba.com/atlassitemapsitenet/static/www_alibaba_com/robots.txt","bytes":5406,"groups":6},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["partial","*",43,10],"OAI-SearchBot":["partial","*",43,10],"ClaudeBot":["partial","*",43,10],"Claude-User":["partial","*",43,10],"Claude-SearchBot":["partial","*",43,10],"anthropic-ai":["partial","*",43,10],"PerplexityBot":["partial","*",43,10],"Perplexity-User":["partial","*",43,10],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["partial","*",43,10],"CCBot":["partial","*",43,10],"Bytespider":["partial","*",43,10],"Amazonbot":["partial","*",43,10],"meta-externalagent":["partial","*",43,10],"cohere-ai":["partial","*",43,10],"Diffbot":["partial","*",43,10],"ContextFetchBot":["partial","*",43,10]},"llms":{"status":"disallowed","error":"robots.txt of error.alibaba.com disallows /error404.htm"},"home":{"status":"ok","http":200,"url":"https://www.alibaba.com/","html_tokens":53057,"markdown_tokens":8,"pruned_tokens":8}},
{"rank":368,"domain":"bmj.com","origin":"https://bmj.com","robots":{"status":"ok","http":200,"url":"https://www.bmj.com/robots.txt","bytes":3104,"groups":3},"bots":{"GPTBot":["partial","*",69,2],"ChatGPT-User":["partial","*",69,2],"OAI-SearchBot":["partial","*",69,2],"ClaudeBot":["partial","*",69,2],"Claude-User":["partial","*",69,2],"Claude-SearchBot":["partial","*",69,2],"anthropic-ai":["partial","*",69,2],"PerplexityBot":["partial","*",69,2],"Perplexity-User":["partial","*",69,2],"Google-Extended":["partial","*",69,2],"Applebot-Extended":["partial","*",69,2],"CCBot":["partial","*",69,2],"Bytespider":["partial","*",69,2],"Amazonbot":["partial","*",69,2],"meta-externalagent":["partial","*",69,2],"cohere-ai":["partial","*",69,2],"Diffbot":["partial","*",69,2],"ContextFetchBot":["partial","*",69,2]},"llms":{"status":"missing","http":404,"bytes":43855},"home":{"status":"http_error","http":403,"url":"https://www.bmj.com/"}},
{"rank":369,"domain":"netlify.app","origin":"https://netlify.app","robots":{"status":"ok","http":200,"url":"https://netlify.app/robots.txt","bytes":147,"groups":1},"bots":{"GPTBot":["allowed","*",0,1],"ChatGPT-User":["allowed","*",0,1],"OAI-SearchBot":["allowed","*",0,1],"ClaudeBot":["allowed","*",0,1],"Claude-User":["allowed","*",0,1],"Claude-SearchBot":["allowed","*",0,1],"anthropic-ai":["allowed","*",0,1],"PerplexityBot":["allowed","*",0,1],"Perplexity-User":["allowed","*",0,1],"Google-Extended":["allowed","*",0,1],"Applebot-Extended":["allowed","*",0,1],"CCBot":["allowed","*",0,1],"Bytespider":["allowed","*",0,1],"Amazonbot":["allowed","*",0,1],"meta-externalagent":["allowed","*",0,1],"cohere-ai":["allowed","*",0,1],"Diffbot":["allowed","*",0,1],"ContextFetchBot":["allowed","*",0,1]},"llms":{"status":"present","http":200,"bytes":2818},"home":{"status":"not_html","error":"unsupported content type text/markdown"}},
{"rank":370,"domain":"sun.com","origin":"https://sun.com","robots":{"status":"refused","http":403,"url":"https://www.oracle.com/us/sun/index.htm","bytes":1448,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":371,"domain":"fortune.com","origin":"https://fortune.com","robots":{"status":"ok","http":200,"url":"https://fortune.com/robots.txt","bytes":1774,"groups":7},"bots":{"GPTBot":["partial","*",19,3],"ChatGPT-User":["partial","*",19,3],"OAI-SearchBot":["partial","*",19,3],"ClaudeBot":["partial","*",19,3],"Claude-User":["partial","*",19,3],"Claude-SearchBot":["partial","*",19,3],"anthropic-ai":["partial","*",19,3],"PerplexityBot":["partial","*",19,3],"Perplexity-User":["partial","*",19,3],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["partial","*",19,3],"CCBot":["blocked","named",1,0],"Bytespider":["partial","*",19,3],"Amazonbot":["partial","*",19,3],"meta-externalagent":["partial","*",19,3],"cohere-ai":["partial","*",19,3],"Diffbot":["partial","*",19,3],"ContextFetchBot":["partial","*",19,3]},"llms":{"status":"missing","http":404,"bytes":19561},"home":{"status":"ok","http":200,"url":"https://fortune.com/","html_tokens":249706,"markdown_tokens":74,"pruned_tokens":74}},
{"rank":372,"domain":"firefox.com","origin":"https://firefox.com","robots":{"status":"ok","http":200,"url":"https://www.firefox.com/robots.txt","bytes":78,"groups":1},"bots":{"GPTBot":["partial","*",1,0],"ChatGPT-User":["partial","*",1,0],"OAI-SearchBot":["partial","*",1,0],"ClaudeBot":["partial","*",1,0],"Claude-User":["partial","*",1,0],"Claude-SearchBot":["partial","*",1,0],"anthropic-ai":["partial","*",1,0],"PerplexityBot":["partial","*",1,0],"Perplexity-User":["partial","*",1,0],"Google-Extended":["partial","*",1,0],"Applebot-Extended":["partial","*",1,0],"CCBot":["partial","*",1,0],"Bytespider":["partial","*",1,0],"Amazonbot":["partial","*",1,0],"meta-externalagent":["partial","*",1,0],"cohere-ai":["partial","*",1,0],"Diffbot":["partial","*",1,0],"ContextFetchBot":["partial","*",1,0]},"llms":{"status":"missing","http":404,"bytes":8978},"home":{"status":"ok","http":200,"url":"https://www.firefox.com/en-US/","html_tokens":35421,"markdown_tokens":222,"pruned_tokens":84}},
{"rank":373,"domain":"bbb.org","origin":"https://bbb.org","robots":{"status":"ok","http":200,"url":"https://www.bbb.org/robots.txt","bytes":3004,"groups":3},"bots":{"GPTBot":["partial","named",20,2],"ChatGPT-User":["partial","named",20,2],"OAI-SearchBot":["partial","named",20,2],"ClaudeBot":["partial","named",20,2],"Claude-User":["partial","named",20,2],"Claude-SearchBot":["partial","named",20,2],"anthropic-ai":["partial","named",20,2],"PerplexityBot":["partial","named",20,2],"Perplexity-User":["partial","named",20,2],"Google-Extended":["partial","named",20,2],"Applebot-Extended":["partial","named",20,2],"CCBot":["partial","named",20,2],"Bytespider":["partial","*",20,2],"Amazonbot":["partial","*",20,2],"meta-externalagent":["partial","*",20,2],"cohere-ai":["partial","*",20,2],"Diffbot":["partial","*",20,2],"ContextFetchBot":["partial","*",20,2]},"llms":{"status":"present","http":200,"bytes":7690},"home":{"status":"ok","http":200,"url":"https://www.bbb.org/","html_tokens":37735,"markdown_tokens":46,"pruned_tokens":46}},
{"rank":374,"domain":"googleapis.com","origin":"https://googleapis.com","robots":{"status":"missing","http":404,"url":"https://googleapis.com/robots.txt","bytes":1571,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":1569},"home":{"status":"http_error","http":404,"url":"https://googleapis.com/"}},
{"rank":375,"domain":"upenn.edu","origin":"https://upenn.edu","robots":{"status":"refused","http":403,"url":"https://upenn.edu/robots.txt","bytes":560,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":376,"domain":"enable-javascript.com","origin":"https://enable-javascript.com","robots":{"status":"missing","http":404,"url":"https://www.enable-javascript.com/robots.txt","bytes":146,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":146},"home":{"status":"ok","http":200,"url":"https://www.enable-javascript.com/","html_tokens":6731,"markdown_tokens":347,"pruned_tokens":114}},
{"rank":377,"domain":"typeform.com","origin":"https://typeform.com","robots":{"status":"ok","http":200,"url":"https://www.typeform.com/robots.txt","bytes":136,"groups":1},"bots":{"GPTBot":["partial","*",3,0],"ChatGPT-User":["partial","*",3,0],"OAI-SearchBot":["partial","*",3,0],"ClaudeBot":["partial","*",3,0],"Claude-User":["partial","*",3,0],"Claude-SearchBot":["partial","*",3,0],"anthropic-ai":["partial","*",3,0],"PerplexityBot":["partial","*",3,0],"Perplexity-User":["partial","*",3,0],"Google-Extended":["partial","*",3,0],"Applebot-Extended":["partial","*",3,0],"CCBot":["partial","*",3,0],"Bytespider":["partial","*",3,0],"Amazonbot":["partial","*",3,0],"meta-externalagent":["partial","*",3,0],"cohere-ai":["partial","*",3,0],"Diffbot":["partial","*",3,0],"ContextFetchBot":["partial","*",3,0]},"llms":{"status":"present","http":200,"bytes":4720},"home":{"status":"ok","http":200,"url":"https://www.typeform.com/","html_tokens":79805,"markdown_tokens":234,"pruned_tokens":92}},
{"rank":378,"domain":"psu.edu","origin":"https://psu.edu","robots":{"status":"ok","http":200,"url":"https://www.psu.edu/robots.txt","bytes":24182,"groups":10},"bots":{"GPTBot":["allowed","named",0,1],"ChatGPT-User":["allowed","named",0,1],"OAI-SearchBot":["partial","*",428,1],"ClaudeBot":["allowed","named",0,1],"Claude-User":["partial","*",428,1],"Claude-SearchBot":["partial","*",428,1],"anthropic-ai":["allowed","named",0,1],"PerplexityBot":["allowed","named",0,1],"Perplexity-User":["partial","*",428,1],"Google-Extended":["allowed","named",0,1],"Applebot-Extended":["partial","*",428,1],"CCBot":["allowed","named",0,1],"Bytespider":["allowed","named",0,1],"Amazonbot":["allowed","named",0,1],"meta-externalagent":["partial","*",428,1],"cohere-ai":["partial","*",428,1],"Diffbot":["partial","*",428,1],"ContextFetchBot":["partial","*",428,1]},"llms":{"status":"missing","http":404,"bytes":64256},"home":{"status":"ok","http":200,"url":"https://www.psu.edu/","html_tokens":73328,"markdown_tokens":120,"pruned_tokens":120}},
{"rank":379,"domain":"es.wikipedia.org","origin":"https://es.wikipedia.org","robots":{"status":"ok","http":200,"url":"https://es.wikipedia.org/robots.txt","bytes":19741,"groups":33},"bots":{"GPTBot":["partial","*",258,4],"ChatGPT-User":["partial","*",258,4],"OAI-SearchBot":["partial","*",258,4],"ClaudeBot":["partial","*",258,4],"Claude-User":["partial","*",258,4],"Claude-SearchBot":["partial","*",258,4],"anthropic-ai":["partial","*",258,4],"PerplexityBot":["partial","*",258,4],"Perplexity-User":["partial","*",258,4],"Google-Extended":["partial","*",258,4],"Applebot-Extended":["partial","*",258,4],"CCBot":["partial","*",258,4],"Bytespider":["partial","*",258,4],"Amazonbot":["partial","*",258,4],"meta-externalagent":["partial","*",258,4],"cohere-ai":["partial","*",258,4],"Diffbot":["partial","*",258,4],"ContextFetchBot":["partial","*",258,4]},"llms":{"status":"missing","http":404,"bytes":1868},"home":{"status":"ok","http":200,"url":"https://es.wikipedia.org/wiki/Wikipedia:Portada","html_tokens":39196,"markdown_tokens":1370,"pruned_tokens":88}},
{"rank":380,"domain":"admin.ch","origin":"https://www.admin.ch","robots":{"status":"ok","http":200,"url":"https://www.admin.ch/robots.txt","bytes":65,"groups":1},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"missing","http":404,"bytes":96934},"home":{"status":"ok","http":200,"url":"https://www.admin.ch/de","html_tokens":33161,"markdown_tokens":334,"pruned_tokens":156}},
{"rank":381,"domain":"skype.com","origin":"https://skype.com","robots":{"status":"html","http":200,"url":"https://teams.live.com/free?source=skype","bytes":61987,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":382,"domain":"ovh.com","origin":"https://ovh.com","robots":{"status":"ok","http":200,"url":"https://www.ovh.com/robots.txt","bytes":6531,"groups":6},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["blocked","named",1,0],"OAI-SearchBot":["partial","*",154,7],"ClaudeBot":["partial","*",154,7],"Claude-User":["partial","*",154,7],"Claude-SearchBot":["partial","*",154,7],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["partial","*",154,7],"Perplexity-User":["partial","*",154,7],"Google-Extended":["partial","*",154,7],"Applebot-Extended":["partial","*",154,7],"CCBot":["blocked","named",1,0],"Bytespider":["partial","*",154,7],"Amazonbot":["partial","*",154,7],"meta-externalagent":["partial","*",154,7],"cohere-ai":["partial","*",154,7],"Diffbot":["partial","*",154,7],"ContextFetchBot":["partial","*",154,7]},"llms":{"status":"missing","http":404,"bytes":146},"home":{"status":"ok","http":200,"url":"https://www.ovhcloud.com/en/","html_tokens":260920,"markdown_tokens":487,"pruned_tokens":107}},
{"rank":383,"domain":"gartner.com","origin":"https://gartner.com","robots":{"status":"ok","http":200,"url":"https://www.gartner.com/robots.txt","bytes":4195,"groups":5},"bots":{"GPTBot":["partial","named",4,93],"ChatGPT-User":["partial","named",4,95],"OAI-SearchBot":["partial","named",4,95],"ClaudeBot":["partial","named",3,0],"Claude-User":["partial","*",23,0],"Claude-SearchBot":["partial","*",23,0],"anthropic-ai":["partial","*",23,0],"PerplexityBot":["partial","*",23,0],"Perplexity-User":["partial","*",23,0],"Google-Extended":["partial","named",7,95],"Applebot-Extended":["partial","named",3,0],"CCBot":["blocked","named",4,0],"Bytespider":["partial","named",3,0],"Amazonbot":["partial","*",23,0],"meta-externalagent":["partial","named",3,0],"cohere-ai":["partial","*",23,0],"Diffbot":["partial","named",3,0],"ContextFetchBot":["partial","*",23,0]},"llms":{"status":"missing","http":404,"bytes":8811},"home":{"status":"http_error","http":403,"url":"https://www.gartner.com/"}},
{"rank":384,"domain":"jimdo.com","origin":"https://jimdo.com","robots":{"status":"ok","http":200,"url":"https://www.jimdo.com/robots.txt","bytes":2319,"groups":2},"bots":{"GPTBot":["partial","named",7,1],"ChatGPT-User":["partial","named",7,1],"OAI-SearchBot":["partial","named",7,1],"ClaudeBot":["partial","named",7,1],"Claude-User":["partial","named",7,1],"Claude-SearchBot":["partial","named",7,1],"anthropic-ai":["partial","named",7,1],"PerplexityBot":["partial","named",7,1],"Perplexity-User":["partial","named",7,1],"Google-Extended":["partial","named",7,1],"Applebot-Extended":["partial","named",7,1],"CCBot":["partial","named",7,1],"Bytespider":["partial","named",7,1],"Amazonbot":["partial","named",7,1],"meta-externalagent":["partial","named",7,1],"cohere-ai":["partial","named",7,1],"Diffbot":["partial","named",7,1],"ContextFetchBot":["partial","*",23,20]},"llms":{"status":"present","http":200,"bytes":1891},"home":{"status":"ok","http":200,"url":"https://www.jimdo.com/","html_tokens":123247,"markdown_tokens":780,"pruned_tokens":431}},
{"rank":385,"domain":"pewresearch.org","origin":"https://pewresearch.org","robots":{"status":"ok","http":200,"url":"https://www.pewresearch.org/robots.txt","bytes":1056,"groups":2},"bots":{"GPTBot":["partial","*",2,1],"ChatGPT-User":["partial","*",2,1],"OAI-SearchBot":["partial","*",2,1],"ClaudeBot":["partial","*",2,1],"Claude-User":["partial","*",2,1],"Claude-SearchBot":["partial","*",2,1],"anthropic-ai":["partial","*",2,1],"PerplexityBot":["partial","*",2,1],"Perplexity-User":["partial","*",2,1],"Google-Extended":["partial","*",2,1],"Applebot-Extended":["partial","*",2,1],"CCBot":["partial","*",2,1],"Bytespider":["partial","*",2,1],"Amazonbot":["partial","*",2,1],"meta-externalagent":["partial","*",2,1],"cohere-ai":["partial","*",2,1],"Diffbot":["partial","*",2,1],"ContextFetchBot":["partial","*",2,1]},"llms":{"status":"present","http":200,"bytes":15263},"home":{"status":"ok","http":200,"url":"https://www.pewresearch.org/","html_tokens":107037,"markdown_tokens":892,"pruned_tokens":175}},
{"rank":386,"domain":"psychologytoday.com","origin":"https://psychologytoday.com","robots":{"status":"ok","http":200,"url":"https://www.psychologytoday.com/robots.txt","bytes":4915,"groups":10},"bots":{"GPTBot":["partial","*",147,32],"ChatGPT-User":["partial","*",147,32],"OAI-SearchBot":["partial","*",147,32],"ClaudeBot":["partial","*",147,32],"Claude-User":["partial","*",147,32],"Claude-SearchBot":["partial","*",147,32],"anthropic-ai":["partial","*",147,32],"PerplexityBot":["partial","*",147,32],"Perplexity-User":["partial","*",147,32],"Google-Extended":["partial","*",147,32],"Applebot-Extended":["partial","*",147,32],"CCBot":["partial","*",147,32],"Bytespider":["partial","*",147,32],"Amazonbot":["partial","*",147,32],"meta-externalagent":["partial","*",147,32],"cohere-ai":["partial","*",147,32],"Diffbot":["partial","*",147,32],"ContextFetchBot":["partial","*",147,32]},"llms":{"status":"http_error","http":403,"bytes":199},"home":{"status":"ok","http":200,"url":"https://www.psychologytoday.com/us","html_tokens":44153,"markdown_tokens":91,"pruned_tokens":90}},
{"rank":387,"domain":"cbc.ca","origin":"https://cbc.ca","robots":{"status":"unreachable","http":null,"url":"https://cbc.ca/robots.txt","bytes":0,"groups":0,"error":"<StreamReset stream_id:1, error_code:2, remote_reset:True>"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":388,"domain":"foxnews.com","origin":"https://foxnews.com","robots":{"status":"ok","http":200,"url":"https://www.foxnews.com/robots.txt","bytes":301,"groups":1},"bots":{"GPTBot":["partial","*",7,0],"ChatGPT-User":["partial","*",7,0],"OAI-SearchBot":["partial","*",7,0],"ClaudeBot":["partial","*",7,0],"Claude-User":["partial","*",7,0],"Claude-SearchBot":["partial","*",7,0],"anthropic-ai":["partial","*",7,0],"PerplexityBot":["partial","*",7,0],"Perplexity-User":["partial","*",7,0],"Google-Extended":["partial","*",7,0],"Applebot-Extended":["partial","*",7,0],"CCBot":["partial","*",7,0],"Bytespider":["partial","*",7,0],"Amazonbot":["partial","*",7,0],"meta-externalagent":["partial","*",7,0],"cohere-ai":["partial","*",7,0],"Diffbot":["partial","*",7,0],"ContextFetchBot":["partial","*",7,0]},"llms":{"status":"present","http":200,"bytes":2191},"home":{"status":"ok","http":200,"url":"https://www.foxnews.com/","html_tokens":202632,"markdown_tokens":11,"pruned_tokens":11}},
{"rank":389,"domain":"threads.net","origin":"https://threads.net","robots":{"status":"ok","http":200,"url":"https://www.threads.com/robots.txt","bytes":8228,"groups":36},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["blocked","*",1,0],"OAI-SearchBot":["blocked","*",1,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["blocked","*",1,0],"Claude-SearchBot":["blocked","*",1,0],"anthropic-ai":["blocked","*",1,0],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["blocked","*",1,0],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","*",1,0],"Bytespider":["blocked","*",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","*",1,0],"cohere-ai":["blocked","*",1,0],"Diffbot":["blocked","*",1,0],"ContextFetchBot":["blocked","*",1,0]},"llms":{"status":"disallowed"},"home":{"status":"disallowed"}},
{"rank":390,"domain":"fao.org","origin":"https://fao.org","robots":{"status":"ok","http":200,"url":"https://www.fao.org/robots.txt","bytes":1056,"groups":1},"bots":{"GPTBot":["partial","*",7,0],"ChatGPT-User":["partial","*",7,0],"OAI-SearchBot":["partial","*",7,0],"ClaudeBot":["partial","*",7,0],"Claude-User":["partial","*",7,0],"Claude-SearchBot":["partial","*",7,0],"anthropic-ai":["partial","*",7,0],"PerplexityBot":["partial","*",7,0],"Perplexity-User":["partial","*",7,0],"Google-Extended":["partial","*",7,0],"Applebot-Extended":["partial","*",7,0],"CCBot":["partial","*",7,0],"Bytespider":["partial","*",7,0],"Amazonbot":["partial","*",7,0],"meta-externalagent":["partial","*",7,0],"cohere-ai":["partial","*",7,0],"Diffbot":["partial","*",7,0],"ContextFetchBot":["partial","*",7,0]},"llms":{"status":"missing","http":404,"bytes":58233},"home":{"status":"ok","http":200,"url":"https://www.fao.org/home/en","html_tokens":296489,"markdown_tokens":1123,"pruned_tokens":759}},
{"rank":391,"domain":"zhihu.com","origin":"https://zhihu.com","robots":{"status":"refused","http":403,"url":"https://www.zhihu.com/","bytes":214,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":392,"domain":"wpastra.com","origin":"https://wpastra.com","robots":{"status":"ok","http":200,"url":"https://wpastra.com/robots.txt","bytes":386,"groups":2},"bots":{"GPTBot":["partial","*",7,1],"ChatGPT-User":["partial","*",7,1],"OAI-SearchBot":["partial","*",7,1],"ClaudeBot":["partial","*",7,1],"Claude-User":["partial","*",7,1],"Claude-SearchBot":["partial","*",7,1],"anthropic-ai":["partial","*",7,1],"PerplexityBot":["partial","*",7,1],"Perplexity-User":["partial","*",7,1],"Google-Extended":["partial","*",7,1],"Applebot-Extended":["partial","*",7,1],"CCBot":["partial","*",7,1],"Bytespider":["partial","*",7,1],"Amazonbot":["partial","*",7,1],"meta-externalagent":["partial","*",7,1],"cohere-ai":["partial","*",7,1],"Diffbot":["partial","*",7,1],"ContextFetchBot":["partial","*",7,1]},"llms":{"status":"present","http":200,"bytes":10040},"home":{"status":"ok","http":200,"url":"https://wpastra.com/","html_tokens":441621,"markdown_tokens":796,"pruned_tokens":119}},
{"rank":393,"domain":"deviantart.com","origin":"https://deviantart.com","robots":{"status":"ok","http":200,"url":"https://www.deviantart.com/robots.txt","bytes":524,"groups":2},"bots":{"GPTBot":["partial","*",11,3],"ChatGPT-User":["partial","*",11,3],"OAI-SearchBot":["partial","*",11,3],"ClaudeBot":["partial","*",11,3],"Claude-User":["partial","*",11,3],"Claude-SearchBot":["partial","*",11,3],"anthropic-ai":["partial","*",11,3],"PerplexityBot":["partial","*",11,3],"Perplexity-User":["partial","*",11,3],"Google-Extended":["partial","*",11,3],"Applebot-Extended":["partial","*",11,3],"CCBot":["partial","*",11,3],"Bytespider":["partial","*",11,3],"Amazonbot":["partial","*",11,3],"meta-externalagent":["partial","*",11,3],"cohere-ai":["partial","*",11,3],"Diffbot":["partial","*",11,3],"ContextFetchBot":["partial","*",11,3]},"llms":{"status":"missing","http":404,"bytes":27938},"home":{"status":"ok","http":200,"url":"https://www.deviantart.com/","html_tokens":298179,"markdown_tokens":544,"pruned_tokens":138}},
{"rank":394,"domain":"bund.de","origin":"https://bund.de","robots":{"status":"ok","http":200,"url":"https://verwaltung.bund.de/robots.txt","bytes":238,"groups":1},"bots":{"GPTBot":["partial","*",4,1],"ChatGPT-User":["partial","*",4,1],"OAI-SearchBot":["partial","*",4,1],"ClaudeBot":["partial","*",4,1],"Claude-User":["partial","*",4,1],"Claude-SearchBot":["partial","*",4,1],"anthropic-ai":["partial","*",4,1],"PerplexityBot":["partial","*",4,1],"Perplexity-User":["partial","*",4,1],"Google-Extended":["partial","*",4,1],"Applebot-Extended":["partial","*",4,1],"CCBot":["partial","*",4,1],"Bytespider":["partial","*",4,1],"Amazonbot":["partial","*",4,1],"meta-externalagent":["partial","*",4,1],"cohere-ai":["partial","*",4,1],"Diffbot":["partial","*",4,1],"ContextFetchBot":["partial","*",4,1]},"llms":{"status":"missing","http":404,"bytes":5064},"home":{"status":"ok","http":200,"url":"https://verwaltung.bund.de/portal/","html_tokens":676,"markdown_tokens":49,"pruned_tokens":49}},
{"rank":395,"domain":"jstor.org","origin":"https://jstor.org","robots":{"status":"ok","http":200,"url":"https://www.jstor.org/robots.txt","bytes":1605,"groups":3},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["partial","*",19,5],"OAI-SearchBot":["partial","*",19,5],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",19,5],"Claude-SearchBot":["partial","*",19,5],"anthropic-ai":["partial","*",19,5],"PerplexityBot":["partial","*",19,5],"Perplexity-User":["partial","*",19,5],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["partial","*",19,5],"CCBot":["blocked","named",1,0],"Bytespider":["partial","*",19,5],"Amazonbot":["partial","*",19,5],"meta-externalagent":["partial","*",19,5],"cohere-ai":["partial","*",19,5],"Diffbot":["partial","*",19,5],"ContextFetchBot":["partial","*",19,5]},"llms":{"status":"html","http":200,"bytes":3038},"home":{"status":"ok","http":200,"url":"https://www.jstor.org/","html_tokens":759,"markdown_tokens":56,"pruned_tokens":56}},
{"rank":396,"domain":"usnews.com","origin":"https://usnews.com","robots":{"status":"ok","http":200,"url":"https://www.usnews.com/robots.txt","bytes":6767,"groups":61},"bots":{"GPTBot":["partial","named",1,5],"ChatGPT-User":["partial","named",1,5],"OAI-SearchBot":["partial","named",1,5],"ClaudeBot":["partial","named",1,5],"Claude-User":["partial","*",2,0],"Claude-SearchBot":["partial","*",2,0],"anthropic-ai":["partial","named",1,5],"PerplexityBot":["partial","named",1,5],"Perplexity-User":["blocked","named",1,0],"Google-Extended":["partial","named",1,5],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",2,0]},"llms":{"status":"timeout","error":"timed out after 10 s"},"home":{"status":"timeout","error":"timed out after 15 s"}},
{"rank":397,"domain":"cam.ac.uk","origin":"https://cam.ac.uk","robots":{"status":"refused","http":403,"url":"https://cam.ac.uk/robots.txt","bytes":560,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":398,"domain":"moodle.org","origin":"https://moodle.org","robots":{"status":"ok","http":200,"url":"https://moodle.org/robots.txt","bytes":267,"groups":1},"bots":{"GPTBot":["partial","*",5,0],"ChatGPT-User":["partial","*",5,0],"OAI-SearchBot":["partial","*",5,0],"ClaudeBot":["partial","*",5,0],"Claude-User":["partial","*",5,0],"Claude-SearchBot":["partial","*",5,0],"anthropic-ai":["partial","*",5,0],"PerplexityBot":["partial","*",5,0],"Perplexity-User":["partial","*",5,0],"Google-Extended":["partial","*",5,0],"Applebot-Extended":["partial","*",5,0],"CCBot":["partial","*",5,0],"Bytespider":["partial","*",5,0],"Amazonbot":["partial","*",5,0],"meta-externalagent":["partial","*",5,0],"cohere-ai":["partial","*",5,0],"Diffbot":["partial","*",5,0],"ContextFetchBot":["partial","*",5,0]},"llms":{"status":"missing","http":404,"bytes":70596},"home":{"status":"http_error","http":403,"url":"https://moodle.org/"}},
{"rank":399,"domain":"geocities.com","origin":"http://geocities.com","robots":{"status":"refused","http":429,"url":"https://www.yahoo.com/","bytes":23,"groups":0,"error":"HTTP 429"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":400,"domain":"letsencrypt.org","origin":"https://letsencrypt.org","robots":{"status":"ok","http":200,"url":"https://letsencrypt.org/robots.txt","bytes":14,"groups":1},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"missing","http":404,"bytes":22808},"home":{"status":"ok","http":200,"url":"https://letsencrypt.org/","html_tokens":22564,"markdown_tokens":279,"pruned_tokens":112}},
{"rank":401,"domain":"uci.edu","origin":"https://uci.edu","robots":{"status":"missing","http":404,"url":"https://uci.edu/robots.txt","bytes":16,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":16},"home":{"status":"ok","http":200,"url":"https://uci.edu/","html_tokens":30751,"markdown_tokens":276,"pruned_tokens":73}},
{"rank":402,"domain":"chat.whatsapp.com","origin":"https://chat.whatsapp.com","robots":{"status":"ok","http":200,"url":"https://chat.whatsapp.com/robots.txt","bytes":160,"groups":1},"bots":{"GPTBot":["blocked","*",1,0],"ChatGPT-User":["blocked","*",1,0],"OAI-SearchBot":["blocked","*",1,0],"ClaudeBot":["blocked","*",1,0],"Claude-User":["blocked","*",1,0],"Claude-SearchBot":["blocked","*",1,0],"anthropic-ai":["blocked","*",1,0],"PerplexityBot":["blocked","*",1,0],"Perplexity-User":["blocked","*",1,0],"Google-Extended":["blocked","*",1,0],"Applebot-Extended":["blocked","*",1,0],"CCBot":["blocked","*",1,0],"Bytespider":["blocked","*",1,0],"Amazonbot":["blocked","*",1,0],"meta-externalagent":["blocked","*",1,0],"cohere-ai":["blocked","*",1,0],"Diffbot":["blocked","*",1,0],"ContextFetchBot":["blocked","*",1,0]},"llms":{"status":"disallowed"},"home":{"status":"disallowed"}},
{"rank":403,"domain":"zdnet.com","origin":"https://zdnet.com","robots":{"status":"ok","http":200,"url":"https://www.zdnet.com/robots.txt","bytes":6827,"groups":3},"bots":{"GPTBot":["partial","named",1,12],"ChatGPT-User":["partial","named",1,12],"OAI-SearchBot":["partial","named",1,12],"ClaudeBot":["partial","named",1,12],"Claude-User":["partial","named",1,12],"Claude-SearchBot":["partial","named",1,12],"anthropic-ai":["partial","named",1,12],"PerplexityBot":["partial","named",1,12],"Perplexity-User":["partial","named",1,12],"Google-Extended":["partial","*",15,0],"Applebot-Extended":["partial","named",1,12],"CCBot":["partial","named",1,12],"Bytespider":["partial","named",1,12],"Amazonbot":["partial","named",1,12],"meta-externalagent":["partial","named",1,12],"cohere-ai":["partial","named",1,12],"Diffbot":["partial","named",1,12],"ContextFetchBot":["partial","*",15,0]},"llms":{"status":"missing","http":404,"bytes":201280},"home":{"status":"ok","http":200,"url":"https://www.zdnet.com/","html_tokens":83044,"markdown_tokens":150,"pruned_tokens":150}},
{"rank":404,"domain":"akismet.com","origin":"https://akismet.com","robots":{"status":"ok","http":200,"url":"https://akismet.com/robots.txt","bytes":3279,"groups":2},"bots":{"GPTBot":["partial","*",92,18],"ChatGPT-User":["partial","*",92,18],"OAI-SearchBot":["partial","*",92,18],"ClaudeBot":["partial","*",92,18],"Claude-User":["partial","*",92,18],"Claude-SearchBot":["partial","*",92,18],"anthropic-ai":["partial","*",92,18],"PerplexityBot":["partial","*",92,18],"Perplexity-User":["partial","*",92,18],"Google-Extended":["partial","*",92,18],"Applebot-Extended":["partial","*",92,18],"CCBot":["partial","*",92,18],"Bytespider":["partial","*",92,18],"Amazonbot":["partial","*",92,18],"meta-externalagent":["partial","*",92,18],"cohere-ai":["partial","*",92,18],"Diffbot":["partial","*",92,18],"ContextFetchBot":["partial","*",92,18]},"llms":{"status":"present","http":200,"bytes":4434},"home":{"status":"ok","http":200,"url":"https://akismet.com/","html_tokens":34562,"markdown_tokens":274,"pruned_tokens":178}},
{"rank":405,"domain":"us.com","origin":"https://us.com","robots":{"status":"ok","http":200,"url":"https://us.com/robots.txt","bytes":0,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":7806},"home":{"status":"ok","http":200,"url":"https://us.com/","html_tokens":10522,"markdown_tokens":820,"pruned_tokens":806}},
{"rank":406,"domain":"abc.net.au","origin":"https://abc.net.au","robots":{"status":"ok","http":200,"url":"https://www.abc.net.au/robots.txt","bytes":2311,"groups":28},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["blocked","named",1,0],"OAI-SearchBot":["partial","*",24,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",24,0],"Claude-SearchBot":["partial","*",24,0],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["partial","*",24,0],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["partial","*",24,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["partial","*",24,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",24,0]},"llms":{"status":"missing","http":404,"bytes":25668},"home":{"status":"ok","http":200,"url":"https://www.abc.net.au/","html_tokens":211776,"markdown_tokens":80,"pruned_tokens":80}},
{"rank":407,"domain":"gofundme.com","origin":"https://gofundme.com","robots":{"status":"ok","http":200,"url":"https://www.gofundme.com/robots.txt","bytes":3979,"groups":5},"bots":{"GPTBot":["partial","*",32,0],"ChatGPT-User":["partial","*",32,0],"OAI-SearchBot":["partial","*",32,0],"ClaudeBot":["partial","*",32,0],"Claude-User":["partial","*",32,0],"Claude-SearchBot":["partial","*",32,0],"anthropic-ai":["partial","*",32,0],"PerplexityBot":["partial","*",32,0],"Perplexity-User":["partial","*",32,0],"Google-Extended":["partial","*",32,0],"Applebot-Extended":["partial","*",32,0],"CCBot":["partial","*",32,0],"Bytespider":["partial","*",32,0],"Amazonbot":["partial","*",32,0],"meta-externalagent":["partial","*",32,0],"cohere-ai":["partial","*",32,0],"Diffbot":["partial","*",32,0],"ContextFetchBot":["partial","*",32,0]},"llms":{"status":"missing","http":404,"bytes":569623},"home":{"status":"ok","http":200,"url":"https://www.gofundme.com/","html_tokens":145273,"markdown_tokens":319,"pruned_tokens":40}},
{"rank":408,"domain":"sciencedaily.com","origin":"https://sciencedaily.com","robots":{"status":"ok","http":200,"url":"https://sciencedaily.com/robots.txt","bytes":87,"groups":1},"bots":{"GPTBot":["partial","*",1,0],"ChatGPT-User":["partial","*",1,0],"OAI-SearchBot":["partial","*",1,0],"ClaudeBot":["partial","*",1,0],"Claude-User":["partial","*",1,0],"Claude-SearchBot":["partial","*",1,0],"anthropic-ai":["partial","*",1,0],"PerplexityBot":["partial","*",1,0],"Perplexity-User":["partial","*",1,0],"Google-Extended":["partial","*",1,0],"Applebot-Extended":["partial","*",1,0],"CCBot":["partial","*",1,0],"Bytespider":["partial","*",1,0],"Amazonbot":["partial","*",1,0],"meta-externalagent":["partial","*",1,0],"cohere-ai":["partial","*",1,0],"Diffbot":["partial","*",1,0],"ContextFetchBot":["partial","*",1,0]},"llms":{"status":"missing","http":404,"bytes":355},"home":{"status":"ok","http":200,"url":"https://sciencedaily.com/","html_tokens":38141,"markdown_tokens":1191,"pruned_tokens":104}},
{"rank":409,"domain":"coursera.org","origin":"https://coursera.org","robots":{"status":"ok","http":200,"url":"https://www.coursera.org/robots.txt","bytes":5345,"groups":6},"bots":{"GPTBot":["partial","named",26,22],"ChatGPT-User":["partial","named",25,22],"OAI-SearchBot":["partial","named",25,22],"ClaudeBot":["partial","named",26,22],"Claude-User":["partial","*",25,22],"Claude-SearchBot":["partial","*",25,22],"anthropic-ai":["partial","*",25,22],"PerplexityBot":["partial","named",26,22],"Perplexity-User":["partial","*",25,22],"Google-Extended":["partial","named",26,22],"Applebot-Extended":["partial","*",25,22],"CCBot":["partial","named",26,22],"Bytespider":["partial","*",25,22],"Amazonbot":["partial","*",25,22],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["partial","*",25,22],"Diffbot":["partial","*",25,22],"ContextFetchBot":["partial","*",25,22]},"llms":{"status":"present","http":200,"bytes":6773},"home":{"status":"ok","http":200,"url":"https://www.coursera.org/","html_tokens":196431,"markdown_tokens":127,"pruned_tokens":27}},
{"rank":410,"domain":"nypost.com","origin":"https://nypost.com","robots":{"status":"ok","http":200,"url":"https://nypost.com/robots.txt","bytes":7173,"groups":66},"bots":{"GPTBot":["partial","*",106,2],"ChatGPT-User":["partial","*",106,2],"OAI-SearchBot":["partial","*",106,2],"ClaudeBot":["blocked","named",1,0],"Claude-User":["blocked","named",1,0],"Claude-SearchBot":["blocked","named",1,0],"anthropic-ai":["blocked","named",2,0],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["partial","*",106,2],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",3,0],"cohere-ai":["blocked","named",2,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",106,2]},"llms":{"status":"missing","http":404,"bytes":468454},"home":{"status":"ok","http":200,"url":"https://nypost.com/","html_tokens":430361,"markdown_tokens":62,"pruned_tokens":62}},
{"rank":411,"domain":"books.google.com","origin":"https://books.google.com","robots":{"status":"ok","http":200,"url":"https://books.google.com/robots.txt","bytes":6624,"groups":4},"bots":{"GPTBot":["partial","*",169,64],"ChatGPT-User":["partial","*",169,64],"OAI-SearchBot":["partial","*",169,64],"ClaudeBot":["partial","*",169,64],"Claude-User":["partial","*",169,64],"Claude-SearchBot":["partial","*",169,64],"anthropic-ai":["partial","*",169,64],"PerplexityBot":["partial","*",169,64],"Perplexity-User":["partial","*",169,64],"Google-Extended":["partial","*",169,64],"Applebot-Extended":["partial","*",169,64],"CCBot":["partial","*",169,64],"Bytespider":["partial","*",169,64],"Amazonbot":["partial","*",169,64],"meta-externalagent":["partial","*",169,64],"cohere-ai":["partial","*",169,64],"Diffbot":["partial","*",169,64],"ContextFetchBot":["partial","*",169,64]},"llms":{"status":"missing","http":404,"bytes":1569},"home":{"status":"ok","http":200,"url":"https://books.google.com/","html_tokens":93287,"markdown_tokens":37,"pruned_tokens":37}},
{"rank":412,"domain":"umn.edu","origin":"https://www.umn.edu","robots":{"status":"ok","http":200,"url":"https://www.umn.edu/robots.txt","bytes":1797,"groups":3},"bots":{"GPTBot":["partial","*",23,18],"ChatGPT-User":["partial","*",23,18],"OAI-SearchBot":["partial","*",23,18],"ClaudeBot":["partial","*",23,18],"Claude-User":["partial","*",23,18],"Claude-SearchBot":["partial","*",23,18],"anthropic-ai":["partial","*",23,18],"PerplexityBot":["partial","*",23,18],"Perplexity-User":["partial","*",23,18],"Google-Extended":["partial","*",23,18],"Applebot-Extended":["partial","*",23,18],"CCBot":["partial","*",23,18],"Bytespider":["partial","*",23,18],"Amazonbot":["partial","*",23,18],"meta-externalagent":["partial","*",23,18],"cohere-ai":["partial","*",23,18],"Diffbot":["partial","*",23,18],"ContextFetchBot":["partial","*",23,18]},"llms":{"status":"missing","http":404,"bytes":185},"home":{"status":"ok","http":200,"url":"https://www.umn.edu/","html_tokens":14554,"markdown_tokens":715,"pruned_tokens":76}},
{"rank":413,"domain":"elsevier.com","origin":"https://elsevier.com","robots":{"status":"ok","http":200,"url":"https://www.elsevier.com/robots.txt","bytes":490,"groups":8},"bots":{"GPTBot":["allowed","*",0,1],"ChatGPT-User":["allowed","*",0,1],"OAI-SearchBot":["allowed","*",0,1],"ClaudeBot":["allowed","*",0,1],"Claude-User":["allowed","*",0,1],"Claude-SearchBot":["allowed","*",0,1],"anthropic-ai":["allowed","*",0,1],"PerplexityBot":["allowed","*",0,1],"Perplexity-User":["allowed","*",0,1],"Google-Extended":["allowed","*",0,1],"Applebot-Extended":["allowed","*",0,1],"CCBot":["allowed","*",0,1],"Bytespider":["allowed","*",0,1],"Amazonbot":["allowed","*",0,1],"meta-externalagent":["allowed","*",0,1],"cohere-ai":["allowed","*",0,1],"Diffbot":["allowed","*",0,1],"ContextFetchBot":["allowed","*",0,1]},"llms":{"status":"missing","http":404,"bytes":176889},"home":{"status":"ok","http":200,"url":"https://www.elsevier.com/","html_tokens":94478,"markdown_tokens":1021,"pruned_tokens":721}},
{"rank":414,"domain":"news.google.com","origin":"https://news.google.com","robots":{"status":"ok","http":200,"url":"https://news.google.com/robots.txt","bytes":515,"groups":3},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["blocked","named",1,0],"OAI-SearchBot":["partial","*",1,13],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",1,13],"Claude-SearchBot":["partial","*",1,13],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["partial","*",1,13],"Google-Extended":["partial","*",1,13],"Applebot-Extended":["partial","*",1,13],"CCBot":["blocked","named",1,0],"Bytespider":["partial","*",1,13],"Amazonbot":["partial","*",1,13],"meta-externalagent":["partial","*",1,13],"cohere-ai":["partial","*",1,13],"Diffbot":["partial","*",1,13],"ContextFetchBot":["partial","*",1,13]},"llms":{"status":"disallowed"},"home":{"status":"ok","http":200,"url":"https://news.google.com/home?hl=en-US&gl=US&ceid=US:en","html_tokens":464484,"markdown_tokens":109,"pruned_tokens":109}},
{"rank":415,"domain":"acm.org","origin":"https://acm.org","robots":{"status":"ok","http":200,"url":"https://www.acm.org/robots.txt","bytes":639,"groups":2},"bots":{"GPTBot":["partial","*",18,0],"ChatGPT-User":["partial","*",18,0],"OAI-SearchBot":["partial","*",18,0],"ClaudeBot":["partial","*",18,0],"Claude-User":["partial","*",18,0],"Claude-SearchBot":["partial","*",18,0],"anthropic-ai":["partial","*",18,0],"PerplexityBot":["partial","*",18,0],"Perplexity-User":["partial","*",18,0],"Google-Extended":["partial","*",18,0],"Applebot-Extended":["partial","*",18,0],"CCBot":["partial","*",18,0],"Bytespider":["partial","*",18,0],"Amazonbot":["partial","*",18,0],"meta-externalagent":["partial","*",18,0],"cohere-ai":["partial","*",18,0],"Diffbot":["partial","*",18,0],"ContextFetchBot":["partial","*",18,0]},"llms":{"status":"missing","http":404,"bytes":64931},"home":{"status":"http_error","http":403,"url":"https://acm.org/"}},
{"rank":416,"domain":"xinhuanet.com","origin":"https://xinhuanet.com","robots":{"status":"ok","http":200,"url":"https://www.xinhuanet.com/robots.txt","bytes":74,"groups":1},"bots":{"GPTBot":["allowed","*",0,1],"ChatGPT-User":["allowed","*",0,1],"OAI-SearchBot":["allowed","*",0,1],"ClaudeBot":["allowed","*",0,1],"Claude-User":["allowed","*",0,1],"Claude-SearchBot":["allowed","*",0,1],"anthropic-ai":["allowed","*",0,1],"PerplexityBot":["allowed","*",0,1],"Perplexity-User":["allowed","*",0,1],"Google-Extended":["allowed","*",0,1],"Applebot-Extended":["allowed","*",0,1],"CCBot":["allowed","*",0,1],"Bytespider":["allowed","*",0,1],"Amazonbot":["allowed","*",0,1],"meta-externalagent":["allowed","*",0,1],"cohere-ai":["allowed","*",0,1],"Diffbot":["allowed","*",0,1],"ContextFetchBot":["allowed","*",0,1]},"llms":{"status":"missing","http":404,"bytes":4655},"home":{"status":"ok","http":200,"url":"https://www.xinhuanet.com/","html_tokens":37880,"markdown_tokens":81,"pruned_tokens":69}},
{"rank":417,"domain":"elegantthemes.com","origin":"https://elegantthemes.com","robots":{"status":"ok","http":200,"url":"https://www.elegantthemes.com/robots.txt","bytes":150,"groups":1},"bots":{"GPTBot":["partial","*",5,0],"ChatGPT-User":["partial","*",5,0],"OAI-SearchBot":["partial","*",5,0],"ClaudeBot":["partial","*",5,0],"Claude-User":["partial","*",5,0],"Claude-SearchBot":["partial","*",5,0],"anthropic-ai":["partial","*",5,0],"PerplexityBot":["partial","*",5,0],"Perplexity-User":["partial","*",5,0],"Google-Extended":["partial","*",5,0],"Applebot-Extended":["partial","*",5,0],"CCBot":["partial","*",5,0],"Bytespider":["partial","*",5,0],"Amazonbot":["partial","*",5,0],"meta-externalagent":["partial","*",5,0],"cohere-ai":["partial","*",5,0],"Diffbot":["partial","*",5,0],"ContextFetchBot":["partial","*",5,0]},"llms":{"status":"missing","http":404,"bytes":227110},"home":{"status":"ok","http":200,"url":"https://www.elegantthemes.com/","html_tokens":67562,"markdown_tokens":861,"pruned_tokens":39}},
{"rank":418,"domain":"dailymail.com","origin":"https://dailymail.com","robots":{"status":"ok","http":200,"url":"https://www.dailymail.com/robots.txt","bytes":5435,"groups":7},"bots":{"GPTBot":["partial","named",1,1],"ChatGPT-User":["partial","named",1,1],"OAI-SearchBot":["partial","named",1,1],"ClaudeBot":["partial","named",1,1],"Claude-User":["partial","named",1,1],"Claude-SearchBot":["partial","named",1,1],"anthropic-ai":["partial","named",1,1],"PerplexityBot":["partial","named",1,1],"Perplexity-User":["partial","named",1,1],"Google-Extended":["partial","named",1,1],"Applebot-Extended":["partial","named",1,1],"CCBot":["partial","named",1,1],"Bytespider":["partial","named",1,1],"Amazonbot":["partial","named",1,1],"meta-externalagent":["partial","named",1,1],"cohere-ai":["partial","named",1,1],"Diffbot":["partial","named",1,1],"ContextFetchBot":["partial","*",21,0]},"llms":{"status":"html","http":200,"bytes":1406681},"home":{"status":"ok","http":200,"url":"https://www.dailymail.com/ushome/index.html","html_tokens":241833,"markdown_tokens":1486,"pruned_tokens":122}},
{"rank":419,"domain":"newyorker.com","origin":"https://newyorker.com","robots":{"status":"ok","http":200,"url":"https://www.newyorker.com/robots.txt","bytes":2209,"groups":3},"bots":{"GPTBot":["partial","*",15,15],"ChatGPT-User":["partial","*",15,15],"OAI-SearchBot":["partial","*",15,15],"ClaudeBot":["blocked","named",1,0],"Claude-User":["blocked","named",1,0],"Claude-SearchBot":["blocked","named",1,0],"anthropic-ai":["partial","*",15,15],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["blocked","named",1,0],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",15,15]},"llms":{"status":"missing","http":404,"bytes":537520},"home":{"status":"ok","http":200,"url":"https://www.newyorker.com/","html_tokens":604323,"markdown_tokens":1677,"pruned_tokens":133}},
{"rank":420,"domain":"windows.net","origin":"http://windows.net","robots":{"status":"html","http":200,"url":"https://dotnet.microsoft.com/en-us/","bytes":254371,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":421,"domain":"gob.mx","origin":"https://www.gob.mx","robots":{"status":"ok","http":200,"url":"https://www.gob.mx/robots.txt","bytes":150,"groups":4},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":2156},"home":{"status":"ok","http":200,"url":"https://www.gob.mx/","html_tokens":24806,"markdown_tokens":321,"pruned_tokens":289}},
{"rank":422,"domain":"vice.com","origin":"https://vice.com","robots":{"status":"ok","http":200,"url":"https://www.vice.com/robots.txt","bytes":1773,"groups":1},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"missing","http":404,"bytes":271617},"home":{"status":"ok","http":200,"url":"https://www.vice.com/","html_tokens":75040,"markdown_tokens":14,"pruned_tokens":14}},
{"rank":423,"domain":"newsweek.com","origin":"https://newsweek.com","robots":{"status":"ok","http":200,"url":"https://www.newsweek.com/robots.txt","bytes":3851,"groups":26},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["blocked","named",1,0],"OAI-SearchBot":["partial","*",6,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",6,0],"Claude-SearchBot":["partial","*",6,0],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["partial","*",6,0],"Google-Extended":["partial","*",6,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["partial","*",6,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",6,0]},"llms":{"status":"missing","http":404,"bytes":23774},"home":{"status":"ok","http":200,"url":"https://www.newsweek.com/","html_tokens":314549,"markdown_tokens":788,"pruned_tokens":120}},
{"rank":424,"domain":"fb.watch","origin":"https://fb.watch","robots":{"status":"ok","http":200,"url":"https://fb.watch/robots.txt","bytes":308,"groups":1},"bots":{"GPTBot":["allowed","*",0,1],"ChatGPT-User":["allowed","*",0,1],"OAI-SearchBot":["allowed","*",0,1],"ClaudeBot":["allowed","*",0,1],"Claude-User":["allowed","*",0,1],"Claude-SearchBot":["allowed","*",0,1],"anthropic-ai":["allowed","*",0,1],"PerplexityBot":["allowed","*",0,1],"Perplexity-User":["allowed","*",0,1],"Google-Extended":["allowed","*",0,1],"Applebot-Extended":["allowed","*",0,1],"CCBot":["allowed","*",0,1],"Bytespider":["allowed","*",0,1],"Amazonbot":["allowed","*",0,1],"meta-externalagent":["allowed","*",0,1],"cohere-ai":["allowed","*",0,1],"Diffbot":["allowed","*",0,1],"ContextFetchBot":["allowed","*",0,1]},"llms":{"status":"disallowed","error":"robots.txt of www.facebook.com disallows /watch/"},"home":{"status":"disallowed","error":"robots.txt of www.facebook.com disallows /watch/"}},
{"rank":425,"domain":"thelancet.com","origin":"https://thelancet.com","robots":{"status":"ok","http":200,"url":"https://www.thelancet.com/robots.txt","bytes":1834,"groups":22},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["blocked","named",1,0],"OAI-SearchBot":["partial","*",19,6],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",19,6],"Claude-SearchBot":["partial","*",19,6],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["partial","*",19,6],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["partial","*",19,6],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",19,6]},"llms":{"status":"http_error","http":403,"bytes":5545},"home":{"status":"http_error","http":403,"url":"https://thelancet.com/"}},
{"rank":426,"domain":"music.apple.com","origin":"https://music.apple.com","robots":{"status":"ok","http":200,"url":"https://music.apple.com/robots.txt","bytes":850,"groups":1},"bots":{"GPTBot":["partial","*",5,1],"ChatGPT-User":["partial","*",5,1],"OAI-SearchBot":["partial","*",5,1],"ClaudeBot":["partial","*",5,1],"Claude-User":["partial","*",5,1],"Claude-SearchBot":["partial","*",5,1],"anthropic-ai":["partial","*",5,1],"PerplexityBot":["partial","*",5,1],"Perplexity-User":["partial","*",5,1],"Google-Extended":["partial","*",5,1],"Applebot-Extended":["partial","*",5,1],"CCBot":["partial","*",5,1],"Bytespider":["partial","*",5,1],"Amazonbot":["partial","*",5,1],"meta-externalagent":["partial","*",5,1],"cohere-ai":["partial","*",5,1],"Diffbot":["partial","*",5,1],"ContextFetchBot":["partial","*",5,1]},"llms":{"status":"present","http":200,"bytes":11583},"home":{"status":"ok","http":200,"url":"https://music.apple.com/us/new","html_tokens":586497,"markdown_tokens":138,"pruned_tokens":138}},
{"rank":427,"domain":"scientificamerican.com","origin":"https://scientificamerican.com","robots":{"status":"ok","http":200,"url":"https://www.scientificamerican.com/robots.txt","bytes":642,"groups":3},"bots":{"GPTBot":["partial","named",1,2],"ChatGPT-User":["partial","named",1,2],"OAI-SearchBot":["partial","named",1,2],"ClaudeBot":["partial","named",1,2],"Claude-User":["partial","named",1,2],"Claude-SearchBot":["partial","named",1,2],"anthropic-ai":["partial","named",1,2],"PerplexityBot":["partial","named",1,2],"Perplexity-User":["partial","*",5,0],"Google-Extended":["partial","named",1,2],"Applebot-Extended":["partial","*",5,0],"CCBot":["partial","named",1,2],"Bytespider":["partial","*",5,0],"Amazonbot":["partial","*",5,0],"meta-externalagent":["partial","*",5,0],"cohere-ai":["partial","named",1,2],"Diffbot":["partial","*",5,0],"ContextFetchBot":["partial","*",5,0]},"llms":{"status":"missing","http":404,"bytes":89542},"home":{"status":"ok","http":200,"url":"https://www.scientificamerican.com/","html_tokens":88652,"markdown_tokens":1051,"pruned_tokens":43}},
{"rank":428,"domain":"kernel.org","origin":"https://kernel.org","robots":{"status":"missing","http":404,"url":"https://kernel.org/robots.txt","bytes":146,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":146},"home":{"status":"ok","http":200,"url":"https://kernel.org/","html_tokens":4788,"markdown_tokens":11,"pruned_tokens":11}},
{"rank":429,"domain":"arstechnica.com","origin":"https://arstechnica.com","robots":{"status":"ok","http":200,"url":"https://arstechnica.com/robots.txt","bytes":1912,"groups":5},"bots":{"GPTBot":["partial","*",30,0],"ChatGPT-User":["partial","*",30,0],"OAI-SearchBot":["partial","*",30,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["blocked","named",1,0],"Claude-SearchBot":["blocked","named",1,0],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["blocked","named",1,0],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["partial","named",1,1],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",30,0]},"llms":{"status":"missing","http":404,"bytes":5565},"home":{"status":"ok","http":200,"url":"https://arstechnica.com/","html_tokens":87557,"markdown_tokens":53,"pruned_tokens":53}},
{"rank":430,"domain":"irs.gov","origin":"https://irs.gov","robots":{"status":"ok","http":200,"url":"https://www.irs.gov/robots.txt","bytes":3899,"groups":1},"bots":{"GPTBot":["partial","*",82,18],"ChatGPT-User":["partial","*",82,18],"OAI-SearchBot":["partial","*",82,18],"ClaudeBot":["partial","*",82,18],"Claude-User":["partial","*",82,18],"Claude-SearchBot":["partial","*",82,18],"anthropic-ai":["partial","*",82,18],"PerplexityBot":["partial","*",82,18],"Perplexity-User":["partial","*",82,18],"Google-Extended":["partial","*",82,18],"Applebot-Extended":["partial","*",82,18],"CCBot":["partial","*",82,18],"Bytespider":["partial","*",82,18],"Amazonbot":["partial","*",82,18],"meta-externalagent":["partial","*",82,18],"cohere-ai":["partial","*",82,18],"Diffbot":["partial","*",82,18],"ContextFetchBot":["partial","*",82,18]},"llms":{"status":"missing","http":404,"bytes":86144},"home":{"status":"ok","http":200,"url":"https://www.irs.gov/","html_tokens":36827,"markdown_tokens":434,"pruned_tokens":105}},
{"rank":431,"domain":"oreilly.com","origin":"https://oreilly.com","robots":{"status":"ok","http":200,"url":"https://www.oreilly.com/robots.txt","bytes":3450,"groups":7},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["partial","named",1,1],"OAI-SearchBot":["partial","named",1,1],"ClaudeBot":["partial","named",1,1],"Claude-User":["partial","*",19,0],"Claude-SearchBot":["partial","named",1,1],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["partial","named",1,1],"Perplexity-User":["partial","named",1,1],"Google-Extended":["partial","named",1,1],"Applebot-Extended":["partial","named",1,1],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",19,0]},"llms":{"status":"present","http":200,"bytes":18762},"home":{"status":"ok","http":200,"url":"https://www.oreilly.com/","html_tokens":79011,"markdown_tokens":458,"pruned_tokens":148}},
{"rank":432,"domain":"dw.com","origin":"https://dw.com","robots":{"status":"ok","http":200,"url":"https://www.dw.com/robots.txt","bytes":2757,"groups":51},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["blocked","named",1,0],"OAI-SearchBot":["blocked","named",1,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",18,0],"Claude-SearchBot":["partial","*",18,0],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["partial","*",18,0],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",18,0]},"llms":{"status":"disallowed","error":"robots.txt of go.dw.com disallows /llms.txt"},"home":{"status":"ok","http":200,"url":"https://www.dw.com/","html_tokens":3047,"markdown_tokens":12,"pruned_tokens":12}},
{"rank":433,"domain":"dell.com","origin":"https://dell.com","robots":{"status":"ok","http":200,"url":"https://www.dell.com/robots.txt","bytes":28897,"groups":9},"bots":{"GPTBot":["partial","*",411,551],"ChatGPT-User":["partial","*",411,551],"OAI-SearchBot":["partial","*",411,551],"ClaudeBot":["partial","*",411,551],"Claude-User":["partial","*",411,551],"Claude-SearchBot":["partial","*",411,551],"anthropic-ai":["partial","*",411,551],"PerplexityBot":["partial","*",411,551],"Perplexity-User":["partial","*",411,551],"Google-Extended":["partial","*",411,551],"Applebot-Extended":["partial","*",411,551],"CCBot":["partial","*",411,551],"Bytespider":["partial","named",13,0],"Amazonbot":["partial","*",411,551],"meta-externalagent":["partial","*",411,551],"cohere-ai":["partial","*",411,551],"Diffbot":["partial","*",411,551],"ContextFetchBot":["partial","*",411,551]},"llms":{"status":"present","http":200,"bytes":90245},"home":{"status":"ok","http":200,"url":"https://www.dell.com/en-us","html_tokens":86736,"markdown_tokens":323,"pruned_tokens":90}},
{"rank":434,"domain":"nps.gov","origin":"https://nps.gov","robots":{"status":"ok","http":200,"url":"https://www.nps.gov/robots.txt","bytes":178,"groups":2},"bots":{"GPTBot":["partial","*",4,0],"ChatGPT-User":["partial","*",4,0],"OAI-SearchBot":["partial","*",4,0],"ClaudeBot":["partial","*",4,0],"Claude-User":["partial","*",4,0],"Claude-SearchBot":["partial","*",4,0],"anthropic-ai":["partial","*",4,0],"PerplexityBot":["partial","*",4,0],"Perplexity-User":["partial","*",4,0],"Google-Extended":["partial","*",4,0],"Applebot-Extended":["partial","*",4,0],"CCBot":["partial","*",4,0],"Bytespider":["partial","*",4,0],"Amazonbot":["partial","*",4,0],"meta-externalagent":["partial","*",4,0],"cohere-ai":["partial","*",4,0],"Diffbot":["partial","*",4,0],"ContextFetchBot":["partial","*",4,0]},"llms":{"status":"missing","http":404,"bytes":18909},"home":{"status":"ok","http":200,"url":"https://www.nps.gov/index.htm","html_tokens":18013,"markdown_tokens":89,"pruned_tokens":70}},
{"rank":435,"domain":"orcid.org","origin":"https://orcid.org","robots":{"status":"ok","http":200,"url":"https://orcid.org/robots.txt","bytes":550,"groups":1},"bots":{"GPTBot":["partial","*",22,0],"ChatGPT-User":["partial","*",22,0],"OAI-SearchBot":["partial","*",22,0],"ClaudeBot":["partial","*",22,0],"Claude-User":["partial","*",22,0],"Claude-SearchBot":["partial","*",22,0],"anthropic-ai":["partial","*",22,0],"PerplexityBot":["partial","*",22,0],"Perplexity-User":["partial","*",22,0],"Google-Extended":["partial","*",22,0],"Applebot-Extended":["partial","*",22,0],"CCBot":["partial","*",22,0],"Bytespider":["partial","*",22,0],"Amazonbot":["partial","*",22,0],"meta-externalagent":["partial","*",22,0],"cohere-ai":["partial","*",22,0],"Diffbot":["partial","*",22,0],"ContextFetchBot":["partial","*",22,0]},"llms":{"status":"html","http":200,"bytes":67250},"home":{"status":"ok","http":200,"url":"https://orcid.org/","html_tokens":16813,"markdown_tokens":15,"pruned_tokens":15}},
{"rank":436,"domain":"threads.com","origin":"https://threads.com","robots":{"status":"ok","http":200,"url":"https://www.threads.com/robots.txt","bytes":8228,"groups":36},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["blocked","*",1,0],"OAI-SearchBot":["blocked","*",1,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["blocked","*",1,0],"Claude-SearchBot":["blocked","*",1,0],"anthropic-ai":["blocked","*",1,0],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["blocked","*",1,0],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","*",1,0],"Bytespider":["blocked","*",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","*",1,0],"cohere-ai":["blocked","*",1,0],"Diffbot":["blocked","*",1,0],"ContextFetchBot":["blocked","*",1,0]},"llms":{"status":"disallowed"},"home":{"status":"disallowed"}},
{"rank":437,"domain":"themeisle.com","origin":"https://themeisle.com","robots":{"status":"ok","http":200,"url":"https://themeisle.com/robots.txt","bytes":171,"groups":1},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"missing","http":404,"bytes":153463},"home":{"status":"ok","http":200,"url":"https://themeisle.com/","html_tokens":63888,"markdown_tokens":471,"pruned_tokens":42}},
{"rank":438,"domain":"uchicago.edu","origin":"https://uchicago.edu","robots":{"status":"refused","http":403,"url":"https://www.uchicago.edu/robots.txt","bytes":17594,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":439,"domain":"jigsaw.w3.org","origin":"https://jigsaw.w3.org","robots":{"status":"ok","http":200,"url":"https://jigsaw.w3.org/robots.txt","bytes":403,"groups":1},"bots":{"GPTBot":["partial","*",11,0],"ChatGPT-User":["partial","*",11,0],"OAI-SearchBot":["partial","*",11,0],"ClaudeBot":["partial","*",11,0],"Claude-User":["partial","*",11,0],"Claude-SearchBot":["partial","*",11,0],"anthropic-ai":["partial","*",11,0],"PerplexityBot":["partial","*",11,0],"Perplexity-User":["partial","*",11,0],"Google-Extended":["partial","*",11,0],"Applebot-Extended":["partial","*",11,0],"CCBot":["partial","*",11,0],"Bytespider":["partial","*",11,0],"Amazonbot":["partial","*",11,0],"meta-externalagent":["partial","*",11,0],"cohere-ai":["partial","*",11,0],"Diffbot":["partial","*",11,0],"ContextFetchBot":["partial","*",11,0]},"llms":{"status":"missing","http":404,"bytes":147},"home":{"status":"ok","http":200,"url":"https://jigsaw.w3.org/","html_tokens":1294,"markdown_tokens":239,"pruned_tokens":116}},
{"rank":440,"domain":"jamanetwork.com","origin":"https://jamanetwork.com","robots":{"status":"ok","http":200,"url":"https://jamanetwork.com/robots.txt","bytes":3431,"groups":40},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["partial","*",27,0],"OAI-SearchBot":["partial","*",27,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",27,0],"Claude-SearchBot":["partial","*",27,0],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["partial","*",27,0],"Perplexity-User":["partial","*",27,0],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["partial","*",27,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["partial","*",27,0],"ContextFetchBot":["partial","*",27,0]},"llms":{"status":"http_error","http":403,"bytes":5526},"home":{"status":"http_error","http":403,"url":"https://jamanetwork.com/"}},
{"rank":441,"domain":"git-scm.com","origin":"https://git-scm.com","robots":{"status":"missing","http":404,"url":"https://git-scm.com/robots.txt","bytes":6697,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":6697},"home":{"status":"ok","http":200,"url":"https://git-scm.com/","html_tokens":1536,"markdown_tokens":64,"pruned_tokens":64}},
{"rank":442,"domain":"lnkd.in","origin":"https://lnkd.in","robots":{"status":"ok","http":200,"url":"https://www.linkedin.com/robots.txt","bytes":120190,"groups":77},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["blocked","named",1,0],"OAI-SearchBot":["partial","named",111,9],"ClaudeBot":["blocked","named",1,0],"Claude-User":["blocked","named",1,0],"Claude-SearchBot":["partial","named",111,9],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["blocked","named",1,0],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","*",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","*",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["blocked","*",1,0]},"llms":{"status":"disallowed"},"home":{"status":"disallowed"}},
{"rank":443,"domain":"wa.link","origin":"https://wa.link","robots":{"status":"ok","http":200,"url":"https://wa.link/robots.txt","bytes":23,"groups":1},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"missing","http":404,"bytes":7338},"home":{"status":"ok","http":200,"url":"https://create.wa.link","html_tokens":25381,"markdown_tokens":1197,"pruned_tokens":349}},
{"rank":444,"domain":"m.me","origin":"https://m.me","robots":{"status":"html","http":200,"url":"https://www.messenger.com/","bytes":109878,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":445,"domain":"stackexchange.com","origin":"https://stackexchange.com","robots":{"status":"refused","http":403,"url":"https://stackexchange.com/robots.txt","bytes":5512,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":446,"domain":"cmu.edu","origin":"https://cmu.edu","robots":{"status":"ok","http":200,"url":"https://www.cmu.edu/robots.txt","bytes":110,"groups":1},"bots":{"GPTBot":["partial","*",3,0],"ChatGPT-User":["partial","*",3,0],"OAI-SearchBot":["partial","*",3,0],"ClaudeBot":["partial","*",3,0],"Claude-User":["partial","*",3,0],"Claude-SearchBot":["partial","*",3,0],"anthropic-ai":["partial","*",3,0],"PerplexityBot":["partial","*",3,0],"Perplexity-User":["partial","*",3,0],"Google-Extended":["partial","*",3,0],"Applebot-Extended":["partial","*",3,0],"CCBot":["partial","*",3,0],"Bytespider":["partial","*",3,0],"Amazonbot":["partial","*",3,0],"meta-externalagent":["partial","*",3,0],"cohere-ai":["partial","*",3,0],"Diffbot":["partial","*",3,0],"ContextFetchBot":["partial","*",3,0]},"llms":{"status":"missing","http":404,"bytes":2625},"home":{"status":"ok","http":200,"url":"https://www.cmu.edu/","html_tokens":25883,"markdown_tokens":668,"pruned_tokens":256}},
{"rank":447,"domain":"go.kr","origin":"https://go.kr","robots":{"status":"unreachable","http":null,"url":"https://go.kr/robots.txt","bytes":0,"groups":0,"error":"cannot resolve go.kr"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":448,"domain":"zoom.com","origin":"https://zoom.com","robots":{"status":"ok","http":200,"url":"https://zoom.us/robots.txt","bytes":2005,"groups":13},"bots":{"GPTBot":["partial","*",28,0],"ChatGPT-User":["partial","*",28,0],"OAI-SearchBot":["partial","*",28,0],"ClaudeBot":["partial","*",28,0],"Claude-User":["partial","*",28,0],"Claude-SearchBot":["partial","*",28,0],"anthropic-ai":["partial","*",28,0],"PerplexityBot":["partial","*",28,0],"Perplexity-User":["partial","*",28,0],"Google-Extended":["partial","*",28,0],"Applebot-Extended":["partial","*",28,0],"CCBot":["partial","*",28,0],"Bytespider":["partial","*",28,0],"Amazonbot":["partial","*",28,0],"meta-externalagent":["partial","*",28,0],"cohere-ai":["partial","*",28,0],"Diffbot":["partial","*",28,0],"ContextFetchBot":["partial","*",28,0]},"llms":{"status":"missing","http":404,"bytes":526015},"home":{"status":"ok","http":200,"url":"https://www.zoom.com","html_tokens":79701,"markdown_tokens":938,"pruned_tokens":459}},
{"rank":449,"domain":"magnific.com","origin":"https://magnific.com","robots":{"status":"ok","http":200,"url":"https://www.magnific.com/robots.txt","bytes":5416,"groups":4},"bots":{"GPTBot":["partial","*",111,0],"ChatGPT-User":["partial","*",111,0],"OAI-SearchBot":["partial","*",111,0],"ClaudeBot":["partial","*",111,0],"Claude-User":["partial","*",111,0],"Claude-SearchBot":["partial","*",111,0],"anthropic-ai":["partial","*",111,0],"PerplexityBot":["partial","*",111,0],"Perplexity-User":["partial","*",111,0],"Google-Extended":["partial","*",111,0],"Applebot-Extended":["partial","*",111,0],"CCBot":["partial","*",111,0],"Bytespider":["partial","*",111,0],"Amazonbot":["partial","*",111,0],"meta-externalagent":["partial","*",111,0],"cohere-ai":["partial","*",111,0],"Diffbot":["partial","*",111,0],"ContextFetchBot":["partial","*",111,0]},"llms":{"status":"http_error","http":403,"bytes":7530},"home":{"status":"http_error","http":403,"url":"https://magnific.com/"}},
{"rank":450,"domain":"mitre.org","origin":"https://mitre.org","robots":{"status":"refused","http":403,"url":"https://www.mitre.org/robots.txt","bytes":381,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":451,"domain":"fastcompany.com","origin":"https://fastcompany.com","robots":{"status":"ok","http":200,"url":"https://www.fastcompany.com/robots.txt","bytes":2075,"groups":38},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["partial","named",1,1],"OAI-SearchBot":["partial","named",1,1],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","named",1,1],"Claude-SearchBot":["partial","named",1,1],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["partial","named",1,1],"Google-Extended":["partial","named",1,1],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",2,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["partial","named",1,1],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",1,1]},"llms":{"status":"http_error","http":403,"bytes":775},"home":{"status":"http_error","http":403,"url":"https://www.fastcompany.com/"}},
{"rank":452,"domain":"naver.com","origin":"https://naver.com","robots":{"status":"ok","http":200,"url":"https://www.naver.com/robots.txt","bytes":91,"groups":1},"bots":{"GPTBot":["partial","*",1,2],"ChatGPT-User":["partial","*",1,2],"OAI-SearchBot":["partial","*",1,2],"ClaudeBot":["partial","*",1,2],"Claude-User":["partial","*",1,2],"Claude-SearchBot":["partial","*",1,2],"anthropic-ai":["partial","*",1,2],"PerplexityBot":["partial","*",1,2],"Perplexity-User":["partial","*",1,2],"Google-Extended":["partial","*",1,2],"Applebot-Extended":["partial","*",1,2],"CCBot":["partial","*",1,2],"Bytespider":["partial","*",1,2],"Amazonbot":["partial","*",1,2],"meta-externalagent":["partial","*",1,2],"cohere-ai":["partial","*",1,2],"Diffbot":["partial","*",1,2],"ContextFetchBot":["partial","*",1,2]},"llms":{"status":"disallowed"},"home":{"status":"ok","http":200,"url":"https://www.naver.com/","html_tokens":55626,"markdown_tokens":27,"pruned_tokens":27}},
{"rank":453,"domain":"slack.com","origin":"https://slack.com","robots":{"status":"ok","http":200,"url":"https://slack.com/robots.txt","bytes":520,"groups":1},"bots":{"GPTBot":["partial","*",15,0],"ChatGPT-User":["partial","*",15,0],"OAI-SearchBot":["partial","*",15,0],"ClaudeBot":["partial","*",15,0],"Claude-User":["partial","*",15,0],"Claude-SearchBot":["partial","*",15,0],"anthropic-ai":["partial","*",15,0],"PerplexityBot":["partial","*",15,0],"Perplexity-User":["partial","*",15,0],"Google-Extended":["partial","*",15,0],"Applebot-Extended":["partial","*",15,0],"CCBot":["partial","*",15,0],"Bytespider":["partial","*",15,0],"Amazonbot":["partial","*",15,0],"meta-externalagent":["partial","*",15,0],"cohere-ai":["partial","*",15,0],"Diffbot":["partial","*",15,0],"ContextFetchBot":["partial","*",15,0]},"llms":{"status":"present","http":200,"bytes":40672},"home":{"status":"ok","http":200,"url":"https://slack.com/","html_tokens":63480,"markdown_tokens":695,"pruned_tokens":416}},
{"rank":454,"domain":"wisc.edu","origin":"https://wisc.edu","robots":{"status":"ok","http":200,"url":"https://www.wisc.edu/robots.txt","bytes":23,"groups":1},"bots":{"GPTBot":["allowed","*",0,1],"ChatGPT-User":["allowed","*",0,1],"OAI-SearchBot":["allowed","*",0,1],"ClaudeBot":["allowed","*",0,1],"Claude-User":["allowed","*",0,1],"Claude-SearchBot":["allowed","*",0,1],"anthropic-ai":["allowed","*",0,1],"PerplexityBot":["allowed","*",0,1],"Perplexity-User":["allowed","*",0,1],"Google-Extended":["allowed","*",0,1],"Applebot-Extended":["allowed","*",0,1],"CCBot":["allowed","*",0,1],"Bytespider":["allowed","*",0,1],"Amazonbot":["allowed","*",0,1],"meta-externalagent":["allowed","*",0,1],"cohere-ai":["allowed","*",0,1],"Diffbot":["allowed","*",0,1],"ContextFetchBot":["allowed","*",0,1]},"llms":{"status":"missing","http":404,"bytes":59594},"home":{"status":"ok","http":200,"url":"https://www.wisc.edu/","html_tokens":20687,"markdown_tokens":547,"pruned_tokens":165}},
{"rank":455,"domain":"tripod.com","origin":"https://tripod.com","robots":{"status":"unreachable","http":null,"url":"https://tripod.com/robots.txt","bytes":0,"groups":0,"error":"cannot resolve tripod.com"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":456,"domain":"about.com","origin":"https://about.com","robots":{"status":"html","http":200,"url":"https://www.people.inc/","bytes":299754,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":457,"domain":"blog.google","origin":"https://blog.google","robots":{"status":"ok","http":200,"url":"https://blog.google/robots.txt","bytes":153,"groups":1},"bots":{"GPTBot":["partial","*",4,0],"ChatGPT-User":["partial","*",4,0],"OAI-SearchBot":["partial","*",4,0],"ClaudeBot":["partial","*",4,0],"Claude-User":["partial","*",4,0],"Claude-SearchBot":["partial","*",4,0],"anthropic-ai":["partial","*",4,0],"PerplexityBot":["partial","*",4,0],"Perplexity-User":["partial","*",4,0],"Google-Extended":["partial","*",4,0],"Applebot-Extended":["partial","*",4,0],"CCBot":["partial","*",4,0],"Bytespider":["partial","*",4,0],"Amazonbot":["partial","*",4,0],"meta-externalagent":["partial","*",4,0],"cohere-ai":["partial","*",4,0],"Diffbot":["partial","*",4,0],"ContextFetchBot":["partial","*",4,0]},"llms":{"status":"missing","http":404,"bytes":233079},"home":{"status":"ok","http":200,"url":"https://blog.google/","html_tokens":90758,"markdown_tokens":45,"pruned_tokens":45}},
{"rank":458,"domain":"ucla.edu","origin":"https://ucla.edu","robots":{"status":"ok","http":200,"url":"https://www.ucla.edu/robots.txt","bytes":94,"groups":1},"bots":{"GPTBot":["partial","*",1,0],"ChatGPT-User":["partial","*",1,0],"OAI-SearchBot":["partial","*",1,0],"ClaudeBot":["partial","*",1,0],"Claude-User":["partial","*",1,0],"Claude-SearchBot":["partial","*",1,0],"anthropic-ai":["partial","*",1,0],"PerplexityBot":["partial","*",1,0],"Perplexity-User":["partial","*",1,0],"Google-Extended":["partial","*",1,0],"Applebot-Extended":["partial","*",1,0],"CCBot":["partial","*",1,0],"Bytespider":["partial","*",1,0],"Amazonbot":["partial","*",1,0],"meta-externalagent":["partial","*",1,0],"cohere-ai":["partial","*",1,0],"Diffbot":["partial","*",1,0],"ContextFetchBot":["partial","*",1,0]},"llms":{"status":"missing","http":404,"bytes":33729},"home":{"status":"ok","http":200,"url":"https://www.ucla.edu/","html_tokens":27072,"markdown_tokens":2079,"pruned_tokens":308}},
{"rank":459,"domain":"jd.com","origin":"https://jd.com","robots":{"status":"ok","http":200,"url":"https://jd.com/robots.txt","bytes":786,"groups":21},"bots":{"GPTBot":["partial","*",1,0],"ChatGPT-User":["partial","*",1,0],"OAI-SearchBot":["partial","*",1,0],"ClaudeBot":["partial","*",1,0],"Claude-User":["partial","*",1,0],"Claude-SearchBot":["partial","*",1,0],"anthropic-ai":["partial","*",1,0],"PerplexityBot":["partial","*",1,0],"Perplexity-User":["partial","*",1,0],"Google-Extended":["partial","*",1,0],"Applebot-Extended":["partial","*",1,0],"CCBot":["partial","*",1,0],"Bytespider":["partial","*",1,0],"Amazonbot":["partial","*",1,0],"meta-externalagent":["partial","*",1,0],"cohere-ai":["partial","*",1,0],"Diffbot":["partial","*",1,0],"ContextFetchBot":["partial","*",1,0]},"llms":{"status":"disallowed","error":"robots.txt of www.jd.com not readable (html)"},"home":{"status":"disallowed","error":"robots.txt of www.jd.com not readable (html)"}},
{"rank":460,"domain":"acs.org","origin":"https://acs.org","robots":{"status":"html","http":200,"url":"https://www.acs.org/robots.txt","bytes":850,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":461,"domain":"pnas.org","origin":"https://pnas.org","robots":{"status":"ok","http":200,"url":"https://www.pnas.org/robots.txt","bytes":559,"groups":3},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["partial","*",13,4],"OAI-SearchBot":["partial","*",13,4],"ClaudeBot":["partial","*",13,4],"Claude-User":["partial","*",13,4],"Claude-SearchBot":["partial","*",13,4],"anthropic-ai":["partial","*",13,4],"PerplexityBot":["partial","*",13,4],"Perplexity-User":["partial","*",13,4],"Google-Extended":["partial","*",13,4],"Applebot-Extended":["partial","*",13,4],"CCBot":["partial","*",13,4],"Bytespider":["partial","*",13,4],"Amazonbot":["partial","*",13,4],"meta-externalagent":["partial","*",13,4],"cohere-ai":["partial","*",13,4],"Diffbot":["partial","*",13,4],"ContextFetchBot":["partial","*",13,4]},"llms":{"status":"http_error","http":403,"bytes":5518},"home":{"status":"http_error","http":403,"url":"https://pnas.org/"}},
{"rank":462,"domain":"typepad.com","origin":"https://typepad.com","robots":{"status":"refused","http":403,"url":"https://www.networksolutions.com/typepad?siteID=100&channelID=P99C100S653N0B5A1D0E0000V111","bytes":5026,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":463,"domain":"business.site","origin":"https://business.site","robots":{"status":"missing","http":404,"url":"https://business.site/robots.txt","bytes":1571,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":1569},"home":{"status":"ok","http":200,"url":"https://business.google.com/us/business-profile/","html_tokens":75339,"markdown_tokens":66,"pruned_tokens":66}},
{"rank":464,"domain":"wiktionary.org","origin":"https://wiktionary.org","robots":{"status":"ok","http":200,"url":"https://en.wiktionary.org/robots.txt","bytes":15254,"groups":33},"bots":{"GPTBot":["partial","*",200,4],"ChatGPT-User":["partial","*",200,4],"OAI-SearchBot":["partial","*",200,4],"ClaudeBot":["partial","*",200,4],"Claude-User":["partial","*",200,4],"Claude-SearchBot":["partial","*",200,4],"anthropic-ai":["partial","*",200,4],"PerplexityBot":["partial","*",200,4],"Perplexity-User":["partial","*",200,4],"Google-Extended":["partial","*",200,4],"Applebot-Extended":["partial","*",200,4],"CCBot":["partial","*",200,4],"Bytespider":["partial","*",200,4],"Amazonbot":["partial","*",200,4],"meta-externalagent":["partial","*",200,4],"cohere-ai":["partial","*",200,4],"Diffbot":["partial","*",200,4],"ContextFetchBot":["partial","*",200,4]},"llms":{"status":"missing","http":404,"bytes":1868},"home":{"status":"ok","http":200,"url":"https://www.wiktionary.org/","html_tokens":18871,"markdown_tokens":8,"pruned_tokens":9}},
{"rank":465,"domain":"anthropic.com","origin":"https://anthropic.com","robots":{"status":"ok","http":200,"url":"https://www.anthropic.com/robots.txt","bytes":71,"groups":1},"bots":{"GPTBot":["allowed","*",0,1],"ChatGPT-User":["allowed","*",0,1],"OAI-SearchBot":["allowed","*",0,1],"ClaudeBot":["allowed","*",0,1],"Claude-User":["allowed","*",0,1],"Claude-SearchBot":["allowed","*",0,1],"anthropic-ai":["allowed","*",0,1],"PerplexityBot":["allowed","*",0,1],"Perplexity-User":["allowed","*",0,1],"Google-Extended":["allowed","*",0,1],"Applebot-Extended":["allowed","*",0,1],"CCBot":["allowed","*",0,1],"Bytespider":["allowed","*",0,1],"Amazonbot":["allowed","*",0,1],"meta-externalagent":["allowed","*",0,1],"cohere-ai":["allowed","*",0,1],"Diffbot":["allowed","*",0,1],"ContextFetchBot":["allowed","*",0,1]},"llms":{"status":"missing","http":404,"bytes":20610},"home":{"status":"ok","http":200,"url":"https://www.anthropic.com/","html_tokens":50342,"markdown_tokens":13,"pruned_tokens":13}},
{"rank":466,"domain":"hilton.com","origin":"https://hilton.com","robots":{"status":"ok","http":200,"url":"https://www.hilton.com/robots.txt","bytes":770,"groups":6},"bots":{"GPTBot":["partial","*",20,0],"ChatGPT-User":["partial","*",20,0],"OAI-SearchBot":["partial","*",20,0],"ClaudeBot":["partial","*",20,0],"Claude-User":["partial","*",20,0],"Claude-SearchBot":["partial","*",20,0],"anthropic-ai":["partial","*",20,0],"PerplexityBot":["partial","*",20,0],"Perplexity-User":["partial","*",20,0],"Google-Extended":["partial","*",20,0],"Applebot-Extended":["partial","*",20,0],"CCBot":["partial","*",20,0],"Bytespider":["partial","*",20,0],"Amazonbot":["partial","*",20,0],"meta-externalagent":["partial","*",20,0],"cohere-ai":["partial","*",20,0],"Diffbot":["partial","*",20,0],"ContextFetchBot":["partial","*",20,0]},"llms":{"status":"timeout","error":"timed out after 10 s"},"home":{"status":"http_error","http":403,"url":"https://www.hilton.com/en/"}},
{"rank":467,"domain":"bls.gov","origin":"https://bls.gov","robots":{"status":"html","http":200,"url":"https://www.bls.gov","bytes":78932,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":468,"domain":"gizmodo.com","origin":"https://gizmodo.com","robots":{"status":"ok","http":200,"url":"https://gizmodo.com/robots.txt","bytes":1223,"groups":18},"bots":{"GPTBot":["partial","*",16,1],"ChatGPT-User":["partial","*",16,1],"OAI-SearchBot":["partial","*",16,1],"ClaudeBot":["partial","*",16,1],"Claude-User":["partial","*",16,1],"Claude-SearchBot":["partial","*",16,1],"anthropic-ai":["partial","*",16,1],"PerplexityBot":["partial","*",16,1],"Perplexity-User":["partial","*",16,1],"Google-Extended":["partial","*",16,1],"Applebot-Extended":["partial","*",16,1],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["partial","*",16,1],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",16,1]},"llms":{"status":"missing","http":404,"bytes":146541},"home":{"status":"ok","http":200,"url":"https://gizmodo.com/","html_tokens":98161,"markdown_tokens":64,"pruned_tokens":60}},
{"rank":469,"domain":"in.ua","origin":"https://in.ua","robots":{"status":"unreachable","http":null,"url":"https://in.ua/robots.txt","bytes":0,"groups":0,"error":"cannot resolve in.ua"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":470,"domain":"getbootstrap.com","origin":"https://getbootstrap.com","robots":{"status":"ok","http":200,"url":"https://getbootstrap.com/robots.txt","bytes":129,"groups":1},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"present","http":200,"bytes":26304},"home":{"status":"ok","http":200,"url":"https://getbootstrap.com/","html_tokens":11430,"markdown_tokens":87,"pruned_tokens":87}},
{"rank":471,"domain":"si.edu","origin":"https://si.edu","robots":{"status":"html","http":200,"url":"https://www.si.edu/robots.txt","bytes":606,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":472,"domain":"globenewswire.com","origin":"https://globenewswire.com","robots":{"status":"refused","http":403,"url":"https://www.globenewswire.com/robots.txt","bytes":387,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":473,"domain":"jhu.edu","origin":"https://jhu.edu","robots":{"status":"refused","http":403,"url":"https://www.jhu.edu/robots.txt","bytes":4544,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":474,"domain":"wikihow.com","origin":"https://wikihow.com","robots":{"status":"ok","http":200,"url":"https://www.wikihow.com/robots.txt","bytes":4514,"groups":76},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["blocked","named",1,0],"OAI-SearchBot":["blocked","named",1,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["blocked","named",1,0],"Claude-SearchBot":["blocked","named",1,0],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["partial","*",7,16],"Google-Extended":["partial","*",7,16],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",7,16]},"llms":{"status":"missing","http":404,"bytes":153},"home":{"status":"ok","http":200,"url":"https://www.wikihow.com/Main-Page","html_tokens":94601,"markdown_tokens":434,"pruned_tokens":170}},
{"rank":475,"domain":"usgs.gov","origin":"https://www.usgs.gov","robots":{"status":"ok","http":200,"url":"https://www.usgs.gov/robots.txt","bytes":1636,"groups":1},"bots":{"GPTBot":["partial","*",22,18],"ChatGPT-User":["partial","*",22,18],"OAI-SearchBot":["partial","*",22,18],"ClaudeBot":["partial","*",22,18],"Claude-User":["partial","*",22,18],"Claude-SearchBot":["partial","*",22,18],"anthropic-ai":["partial","*",22,18],"PerplexityBot":["partial","*",22,18],"Perplexity-User":["partial","*",22,18],"Google-Extended":["partial","*",22,18],"Applebot-Extended":["partial","*",22,18],"CCBot":["partial","*",22,18],"Bytespider":["partial","*",22,18],"Amazonbot":["partial","*",22,18],"meta-externalagent":["partial","*",22,18],"cohere-ai":["partial","*",22,18],"Diffbot":["partial","*",22,18],"ContextFetchBot":["partial","*",22,18]},"llms":{"status":"missing","http":404,"bytes":291},"home":{"status":"ok","http":200,"url":"https://www.usgs.gov/","html_tokens":32020,"markdown_tokens":411,"pruned_tokens":294}},
{"rank":476,"domain":"people.com.cn","origin":"https://people.com.cn","robots":{"status":"ok","http":200,"url":"https://www.people.com.cn/robots.txt","bytes":90,"groups":1},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"crawl_delay","error":"Crawl-delay 120 s"},"home":{"status":"crawl_delay","error":"Crawl-delay 120 s"}},
{"rank":477,"domain":"techtarget.com","origin":"https://techtarget.com","robots":{"status":"ok","http":200,"url":"https://www.techtarget.com/robots.txt","bytes":10561,"groups":7},"bots":{"GPTBot":["partial","*",60,0],"ChatGPT-User":["partial","*",60,0],"OAI-SearchBot":["partial","*",60,0],"ClaudeBot":["partial","*",60,0],"Claude-User":["partial","*",60,0],"Claude-SearchBot":["partial","*",60,0],"anthropic-ai":["partial","*",60,0],"PerplexityBot":["partial","*",60,0],"Perplexity-User":["partial","*",60,0],"Google-Extended":["partial","*",60,0],"Applebot-Extended":["partial","*",60,0],"CCBot":["partial","*",60,0],"Bytespider":["partial","*",60,0],"Amazonbot":["partial","*",60,0],"meta-externalagent":["partial","*",60,0],"cohere-ai":["partial","*",60,0],"Diffbot":["partial","*",60,0],"ContextFetchBot":["partial","*",60,0]},"llms":{"status":"missing","http":404,"bytes":16431},"home":{"status":"ok","http":200,"url":"https://www.techtarget.com/","html_tokens":37128,"markdown_tokens":729,"pruned_tokens":159}},
{"rank":478,"domain":"go.id","origin":"https://go.id","robots":{"status":"unreachable","http":null,"url":"https://go.id/robots.txt","bytes":0,"groups":0,"error":"cannot resolve go.id"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":479,"domain":"merriam-webster.com","origin":"https://merriam-webster.com","robots":{"status":"ok","http":200,"url":"https://www.merriam-webster.com/robots.txt","bytes":708,"groups":1},"bots":{"GPTBot":["partial","*",7,0],"ChatGPT-User":["partial","*",7,0],"OAI-SearchBot":["partial","*",7,0],"ClaudeBot":["partial","*",7,0],"Claude-User":["partial","*",7,0],"Claude-SearchBot":["partial","*",7,0],"anthropic-ai":["partial","*",7,0],"PerplexityBot":["partial","*",7,0],"Perplexity-User":["partial","*",7,0],"Google-Extended":["partial","*",7,0],"Applebot-Extended":["partial","*",7,0],"CCBot":["partial","*",7,0],"Bytespider":["partial","*",7,0],"Amazonbot":["partial","*",7,0],"meta-externalagent":["partial","*",7,0],"cohere-ai":["partial","*",7,0],"Diffbot":["partial","*",7,0],"ContextFetchBot":["partial","*",7,0]},"llms":{"status":"missing","http":404,"bytes":146},"home":{"status":"http_error","http":403,"url":"https://www.merriam-webster.com/"}},
{"rank":480,"domain":"elpais.com","origin":"https://elpais.com","robots":{"status":"ok","http":200,"url":"https://elpais.com/robots.txt","bytes":7777,"groups":201},"bots":{"GPTBot":["partial","*",16,0],"ChatGPT-User":["partial","*",16,0],"OAI-SearchBot":["partial","*",16,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["blocked","named",1,0],"Claude-SearchBot":["blocked","named",1,0],"anthropic-ai":["partial","*",16,0],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["blocked","named",1,0],"Google-Extended":["partial","*",16,0],"Applebot-Extended":["partial","*",16,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["partial","*",16,0],"cohere-ai":["partial","*",16,0],"Diffbot":["partial","*",16,0],"ContextFetchBot":["partial","*",16,0]},"llms":{"status":"missing","http":404,"bytes":64264},"home":{"status":"ok","http":200,"url":"https://elpais.com/","html_tokens":118447,"markdown_tokens":1167,"pruned_tokens":169}},
{"rank":481,"domain":"science.org","origin":"https://science.org","robots":{"status":"ok","http":200,"url":"https://www.science.org/robots.txt","bytes":930,"groups":9},"bots":{"GPTBot":["blocked","named",2,0],"ChatGPT-User":["blocked","named",1,0],"OAI-SearchBot":["partial","*",14,8],"ClaudeBot":["partial","*",14,8],"Claude-User":["partial","*",14,8],"Claude-SearchBot":["partial","*",14,8],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["partial","*",14,8],"Perplexity-User":["partial","*",14,8],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["partial","*",14,8],"Amazonbot":["partial","*",14,8],"meta-externalagent":["partial","*",14,8],"cohere-ai":["partial","*",14,8],"Diffbot":["partial","*",14,8],"ContextFetchBot":["partial","*",14,8]},"llms":{"status":"http_error","http":403,"bytes":5568},"home":{"status":"http_error","http":403,"url":"https://www.science.org/"}},
{"rank":482,"domain":"aljazeera.com","origin":"https://aljazeera.com","robots":{"status":"ok","http":200,"url":"https://www.aljazeera.com/robots.txt","bytes":1834,"groups":10},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["blocked","named",1,0],"OAI-SearchBot":["partial","*",15,1],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",15,1],"Claude-SearchBot":["partial","*",15,1],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["partial","*",15,1],"Google-Extended":["partial","*",15,1],"Applebot-Extended":["partial","*",15,1],"CCBot":["partial","*",15,1],"Bytespider":["blocked","named",1,0],"Amazonbot":["partial","*",15,1],"meta-externalagent":["partial","*",15,1],"cohere-ai":["blocked","named",1,0],"Diffbot":["partial","*",15,1],"ContextFetchBot":["partial","*",15,1]},"llms":{"status":"missing","http":404,"bytes":40662},"home":{"status":"ok","http":200,"url":"https://www.aljazeera.com/","html_tokens":124346,"markdown_tokens":150,"pruned_tokens":150}},
{"rank":483,"domain":"unicef.org","origin":"https://unicef.org","robots":{"status":"ok","http":200,"url":"https://www.unicef.org/robots.txt","bytes":1945,"groups":1},"bots":{"GPTBot":["partial","*",34,18],"ChatGPT-User":["partial","*",34,18],"OAI-SearchBot":["partial","*",34,18],"ClaudeBot":["partial","*",34,18],"Claude-User":["partial","*",34,18],"Claude-SearchBot":["partial","*",34,18],"anthropic-ai":["partial","*",34,18],"PerplexityBot":["partial","*",34,18],"Perplexity-User":["partial","*",34,18],"Google-Extended":["partial","*",34,18],"Applebot-Extended":["partial","*",34,18],"CCBot":["partial","*",34,18],"Bytespider":["partial","*",34,18],"Amazonbot":["partial","*",34,18],"meta-externalagent":["partial","*",34,18],"cohere-ai":["partial","*",34,18],"Diffbot":["partial","*",34,18],"ContextFetchBot":["partial","*",34,18]},"llms":{"status":"missing","http":404,"bytes":515},"home":{"status":"ok","http":200,"url":"https://www.unicef.org/","html_tokens":58962,"markdown_tokens":102,"pruned_tokens":102}},
{"rank":484,"domain":"huggingface.co","origin":"https://huggingface.co","robots":{"status":"ok","http":200,"url":"https://huggingface.co/robots.txt","bytes":68,"groups":1},"bots":{"GPTBot":["allowed","*",0,1],"ChatGPT-User":["allowed","*",0,1],"OAI-SearchBot":["allowed","*",0,1],"ClaudeBot":["allowed","*",0,1],"Claude-User":["allowed","*",0,1],"Claude-SearchBot":["allowed","*",0,1],"anthropic-ai":["allowed","*",0,1],"PerplexityBot":["allowed","*",0,1],"Perplexity-User":["allowed","*",0,1],"Google-Extended":["allowed","*",0,1],"Applebot-Extended":["allowed","*",0,1],"CCBot":["allowed","*",0,1],"Bytespider":["allowed","*",0,1],"Amazonbot":["allowed","*",0,1],"meta-externalagent":["allowed","*",0,1],"cohere-ai":["allowed","*",0,1],"Diffbot":["allowed","*",0,1],"ContextFetchBot":["allowed","*",0,1]},"llms":{"status":"missing","http":404,"bytes":52349},"home":{"status":"ok","http":200,"url":"https://huggingface.co/","html_tokens":44463,"markdown_tokens":229,"pruned_tokens":96}},
{"rank":485,"domain":"openssl.org","origin":"https://openssl.org","robots":{"status":"ok","http":200,"url":"https://openssl.org/robots.txt","bytes":26,"groups":1},"bots":{"GPTBot":["blocked","*",1,0],"ChatGPT-User":["blocked","*",1,0],"OAI-SearchBot":["blocked","*",1,0],"ClaudeBot":["blocked","*",1,0],"Claude-User":["blocked","*",1,0],"Claude-SearchBot":["blocked","*",1,0],"anthropic-ai":["blocked","*",1,0],"PerplexityBot":["blocked","*",1,0],"Perplexity-User":["blocked","*",1,0],"Google-Extended":["blocked","*",1,0],"Applebot-Extended":["blocked","*",1,0],"CCBot":["blocked","*",1,0],"Bytespider":["blocked","*",1,0],"Amazonbot":["blocked","*",1,0],"meta-externalagent":["blocked","*",1,0],"cohere-ai":["blocked","*",1,0],"Diffbot":["blocked","*",1,0],"ContextFetchBot":["blocked","*",1,0]},"llms":{"status":"disallowed"},"home":{"status":"disallowed"}},
{"rank":486,"domain":"walmart.com","origin":"https://walmart.com","robots":{"status":"ok","http":200,"url":"https://www.walmart.com/robots.txt","bytes":3479,"groups":2},"bots":{"GPTBot":["partial","*",57,3],"ChatGPT-User":["partial","*",57,3],"OAI-SearchBot":["partial","*",57,3],"ClaudeBot":["partial","*",57,3],"Claude-User":["partial","*",57,3],"Claude-SearchBot":["partial","*",57,3],"anthropic-ai":["partial","*",57,3],"PerplexityBot":["partial","*",57,3],"Perplexity-User":["partial","*",57,3],"Google-Extended":["partial","*",57,3],"Applebot-Extended":["partial","*",57,3],"CCBot":["partial","*",57,3],"Bytespider":["partial","*",57,3],"Amazonbot":["partial","*",57,3],"meta-externalagent":["partial","*",57,3],"cohere-ai":["partial","*",57,3],"Diffbot":["partial","*",57,3],"ContextFetchBot":["partial","*",57,3]},"llms":{"status":"html","http":200,"bytes":244},"home":{"status":"ok","http":200,"url":"https://www.walmart.com/","html_tokens":112717,"markdown_tokens":415,"pruned_tokens":172}},
{"rank":487,"domain":"utexas.edu","origin":"https://utexas.edu","robots":{"status":"refused","http":403,"url":"https://utexas.edu/robots.txt","bytes":560,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":488,"domain":"intuit.com","origin":"https://intuit.com","robots":{"status":"ok","http":200,"url":"https://www.intuit.com/robots.txt","bytes":3086,"groups":16},"bots":{"GPTBot":["partial","*",11,0],"ChatGPT-User":["partial","*",11,0],"OAI-SearchBot":["partial","*",11,0],"ClaudeBot":["partial","*",11,0],"Claude-User":["partial","*",11,0],"Claude-SearchBot":["partial","*",11,0],"anthropic-ai":["partial","*",11,0],"PerplexityBot":["partial","*",11,0],"Perplexity-User":["partial","*",11,0],"Google-Extended":["partial","*",11,0],"Applebot-Extended":["partial","*",11,0],"CCBot":["partial","*",11,0],"Bytespider":["partial","*",11,0],"Amazonbot":["partial","*",11,0],"meta-externalagent":["partial","*",11,0],"cohere-ai":["partial","*",11,0],"Diffbot":["partial","*",11,0],"ContextFetchBot":["partial","*",11,0]},"llms":{"status":"present","http":200,"bytes":5786},"home":{"status":"not_html","error":"unsupported content type text/xml"}},
{"rank":489,"domain":"itu.int","origin":"https://itu.int","robots":{"status":"ok","http":200,"url":"https://www.itu.int/robots.txt","bytes":1574,"groups":1},"bots":{"GPTBot":["partial","*",35,0],"ChatGPT-User":["partial","*",35,0],"OAI-SearchBot":["partial","*",35,0],"ClaudeBot":["partial","*",35,0],"Claude-User":["partial","*",35,0],"Claude-SearchBot":["partial","*",35,0],"anthropic-ai":["partial","*",35,0],"PerplexityBot":["partial","*",35,0],"Perplexity-User":["partial","*",35,0],"Google-Extended":["partial","*",35,0],"Applebot-Extended":["partial","*",35,0],"CCBot":["partial","*",35,0],"Bytespider":["partial","*",35,0],"Amazonbot":["partial","*",35,0],"meta-externalagent":["partial","*",35,0],"cohere-ai":["partial","*",35,0],"Diffbot":["partial","*",35,0],"ContextFetchBot":["partial","*",35,0]},"llms":{"status":"missing","http":404,"bytes":657},"home":{"status":"ok","http":200,"url":"https://www.itu.int/en/Pages/default.aspx","html_tokens":24871,"markdown_tokens":127,"pruned_tokens":127}},
{"rank":490,"domain":"www.nhs.uk","origin":"https://www.nhs.uk","robots":{"status":"ok","http":200,"url":"https://www.nhs.uk/robots.txt","bytes":2674,"groups":4},"bots":{"GPTBot":["partial","*",31,0],"ChatGPT-User":["partial","*",31,0],"OAI-SearchBot":["partial","*",31,0],"ClaudeBot":["partial","*",31,0],"Claude-User":["partial","*",31,0],"Claude-SearchBot":["partial","*",31,0],"anthropic-ai":["partial","*",31,0],"PerplexityBot":["partial","*",31,0],"Perplexity-User":["partial","*",31,0],"Google-Extended":["partial","*",31,0],"Applebot-Extended":["partial","*",31,0],"CCBot":["partial","*",31,0],"Bytespider":["partial","*",31,0],"Amazonbot":["partial","*",31,0],"meta-externalagent":["partial","*",31,0],"cohere-ai":["partial","*",31,0],"Diffbot":["partial","*",31,0],"ContextFetchBot":["partial","*",31,0]},"llms":{"status":"missing","http":404,"bytes":15061},"home":{"status":"ok","http":200,"url":"https://www.nhs.uk/","html_tokens":7864,"markdown_tokens":51,"pruned_tokens":51}},
{"rank":491,"domain":"ed.gov","origin":"https://ed.gov","robots":{"status":"refused","http":403,"url":"https://www.ed.gov/robots.txt","bytes":11622,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":492,"domain":"euronews.com","origin":"https://euronews.com","robots":{"status":"ok","http":200,"url":"https://www.euronews.com/robots.txt","bytes":2387,"groups":3},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["partial","named",27,1],"OAI-SearchBot":["partial","named",27,1],"ClaudeBot":["partial","*",29,0],"Claude-User":["partial","*",29,0],"Claude-SearchBot":["partial","*",29,0],"anthropic-ai":["partial","*",29,0],"PerplexityBot":["partial","named",27,1],"Perplexity-User":["partial","*",29,0],"Google-Extended":["partial","*",29,0],"Applebot-Extended":["partial","*",29,0],"CCBot":["blocked","named",1,0],"Bytespider":["partial","*",29,0],"Amazonbot":["partial","*",29,0],"meta-externalagent":["partial","*",29,0],"cohere-ai":["partial","*",29,0],"Diffbot":["partial","*",29,0],"ContextFetchBot":["partial","*",29,0]},"llms":{"status":"missing","http":404,"bytes":199506},"home":{"status":"ok","http":200,"url":"https://www.euronews.com/","html_tokens":126129,"markdown_tokens":117,"pruned_tokens":114}},
{"rank":493,"domain":"ssrn.com","origin":"https://ssrn.com","robots":{"status":"refused","http":403,"url":"https://www.ssrn.com/robots.txt","bytes":896795,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":494,"domain":"sciencemag.org","origin":"https://sciencemag.org","robots":{"status":"ok","http":200,"url":"https://www.science.org/robots.txt","bytes":930,"groups":9},"bots":{"GPTBot":["blocked","named",2,0],"ChatGPT-User":["blocked","named",1,0],"OAI-SearchBot":["partial","*",14,8],"ClaudeBot":["partial","*",14,8],"Claude-User":["partial","*",14,8],"Claude-SearchBot":["partial","*",14,8],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["partial","*",14,8],"Perplexity-User":["partial","*",14,8],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["partial","*",14,8],"Amazonbot":["partial","*",14,8],"meta-externalagent":["partial","*",14,8],"cohere-ai":["partial","*",14,8],"Diffbot":["partial","*",14,8],"ContextFetchBot":["partial","*",14,8]},"llms":{"status":"http_error","http":403,"bytes":5571},"home":{"status":"http_error","http":403,"url":"https://www.sciencemag.org/"}},
{"rank":495,"domain":"giphy.com","origin":"https://giphy.com","robots":{"status":"ok","http":200,"url":"https://giphy.com/robots.txt","bytes":119,"groups":3},"bots":{"GPTBot":["allowed","*",0,2],"ChatGPT-User":["allowed","*",0,2],"OAI-SearchBot":["allowed","*",0,2],"ClaudeBot":["allowed","*",0,2],"Claude-User":["allowed","*",0,2],"Claude-SearchBot":["allowed","*",0,2],"anthropic-ai":["allowed","*",0,2],"PerplexityBot":["allowed","*",0,2],"Perplexity-User":["allowed","*",0,2],"Google-Extended":["allowed","*",0,2],"Applebot-Extended":["allowed","*",0,2],"CCBot":["allowed","*",0,2],"Bytespider":["allowed","*",0,2],"Amazonbot":["allowed","*",0,2],"meta-externalagent":["allowed","*",0,2],"cohere-ai":["allowed","*",0,2],"Diffbot":["allowed","*",0,2],"ContextFetchBot":["allowed","*",0,2]},"llms":{"status":"missing","http":404,"bytes":13808},"home":{"status":"ok","http":200,"url":"https://giphy.com/","html_tokens":84131,"markdown_tokens":73,"pruned_tokens":73}},
{"rank":496,"domain":"iana.org","origin":"https://iana.org","robots":{"status":"ok","http":200,"url":"https://www.iana.org/robots.txt","bytes":24,"groups":1},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"missing","http":404,"bytes":4216},"home":{"status":"ok","http":200,"url":"https://www.iana.org/","html_tokens":1563,"markdown_tokens":45,"pruned_tokens":45}},
{"rank":497,"domain":"aboutcookies.org","origin":"https://aboutcookies.org","robots":{"status":"ok","http":200,"url":"https://www.aboutcookies.org/robots.txt","bytes":64,"groups":1},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"missing","http":404,"bytes":165609},"home":{"status":"ok","http":200,"url":"https://www.aboutcookies.org/","html_tokens":42329,"markdown_tokens":157,"pruned_tokens":151}},
{"rank":498,"domain":"azurewebsites.net","origin":"https://azurewebsites.net","robots":{"status":"unreachable","http":null,"url":"https://azurewebsites.net/robots.txt","bytes":0,"groups":0,"error":"cannot resolve azurewebsites.net"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":499,"domain":"sec.gov","origin":"https://sec.gov","robots":{"status":"ok","http":200,"url":"https://www.sec.gov/robots.txt","bytes":2736,"groups":1},"bots":{"GPTBot":["partial","*",48,23],"ChatGPT-User":["partial","*",48,23],"OAI-SearchBot":["partial","*",48,23],"ClaudeBot":["partial","*",48,23],"Claude-User":["partial","*",48,23],"Claude-SearchBot":["partial","*",48,23],"anthropic-ai":["partial","*",48,23],"PerplexityBot":["partial","*",48,23],"Perplexity-User":["partial","*",48,23],"Google-Extended":["partial","*",48,23],"Applebot-Extended":["partial","*",48,23],"CCBot":["partial","*",48,23],"Bytespider":["partial","*",48,23],"Amazonbot":["partial","*",48,23],"meta-externalagent":["partial","*",48,23],"cohere-ai":["partial","*",48,23],"Diffbot":["partial","*",48,23],"ContextFetchBot":["partial","*",48,23]},"llms":{"status":"missing","http":404,"bytes":291},"home":{"status":"ok","http":200,"url":"https://www.sec.gov/","html_tokens":56382,"markdown_tokens":316,"pruned_tokens":108}},
{"rank":500,"domain":"atlassian.com","origin":"https://atlassian.com","robots":{"status":"ok","http":200,"url":"https://www.atlassian.com/robots.txt","bytes":1292,"groups":2},"bots":{"GPTBot":["partial","*",10,1],"ChatGPT-User":["partial","*",10,1],"OAI-SearchBot":["partial","*",10,1],"ClaudeBot":["partial","*",10,1],"Claude-User":["partial","*",10,1],"Claude-SearchBot":["partial","*",10,1],"anthropic-ai":["partial","*",10,1],"PerplexityBot":["partial","*",10,1],"Perplexity-User":["partial","*",10,1],"Google-Extended":["partial","*",10,1],"Applebot-Extended":["partial","*",10,1],"CCBot":["partial","*",10,1],"Bytespider":["partial","*",10,1],"Amazonbot":["partial","*",10,1],"meta-externalagent":["partial","*",10,1],"cohere-ai":["partial","*",10,1],"Diffbot":["partial","*",10,1],"ContextFetchBot":["partial","*",10,1]},"llms":{"status":"present","http":200,"bytes":26607},"home":{"status":"ok","http":200,"url":"https://www.atlassian.com/","html_tokens":659059,"markdown_tokens":109,"pruned_tokens":109}},
{"rank":501,"domain":"lin.ee","origin":"https://lin.ee","robots":{"status":"ok","http":200,"url":"https://lin.ee/robots.txt","bytes":31,"groups":1},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":13},"home":{"status":"http_error","http":404,"url":"https://lin.ee/"}},
{"rank":502,"domain":"hhs.gov","origin":"https://hhs.gov","robots":{"status":"refused","http":403,"url":"https://www.hhs.gov/robots.txt","bytes":379,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":503,"domain":"mashable.com","origin":"https://mashable.com","robots":{"status":"ok","http":200,"url":"https://mashable.com/robots.txt","bytes":6394,"groups":3},"bots":{"GPTBot":["partial","named",2,7],"ChatGPT-User":["partial","named",1,4],"OAI-SearchBot":["partial","named",1,4],"ClaudeBot":["partial","named",1,4],"Claude-User":["partial","named",1,4],"Claude-SearchBot":["partial","named",1,4],"anthropic-ai":["partial","named",1,4],"PerplexityBot":["partial","named",1,4],"Perplexity-User":["partial","named",1,4],"Google-Extended":["partial","*",3,1],"Applebot-Extended":["partial","named",1,4],"CCBot":["partial","named",1,4],"Bytespider":["partial","named",1,4],"Amazonbot":["partial","named",1,4],"meta-externalagent":["partial","named",1,4],"cohere-ai":["partial","named",1,4],"Diffbot":["partial","named",1,4],"ContextFetchBot":["partial","*",3,1]},"llms":{"status":"missing","http":404,"bytes":154384},"home":{"status":"ok","http":200,"url":"https://mashable.com/","html_tokens":81583,"markdown_tokens":23,"pruned_tokens":23}},
{"rank":504,"domain":"gob.es","origin":"https://gob.es","robots":{"status":"unreachable","http":null,"url":"https://gob.es/robots.txt","bytes":0,"groups":0,"error":"cannot resolve gob.es"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":505,"domain":"engadget.com","origin":"https://engadget.com","robots":{"status":"ok","http":200,"url":"https://engadget.com/robots.txt","bytes":207,"groups":1},"bots":{"GPTBot":["partial","*",4,0],"ChatGPT-User":["partial","*",4,0],"OAI-SearchBot":["partial","*",4,0],"ClaudeBot":["partial","*",4,0],"Claude-User":["partial","*",4,0],"Claude-SearchBot":["partial","*",4,0],"anthropic-ai":["partial","*",4,0],"PerplexityBot":["partial","*",4,0],"Perplexity-User":["partial","*",4,0],"Google-Extended":["partial","*",4,0],"Applebot-Extended":["partial","*",4,0],"CCBot":["partial","*",4,0],"Bytespider":["partial","*",4,0],"Amazonbot":["partial","*",4,0],"meta-externalagent":["partial","*",4,0],"cohere-ai":["partial","*",4,0],"Diffbot":["partial","*",4,0],"ContextFetchBot":["partial","*",4,0]},"llms":{"status":"html","http":200,"bytes":109673},"home":{"status":"ok","http":200,"url":"https://www.engadget.com/","html_tokens":27416,"markdown_tokens":408,"pruned_tokens":146}},
{"rank":506,"domain":"aliexpress.com","origin":"https://aliexpress.com","robots":{"status":"ok","http":200,"url":"https://www.aliexpress.com/robots.txt","bytes":3025,"groups":9},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["partial","*",35,4],"OAI-SearchBot":["partial","*",35,4],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",35,4],"Claude-SearchBot":["partial","*",35,4],"anthropic-ai":["partial","*",35,4],"PerplexityBot":["partial","*",35,4],"Perplexity-User":["partial","*",35,4],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["partial","*",35,4],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["partial","*",35,4],"cohere-ai":["partial","*",35,4],"Diffbot":["partial","*",35,4],"ContextFetchBot":["partial","*",35,4]},"llms":{"status":"html","http":200,"bytes":13874},"home":{"status":"disallowed","error":"robots.txt of login.aliexpress.com not readable (html)"}},
{"rank":507,"domain":"w3schools.com","origin":"https://w3schools.com","robots":{"status":"ok","http":200,"url":"https://www.w3schools.com/robots.txt","bytes":132,"groups":2},"bots":{"GPTBot":["partial","*",4,0],"ChatGPT-User":["partial","*",4,0],"OAI-SearchBot":["partial","*",4,0],"ClaudeBot":["partial","*",4,0],"Claude-User":["partial","*",4,0],"Claude-SearchBot":["partial","*",4,0],"anthropic-ai":["partial","*",4,0],"PerplexityBot":["partial","*",4,0],"Perplexity-User":["partial","*",4,0],"Google-Extended":["partial","*",4,0],"Applebot-Extended":["partial","*",4,0],"CCBot":["partial","*",4,0],"Bytespider":["partial","*",4,0],"Amazonbot":["partial","*",4,0],"meta-externalagent":["partial","*",4,0],"cohere-ai":["partial","*",4,0],"Diffbot":["partial","*",4,0],"ContextFetchBot":["partial","*",4,0]},"llms":{"status":"missing","http":404,"bytes":9496},"home":{"status":"ok","http":200,"url":"https://www.w3schools.com/","html_tokens":93246,"markdown_tokens":198,"pruned_tokens":26}},
{"rank":508,"domain":"meetup.com","origin":"https://meetup.com","robots":{"status":"ok","http":200,"url":"https://www.meetup.com/robots.txt","bytes":3770,"groups":10},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["partial","*",115,2],"OAI-SearchBot":["partial","*",115,2],"ClaudeBot":["partial","*",115,2],"Claude-User":["partial","*",115,2],"Claude-SearchBot":["partial","*",115,2],"anthropic-ai":["partial","*",115,2],"PerplexityBot":["partial","*",115,2],"Perplexity-User":["partial","*",115,2],"Google-Extended":["partial","*",115,2],"Applebot-Extended":["partial","*",115,2],"CCBot":["partial","*",115,2],"Bytespider":["partial","*",115,2],"Amazonbot":["partial","*",115,2],"meta-externalagent":["partial","*",115,2],"cohere-ai":["partial","*",115,2],"Diffbot":["partial","*",115,2],"ContextFetchBot":["partial","*",115,2]},"llms":{"status":"present","http":200,"bytes":9414},"home":{"status":"ok","http":200,"url":"https://www.meetup.com/","html_tokens":167349,"markdown_tokens":143,"pruned_tokens":143}},
{"rank":509,"domain":"matterport.com","origin":"https://matterport.com","robots":{"status":"ok","http":200,"url":"https://matterport.com/robots.txt","bytes":68,"groups":1},"bots":{"GPTBot":["allowed","*",0,1],"ChatGPT-User":["allowed","*",0,1],"OAI-SearchBot":["allowed","*",0,1],"ClaudeBot":["allowed","*",0,1],"Claude-User":["allowed","*",0,1],"Claude-SearchBot":["allowed","*",0,1],"anthropic-ai":["allowed","*",0,1],"PerplexityBot":["allowed","*",0,1],"Perplexity-User":["allowed","*",0,1],"Google-Extended":["allowed","*",0,1],"Applebot-Extended":["allowed","*",0,1],"CCBot":["allowed","*",0,1],"Bytespider":["allowed","*",0,1],"Amazonbot":["allowed","*",0,1],"meta-externalagent":["allowed","*",0,1],"cohere-ai":["allowed","*",0,1],"Diffbot":["allowed","*",0,1],"ContextFetchBot":["allowed","*",0,1]},"llms":{"status":"missing","http":404,"bytes":398881},"home":{"status":"ok","http":200,"url":"https://matterport.com/","html_tokens":128468,"markdown_tokens":162,"pruned_tokens":162}},
{"rank":510,"domain":"storage.googleapis.com","origin":"https://storage.googleapis.com","robots":{"status":"missing","http":404,"url":"https://storage.googleapis.com/robots.txt","bytes":133,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":133},"home":{"status":"not_html","error":"unsupported content type application/xml"}},
{"rank":511,"domain":"lemonde.fr","origin":"https://lemonde.fr","robots":{"status":"ok","http":200,"url":"https://www.lemonde.fr/robots.txt","bytes":13226,"groups":304},"bots":{"GPTBot":["partial","*",30,2],"ChatGPT-User":["partial","*",30,2],"OAI-SearchBot":["partial","*",30,2],"ClaudeBot":["blocked","named",1,0],"Claude-User":["blocked","named",1,0],"Claude-SearchBot":["blocked","named",1,0],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["partial","*",30,2],"Perplexity-User":["partial","*",30,2],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["partial","*",30,2],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",30,2]},"llms":{"status":"html","http":200,"bytes":3038},"home":{"status":"ok","http":200,"url":"https://www.lemonde.fr/","html_tokens":491019,"markdown_tokens":1160,"pruned_tokens":173}},
{"rank":512,"domain":"ikea.com","origin":"https://ikea.com","robots":{"status":"ok","http":200,"url":"https://www.ikea.com/robots.txt","bytes":4054,"groups":4},"bots":{"GPTBot":["partial","*",92,3],"ChatGPT-User":["partial","*",92,3],"OAI-SearchBot":["partial","*",92,3],"ClaudeBot":["partial","*",92,3],"Claude-User":["partial","*",92,3],"Claude-SearchBot":["partial","*",92,3],"anthropic-ai":["partial","*",92,3],"PerplexityBot":["partial","*",92,3],"Perplexity-User":["partial","*",92,3],"Google-Extended":["partial","*",92,3],"Applebot-Extended":["partial","*",92,3],"CCBot":["partial","*",92,3],"Bytespider":["partial","*",92,3],"Amazonbot":["partial","*",92,3],"meta-externalagent":["partial","*",92,3],"cohere-ai":["partial","*",92,3],"Diffbot":["partial","*",92,3],"ContextFetchBot":["partial","*",92,3]},"llms":{"status":"missing","http":404,"bytes":9},"home":{"status":"ok","http":200,"url":"https://www.ikea.com/","html_tokens":145192,"markdown_tokens":92,"pruned_tokens":47}},
{"rank":513,"domain":"uiuc.edu","origin":"https://uiuc.edu","robots":{"status":"ok","http":200,"url":"https://illinois.edu/robots.txt","bytes":110,"groups":1},"bots":{"GPTBot":["partial","*",1,1],"ChatGPT-User":["partial","*",1,1],"OAI-SearchBot":["partial","*",1,1],"ClaudeBot":["partial","*",1,1],"Claude-User":["partial","*",1,1],"Claude-SearchBot":["partial","*",1,1],"anthropic-ai":["partial","*",1,1],"PerplexityBot":["partial","*",1,1],"Perplexity-User":["partial","*",1,1],"Google-Extended":["partial","*",1,1],"Applebot-Extended":["partial","*",1,1],"CCBot":["partial","*",1,1],"Bytespider":["partial","*",1,1],"Amazonbot":["partial","*",1,1],"meta-externalagent":["partial","*",1,1],"cohere-ai":["partial","*",1,1],"Diffbot":["partial","*",1,1],"ContextFetchBot":["partial","*",1,1]},"llms":{"status":"html","http":200,"bytes":57478},"home":{"status":"ok","http":200,"url":"https://illinois.edu/","html_tokens":37093,"markdown_tokens":836,"pruned_tokens":147}},
{"rank":514,"domain":"justice.gov","origin":"https://justice.gov","robots":{"status":"ok","http":200,"url":"https://www.justice.gov/robots.txt","bytes":2651,"groups":2},"bots":{"GPTBot":["partial","*",50,24],"ChatGPT-User":["partial","*",50,24],"OAI-SearchBot":["partial","*",50,24],"ClaudeBot":["partial","*",50,24],"Claude-User":["partial","*",50,24],"Claude-SearchBot":["partial","*",50,24],"anthropic-ai":["partial","*",50,24],"PerplexityBot":["partial","*",50,24],"Perplexity-User":["partial","*",50,24],"Google-Extended":["partial","*",50,24],"Applebot-Extended":["partial","*",50,24],"CCBot":["partial","*",50,24],"Bytespider":["partial","*",50,24],"Amazonbot":["partial","*",50,24],"meta-externalagent":["partial","*",50,24],"cohere-ai":["partial","*",50,24],"Diffbot":["partial","*",50,24],"ContextFetchBot":["partial","*",50,24]},"llms":{"status":"missing","http":404,"bytes":189},"home":{"status":"ok","http":200,"url":"https://www.justice.gov/","html_tokens":25670,"markdown_tokens":382,"pruned_tokens":32}},
{"rank":515,"domain":"qc.ca","origin":"https://qc.ca","robots":{"status":"unreachable","http":null,"url":"https://qc.ca/robots.txt","bytes":0,"groups":0,"error":"cannot resolve qc.ca"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":516,"domain":"ru.wikipedia.org","origin":"https://ru.wikipedia.org","robots":{"status":"ok","http":200,"url":"https://ru.wikipedia.org/robots.txt","bytes":21125,"groups":33},"bots":{"GPTBot":["partial","*",248,4],"ChatGPT-User":["partial","*",248,4],"OAI-SearchBot":["partial","*",248,4],"ClaudeBot":["partial","*",248,4],"Claude-User":["partial","*",248,4],"Claude-SearchBot":["partial","*",248,4],"anthropic-ai":["partial","*",248,4],"PerplexityBot":["partial","*",248,4],"Perplexity-User":["partial","*",248,4],"Google-Extended":["partial","*",248,4],"Applebot-Extended":["partial","*",248,4],"CCBot":["partial","*",248,4],"Bytespider":["partial","*",248,4],"Amazonbot":["partial","*",248,4],"meta-externalagent":["partial","*",248,4],"cohere-ai":["partial","*",248,4],"Diffbot":["partial","*",248,4],"ContextFetchBot":["partial","*",248,4]},"llms":{"status":"missing","http":404,"bytes":1868},"home":{"status":"ok","http":200,"url":"https://ru.wikipedia.org/wiki/%D0%97%D0%B0%D0%B3%D0%BB%D0%B0%D0%B2%D0%BD%D0%B0%D1%8F_%D1%81%D1%82%D1%80%D0%B0%D0%BD%D0%B8%D1%86%D0%B0","html_tokens":104025,"markdown_tokens":1637,"pruned_tokens":164}},
{"rank":517,"domain":"automattic.com","origin":"https://automattic.com","robots":{"status":"ok","http":200,"url":"https://automattic.com/robots.txt","bytes":628,"groups":1},"bots":{"GPTBot":["partial","*",10,1],"ChatGPT-User":["partial","*",10,1],"OAI-SearchBot":["partial","*",10,1],"ClaudeBot":["partial","*",10,1],"Claude-User":["partial","*",10,1],"Claude-SearchBot":["partial","*",10,1],"anthropic-ai":["partial","*",10,1],"PerplexityBot":["partial","*",10,1],"Perplexity-User":["partial","*",10,1],"Google-Extended":["partial","*",10,1],"Applebot-Extended":["partial","*",10,1],"CCBot":["partial","*",10,1],"Bytespider":["partial","*",10,1],"Amazonbot":["partial","*",10,1],"meta-externalagent":["partial","*",10,1],"cohere-ai":["partial","*",10,1],"Diffbot":["partial","*",10,1],"ContextFetchBot":["partial","*",10,1]},"llms":{"status":"missing","http":404,"bytes":109286},"home":{"status":"ok","http":200,"url":"https://automattic.com/","html_tokens":34772,"markdown_tokens":527,"pruned_tokens":109}},
{"rank":518,"domain":"www.gov.cn","origin":"https://www.gov.cn","robots":{"status":"ok","http":200,"url":"https://www.gov.cn/robots.txt","bytes":811,"groups":1},"bots":{"GPTBot":["partial","*",22,1],"ChatGPT-User":["partial","*",22,1],"OAI-SearchBot":["partial","*",22,1],"ClaudeBot":["partial","*",22,1],"Claude-User":["partial","*",22,1],"Claude-SearchBot":["partial","*",22,1],"anthropic-ai":["partial","*",22,1],"PerplexityBot":["partial","*",22,1],"Perplexity-User":["partial","*",22,1],"Google-Extended":["partial","*",22,1],"Applebot-Extended":["partial","*",22,1],"CCBot":["partial","*",22,1],"Bytespider":["partial","*",22,1],"Amazonbot":["partial","*",22,1],"meta-externalagent":["partial","*",22,1],"cohere-ai":["partial","*",22,1],"Diffbot":["partial","*",22,1],"ContextFetchBot":["partial","*",22,1]},"llms":{"status":"missing","http":404,"bytes":15556},"home":{"status":"ok","http":200,"url":"https://www.gov.cn/","html_tokens":15817,"markdown_tokens":16,"pruned_tokens":16}},
{"rank":519,"domain":"house.gov","origin":"https://house.gov","robots":{"status":"ok","http":200,"url":"https://www.house.gov/robots.txt","bytes":2141,"groups":1},"bots":{"GPTBot":["partial","*",35,22],"ChatGPT-User":["partial","*",35,22],"OAI-SearchBot":["partial","*",35,22],"ClaudeBot":["partial","*",35,22],"Claude-User":["partial","*",35,22],"Claude-SearchBot":["partial","*",35,22],"anthropic-ai":["partial","*",35,22],"PerplexityBot":["partial","*",35,22],"Perplexity-User":["partial","*",35,22],"Google-Extended":["partial","*",35,22],"Applebot-Extended":["partial","*",35,22],"CCBot":["partial","*",35,22],"Bytespider":["partial","*",35,22],"Amazonbot":["partial","*",35,22],"meta-externalagent":["partial","*",35,22],"cohere-ai":["partial","*",35,22],"Diffbot":["partial","*",35,22],"ContextFetchBot":["partial","*",35,22]},"llms":{"status":"missing","http":404,"bytes":22034},"home":{"status":"ok","http":200,"url":"https://www.house.gov/","html_tokens":43208,"markdown_tokens":142,"pruned_tokens":142}},
{"rank":520,"domain":"teamviewer.com","origin":"https://teamviewer.com","robots":{"status":"ok","http":200,"url":"https://www.teamviewer.com/robots.txt","bytes":581,"groups":1},"bots":{"GPTBot":["partial","*",18,0],"ChatGPT-User":["partial","*",18,0],"OAI-SearchBot":["partial","*",18,0],"ClaudeBot":["partial","*",18,0],"Claude-User":["partial","*",18,0],"Claude-SearchBot":["partial","*",18,0],"anthropic-ai":["partial","*",18,0],"PerplexityBot":["partial","*",18,0],"Perplexity-User":["partial","*",18,0],"Google-Extended":["partial","*",18,0],"Applebot-Extended":["partial","*",18,0],"CCBot":["partial","*",18,0],"Bytespider":["partial","*",18,0],"Amazonbot":["partial","*",18,0],"meta-externalagent":["partial","*",18,0],"cohere-ai":["partial","*",18,0],"Diffbot":["partial","*",18,0],"ContextFetchBot":["partial","*",18,0]},"llms":{"status":"missing","http":404,"bytes":244527},"home":{"status":"ok","http":200,"url":"https://www.teamviewer.com/en-us/","html_tokens":112355,"markdown_tokens":188,"pruned_tokens":24}},
{"rank":521,"domain":"service.gov.uk","origin":"https://service.gov.uk","robots":{"status":"html","http":200,"url":"https://www.gov.uk","bytes":85520,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":522,"domain":"pwc.com","origin":"https://pwc.com","robots":{"status":"refused","http":403,"url":"https://www.pwc.com/robots.txt","bytes":379,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":523,"domain":"teams.live.com","origin":"https://teams.live.com","robots":{"status":"ok","http":200,"url":"https://teams.live.com/robots.txt","bytes":240,"groups":1},"bots":{"GPTBot":["partial","*",2,0],"ChatGPT-User":["partial","*",2,0],"OAI-SearchBot":["partial","*",2,0],"ClaudeBot":["partial","*",2,0],"Claude-User":["partial","*",2,0],"Claude-SearchBot":["partial","*",2,0],"anthropic-ai":["partial","*",2,0],"PerplexityBot":["partial","*",2,0],"Perplexity-User":["partial","*",2,0],"Google-Extended":["partial","*",2,0],"Applebot-Extended":["partial","*",2,0],"CCBot":["partial","*",2,0],"Bytespider":["partial","*",2,0],"Amazonbot":["partial","*",2,0],"meta-externalagent":["partial","*",2,0],"cohere-ai":["partial","*",2,0],"Diffbot":["partial","*",2,0],"ContextFetchBot":["partial","*",2,0]},"llms":{"status":"missing","http":404,"bytes":0},"home":{"status":"ok","http":200,"url":"https://teams.live.com/v2/unsupported-browser#env=life&browserOverrides={%22firefox%22:{%22isSupported%22:true,%22minVersion%22:%2297.0%22},%22safari%22:{%22isSupported%22:true,%22minVersion%22:%2216.0%22}}","html_tokens":57939,"markdown_tokens":10,"pruned_tokens":10}},
{"rank":524,"domain":"spiegel.de","origin":"https://spiegel.de","robots":{"status":"ok","http":200,"url":"https://www.spiegel.de/robots.txt","bytes":6634,"groups":111},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["partial","named",1,8],"OAI-SearchBot":["partial","named",1,8],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","named",1,8],"Claude-SearchBot":["partial","named",1,8],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["partial","*",1,1],"Perplexity-User":["partial","*",1,1],"Google-Extended":["partial","*",1,1],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["partial","*",1,1],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",1,1]},"llms":{"status":"missing","http":404,"bytes":19180},"home":{"status":"ok","http":200,"url":"https://www.spiegel.de/","html_tokens":493263,"markdown_tokens":186,"pruned_tokens":174}},
{"rank":525,"domain":"govt.nz","origin":"https://www.govt.nz","robots":{"status":"missing","http":404,"url":"https://www.govt.nz/robots.txt","bytes":278,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":279},"home":{"status":"ok","http":200,"url":"https://www.govt.nz/","html_tokens":8528,"markdown_tokens":21,"pruned_tokens":21}},
{"rank":526,"domain":"laravel.com","origin":"https://laravel.com","robots":{"status":"ok","http":200,"url":"https://laravel.com/robots.txt","bytes":88,"groups":1},"bots":{"GPTBot":["partial","*",1,0],"ChatGPT-User":["partial","*",1,0],"OAI-SearchBot":["partial","*",1,0],"ClaudeBot":["partial","*",1,0],"Claude-User":["partial","*",1,0],"Claude-SearchBot":["partial","*",1,0],"anthropic-ai":["partial","*",1,0],"PerplexityBot":["partial","*",1,0],"Perplexity-User":["partial","*",1,0],"Google-Extended":["partial","*",1,0],"Applebot-Extended":["partial","*",1,0],"CCBot":["partial","*",1,0],"Bytespider":["partial","*",1,0],"Amazonbot":["partial","*",1,0],"meta-externalagent":["partial","*",1,0],"cohere-ai":["partial","*",1,0],"Diffbot":["partial","*",1,0],"ContextFetchBot":["partial","*",1,0]},"llms":{"status":"missing","http":404,"bytes":85910},"home":{"status":"ok","http":200,"url":"https://laravel.com/","html_tokens":135041,"markdown_tokens":415,"pruned_tokens":250}},
{"rank":527,"domain":"freebsd.org","origin":"https://freebsd.org","robots":{"status":"ok","http":200,"url":"https://www.freebsd.org/robots.txt","bytes":468,"groups":9},"bots":{"GPTBot":["partial","named",1,0],"ChatGPT-User":["partial","*",5,0],"OAI-SearchBot":["partial","*",5,0],"ClaudeBot":["partial","*",5,0],"Claude-User":["partial","*",5,0],"Claude-SearchBot":["partial","*",5,0],"anthropic-ai":["partial","*",5,0],"PerplexityBot":["partial","*",5,0],"Perplexity-User":["partial","*",5,0],"Google-Extended":["partial","*",5,0],"Applebot-Extended":["partial","*",5,0],"CCBot":["partial","named",1,0],"Bytespider":["partial","*",5,0],"Amazonbot":["partial","*",5,0],"meta-externalagent":["partial","*",5,0],"cohere-ai":["partial","*",5,0],"Diffbot":["partial","*",5,0],"ContextFetchBot":["partial","*",5,0]},"llms":{"status":"missing","http":404,"bytes":14140},"home":{"status":"ok","http":200,"url":"https://www.freebsd.org/","html_tokens":5507,"markdown_tokens":553,"pruned_tokens":142}},
{"rank":528,"domain":"web.mit.edu","origin":"https://web.mit.edu","robots":{"status":"ok","http":200,"url":"https://web.mit.edu/robots.txt","bytes":199,"groups":1},"bots":{"GPTBot":["partial","*",10,0],"ChatGPT-User":["partial","*",10,0],"OAI-SearchBot":["partial","*",10,0],"ClaudeBot":["partial","*",10,0],"Claude-User":["partial","*",10,0],"Claude-SearchBot":["partial","*",10,0],"anthropic-ai":["partial","*",10,0],"PerplexityBot":["partial","*",10,0],"Perplexity-User":["partial","*",10,0],"Google-Extended":["partial","*",10,0],"Applebot-Extended":["partial","*",10,0],"CCBot":["partial","*",10,0],"Bytespider":["partial","*",10,0],"Amazonbot":["partial","*",10,0],"meta-externalagent":["partial","*",10,0],"cohere-ai":["partial","*",10,0],"Diffbot":["partial","*",10,0],"ContextFetchBot":["partial","*",10,0]},"llms":{"status":"missing","http":404,"bytes":25731},"home":{"status":"ok","http":200,"url":"https://web.mit.edu/","html_tokens":11988,"markdown_tokens":69,"pruned_tokens":69}},
{"rank":529,"domain":"airbnb.com","origin":"https://airbnb.com","robots":{"status":"ok","http":200,"url":"https://www.airbnb.com/robots.txt","bytes":20930,"groups":18},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["partial","named",61,2],"OAI-SearchBot":["partial","named",61,2],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",61,2],"Claude-SearchBot":["partial","*",61,2],"anthropic-ai":["partial","named",63,2],"PerplexityBot":["partial","named",63,2],"Perplexity-User":["partial","*",61,2],"Google-Extended":["partial","*",61,2],"Applebot-Extended":["blocked","named",1,0],"CCBot":["partial","*",61,2],"Bytespider":["partial","*",61,2],"Amazonbot":["partial","*",61,2],"meta-externalagent":["partial","named",61,2],"cohere-ai":["partial","named",66,2],"Diffbot":["partial","*",61,2],"ContextFetchBot":["partial","*",61,2]},"llms":{"status":"missing","http":404,"bytes":2947},"home":{"status":"ok","http":200,"url":"https://www.airbnb.com/","html_tokens":170812,"markdown_tokens":804,"pruned_tokens":175}},
{"rank":530,"domain":"snapchat.com","origin":"https://snapchat.com","robots":{"status":"ok","http":200,"url":"https://www.snapchat.com/robots.txt","bytes":3997,"groups":51},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["partial","named",2,1],"OAI-SearchBot":["partial","named",2,1],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","named",2,1],"Claude-SearchBot":["partial","named",2,1],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["partial","named",2,1],"Perplexity-User":["partial","named",2,1],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",4,0]},"llms":{"status":"missing","http":404,"bytes":66311},"home":{"status":"ok","http":200,"url":"https://www.snapchat.com/","html_tokens":32634,"markdown_tokens":55,"pruned_tokens":55}},
{"rank":531,"domain":"yahoo.co.jp","origin":"https://yahoo.co.jp","robots":{"status":"ok","http":200,"url":"https://www.yahoo.co.jp/robots.txt","bytes":87,"groups":3},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"html","http":200,"bytes":33488},"home":{"status":"ok","http":200,"url":"https://www.yahoo.co.jp/","html_tokens":7975,"markdown_tokens":53,"pruned_tokens":34}},
{"rank":532,"domain":"fedoraproject.org","origin":"https://fedoraproject.org","robots":{"status":"ok","http":200,"url":"https://fedoraproject.org/robots.txt","bytes":185,"groups":1},"bots":{"GPTBot":["partial","*",2,0],"ChatGPT-User":["partial","*",2,0],"OAI-SearchBot":["partial","*",2,0],"ClaudeBot":["partial","*",2,0],"Claude-User":["partial","*",2,0],"Claude-SearchBot":["partial","*",2,0],"anthropic-ai":["partial","*",2,0],"PerplexityBot":["partial","*",2,0],"Perplexity-User":["partial","*",2,0],"Google-Extended":["partial","*",2,0],"Applebot-Extended":["partial","*",2,0],"CCBot":["partial","*",2,0],"Bytespider":["partial","*",2,0],"Amazonbot":["partial","*",2,0],"meta-externalagent":["partial","*",2,0],"cohere-ai":["partial","*",2,0],"Diffbot":["partial","*",2,0],"ContextFetchBot":["partial","*",2,0]},"llms":{"status":"missing","http":404,"bytes":236769},"home":{"status":"ok","http":200,"url":"https://fedoraproject.org/","html_tokens":65371,"markdown_tokens":201,"pruned_tokens":144}},
{"rank":533,"domain":"telegra.ph","origin":"https://telegra.ph","robots":{"status":"missing","http":404,"url":"https://telegra.ph/robots.txt","bytes":951,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":951},"home":{"status":"ok","http":200,"url":"https://telegra.ph/","html_tokens":1144,"markdown_tokens":0,"pruned_tokens":0}},
{"rank":534,"domain":"clevelandclinic.org","origin":"https://clevelandclinic.org","robots":{"status":"ok","http":200,"url":"https://my.clevelandclinic.org/robots.txt","bytes":1064,"groups":2},"bots":{"GPTBot":["partial","*",23,0],"ChatGPT-User":["partial","*",23,0],"OAI-SearchBot":["partial","*",23,0],"ClaudeBot":["partial","*",23,0],"Claude-User":["partial","*",23,0],"Claude-SearchBot":["partial","*",23,0],"anthropic-ai":["partial","*",23,0],"PerplexityBot":["partial","*",23,0],"Perplexity-User":["partial","*",23,0],"Google-Extended":["partial","*",23,0],"Applebot-Extended":["partial","*",23,0],"CCBot":["partial","*",23,0],"Bytespider":["partial","*",23,0],"Amazonbot":["partial","*",23,0],"meta-externalagent":["partial","*",23,0],"cohere-ai":["partial","*",23,0],"Diffbot":["partial","*",23,0],"ContextFetchBot":["partial","*",23,0]},"llms":{"status":"present","http":200,"bytes":1925},"home":{"status":"ok","http":200,"url":"https://my.clevelandclinic.org/","html_tokens":75134,"markdown_tokens":135,"pruned_tokens":48}},
{"rank":535,"domain":"visualstudio.com","origin":"https://visualstudio.com","robots":{"status":"ok","http":200,"url":"https://visualstudio.microsoft.com/robots.txt","bytes":234,"groups":1},"bots":{"GPTBot":["partial","*",7,0],"ChatGPT-User":["partial","*",7,0],"OAI-SearchBot":["partial","*",7,0],"ClaudeBot":["partial","*",7,0],"Claude-User":["partial","*",7,0],"Claude-SearchBot":["partial","*",7,0],"anthropic-ai":["partial","*",7,0],"PerplexityBot":["partial","*",7,0],"Perplexity-User":["partial","*",7,0],"Google-Extended":["partial","*",7,0],"Applebot-Extended":["partial","*",7,0],"CCBot":["partial","*",7,0],"Bytespider":["partial","*",7,0],"Amazonbot":["partial","*",7,0],"meta-externalagent":["partial","*",7,0],"cohere-ai":["partial","*",7,0],"Diffbot":["partial","*",7,0],"ContextFetchBot":["partial","*",7,0]},"llms":{"status":"missing","http":404,"bytes":146},"home":{"status":"ok","http":200,"url":"https://visualstudio.microsoft.com/","html_tokens":85400,"markdown_tokens":154,"pruned_tokens":123}},
{"rank":536,"domain":"npmjs.com","origin":"https://npmjs.com","robots":{"status":"refused","http":403,"url":"https://www.npmjs.com/robots.txt","bytes":5594,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":537,"domain":"generatepress.com","origin":"https://generatepress.com","robots":{"status":"ok","http":200,"url":"https://generatepress.com/robots.txt","bytes":463,"groups":3},"bots":{"GPTBot":["partial","*",4,0],"ChatGPT-User":["partial","*",4,0],"OAI-SearchBot":["partial","*",4,0],"ClaudeBot":["partial","*",4,0],"Claude-User":["partial","*",4,0],"Claude-SearchBot":["partial","*",4,0],"anthropic-ai":["partial","*",4,0],"PerplexityBot":["partial","*",4,0],"Perplexity-User":["partial","*",4,0],"Google-Extended":["partial","*",4,0],"Applebot-Extended":["partial","*",4,0],"CCBot":["partial","*",4,0],"Bytespider":["partial","*",4,0],"Amazonbot":["partial","*",4,0],"meta-externalagent":["partial","*",4,0],"cohere-ai":["partial","*",4,0],"Diffbot":["partial","*",4,0],"ContextFetchBot":["partial","*",4,0]},"llms":{"status":"missing","http":404,"bytes":77160},"home":{"status":"ok","http":200,"url":"https://generatepress.com/","html_tokens":35421,"markdown_tokens":854,"pruned_tokens":480}},
{"rank":538,"domain":"siemens.com","origin":"https://siemens.com","robots":{"status":"ok","http":200,"url":"https://www.siemens.com/robots.txt","bytes":130,"groups":1},"bots":{"GPTBot":["partial","*",3,0],"ChatGPT-User":["partial","*",3,0],"OAI-SearchBot":["partial","*",3,0],"ClaudeBot":["partial","*",3,0],"Claude-User":["partial","*",3,0],"Claude-SearchBot":["partial","*",3,0],"anthropic-ai":["partial","*",3,0],"PerplexityBot":["partial","*",3,0],"Perplexity-User":["partial","*",3,0],"Google-Extended":["partial","*",3,0],"Applebot-Extended":["partial","*",3,0],"CCBot":["partial","*",3,0],"Bytespider":["partial","*",3,0],"Amazonbot":["partial","*",3,0],"meta-externalagent":["partial","*",3,0],"cohere-ai":["partial","*",3,0],"Diffbot":["partial","*",3,0],"ContextFetchBot":["partial","*",3,0]},"llms":{"status":"present","http":200,"bytes":2737},"home":{"status":"ok","http":200,"url":"https://www.siemens.com/en-us/","html_tokens":73105,"markdown_tokens":151,"pruned_tokens":92}},
{"rank":539,"domain":"census.gov","origin":"https://census.gov","robots":{"status":"ok","http":200,"url":"https://www.census.gov/robots.txt","bytes":1206,"groups":6},"bots":{"GPTBot":["partial","*",1,0],"ChatGPT-User":["partial","*",1,0],"OAI-SearchBot":["partial","*",1,0],"ClaudeBot":["partial","*",1,0],"Claude-User":["partial","*",1,0],"Claude-SearchBot":["partial","*",1,0],"anthropic-ai":["partial","*",1,0],"PerplexityBot":["partial","*",1,0],"Perplexity-User":["partial","*",1,0],"Google-Extended":["partial","*",1,0],"Applebot-Extended":["partial","*",1,0],"CCBot":["partial","*",1,0],"Bytespider":["partial","*",1,0],"Amazonbot":["partial","*",1,0],"meta-externalagent":["partial","*",1,0],"cohere-ai":["partial","*",1,0],"Diffbot":["partial","*",1,0],"ContextFetchBot":["partial","*",1,0]},"llms":{"status":"missing","http":404,"bytes":26134},"home":{"status":"ok","http":200,"url":"https://www.census.gov/","html_tokens":97321,"markdown_tokens":64,"pruned_tokens":64}},
{"rank":540,"domain":"amazon.co.jp","origin":"https://amazon.co.jp","robots":{"status":"ok","http":200,"url":"https://www.amazon.co.jp/robots.txt","bytes":10234,"groups":101},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["blocked","named",1,0],"OAI-SearchBot":["blocked","named",1,0],"ClaudeBot":["blocked","named",2,0],"Claude-User":["blocked","named",1,0],"Claude-SearchBot":["blocked","named",1,0],"anthropic-ai":["partial","*",176,17],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["blocked","named",1,0],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["partial","*",176,17],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["partial","*",176,17],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",176,17]},"llms":{"status":"missing","http":404,"bytes":2166},"home":{"status":"ok","http":200,"url":"https://www.amazon.co.jp/-/en/","html_tokens":166796,"markdown_tokens":71,"pruned_tokens":27}},
{"rank":541,"domain":"nodejs.org","origin":"https://nodejs.org","robots":{"status":"ok","http":200,"url":"https://nodejs.org/robots.txt","bytes":199,"groups":1},"bots":{"GPTBot":["partial","*",2,3],"ChatGPT-User":["partial","*",2,3],"OAI-SearchBot":["partial","*",2,3],"ClaudeBot":["partial","*",2,3],"Claude-User":["partial","*",2,3],"Claude-SearchBot":["partial","*",2,3],"anthropic-ai":["partial","*",2,3],"PerplexityBot":["partial","*",2,3],"Perplexity-User":["partial","*",2,3],"Google-Extended":["partial","*",2,3],"Applebot-Extended":["partial","*",2,3],"CCBot":["partial","*",2,3],"Bytespider":["partial","*",2,3],"Amazonbot":["partial","*",2,3],"meta-externalagent":["partial","*",2,3],"cohere-ai":["partial","*",2,3],"Diffbot":["partial","*",2,3],"ContextFetchBot":["partial","*",2,3]},"llms":{"status":"present","http":200,"bytes":15010},"home":{"status":"ok","http":200,"url":"https://nodejs.org/en","html_tokens":132910,"markdown_tokens":526,"pruned_tokens":141}},
{"rank":542,"domain":"web.dev","origin":"https://web.dev","robots":{"status":"ok","http":200,"url":"https://web.dev/robots.txt","bytes":61,"groups":1},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"missing","http":404,"bytes":64320},"home":{"status":"ok","http":200,"url":"https://web.dev/","html_tokens":33121,"markdown_tokens":337,"pruned_tokens":63}},
{"rank":543,"domain":"timesofindia.indiatimes.com","origin":"https://timesofindia.indiatimes.com","robots":{"status":"ok","http":200,"url":"https://timesofindia.indiatimes.com/robots.txt","bytes":11377,"groups":7},"bots":{"GPTBot":["partial","*",167,1],"ChatGPT-User":["partial","*",167,1],"OAI-SearchBot":["partial","*",167,1],"ClaudeBot":["partial","*",167,1],"Claude-User":["partial","*",167,1],"Claude-SearchBot":["partial","*",167,1],"anthropic-ai":["partial","*",167,1],"PerplexityBot":["partial","*",167,1],"Perplexity-User":["partial","*",167,1],"Google-Extended":["partial","*",167,1],"Applebot-Extended":["partial","*",167,1],"CCBot":["partial","*",167,1],"Bytespider":["partial","*",167,1],"Amazonbot":["partial","*",167,1],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["partial","*",167,1],"Diffbot":["partial","*",167,1],"ContextFetchBot":["partial","*",167,1]},"llms":{"status":"present","http":200,"bytes":3589},"home":{"status":"ok","http":200,"url":"https://timesofindia.indiatimes.com/us","html_tokens":187507,"markdown_tokens":141,"pruned_tokens":141}},
{"rank":544,"domain":"energy.gov","origin":"https://www.energy.gov","robots":{"status":"ok","http":200,"url":"https://www.energy.gov/robots.txt","bytes":2141,"groups":1},"bots":{"GPTBot":["partial","*",35,22],"ChatGPT-User":["partial","*",35,22],"OAI-SearchBot":["partial","*",35,22],"ClaudeBot":["partial","*",35,22],"Claude-User":["partial","*",35,22],"Claude-SearchBot":["partial","*",35,22],"anthropic-ai":["partial","*",35,22],"PerplexityBot":["partial","*",35,22],"Perplexity-User":["partial","*",35,22],"Google-Extended":["partial","*",35,22],"Applebot-Extended":["partial","*",35,22],"CCBot":["partial","*",35,22],"Bytespider":["partial","*",35,22],"Amazonbot":["partial","*",35,22],"meta-externalagent":["partial","*",35,22],"cohere-ai":["partial","*",35,22],"Diffbot":["partial","*",35,22],"ContextFetchBot":["partial","*",35,22]},"llms":{"status":"missing","http":404,"bytes":35385},"home":{"status":"ok","http":200,"url":"https://www.energy.gov/","html_tokens":32660,"markdown_tokens":194,"pruned_tokens":80}},
{"rank":545,"domain":"buzzfeed.com","origin":"https://buzzfeed.com","robots":{"status":"ok","http":200,"url":"https://www.buzzfeed.com/robots.txt","bytes":4496,"groups":34},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["partial","*",25,0],"OAI-SearchBot":["partial","*",25,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",25,0],"Claude-SearchBot":["partial","*",25,0],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["blocked","named",1,0],"Google-Extended":["blocked","named",2,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["partial","named",1,1],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",25,0]},"llms":{"status":"missing","http":404,"bytes":12927},"home":{"status":"ok","http":200,"url":"https://www.buzzfeed.com/","html_tokens":501358,"markdown_tokens":56,"pruned_tokens":56}},
{"rank":546,"domain":"guardian.co.uk","origin":"https://guardian.co.uk","robots":{"status":"ok","http":200,"url":"https://www.theguardian.com/robots.txt","bytes":2873,"groups":3},"bots":{"GPTBot":["partial","*",48,0],"ChatGPT-User":["partial","*",48,0],"OAI-SearchBot":["partial","*",48,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["blocked","named",1,0],"Claude-SearchBot":["blocked","named",1,0],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["partial","*",48,0],"Google-Extended":["partial","*",48,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["partial","*",48,0],"Diffbot":["partial","*",48,0],"ContextFetchBot":["partial","*",48,0]},"llms":{"status":"missing","http":404,"bytes":24345},"home":{"status":"ok","http":200,"url":"https://www.theguardian.com/us","html_tokens":349444,"markdown_tokens":102,"pruned_tokens":102}},
{"rank":547,"domain":"opensource.org","origin":"https://opensource.org","robots":{"status":"ok","http":200,"url":"https://opensource.org/robots.txt","bytes":160,"groups":1},"bots":{"GPTBot":["partial","*",1,1],"ChatGPT-User":["partial","*",1,1],"OAI-SearchBot":["partial","*",1,1],"ClaudeBot":["partial","*",1,1],"Claude-User":["partial","*",1,1],"Claude-SearchBot":["partial","*",1,1],"anthropic-ai":["partial","*",1,1],"PerplexityBot":["partial","*",1,1],"Perplexity-User":["partial","*",1,1],"Google-Extended":["partial","*",1,1],"Applebot-Extended":["partial","*",1,1],"CCBot":["partial","*",1,1],"Bytespider":["partial","*",1,1],"Amazonbot":["partial","*",1,1],"meta-externalagent":["partial","*",1,1],"cohere-ai":["partial","*",1,1],"Diffbot":["partial","*",1,1],"ContextFetchBot":["partial","*",1,1]},"llms":{"status":"missing","http":404,"bytes":190385},"home":{"status":"ok","http":200,"url":"https://opensource.org/","html_tokens":56363,"markdown_tokens":230,"pruned_tokens":161}},
{"rank":548,"domain":"vox.com","origin":"https://vox.com","robots":{"status":"ok","http":200,"url":"https://www.vox.com/robots.txt","bytes":7548,"groups":106},"bots":{"GPTBot":["allowed","named",0,1],"ChatGPT-User":["partial","named",1,2],"OAI-SearchBot":["partial","*",15,2],"ClaudeBot":["partial","named",1,2],"Claude-User":["partial","named",1,2],"Claude-SearchBot":["partial","named",1,2],"anthropic-ai":["partial","named",1,2],"PerplexityBot":["partial","named",1,2],"Perplexity-User":["partial","named",1,2],"Google-Extended":["partial","named",1,2],"Applebot-Extended":["partial","named",1,2],"CCBot":["partial","named",1,2],"Bytespider":["partial","named",1,2],"Amazonbot":["allowed","named",0,1],"meta-externalagent":["partial","named",1,2],"cohere-ai":["partial","named",1,2],"Diffbot":["partial","named",1,2],"ContextFetchBot":["partial","*",15,2]},"llms":{"status":"missing","http":404,"bytes":53166},"home":{"status":"ok","http":200,"url":"https://www.vox.com/","html_tokens":260850,"markdown_tokens":453,"pruned_tokens":145}},
{"rank":549,"domain":"withgoogle.com","origin":"https://withgoogle.com","robots":{"status":"html","http":200,"url":"https://www.google.com/","bytes":85216,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":550,"domain":"box.com","origin":"https://box.com","robots":{"status":"ok","http":200,"url":"https://www.box.com/robots.txt","bytes":1484,"groups":1},"bots":{"GPTBot":["partial","*",28,18],"ChatGPT-User":["partial","*",28,18],"OAI-SearchBot":["partial","*",28,18],"ClaudeBot":["partial","*",28,18],"Claude-User":["partial","*",28,18],"Claude-SearchBot":["partial","*",28,18],"anthropic-ai":["partial","*",28,18],"PerplexityBot":["partial","*",28,18],"Perplexity-User":["partial","*",28,18],"Google-Extended":["partial","*",28,18],"Applebot-Extended":["partial","*",28,18],"CCBot":["partial","*",28,18],"Bytespider":["partial","*",28,18],"Amazonbot":["partial","*",28,18],"meta-externalagent":["partial","*",28,18],"cohere-ai":["partial","*",28,18],"Diffbot":["partial","*",28,18],"ContextFetchBot":["partial","*",28,18]},"llms":{"status":"http_error","http":403,"bytes":5542},"home":{"status":"ok","http":200,"url":"https://www.box.com/","html_tokens":126998,"markdown_tokens":649,"pruned_tokens":88}},
{"rank":551,"domain":"netscape.com","origin":"https://netscape.com","robots":{"status":"html","http":200,"url":"https://www.aol.com/","bytes":1401937,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":552,"domain":"ok.ru","origin":"https://ok.ru","robots":{"status":"ok","http":200,"url":"https://ok.ru/robots.txt","bytes":6051,"groups":1},"bots":{"GPTBot":["partial","*",34,0],"ChatGPT-User":["partial","*",34,0],"OAI-SearchBot":["partial","*",34,0],"ClaudeBot":["partial","*",34,0],"Claude-User":["partial","*",34,0],"Claude-SearchBot":["partial","*",34,0],"anthropic-ai":["partial","*",34,0],"PerplexityBot":["partial","*",34,0],"Perplexity-User":["partial","*",34,0],"Google-Extended":["partial","*",34,0],"Applebot-Extended":["partial","*",34,0],"CCBot":["partial","*",34,0],"Bytespider":["partial","*",34,0],"Amazonbot":["partial","*",34,0],"meta-externalagent":["partial","*",34,0],"cohere-ai":["partial","*",34,0],"Diffbot":["partial","*",34,0],"ContextFetchBot":["partial","*",34,0]},"llms":{"status":"missing","http":404,"bytes":0},"home":{"status":"ok","http":200,"url":"https://ok.ru/","html_tokens":64451,"markdown_tokens":45,"pruned_tokens":30}},
{"rank":553,"domain":"moodle.com","origin":"https://moodle.com","robots":{"status":"ok","http":200,"url":"https://moodle.com/robots.txt","bytes":70,"groups":1},"bots":{"GPTBot":["allowed","*",0,1],"ChatGPT-User":["allowed","*",0,1],"OAI-SearchBot":["allowed","*",0,1],"ClaudeBot":["allowed","*",0,1],"Claude-User":["allowed","*",0,1],"Claude-SearchBot":["allowed","*",0,1],"anthropic-ai":["allowed","*",0,1],"PerplexityBot":["allowed","*",0,1],"Perplexity-User":["allowed","*",0,1],"Google-Extended":["allowed","*",0,1],"Applebot-Extended":["allowed","*",0,1],"CCBot":["allowed","*",0,1],"Bytespider":["allowed","*",0,1],"Amazonbot":["allowed","*",0,1],"meta-externalagent":["allowed","*",0,1],"cohere-ai":["allowed","*",0,1],"Diffbot":["allowed","*",0,1],"ContextFetchBot":["allowed","*",0,1]},"llms":{"status":"missing","http":404,"bytes":60},"home":{"status":"ok","http":200,"url":"https://moodle.com/","html_tokens":35264,"markdown_tokens":354,"pruned_tokens":113}},
{"rank":554,"domain":"nyu.edu","origin":"https://www.nyu.edu","robots":{"status":"ok","http":200,"url":"https://www.nyu.edu/robots.txt","bytes":3187,"groups":51},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["blocked","named",1,0],"OAI-SearchBot":["partial","*",19,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",19,0],"Claude-SearchBot":["partial","*",19,0],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["partial","*",19,0],"Perplexity-User":["partial","*",19,0],"Google-Extended":["partial","named",4,0],"Applebot-Extended":["partial","*",19,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["partial","*",19,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",19,0]},"llms":{"status":"missing","http":404,"bytes":33616},"home":{"status":"ok","http":200,"url":"https://www.nyu.edu/","html_tokens":25629,"markdown_tokens":662,"pruned_tokens":132}},
{"rank":555,"domain":"gv.at","origin":"https://gv.at","robots":{"status":"unreachable","http":null,"url":"https://gv.at/robots.txt","bytes":0,"groups":0,"error":"cannot resolve gv.at"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":556,"domain":"disqus.com","origin":"https://disqus.com","robots":{"status":"ok","http":200,"url":"https://disqus.com/robots.txt","bytes":49,"groups":1},"bots":{"GPTBot":["partial","*",2,0],"ChatGPT-User":["partial","*",2,0],"OAI-SearchBot":["partial","*",2,0],"ClaudeBot":["partial","*",2,0],"Claude-User":["partial","*",2,0],"Claude-SearchBot":["partial","*",2,0],"anthropic-ai":["partial","*",2,0],"PerplexityBot":["partial","*",2,0],"Perplexity-User":["partial","*",2,0],"Google-Extended":["partial","*",2,0],"Applebot-Extended":["partial","*",2,0],"CCBot":["partial","*",2,0],"Bytespider":["partial","*",2,0],"Amazonbot":["partial","*",2,0],"meta-externalagent":["partial","*",2,0],"cohere-ai":["partial","*",2,0],"Diffbot":["partial","*",2,0],"ContextFetchBot":["partial","*",2,0]},"llms":{"status":"missing","http":404,"bytes":5108},"home":{"status":"ok","http":200,"url":"https://disqus.com/","html_tokens":1217,"markdown_tokens":45,"pruned_tokens":45}},
{"rank":557,"domain":"cctv.com","origin":"https://cctv.com","robots":{"status":"ok","http":200,"url":"https://www.cctv.com/robots.txt","bytes":9053,"groups":1},"bots":{"GPTBot":["partial","*",363,0],"ChatGPT-User":["partial","*",363,0],"OAI-SearchBot":["partial","*",363,0],"ClaudeBot":["partial","*",363,0],"Claude-User":["partial","*",363,0],"Claude-SearchBot":["partial","*",363,0],"anthropic-ai":["partial","*",363,0],"PerplexityBot":["partial","*",363,0],"Perplexity-User":["partial","*",363,0],"Google-Extended":["partial","*",363,0],"Applebot-Extended":["partial","*",363,0],"CCBot":["partial","*",363,0],"Bytespider":["partial","*",363,0],"Amazonbot":["partial","*",363,0],"meta-externalagent":["partial","*",363,0],"cohere-ai":["partial","*",363,0],"Diffbot":["partial","*",363,0],"ContextFetchBot":["partial","*",363,0]},"llms":{"status":"timeout","error":"timed out after 10 s"},"home":{"status":"ok","http":200,"url":"https://www.cctv.com/","html_tokens":75117,"markdown_tokens":82,"pruned_tokens":82}},
{"rank":558,"domain":"shorturl.at","origin":"https://shorturl.at","robots":{"status":"missing","http":404,"url":"https://www.shorturl.at/robots.txt","bytes":14110,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":14110},"home":{"status":"http_error","http":403,"url":"https://shorturl.at/"}},
{"rank":559,"domain":"ionos.de","origin":"https://ionos.de","robots":{"status":"ok","http":200,"url":"https://www.ionos.de/robots.txt","bytes":449,"groups":2},"bots":{"GPTBot":["partial","*",8,1],"ChatGPT-User":["partial","*",8,1],"OAI-SearchBot":["partial","*",8,1],"ClaudeBot":["partial","*",8,1],"Claude-User":["partial","*",8,1],"Claude-SearchBot":["partial","*",8,1],"anthropic-ai":["partial","*",8,1],"PerplexityBot":["partial","*",8,1],"Perplexity-User":["partial","*",8,1],"Google-Extended":["partial","*",8,1],"Applebot-Extended":["partial","*",8,1],"CCBot":["partial","*",8,1],"Bytespider":["partial","*",8,1],"Amazonbot":["partial","*",8,1],"meta-externalagent":["partial","*",8,1],"cohere-ai":["partial","*",8,1],"Diffbot":["partial","*",8,1],"ContextFetchBot":["partial","*",8,1]},"llms":{"status":"missing","http":404,"bytes":740637},"home":{"status":"ok","http":200,"url":"https://www.ionos.de/","html_tokens":233083,"markdown_tokens":138,"pruned_tokens":115}},
{"rank":560,"domain":"jquery.com","origin":"https://jquery.com","robots":{"status":"error","http":null,"url":"https://jquery.com/robots.txt","bytes":0,"groups":0,"error":"more than 5 redirects"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":561,"domain":"strato.de","origin":"https://strato.de","robots":{"status":"ok","http":200,"url":"https://www.strato.de/robots.txt","bytes":199,"groups":1},"bots":{"GPTBot":["partial","*",2,1],"ChatGPT-User":["partial","*",2,1],"OAI-SearchBot":["partial","*",2,1],"ClaudeBot":["partial","*",2,1],"Claude-User":["partial","*",2,1],"Claude-SearchBot":["partial","*",2,1],"anthropic-ai":["partial","*",2,1],"PerplexityBot":["partial","*",2,1],"Perplexity-User":["partial","*",2,1],"Google-Extended":["partial","*",2,1],"Applebot-Extended":["partial","*",2,1],"CCBot":["partial","*",2,1],"Bytespider":["partial","*",2,1],"Amazonbot":["partial","*",2,1],"meta-externalagent":["partial","*",2,1],"cohere-ai":["partial","*",2,1],"Diffbot":["partial","*",2,1],"ContextFetchBot":["partial","*",2,1]},"llms":{"status":"missing","http":404,"bytes":82080},"home":{"status":"ok","http":200,"url":"https://www.strato.de/","html_tokens":34640,"markdown_tokens":634,"pruned_tokens":163}},
{"rank":562,"domain":"bigcartel.com","origin":"https://bigcartel.com","robots":{"status":"ok","http":200,"url":"https://www.bigcartel.com/robots.txt","bytes":62,"groups":1},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"missing","http":404,"bytes":334811},"home":{"status":"ok","http":200,"url":"https://www.bigcartel.com/","html_tokens":131694,"markdown_tokens":471,"pruned_tokens":145}},
{"rank":563,"domain":"marketwatch.com","origin":"https://marketwatch.com","robots":{"status":"ok","http":200,"url":"https://www.marketwatch.com/robots.txt","bytes":4380,"groups":3},"bots":{"GPTBot":["partial","named",53,1],"ChatGPT-User":["partial","named",53,1],"OAI-SearchBot":["partial","named",53,1],"ClaudeBot":["blocked","*",1,0],"Claude-User":["blocked","*",1,0],"Claude-SearchBot":["blocked","*",1,0],"anthropic-ai":["blocked","*",1,0],"PerplexityBot":["blocked","*",1,0],"Perplexity-User":["blocked","*",1,0],"Google-Extended":["blocked","*",1,0],"Applebot-Extended":["blocked","*",1,0],"CCBot":["blocked","*",1,0],"Bytespider":["blocked","*",1,0],"Amazonbot":["blocked","*",1,0],"meta-externalagent":["blocked","*",1,0],"cohere-ai":["blocked","*",1,0],"Diffbot":["blocked","*",1,0],"ContextFetchBot":["blocked","*",1,0]},"llms":{"status":"disallowed"},"home":{"status":"disallowed"}},
{"rank":564,"domain":"doubleclick.net","origin":"https://doubleclick.net","robots":{"status":"ok","http":200,"url":"https://doubleclick.net/robots.txt","bytes":6624,"groups":4},"bots":{"GPTBot":["partial","*",169,64],"ChatGPT-User":["partial","*",169,64],"OAI-SearchBot":["partial","*",169,64],"ClaudeBot":["partial","*",169,64],"Claude-User":["partial","*",169,64],"Claude-SearchBot":["partial","*",169,64],"anthropic-ai":["partial","*",169,64],"PerplexityBot":["partial","*",169,64],"Perplexity-User":["partial","*",169,64],"Google-Extended":["partial","*",169,64],"Applebot-Extended":["partial","*",169,64],"CCBot":["partial","*",169,64],"Bytespider":["partial","*",169,64],"Amazonbot":["partial","*",169,64],"meta-externalagent":["partial","*",169,64],"cohere-ai":["partial","*",169,64],"Diffbot":["partial","*",169,64],"ContextFetchBot":["partial","*",169,64]},"llms":{"status":"missing","http":404,"bytes":1569},"home":{"status":"ok","http":200,"url":"https://marketingplatform.google.com/about/enterprise/","html_tokens":31309,"markdown_tokens":821,"pruned_tokens":171}},
{"rank":565,"domain":"jimdofree.com","origin":"https://www.jimdofree.com","robots":{"status":"refused","http":403,"url":"https://www.jimdofree.com/robots.txt","bytes":3553,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":566,"domain":"inc.com","origin":"https://inc.com","robots":{"status":"ok","http":200,"url":"https://www.inc.com/robots.txt","bytes":2070,"groups":38},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["partial","named",1,1],"OAI-SearchBot":["partial","named",1,1],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","named",1,1],"Claude-SearchBot":["partial","named",1,1],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["partial","named",1,1],"Google-Extended":["partial","named",1,1],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",2,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["partial","named",1,1],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",1,1]},"llms":{"status":"http_error","http":403,"bytes":767},"home":{"status":"http_error","http":403,"url":"https://www.inc.com/"}},
{"rank":567,"domain":"google.fr","origin":"https://google.fr","robots":{"status":"ok","http":200,"url":"https://www.google.fr/robots.txt","bytes":6624,"groups":4},"bots":{"GPTBot":["partial","*",169,64],"ChatGPT-User":["partial","*",169,64],"OAI-SearchBot":["partial","*",169,64],"ClaudeBot":["partial","*",169,64],"Claude-User":["partial","*",169,64],"Claude-SearchBot":["partial","*",169,64],"anthropic-ai":["partial","*",169,64],"PerplexityBot":["partial","*",169,64],"Perplexity-User":["partial","*",169,64],"Google-Extended":["partial","*",169,64],"Applebot-Extended":["partial","*",169,64],"CCBot":["partial","*",169,64],"Bytespider":["partial","*",169,64],"Amazonbot":["partial","*",169,64],"meta-externalagent":["partial","*",169,64],"cohere-ai":["partial","*",169,64],"Diffbot":["partial","*",169,64],"ContextFetchBot":["partial","*",169,64]},"llms":{"status":"missing","http":404,"bytes":1569},"home":{"status":"ok","http":200,"url":"https://www.google.com/","html_tokens":21352,"markdown_tokens":30,"pruned_tokens":30}},
{"rank":568,"domain":"kde.org","origin":"https://kde.org","robots":{"status":"ok","http":200,"url":"https://kde.org/robots.txt","bytes":150,"groups":1},"bots":{"GPTBot":["partial","*",2,0],"ChatGPT-User":["partial","*",2,0],"OAI-SearchBot":["partial","*",2,0],"ClaudeBot":["partial","*",2,0],"Claude-User":["partial","*",2,0],"Claude-SearchBot":["partial","*",2,0],"anthropic-ai":["partial","*",2,0],"PerplexityBot":["partial","*",2,0],"Perplexity-User":["partial","*",2,0],"Google-Extended":["partial","*",2,0],"Applebot-Extended":["partial","*",2,0],"CCBot":["partial","*",2,0],"Bytespider":["partial","*",2,0],"Amazonbot":["partial","*",2,0],"meta-externalagent":["partial","*",2,0],"cohere-ai":["partial","*",2,0],"Diffbot":["partial","*",2,0],"ContextFetchBot":["partial","*",2,0]},"llms":{"status":"missing","http":404,"bytes":37278},"home":{"status":"ok","http":200,"url":"https://kde.org/","html_tokens":13607,"markdown_tokens":600,"pruned_tokens":275}},
{"rank":569,"domain":"variety.com","origin":"https://variety.com","robots":{"status":"ok","http":200,"url":"https://variety.com/robots.txt","bytes":1869,"groups":26},"bots":{"GPTBot":["partial","named",1,1],"ChatGPT-User":["partial","named",1,1],"OAI-SearchBot":["partial","named",1,1],"ClaudeBot":["partial","named",1,1],"Claude-User":["partial","*",11,1],"Claude-SearchBot":["partial","*",11,1],"anthropic-ai":["partial","named",1,1],"PerplexityBot":["partial","named",1,1],"Perplexity-User":["partial","*",11,1],"Google-Extended":["partial","named",1,1],"Applebot-Extended":["partial","named",1,1],"CCBot":["partial","named",1,1],"Bytespider":["partial","named",1,1],"Amazonbot":["partial","*",11,1],"meta-externalagent":["partial","named",1,1],"cohere-ai":["partial","named",1,1],"Diffbot":["partial","named",1,1],"ContextFetchBot":["partial","*",11,1]},"llms":{"status":"missing","http":404,"bytes":455230},"home":{"status":"ok","http":200,"url":"https://variety.com/","html_tokens":180162,"markdown_tokens":159,"pruned_tokens":157}},
{"rank":570,"domain":"senate.gov","origin":"https://senate.gov","robots":{"status":"html","http":200,"url":"https://www.senate.gov/","bytes":24039,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":571,"domain":"congress.gov","origin":"https://congress.gov","robots":{"status":"ok","http":200,"url":"https://www.congress.gov/robots.txt","bytes":3529,"groups":2},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["blocked","named",1,0],"OAI-SearchBot":["blocked","named",1,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["blocked","named",1,0],"Claude-SearchBot":["blocked","named",1,0],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["blocked","named",1,0],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["partial","*",11,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",11,0]},"llms":{"status":"missing","http":404,"bytes":341482},"home":{"status":"http_error","http":403,"url":"https://congress.gov/"}},
{"rank":572,"domain":"gallup.com","origin":"https://gallup.com","robots":{"status":"ok","http":200,"url":"https://www.gallup.com/robots.txt","bytes":1001,"groups":15},"bots":{"GPTBot":["allowed","named",0,1],"ChatGPT-User":["allowed","named",0,1],"OAI-SearchBot":["allowed","named",0,1],"ClaudeBot":["allowed","named",0,1],"Claude-User":["allowed","named",0,1],"Claude-SearchBot":["allowed","named",0,1],"anthropic-ai":["partial","*",15,0],"PerplexityBot":["allowed","named",0,1],"Perplexity-User":["allowed","named",0,1],"Google-Extended":["allowed","named",0,1],"Applebot-Extended":["partial","*",15,0],"CCBot":["partial","*",15,0],"Bytespider":["partial","*",15,0],"Amazonbot":["partial","*",15,0],"meta-externalagent":["partial","*",15,0],"cohere-ai":["allowed","named",0,1],"Diffbot":["partial","*",15,0],"ContextFetchBot":["partial","*",15,0]},"llms":{"status":"missing","http":404,"bytes":54647},"home":{"status":"ok","http":200,"url":"https://www.gallup.com/home.aspx","html_tokens":23927,"markdown_tokens":355,"pruned_tokens":34}},
{"rank":573,"domain":"imf.org","origin":"https://imf.org","robots":{"status":"missing","http":404,"url":"https://www.imf.org/robots.txt","bytes":48,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":48},"home":{"status":"http_error","http":429,"url":"https://www.imf.org/"}},
{"rank":574,"domain":"douyin.com","origin":"https://douyin.com","robots":{"status":"ok","http":200,"url":"https://www.douyin.com/robots.txt","bytes":1044,"groups":3},"bots":{"GPTBot":["partial","named",6,0],"ChatGPT-User":["partial","named",6,0],"OAI-SearchBot":["partial","named",6,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["partial","named",6,0],"Perplexity-User":["partial","named",6,0],"Google-Extended":["partial","named",6,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["partial","named",6,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":210347},"home":{"status":"ok","http":200,"url":"https://www.douyin.com/","html_tokens":18229,"markdown_tokens":0,"pruned_tokens":0}},
{"rank":575,"domain":"feedburner.com","origin":"https://feedburner.com","robots":{"status":"missing","http":404,"url":"https://feedburner.com/robots.txt","bytes":1652,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":1652},"home":{"status":"ok","http":200,"url":"https://feedburner.google.com/","html_tokens":341,"markdown_tokens":0,"pruned_tokens":0}},
{"rank":576,"domain":"notion.site","origin":"https://notion.site","robots":{"status":"html","http":200,"url":"https://www.notion.com/","bytes":210280,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":577,"domain":"medicalnewstoday.com","origin":"https://medicalnewstoday.com","robots":{"status":"ok","http":200,"url":"https://www.medicalnewstoday.com/robots.txt","bytes":1713,"groups":29},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["partial","*",8,0],"OAI-SearchBot":["partial","*",8,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",8,0],"Claude-SearchBot":["partial","*",8,0],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["partial","*",8,0],"Perplexity-User":["partial","*",8,0],"Google-Extended":["partial","*",8,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["partial","*",8,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",8,0]},"llms":{"status":"missing","http":404,"bytes":89372},"home":{"status":"ok","http":200,"url":"https://www.medicalnewstoday.com/","html_tokens":79126,"markdown_tokens":96,"pruned_tokens":96}},
{"rank":578,"domain":"cnil.fr","origin":"https://cnil.fr","robots":{"status":"ok","http":200,"url":"https://cnil.fr/robots.txt","bytes":5155,"groups":1},"bots":{"GPTBot":["partial","*",102,54],"ChatGPT-User":["partial","*",102,54],"OAI-SearchBot":["partial","*",102,54],"ClaudeBot":["partial","*",102,54],"Claude-User":["partial","*",102,54],"Claude-SearchBot":["partial","*",102,54],"anthropic-ai":["partial","*",102,54],"PerplexityBot":["partial","*",102,54],"Perplexity-User":["partial","*",102,54],"Google-Extended":["partial","*",102,54],"Applebot-Extended":["partial","*",102,54],"CCBot":["partial","*",102,54],"Bytespider":["partial","*",102,54],"Amazonbot":["partial","*",102,54],"meta-externalagent":["partial","*",102,54],"cohere-ai":["partial","*",102,54],"Diffbot":["partial","*",102,54],"ContextFetchBot":["partial","*",102,54]},"llms":{"status":"missing","http":404,"bytes":181},"home":{"status":"ok","http":200,"url":"https://cnil.fr/fr","html_tokens":21139,"markdown_tokens":160,"pruned_tokens":74}},
{"rank":579,"domain":"nejm.org","origin":"https://nejm.org","robots":{"status":"ok","http":200,"url":"https://www.nejm.org/robots.txt","bytes":1423,"groups":14},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["blocked","named",1,0],"OAI-SearchBot":["partial","*",12,7],"ClaudeBot":["partial","*",12,7],"Claude-User":["partial","*",12,7],"Claude-SearchBot":["partial","*",12,7],"anthropic-ai":["partial","*",12,7],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["partial","*",12,7],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["partial","*",12,7],"CCBot":["blocked","named",1,0],"Bytespider":["partial","*",12,7],"Amazonbot":["partial","*",12,7],"meta-externalagent":["partial","*",12,7],"cohere-ai":["partial","*",12,7],"Diffbot":["partial","*",12,7],"ContextFetchBot":["partial","*",12,7]},"llms":{"status":"http_error","http":403,"bytes":5544},"home":{"status":"http_error","http":403,"url":"https://www.nejm.org/"}},
{"rank":580,"domain":"ucsd.edu","origin":"https://ucsd.edu","robots":{"status":"ok","http":200,"url":"https://ucsd.edu/robots.txt","bytes":106,"groups":1},"bots":{"GPTBot":["partial","*",2,0],"ChatGPT-User":["partial","*",2,0],"OAI-SearchBot":["partial","*",2,0],"ClaudeBot":["partial","*",2,0],"Claude-User":["partial","*",2,0],"Claude-SearchBot":["partial","*",2,0],"anthropic-ai":["partial","*",2,0],"PerplexityBot":["partial","*",2,0],"Perplexity-User":["partial","*",2,0],"Google-Extended":["partial","*",2,0],"Applebot-Extended":["partial","*",2,0],"CCBot":["partial","*",2,0],"Bytespider":["partial","*",2,0],"Amazonbot":["partial","*",2,0],"meta-externalagent":["partial","*",2,0],"cohere-ai":["partial","*",2,0],"Diffbot":["partial","*",2,0],"ContextFetchBot":["partial","*",2,0]},"llms":{"status":"missing","http":404,"bytes":39045},"home":{"status":"ok","http":200,"url":"https://ucsd.edu/","html_tokens":16080,"markdown_tokens":615,"pruned_tokens":128}},
{"rank":581,"domain":"digg.com","origin":"https://digg.com","robots":{"status":"ok","http":200,"url":"https://digg.com/robots.txt","bytes":922,"groups":2},"bots":{"GPTBot":["partial","*",3,1],"ChatGPT-User":["partial","named",3,1],"OAI-SearchBot":["partial","named",3,1],"ClaudeBot":["partial","named",3,1],"Claude-User":["partial","named",3,1],"Claude-SearchBot":["partial","named",3,1],"anthropic-ai":["partial","*",3,1],"PerplexityBot":["partial","named",3,1],"Perplexity-User":["partial","named",3,1],"Google-Extended":["partial","named",3,1],"Applebot-Extended":["partial","named",3,1],"CCBot":["partial","*",3,1],"Bytespider":["partial","*",3,1],"Amazonbot":["partial","*",3,1],"meta-externalagent":["partial","*",3,1],"cohere-ai":["partial","*",3,1],"Diffbot":["partial","*",3,1],"ContextFetchBot":["partial","*",3,1]},"llms":{"status":"present","http":200,"bytes":3231},"home":{"status":"ok","http":200,"url":"https://digg.com/","html_tokens":202794,"markdown_tokens":514,"pruned_tokens":175}},
{"rank":582,"domain":"purdue.edu","origin":"https://purdue.edu","robots":{"status":"ok","http":200,"url":"https://purdue.edu/robots.txt","bytes":2855,"groups":1},"bots":{"GPTBot":["partial","*",64,0],"ChatGPT-User":["partial","*",64,0],"OAI-SearchBot":["partial","*",64,0],"ClaudeBot":["partial","*",64,0],"Claude-User":["partial","*",64,0],"Claude-SearchBot":["partial","*",64,0],"anthropic-ai":["partial","*",64,0],"PerplexityBot":["partial","*",64,0],"Perplexity-User":["partial","*",64,0],"Google-Extended":["partial","*",64,0],"Applebot-Extended":["partial","*",64,0],"CCBot":["partial","*",64,0],"Bytespider":["partial","*",64,0],"Amazonbot":["partial","*",64,0],"meta-externalagent":["partial","*",64,0],"cohere-ai":["partial","*",64,0],"Diffbot":["partial","*",64,0],"ContextFetchBot":["partial","*",64,0]},"llms":{"status":"missing","http":404,"bytes":41041},"home":{"status":"ok","http":200,"url":"https://purdue.edu/","html_tokens":8912,"markdown_tokens":40,"pruned_tokens":35}},
{"rank":583,"domain":"trustindex.io","origin":"https://trustindex.io","robots":{"status":"ok","http":200,"url":"https://www.trustindex.io/robots.txt","bytes":175,"groups":1},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"missing","http":404,"bytes":97833},"home":{"status":"ok","http":200,"url":"https://www.trustindex.io/","html_tokens":29565,"markdown_tokens":452,"pruned_tokens":89}},
{"rank":584,"domain":"constantcontact.com","origin":"https://constantcontact.com","robots":{"status":"ok","http":200,"url":"https://www.constantcontact.com/robots.txt","bytes":350,"groups":1},"bots":{"GPTBot":["partial","*",10,0],"ChatGPT-User":["partial","*",10,0],"OAI-SearchBot":["partial","*",10,0],"ClaudeBot":["partial","*",10,0],"Claude-User":["partial","*",10,0],"Claude-SearchBot":["partial","*",10,0],"anthropic-ai":["partial","*",10,0],"PerplexityBot":["partial","*",10,0],"Perplexity-User":["partial","*",10,0],"Google-Extended":["partial","*",10,0],"Applebot-Extended":["partial","*",10,0],"CCBot":["partial","*",10,0],"Bytespider":["partial","*",10,0],"Amazonbot":["partial","*",10,0],"meta-externalagent":["partial","*",10,0],"cohere-ai":["partial","*",10,0],"Diffbot":["partial","*",10,0],"ContextFetchBot":["partial","*",10,0]},"llms":{"status":"other_text","http":200,"bytes":3272},"home":{"status":"http_error","http":403,"url":"https://www.constantcontact.com/"}},
{"rank":585,"domain":"service.weibo.com","origin":"https://service.weibo.com","robots":{"status":"error","http":null,"url":"https://service.weibo.com/robots.txt","bytes":0,"groups":0,"error":"more than 5 redirects"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":586,"domain":"barnesandnoble.com","origin":"https://barnesandnoble.com","robots":{"status":"ok","http":200,"url":"https://www.barnesandnoble.com/robots.txt","bytes":4352,"groups":7},"bots":{"GPTBot":["partial","*",34,0],"ChatGPT-User":["partial","*",34,0],"OAI-SearchBot":["partial","*",34,0],"ClaudeBot":["partial","*",34,0],"Claude-User":["partial","*",34,0],"Claude-SearchBot":["partial","*",34,0],"anthropic-ai":["partial","*",34,0],"PerplexityBot":["partial","*",34,0],"Perplexity-User":["partial","*",34,0],"Google-Extended":["partial","*",34,0],"Applebot-Extended":["partial","*",34,0],"CCBot":["partial","*",34,0],"Bytespider":["partial","*",34,0],"Amazonbot":["partial","*",34,0],"meta-externalagent":["partial","*",34,0],"cohere-ai":["partial","*",34,0],"Diffbot":["partial","*",34,0],"ContextFetchBot":["partial","*",34,0]},"llms":{"status":"missing","http":404,"bytes":29611},"home":{"status":"http_error","http":403,"url":"https://www.barnesandnoble.com/"}},
{"rank":587,"domain":"google.es","origin":"https://google.es","robots":{"status":"ok","http":200,"url":"https://www.google.es/robots.txt","bytes":6624,"groups":4},"bots":{"GPTBot":["partial","*",169,64],"ChatGPT-User":["partial","*",169,64],"OAI-SearchBot":["partial","*",169,64],"ClaudeBot":["partial","*",169,64],"Claude-User":["partial","*",169,64],"Claude-SearchBot":["partial","*",169,64],"anthropic-ai":["partial","*",169,64],"PerplexityBot":["partial","*",169,64],"Perplexity-User":["partial","*",169,64],"Google-Extended":["partial","*",169,64],"Applebot-Extended":["partial","*",169,64],"CCBot":["partial","*",169,64],"Bytespider":["partial","*",169,64],"Amazonbot":["partial","*",169,64],"meta-externalagent":["partial","*",169,64],"cohere-ai":["partial","*",169,64],"Diffbot":["partial","*",169,64],"ContextFetchBot":["partial","*",169,64]},"llms":{"status":"missing","http":404,"bytes":1569},"home":{"status":"ok","http":200,"url":"https://www.google.com/","html_tokens":21336,"markdown_tokens":30,"pruned_tokens":30}},
{"rank":588,"domain":"co.com","origin":"https://co.com","robots":{"status":"ok","http":200,"url":"https://co.com/robots.txt","bytes":26,"groups":1},"bots":{"GPTBot":["blocked","*",1,0],"ChatGPT-User":["blocked","*",1,0],"OAI-SearchBot":["blocked","*",1,0],"ClaudeBot":["blocked","*",1,0],"Claude-User":["blocked","*",1,0],"Claude-SearchBot":["blocked","*",1,0],"anthropic-ai":["blocked","*",1,0],"PerplexityBot":["blocked","*",1,0],"Perplexity-User":["blocked","*",1,0],"Google-Extended":["blocked","*",1,0],"Applebot-Extended":["blocked","*",1,0],"CCBot":["blocked","*",1,0],"Bytespider":["blocked","*",1,0],"Amazonbot":["blocked","*",1,0],"meta-externalagent":["blocked","*",1,0],"cohere-ai":["blocked","*",1,0],"Diffbot":["blocked","*",1,0],"ContextFetchBot":["blocked","*",1,0]},"llms":{"status":"disallowed"},"home":{"status":"disallowed"}},
{"rank":589,"domain":"google.co.jp","origin":"https://google.co.jp","robots":{"status":"ok","http":200,"url":"https://www.google.co.jp/robots.txt","bytes":6624,"groups":4},"bots":{"GPTBot":["partial","*",169,64],"ChatGPT-User":["partial","*",169,64],"OAI-SearchBot":["partial","*",169,64],"ClaudeBot":["partial","*",169,64],"Claude-User":["partial","*",169,64],"Claude-SearchBot":["partial","*",169,64],"anthropic-ai":["partial","*",169,64],"PerplexityBot":["partial","*",169,64],"Perplexity-User":["partial","*",169,64],"Google-Extended":["partial","*",169,64],"Applebot-Extended":["partial","*",169,64],"CCBot":["partial","*",169,64],"Bytespider":["partial","*",169,64],"Amazonbot":["partial","*",169,64],"meta-externalagent":["partial","*",169,64],"cohere-ai":["partial","*",169,64],"Diffbot":["partial","*",169,64],"ContextFetchBot":["partial","*",169,64]},"llms":{"status":"missing","http":404,"bytes":1569},"home":{"status":"ok","http":200,"url":"https://www.google.com/","html_tokens":21466,"markdown_tokens":30,"pruned_tokens":30}},
{"rank":590,"domain":"steamcommunity.com","origin":"https://steamcommunity.com","robots":{"status":"ok","http":200,"url":"https://steamcommunity.com/robots.txt","bytes":141,"groups":1},"bots":{"GPTBot":["partial","*",5,0],"ChatGPT-User":["partial","*",5,0],"OAI-SearchBot":["partial","*",5,0],"ClaudeBot":["partial","*",5,0],"Claude-User":["partial","*",5,0],"Claude-SearchBot":["partial","*",5,0],"anthropic-ai":["partial","*",5,0],"PerplexityBot":["partial","*",5,0],"Perplexity-User":["partial","*",5,0],"Google-Extended":["partial","*",5,0],"Applebot-Extended":["partial","*",5,0],"CCBot":["partial","*",5,0],"Bytespider":["partial","*",5,0],"Amazonbot":["partial","*",5,0],"meta-externalagent":["partial","*",5,0],"cohere-ai":["partial","*",5,0],"Diffbot":["partial","*",5,0],"ContextFetchBot":["partial","*",5,0]},"llms":{"status":"html","http":200,"bytes":47657},"home":{"status":"ok","http":200,"url":"https://steamcommunity.com/","html_tokens":11893,"markdown_tokens":72,"pruned_tokens":72}},
{"rank":591,"domain":"hatena.ne.jp","origin":"https://hatena.ne.jp","robots":{"status":"html","http":200,"url":"https://www.hatena.ne.jp/","bytes":610879,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":592,"domain":"techradar.com","origin":"https://techradar.com","robots":{"status":"ok","http":200,"url":"https://www.techradar.com/robots.txt","bytes":3408,"groups":5},"bots":{"GPTBot":["partial","*",20,0],"ChatGPT-User":["partial","*",20,0],"OAI-SearchBot":["partial","*",20,0],"ClaudeBot":["partial","*",20,0],"Claude-User":["partial","*",20,0],"Claude-SearchBot":["partial","*",20,0],"anthropic-ai":["partial","*",20,0],"PerplexityBot":["partial","*",20,0],"Perplexity-User":["partial","*",20,0],"Google-Extended":["partial","*",20,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["partial","*",20,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["partial","*",20,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",20,0]},"llms":{"status":"missing","http":404,"bytes":1343058},"home":{"status":"ok","http":200,"url":"https://www.techradar.com/","html_tokens":555746,"markdown_tokens":2219,"pruned_tokens":148}},
{"rank":593,"domain":"java.com","origin":"https://java.com","robots":{"status":"ok","http":200,"url":"https://www.java.com/robots.txt","bytes":576,"groups":1},"bots":{"GPTBot":["partial","*",19,0],"ChatGPT-User":["partial","*",19,0],"OAI-SearchBot":["partial","*",19,0],"ClaudeBot":["partial","*",19,0],"Claude-User":["partial","*",19,0],"Claude-SearchBot":["partial","*",19,0],"anthropic-ai":["partial","*",19,0],"PerplexityBot":["partial","*",19,0],"Perplexity-User":["partial","*",19,0],"Google-Extended":["partial","*",19,0],"Applebot-Extended":["partial","*",19,0],"CCBot":["partial","*",19,0],"Bytespider":["partial","*",19,0],"Amazonbot":["partial","*",19,0],"meta-externalagent":["partial","*",19,0],"cohere-ai":["partial","*",19,0],"Diffbot":["partial","*",19,0],"ContextFetchBot":["partial","*",19,0]},"llms":{"status":"missing","http":404,"bytes":3208},"home":{"status":"ok","http":200,"url":"https://www.java.com/en/","html_tokens":4472,"markdown_tokens":66,"pruned_tokens":66}},
{"rank":594,"domain":"perl.org","origin":"https://perl.org","robots":{"status":"ok","http":200,"url":"https://www.perl.org/robots.txt","bytes":107,"groups":1},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":5800},"home":{"status":"ok","http":200,"url":"https://www.perl.org/","html_tokens":3093,"markdown_tokens":92,"pruned_tokens":92}},
{"rank":595,"domain":"picasaweb.google.com","origin":"https://picasaweb.google.com","robots":{"status":"missing","http":404,"url":"https://get.google.com/albumarchive/pwa/robots.txt","bytes":1588,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"disallowed","error":"robots.txt of get.google.com disallows /albumarchive/pwa/llms.txt"},"home":{"status":"disallowed","error":"robots.txt of get.google.com disallows /albumarchive/pwa/"}},
{"rank":596,"domain":"fliphtml5.com","origin":"https://fliphtml5.com","robots":{"status":"ok","http":200,"url":"https://fliphtml5.com/robots.txt","bytes":10412,"groups":146},"bots":{"GPTBot":["partial","*",45,0],"ChatGPT-User":["partial","*",45,0],"OAI-SearchBot":["partial","*",45,0],"ClaudeBot":["partial","*",45,0],"Claude-User":["partial","*",45,0],"Claude-SearchBot":["partial","*",45,0],"anthropic-ai":["partial","*",45,0],"PerplexityBot":["partial","*",45,0],"Perplexity-User":["partial","*",45,0],"Google-Extended":["partial","*",45,0],"Applebot-Extended":["partial","*",45,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",2,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",45,0]},"llms":{"status":"present","http":200,"bytes":49572},"home":{"status":"http_error","http":403,"url":"https://fliphtml5.com/"}},
{"rank":597,"domain":"rakuten.co.jp","origin":"https://rakuten.co.jp","robots":{"status":"ok","http":200,"url":"https://www.rakuten.co.jp/robots.txt","bytes":490,"groups":3},"bots":{"GPTBot":["partial","*",12,2],"ChatGPT-User":["partial","*",12,2],"OAI-SearchBot":["partial","*",12,2],"ClaudeBot":["partial","*",12,2],"Claude-User":["partial","*",12,2],"Claude-SearchBot":["partial","*",12,2],"anthropic-ai":["partial","*",12,2],"PerplexityBot":["partial","*",12,2],"Perplexity-User":["partial","*",12,2],"Google-Extended":["partial","*",12,2],"Applebot-Extended":["partial","*",12,2],"CCBot":["partial","*",12,2],"Bytespider":["partial","*",12,2],"Amazonbot":["partial","*",12,2],"meta-externalagent":["partial","*",12,2],"cohere-ai":["partial","*",12,2],"Diffbot":["partial","*",12,2],"ContextFetchBot":["partial","*",12,2]},"llms":{"status":"timeout","error":"timed out after 10 s"},"home":{"status":"ok","http":200,"url":"https://www.rakuten.co.jp/","html_tokens":42127,"markdown_tokens":20,"pruned_tokens":20}},
{"rank":598,"domain":"technologyreview.com","origin":"https://technologyreview.com","robots":{"status":"ok","http":200,"url":"https://www.technologyreview.com/robots.txt","bytes":1920,"groups":35},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["blocked","named",1,0],"OAI-SearchBot":["blocked","named",1,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",2,1],"Claude-SearchBot":["partial","*",2,1],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["partial","*",2,1],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",2,1]},"llms":{"status":"missing","http":404,"bytes":204297},"home":{"status":"ok","http":200,"url":"https://www.technologyreview.com/","html_tokens":220341,"markdown_tokens":3336,"pruned_tokens":147}},
{"rank":599,"domain":"politico.com","origin":"https://politico.com","robots":{"status":"refused","http":403,"url":"https://www.politico.com/robots.txt","bytes":5554,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":600,"domain":"photobucket.com","origin":"https://photobucket.com","robots":{"status":"ok","http":200,"url":"https://photobucket.com/robots.txt","bytes":988,"groups":2},"bots":{"GPTBot":["partial","*",21,2],"ChatGPT-User":["partial","*",21,2],"OAI-SearchBot":["partial","*",21,2],"ClaudeBot":["partial","*",21,2],"Claude-User":["partial","*",21,2],"Claude-SearchBot":["partial","*",21,2],"anthropic-ai":["partial","*",21,2],"PerplexityBot":["partial","*",21,2],"Perplexity-User":["partial","*",21,2],"Google-Extended":["partial","*",21,2],"Applebot-Extended":["partial","*",21,2],"CCBot":["partial","*",21,2],"Bytespider":["blocked","named",1,0],"Amazonbot":["partial","*",21,2],"meta-externalagent":["partial","*",21,2],"cohere-ai":["partial","*",21,2],"Diffbot":["partial","*",21,2],"ContextFetchBot":["partial","*",21,2]},"llms":{"status":"html","http":200,"bytes":17468},"home":{"status":"ok","http":200,"url":"https://photobucket.com/","html_tokens":4367,"markdown_tokens":12,"pruned_tokens":12}},
{"rank":601,"domain":"trello.com","origin":"https://trello.com","robots":{"status":"ok","http":200,"url":"https://trello.com/robots.txt","bytes":657,"groups":1},"bots":{"GPTBot":["partial","*",23,1],"ChatGPT-User":["partial","*",23,1],"OAI-SearchBot":["partial","*",23,1],"ClaudeBot":["partial","*",23,1],"Claude-User":["partial","*",23,1],"Claude-SearchBot":["partial","*",23,1],"anthropic-ai":["partial","*",23,1],"PerplexityBot":["partial","*",23,1],"Perplexity-User":["partial","*",23,1],"Google-Extended":["partial","*",23,1],"Applebot-Extended":["partial","*",23,1],"CCBot":["partial","*",23,1],"Bytespider":["partial","*",23,1],"Amazonbot":["partial","*",23,1],"meta-externalagent":["partial","*",23,1],"cohere-ai":["partial","*",23,1],"Diffbot":["partial","*",23,1],"ContextFetchBot":["partial","*",23,1]},"llms":{"status":"html","http":200,"bytes":161911},"home":{"status":"ok","http":200,"url":"https://trello.com/","html_tokens":133012,"markdown_tokens":91,"pruned_tokens":46}},
{"rank":602,"domain":"apachefriends.org","origin":"https://apachefriends.org","robots":{"status":"ok","http":200,"url":"https://www.apachefriends.org/robots.txt","bytes":357,"groups":1},"bots":{"GPTBot":["partial","*",15,0],"ChatGPT-User":["partial","*",15,0],"OAI-SearchBot":["partial","*",15,0],"ClaudeBot":["partial","*",15,0],"Claude-User":["partial","*",15,0],"Claude-SearchBot":["partial","*",15,0],"anthropic-ai":["partial","*",15,0],"PerplexityBot":["partial","*",15,0],"Perplexity-User":["partial","*",15,0],"Google-Extended":["partial","*",15,0],"Applebot-Extended":["partial","*",15,0],"CCBot":["partial","*",15,0],"Bytespider":["partial","*",15,0],"Amazonbot":["partial","*",15,0],"meta-externalagent":["partial","*",15,0],"cohere-ai":["partial","*",15,0],"Diffbot":["partial","*",15,0],"ContextFetchBot":["partial","*",15,0]},"llms":{"status":"missing","http":404,"bytes":5388},"home":{"status":"ok","http":200,"url":"https://www.apachefriends.org/","html_tokens":2848,"markdown_tokens":206,"pruned_tokens":153}},
{"rank":603,"domain":"amd.com","origin":"https://amd.com","robots":{"status":"unreachable","http":null,"url":"https://amd.com/robots.txt","bytes":0,"groups":0,"error":"<StreamReset stream_id:1, error_code:2, remote_reset:True>"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":604,"domain":"namepros.com","origin":"https://namepros.com","robots":{"status":"ok","http":200,"url":"https://www.namepros.com/robots.txt","bytes":3196,"groups":3},"bots":{"GPTBot":["partial","*",54,1],"ChatGPT-User":["partial","*",54,1],"OAI-SearchBot":["partial","*",54,1],"ClaudeBot":["partial","*",54,1],"Claude-User":["partial","*",54,1],"Claude-SearchBot":["partial","*",54,1],"anthropic-ai":["partial","*",54,1],"PerplexityBot":["partial","*",54,1],"Perplexity-User":["partial","*",54,1],"Google-Extended":["partial","*",54,1],"Applebot-Extended":["partial","*",54,1],"CCBot":["partial","*",54,1],"Bytespider":["partial","*",54,1],"Amazonbot":["partial","*",54,1],"meta-externalagent":["partial","*",54,1],"cohere-ai":["partial","*",54,1],"Diffbot":["partial","*",54,1],"ContextFetchBot":["partial","*",54,1]},"llms":{"status":"missing","http":404,"bytes":126875},"home":{"status":"http_error","http":403,"url":"https://namepros.com/"}},
{"rank":605,"domain":"matomo.org","origin":"https://matomo.org","robots":{"status":"ok","http":200,"url":"https://matomo.org/robots.txt","bytes":345,"groups":1},"bots":{"GPTBot":["partial","*",9,1],"ChatGPT-User":["partial","*",9,1],"OAI-SearchBot":["partial","*",9,1],"ClaudeBot":["partial","*",9,1],"Claude-User":["partial","*",9,1],"Claude-SearchBot":["partial","*",9,1],"anthropic-ai":["partial","*",9,1],"PerplexityBot":["partial","*",9,1],"Perplexity-User":["partial","*",9,1],"Google-Extended":["partial","*",9,1],"Applebot-Extended":["partial","*",9,1],"CCBot":["partial","*",9,1],"Bytespider":["partial","*",9,1],"Amazonbot":["partial","*",9,1],"meta-externalagent":["partial","*",9,1],"cohere-ai":["partial","*",9,1],"Diffbot":["partial","*",9,1],"ContextFetchBot":["partial","*",9,1]},"llms":{"status":"timeout","error":"timed out after 10 s"},"home":{"status":"ok","http":200,"url":"https://matomo.org/","html_tokens":79711,"markdown_tokens":1379,"pruned_tokens":118}},
{"rank":606,"domain":"entrepreneur.com","origin":"https://entrepreneur.com","robots":{"status":"ok","http":200,"url":"https://www.entrepreneur.com/robots.txt","bytes":886,"groups":5},"bots":{"GPTBot":["partial","*",8,1],"ChatGPT-User":["partial","*",8,1],"OAI-SearchBot":["partial","*",8,1],"ClaudeBot":["partial","*",8,1],"Claude-User":["partial","*",8,1],"Claude-SearchBot":["partial","*",8,1],"anthropic-ai":["partial","*",8,1],"PerplexityBot":["partial","*",8,1],"Perplexity-User":["partial","*",8,1],"Google-Extended":["partial","*",8,1],"Applebot-Extended":["partial","*",8,1],"CCBot":["partial","*",8,1],"Bytespider":["partial","*",8,1],"Amazonbot":["partial","*",8,1],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["partial","*",8,1],"Diffbot":["partial","*",8,1],"ContextFetchBot":["partial","*",8,1]},"llms":{"status":"missing","http":404,"bytes":50503},"home":{"status":"ok","http":200,"url":"https://www.entrepreneur.com/","html_tokens":78424,"markdown_tokens":123,"pruned_tokens":123}},
{"rank":607,"domain":"cve.org","origin":"https://cve.org","robots":{"status":"ok","http":200,"url":"https://www.cve.org/robots.txt","bytes":74,"groups":1},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"html","http":200,"bytes":880},"home":{"status":"ok","http":200,"url":"https://www.cve.org/","html_tokens":220,"markdown_tokens":11,"pruned_tokens":11}},
{"rank":608,"domain":"qz.com","origin":"https://qz.com","robots":{"status":"ok","http":200,"url":"https://qz.com/robots.txt","bytes":11656,"groups":1},"bots":{"GPTBot":["partial","*",2,0],"ChatGPT-User":["partial","*",2,0],"OAI-SearchBot":["partial","*",2,0],"ClaudeBot":["partial","*",2,0],"Claude-User":["partial","*",2,0],"Claude-SearchBot":["partial","*",2,0],"anthropic-ai":["partial","*",2,0],"PerplexityBot":["partial","*",2,0],"Perplexity-User":["partial","*",2,0],"Google-Extended":["partial","*",2,0],"Applebot-Extended":["partial","*",2,0],"CCBot":["partial","*",2,0],"Bytespider":["partial","*",2,0],"Amazonbot":["partial","*",2,0],"meta-externalagent":["partial","*",2,0],"cohere-ai":["partial","*",2,0],"Diffbot":["partial","*",2,0],"ContextFetchBot":["partial","*",2,0]},"llms":{"status":"html","http":200,"bytes":116745},"home":{"status":"http_error","http":403,"url":"https://qz.com/"}},
{"rank":609,"domain":"iubenda.com","origin":"https://iubenda.com","robots":{"status":"ok","http":200,"url":"https://www.iubenda.com/robots.txt","bytes":2666,"groups":1},"bots":{"GPTBot":["partial","*",51,2],"ChatGPT-User":["partial","*",51,2],"OAI-SearchBot":["partial","*",51,2],"ClaudeBot":["partial","*",51,2],"Claude-User":["partial","*",51,2],"Claude-SearchBot":["partial","*",51,2],"anthropic-ai":["partial","*",51,2],"PerplexityBot":["partial","*",51,2],"Perplexity-User":["partial","*",51,2],"Google-Extended":["partial","*",51,2],"Applebot-Extended":["partial","*",51,2],"CCBot":["partial","*",51,2],"Bytespider":["partial","*",51,2],"Amazonbot":["partial","*",51,2],"meta-externalagent":["partial","*",51,2],"cohere-ai":["partial","*",51,2],"Diffbot":["partial","*",51,2],"ContextFetchBot":["partial","*",51,2]},"llms":{"status":"present","http":200,"bytes":6132},"home":{"status":"ok","http":200,"url":"https://www.iubenda.com/en/","html_tokens":93839,"markdown_tokens":822,"pruned_tokens":142}},
{"rank":610,"domain":"scmp.com","origin":"https://scmp.com","robots":{"status":"ok","http":200,"url":"https://www.scmp.com/robots.txt","bytes":4518,"groups":5},"bots":{"GPTBot":["partial","*",116,0],"ChatGPT-User":["partial","*",116,0],"OAI-SearchBot":["partial","*",116,0],"ClaudeBot":["partial","*",116,0],"Claude-User":["partial","*",116,0],"Claude-SearchBot":["partial","*",116,0],"anthropic-ai":["partial","*",116,0],"PerplexityBot":["partial","*",116,0],"Perplexity-User":["partial","*",116,0],"Google-Extended":["partial","*",116,0],"Applebot-Extended":["partial","*",116,0],"CCBot":["partial","*",116,0],"Bytespider":["partial","*",116,0],"Amazonbot":["partial","*",116,0],"meta-externalagent":["partial","*",116,0],"cohere-ai":["partial","*",116,0],"Diffbot":["partial","*",116,0],"ContextFetchBot":["partial","*",116,0]},"llms":{"status":"missing","http":404,"bytes":90950},"home":{"status":"ok","http":200,"url":"https://www.scmp.com/","html_tokens":543446,"markdown_tokens":46,"pruned_tokens":46}},
{"rank":611,"domain":"ufl.edu","origin":"https://ufl.edu","robots":{"status":"ok","http":200,"url":"https://www.ufl.edu/robots.txt","bytes":213,"groups":1},"bots":{"GPTBot":["partial","*",6,0],"ChatGPT-User":["partial","*",6,0],"OAI-SearchBot":["partial","*",6,0],"ClaudeBot":["partial","*",6,0],"Claude-User":["partial","*",6,0],"Claude-SearchBot":["partial","*",6,0],"anthropic-ai":["partial","*",6,0],"PerplexityBot":["partial","*",6,0],"Perplexity-User":["partial","*",6,0],"Google-Extended":["partial","*",6,0],"Applebot-Extended":["partial","*",6,0],"CCBot":["partial","*",6,0],"Bytespider":["partial","*",6,0],"Amazonbot":["partial","*",6,0],"meta-externalagent":["partial","*",6,0],"cohere-ai":["partial","*",6,0],"Diffbot":["partial","*",6,0],"ContextFetchBot":["partial","*",6,0]},"llms":{"status":"missing","http":404,"bytes":214733},"home":{"status":"ok","http":200,"url":"https://www.ufl.edu/","html_tokens":104962,"markdown_tokens":285,"pruned_tokens":164}},
{"rank":612,"domain":"zend.com","origin":"https://zend.com","robots":{"status":"refused","http":403,"url":"https://zend.com/robots.txt","bytes":560,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":613,"domain":"so.com","origin":"https://so.com","robots":{"status":"ok","http":200,"url":"https://www.so.com/robots.txt","bytes":58,"groups":1},"bots":{"GPTBot":["partial","*",3,0],"ChatGPT-User":["partial","*",3,0],"OAI-SearchBot":["partial","*",3,0],"ClaudeBot":["partial","*",3,0],"Claude-User":["partial","*",3,0],"Claude-SearchBot":["partial","*",3,0],"anthropic-ai":["partial","*",3,0],"PerplexityBot":["partial","*",3,0],"Perplexity-User":["partial","*",3,0],"Google-Extended":["partial","*",3,0],"Applebot-Extended":["partial","*",3,0],"CCBot":["partial","*",3,0],"Bytespider":["partial","*",3,0],"Amazonbot":["partial","*",3,0],"meta-externalagent":["partial","*",3,0],"cohere-ai":["partial","*",3,0],"Diffbot":["partial","*",3,0],"ContextFetchBot":["partial","*",3,0]},"llms":{"status":"missing","http":404,"bytes":25482},"home":{"status":"ok","http":200,"url":"https://www.so.com?src=so.com","html_tokens":65630,"markdown_tokens":23,"pruned_tokens":23}},
{"rank":614,"domain":"slate.com","origin":"https://slate.com","robots":{"status":"ok","http":200,"url":"https://slate.com/robots.txt","bytes":11372,"groups":4},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["partial","*",3,0],"OAI-SearchBot":["partial","*",3,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",3,0],"Claude-SearchBot":["partial","*",3,0],"anthropic-ai":["partial","*",3,0],"PerplexityBot":["partial","*",3,0],"Perplexity-User":["partial","*",3,0],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["partial","*",3,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",3,0]},"llms":{"status":"missing","http":404,"bytes":286817},"home":{"status":"ok","http":200,"url":"https://slate.com/","html_tokens":121919,"markdown_tokens":230,"pruned_tokens":84}},
{"rank":615,"domain":"meta.com","origin":"https://meta.com","robots":{"status":"ok","http":200,"url":"https://www.meta.com/robots.txt","bytes":3495,"groups":10},"bots":{"GPTBot":["partial","*",31,0],"ChatGPT-User":["partial","*",31,0],"OAI-SearchBot":["partial","*",31,0],"ClaudeBot":["partial","*",31,0],"Claude-User":["partial","*",31,0],"Claude-SearchBot":["partial","*",31,0],"anthropic-ai":["partial","*",31,0],"PerplexityBot":["partial","*",31,0],"Perplexity-User":["partial","*",31,0],"Google-Extended":["partial","*",31,0],"Applebot-Extended":["partial","*",31,0],"CCBot":["partial","*",31,0],"Bytespider":["partial","*",31,0],"Amazonbot":["partial","*",31,0],"meta-externalagent":["partial","*",31,0],"cohere-ai":["partial","*",31,0],"Diffbot":["partial","*",31,0],"ContextFetchBot":["partial","*",31,0]},"llms":{"status":"missing","http":404,"bytes":214438},"home":{"status":"ok","http":200,"url":"https://www.meta.com/","html_tokens":153474,"markdown_tokens":14,"pruned_tokens":14}},
{"rank":616,"domain":"drupal.org","origin":"https://drupal.org","robots":{"status":"ok","http":200,"url":"https://www.drupal.org/robots.txt","bytes":2523,"groups":1},"bots":{"GPTBot":["partial","*",56,0],"ChatGPT-User":["partial","*",56,0],"OAI-SearchBot":["partial","*",56,0],"ClaudeBot":["partial","*",56,0],"Claude-User":["partial","*",56,0],"Claude-SearchBot":["partial","*",56,0],"anthropic-ai":["partial","*",56,0],"PerplexityBot":["partial","*",56,0],"Perplexity-User":["partial","*",56,0],"Google-Extended":["partial","*",56,0],"Applebot-Extended":["partial","*",56,0],"CCBot":["partial","*",56,0],"Bytespider":["partial","*",56,0],"Amazonbot":["partial","*",56,0],"meta-externalagent":["partial","*",56,0],"cohere-ai":["partial","*",56,0],"Diffbot":["partial","*",56,0],"ContextFetchBot":["partial","*",56,0]},"llms":{"status":"missing","http":404,"bytes":188},"home":{"status":"ok","http":200,"url":"https://new.drupal.org/","html_tokens":20344,"markdown_tokens":573,"pruned_tokens":158}},
{"rank":617,"domain":"chinadaily.com.cn","origin":"https://chinadaily.com.cn","robots":{"status":"html","http":200,"url":"https://www.chinadaily.com.cn","bytes":118863,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":618,"domain":"kiev.ua","origin":"https://kiev.ua","robots":{"status":"unreachable","http":null,"url":"https://kiev.ua/robots.txt","bytes":0,"groups":0,"error":"cannot resolve kiev.ua"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":619,"domain":"udemy.com","origin":"https://udemy.com","robots":{"status":"refused","http":403,"url":"https://udemy.com/robots.txt","bytes":5525,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":620,"domain":"duckduckgo.com","origin":"https://duckduckgo.com","robots":{"status":"ok","http":200,"url":"https://duckduckgo.com/robots.txt","bytes":757,"groups":2},"bots":{"GPTBot":["partial","*",5,5],"ChatGPT-User":["partial","*",5,5],"OAI-SearchBot":["partial","*",5,5],"ClaudeBot":["partial","*",5,5],"Claude-User":["partial","*",5,5],"Claude-SearchBot":["partial","*",5,5],"anthropic-ai":["partial","*",5,5],"PerplexityBot":["partial","*",5,5],"Perplexity-User":["partial","*",5,5],"Google-Extended":["partial","*",5,5],"Applebot-Extended":["partial","*",5,5],"CCBot":["partial","*",5,5],"Bytespider":["partial","*",5,5],"Amazonbot":["partial","*",5,5],"meta-externalagent":["partial","*",5,5],"cohere-ai":["partial","*",5,5],"Diffbot":["partial","*",5,5],"ContextFetchBot":["partial","*",5,5]},"llms":{"status":"html","http":200,"bytes":18403},"home":{"status":"ok","http":200,"url":"https://duckduckgo.com/","html_tokens":58889,"markdown_tokens":110,"pruned_tokens":110}},
{"rank":621,"domain":"lww.com","origin":"https://lww.com","robots":{"status":"refused","http":403,"url":"https://lww.com/robots.txt","bytes":4901,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":622,"domain":"earthlink.net","origin":"https://earthlink.net","robots":{"status":"refused","http":403,"url":"https://earthlink.net/robots.txt","bytes":26203,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":623,"domain":"claude.ai","origin":"https://claude.ai","robots":{"status":"ok","http":200,"url":"https://claude.ai/robots.txt","bytes":281,"groups":3},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["blocked","named",1,0],"OAI-SearchBot":["blocked","named",1,0],"ClaudeBot":["partial","*",2,1],"Claude-User":["partial","*",2,1],"Claude-SearchBot":["partial","*",2,1],"anthropic-ai":["partial","*",2,1],"PerplexityBot":["partial","*",2,1],"Perplexity-User":["partial","*",2,1],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["partial","*",2,1],"CCBot":["partial","*",2,1],"Bytespider":["partial","*",2,1],"Amazonbot":["partial","*",2,1],"meta-externalagent":["partial","*",2,1],"cohere-ai":["partial","*",2,1],"Diffbot":["partial","*",2,1],"ContextFetchBot":["partial","*",2,1]},"llms":{"status":"html","http":200,"bytes":143961},"home":{"status":"http_error","http":403,"url":"https://claude.ai/"}},
{"rank":624,"domain":"duke.edu","origin":"https://duke.edu","robots":{"status":"ok","http":200,"url":"https://www.duke.edu/robots.txt","bytes":170,"groups":1},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"missing","http":404,"bytes":133576},"home":{"status":"ok","http":200,"url":"https://www.duke.edu/","html_tokens":59525,"markdown_tokens":1003,"pruned_tokens":78}},
{"rank":625,"domain":"i.imgur.com","origin":"https://i.imgur.com","robots":{"status":"ok","http":200,"url":"https://imgur.com/robots.txt","bytes":352,"groups":2},"bots":{"GPTBot":["partial","*",9,5],"ChatGPT-User":["partial","*",9,5],"OAI-SearchBot":["partial","*",9,5],"ClaudeBot":["partial","*",9,5],"Claude-User":["partial","*",9,5],"Claude-SearchBot":["partial","*",9,5],"anthropic-ai":["partial","*",9,5],"PerplexityBot":["partial","*",9,5],"Perplexity-User":["partial","*",9,5],"Google-Extended":["partial","*",9,5],"Applebot-Extended":["partial","*",9,5],"CCBot":["partial","*",9,5],"Bytespider":["partial","*",9,5],"Amazonbot":["partial","*",9,5],"meta-externalagent":["partial","*",9,5],"cohere-ai":["partial","*",9,5],"Diffbot":["partial","*",9,5],"ContextFetchBot":["partial","*",9,5]},"llms":{"status":"html","http":200,"bytes":6675},"home":{"status":"ok","http":200,"url":"https://imgur.com/","html_tokens":1669,"markdown_tokens":32,"pruned_tokens":32}},
{"rank":626,"domain":"usc.edu","origin":"https://usc.edu","robots":{"status":"refused","http":403,"url":"https://usc.edu/robots.txt","bytes":560,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":627,"domain":"wikia.com","origin":"https://wikia.com","robots":{"status":"refused","http":403,"url":"https://community.fandom.com/robots.txt","bytes":5579,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":628,"domain":"vmware.com","origin":"https://vmware.com","robots":{"status":"ok","http":200,"url":"https://www.vmware.com/robots.txt","bytes":477,"groups":1},"bots":{"GPTBot":["partial","*",1,1],"ChatGPT-User":["partial","*",1,1],"OAI-SearchBot":["partial","*",1,1],"ClaudeBot":["partial","*",1,1],"Claude-User":["partial","*",1,1],"Claude-SearchBot":["partial","*",1,1],"anthropic-ai":["partial","*",1,1],"PerplexityBot":["partial","*",1,1],"Perplexity-User":["partial","*",1,1],"Google-Extended":["partial","*",1,1],"Applebot-Extended":["partial","*",1,1],"CCBot":["partial","*",1,1],"Bytespider":["partial","*",1,1],"Amazonbot":["partial","*",1,1],"meta-externalagent":["partial","*",1,1],"cohere-ai":["partial","*",1,1],"Diffbot":["partial","*",1,1],"ContextFetchBot":["partial","*",1,1]},"llms":{"status":"missing","http":404,"bytes":3751},"home":{"status":"ok","http":200,"url":"https://www.vmware.com/","html_tokens":1215,"markdown_tokens":0,"pruned_tokens":0}},
{"rank":629,"domain":"vk.ru","origin":"https://vk.ru","robots":{"status":"ok","http":200,"url":"https://vk.ru/robots.txt","bytes":6175,"groups":3},"bots":{"GPTBot":["partial","*",79,11],"ChatGPT-User":["partial","*",79,11],"OAI-SearchBot":["partial","*",79,11],"ClaudeBot":["partial","*",79,11],"Claude-User":["partial","*",79,11],"Claude-SearchBot":["partial","*",79,11],"anthropic-ai":["partial","*",79,11],"PerplexityBot":["partial","*",79,11],"Perplexity-User":["partial","*",79,11],"Google-Extended":["partial","*",79,11],"Applebot-Extended":["partial","*",79,11],"CCBot":["partial","*",79,11],"Bytespider":["partial","*",79,11],"Amazonbot":["partial","*",79,11],"meta-externalagent":["partial","*",79,11],"cohere-ai":["partial","*",79,11],"Diffbot":["partial","*",79,11],"ContextFetchBot":["partial","*",79,11]},"llms":{"status":"missing","http":404,"bytes":13067},"home":{"status":"ok","http":200,"url":"https://m.vk.ru/","html_tokens":27051,"markdown_tokens":27,"pruned_tokens":27}},
{"rank":630,"domain":"sap.com","origin":"https://sap.com","robots":{"status":"ok","http":200,"url":"https://www.sap.com/robots.txt","bytes":4487,"groups":4},"bots":{"GPTBot":["partial","named",32,3],"ChatGPT-User":["partial","named",32,3],"OAI-SearchBot":["partial","named",29,3],"ClaudeBot":["partial","named",32,3],"Claude-User":["partial","named",32,3],"Claude-SearchBot":["partial","named",32,3],"anthropic-ai":["blocked","*",1,0],"PerplexityBot":["partial","named",29,3],"Perplexity-User":["partial","named",29,3],"Google-Extended":["partial","named",29,3],"Applebot-Extended":["blocked","*",1,0],"CCBot":["partial","named",29,3],"Bytespider":["blocked","*",1,0],"Amazonbot":["partial","named",32,3],"meta-externalagent":["blocked","*",1,0],"cohere-ai":["blocked","*",1,0],"Diffbot":["blocked","*",1,0],"ContextFetchBot":["blocked","*",1,0]},"llms":{"status":"disallowed"},"home":{"status":"disallowed"}},
{"rank":631,"domain":"google.ca","origin":"https://google.ca","robots":{"status":"ok","http":200,"url":"https://www.google.ca/robots.txt","bytes":6624,"groups":4},"bots":{"GPTBot":["partial","*",169,64],"ChatGPT-User":["partial","*",169,64],"OAI-SearchBot":["partial","*",169,64],"ClaudeBot":["partial","*",169,64],"Claude-User":["partial","*",169,64],"Claude-SearchBot":["partial","*",169,64],"anthropic-ai":["partial","*",169,64],"PerplexityBot":["partial","*",169,64],"Perplexity-User":["partial","*",169,64],"Google-Extended":["partial","*",169,64],"Applebot-Extended":["partial","*",169,64],"CCBot":["partial","*",169,64],"Bytespider":["partial","*",169,64],"Amazonbot":["partial","*",169,64],"meta-externalagent":["partial","*",169,64],"cohere-ai":["partial","*",169,64],"Diffbot":["partial","*",169,64],"ContextFetchBot":["partial","*",169,64]},"llms":{"status":"missing","http":404,"bytes":1569},"home":{"status":"ok","http":200,"url":"https://www.google.com/","html_tokens":21331,"markdown_tokens":30,"pruned_tokens":30}},
{"rank":632,"domain":"rollingstone.com","origin":"https://rollingstone.com","robots":{"status":"ok","http":200,"url":"https://www.rollingstone.com/robots.txt","bytes":2090,"groups":26},"bots":{"GPTBot":["partial","named",1,1],"ChatGPT-User":["partial","named",1,1],"OAI-SearchBot":["partial","named",1,1],"ClaudeBot":["partial","named",1,1],"Claude-User":["partial","*",9,1],"Claude-SearchBot":["partial","*",9,1],"anthropic-ai":["partial","named",1,1],"PerplexityBot":["partial","named",1,1],"Perplexity-User":["partial","*",9,1],"Google-Extended":["partial","named",1,1],"Applebot-Extended":["partial","named",1,1],"CCBot":["partial","named",1,1],"Bytespider":["partial","named",1,1],"Amazonbot":["partial","*",9,1],"meta-externalagent":["partial","named",1,1],"cohere-ai":["partial","named",1,1],"Diffbot":["partial","named",1,1],"ContextFetchBot":["partial","*",9,1]},"llms":{"status":"missing","http":404,"bytes":418507},"home":{"status":"ok","http":200,"url":"https://www.rollingstone.com/","html_tokens":159199,"markdown_tokens":172,"pruned_tokens":172}},
{"rank":633,"domain":"pcmag.com","origin":"https://pcmag.com","robots":{"status":"ok","http":200,"url":"https://www.pcmag.com/robots.txt","bytes":19340,"groups":3},"bots":{"GPTBot":["partial","named",2,166],"ChatGPT-User":["partial","named",2,166],"OAI-SearchBot":["partial","named",2,166],"ClaudeBot":["partial","named",2,166],"Claude-User":["partial","named",1,166],"Claude-SearchBot":["partial","named",1,166],"anthropic-ai":["partial","named",2,166],"PerplexityBot":["partial","named",2,166],"Perplexity-User":["partial","named",2,166],"Google-Extended":["partial","*",7,0],"Applebot-Extended":["partial","named",2,166],"CCBot":["partial","named",2,166],"Bytespider":["partial","named",2,166],"Amazonbot":["partial","named",2,166],"meta-externalagent":["partial","named",2,166],"cohere-ai":["partial","named",2,166],"Diffbot":["partial","named",2,166],"ContextFetchBot":["partial","*",7,0]},"llms":{"status":"missing","http":404,"bytes":593127},"home":{"status":"ok","http":200,"url":"https://www.pcmag.com/","html_tokens":174211,"markdown_tokens":126,"pruned_tokens":101}},
{"rank":634,"domain":"coe.int","origin":"https://coe.int","robots":{"status":"refused","http":403,"url":"https://www.coe.int/robots.txt","bytes":4544,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":635,"domain":"shutterstock.com","origin":"https://shutterstock.com","robots":{"status":"ok","http":200,"url":"https://www.shutterstock.com/robots.txt","bytes":9159,"groups":11},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["partial","*",42,10],"OAI-SearchBot":["partial","named",42,10],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",42,10],"Claude-SearchBot":["partial","named",42,10],"anthropic-ai":["partial","*",42,10],"PerplexityBot":["partial","named",42,10],"Perplexity-User":["partial","*",42,10],"Google-Extended":["partial","*",42,10],"Applebot-Extended":["partial","*",42,10],"CCBot":["blocked","named",1,0],"Bytespider":["partial","*",42,10],"Amazonbot":["partial","*",42,10],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["partial","*",42,10],"Diffbot":["partial","*",42,10],"ContextFetchBot":["partial","*",42,10]},"llms":{"status":"http_error","http":403,"bytes":784},"home":{"status":"http_error","http":403,"url":"https://www.shutterstock.com/"}},
{"rank":636,"domain":"feeds.feedburner.com","origin":"https://feeds.feedburner.com","robots":{"status":"html","http":200,"url":"https://feeds.feedburner.com/robots.txt","bytes":114,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":637,"domain":"qualtrics.com","origin":"https://qualtrics.com","robots":{"status":"ok","http":200,"url":"https://www.qualtrics.com/robots.txt","bytes":30643,"groups":1},"bots":{"GPTBot":["partial","*",744,252],"ChatGPT-User":["partial","*",744,252],"OAI-SearchBot":["partial","*",744,252],"ClaudeBot":["partial","*",744,252],"Claude-User":["partial","*",744,252],"Claude-SearchBot":["partial","*",744,252],"anthropic-ai":["partial","*",744,252],"PerplexityBot":["partial","*",744,252],"Perplexity-User":["partial","*",744,252],"Google-Extended":["partial","*",744,252],"Applebot-Extended":["partial","*",744,252],"CCBot":["partial","*",744,252],"Bytespider":["partial","*",744,252],"Amazonbot":["partial","*",744,252],"meta-externalagent":["partial","*",744,252],"cohere-ai":["partial","*",744,252],"Diffbot":["partial","*",744,252],"ContextFetchBot":["partial","*",744,252]},"llms":{"status":"missing","http":404,"bytes":191},"home":{"status":"ok","http":200,"url":"https://www.qualtrics.com/","html_tokens":129843,"markdown_tokens":750,"pruned_tokens":158}},
{"rank":638,"domain":"semrush.com","origin":"https://semrush.com","robots":{"status":"ok","http":200,"url":"https://www.semrush.com/robots.txt","bytes":3599,"groups":11},"bots":{"GPTBot":["partial","*",53,34],"ChatGPT-User":["partial","*",53,34],"OAI-SearchBot":["partial","*",53,34],"ClaudeBot":["partial","*",53,34],"Claude-User":["partial","*",53,34],"Claude-SearchBot":["partial","*",53,34],"anthropic-ai":["partial","*",53,34],"PerplexityBot":["partial","*",53,34],"Perplexity-User":["partial","*",53,34],"Google-Extended":["partial","*",53,34],"Applebot-Extended":["partial","*",53,34],"CCBot":["partial","*",53,34],"Bytespider":["partial","*",53,34],"Amazonbot":["partial","*",53,34],"meta-externalagent":["partial","*",53,34],"cohere-ai":["partial","*",53,34],"Diffbot":["partial","*",53,34],"ContextFetchBot":["partial","*",53,34]},"llms":{"status":"present","http":200,"bytes":14952},"home":{"status":"ok","http":200,"url":"https://www.semrush.com/","html_tokens":48572,"markdown_tokens":247,"pruned_tokens":45}},
{"rank":639,"domain":"umd.edu","origin":"https://umd.edu","robots":{"status":"ok","http":200,"url":"https://umd.edu/robots.txt","bytes":642,"groups":4},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["partial","*",4,0],"OAI-SearchBot":["partial","*",4,0],"ClaudeBot":["partial","*",4,0],"Claude-User":["partial","*",4,0],"Claude-SearchBot":["partial","*",4,0],"anthropic-ai":["partial","*",4,0],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["partial","*",4,0],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["partial","*",4,0],"CCBot":["partial","*",4,0],"Bytespider":["partial","*",4,0],"Amazonbot":["partial","*",4,0],"meta-externalagent":["partial","*",4,0],"cohere-ai":["partial","*",4,0],"Diffbot":["partial","*",4,0],"ContextFetchBot":["partial","*",4,0]},"llms":{"status":"missing","http":404,"bytes":28825},"home":{"status":"ok","http":200,"url":"https://umd.edu/","html_tokens":19117,"markdown_tokens":684,"pruned_tokens":171}},
{"rank":640,"domain":"ifeng.com","origin":"https://ifeng.com","robots":{"status":"ok","http":200,"url":"https://www.ifeng.com/robots.txt","bytes":123,"groups":1},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"present","http":200,"bytes":74174},"home":{"status":"ok","http":200,"url":"https://www.ifeng.com/","html_tokens":68991,"markdown_tokens":1682,"pruned_tokens":172}},
{"rank":641,"domain":"mail.ru","origin":"https://mail.ru","robots":{"status":"ok","http":200,"url":"https://mail.ru/robots.txt","bytes":433,"groups":4},"bots":{"GPTBot":["partial","*",1,6],"ChatGPT-User":["partial","*",1,6],"OAI-SearchBot":["partial","*",1,6],"ClaudeBot":["partial","*",1,6],"Claude-User":["partial","*",1,6],"Claude-SearchBot":["partial","*",1,6],"anthropic-ai":["partial","*",1,6],"PerplexityBot":["partial","*",1,6],"Perplexity-User":["partial","*",1,6],"Google-Extended":["partial","*",1,6],"Applebot-Extended":["partial","*",1,6],"CCBot":["partial","*",1,6],"Bytespider":["partial","*",1,6],"Amazonbot":["partial","*",1,6],"meta-externalagent":["partial","*",1,6],"cohere-ai":["partial","*",1,6],"Diffbot":["partial","*",1,6],"ContextFetchBot":["partial","*",1,6]},"llms":{"status":"disallowed"},"home":{"status":"disallowed","error":"robots.txt of login.vk.ru disallows /?act=autologin&app_id=7539952&redirect_uri=https%3A%2F%2Faccount.mail.ru%2Flogin%3Fautologin_exp%3D"}},
{"rank":642,"domain":"ccc.de","origin":"https://ccc.de","robots":{"status":"ok","http":200,"url":"https://www.ccc.de/robots.txt","bytes":204,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":5201},"home":{"status":"ok","http":200,"url":"https://www.ccc.de/","html_tokens":18104,"markdown_tokens":953,"pruned_tokens":133}},
{"rank":643,"domain":"ycombinator.com","origin":"https://ycombinator.com","robots":{"status":"ok","http":200,"url":"https://www.ycombinator.com/robots.txt","bytes":283,"groups":4},"bots":{"GPTBot":["partial","*",4,2],"ChatGPT-User":["partial","*",4,2],"OAI-SearchBot":["partial","*",4,2],"ClaudeBot":["partial","*",4,2],"Claude-User":["partial","*",4,2],"Claude-SearchBot":["partial","*",4,2],"anthropic-ai":["partial","*",4,2],"PerplexityBot":["partial","*",4,2],"Perplexity-User":["partial","*",4,2],"Google-Extended":["partial","*",4,2],"Applebot-Extended":["partial","*",4,2],"CCBot":["partial","*",4,2],"Bytespider":["partial","*",4,2],"Amazonbot":["partial","*",4,2],"meta-externalagent":["partial","*",4,2],"cohere-ai":["partial","*",4,2],"Diffbot":["partial","*",4,2],"ContextFetchBot":["partial","*",4,2]},"llms":{"status":"missing","http":404,"bytes":0},"home":{"status":"ok","http":200,"url":"https://www.ycombinator.com/","html_tokens":81932,"markdown_tokens":150,"pruned_tokens":150}},
{"rank":644,"domain":"ny.gov","origin":"https://ny.gov","robots":{"status":"ok","http":200,"url":"https://www.ny.gov/robots.txt","bytes":2150,"groups":1},"bots":{"GPTBot":["partial","*",34,18],"ChatGPT-User":["partial","*",34,18],"OAI-SearchBot":["partial","*",34,18],"ClaudeBot":["partial","*",34,18],"Claude-User":["partial","*",34,18],"Claude-SearchBot":["partial","*",34,18],"anthropic-ai":["partial","*",34,18],"PerplexityBot":["partial","*",34,18],"Perplexity-User":["partial","*",34,18],"Google-Extended":["partial","*",34,18],"Applebot-Extended":["partial","*",34,18],"CCBot":["partial","*",34,18],"Bytespider":["partial","*",34,18],"Amazonbot":["partial","*",34,18],"meta-externalagent":["partial","*",34,18],"cohere-ai":["partial","*",34,18],"Diffbot":["partial","*",34,18],"ContextFetchBot":["partial","*",34,18]},"llms":{"status":"missing","http":404,"bytes":184},"home":{"status":"ok","http":200,"url":"https://www.ny.gov/","html_tokens":25733,"markdown_tokens":158,"pruned_tokens":158}},
{"rank":645,"domain":"gutenberg.org","origin":"https://gutenberg.org","robots":{"status":"ok","http":200,"url":"https://www.gutenberg.org/robots.txt","bytes":40,"groups":1},"bots":{"GPTBot":["partial","*",1,0],"ChatGPT-User":["partial","*",1,0],"OAI-SearchBot":["partial","*",1,0],"ClaudeBot":["partial","*",1,0],"Claude-User":["partial","*",1,0],"Claude-SearchBot":["partial","*",1,0],"anthropic-ai":["partial","*",1,0],"PerplexityBot":["partial","*",1,0],"Perplexity-User":["partial","*",1,0],"Google-Extended":["partial","*",1,0],"Applebot-Extended":["partial","*",1,0],"CCBot":["partial","*",1,0],"Bytespider":["partial","*",1,0],"Amazonbot":["partial","*",1,0],"meta-externalagent":["partial","*",1,0],"cohere-ai":["partial","*",1,0],"Diffbot":["partial","*",1,0],"ContextFetchBot":["partial","*",1,0]},"llms":{"status":"missing","http":404,"bytes":6559},"home":{"status":"ok","http":200,"url":"https://www.gutenberg.org/","html_tokens":7561,"markdown_tokens":282,"pruned_tokens":163}},
{"rank":646,"domain":"utoronto.ca","origin":"https://utoronto.ca","robots":{"status":"refused","http":403,"url":"https://utoronto.ca/robots.txt","bytes":560,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":647,"domain":"de.com","origin":"https://de.com","robots":{"status":"ok","http":200,"url":"https://de.com/robots.txt","bytes":0,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":7852},"home":{"status":"ok","http":200,"url":"https://de.com/","html_tokens":10659,"markdown_tokens":809,"pruned_tokens":795}},
{"rank":648,"domain":"timeanddate.com","origin":"https://timeanddate.com","robots":{"status":"ok","http":200,"url":"https://www.timeanddate.com/robots.txt","bytes":7999,"groups":15},"bots":{"GPTBot":["partial","*",135,4],"ChatGPT-User":["partial","*",135,4],"OAI-SearchBot":["partial","*",135,4],"ClaudeBot":["partial","*",135,4],"Claude-User":["partial","*",135,4],"Claude-SearchBot":["partial","*",135,4],"anthropic-ai":["partial","*",135,4],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["partial","*",135,4],"Google-Extended":["partial","*",135,4],"Applebot-Extended":["blocked","named",1,0],"CCBot":["partial","*",135,4],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["partial","*",135,4],"Diffbot":["partial","*",135,4],"ContextFetchBot":["partial","*",135,4]},"llms":{"status":"missing","http":404,"bytes":20226},"home":{"status":"http_error","http":403,"url":"https://timeanddate.com/"}},
{"rank":649,"domain":"on.ca","origin":"https://on.ca","robots":{"status":"unreachable","http":null,"url":"https://on.ca/robots.txt","bytes":0,"groups":0,"error":"cannot resolve on.ca"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":650,"domain":"fifa.com","origin":"https://fifa.com","robots":{"status":"ok","http":200,"url":"https://www.fifa.com/robots.txt","bytes":216,"groups":1},"bots":{"GPTBot":["partial","*",3,0],"ChatGPT-User":["partial","*",3,0],"OAI-SearchBot":["partial","*",3,0],"ClaudeBot":["partial","*",3,0],"Claude-User":["partial","*",3,0],"Claude-SearchBot":["partial","*",3,0],"anthropic-ai":["partial","*",3,0],"PerplexityBot":["partial","*",3,0],"Perplexity-User":["partial","*",3,0],"Google-Extended":["partial","*",3,0],"Applebot-Extended":["partial","*",3,0],"CCBot":["partial","*",3,0],"Bytespider":["partial","*",3,0],"Amazonbot":["partial","*",3,0],"meta-externalagent":["partial","*",3,0],"cohere-ai":["partial","*",3,0],"Diffbot":["partial","*",3,0],"ContextFetchBot":["partial","*",3,0]},"llms":{"status":"html","http":200,"bytes":4440},"home":{"status":"ok","http":200,"url":"https://www.fifa.com/","html_tokens":1139,"markdown_tokens":0,"pruned_tokens":0}},
{"rank":651,"domain":"ucl.ac.uk","origin":"https://ucl.ac.uk","robots":{"status":"ok","http":200,"url":"https://www.ucl.ac.uk/robots.txt","bytes":18458,"groups":4},"bots":{"GPTBot":["partial","*",136,64],"ChatGPT-User":["partial","*",136,64],"OAI-SearchBot":["partial","*",136,64],"ClaudeBot":["partial","*",136,64],"Claude-User":["partial","*",136,64],"Claude-SearchBot":["partial","*",136,64],"anthropic-ai":["partial","*",136,64],"PerplexityBot":["partial","*",136,64],"Perplexity-User":["partial","*",136,64],"Google-Extended":["partial","*",136,64],"Applebot-Extended":["partial","*",136,64],"CCBot":["partial","*",136,64],"Bytespider":["partial","*",136,64],"Amazonbot":["partial","*",136,64],"meta-externalagent":["partial","*",136,64],"cohere-ai":["partial","*",136,64],"Diffbot":["partial","*",136,64],"ContextFetchBot":["partial","*",136,64]},"llms":{"status":"http_error","http":403,"bytes":5549},"home":{"status":"http_error","http":403,"url":"https://www.ucl.ac.uk/"}},
{"rank":652,"domain":"autodesk.com","origin":"https://autodesk.com","robots":{"status":"refused","http":403,"url":"https://www.autodesk.com/robots.txt","bytes":384,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":653,"domain":"axios.com","origin":"https://axios.com","robots":{"status":"ok","http":200,"url":"https://www.axios.com/robots.txt","bytes":645,"groups":10},"bots":{"GPTBot":["partial","*",5,0],"ChatGPT-User":["partial","*",5,0],"OAI-SearchBot":["partial","*",5,0],"ClaudeBot":["partial","*",5,0],"Claude-User":["partial","*",5,0],"Claude-SearchBot":["partial","*",5,0],"anthropic-ai":["partial","*",5,0],"PerplexityBot":["partial","*",5,0],"Perplexity-User":["partial","*",5,0],"Google-Extended":["partial","named",5,1],"Applebot-Extended":["partial","*",5,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["partial","*",5,0],"cohere-ai":["partial","*",5,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",5,0]},"llms":{"status":"missing","http":404,"bytes":39818},"home":{"status":"http_error","http":403,"url":"https://www.axios.com/"}},
{"rank":654,"domain":"mixcloud.com","origin":"https://mixcloud.com","robots":{"status":"ok","http":200,"url":"https://www.mixcloud.com/robots.txt","bytes":463,"groups":1},"bots":{"GPTBot":["partial","*",3,1],"ChatGPT-User":["partial","*",3,1],"OAI-SearchBot":["partial","*",3,1],"ClaudeBot":["partial","*",3,1],"Claude-User":["partial","*",3,1],"Claude-SearchBot":["partial","*",3,1],"anthropic-ai":["partial","*",3,1],"PerplexityBot":["partial","*",3,1],"Perplexity-User":["partial","*",3,1],"Google-Extended":["partial","*",3,1],"Applebot-Extended":["partial","*",3,1],"CCBot":["partial","*",3,1],"Bytespider":["partial","*",3,1],"Amazonbot":["partial","*",3,1],"meta-externalagent":["partial","*",3,1],"cohere-ai":["partial","*",3,1],"Diffbot":["partial","*",3,1],"ContextFetchBot":["partial","*",3,1]},"llms":{"status":"missing","http":404,"bytes":119566},"home":{"status":"ok","http":200,"url":"https://www.mixcloud.com/","html_tokens":166471,"markdown_tokens":40,"pruned_tokens":40}},
{"rank":655,"domain":"hetzner.com","origin":"https://hetzner.com","robots":{"status":"ok","http":200,"url":"https://www.hetzner.com/robots.txt","bytes":196,"groups":1},"bots":{"GPTBot":["partial","*",2,1],"ChatGPT-User":["partial","*",2,1],"OAI-SearchBot":["partial","*",2,1],"ClaudeBot":["partial","*",2,1],"Claude-User":["partial","*",2,1],"Claude-SearchBot":["partial","*",2,1],"anthropic-ai":["partial","*",2,1],"PerplexityBot":["partial","*",2,1],"Perplexity-User":["partial","*",2,1],"Google-Extended":["partial","*",2,1],"Applebot-Extended":["partial","*",2,1],"CCBot":["partial","*",2,1],"Bytespider":["partial","*",2,1],"Amazonbot":["partial","*",2,1],"meta-externalagent":["partial","*",2,1],"cohere-ai":["partial","*",2,1],"Diffbot":["partial","*",2,1],"ContextFetchBot":["partial","*",2,1]},"llms":{"status":"missing","http":404,"bytes":111507},"home":{"status":"ok","http":200,"url":"https://www.hetzner.com/","html_tokens":25205,"markdown_tokens":448,"pruned_tokens":281}},
{"rank":656,"domain":"chromewebstore.google.com","origin":"https://chromewebstore.google.com","robots":{"status":"ok","http":200,"url":"https://chromewebstore.google.com/robots.txt","bytes":503,"groups":1},"bots":{"GPTBot":["partial","*",15,0],"ChatGPT-User":["partial","*",15,0],"OAI-SearchBot":["partial","*",15,0],"ClaudeBot":["partial","*",15,0],"Claude-User":["partial","*",15,0],"Claude-SearchBot":["partial","*",15,0],"anthropic-ai":["partial","*",15,0],"PerplexityBot":["partial","*",15,0],"Perplexity-User":["partial","*",15,0],"Google-Extended":["partial","*",15,0],"Applebot-Extended":["partial","*",15,0],"CCBot":["partial","*",15,0],"Bytespider":["partial","*",15,0],"Amazonbot":["partial","*",15,0],"meta-externalagent":["partial","*",15,0],"cohere-ai":["partial","*",15,0],"Diffbot":["partial","*",15,0],"ContextFetchBot":["partial","*",15,0]},"llms":{"status":"missing","http":404,"bytes":117},"home":{"status":"ok","http":200,"url":"https://chromewebstore.google.com/","html_tokens":261081,"markdown_tokens":270,"pruned_tokens":105}},
{"rank":657,"domain":"slashdot.org","origin":"https://slashdot.org","robots":{"status":"ok","http":200,"url":"https://slashdot.org/robots.txt","bytes":4113,"groups":1},"bots":{"GPTBot":["partial","*",85,1],"ChatGPT-User":["partial","*",85,1],"OAI-SearchBot":["partial","*",85,1],"ClaudeBot":["partial","*",85,1],"Claude-User":["partial","*",85,1],"Claude-SearchBot":["partial","*",85,1],"anthropic-ai":["partial","*",85,1],"PerplexityBot":["partial","*",85,1],"Perplexity-User":["partial","*",85,1],"Google-Extended":["partial","*",85,1],"Applebot-Extended":["partial","*",85,1],"CCBot":["partial","*",85,1],"Bytespider":["partial","*",85,1],"Amazonbot":["partial","*",85,1],"meta-externalagent":["partial","*",85,1],"cohere-ai":["partial","*",85,1],"Diffbot":["partial","*",85,1],"ContextFetchBot":["partial","*",85,1]},"llms":{"status":"present","http":200,"bytes":3603},"home":{"status":"ok","http":200,"url":"https://slashdot.org/","html_tokens":39790,"markdown_tokens":885,"pruned_tokens":163}},
{"rank":658,"domain":"zoho.com","origin":"https://zoho.com","robots":{"status":"ok","http":200,"url":"https://www.zoho.com/robots.txt","bytes":4631,"groups":3},"bots":{"GPTBot":["partial","*",106,2],"ChatGPT-User":["partial","*",106,2],"OAI-SearchBot":["partial","*",106,2],"ClaudeBot":["partial","*",106,2],"Claude-User":["partial","*",106,2],"Claude-SearchBot":["partial","*",106,2],"anthropic-ai":["partial","*",106,2],"PerplexityBot":["partial","*",106,2],"Perplexity-User":["partial","*",106,2],"Google-Extended":["partial","*",106,2],"Applebot-Extended":["partial","*",106,2],"CCBot":["partial","*",106,2],"Bytespider":["partial","*",106,2],"Amazonbot":["partial","*",106,2],"meta-externalagent":["partial","*",106,2],"cohere-ai":["partial","*",106,2],"Diffbot":["partial","*",106,2],"ContextFetchBot":["partial","*",106,2]},"llms":{"status":"present","http":200,"bytes":50513},"home":{"status":"ok","http":200,"url":"https://www.zoho.com/","html_tokens":4495,"markdown_tokens":418,"pruned_tokens":87}},
{"rank":659,"domain":"ey.com","origin":"https://ey.com","robots":{"status":"ok","http":200,"url":"https://www.ey.com/robots.txt","bytes":7981,"groups":13},"bots":{"GPTBot":["allowed","named",0,1],"ChatGPT-User":["allowed","named",0,1],"OAI-SearchBot":["partial","*",3,0],"ClaudeBot":["partial","*",3,0],"Claude-User":["partial","*",3,0],"Claude-SearchBot":["partial","*",3,0],"anthropic-ai":["allowed","named",0,1],"PerplexityBot":["allowed","named",0,1],"Perplexity-User":["allowed","named",0,1],"Google-Extended":["allowed","named",0,1],"Applebot-Extended":["partial","*",3,0],"CCBot":["allowed","named",0,1],"Bytespider":["partial","*",3,0],"Amazonbot":["partial","*",3,0],"meta-externalagent":["partial","*",3,0],"cohere-ai":["partial","*",3,0],"Diffbot":["partial","*",3,0],"ContextFetchBot":["partial","*",3,0]},"llms":{"status":"missing","http":404,"bytes":265691},"home":{"status":"ok","http":200,"url":"https://www.ey.com/en_us","html_tokens":105604,"markdown_tokens":67,"pruned_tokens":67}},
{"rank":660,"domain":"b-cdn.net","origin":"https://b-cdn.net","robots":{"status":"ok","http":200,"url":"https://bunny.net","bytes":15244,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"other_text","http":200,"bytes":15244},"home":{"status":"not_html","error":"unsupported content type text/markdown"}},
{"rank":661,"domain":"fc2.com","origin":"https://fc2.com","robots":{"status":"ok","http":200,"url":"https://fc2.com/robots.txt","bytes":25,"groups":1},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"missing","http":404,"bytes":4260},"home":{"status":"ok","http":200,"url":"https://fc2.com/","html_tokens":3523,"markdown_tokens":199,"pruned_tokens":170}},
{"rank":662,"domain":"parallels.com","origin":"https://parallels.com","robots":{"status":"ok","http":200,"url":"https://www.parallels.com/robots.txt","bytes":2211,"groups":4},"bots":{"GPTBot":["partial","*",47,2],"ChatGPT-User":["partial","*",47,2],"OAI-SearchBot":["partial","*",47,2],"ClaudeBot":["partial","*",47,2],"Claude-User":["partial","*",47,2],"Claude-SearchBot":["partial","*",47,2],"anthropic-ai":["partial","*",47,2],"PerplexityBot":["partial","*",47,2],"Perplexity-User":["partial","*",47,2],"Google-Extended":["partial","*",47,2],"Applebot-Extended":["partial","*",47,2],"CCBot":["partial","*",47,2],"Bytespider":["partial","*",47,2],"Amazonbot":["partial","*",47,2],"meta-externalagent":["partial","*",47,2],"cohere-ai":["partial","*",47,2],"Diffbot":["partial","*",47,2],"ContextFetchBot":["partial","*",47,2]},"llms":{"status":"present","http":200,"bytes":5381},"home":{"status":"ok","http":200,"url":"https://www.parallels.com/","html_tokens":29720,"markdown_tokens":292,"pruned_tokens":219}},
{"rank":663,"domain":"qcloud.com","origin":"https://qcloud.com","robots":{"status":"ok","http":200,"url":"https://qcloud.com/robots.txt","bytes":26,"groups":1},"bots":{"GPTBot":["blocked","*",1,0],"ChatGPT-User":["blocked","*",1,0],"OAI-SearchBot":["blocked","*",1,0],"ClaudeBot":["blocked","*",1,0],"Claude-User":["blocked","*",1,0],"Claude-SearchBot":["blocked","*",1,0],"anthropic-ai":["blocked","*",1,0],"PerplexityBot":["blocked","*",1,0],"Perplexity-User":["blocked","*",1,0],"Google-Extended":["blocked","*",1,0],"Applebot-Extended":["blocked","*",1,0],"CCBot":["blocked","*",1,0],"Bytespider":["blocked","*",1,0],"Amazonbot":["blocked","*",1,0],"meta-externalagent":["blocked","*",1,0],"cohere-ai":["blocked","*",1,0],"Diffbot":["blocked","*",1,0],"ContextFetchBot":["blocked","*",1,0]},"llms":{"status":"disallowed"},"home":{"status":"disallowed"}},
{"rank":664,"domain":"semanticscholar.org","origin":"https://semanticscholar.org","robots":{"status":"ok","http":200,"url":"https://www.semanticscholar.org/robots.txt","bytes":574,"groups":1},"bots":{"GPTBot":["partial","*",8,1],"ChatGPT-User":["partial","*",8,1],"OAI-SearchBot":["partial","*",8,1],"ClaudeBot":["partial","*",8,1],"Claude-User":["partial","*",8,1],"Claude-SearchBot":["partial","*",8,1],"anthropic-ai":["partial","*",8,1],"PerplexityBot":["partial","*",8,1],"Perplexity-User":["partial","*",8,1],"Google-Extended":["partial","*",8,1],"Applebot-Extended":["partial","*",8,1],"CCBot":["partial","*",8,1],"Bytespider":["partial","*",8,1],"Amazonbot":["partial","*",8,1],"meta-externalagent":["partial","*",8,1],"cohere-ai":["partial","*",8,1],"Diffbot":["partial","*",8,1],"ContextFetchBot":["partial","*",8,1]},"llms":{"status":"missing","http":404,"bytes":97358},"home":{"status":"ok","http":200,"url":"https://www.semanticscholar.org/","html_tokens":47000,"markdown_tokens":99,"pruned_tokens":43}},
{"rank":665,"domain":"google.it","origin":"https://google.it","robots":{"status":"ok","http":200,"url":"https://www.google.it/robots.txt","bytes":6624,"groups":4},"bots":{"GPTBot":["partial","*",169,64],"ChatGPT-User":["partial","*",169,64],"OAI-SearchBot":["partial","*",169,64],"ClaudeBot":["partial","*",169,64],"Claude-User":["partial","*",169,64],"Claude-SearchBot":["partial","*",169,64],"anthropic-ai":["partial","*",169,64],"PerplexityBot":["partial","*",169,64],"Perplexity-User":["partial","*",169,64],"Google-Extended":["partial","*",169,64],"Applebot-Extended":["partial","*",169,64],"CCBot":["partial","*",169,64],"Bytespider":["partial","*",169,64],"Amazonbot":["partial","*",169,64],"meta-externalagent":["partial","*",169,64],"cohere-ai":["partial","*",169,64],"Diffbot":["partial","*",169,64],"ContextFetchBot":["partial","*",169,64]},"llms":{"status":"missing","http":404,"bytes":1569},"home":{"status":"ok","http":200,"url":"https://www.google.com/","html_tokens":21341,"markdown_tokens":30,"pruned_tokens":30}},
{"rank":666,"domain":"woocommerce.com","origin":"https://woocommerce.com","robots":{"status":"ok","http":200,"url":"https://woocommerce.com/robots.txt","bytes":679,"groups":2},"bots":{"GPTBot":["partial","*",15,1],"ChatGPT-User":["partial","*",15,1],"OAI-SearchBot":["partial","*",15,1],"ClaudeBot":["partial","*",15,1],"Claude-User":["partial","*",15,1],"Claude-SearchBot":["partial","*",15,1],"anthropic-ai":["partial","*",15,1],"PerplexityBot":["partial","*",15,1],"Perplexity-User":["partial","*",15,1],"Google-Extended":["partial","*",15,1],"Applebot-Extended":["partial","*",15,1],"CCBot":["partial","*",15,1],"Bytespider":["partial","*",15,1],"Amazonbot":["partial","*",15,1],"meta-externalagent":["partial","*",15,1],"cohere-ai":["partial","*",15,1],"Diffbot":["partial","*",15,1],"ContextFetchBot":["partial","*",15,1]},"llms":{"status":"other_text","http":200,"bytes":10148},"home":{"status":"ok","http":200,"url":"https://woocommerce.com/","html_tokens":88812,"markdown_tokens":220,"pruned_tokens":176}},
{"rank":667,"domain":"icann.org","origin":"https://icann.org","robots":{"status":"ok","http":200,"url":"https://www.icann.org/robots.txt","bytes":30,"groups":1},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"missing","http":404,"bytes":4391},"home":{"status":"http_error","http":403,"url":"https://www.icann.org/"}},
{"rank":668,"domain":"digitalocean.com","origin":"https://digitalocean.com","robots":{"status":"ok","http":200,"url":"https://www.digitalocean.com/robots.txt","bytes":172,"groups":1},"bots":{"GPTBot":["partial","*",4,1],"ChatGPT-User":["partial","*",4,1],"OAI-SearchBot":["partial","*",4,1],"ClaudeBot":["partial","*",4,1],"Claude-User":["partial","*",4,1],"Claude-SearchBot":["partial","*",4,1],"anthropic-ai":["partial","*",4,1],"PerplexityBot":["partial","*",4,1],"Perplexity-User":["partial","*",4,1],"Google-Extended":["partial","*",4,1],"Applebot-Extended":["partial","*",4,1],"CCBot":["partial","*",4,1],"Bytespider":["partial","*",4,1],"Amazonbot":["partial","*",4,1],"meta-externalagent":["partial","*",4,1],"cohere-ai":["partial","*",4,1],"Diffbot":["partial","*",4,1],"ContextFetchBot":["partial","*",4,1]},"llms":{"status":"missing","http":404,"bytes":67531},"home":{"status":"ok","http":200,"url":"https://www.digitalocean.com/","html_tokens":74433,"markdown_tokens":476,"pruned_tokens":144}},
{"rank":669,"domain":"newscientist.com","origin":"https://newscientist.com","robots":{"status":"ok","http":200,"url":"https://www.newscientist.com/robots.txt","bytes":6967,"groups":4},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["blocked","named",1,0],"OAI-SearchBot":["blocked","named",1,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["blocked","named",1,0],"Claude-SearchBot":["blocked","named",1,0],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["blocked","named",1,0],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",31,0]},"llms":{"status":"missing","http":404,"bytes":207057},"home":{"status":"ok","http":200,"url":"https://www.newscientist.com/","html_tokens":74419,"markdown_tokens":67,"pruned_tokens":67}},
{"rank":670,"domain":"appspot.com","origin":"https://appspot.com","robots":{"status":"html","http":200,"url":"https://accounts.google.com/v3/signin/identifier?continue=https://console.cloud.google.com/appengine&followup=https://console.cloud.google.com/appengine&osid=1&passive=1209600&service=cloudconsole&flowName=WebLiteSignIn&flowEntry=ServiceLogin&dsh=S118168547:1792374191915712","bytes":946689,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":671,"domain":"ucdavis.edu","origin":"https://ucdavis.edu","robots":{"status":"refused","http":403,"url":"https://ucdavis.edu/robots.txt","bytes":560,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":672,"domain":"ilo.org","origin":"https://ilo.org","robots":{"status":"ok","http":200,"url":"https://www.ilo.org/robots.txt","bytes":2987,"groups":1},"bots":{"GPTBot":["partial","*",43,22],"ChatGPT-User":["partial","*",43,22],"OAI-SearchBot":["partial","*",43,22],"ClaudeBot":["partial","*",43,22],"Claude-User":["partial","*",43,22],"Claude-SearchBot":["partial","*",43,22],"anthropic-ai":["partial","*",43,22],"PerplexityBot":["partial","*",43,22],"Perplexity-User":["partial","*",43,22],"Google-Extended":["partial","*",43,22],"Applebot-Extended":["partial","*",43,22],"CCBot":["partial","*",43,22],"Bytespider":["partial","*",43,22],"Amazonbot":["partial","*",43,22],"meta-externalagent":["partial","*",43,22],"cohere-ai":["partial","*",43,22],"Diffbot":["partial","*",43,22],"ContextFetchBot":["partial","*",43,22]},"llms":{"status":"http_error","http":403,"bytes":4901},"home":{"status":"ok","http":200,"url":"https://www.ilo.org/","html_tokens":22202,"markdown_tokens":28,"pruned_tokens":28}},
{"rank":673,"domain":"cell.com","origin":"https://cell.com","robots":{"status":"ok","http":200,"url":"https://www.cell.com/robots.txt","bytes":1824,"groups":22},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["blocked","named",1,0],"OAI-SearchBot":["partial","*",19,6],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",19,6],"Claude-SearchBot":["partial","*",19,6],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["partial","*",19,6],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["partial","*",19,6],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",19,6]},"llms":{"status":"http_error","http":403,"bytes":5518},"home":{"status":"http_error","http":403,"url":"https://cell.com/"}},
{"rank":674,"domain":"4.bp.blogspot.com","origin":"https://4.bp.blogspot.com","robots":{"status":"refused","http":400,"url":"https://4.bp.blogspot.com/robots.txt","bytes":1555,"groups":0,"error":"HTTP 400"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":675,"domain":"unc.edu","origin":"https://unc.edu","robots":{"status":"refused","http":403,"url":"https://unc.edu/robots.txt","bytes":560,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":676,"domain":"indiegogo.com","origin":"https://indiegogo.com","robots":{"status":"refused","http":403,"url":"https://www.indiegogo.com/robots.txt","bytes":5534,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":677,"domain":"livescience.com","origin":"https://livescience.com","robots":{"status":"ok","http":200,"url":"https://www.livescience.com/robots.txt","bytes":1637,"groups":4},"bots":{"GPTBot":["partial","*",18,0],"ChatGPT-User":["partial","*",18,0],"OAI-SearchBot":["partial","*",18,0],"ClaudeBot":["partial","*",18,0],"Claude-User":["partial","*",18,0],"Claude-SearchBot":["partial","*",18,0],"anthropic-ai":["partial","*",18,0],"PerplexityBot":["partial","*",18,0],"Perplexity-User":["partial","*",18,0],"Google-Extended":["partial","*",18,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["partial","*",18,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["partial","*",18,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",18,0]},"llms":{"status":"missing","http":404,"bytes":1514785},"home":{"status":"ok","http":200,"url":"https://www.livescience.com/","html_tokens":538502,"markdown_tokens":118,"pruned_tokens":118}},
{"rank":678,"domain":"translate.goog","origin":"https://translate.goog","robots":{"status":"missing","http":404,"url":"https://translate.goog/robots.txt","bytes":1571,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":1569},"home":{"status":"http_error","http":404,"url":"https://translate.goog/"}},
{"rank":679,"domain":"gob.ar","origin":"https://gob.ar","robots":{"status":"unreachable","http":null,"url":"https://gob.ar/robots.txt","bytes":0,"groups":0,"error":"cannot resolve gob.ar"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":680,"domain":"sfgate.com","origin":"https://sfgate.com","robots":{"status":"ok","http":200,"url":"https://www.sfgate.com/robots.txt","bytes":1116,"groups":3},"bots":{"GPTBot":["partial","*",14,0],"ChatGPT-User":["partial","*",14,0],"OAI-SearchBot":["partial","*",14,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["blocked","named",1,0],"Claude-SearchBot":["partial","*",14,0],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["blocked","named",1,0],"Google-Extended":["partial","*",14,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["partial","*",14,0],"meta-externalagent":["partial","*",14,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",14,0]},"llms":{"status":"missing","http":404,"bytes":48068},"home":{"status":"ok","http":200,"url":"https://www.sfgate.com/","html_tokens":207874,"markdown_tokens":69,"pruned_tokens":56}},
{"rank":681,"domain":"nikkei.com","origin":"https://nikkei.com","robots":{"status":"html","http":200,"url":"https://www.nikkei.com/","bytes":729916,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":682,"domain":"privacyshield.gov","origin":"https://www.privacyshield.gov","robots":{"status":"ok","http":200,"url":"https://www.privacyshield.gov/ps/robots.txt","bytes":106,"groups":2},"bots":{"GPTBot":["allowed","*",0,1],"ChatGPT-User":["allowed","*",0,1],"OAI-SearchBot":["allowed","*",0,1],"ClaudeBot":["allowed","*",0,1],"Claude-User":["allowed","*",0,1],"Claude-SearchBot":["allowed","*",0,1],"anthropic-ai":["allowed","*",0,1],"PerplexityBot":["allowed","*",0,1],"Perplexity-User":["allowed","*",0,1],"Google-Extended":["allowed","*",0,1],"Applebot-Extended":["allowed","*",0,1],"CCBot":["allowed","*",0,1],"Bytespider":["allowed","*",0,1],"Amazonbot":["allowed","*",0,1],"meta-externalagent":["allowed","*",0,1],"cohere-ai":["allowed","*",0,1],"Diffbot":["allowed","*",0,1],"ContextFetchBot":["allowed","*",0,1]},"llms":{"status":"http_error","http":401,"bytes":69229},"home":{"status":"ok","http":200,"url":"https://www.dataprivacyframework.gov/","html_tokens":529,"markdown_tokens":12,"pruned_tokens":12}},
{"rank":683,"domain":"espn.com","origin":"https://espn.com","robots":{"status":"ok","http":200,"url":"https://www.espn.com/robots.txt","bytes":3223,"groups":11},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["blocked","named",1,0],"OAI-SearchBot":["partial","*",103,4],"ClaudeBot":["partial","*",103,4],"Claude-User":["partial","*",103,4],"Claude-SearchBot":["partial","*",103,4],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["partial","*",103,4],"Perplexity-User":["partial","*",103,4],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["partial","*",103,4],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["partial","*",103,4],"meta-externalagent":["partial","*",103,4],"cohere-ai":["partial","*",103,4],"Diffbot":["partial","*",103,4],"ContextFetchBot":["partial","*",103,4]},"llms":{"status":"missing","http":404,"bytes":1},"home":{"status":"ok","http":200,"url":"https://www.espn.com/","html_tokens":62526,"markdown_tokens":455,"pruned_tokens":103}},
{"rank":684,"domain":"getpocket.com","origin":"https://getpocket.com","robots":{"status":"ok","http":200,"url":"https://getpocket.com/robots.txt","bytes":626,"groups":12},"bots":{"GPTBot":["partial","*",13,0],"ChatGPT-User":["partial","*",13,0],"OAI-SearchBot":["partial","*",13,0],"ClaudeBot":["partial","*",13,0],"Claude-User":["partial","*",13,0],"Claude-SearchBot":["partial","*",13,0],"anthropic-ai":["partial","*",13,0],"PerplexityBot":["partial","*",13,0],"Perplexity-User":["partial","*",13,0],"Google-Extended":["partial","*",13,0],"Applebot-Extended":["partial","*",13,0],"CCBot":["partial","*",13,0],"Bytespider":["partial","*",13,0],"Amazonbot":["partial","*",13,0],"meta-externalagent":["partial","*",13,0],"cohere-ai":["partial","*",13,0],"Diffbot":["partial","*",13,0],"ContextFetchBot":["partial","*",13,0]},"llms":{"status":"http_error","http":503,"bytes":955},"home":{"status":"ok","http":200,"url":"https://getpocket.com/home","html_tokens":9355,"markdown_tokens":516,"pruned_tokens":166}},
{"rank":685,"domain":"mediafire.com","origin":"https://mediafire.com","robots":{"status":"ok","http":200,"url":"https://mediafire.com/robots.txt","bytes":212,"groups":2},"bots":{"GPTBot":["partial","*",8,0],"ChatGPT-User":["partial","*",8,0],"OAI-SearchBot":["partial","*",8,0],"ClaudeBot":["partial","*",8,0],"Claude-User":["partial","*",8,0],"Claude-SearchBot":["partial","*",8,0],"anthropic-ai":["partial","*",8,0],"PerplexityBot":["partial","*",8,0],"Perplexity-User":["partial","*",8,0],"Google-Extended":["partial","*",8,0],"Applebot-Extended":["partial","*",8,0],"CCBot":["partial","*",8,0],"Bytespider":["partial","*",8,0],"Amazonbot":["partial","*",8,0],"meta-externalagent":["partial","*",8,0],"cohere-ai":["partial","*",8,0],"Diffbot":["partial","*",8,0],"ContextFetchBot":["partial","*",8,0]},"llms":{"status":"missing","http":404,"bytes":31944},"home":{"status":"ok","http":200,"url":"https://www.mediafire.com/","html_tokens":9608,"markdown_tokens":1073,"pruned_tokens":548}},
{"rank":686,"domain":"illinois.edu","origin":"https://illinois.edu","robots":{"status":"ok","http":200,"url":"https://illinois.edu/robots.txt","bytes":110,"groups":1},"bots":{"GPTBot":["partial","*",1,1],"ChatGPT-User":["partial","*",1,1],"OAI-SearchBot":["partial","*",1,1],"ClaudeBot":["partial","*",1,1],"Claude-User":["partial","*",1,1],"Claude-SearchBot":["partial","*",1,1],"anthropic-ai":["partial","*",1,1],"PerplexityBot":["partial","*",1,1],"Perplexity-User":["partial","*",1,1],"Google-Extended":["partial","*",1,1],"Applebot-Extended":["partial","*",1,1],"CCBot":["partial","*",1,1],"Bytespider":["partial","*",1,1],"Amazonbot":["partial","*",1,1],"meta-externalagent":["partial","*",1,1],"cohere-ai":["partial","*",1,1],"Diffbot":["partial","*",1,1],"ContextFetchBot":["partial","*",1,1]},"llms":{"status":"html","http":200,"bytes":57478},"home":{"status":"ok","http":200,"url":"https://illinois.edu/","html_tokens":37093,"markdown_tokens":836,"pruned_tokens":82}},
{"rank":687,"domain":"thesun.co.uk","origin":"https://thesun.co.uk","robots":{"status":"ok","http":200,"url":"https://www.thesun.co.uk/robots.txt","bytes":4803,"groups":2},"bots":{"GPTBot":["partial","named",101,3],"ChatGPT-User":["partial","named",101,3],"OAI-SearchBot":["partial","named",101,3],"ClaudeBot":["blocked","*",1,0],"Claude-User":["blocked","*",1,0],"Claude-SearchBot":["blocked","*",1,0],"anthropic-ai":["blocked","*",1,0],"PerplexityBot":["blocked","*",1,0],"Perplexity-User":["blocked","*",1,0],"Google-Extended":["blocked","*",1,0],"Applebot-Extended":["blocked","*",1,0],"CCBot":["blocked","*",1,0],"Bytespider":["blocked","*",1,0],"Amazonbot":["blocked","*",1,0],"meta-externalagent":["blocked","*",1,0],"cohere-ai":["blocked","*",1,0],"Diffbot":["blocked","*",1,0],"ContextFetchBot":["blocked","*",1,0]},"llms":{"status":"disallowed"},"home":{"status":"disallowed"}},
{"rank":688,"domain":"in.net","origin":"https://in.net","robots":{"status":"unreachable","http":null,"url":"https://in.net/robots.txt","bytes":0,"groups":0,"error":"cannot resolve in.net"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":689,"domain":"venturebeat.com","origin":"https://venturebeat.com","robots":{"status":"ok","http":200,"url":"https://venturebeat.com/robots.txt","bytes":3131,"groups":30},"bots":{"GPTBot":["partial","named",4,1],"ChatGPT-User":["partial","*",7,0],"OAI-SearchBot":["partial","*",7,0],"ClaudeBot":["partial","named",4,1],"Claude-User":["partial","*",7,0],"Claude-SearchBot":["partial","*",7,0],"anthropic-ai":["partial","named",4,1],"PerplexityBot":["partial","named",4,1],"Perplexity-User":["partial","*",7,0],"Google-Extended":["partial","named",4,1],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["partial","*",7,0],"cohere-ai":["partial","*",7,0],"Diffbot":["partial","*",7,0],"ContextFetchBot":["partial","*",7,0]},"llms":{"status":"http_error","http":429,"bytes":34217},"home":{"status":"http_error","http":429,"url":"https://venturebeat.com/"}},
{"rank":690,"domain":"howstuffworks.com","origin":"https://howstuffworks.com","robots":{"status":"ok","http":200,"url":"https://animals.howstuffworks.com/robots.txt","bytes":175,"groups":2},"bots":{"GPTBot":["partial","*",2,0],"ChatGPT-User":["partial","*",2,0],"OAI-SearchBot":["partial","*",2,0],"ClaudeBot":["partial","*",2,0],"Claude-User":["partial","*",2,0],"Claude-SearchBot":["partial","*",2,0],"anthropic-ai":["partial","*",2,0],"PerplexityBot":["partial","*",2,0],"Perplexity-User":["partial","*",2,0],"Google-Extended":["partial","*",2,0],"Applebot-Extended":["partial","*",2,0],"CCBot":["partial","*",2,0],"Bytespider":["partial","*",2,0],"Amazonbot":["partial","*",2,0],"meta-externalagent":["partial","*",2,0],"cohere-ai":["partial","*",2,0],"Diffbot":["partial","*",2,0],"ContextFetchBot":["partial","*",2,0]},"llms":{"status":"html","http":202,"bytes":2044},"home":{"status":"ok","http":202,"url":"https://howstuffworks.com/","html_tokens":511,"markdown_tokens":40,"pruned_tokens":40}},
{"rank":691,"domain":"cutt.ly","origin":"https://cutt.ly","robots":{"status":"ok","http":200,"url":"https://cutt.ly/robots.txt","bytes":106,"groups":1},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"present","http":200,"bytes":20023},"home":{"status":"ok","http":200,"url":"https://cutt.ly/","html_tokens":56081,"markdown_tokens":3820,"pruned_tokens":299}},
{"rank":692,"domain":"mirror.co.uk","origin":"https://mirror.co.uk","robots":{"status":"ok","http":200,"url":"https://www.mirror.co.uk/robots.txt","bytes":1900,"groups":18},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["partial","*",24,0],"OAI-SearchBot":["blocked","named",1,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",24,0],"Claude-SearchBot":["partial","*",24,0],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["partial","*",24,0],"Google-Extended":["partial","*",24,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["partial","*",24,0],"Amazonbot":["partial","*",24,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["partial","*",24,0],"Diffbot":["partial","*",24,0],"ContextFetchBot":["partial","*",24,0]},"llms":{"status":"missing","http":404,"bytes":783293},"home":{"status":"ok","http":200,"url":"https://www.mirror.co.uk/","html_tokens":180208,"markdown_tokens":153,"pruned_tokens":153}},
{"rank":693,"domain":"go.th","origin":"https://go.th","robots":{"status":"unreachable","http":null,"url":"https://go.th/robots.txt","bytes":0,"groups":0,"error":"cannot resolve go.th"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":694,"domain":"githubusercontent.com","origin":"https://www.githubusercontent.com","robots":{"status":"server_error","http":500,"url":"https://www.githubusercontent.com/robots.txt","bytes":296,"groups":0,"error":"HTTP 500"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":695,"domain":"sky.com","origin":"https://sky.com","robots":{"status":"ok","http":200,"url":"https://www.sky.com/robots.txt","bytes":4185,"groups":4},"bots":{"GPTBot":["partial","*",112,0],"ChatGPT-User":["partial","*",112,0],"OAI-SearchBot":["allowed","named",0,1],"ClaudeBot":["partial","*",112,0],"Claude-User":["partial","*",112,0],"Claude-SearchBot":["partial","*",112,0],"anthropic-ai":["partial","*",112,0],"PerplexityBot":["partial","*",112,0],"Perplexity-User":["partial","*",112,0],"Google-Extended":["partial","*",112,0],"Applebot-Extended":["partial","*",112,0],"CCBot":["partial","*",112,0],"Bytespider":["partial","*",112,0],"Amazonbot":["partial","*",112,0],"meta-externalagent":["partial","*",112,0],"cohere-ai":["partial","*",112,0],"Diffbot":["partial","*",112,0],"ContextFetchBot":["partial","*",112,0]},"llms":{"status":"missing","http":404,"bytes":155019},"home":{"status":"ok","http":200,"url":"https://www.sky.com/","html_tokens":78213,"markdown_tokens":5473,"pruned_tokens":51}},
{"rank":696,"domain":"msu.edu","origin":"https://msu.edu","robots":{"status":"ok","http":200,"url":"https://msu.edu/robots.txt","bytes":647,"groups":1},"bots":{"GPTBot":["partial","*",17,0],"ChatGPT-User":["partial","*",17,0],"OAI-SearchBot":["partial","*",17,0],"ClaudeBot":["partial","*",17,0],"Claude-User":["partial","*",17,0],"Claude-SearchBot":["partial","*",17,0],"anthropic-ai":["partial","*",17,0],"PerplexityBot":["partial","*",17,0],"Perplexity-User":["partial","*",17,0],"Google-Extended":["partial","*",17,0],"Applebot-Extended":["partial","*",17,0],"CCBot":["partial","*",17,0],"Bytespider":["partial","*",17,0],"Amazonbot":["partial","*",17,0],"meta-externalagent":["partial","*",17,0],"cohere-ai":["partial","*",17,0],"Diffbot":["partial","*",17,0],"ContextFetchBot":["partial","*",17,0]},"llms":{"status":"present","http":200,"bytes":859},"home":{"status":"ok","http":200,"url":"https://msu.edu/","html_tokens":211,"markdown_tokens":21,"pruned_tokens":21}},
{"rank":697,"domain":"playstation.com","origin":"https://playstation.com","robots":{"status":"ok","http":200,"url":"https://www.playstation.com/robots.txt","bytes":655,"groups":4},"bots":{"GPTBot":["partial","*",7,0],"ChatGPT-User":["partial","*",7,0],"OAI-SearchBot":["partial","*",7,0],"ClaudeBot":["partial","*",7,0],"Claude-User":["partial","*",7,0],"Claude-SearchBot":["partial","*",7,0],"anthropic-ai":["partial","*",7,0],"PerplexityBot":["partial","*",7,0],"Perplexity-User":["partial","*",7,0],"Google-Extended":["partial","*",7,0],"Applebot-Extended":["partial","*",7,0],"CCBot":["partial","*",7,0],"Bytespider":["partial","*",7,0],"Amazonbot":["partial","*",7,0],"meta-externalagent":["partial","*",7,0],"cohere-ai":["partial","*",7,0],"Diffbot":["partial","*",7,0],"ContextFetchBot":["partial","*",7,0]},"llms":{"status":"other_text","http":200,"bytes":3842},"home":{"status":"ok","http":200,"url":"https://www.playstation.com/en-us/","html_tokens":91635,"markdown_tokens":97,"pruned_tokens":75}},
{"rank":698,"domain":"yandex.com","origin":"https://yandex.com","robots":{"status":"ok","http":200,"url":"https://yandex.com/robots.txt","bytes":21167,"groups":4},"bots":{"GPTBot":["partial","*",221,27],"ChatGPT-User":["partial","*",221,27],"OAI-SearchBot":["partial","*",221,27],"ClaudeBot":["partial","*",221,27],"Claude-User":["partial","*",221,27],"Claude-SearchBot":["partial","*",221,27],"anthropic-ai":["partial","*",221,27],"PerplexityBot":["partial","*",221,27],"Perplexity-User":["partial","*",221,27],"Google-Extended":["partial","*",221,27],"Applebot-Extended":["partial","*",221,27],"CCBot":["partial","*",221,27],"Bytespider":["partial","*",221,27],"Amazonbot":["partial","*",221,27],"meta-externalagent":["partial","*",221,27],"cohere-ai":["partial","*",221,27],"Diffbot":["partial","*",221,27],"ContextFetchBot":["partial","*",221,27]},"llms":{"status":"missing","http":404,"bytes":12536},"home":{"status":"ok","http":200,"url":"https://yandex.com/","html_tokens":96124,"markdown_tokens":43,"pruned_tokens":43}},
{"rank":699,"domain":"altervista.org","origin":"https://altervista.org","robots":{"status":"html","http":200,"url":"https://en.altervista.org/","bytes":44516,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":700,"domain":"lenovo.com","origin":"https://lenovo.com","robots":{"status":"refused","http":403,"url":"https://www.lenovo.com/robots.txt","bytes":380,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":701,"domain":"smh.com.au","origin":"https://smh.com.au","robots":{"status":"ok","http":200,"url":"https://www.smh.com.au/robots.txt","bytes":4048,"groups":52},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["blocked","named",1,0],"OAI-SearchBot":["blocked","named",1,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",5,1],"Claude-SearchBot":["partial","*",5,1],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["partial","*",5,1],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",5,1]},"llms":{"status":"missing","http":404,"bytes":171625},"home":{"status":"ok","http":200,"url":"https://www.smh.com.au/","html_tokens":296158,"markdown_tokens":930,"pruned_tokens":144}},
{"rank":702,"domain":"archives.gov","origin":"https://archives.gov","robots":{"status":"ok","http":200,"url":"https://www.archives.gov/robots.txt","bytes":1048,"groups":2},"bots":{"GPTBot":["partial","*",4,0],"ChatGPT-User":["partial","*",4,0],"OAI-SearchBot":["partial","*",4,0],"ClaudeBot":["partial","*",4,0],"Claude-User":["partial","*",4,0],"Claude-SearchBot":["partial","*",4,0],"anthropic-ai":["partial","*",4,0],"PerplexityBot":["partial","*",4,0],"Perplexity-User":["partial","*",4,0],"Google-Extended":["partial","*",4,0],"Applebot-Extended":["partial","*",4,0],"CCBot":["partial","*",4,0],"Bytespider":["partial","*",4,0],"Amazonbot":["partial","*",4,0],"meta-externalagent":["partial","*",4,0],"cohere-ai":["partial","*",4,0],"Diffbot":["partial","*",4,0],"ContextFetchBot":["partial","*",4,0]},"llms":{"status":"timeout","error":"timed out after 10 s"},"home":{"status":"ok","http":200,"url":"https://www.archives.gov/","html_tokens":7338,"markdown_tokens":238,"pruned_tokens":165}},
{"rank":703,"domain":"vic.gov.au","origin":"https://vic.gov.au","robots":{"status":"ok","http":200,"url":"https://www.vic.gov.au/robots.txt","bytes":627,"groups":3},"bots":{"GPTBot":["partial","*",15,0],"ChatGPT-User":["partial","*",15,0],"OAI-SearchBot":["partial","*",15,0],"ClaudeBot":["partial","*",15,0],"Claude-User":["partial","*",15,0],"Claude-SearchBot":["partial","*",15,0],"anthropic-ai":["partial","*",15,0],"PerplexityBot":["partial","*",15,0],"Perplexity-User":["partial","*",15,0],"Google-Extended":["partial","*",15,0],"Applebot-Extended":["partial","*",15,0],"CCBot":["partial","*",15,0],"Bytespider":["partial","*",15,0],"Amazonbot":["partial","*",15,0],"meta-externalagent":["partial","*",15,0],"cohere-ai":["partial","*",15,0],"Diffbot":["partial","*",15,0],"ContextFetchBot":["partial","*",15,0]},"llms":{"status":"missing","http":404,"bytes":115972},"home":{"status":"ok","http":200,"url":"https://www.vic.gov.au/","html_tokens":41587,"markdown_tokens":47,"pruned_tokens":47}},
{"rank":704,"domain":"ubc.ca","origin":"https://ubc.ca","robots":{"status":"ok","http":200,"url":"https://www.ubc.ca/robots.txt","bytes":187,"groups":1},"bots":{"GPTBot":["partial","*",5,0],"ChatGPT-User":["partial","*",5,0],"OAI-SearchBot":["partial","*",5,0],"ClaudeBot":["partial","*",5,0],"Claude-User":["partial","*",5,0],"Claude-SearchBot":["partial","*",5,0],"anthropic-ai":["partial","*",5,0],"PerplexityBot":["partial","*",5,0],"Perplexity-User":["partial","*",5,0],"Google-Extended":["partial","*",5,0],"Applebot-Extended":["partial","*",5,0],"CCBot":["partial","*",5,0],"Bytespider":["partial","*",5,0],"Amazonbot":["partial","*",5,0],"meta-externalagent":["partial","*",5,0],"cohere-ai":["partial","*",5,0],"Diffbot":["partial","*",5,0],"ContextFetchBot":["partial","*",5,0]},"llms":{"status":"missing","http":404,"bytes":14837},"home":{"status":"ok","http":200,"url":"https://www.ubc.ca/","html_tokens":7430,"markdown_tokens":56,"pruned_tokens":56}},
{"rank":705,"domain":"undp.org","origin":"https://undp.org","robots":{"status":"ok","http":200,"url":"https://www.undp.org/robots.txt","bytes":2124,"groups":1},"bots":{"GPTBot":["partial","*",36,18],"ChatGPT-User":["partial","*",36,18],"OAI-SearchBot":["partial","*",36,18],"ClaudeBot":["partial","*",36,18],"Claude-User":["partial","*",36,18],"Claude-SearchBot":["partial","*",36,18],"anthropic-ai":["partial","*",36,18],"PerplexityBot":["partial","*",36,18],"Perplexity-User":["partial","*",36,18],"Google-Extended":["partial","*",36,18],"Applebot-Extended":["partial","*",36,18],"CCBot":["partial","*",36,18],"Bytespider":["partial","*",36,18],"Amazonbot":["partial","*",36,18],"meta-externalagent":["partial","*",36,18],"cohere-ai":["partial","*",36,18],"Diffbot":["partial","*",36,18],"ContextFetchBot":["partial","*",36,18]},"llms":{"status":"missing","http":404,"bytes":93010},"home":{"status":"ok","http":200,"url":"https://www.undp.org/","html_tokens":44418,"markdown_tokens":1343,"pruned_tokens":717}},
{"rank":706,"domain":"curl.se","origin":"https://curl.se","robots":{"status":"ok","http":200,"url":"https://curl.se/robots.txt","bytes":66,"groups":1},"bots":{"GPTBot":["partial","*",1,0],"ChatGPT-User":["partial","*",1,0],"OAI-SearchBot":["partial","*",1,0],"ClaudeBot":["partial","*",1,0],"Claude-User":["partial","*",1,0],"Claude-SearchBot":["partial","*",1,0],"anthropic-ai":["partial","*",1,0],"PerplexityBot":["partial","*",1,0],"Perplexity-User":["partial","*",1,0],"Google-Extended":["partial","*",1,0],"Applebot-Extended":["partial","*",1,0],"CCBot":["partial","*",1,0],"Bytespider":["partial","*",1,0],"Amazonbot":["partial","*",1,0],"meta-externalagent":["partial","*",1,0],"cohere-ai":["partial","*",1,0],"Diffbot":["partial","*",1,0],"ContextFetchBot":["partial","*",1,0]},"llms":{"status":"missing","http":404,"bytes":8017},"home":{"status":"ok","http":200,"url":"https://curl.se/","html_tokens":3469,"markdown_tokens":703,"pruned_tokens":170}},
{"rank":707,"domain":"itch.io","origin":"https://itch.io","robots":{"status":"ok","http":200,"url":"https://itch.io/robots.txt","bytes":249,"groups":1},"bots":{"GPTBot":["partial","*",8,0],"ChatGPT-User":["partial","*",8,0],"OAI-SearchBot":["partial","*",8,0],"ClaudeBot":["partial","*",8,0],"Claude-User":["partial","*",8,0],"Claude-SearchBot":["partial","*",8,0],"anthropic-ai":["partial","*",8,0],"PerplexityBot":["partial","*",8,0],"Perplexity-User":["partial","*",8,0],"Google-Extended":["partial","*",8,0],"Applebot-Extended":["partial","*",8,0],"CCBot":["partial","*",8,0],"Bytespider":["partial","*",8,0],"Amazonbot":["partial","*",8,0],"meta-externalagent":["partial","*",8,0],"cohere-ai":["partial","*",8,0],"Diffbot":["partial","*",8,0],"ContextFetchBot":["partial","*",8,0]},"llms":{"status":"missing","http":404,"bytes":6706},"home":{"status":"ok","http":200,"url":"https://itch.io/","html_tokens":28270,"markdown_tokens":300,"pruned_tokens":300}},
{"rank":708,"domain":"iea.org","origin":"https://iea.org","robots":{"status":"ok","http":200,"url":"https://www.iea.org/robots.txt","bytes":116,"groups":1},"bots":{"GPTBot":["partial","*",3,0],"ChatGPT-User":["partial","*",3,0],"OAI-SearchBot":["partial","*",3,0],"ClaudeBot":["partial","*",3,0],"Claude-User":["partial","*",3,0],"Claude-SearchBot":["partial","*",3,0],"anthropic-ai":["partial","*",3,0],"PerplexityBot":["partial","*",3,0],"Perplexity-User":["partial","*",3,0],"Google-Extended":["partial","*",3,0],"Applebot-Extended":["partial","*",3,0],"CCBot":["partial","*",3,0],"Bytespider":["partial","*",3,0],"Amazonbot":["partial","*",3,0],"meta-externalagent":["partial","*",3,0],"cohere-ai":["partial","*",3,0],"Diffbot":["partial","*",3,0],"ContextFetchBot":["partial","*",3,0]},"llms":{"status":"missing","http":404,"bytes":218315},"home":{"status":"ok","http":200,"url":"https://www.iea.org/","html_tokens":77222,"markdown_tokens":98,"pruned_tokens":59}},
{"rank":709,"domain":"addthis.com","origin":"https://addthis.com","robots":{"status":"refused","http":403,"url":"https://www.oracle.com/","bytes":1448,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":710,"domain":"amazon.fr","origin":"https://amazon.fr","robots":{"status":"ok","http":200,"url":"https://www.amazon.fr/robots.txt","bytes":7161,"groups":101},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["blocked","named",1,0],"OAI-SearchBot":["blocked","named",1,0],"ClaudeBot":["blocked","named",2,0],"Claude-User":["blocked","named",1,0],"Claude-SearchBot":["blocked","named",1,0],"anthropic-ai":["partial","*",90,8],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["blocked","named",1,0],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["partial","*",90,8],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["partial","*",90,8],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",90,8]},"llms":{"status":"missing","http":404,"bytes":2321},"home":{"status":"ok","http":200,"url":"https://www.amazon.fr/-/en/","html_tokens":223593,"markdown_tokens":1106,"pruned_tokens":174}},
{"rank":711,"domain":"mastodon.social","origin":"https://mastodon.social","robots":{"status":"ok","http":200,"url":"https://mastodon.social/robots.txt","bytes":231,"groups":2},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["partial","*",3,0],"OAI-SearchBot":["partial","*",3,0],"ClaudeBot":["partial","*",3,0],"Claude-User":["partial","*",3,0],"Claude-SearchBot":["partial","*",3,0],"anthropic-ai":["partial","*",3,0],"PerplexityBot":["partial","*",3,0],"Perplexity-User":["partial","*",3,0],"Google-Extended":["partial","*",3,0],"Applebot-Extended":["partial","*",3,0],"CCBot":["partial","*",3,0],"Bytespider":["partial","*",3,0],"Amazonbot":["partial","*",3,0],"meta-externalagent":["partial","*",3,0],"cohere-ai":["partial","*",3,0],"Diffbot":["partial","*",3,0],"ContextFetchBot":["partial","*",3,0]},"llms":{"status":"missing","http":404,"bytes":1539},"home":{"status":"ok","http":200,"url":"https://mastodon.social/","html_tokens":12580,"markdown_tokens":34,"pruned_tokens":34}},
{"rank":712,"domain":"smithsonianmag.com","origin":"https://smithsonianmag.com","robots":{"status":"ok","http":200,"url":"https://www.smithsonianmag.com/robots.txt","bytes":407,"groups":2},"bots":{"GPTBot":["partial","*",13,0],"ChatGPT-User":["partial","*",13,0],"OAI-SearchBot":["partial","*",13,0],"ClaudeBot":["partial","*",13,0],"Claude-User":["partial","*",13,0],"Claude-SearchBot":["partial","*",13,0],"anthropic-ai":["partial","*",13,0],"PerplexityBot":["partial","*",13,0],"Perplexity-User":["partial","*",13,0],"Google-Extended":["partial","*",13,0],"Applebot-Extended":["partial","*",13,0],"CCBot":["partial","*",13,0],"Bytespider":["partial","*",13,0],"Amazonbot":["partial","*",13,0],"meta-externalagent":["partial","*",13,0],"cohere-ai":["partial","*",13,0],"Diffbot":["partial","*",13,0],"ContextFetchBot":["partial","*",13,0]},"llms":{"status":"http_error","http":403,"bytes":5536},"home":{"status":"http_error","http":403,"url":"https://www.smithsonianmag.com/"}},
{"rank":713,"domain":"ethz.ch","origin":"https://ethz.ch","robots":{"status":"missing","http":404,"url":"https://ethz.ch/robots.txt","bytes":4121,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":7870},"home":{"status":"ok","http":200,"url":"https://ethz.ch/de.html","html_tokens":17257,"markdown_tokens":246,"pruned_tokens":77}},
{"rank":714,"domain":"hollywoodreporter.com","origin":"https://hollywoodreporter.com","robots":{"status":"ok","http":200,"url":"https://www.hollywoodreporter.com/robots.txt","bytes":1859,"groups":26},"bots":{"GPTBot":["partial","named",1,1],"ChatGPT-User":["partial","named",1,1],"OAI-SearchBot":["partial","named",1,1],"ClaudeBot":["partial","named",1,1],"Claude-User":["partial","*",9,1],"Claude-SearchBot":["partial","*",9,1],"anthropic-ai":["partial","named",1,1],"PerplexityBot":["partial","named",1,1],"Perplexity-User":["partial","*",9,1],"Google-Extended":["partial","named",1,1],"Applebot-Extended":["partial","named",1,1],"CCBot":["partial","named",1,1],"Bytespider":["partial","named",1,1],"Amazonbot":["partial","*",9,1],"meta-externalagent":["partial","named",1,1],"cohere-ai":["partial","named",1,1],"Diffbot":["partial","named",1,1],"ContextFetchBot":["partial","*",9,1]},"llms":{"status":"missing","http":404,"bytes":497258},"home":{"status":"ok","http":200,"url":"https://www.hollywoodreporter.com/","html_tokens":174541,"markdown_tokens":209,"pruned_tokens":71}},
{"rank":715,"domain":"mi.com","origin":"https://mi.com","robots":{"status":"refused","http":403,"url":"https://www.mi.com/robots.txt","bytes":378,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":716,"domain":"iop.org","origin":"https://iop.org","robots":{"status":"refused","http":403,"url":"https://iop.org/robots.txt","bytes":560,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":717,"domain":"claude.com","origin":"https://claude.com","robots":{"status":"ok","http":200,"url":"https://claude.com/robots.txt","bytes":107,"groups":1},"bots":{"GPTBot":["allowed","*",0,1],"ChatGPT-User":["allowed","*",0,1],"OAI-SearchBot":["allowed","*",0,1],"ClaudeBot":["allowed","*",0,1],"Claude-User":["allowed","*",0,1],"Claude-SearchBot":["allowed","*",0,1],"anthropic-ai":["allowed","*",0,1],"PerplexityBot":["allowed","*",0,1],"Perplexity-User":["allowed","*",0,1],"Google-Extended":["allowed","*",0,1],"Applebot-Extended":["allowed","*",0,1],"CCBot":["allowed","*",0,1],"Bytespider":["allowed","*",0,1],"Amazonbot":["allowed","*",0,1],"meta-externalagent":["allowed","*",0,1],"cohere-ai":["allowed","*",0,1],"Diffbot":["allowed","*",0,1],"ContextFetchBot":["allowed","*",0,1]},"llms":{"status":"missing","http":404,"bytes":107137},"home":{"status":"ok","http":200,"url":"https://claude.com/","html_tokens":86955,"markdown_tokens":459,"pruned_tokens":324}},
{"rank":718,"domain":"proofpoint.com","origin":"https://proofpoint.com","robots":{"status":"ok","http":200,"url":"https://www.proofpoint.com/robots.txt","bytes":105022,"groups":7},"bots":{"GPTBot":["partial","*",1215,50],"ChatGPT-User":["partial","*",1215,50],"OAI-SearchBot":["partial","*",1215,50],"ClaudeBot":["partial","*",1215,50],"Claude-User":["partial","*",1215,50],"Claude-SearchBot":["partial","*",1215,50],"anthropic-ai":["partial","*",1215,50],"PerplexityBot":["partial","*",1215,50],"Perplexity-User":["partial","*",1215,50],"Google-Extended":["partial","*",1215,50],"Applebot-Extended":["partial","*",1215,50],"CCBot":["partial","*",1215,50],"Bytespider":["partial","*",1215,50],"Amazonbot":["partial","*",1215,50],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["partial","*",1215,50],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",1215,50]},"llms":{"status":"missing","http":404,"bytes":192},"home":{"status":"ok","http":200,"url":"https://www.proofpoint.com/us","html_tokens":146506,"markdown_tokens":239,"pruned_tokens":45}},
{"rank":719,"domain":"hostinger.com","origin":"https://hostinger.com","robots":{"status":"ok","http":200,"url":"https://www.hostinger.com/robots.txt","bytes":21574,"groups":4},"bots":{"GPTBot":["partial","named",212,1],"ChatGPT-User":["partial","named",212,1],"OAI-SearchBot":["partial","named",212,1],"ClaudeBot":["partial","named",212,1],"Claude-User":["partial","named",212,1],"Claude-SearchBot":["partial","named",212,1],"anthropic-ai":["partial","*",279,0],"PerplexityBot":["partial","named",212,1],"Perplexity-User":["partial","named",212,1],"Google-Extended":["partial","named",212,1],"Applebot-Extended":["partial","*",279,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["partial","*",279,0],"meta-externalagent":["partial","named",212,1],"cohere-ai":["partial","*",279,0],"Diffbot":["partial","*",279,0],"ContextFetchBot":["partial","*",279,0]},"llms":{"status":"present","http":200,"bytes":12481},"home":{"status":"ok","http":200,"url":"https://www.hostinger.com/","html_tokens":240993,"markdown_tokens":1258,"pruned_tokens":723}},
{"rank":720,"domain":"phys.org","origin":"https://phys.org","robots":{"status":"ok","http":200,"url":"https://phys.org/robots.txt","bytes":1146,"groups":15},"bots":{"GPTBot":["partial","named",4,0],"ChatGPT-User":["partial","*",5,1],"OAI-SearchBot":["partial","*",5,1],"ClaudeBot":["partial","named",4,0],"Claude-User":["partial","*",5,1],"Claude-SearchBot":["partial","*",5,1],"anthropic-ai":["partial","*",5,1],"PerplexityBot":["partial","*",5,1],"Perplexity-User":["partial","*",5,1],"Google-Extended":["partial","named",2,0],"Applebot-Extended":["partial","named",2,0],"CCBot":["partial","named",4,0],"Bytespider":["partial","named",4,0],"Amazonbot":["partial","*",5,1],"meta-externalagent":["partial","*",5,1],"cohere-ai":["partial","named",4,0],"Diffbot":["partial","*",5,1],"ContextFetchBot":["partial","*",5,1]},"llms":{"status":"present","http":200,"bytes":2417},"home":{"status":"ok","http":200,"url":"https://phys.org/","html_tokens":51440,"markdown_tokens":2180,"pruned_tokens":162}},
{"rank":721,"domain":"pypi.org","origin":"https://pypi.org","robots":{"status":"ok","http":200,"url":"https://pypi.org/robots.txt","bytes":325,"groups":1},"bots":{"GPTBot":["partial","*",12,0],"ChatGPT-User":["partial","*",12,0],"OAI-SearchBot":["partial","*",12,0],"ClaudeBot":["partial","*",12,0],"Claude-User":["partial","*",12,0],"Claude-SearchBot":["partial","*",12,0],"anthropic-ai":["partial","*",12,0],"PerplexityBot":["partial","*",12,0],"Perplexity-User":["partial","*",12,0],"Google-Extended":["partial","*",12,0],"Applebot-Extended":["partial","*",12,0],"CCBot":["partial","*",12,0],"Bytespider":["partial","*",12,0],"Amazonbot":["partial","*",12,0],"meta-externalagent":["partial","*",12,0],"cohere-ai":["partial","*",12,0],"Diffbot":["partial","*",12,0],"ContextFetchBot":["partial","*",12,0]},"llms":{"status":"missing","http":404,"bytes":4820},"home":{"status":"ok","http":200,"url":"https://pypi.org/","html_tokens":6982,"markdown_tokens":81,"pruned_tokens":81}},
{"rank":722,"domain":"caniuse.com","origin":"https://caniuse.com","robots":{"status":"html","http":200,"url":"https://caniuse.com/?search=robots.txt","bytes":8790,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":723,"domain":"wipo.int","origin":"https://wipo.int","robots":{"status":"ok","http":200,"url":"https://www.wipo.int/robots.txt","bytes":3889,"groups":1},"bots":{"GPTBot":["partial","*",97,20],"ChatGPT-User":["partial","*",97,20],"OAI-SearchBot":["partial","*",97,20],"ClaudeBot":["partial","*",97,20],"Claude-User":["partial","*",97,20],"Claude-SearchBot":["partial","*",97,20],"anthropic-ai":["partial","*",97,20],"PerplexityBot":["partial","*",97,20],"Perplexity-User":["partial","*",97,20],"Google-Extended":["partial","*",97,20],"Applebot-Extended":["partial","*",97,20],"CCBot":["partial","*",97,20],"Bytespider":["partial","*",97,20],"Amazonbot":["partial","*",97,20],"meta-externalagent":["partial","*",97,20],"cohere-ai":["partial","*",97,20],"Diffbot":["partial","*",97,20],"ContextFetchBot":["partial","*",97,20]},"llms":{"status":"missing","http":404,"bytes":11969},"home":{"status":"ok","http":200,"url":"https://www.wipo.int/","html_tokens":139,"markdown_tokens":0,"pruned_tokens":0}},
{"rank":724,"domain":"medlineplus.gov","origin":"https://medlineplus.gov","robots":{"status":"ok","http":200,"url":"https://medlineplus.gov/robots.txt","bytes":447,"groups":1},"bots":{"GPTBot":["partial","*",15,0],"ChatGPT-User":["partial","*",15,0],"OAI-SearchBot":["partial","*",15,0],"ClaudeBot":["partial","*",15,0],"Claude-User":["partial","*",15,0],"Claude-SearchBot":["partial","*",15,0],"anthropic-ai":["partial","*",15,0],"PerplexityBot":["partial","*",15,0],"Perplexity-User":["partial","*",15,0],"Google-Extended":["partial","*",15,0],"Applebot-Extended":["partial","*",15,0],"CCBot":["partial","*",15,0],"Bytespider":["partial","*",15,0],"Amazonbot":["partial","*",15,0],"meta-externalagent":["partial","*",15,0],"cohere-ai":["partial","*",15,0],"Diffbot":["partial","*",15,0],"ContextFetchBot":["partial","*",15,0]},"llms":{"status":"missing","http":404,"bytes":17607},"home":{"status":"ok","http":200,"url":"https://medlineplus.gov/","html_tokens":6619,"markdown_tokens":133,"pruned_tokens":133}},
{"rank":725,"domain":"webs.com","origin":"https://webs.com","robots":{"status":"html","http":200,"url":"https://www.vistaprint.com/digital-marketing/webs-shutdown","bytes":173973,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":726,"domain":"dataprivacyframework.gov","origin":"https://dataprivacyframework.gov","robots":{"status":"ok","http":200,"url":"https://dataprivacyframework.gov/robots.txt","bytes":67,"groups":1},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"html","http":200,"bytes":2116},"home":{"status":"ok","http":200,"url":"https://dataprivacyframework.gov/","html_tokens":529,"markdown_tokens":12,"pruned_tokens":12}},
{"rank":727,"domain":"fcc.gov","origin":"https://fcc.gov","robots":{"status":"refused","http":403,"url":"https://fcc.gov/robots.txt","bytes":371,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":728,"domain":"stumbleupon.com","origin":"https://stumbleupon.com","robots":{"status":"ok","http":200,"url":"https://stumbleupon.com/robots.txt","bytes":172,"groups":1},"bots":{"GPTBot":["partial","*",8,0],"ChatGPT-User":["partial","*",8,0],"OAI-SearchBot":["partial","*",8,0],"ClaudeBot":["partial","*",8,0],"Claude-User":["partial","*",8,0],"Claude-SearchBot":["partial","*",8,0],"anthropic-ai":["partial","*",8,0],"PerplexityBot":["partial","*",8,0],"Perplexity-User":["partial","*",8,0],"Google-Extended":["partial","*",8,0],"Applebot-Extended":["partial","*",8,0],"CCBot":["partial","*",8,0],"Bytespider":["partial","*",8,0],"Amazonbot":["partial","*",8,0],"meta-externalagent":["partial","*",8,0],"cohere-ai":["partial","*",8,0],"Diffbot":["partial","*",8,0],"ContextFetchBot":["partial","*",8,0]},"llms":{"status":"missing","http":404,"bytes":5903},"home":{"status":"ok","http":200,"url":"https://stumbleupon.com/","html_tokens":1205,"markdown_tokens":36,"pruned_tokens":36}},
{"rank":729,"domain":"ign.com","origin":"https://ign.com","robots":{"status":"ok","http":200,"url":"https://www.ign.com/robots.txt","bytes":14178,"groups":4},"bots":{"GPTBot":["partial","named",2,131],"ChatGPT-User":["partial","named",2,131],"OAI-SearchBot":["partial","named",2,131],"ClaudeBot":["partial","named",1,130],"Claude-User":["partial","named",1,130],"Claude-SearchBot":["partial","named",1,130],"anthropic-ai":["partial","named",1,130],"PerplexityBot":["partial","named",1,130],"Perplexity-User":["partial","named",1,130],"Google-Extended":["partial","*",14,1],"Applebot-Extended":["partial","named",1,130],"CCBot":["partial","named",1,130],"Bytespider":["partial","named",1,130],"Amazonbot":["partial","named",1,130],"meta-externalagent":["partial","named",1,130],"cohere-ai":["partial","named",1,130],"Diffbot":["partial","named",1,130],"ContextFetchBot":["partial","*",14,1]},"llms":{"status":"missing","http":404,"bytes":3395},"home":{"status":"ok","http":200,"url":"https://www.ign.com/","html_tokens":86708,"markdown_tokens":36,"pruned_tokens":36}},
{"rank":730,"domain":"northwestern.edu","origin":"https://northwestern.edu","robots":{"status":"ok","http":200,"url":"https://www.northwestern.edu/robots.txt","bytes":2338,"groups":1},"bots":{"GPTBot":["partial","*",84,0],"ChatGPT-User":["partial","*",84,0],"OAI-SearchBot":["partial","*",84,0],"ClaudeBot":["partial","*",84,0],"Claude-User":["partial","*",84,0],"Claude-SearchBot":["partial","*",84,0],"anthropic-ai":["partial","*",84,0],"PerplexityBot":["partial","*",84,0],"Perplexity-User":["partial","*",84,0],"Google-Extended":["partial","*",84,0],"Applebot-Extended":["partial","*",84,0],"CCBot":["partial","*",84,0],"Bytespider":["partial","*",84,0],"Amazonbot":["partial","*",84,0],"meta-externalagent":["partial","*",84,0],"cohere-ai":["partial","*",84,0],"Diffbot":["partial","*",84,0],"ContextFetchBot":["partial","*",84,0]},"llms":{"status":"missing","http":404,"bytes":66256},"home":{"status":"ok","http":200,"url":"https://www.northwestern.edu/","html_tokens":22153,"markdown_tokens":130,"pruned_tokens":76}},
{"rank":731,"domain":"asu.edu","origin":"https://asu.edu","robots":{"status":"ok","http":200,"url":"https://www.asu.edu/robots.txt","bytes":1552,"groups":1},"bots":{"GPTBot":["partial","*",21,18],"ChatGPT-User":["partial","*",21,18],"OAI-SearchBot":["partial","*",21,18],"ClaudeBot":["partial","*",21,18],"Claude-User":["partial","*",21,18],"Claude-SearchBot":["partial","*",21,18],"anthropic-ai":["partial","*",21,18],"PerplexityBot":["partial","*",21,18],"Perplexity-User":["partial","*",21,18],"Google-Extended":["partial","*",21,18],"Applebot-Extended":["partial","*",21,18],"CCBot":["partial","*",21,18],"Bytespider":["partial","*",21,18],"Amazonbot":["partial","*",21,18],"meta-externalagent":["partial","*",21,18],"cohere-ai":["partial","*",21,18],"Diffbot":["partial","*",21,18],"ContextFetchBot":["partial","*",21,18]},"llms":{"status":"missing","http":404,"bytes":546},"home":{"status":"ok","http":200,"url":"https://www.asu.edu/","html_tokens":19350,"markdown_tokens":295,"pruned_tokens":296}},
{"rank":732,"domain":"cpan.org","origin":"https://cpan.org","robots":{"status":"ok","http":200,"url":"https://www.cpan.org/robots.txt","bytes":116,"groups":2},"bots":{"GPTBot":["partial","*",1,0],"ChatGPT-User":["partial","*",1,0],"OAI-SearchBot":["partial","*",1,0],"ClaudeBot":["partial","*",1,0],"Claude-User":["partial","*",1,0],"Claude-SearchBot":["partial","*",1,0],"anthropic-ai":["partial","*",1,0],"PerplexityBot":["partial","*",1,0],"Perplexity-User":["partial","*",1,0],"Google-Extended":["partial","*",1,0],"Applebot-Extended":["partial","*",1,0],"CCBot":["partial","*",1,0],"Bytespider":["partial","*",1,0],"Amazonbot":["partial","*",1,0],"meta-externalagent":["partial","*",1,0],"cohere-ai":["partial","*",1,0],"Diffbot":["partial","*",1,0],"ContextFetchBot":["partial","*",1,0]},"llms":{"status":"missing","http":404,"bytes":272},"home":{"status":"ok","http":200,"url":"https://www.cpan.org/","html_tokens":2182,"markdown_tokens":75,"pruned_tokens":75}},
{"rank":733,"domain":"thehill.com","origin":"https://thehill.com","robots":{"status":"ok","http":200,"url":"https://thehill.com/robots.txt","bytes":2218,"groups":30},"bots":{"GPTBot":["partial","named",1,1],"ChatGPT-User":["blocked","named",1,0],"OAI-SearchBot":["partial","named",1,1],"ClaudeBot":["partial","named",1,1],"Claude-User":["partial","*",13,2],"Claude-SearchBot":["partial","*",13,2],"anthropic-ai":["partial","named",1,1],"PerplexityBot":["partial","named",1,1],"Perplexity-User":["partial","*",13,2],"Google-Extended":["partial","named",1,1],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["partial","*",13,2],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",13,2]},"llms":{"status":"html","http":200,"bytes":188345},"home":{"status":"ok","http":200,"url":"https://thehill.com/","html_tokens":133531,"markdown_tokens":106,"pruned_tokens":106}},
{"rank":734,"domain":"colorado.edu","origin":"https://colorado.edu","robots":{"status":"ok","http":200,"url":"https://www.colorado.edu/robots.txt","bytes":2141,"groups":1},"bots":{"GPTBot":["partial","*",35,22],"ChatGPT-User":["partial","*",35,22],"OAI-SearchBot":["partial","*",35,22],"ClaudeBot":["partial","*",35,22],"Claude-User":["partial","*",35,22],"Claude-SearchBot":["partial","*",35,22],"anthropic-ai":["partial","*",35,22],"PerplexityBot":["partial","*",35,22],"Perplexity-User":["partial","*",35,22],"Google-Extended":["partial","*",35,22],"Applebot-Extended":["partial","*",35,22],"CCBot":["partial","*",35,22],"Bytespider":["partial","*",35,22],"Amazonbot":["partial","*",35,22],"meta-externalagent":["partial","*",35,22],"cohere-ai":["partial","*",35,22],"Diffbot":["partial","*",35,22],"ContextFetchBot":["partial","*",35,22]},"llms":{"status":"missing","http":404,"bytes":190},"home":{"status":"ok","http":200,"url":"https://www.colorado.edu/","html_tokens":21591,"markdown_tokens":344,"pruned_tokens":160}},
{"rank":735,"domain":"history.com","origin":"https://history.com","robots":{"status":"ok","http":200,"url":"https://www.history.com/robots.txt","bytes":96,"groups":1},"bots":{"GPTBot":["partial","*",2,0],"ChatGPT-User":["partial","*",2,0],"OAI-SearchBot":["partial","*",2,0],"ClaudeBot":["partial","*",2,0],"Claude-User":["partial","*",2,0],"Claude-SearchBot":["partial","*",2,0],"anthropic-ai":["partial","*",2,0],"PerplexityBot":["partial","*",2,0],"Perplexity-User":["partial","*",2,0],"Google-Extended":["partial","*",2,0],"Applebot-Extended":["partial","*",2,0],"CCBot":["partial","*",2,0],"Bytespider":["partial","*",2,0],"Amazonbot":["partial","*",2,0],"meta-externalagent":["partial","*",2,0],"cohere-ai":["partial","*",2,0],"Diffbot":["partial","*",2,0],"ContextFetchBot":["partial","*",2,0]},"llms":{"status":"missing","http":404,"bytes":45573},"home":{"status":"ok","http":200,"url":"https://www.history.com/","html_tokens":155274,"markdown_tokens":188,"pruned_tokens":40}},
{"rank":736,"domain":"news.cn","origin":"https://www.news.cn","robots":{"status":"ok","http":200,"url":"https://www.news.cn/robots.txt","bytes":74,"groups":1},"bots":{"GPTBot":["allowed","*",0,1],"ChatGPT-User":["allowed","*",0,1],"OAI-SearchBot":["allowed","*",0,1],"ClaudeBot":["allowed","*",0,1],"Claude-User":["allowed","*",0,1],"Claude-SearchBot":["allowed","*",0,1],"anthropic-ai":["allowed","*",0,1],"PerplexityBot":["allowed","*",0,1],"Perplexity-User":["allowed","*",0,1],"Google-Extended":["allowed","*",0,1],"Applebot-Extended":["allowed","*",0,1],"CCBot":["allowed","*",0,1],"Bytespider":["allowed","*",0,1],"Amazonbot":["allowed","*",0,1],"meta-externalagent":["allowed","*",0,1],"cohere-ai":["allowed","*",0,1],"Diffbot":["allowed","*",0,1],"ContextFetchBot":["allowed","*",0,1]},"llms":{"status":"missing","http":404,"bytes":4655},"home":{"status":"ok","http":200,"url":"https://www.news.cn/","html_tokens":37880,"markdown_tokens":81,"pruned_tokens":69}},
{"rank":737,"domain":"chicagotribune.com","origin":"https://chicagotribune.com","robots":{"status":"ok","http":200,"url":"https://www.chicagotribune.com/robots.txt","bytes":4493,"groups":68},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["blocked","named",1,0],"OAI-SearchBot":["blocked","named",1,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["blocked","named",1,0],"Claude-SearchBot":["blocked","named",1,0],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["blocked","named",1,0],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["partial","*",8,1],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",8,1]},"llms":{"status":"missing","http":404,"bytes":87158},"home":{"status":"ok","http":200,"url":"https://www.chicagotribune.com/","html_tokens":90860,"markdown_tokens":390,"pruned_tokens":48}},
{"rank":738,"domain":"urldefense.com","origin":"https://urldefense.com","robots":{"status":"ok","http":200,"url":"https://urldefense.com/robots.txt","bytes":26,"groups":1},"bots":{"GPTBot":["blocked","*",1,0],"ChatGPT-User":["blocked","*",1,0],"OAI-SearchBot":["blocked","*",1,0],"ClaudeBot":["blocked","*",1,0],"Claude-User":["blocked","*",1,0],"Claude-SearchBot":["blocked","*",1,0],"anthropic-ai":["blocked","*",1,0],"PerplexityBot":["blocked","*",1,0],"Perplexity-User":["blocked","*",1,0],"Google-Extended":["blocked","*",1,0],"Applebot-Extended":["blocked","*",1,0],"CCBot":["blocked","*",1,0],"Bytespider":["blocked","*",1,0],"Amazonbot":["blocked","*",1,0],"meta-externalagent":["blocked","*",1,0],"cohere-ai":["blocked","*",1,0],"Diffbot":["blocked","*",1,0],"ContextFetchBot":["blocked","*",1,0]},"llms":{"status":"disallowed"},"home":{"status":"disallowed"}},
{"rank":739,"domain":"broadcom.com","origin":"https://broadcom.com","robots":{"status":"ok","http":200,"url":"https://www.broadcom.com/robots.txt","bytes":1366,"groups":1},"bots":{"GPTBot":["partial","*",7,1],"ChatGPT-User":["partial","*",7,1],"OAI-SearchBot":["partial","*",7,1],"ClaudeBot":["partial","*",7,1],"Claude-User":["partial","*",7,1],"Claude-SearchBot":["partial","*",7,1],"anthropic-ai":["partial","*",7,1],"PerplexityBot":["partial","*",7,1],"Perplexity-User":["partial","*",7,1],"Google-Extended":["partial","*",7,1],"Applebot-Extended":["partial","*",7,1],"CCBot":["partial","*",7,1],"Bytespider":["partial","*",7,1],"Amazonbot":["partial","*",7,1],"meta-externalagent":["partial","*",7,1],"cohere-ai":["partial","*",7,1],"Diffbot":["partial","*",7,1],"ContextFetchBot":["partial","*",7,1]},"llms":{"status":"missing","http":404,"bytes":3751},"home":{"status":"ok","http":200,"url":"https://www.broadcom.com/","html_tokens":4186,"markdown_tokens":0,"pruned_tokens":0}},
{"rank":740,"domain":"tmall.com","origin":"https://tmall.com","robots":{"status":"ok","http":200,"url":"https://www.tmall.com/robots.txt","bytes":29,"groups":1},"bots":{"GPTBot":["partial","*",1,0],"ChatGPT-User":["partial","*",1,0],"OAI-SearchBot":["partial","*",1,0],"ClaudeBot":["partial","*",1,0],"Claude-User":["partial","*",1,0],"Claude-SearchBot":["partial","*",1,0],"anthropic-ai":["partial","*",1,0],"PerplexityBot":["partial","*",1,0],"Perplexity-User":["partial","*",1,0],"Google-Extended":["partial","*",1,0],"Applebot-Extended":["partial","*",1,0],"CCBot":["partial","*",1,0],"Bytespider":["partial","*",1,0],"Amazonbot":["partial","*",1,0],"meta-externalagent":["partial","*",1,0],"cohere-ai":["partial","*",1,0],"Diffbot":["partial","*",1,0],"ContextFetchBot":["partial","*",1,0]},"llms":{"status":"disallowed","error":"robots.txt of err.tmall.com disallows /error1.html?c=404&u=https%3A%2F%2Fwww.tmall.com%2Fllms.txt"},"home":{"status":"ok","http":200,"url":"https://www.tmall.com/","html_tokens":26184,"markdown_tokens":152,"pruned_tokens":24}},
{"rank":741,"domain":"dmoz.org","origin":"http://dmoz.org","robots":{"status":"missing","http":404,"url":"http://dmoz.org/robots.txt","bytes":4589,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":4587},"home":{"status":"http_error","http":404,"url":"http://dmoz.org/"}},
{"rank":742,"domain":"laracasts.com","origin":"https://laracasts.com","robots":{"status":"ok","http":200,"url":"https://laracasts.com/robots.txt","bytes":137,"groups":1},"bots":{"GPTBot":["partial","*",4,0],"ChatGPT-User":["partial","*",4,0],"OAI-SearchBot":["partial","*",4,0],"ClaudeBot":["partial","*",4,0],"Claude-User":["partial","*",4,0],"Claude-SearchBot":["partial","*",4,0],"anthropic-ai":["partial","*",4,0],"PerplexityBot":["partial","*",4,0],"Perplexity-User":["partial","*",4,0],"Google-Extended":["partial","*",4,0],"Applebot-Extended":["partial","*",4,0],"CCBot":["partial","*",4,0],"Bytespider":["partial","*",4,0],"Amazonbot":["partial","*",4,0],"meta-externalagent":["partial","*",4,0],"cohere-ai":["partial","*",4,0],"Diffbot":["partial","*",4,0],"ContextFetchBot":["partial","*",4,0]},"llms":{"status":"present","http":200,"bytes":5235},"home":{"status":"ok","http":200,"url":"https://laracasts.com/","html_tokens":147141,"markdown_tokens":457,"pruned_tokens":93}},
{"rank":743,"domain":"csdn.net","origin":"https://www.csdn.net","robots":{"status":"ok","http":200,"url":"https://www.csdn.net/robots.txt","bytes":299,"groups":1},"bots":{"GPTBot":["partial","*",15,0],"ChatGPT-User":["partial","*",15,0],"OAI-SearchBot":["partial","*",15,0],"ClaudeBot":["partial","*",15,0],"Claude-User":["partial","*",15,0],"Claude-SearchBot":["partial","*",15,0],"anthropic-ai":["partial","*",15,0],"PerplexityBot":["partial","*",15,0],"Perplexity-User":["partial","*",15,0],"Google-Extended":["partial","*",15,0],"Applebot-Extended":["partial","*",15,0],"CCBot":["partial","*",15,0],"Bytespider":["partial","*",15,0],"Amazonbot":["partial","*",15,0],"meta-externalagent":["partial","*",15,0],"cohere-ai":["partial","*",15,0],"Diffbot":["partial","*",15,0],"ContextFetchBot":["partial","*",15,0]},"llms":{"status":"http_error","http":521,"bytes":2007},"home":{"status":"http_error","http":521,"url":"https://www.csdn.net/"}},
{"rank":744,"domain":"faqs.org","origin":"http://faqs.org","robots":{"status":"ok","http":200,"url":"http://www.faqs.org/robots.txt","bytes":444,"groups":9},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["partial","*",4,0],"OAI-SearchBot":["partial","*",4,0],"ClaudeBot":["partial","*",4,0],"Claude-User":["partial","*",4,0],"Claude-SearchBot":["partial","*",4,0],"anthropic-ai":["partial","*",4,0],"PerplexityBot":["partial","*",4,0],"Perplexity-User":["partial","*",4,0],"Google-Extended":["partial","*",4,0],"Applebot-Extended":["partial","*",4,0],"CCBot":["partial","*",4,0],"Bytespider":["partial","*",4,0],"Amazonbot":["partial","*",4,0],"meta-externalagent":["partial","*",4,0],"cohere-ai":["partial","*",4,0],"Diffbot":["partial","*",4,0],"ContextFetchBot":["partial","*",4,0]},"llms":{"status":"missing","http":404,"bytes":3300},"home":{"status":"ok","http":200,"url":"http://www.faqs.org/faqs/","html_tokens":1646,"markdown_tokens":205,"pruned_tokens":146}},
{"rank":745,"domain":"haxx.se","origin":"https://haxx.se","robots":{"status":"missing","http":404,"url":"https://haxx.se/robots.txt","bytes":269,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":269},"home":{"status":"ok","http":200,"url":"https://haxx.se/","html_tokens":341,"markdown_tokens":219,"pruned_tokens":219}},
{"rank":746,"domain":"rt.com","origin":"https://rt.com","robots":{"status":"ok","http":200,"url":"https://rt-english.com/robots.txt","bytes":1289,"groups":3},"bots":{"GPTBot":["partial","*",15,3],"ChatGPT-User":["partial","*",15,3],"OAI-SearchBot":["partial","*",15,3],"ClaudeBot":["partial","*",15,3],"Claude-User":["partial","*",15,3],"Claude-SearchBot":["partial","*",15,3],"anthropic-ai":["partial","*",15,3],"PerplexityBot":["partial","*",15,3],"Perplexity-User":["partial","*",15,3],"Google-Extended":["partial","*",15,3],"Applebot-Extended":["partial","*",15,3],"CCBot":["partial","*",15,3],"Bytespider":["partial","*",15,3],"Amazonbot":["partial","*",15,3],"meta-externalagent":["partial","*",15,3],"cohere-ai":["partial","*",15,3],"Diffbot":["partial","*",15,3],"ContextFetchBot":["partial","*",15,3]},"llms":{"status":"missing","http":404,"bytes":136126},"home":{"status":"ok","http":200,"url":"https://rt-english.com/","html_tokens":90477,"markdown_tokens":53,"pruned_tokens":53}},
{"rank":747,"domain":"google.nl","origin":"https://google.nl","robots":{"status":"ok","http":200,"url":"https://www.google.nl/robots.txt","bytes":6624,"groups":4},"bots":{"GPTBot":["partial","*",169,64],"ChatGPT-User":["partial","*",169,64],"OAI-SearchBot":["partial","*",169,64],"ClaudeBot":["partial","*",169,64],"Claude-User":["partial","*",169,64],"Claude-SearchBot":["partial","*",169,64],"anthropic-ai":["partial","*",169,64],"PerplexityBot":["partial","*",169,64],"Perplexity-User":["partial","*",169,64],"Google-Extended":["partial","*",169,64],"Applebot-Extended":["partial","*",169,64],"CCBot":["partial","*",169,64],"Bytespider":["partial","*",169,64],"Amazonbot":["partial","*",169,64],"meta-externalagent":["partial","*",169,64],"cohere-ai":["partial","*",169,64],"Diffbot":["partial","*",169,64],"ContextFetchBot":["partial","*",169,64]},"llms":{"status":"missing","http":404,"bytes":1569},"home":{"status":"ok","http":200,"url":"https://www.google.com/","html_tokens":21299,"markdown_tokens":30,"pruned_tokens":30}},
{"rank":748,"domain":"ohchr.org","origin":"https://ohchr.org","robots":{"status":"refused","http":403,"url":"https://www.ohchr.org/en/ohchr_homepage","bytes":5551,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":749,"domain":"abcnews.com","origin":"https://abcnews.com","robots":{"status":"ok","http":200,"url":"https://abcnews.com/robots.txt","bytes":1702,"groups":12},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["blocked","named",1,0],"OAI-SearchBot":["partial","*",44,6],"ClaudeBot":["partial","*",44,6],"Claude-User":["partial","*",44,6],"Claude-SearchBot":["partial","*",44,6],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["partial","*",44,6],"Perplexity-User":["partial","*",44,6],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["partial","*",44,6],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["partial","*",44,6],"meta-externalagent":["partial","*",44,6],"cohere-ai":["partial","*",44,6],"Diffbot":["partial","*",44,6],"ContextFetchBot":["partial","*",44,6]},"llms":{"status":"missing","http":404,"bytes":1},"home":{"status":"ok","http":200,"url":"https://abcnews.com/","html_tokens":212172,"markdown_tokens":263,"pruned_tokens":112}},
{"rank":750,"domain":"theglobeandmail.com","origin":"https://theglobeandmail.com","robots":{"status":"ok","http":200,"url":"https://www.theglobeandmail.com/robots.txt","bytes":7529,"groups":184},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["blocked","named",1,0],"OAI-SearchBot":["blocked","named",1,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["blocked","named",1,0],"Claude-SearchBot":["blocked","named",1,0],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["blocked","named",1,0],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["partial","*",9,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",2,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",9,0]},"llms":{"status":"missing","http":404,"bytes":462462},"home":{"status":"ok","http":200,"url":"https://www.theglobeandmail.com/","html_tokens":786074,"markdown_tokens":279,"pruned_tokens":149}},
{"rank":751,"domain":"imagemagick.org","origin":"https://imagemagick.org","robots":{"status":"ok","http":200,"url":"https://imagemagick.org/robots.txt","bytes":38,"groups":1},"bots":{"GPTBot":["allowed","*",0,1],"ChatGPT-User":["allowed","*",0,1],"OAI-SearchBot":["allowed","*",0,1],"ClaudeBot":["allowed","*",0,1],"Claude-User":["allowed","*",0,1],"Claude-SearchBot":["allowed","*",0,1],"anthropic-ai":["allowed","*",0,1],"PerplexityBot":["allowed","*",0,1],"Perplexity-User":["allowed","*",0,1],"Google-Extended":["allowed","*",0,1],"Applebot-Extended":["allowed","*",0,1],"CCBot":["allowed","*",0,1],"Bytespider":["allowed","*",0,1],"Amazonbot":["allowed","*",0,1],"meta-externalagent":["allowed","*",0,1],"cohere-ai":["allowed","*",0,1],"Diffbot":["allowed","*",0,1],"ContextFetchBot":["allowed","*",0,1]},"llms":{"status":"present","http":200,"bytes":3463},"home":{"status":"ok","http":200,"url":"https://imagemagick.org/","html_tokens":6550,"markdown_tokens":133,"pruned_tokens":101}},
{"rank":752,"domain":"heise.de","origin":"https://heise.de","robots":{"status":"missing","http":410,"url":"https://heise.de/robots.txt","bytes":322,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":150035},"home":{"status":"ok","http":200,"url":"https://www.heise.de/","html_tokens":229306,"markdown_tokens":331,"pruned_tokens":134}},
{"rank":753,"domain":"thetimes.co.uk","origin":"https://thetimes.co.uk","robots":{"status":"ok","http":200,"url":"https://www.thetimes.co.uk/robots.txt","bytes":1700,"groups":2},"bots":{"GPTBot":["partial","named",8,1],"ChatGPT-User":["partial","named",8,1],"OAI-SearchBot":["partial","named",8,1],"ClaudeBot":["blocked","*",1,0],"Claude-User":["blocked","*",1,0],"Claude-SearchBot":["blocked","*",1,0],"anthropic-ai":["blocked","*",1,0],"PerplexityBot":["blocked","*",1,0],"Perplexity-User":["blocked","*",1,0],"Google-Extended":["blocked","*",1,0],"Applebot-Extended":["blocked","*",1,0],"CCBot":["blocked","*",1,0],"Bytespider":["blocked","*",1,0],"Amazonbot":["blocked","*",1,0],"meta-externalagent":["blocked","*",1,0],"cohere-ai":["blocked","*",1,0],"Diffbot":["blocked","*",1,0],"ContextFetchBot":["blocked","*",1,0]},"llms":{"status":"disallowed"},"home":{"status":"disallowed"}},
{"rank":754,"domain":"bizjournals.com","origin":"https://bizjournals.com","robots":{"status":"ok","http":200,"url":"https://www.bizjournals.com/robots.txt","bytes":5314,"groups":15},"bots":{"GPTBot":["partial","*",61,1],"ChatGPT-User":["partial","*",61,1],"OAI-SearchBot":["partial","*",61,1],"ClaudeBot":["partial","*",61,1],"Claude-User":["partial","*",61,1],"Claude-SearchBot":["partial","*",61,1],"anthropic-ai":["partial","*",61,1],"PerplexityBot":["partial","*",61,1],"Perplexity-User":["partial","*",61,1],"Google-Extended":["partial","*",61,1],"Applebot-Extended":["partial","*",61,1],"CCBot":["partial","*",61,1],"Bytespider":["partial","*",61,1],"Amazonbot":["partial","*",61,1],"meta-externalagent":["partial","*",61,1],"cohere-ai":["blocked","named",1,0],"Diffbot":["partial","*",61,1],"ContextFetchBot":["partial","*",61,1]},"llms":{"status":"missing","http":404,"bytes":201993},"home":{"status":"http_error","http":403,"url":"https://www.bizjournals.com/"}},
{"rank":755,"domain":"nike.com","origin":"https://nike.com","robots":{"status":"ok","http":200,"url":"https://www.nike.com/robots.txt","bytes":3503,"groups":6},"bots":{"GPTBot":["partial","*",18,0],"ChatGPT-User":["partial","*",18,0],"OAI-SearchBot":["partial","*",18,0],"ClaudeBot":["partial","*",18,0],"Claude-User":["partial","*",18,0],"Claude-SearchBot":["partial","*",18,0],"anthropic-ai":["partial","*",18,0],"PerplexityBot":["partial","*",18,0],"Perplexity-User":["partial","*",18,0],"Google-Extended":["partial","*",18,0],"Applebot-Extended":["partial","*",18,0],"CCBot":["partial","*",18,0],"Bytespider":["partial","*",18,0],"Amazonbot":["partial","*",18,0],"meta-externalagent":["partial","*",18,0],"cohere-ai":["partial","*",18,0],"Diffbot":["partial","*",18,0],"ContextFetchBot":["partial","*",18,0]},"llms":{"status":"missing","http":404,"bytes":568693},"home":{"status":"ok","http":200,"url":"https://www.nike.com/","html_tokens":235703,"markdown_tokens":77,"pruned_tokens":77}},
{"rank":756,"domain":"garmin.com","origin":"https://garmin.com","robots":{"status":"ok","http":200,"url":"https://www.garmin.com/robots.txt","bytes":783,"groups":3},"bots":{"GPTBot":["partial","*",4,1],"ChatGPT-User":["partial","*",4,1],"OAI-SearchBot":["partial","*",4,1],"ClaudeBot":["partial","*",4,1],"Claude-User":["partial","*",4,1],"Claude-SearchBot":["partial","*",4,1],"anthropic-ai":["partial","*",4,1],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["partial","*",4,1],"Google-Extended":["partial","*",4,1],"Applebot-Extended":["partial","*",4,1],"CCBot":["partial","*",4,1],"Bytespider":["partial","*",4,1],"Amazonbot":["partial","*",4,1],"meta-externalagent":["partial","*",4,1],"cohere-ai":["partial","*",4,1],"Diffbot":["partial","*",4,1],"ContextFetchBot":["partial","*",4,1]},"llms":{"status":"missing","http":404,"bytes":170918},"home":{"status":"ok","http":200,"url":"https://www.garmin.com/en-US/","html_tokens":60981,"markdown_tokens":125,"pruned_tokens":125}},
{"rank":757,"domain":"square.site","origin":"https://square.site","robots":{"status":"ok","http":200,"url":"https://squareup.com/robots.txt","bytes":2199,"groups":1},"bots":{"GPTBot":["partial","*",39,8],"ChatGPT-User":["partial","*",39,8],"OAI-SearchBot":["partial","*",39,8],"ClaudeBot":["partial","*",39,8],"Claude-User":["partial","*",39,8],"Claude-SearchBot":["partial","*",39,8],"anthropic-ai":["partial","*",39,8],"PerplexityBot":["partial","*",39,8],"Perplexity-User":["partial","*",39,8],"Google-Extended":["partial","*",39,8],"Applebot-Extended":["partial","*",39,8],"CCBot":["partial","*",39,8],"Bytespider":["partial","*",39,8],"Amazonbot":["partial","*",39,8],"meta-externalagent":["partial","*",39,8],"cohere-ai":["partial","*",39,8],"Diffbot":["partial","*",39,8],"ContextFetchBot":["partial","*",39,8]},"llms":{"status":"present","http":200,"bytes":22082},"home":{"status":"ok","http":200,"url":"https://squareup.com/us/en","html_tokens":446283,"markdown_tokens":529,"pruned_tokens":129}},
{"rank":758,"domain":"calameo.com","origin":"https://calameo.com","robots":{"status":"unreachable","http":null,"url":"https://calameo.com/robots.txt","bytes":0,"groups":0,"error":"Server disconnected without sending a response."},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":759,"domain":"kpmg.com","origin":"https://kpmg.com","robots":{"status":"ok","http":200,"url":"https://kpmg.com/robots.txt","bytes":672,"groups":3},"bots":{"GPTBot":["partial","*",14,1],"ChatGPT-User":["partial","*",14,1],"OAI-SearchBot":["partial","*",14,1],"ClaudeBot":["partial","*",14,1],"Claude-User":["partial","*",14,1],"Claude-SearchBot":["partial","*",14,1],"anthropic-ai":["partial","*",14,1],"PerplexityBot":["partial","*",14,1],"Perplexity-User":["partial","*",14,1],"Google-Extended":["partial","*",14,1],"Applebot-Extended":["partial","*",14,1],"CCBot":["partial","*",14,1],"Bytespider":["partial","*",14,1],"Amazonbot":["partial","*",14,1],"meta-externalagent":["partial","*",14,1],"cohere-ai":["partial","*",14,1],"Diffbot":["partial","*",14,1],"ContextFetchBot":["partial","*",14,1]},"llms":{"status":"present","http":200,"bytes":6837},"home":{"status":"ok","http":200,"url":"https://kpmg.com/xx/en.html","html_tokens":53191,"markdown_tokens":70,"pruned_tokens":62}},
{"rank":760,"domain":"treasury.gov","origin":"https://treasury.gov","robots":{"status":"timeout","http":null,"url":"https://treasury.gov/robots.txt","bytes":0,"groups":0,"error":"timed out after 10 s"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":761,"domain":"1drv.ms","origin":"https://1drv.ms","robots":{"status":"ok","http":200,"url":"https://1drv.ms/robots.txt","bytes":26,"groups":1},"bots":{"GPTBot":["blocked","*",1,0],"ChatGPT-User":["blocked","*",1,0],"OAI-SearchBot":["blocked","*",1,0],"ClaudeBot":["blocked","*",1,0],"Claude-User":["blocked","*",1,0],"Claude-SearchBot":["blocked","*",1,0],"anthropic-ai":["blocked","*",1,0],"PerplexityBot":["blocked","*",1,0],"Perplexity-User":["blocked","*",1,0],"Google-Extended":["blocked","*",1,0],"Applebot-Extended":["blocked","*",1,0],"CCBot":["blocked","*",1,0],"Bytespider":["blocked","*",1,0],"Amazonbot":["blocked","*",1,0],"meta-externalagent":["blocked","*",1,0],"cohere-ai":["blocked","*",1,0],"Diffbot":["blocked","*",1,0],"ContextFetchBot":["blocked","*",1,0]},"llms":{"status":"disallowed"},"home":{"status":"disallowed"}},
{"rank":762,"domain":"arizona.edu","origin":"https://arizona.edu","robots":{"status":"refused","http":403,"url":"https://arizona.edu/robots.txt","bytes":560,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":763,"domain":"anchor.fm","origin":"https://anchor.fm","robots":{"status":"html","http":200,"url":"https://creators.spotify.com/pod/profile/robots/.txt","bytes":58431,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":764,"domain":"paypal.me","origin":"https://paypal.me","robots":{"status":"ok","http":200,"url":"https://www.paypal.com/paypalme/robots.txt","bytes":175,"groups":3},"bots":{"GPTBot":["partial","*",1,3],"ChatGPT-User":["partial","*",1,3],"OAI-SearchBot":["partial","*",1,3],"ClaudeBot":["partial","*",1,3],"Claude-User":["partial","*",1,3],"Claude-SearchBot":["partial","*",1,3],"anthropic-ai":["partial","*",1,3],"PerplexityBot":["partial","*",1,3],"Perplexity-User":["partial","*",1,3],"Google-Extended":["partial","*",1,3],"Applebot-Extended":["partial","*",1,3],"CCBot":["partial","*",1,3],"Bytespider":["partial","*",1,3],"Amazonbot":["partial","*",1,3],"meta-externalagent":["partial","*",1,3],"cohere-ai":["partial","*",1,3],"Diffbot":["partial","*",1,3],"ContextFetchBot":["partial","*",1,3]},"llms":{"status":"disallowed"},"home":{"status":"ok","http":200,"url":"https://www.paypal.com/paypalme/","html_tokens":35388,"markdown_tokens":263,"pruned_tokens":98}},
{"rank":765,"domain":"sakura.ne.jp","origin":"https://sakura.ne.jp","robots":{"status":"ok","http":200,"url":"https://rs.sakura.ad.jp/robots.txt","bytes":222,"groups":2},"bots":{"GPTBot":["partial","*",2,1],"ChatGPT-User":["partial","*",2,1],"OAI-SearchBot":["partial","*",2,1],"ClaudeBot":["partial","*",2,1],"Claude-User":["partial","*",2,1],"Claude-SearchBot":["partial","*",2,1],"anthropic-ai":["partial","*",2,1],"PerplexityBot":["partial","*",2,1],"Perplexity-User":["partial","*",2,1],"Google-Extended":["partial","*",2,1],"Applebot-Extended":["partial","*",2,1],"CCBot":["partial","*",2,1],"Bytespider":["partial","*",2,1],"Amazonbot":["partial","*",2,1],"meta-externalagent":["partial","*",2,1],"cohere-ai":["partial","*",2,1],"Diffbot":["partial","*",2,1],"ContextFetchBot":["partial","*",2,1]},"llms":{"status":"missing","http":404,"bytes":10303},"home":{"status":"ok","http":200,"url":"https://rs.sakura.ad.jp/","html_tokens":13801,"markdown_tokens":198,"pruned_tokens":113}},
{"rank":766,"domain":"openldap.org","origin":"https://openldap.org","robots":{"status":"ok","http":200,"url":"https://openldap.org/robots.txt","bytes":462,"groups":1},"bots":{"GPTBot":["partial","*",7,0],"ChatGPT-User":["partial","*",7,0],"OAI-SearchBot":["partial","*",7,0],"ClaudeBot":["partial","*",7,0],"Claude-User":["partial","*",7,0],"Claude-SearchBot":["partial","*",7,0],"anthropic-ai":["partial","*",7,0],"PerplexityBot":["partial","*",7,0],"Perplexity-User":["partial","*",7,0],"Google-Extended":["partial","*",7,0],"Applebot-Extended":["partial","*",7,0],"CCBot":["partial","*",7,0],"Bytespider":["partial","*",7,0],"Amazonbot":["partial","*",7,0],"meta-externalagent":["partial","*",7,0],"cohere-ai":["partial","*",7,0],"Diffbot":["partial","*",7,0],"ContextFetchBot":["partial","*",7,0]},"llms":{"status":"missing","http":404,"bytes":153},"home":{"status":"ok","http":200,"url":"https://openldap.org/","html_tokens":1499,"markdown_tokens":134,"pruned_tokens":133}},
{"rank":767,"domain":"fontawesome.com","origin":"https://fontawesome.com","robots":{"status":"ok","http":200,"url":"https://fontawesome.com/robots.txt","bytes":358,"groups":1},"bots":{"GPTBot":["partial","*",7,0],"ChatGPT-User":["partial","*",7,0],"OAI-SearchBot":["partial","*",7,0],"ClaudeBot":["partial","*",7,0],"Claude-User":["partial","*",7,0],"Claude-SearchBot":["partial","*",7,0],"anthropic-ai":["partial","*",7,0],"PerplexityBot":["partial","*",7,0],"Perplexity-User":["partial","*",7,0],"Google-Extended":["partial","*",7,0],"Applebot-Extended":["partial","*",7,0],"CCBot":["partial","*",7,0],"Bytespider":["partial","*",7,0],"Amazonbot":["partial","*",7,0],"meta-externalagent":["partial","*",7,0],"cohere-ai":["partial","*",7,0],"Diffbot":["partial","*",7,0],"ContextFetchBot":["partial","*",7,0]},"llms":{"status":"present","http":200,"bytes":1413},"home":{"status":"ok","http":200,"url":"https://fontawesome.com/","html_tokens":5277,"markdown_tokens":0,"pruned_tokens":0}},
{"rank":768,"domain":"fbi.gov","origin":"https://fbi.gov","robots":{"status":"ok","http":200,"url":"https://www.fbi.gov/robots.txt","bytes":1017,"groups":2},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"missing","http":404,"bytes":39287},"home":{"status":"http_error","http":403,"url":"https://fbi.gov/"}},
{"rank":769,"domain":"bc.ca","origin":"https://bc.ca","robots":{"status":"unreachable","http":null,"url":"https://bc.ca/robots.txt","bytes":0,"groups":0,"error":"cannot resolve bc.ca"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":770,"domain":"podbean.com","origin":"https://podbean.com","robots":{"status":"ok","http":200,"url":"https://www.podbean.com/robots.txt","bytes":372,"groups":1},"bots":{"GPTBot":["partial","*",11,0],"ChatGPT-User":["partial","*",11,0],"OAI-SearchBot":["partial","*",11,0],"ClaudeBot":["partial","*",11,0],"Claude-User":["partial","*",11,0],"Claude-SearchBot":["partial","*",11,0],"anthropic-ai":["partial","*",11,0],"PerplexityBot":["partial","*",11,0],"Perplexity-User":["partial","*",11,0],"Google-Extended":["partial","*",11,0],"Applebot-Extended":["partial","*",11,0],"CCBot":["partial","*",11,0],"Bytespider":["partial","*",11,0],"Amazonbot":["partial","*",11,0],"meta-externalagent":["partial","*",11,0],"cohere-ai":["partial","*",11,0],"Diffbot":["partial","*",11,0],"ContextFetchBot":["partial","*",11,0]},"llms":{"status":"missing","http":404,"bytes":1284},"home":{"status":"ok","http":200,"url":"https://www.podbean.com/","html_tokens":46604,"markdown_tokens":63,"pruned_tokens":38}},
{"rank":771,"domain":"joomla.org","origin":"https://joomla.org","robots":{"status":"ok","http":200,"url":"https://www.joomla.org/robots.txt","bytes":1307,"groups":1},"bots":{"GPTBot":["partial","*",19,17],"ChatGPT-User":["partial","*",19,17],"OAI-SearchBot":["partial","*",19,17],"ClaudeBot":["partial","*",19,17],"Claude-User":["partial","*",19,17],"Claude-SearchBot":["partial","*",19,17],"anthropic-ai":["partial","*",19,17],"PerplexityBot":["partial","*",19,17],"Perplexity-User":["partial","*",19,17],"Google-Extended":["partial","*",19,17],"Applebot-Extended":["partial","*",19,17],"CCBot":["partial","*",19,17],"Bytespider":["partial","*",19,17],"Amazonbot":["partial","*",19,17],"meta-externalagent":["partial","*",19,17],"cohere-ai":["partial","*",19,17],"Diffbot":["partial","*",19,17],"ContextFetchBot":["partial","*",19,17]},"llms":{"status":"missing","http":404,"bytes":21970},"home":{"status":"ok","http":200,"url":"https://www.joomla.org/","html_tokens":12124,"markdown_tokens":79,"pruned_tokens":79}},
{"rank":772,"domain":"accor.com","origin":"https://accor.com","robots":{"status":"html","http":200,"url":"https://group.accor.com/en","bytes":797945,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":773,"domain":"binance.com","origin":"https://binance.com","robots":{"status":"ok","http":200,"url":"https://www.binance.com/robots.txt","bytes":6722,"groups":3},"bots":{"GPTBot":["partial","*",20,8],"ChatGPT-User":["partial","*",20,8],"OAI-SearchBot":["partial","*",20,8],"ClaudeBot":["partial","*",20,8],"Claude-User":["partial","*",20,8],"Claude-SearchBot":["partial","*",20,8],"anthropic-ai":["partial","*",20,8],"PerplexityBot":["partial","*",20,8],"Perplexity-User":["partial","*",20,8],"Google-Extended":["partial","*",20,8],"Applebot-Extended":["partial","*",20,8],"CCBot":["partial","*",20,8],"Bytespider":["partial","*",20,8],"Amazonbot":["partial","*",20,8],"meta-externalagent":["partial","*",20,8],"cohere-ai":["partial","*",20,8],"Diffbot":["partial","*",20,8],"ContextFetchBot":["partial","*",20,8]},"llms":{"status":"html","http":202,"bytes":2117},"home":{"status":"ok","http":202,"url":"https://www.binance.com/","html_tokens":527,"markdown_tokens":40,"pruned_tokens":40}},
{"rank":774,"domain":"virginia.edu","origin":"https://www.virginia.edu","robots":{"status":"ok","http":200,"url":"https://www.virginia.edu/robots.txt","bytes":152,"groups":1},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"missing","http":404,"bytes":26702},"home":{"status":"ok","http":200,"url":"https://www.virginia.edu/","html_tokens":12868,"markdown_tokens":134,"pruned_tokens":134}},
{"rank":775,"domain":"ameblo.jp","origin":"https://ameblo.jp","robots":{"status":"ok","http":200,"url":"https://ameblo.jp/robots.txt","bytes":1964,"groups":3},"bots":{"GPTBot":["partial","*",8,0],"ChatGPT-User":["partial","*",8,0],"OAI-SearchBot":["partial","*",8,0],"ClaudeBot":["partial","*",8,0],"Claude-User":["partial","*",8,0],"Claude-SearchBot":["partial","*",8,0],"anthropic-ai":["partial","*",8,0],"PerplexityBot":["partial","*",8,0],"Perplexity-User":["partial","*",8,0],"Google-Extended":["partial","*",8,0],"Applebot-Extended":["partial","*",8,0],"CCBot":["partial","*",8,0],"Bytespider":["partial","*",8,0],"Amazonbot":["partial","*",8,0],"meta-externalagent":["partial","*",8,0],"cohere-ai":["partial","*",8,0],"Diffbot":["partial","*",8,0],"ContextFetchBot":["partial","*",8,0]},"llms":{"status":"missing","http":404,"bytes":11074},"home":{"status":"ok","http":200,"url":"https://ameblo.jp/","html_tokens":27332,"markdown_tokens":59,"pruned_tokens":59}},
{"rank":776,"domain":"man7.org","origin":"https://man7.org","robots":{"status":"ok","http":200,"url":"https://man7.org/robots.txt","bytes":110,"groups":1},"bots":{"GPTBot":["partial","*",4,0],"ChatGPT-User":["partial","*",4,0],"OAI-SearchBot":["partial","*",4,0],"ClaudeBot":["partial","*",4,0],"Claude-User":["partial","*",4,0],"Claude-SearchBot":["partial","*",4,0],"anthropic-ai":["partial","*",4,0],"PerplexityBot":["partial","*",4,0],"Perplexity-User":["partial","*",4,0],"Google-Extended":["partial","*",4,0],"Applebot-Extended":["partial","*",4,0],"CCBot":["partial","*",4,0],"Bytespider":["partial","*",4,0],"Amazonbot":["partial","*",4,0],"meta-externalagent":["partial","*",4,0],"cohere-ai":["partial","*",4,0],"Diffbot":["partial","*",4,0],"ContextFetchBot":["partial","*",4,0]},"llms":{"status":"missing","http":404,"bytes":196},"home":{"status":"ok","http":200,"url":"https://man7.org/","html_tokens":1262,"markdown_tokens":11,"pruned_tokens":11}},
{"rank":777,"domain":"notion.so","origin":"https://notion.so","robots":{"status":"ok","http":200,"url":"https://www.notion.so/robots.txt","bytes":1136,"groups":6},"bots":{"GPTBot":["partial","*",11,1],"ChatGPT-User":["partial","*",11,1],"OAI-SearchBot":["partial","*",11,1],"ClaudeBot":["partial","*",11,1],"Claude-User":["partial","*",11,1],"Claude-SearchBot":["partial","*",11,1],"anthropic-ai":["partial","*",11,1],"PerplexityBot":["partial","*",11,1],"Perplexity-User":["partial","*",11,1],"Google-Extended":["partial","*",11,1],"Applebot-Extended":["partial","*",11,1],"CCBot":["partial","*",11,1],"Bytespider":["partial","*",11,1],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["partial","*",11,1],"cohere-ai":["partial","*",11,1],"Diffbot":["partial","*",11,1],"ContextFetchBot":["partial","*",11,1]},"llms":{"status":"other_text","http":200,"bytes":6780},"home":{"status":"ok","http":200,"url":"https://www.notion.com/","html_tokens":52540,"markdown_tokens":42,"pruned_tokens":11}},
{"rank":778,"domain":"khanacademy.org","origin":"https://khanacademy.org","robots":{"status":"html","http":200,"url":"https://www.khanacademy.org/robots.txt","bytes":3038,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":779,"domain":"france24.com","origin":"https://france24.com","robots":{"status":"ok","http":200,"url":"https://www.france24.com/robots.txt","bytes":12929,"groups":327},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["blocked","named",1,0],"OAI-SearchBot":["blocked","named",1,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["blocked","named",1,0],"Claude-SearchBot":["blocked","named",1,0],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["blocked","named",2,0],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["blocked","named",1,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"http_error","http":409,"bytes":248},"home":{"status":"ok","http":200,"url":"https://www.france24.com/en/","html_tokens":202103,"markdown_tokens":1188,"pruned_tokens":92}},
{"rank":780,"domain":"spb.ru","origin":"http://spb.ru","robots":{"status":"server_error","http":503,"url":"http://spb.ru/robots.txt","bytes":48623,"groups":0,"error":"HTTP 503"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":781,"domain":"fr.linkedin.com","origin":"https://fr.linkedin.com","robots":{"status":"ok","http":200,"url":"https://fr.linkedin.com/robots.txt","bytes":120190,"groups":77},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["blocked","named",1,0],"OAI-SearchBot":["partial","named",111,9],"ClaudeBot":["blocked","named",1,0],"Claude-User":["blocked","named",1,0],"Claude-SearchBot":["partial","named",111,9],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["blocked","named",1,0],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","*",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","*",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["blocked","*",1,0]},"llms":{"status":"disallowed"},"home":{"status":"disallowed"}},
{"rank":782,"domain":"deezer.com","origin":"https://deezer.com","robots":{"status":"ok","http":200,"url":"https://www.deezer.com/robots.txt","bytes":2103,"groups":3},"bots":{"GPTBot":["partial","named",5,0],"ChatGPT-User":["partial","named",18,1],"OAI-SearchBot":["partial","named",18,1],"ClaudeBot":["partial","named",5,0],"Claude-User":["partial","named",18,1],"Claude-SearchBot":["partial","named",18,1],"anthropic-ai":["partial","named",5,0],"PerplexityBot":["partial","named",18,1],"Perplexity-User":["partial","named",18,1],"Google-Extended":["partial","named",5,0],"Applebot-Extended":["partial","named",5,0],"CCBot":["partial","named",5,0],"Bytespider":["partial","named",5,0],"Amazonbot":["partial","named",18,1],"meta-externalagent":["partial","named",5,0],"cohere-ai":["partial","named",5,0],"Diffbot":["partial","named",5,0],"ContextFetchBot":["partial","*",18,1]},"llms":{"status":"missing","http":404,"bytes":8173},"home":{"status":"ok","http":200,"url":"https://www.deezer.com/us/","html_tokens":48457,"markdown_tokens":153,"pruned_tokens":153}},
{"rank":783,"domain":"miitbeian.gov.cn","origin":"https://miitbeian.gov.cn","robots":{"status":"unreachable","http":null,"url":"https://miitbeian.gov.cn/robots.txt","bytes":0,"groups":0,"error":"cannot resolve miitbeian.gov.cn"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":784,"domain":"moz.com","origin":"https://moz.com","robots":{"status":"ok","http":200,"url":"https://moz.com/robots.txt","bytes":1207,"groups":3},"bots":{"GPTBot":["partial","named",2,0],"ChatGPT-User":["partial","*",19,12],"OAI-SearchBot":["partial","*",19,12],"ClaudeBot":["partial","*",19,12],"Claude-User":["partial","*",19,12],"Claude-SearchBot":["partial","*",19,12],"anthropic-ai":["partial","*",19,12],"PerplexityBot":["partial","*",19,12],"Perplexity-User":["partial","*",19,12],"Google-Extended":["partial","*",19,12],"Applebot-Extended":["partial","*",19,12],"CCBot":["partial","*",19,12],"Bytespider":["partial","*",19,12],"Amazonbot":["partial","*",19,12],"meta-externalagent":["partial","*",19,12],"cohere-ai":["partial","*",19,12],"Diffbot":["partial","*",19,12],"ContextFetchBot":["partial","*",19,12]},"llms":{"status":"missing","http":404,"bytes":62850},"home":{"status":"ok","http":200,"url":"https://moz.com/","html_tokens":41813,"markdown_tokens":712,"pruned_tokens":325}},
{"rank":785,"domain":"angelfire.com","origin":"https://angelfire.com","robots":{"status":"timeout","http":null,"url":"https://angelfire.com/robots.txt","bytes":0,"groups":0,"error":"ConnectTimeout"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":786,"domain":"heart.org","origin":"https://heart.org","robots":{"status":"refused","http":403,"url":"https://www.heart.org/robots.txt","bytes":4903,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":787,"domain":"office365.com","origin":"https://office365.com","robots":{"status":"refused","http":403,"url":"https://office365.com/robots.txt","bytes":1892,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":788,"domain":"about.me","origin":"https://about.me","robots":{"status":"ok","http":200,"url":"https://about.me/robots.txt","bytes":4745,"groups":112},"bots":{"GPTBot":["partial","*",9,0],"ChatGPT-User":["partial","*",9,0],"OAI-SearchBot":["partial","*",9,0],"ClaudeBot":["partial","*",9,0],"Claude-User":["partial","*",9,0],"Claude-SearchBot":["partial","*",9,0],"anthropic-ai":["partial","*",9,0],"PerplexityBot":["partial","*",9,0],"Perplexity-User":["partial","*",9,0],"Google-Extended":["partial","*",9,0],"Applebot-Extended":["partial","*",9,0],"CCBot":["partial","*",9,0],"Bytespider":["partial","*",9,0],"Amazonbot":["partial","*",9,0],"meta-externalagent":["partial","*",9,0],"cohere-ai":["partial","*",9,0],"Diffbot":["partial","*",9,0],"ContextFetchBot":["partial","*",9,0]},"llms":{"status":"missing","http":404,"bytes":82946},"home":{"status":"ok","http":200,"url":"https://about.me/","html_tokens":58150,"markdown_tokens":1352,"pruned_tokens":102}},
{"rank":789,"domain":"asus.com","origin":"https://asus.com","robots":{"status":"ok","http":200,"url":"https://www.asus.com/robots.txt","bytes":480,"groups":1},"bots":{"GPTBot":["partial","*",12,0],"ChatGPT-User":["partial","*",12,0],"OAI-SearchBot":["partial","*",12,0],"ClaudeBot":["partial","*",12,0],"Claude-User":["partial","*",12,0],"Claude-SearchBot":["partial","*",12,0],"anthropic-ai":["partial","*",12,0],"PerplexityBot":["partial","*",12,0],"Perplexity-User":["partial","*",12,0],"Google-Extended":["partial","*",12,0],"Applebot-Extended":["partial","*",12,0],"CCBot":["partial","*",12,0],"Bytespider":["partial","*",12,0],"Amazonbot":["partial","*",12,0],"meta-externalagent":["partial","*",12,0],"cohere-ai":["partial","*",12,0],"Diffbot":["partial","*",12,0],"ContextFetchBot":["partial","*",12,0]},"llms":{"status":"present","http":200,"bytes":37083},"home":{"status":"ok","http":200,"url":"https://www.asus.com/","html_tokens":114297,"markdown_tokens":15,"pruned_tokens":15}},
{"rank":790,"domain":"olympics.com","origin":"https://olympics.com","robots":{"status":"unreachable","http":null,"url":"https://olympics.com/robots.txt","bytes":0,"groups":0,"error":"<StreamReset stream_id:1, error_code:2, remote_reset:True>"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":791,"domain":"ihg.com","origin":"https://ihg.com","robots":{"status":"refused","http":403,"url":"https://ihg.com/robots.txt","bytes":371,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":792,"domain":"cygwin.com","origin":"https://cygwin.com","robots":{"status":"ok","http":200,"url":"https://cygwin.com/robots.txt","bytes":684,"groups":6},"bots":{"GPTBot":["partial","*",16,0],"ChatGPT-User":["partial","*",16,0],"OAI-SearchBot":["partial","*",16,0],"ClaudeBot":["partial","*",16,0],"Claude-User":["partial","*",16,0],"Claude-SearchBot":["partial","*",16,0],"anthropic-ai":["partial","*",16,0],"PerplexityBot":["partial","*",16,0],"Perplexity-User":["partial","*",16,0],"Google-Extended":["partial","*",16,0],"Applebot-Extended":["partial","*",16,0],"CCBot":["partial","*",16,0],"Bytespider":["partial","*",16,0],"Amazonbot":["partial","*",16,0],"meta-externalagent":["partial","*",16,0],"cohere-ai":["partial","*",16,0],"Diffbot":["partial","*",16,0],"ContextFetchBot":["partial","*",16,0]},"llms":{"status":"missing","http":404,"bytes":196},"home":{"status":"ok","http":200,"url":"https://cygwin.com/","html_tokens":1481,"markdown_tokens":540,"pruned_tokens":163}},
{"rank":793,"domain":"owasp.org","origin":"https://owasp.org","robots":{"status":"ok","http":200,"url":"https://owasp.org/robots.txt","bytes":166,"groups":1},"bots":{"GPTBot":["partial","*",6,1],"ChatGPT-User":["partial","*",6,1],"OAI-SearchBot":["partial","*",6,1],"ClaudeBot":["partial","*",6,1],"Claude-User":["partial","*",6,1],"Claude-SearchBot":["partial","*",6,1],"anthropic-ai":["partial","*",6,1],"PerplexityBot":["partial","*",6,1],"Perplexity-User":["partial","*",6,1],"Google-Extended":["partial","*",6,1],"Applebot-Extended":["partial","*",6,1],"CCBot":["partial","*",6,1],"Bytespider":["partial","*",6,1],"Amazonbot":["partial","*",6,1],"meta-externalagent":["partial","*",6,1],"cohere-ai":["partial","*",6,1],"Diffbot":["partial","*",6,1],"ContextFetchBot":["partial","*",6,1]},"llms":{"status":"missing","http":404,"bytes":47782},"home":{"status":"ok","http":200,"url":"https://owasp.org/","html_tokens":23741,"markdown_tokens":300,"pruned_tokens":248}},
{"rank":794,"domain":"nsw.gov.au","origin":"https://nsw.gov.au","robots":{"status":"ok","http":200,"url":"https://www.nsw.gov.au/robots.txt","bytes":2204,"groups":1},"bots":{"GPTBot":["partial","*",40,18],"ChatGPT-User":["partial","*",40,18],"OAI-SearchBot":["partial","*",40,18],"ClaudeBot":["partial","*",40,18],"Claude-User":["partial","*",40,18],"Claude-SearchBot":["partial","*",40,18],"anthropic-ai":["partial","*",40,18],"PerplexityBot":["partial","*",40,18],"Perplexity-User":["partial","*",40,18],"Google-Extended":["partial","*",40,18],"Applebot-Extended":["partial","*",40,18],"CCBot":["partial","*",40,18],"Bytespider":["partial","*",40,18],"Amazonbot":["partial","*",40,18],"meta-externalagent":["partial","*",40,18],"cohere-ai":["partial","*",40,18],"Diffbot":["partial","*",40,18],"ContextFetchBot":["partial","*",40,18]},"llms":{"status":"other_text","http":200,"bytes":17247},"home":{"status":"ok","http":200,"url":"https://www.nsw.gov.au/","html_tokens":48015,"markdown_tokens":386,"pruned_tokens":174}},
{"rank":795,"domain":"jetbrains.com","origin":"https://jetbrains.com","robots":{"status":"ok","http":200,"url":"https://www.jetbrains.com/robots.txt","bytes":1760,"groups":2},"bots":{"GPTBot":["partial","*",7,0],"ChatGPT-User":["partial","*",7,0],"OAI-SearchBot":["partial","*",7,0],"ClaudeBot":["partial","*",7,0],"Claude-User":["partial","*",7,0],"Claude-SearchBot":["partial","*",7,0],"anthropic-ai":["partial","*",7,0],"PerplexityBot":["partial","*",7,0],"Perplexity-User":["partial","*",7,0],"Google-Extended":["partial","*",7,0],"Applebot-Extended":["partial","*",7,0],"CCBot":["partial","*",7,0],"Bytespider":["partial","*",7,0],"Amazonbot":["partial","*",7,0],"meta-externalagent":["partial","*",7,0],"cohere-ai":["partial","*",7,0],"Diffbot":["partial","*",7,0],"ContextFetchBot":["partial","*",7,0]},"llms":{"status":"missing","http":404,"bytes":40121},"home":{"status":"ok","http":200,"url":"https://www.jetbrains.com/","html_tokens":68572,"markdown_tokens":1003,"pruned_tokens":148}},
{"rank":796,"domain":"uspto.gov","origin":"https://uspto.gov","robots":{"status":"ok","http":200,"url":"https://www.uspto.gov/robots.txt","bytes":356,"groups":1},"bots":{"GPTBot":["partial","*",5,0],"ChatGPT-User":["partial","*",5,0],"OAI-SearchBot":["partial","*",5,0],"ClaudeBot":["partial","*",5,0],"Claude-User":["partial","*",5,0],"Claude-SearchBot":["partial","*",5,0],"anthropic-ai":["partial","*",5,0],"PerplexityBot":["partial","*",5,0],"Perplexity-User":["partial","*",5,0],"Google-Extended":["partial","*",5,0],"Applebot-Extended":["partial","*",5,0],"CCBot":["partial","*",5,0],"Bytespider":["partial","*",5,0],"Amazonbot":["partial","*",5,0],"meta-externalagent":["partial","*",5,0],"cohere-ai":["partial","*",5,0],"Diffbot":["partial","*",5,0],"ContextFetchBot":["partial","*",5,0]},"llms":{"status":"missing","http":404,"bytes":128234},"home":{"status":"ok","http":200,"url":"https://www.uspto.gov/","html_tokens":49553,"markdown_tokens":306,"pruned_tokens":88}},
{"rank":797,"domain":"lefigaro.fr","origin":"https://lefigaro.fr","robots":{"status":"ok","http":200,"url":"https://www.lefigaro.fr/robots.txt","bytes":12666,"groups":246},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["partial","named",1,9],"OAI-SearchBot":["partial","named",1,9],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","named",1,3],"Claude-SearchBot":["partial","named",1,3],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["partial","*",10,0],"Perplexity-User":["partial","*",10,0],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["partial","*",10,0],"CCBot":["blocked","named",1,0],"Bytespider":["partial","*",10,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["partial","*",10,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",10,0]},"llms":{"status":"missing","http":404,"bytes":170750},"home":{"status":"ok","http":200,"url":"https://www.lefigaro.fr/","html_tokens":308215,"markdown_tokens":95,"pruned_tokens":95}},
{"rank":798,"domain":"mozilla.com","origin":"https://mozilla.com","robots":{"status":"ok","http":200,"url":"https://www.mozilla.org/robots.txt","bytes":66,"groups":1},"bots":{"GPTBot":["partial","*",2,0],"ChatGPT-User":["partial","*",2,0],"OAI-SearchBot":["partial","*",2,0],"ClaudeBot":["partial","*",2,0],"Claude-User":["partial","*",2,0],"Claude-SearchBot":["partial","*",2,0],"anthropic-ai":["partial","*",2,0],"PerplexityBot":["partial","*",2,0],"Perplexity-User":["partial","*",2,0],"Google-Extended":["partial","*",2,0],"Applebot-Extended":["partial","*",2,0],"CCBot":["partial","*",2,0],"Bytespider":["partial","*",2,0],"Amazonbot":["partial","*",2,0],"meta-externalagent":["partial","*",2,0],"cohere-ai":["partial","*",2,0],"Diffbot":["partial","*",2,0],"ContextFetchBot":["partial","*",2,0]},"llms":{"status":"missing","http":404,"bytes":8955},"home":{"status":"ok","http":200,"url":"https://www.firefox.com/en-US/","html_tokens":35421,"markdown_tokens":222,"pruned_tokens":84}},
{"rank":799,"domain":"dhs.gov","origin":"https://dhs.gov","robots":{"status":"refused","http":403,"url":"https://dhs.gov/robots.txt","bytes":371,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":800,"domain":"uber.com","origin":"https://uber.com","robots":{"status":"ok","http":200,"url":"https://www.uber.com/robots.txt","bytes":904,"groups":3},"bots":{"GPTBot":["partial","*",27,3],"ChatGPT-User":["partial","*",27,3],"OAI-SearchBot":["partial","*",27,3],"ClaudeBot":["partial","*",27,3],"Claude-User":["partial","*",27,3],"Claude-SearchBot":["partial","*",27,3],"anthropic-ai":["partial","*",27,3],"PerplexityBot":["partial","*",27,3],"Perplexity-User":["partial","*",27,3],"Google-Extended":["partial","*",27,3],"Applebot-Extended":["partial","*",27,3],"CCBot":["partial","*",27,3],"Bytespider":["partial","*",27,3],"Amazonbot":["partial","*",27,3],"meta-externalagent":["partial","*",27,3],"cohere-ai":["partial","*",27,3],"Diffbot":["partial","*",27,3],"ContextFetchBot":["partial","*",27,3]},"llms":{"status":"missing","http":404,"bytes":465332},"home":{"status":"ok","http":200,"url":"https://www.uber.com/","html_tokens":238714,"markdown_tokens":589,"pruned_tokens":375}},
{"rank":801,"domain":"uol.com.br","origin":"https://uol.com.br","robots":{"status":"ok","http":200,"url":"https://www.uol.com.br/robots.txt","bytes":2515,"groups":50},"bots":{"GPTBot":["partial","*",3,1],"ChatGPT-User":["partial","*",3,1],"OAI-SearchBot":["partial","*",3,1],"ClaudeBot":["blocked","named",1,0],"Claude-User":["blocked","named",1,0],"Claude-SearchBot":["blocked","named",1,0],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["blocked","named",1,0],"Google-Extended":["partial","named",1,1],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",3,1]},"llms":{"status":"missing","http":404,"bytes":506},"home":{"status":"ok","http":200,"url":"https://www.uol.com.br/","html_tokens":179062,"markdown_tokens":23,"pruned_tokens":23}},
{"rank":802,"domain":"weather.com","origin":"https://weather.com","robots":{"status":"ok","http":200,"url":"https://weather.com/robots.txt","bytes":3780,"groups":26},"bots":{"GPTBot":["partial","*",65,1],"ChatGPT-User":["allowed","named",0,1],"OAI-SearchBot":["partial","*",65,1],"ClaudeBot":["partial","*",65,1],"Claude-User":["partial","*",65,1],"Claude-SearchBot":["partial","*",65,1],"anthropic-ai":["partial","*",65,1],"PerplexityBot":["partial","*",65,1],"Perplexity-User":["partial","*",65,1],"Google-Extended":["partial","*",65,1],"Applebot-Extended":["partial","*",65,1],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["partial","*",65,1],"cohere-ai":["partial","*",65,1],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",65,1]},"llms":{"status":"present","http":200,"bytes":9101},"home":{"status":"ok","http":200,"url":"https://weather.com/","html_tokens":87823,"markdown_tokens":46,"pruned_tokens":46}},
{"rank":803,"domain":"youronlinechoices.eu","origin":"https://youronlinechoices.eu","robots":{"status":"ok","http":200,"url":"https://youronlinechoices.eu/robots.txt","bytes":63,"groups":1},"bots":{"GPTBot":["partial","*",2,0],"ChatGPT-User":["partial","*",2,0],"OAI-SearchBot":["partial","*",2,0],"ClaudeBot":["partial","*",2,0],"Claude-User":["partial","*",2,0],"Claude-SearchBot":["partial","*",2,0],"anthropic-ai":["partial","*",2,0],"PerplexityBot":["partial","*",2,0],"Perplexity-User":["partial","*",2,0],"Google-Extended":["partial","*",2,0],"Applebot-Extended":["partial","*",2,0],"CCBot":["partial","*",2,0],"Bytespider":["partial","*",2,0],"Amazonbot":["partial","*",2,0],"meta-externalagent":["partial","*",2,0],"cohere-ai":["partial","*",2,0],"Diffbot":["partial","*",2,0],"ContextFetchBot":["partial","*",2,0]},"llms":{"status":"missing","http":404,"bytes":1291},"home":{"status":"ok","http":200,"url":"https://youronlinechoices.eu/","html_tokens":1855,"markdown_tokens":447,"pruned_tokens":121}},
{"rank":804,"domain":"mastercard.com","origin":"https://mastercard.com","robots":{"status":"refused","http":403,"url":"https://www.mastercard.com/robots.txt","bytes":386,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":805,"domain":"evernote.com","origin":"https://evernote.com","robots":{"status":"ok","http":200,"url":"https://evernote.com/robots.txt","bytes":166,"groups":2},"bots":{"GPTBot":["partial","*",1,1],"ChatGPT-User":["partial","*",1,1],"OAI-SearchBot":["partial","*",1,1],"ClaudeBot":["partial","*",1,1],"Claude-User":["partial","*",1,1],"Claude-SearchBot":["partial","*",1,1],"anthropic-ai":["partial","*",1,1],"PerplexityBot":["partial","*",1,1],"Perplexity-User":["partial","*",1,1],"Google-Extended":["partial","*",1,1],"Applebot-Extended":["partial","*",1,1],"CCBot":["partial","*",1,1],"Bytespider":["partial","*",1,1],"Amazonbot":["partial","*",1,1],"meta-externalagent":["partial","*",1,1],"cohere-ai":["partial","*",1,1],"Diffbot":["partial","*",1,1],"ContextFetchBot":["partial","*",1,1]},"llms":{"status":"present","http":200,"bytes":10639},"home":{"status":"ok","http":200,"url":"https://evernote.com/","html_tokens":94911,"markdown_tokens":85,"pruned_tokens":50}},
{"rank":806,"domain":"dot.gov","origin":"https://dot.gov","robots":{"status":"refused","http":403,"url":"https://www.dot.gov","bytes":365,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":807,"domain":"va.gov","origin":"https://va.gov","robots":{"status":"ok","http":200,"url":"https://www.va.gov/robots.txt","bytes":863,"groups":4},"bots":{"GPTBot":["partial","*",7,0],"ChatGPT-User":["partial","*",7,0],"OAI-SearchBot":["partial","*",7,0],"ClaudeBot":["partial","*",7,0],"Claude-User":["partial","*",7,0],"Claude-SearchBot":["partial","*",7,0],"anthropic-ai":["partial","*",7,0],"PerplexityBot":["partial","*",7,0],"Perplexity-User":["partial","*",7,0],"Google-Extended":["partial","*",7,0],"Applebot-Extended":["partial","*",7,0],"CCBot":["partial","*",7,0],"Bytespider":["partial","*",7,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["partial","*",7,0],"cohere-ai":["partial","*",7,0],"Diffbot":["partial","*",7,0],"ContextFetchBot":["partial","*",7,0]},"llms":{"status":"missing","http":404,"bytes":86797},"home":{"status":"ok","http":200,"url":"https://www.va.gov/","html_tokens":30982,"markdown_tokens":540,"pruned_tokens":86}},
{"rank":808,"domain":"cancer.gov","origin":"https://cancer.gov","robots":{"status":"ok","http":200,"url":"https://www.cancer.gov/robots.txt","bytes":57,"groups":1},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"missing","http":404,"bytes":24040},"home":{"status":"ok","http":200,"url":"https://www.cancer.gov/","html_tokens":10592,"markdown_tokens":448,"pruned_tokens":55}},
{"rank":809,"domain":"verisign.com","origin":"https://verisign.com","robots":{"status":"ok","http":200,"url":"https://www.verisign.com/robots.txt","bytes":62,"groups":1},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"missing","http":404,"bytes":101166},"home":{"status":"ok","http":200,"url":"https://www.verisign.com/","html_tokens":30690,"markdown_tokens":130,"pruned_tokens":62}},
{"rank":810,"domain":"brookings.edu","origin":"https://brookings.edu","robots":{"status":"ok","http":200,"url":"https://www.brookings.edu/robots.txt","bytes":16664,"groups":2},"bots":{"GPTBot":["partial","*",287,0],"ChatGPT-User":["partial","*",287,0],"OAI-SearchBot":["partial","*",287,0],"ClaudeBot":["partial","*",287,0],"Claude-User":["partial","*",287,0],"Claude-SearchBot":["partial","*",287,0],"anthropic-ai":["partial","*",287,0],"PerplexityBot":["partial","*",287,0],"Perplexity-User":["partial","*",287,0],"Google-Extended":["partial","*",287,0],"Applebot-Extended":["partial","*",287,0],"CCBot":["partial","*",287,0],"Bytespider":["partial","*",287,0],"Amazonbot":["partial","*",287,0],"meta-externalagent":["partial","*",287,0],"cohere-ai":["partial","*",287,0],"Diffbot":["partial","*",287,0],"ContextFetchBot":["partial","*",287,0]},"llms":{"status":"missing","http":404,"bytes":88923},"home":{"status":"ok","http":200,"url":"https://www.brookings.edu/","html_tokens":43793,"markdown_tokens":203,"pruned_tokens":203}},
{"rank":811,"domain":"webflow.io","origin":"https://webflow.io","robots":{"status":"html","http":200,"url":"https://webflow.com","bytes":524847,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":812,"domain":"kaspersky.com","origin":"https://kaspersky.com","robots":{"status":"ok","http":200,"url":"https://www.kaspersky.com/robots.txt","bytes":1929,"groups":18},"bots":{"GPTBot":["allowed","named",0,1],"ChatGPT-User":["allowed","named",0,1],"OAI-SearchBot":["allowed","named",0,1],"ClaudeBot":["allowed","named",0,1],"Claude-User":["allowed","named",0,1],"Claude-SearchBot":["partial","*",23,4],"anthropic-ai":["partial","*",23,4],"PerplexityBot":["allowed","named",0,1],"Perplexity-User":["allowed","named",0,1],"Google-Extended":["allowed","named",0,1],"Applebot-Extended":["allowed","named",0,1],"CCBot":["allowed","named",0,1],"Bytespider":["partial","*",23,4],"Amazonbot":["partial","*",23,4],"meta-externalagent":["partial","*",23,4],"cohere-ai":["partial","*",23,4],"Diffbot":["allowed","named",0,1],"ContextFetchBot":["partial","*",23,4]},"llms":{"status":"present","http":200,"bytes":44633},"home":{"status":"ok","http":200,"url":"https://www.kaspersky.com/","html_tokens":220415,"markdown_tokens":71,"pruned_tokens":56}},
{"rank":813,"domain":"accenture.com","origin":"https://accenture.com","robots":{"status":"ok","http":200,"url":"https://www.accenture.com/robots.txt","bytes":836,"groups":1},"bots":{"GPTBot":["partial","*",29,0],"ChatGPT-User":["partial","*",29,0],"OAI-SearchBot":["partial","*",29,0],"ClaudeBot":["partial","*",29,0],"Claude-User":["partial","*",29,0],"Claude-SearchBot":["partial","*",29,0],"anthropic-ai":["partial","*",29,0],"PerplexityBot":["partial","*",29,0],"Perplexity-User":["partial","*",29,0],"Google-Extended":["partial","*",29,0],"Applebot-Extended":["partial","*",29,0],"CCBot":["partial","*",29,0],"Bytespider":["partial","*",29,0],"Amazonbot":["partial","*",29,0],"meta-externalagent":["partial","*",29,0],"cohere-ai":["partial","*",29,0],"Diffbot":["partial","*",29,0],"ContextFetchBot":["partial","*",29,0]},"llms":{"status":"missing","http":404,"bytes":167561},"home":{"status":"ok","http":200,"url":"https://www.accenture.com/en","html_tokens":58334,"markdown_tokens":80,"pruned_tokens":80}},
{"rank":814,"domain":"google.cn","origin":"https://google.cn","robots":{"status":"missing","http":404,"url":"https://google.cn/robots.txt","bytes":1571,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":1569},"home":{"status":"ok","http":200,"url":"http://www.google.cn/","html_tokens":346,"markdown_tokens":2,"pruned_tokens":2}},
{"rank":815,"domain":"wa.gov","origin":"https://wa.gov","robots":{"status":"ok","http":200,"url":"https://wa.gov/robots.txt","bytes":139,"groups":3},"bots":{"GPTBot":["blocked","*",1,0],"ChatGPT-User":["blocked","*",1,0],"OAI-SearchBot":["blocked","*",1,0],"ClaudeBot":["blocked","*",1,0],"Claude-User":["blocked","*",1,0],"Claude-SearchBot":["blocked","*",1,0],"anthropic-ai":["blocked","*",1,0],"PerplexityBot":["blocked","*",1,0],"Perplexity-User":["blocked","*",1,0],"Google-Extended":["blocked","*",1,0],"Applebot-Extended":["blocked","*",1,0],"CCBot":["blocked","*",1,0],"Bytespider":["blocked","*",1,0],"Amazonbot":["blocked","*",1,0],"meta-externalagent":["blocked","*",1,0],"cohere-ai":["blocked","*",1,0],"Diffbot":["blocked","*",1,0],"ContextFetchBot":["blocked","*",1,0]},"llms":{"status":"disallowed"},"home":{"status":"disallowed"}},
{"rank":816,"domain":"aliyuncs.com","origin":"https://aliyuncs.com","robots":{"status":"missing","http":404,"url":"https://aliyuncs.com/robots.txt","bytes":611,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":609},"home":{"status":"http_error","http":404,"url":"https://aliyuncs.com/"}},
{"rank":817,"domain":"ed.ac.uk","origin":"https://www.ed.ac.uk","robots":{"status":"refused","http":403,"url":"https://www.ed.ac.uk/robots.txt","bytes":560,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":818,"domain":"mpg.de","origin":"https://mpg.de","robots":{"status":"ok","http":200,"url":"https://www.mpg.de/robots.txt","bytes":87,"groups":1},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"missing","http":404,"bytes":0},"home":{"status":"ok","http":200,"url":"https://www.mpg.de/de","html_tokens":124415,"markdown_tokens":767,"pruned_tokens":171}},
{"rank":819,"domain":"cdn.shopify.com","origin":"https://cdn.shopify.com","robots":{"status":"ok","http":200,"url":"https://cdn.shopify.com/robots.txt","bytes":80,"groups":1},"bots":{"GPTBot":["partial","*",2,0],"ChatGPT-User":["partial","*",2,0],"OAI-SearchBot":["partial","*",2,0],"ClaudeBot":["partial","*",2,0],"Claude-User":["partial","*",2,0],"Claude-SearchBot":["partial","*",2,0],"anthropic-ai":["partial","*",2,0],"PerplexityBot":["partial","*",2,0],"Perplexity-User":["partial","*",2,0],"Google-Extended":["partial","*",2,0],"Applebot-Extended":["partial","*",2,0],"CCBot":["partial","*",2,0],"Bytespider":["partial","*",2,0],"Amazonbot":["partial","*",2,0],"meta-externalagent":["partial","*",2,0],"cohere-ai":["partial","*",2,0],"Diffbot":["partial","*",2,0],"ContextFetchBot":["partial","*",2,0]},"llms":{"status":"missing","http":404,"bytes":2770},"home":{"status":"ok","http":200,"url":"https://cdn.shopify.com/","html_tokens":17611,"markdown_tokens":247,"pruned_tokens":162}},
{"rank":820,"domain":"webex.com","origin":"https://webex.com","robots":{"status":"refused","http":403,"url":"https://www.webex.com/robots.txt","bytes":381,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":821,"domain":"canva.site","origin":"https://canva.site","robots":{"status":"missing","http":404,"url":"https://canva.site/robots.txt","bytes":587,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":587},"home":{"status":"http_error","http":404,"url":"https://canva.site/"}},
{"rank":822,"domain":"texas.gov","origin":"https://texas.gov","robots":{"status":"ok","http":200,"url":"https://www.texas.gov/robots.txt","bytes":328,"groups":1},"bots":{"GPTBot":["partial","*",6,1],"ChatGPT-User":["partial","*",6,1],"OAI-SearchBot":["partial","*",6,1],"ClaudeBot":["partial","*",6,1],"Claude-User":["partial","*",6,1],"Claude-SearchBot":["partial","*",6,1],"anthropic-ai":["partial","*",6,1],"PerplexityBot":["partial","*",6,1],"Perplexity-User":["partial","*",6,1],"Google-Extended":["partial","*",6,1],"Applebot-Extended":["partial","*",6,1],"CCBot":["partial","*",6,1],"Bytespider":["partial","*",6,1],"Amazonbot":["partial","*",6,1],"meta-externalagent":["partial","*",6,1],"cohere-ai":["partial","*",6,1],"Diffbot":["partial","*",6,1],"ContextFetchBot":["partial","*",6,1]},"llms":{"status":"missing","http":404,"bytes":152373},"home":{"status":"ok","http":200,"url":"https://www.texas.gov/","html_tokens":41552,"markdown_tokens":195,"pruned_tokens":106}},
{"rank":823,"domain":"people.com","origin":"https://people.com","robots":{"status":"ok","http":200,"url":"https://people.com/robots.txt","bytes":3457,"groups":6},"bots":{"GPTBot":["partial","named",1,0],"ChatGPT-User":["partial","named",1,0],"OAI-SearchBot":["partial","named",1,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["blocked","named",1,0],"Claude-SearchBot":["partial","named",1,2],"anthropic-ai":["partial","named",1,2],"PerplexityBot":["partial","named",1,2],"Perplexity-User":["blocked","named",1,0],"Google-Extended":["partial","named",1,2],"Applebot-Extended":["blocked","named",1,0],"CCBot":["partial","named",1,2],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["partial","named",1,2],"cohere-ai":["partial","named",1,2],"Diffbot":["partial","*",2,0],"ContextFetchBot":["partial","*",2,0]},"llms":{"status":"http_error","http":402,"bytes":612},"home":{"status":"http_error","http":402,"url":"https://people.com/"}},
{"rank":824,"domain":"globo.com","origin":"https://globo.com","robots":{"status":"ok","http":200,"url":"https://www.globo.com/robots.txt","bytes":408,"groups":4},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["partial","*",10,0],"OAI-SearchBot":["partial","*",10,0],"ClaudeBot":["partial","*",10,0],"Claude-User":["partial","*",10,0],"Claude-SearchBot":["partial","*",10,0],"anthropic-ai":["partial","*",10,0],"PerplexityBot":["partial","*",10,0],"Perplexity-User":["partial","*",10,0],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["partial","*",10,0],"CCBot":["blocked","named",1,0],"Bytespider":["partial","*",10,0],"Amazonbot":["partial","*",10,0],"meta-externalagent":["partial","*",10,0],"cohere-ai":["partial","*",10,0],"Diffbot":["partial","*",10,0],"ContextFetchBot":["partial","*",10,0]},"llms":{"status":"missing","http":404,"bytes":4167},"home":{"status":"ok","http":200,"url":"https://www.globo.com/","html_tokens":669075,"markdown_tokens":18,"pruned_tokens":18}},
{"rank":825,"domain":"amazon.ca","origin":"https://amazon.ca","robots":{"status":"ok","http":200,"url":"https://www.amazon.ca/robots.txt","bytes":6179,"groups":101},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["blocked","named",1,0],"OAI-SearchBot":["blocked","named",1,0],"ClaudeBot":["blocked","named",2,0],"Claude-User":["blocked","named",1,0],"Claude-SearchBot":["blocked","named",1,0],"anthropic-ai":["partial","*",63,8],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["blocked","named",1,0],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["partial","*",63,8],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["partial","*",63,8],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",63,8]},"llms":{"status":"missing","http":404,"bytes":2255},"home":{"status":"ok","http":200,"url":"https://www.amazon.ca/","html_tokens":182650,"markdown_tokens":95,"pruned_tokens":53}},
{"rank":826,"domain":"instructables.com","origin":"https://instructables.com","robots":{"status":"ok","http":200,"url":"https://www.instructables.com/robots.txt","bytes":242,"groups":3},"bots":{"GPTBot":["partial","*",6,0],"ChatGPT-User":["partial","*",6,0],"OAI-SearchBot":["partial","*",6,0],"ClaudeBot":["partial","*",6,0],"Claude-User":["partial","*",6,0],"Claude-SearchBot":["partial","*",6,0],"anthropic-ai":["partial","*",6,0],"PerplexityBot":["partial","*",6,0],"Perplexity-User":["partial","*",6,0],"Google-Extended":["partial","*",6,0],"Applebot-Extended":["partial","*",6,0],"CCBot":["partial","*",6,0],"Bytespider":["partial","*",6,0],"Amazonbot":["partial","*",6,0],"meta-externalagent":["partial","*",6,0],"cohere-ai":["partial","*",6,0],"Diffbot":["partial","*",6,0],"ContextFetchBot":["partial","*",6,0]},"llms":{"status":"disallowed"},"home":{"status":"ok","http":200,"url":"https://www.instructables.com/","html_tokens":3531,"markdown_tokens":0,"pruned_tokens":0}},
{"rank":827,"domain":"figma.com","origin":"https://figma.com","robots":{"status":"ok","http":200,"url":"https://www.figma.com/robots.txt","bytes":399184,"groups":22},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["partial","named",1,9143],"OAI-SearchBot":["partial","named",1,9143],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","named",1,9143],"Claude-SearchBot":["partial","named",1,9143],"anthropic-ai":["partial","*",17,14],"PerplexityBot":["partial","named",1,9143],"Perplexity-User":["partial","*",17,14],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["partial","*",17,14],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["partial","*",17,14],"meta-externalagent":["partial","*",17,14],"cohere-ai":["blocked","named",1,0],"Diffbot":["partial","*",17,14],"ContextFetchBot":["partial","*",17,14]},"llms":{"status":"missing","http":404,"bytes":9586},"home":{"status":"ok","http":200,"url":"https://www.figma.com/","html_tokens":450355,"markdown_tokens":105,"pruned_tokens":30}},
{"rank":828,"domain":"tamu.edu","origin":"https://tamu.edu","robots":{"status":"ok","http":200,"url":"https://www.tamu.edu/robots.txt","bytes":69,"groups":1},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"missing","http":404,"bytes":64339},"home":{"status":"ok","http":200,"url":"https://www.tamu.edu/","html_tokens":28113,"markdown_tokens":771,"pruned_tokens":159}},
{"rank":829,"domain":"redis.io","origin":"https://redis.io","robots":{"status":"ok","http":200,"url":"https://redis.io/robots.txt","bytes":578,"groups":8},"bots":{"GPTBot":["allowed","named",0,1],"ChatGPT-User":["allowed","named",0,1],"OAI-SearchBot":["partial","*",11,1],"ClaudeBot":["partial","*",11,1],"Claude-User":["partial","*",11,1],"Claude-SearchBot":["partial","*",11,1],"anthropic-ai":["partial","*",11,1],"PerplexityBot":["allowed","named",0,1],"Perplexity-User":["partial","*",11,1],"Google-Extended":["allowed","named",0,1],"Applebot-Extended":["partial","*",11,1],"CCBot":["allowed","named",0,1],"Bytespider":["partial","*",11,1],"Amazonbot":["allowed","named",0,1],"meta-externalagent":["partial","*",11,1],"cohere-ai":["partial","*",11,1],"Diffbot":["partial","*",11,1],"ContextFetchBot":["partial","*",11,1]},"llms":{"status":"present","http":200,"bytes":46852},"home":{"status":"not_html","error":"unsupported content type text/markdown"}},
{"rank":830,"domain":"rutgers.edu","origin":"https://rutgers.edu","robots":{"status":"ok","http":200,"url":"https://www.rutgers.edu/robots.txt","bytes":2027,"groups":1},"bots":{"GPTBot":["partial","*",34,18],"ChatGPT-User":["partial","*",34,18],"OAI-SearchBot":["partial","*",34,18],"ClaudeBot":["partial","*",34,18],"Claude-User":["partial","*",34,18],"Claude-SearchBot":["partial","*",34,18],"anthropic-ai":["partial","*",34,18],"PerplexityBot":["partial","*",34,18],"Perplexity-User":["partial","*",34,18],"Google-Extended":["partial","*",34,18],"Applebot-Extended":["partial","*",34,18],"CCBot":["partial","*",34,18],"Bytespider":["partial","*",34,18],"Amazonbot":["partial","*",34,18],"meta-externalagent":["partial","*",34,18],"cohere-ai":["partial","*",34,18],"Diffbot":["partial","*",34,18],"ContextFetchBot":["partial","*",34,18]},"llms":{"status":"missing","http":404,"bytes":111593},"home":{"status":"ok","http":200,"url":"https://www.rutgers.edu/","html_tokens":48259,"markdown_tokens":165,"pruned_tokens":165}},
{"rank":831,"domain":"nobelprize.org","origin":"https://nobelprize.org","robots":{"status":"ok","http":200,"url":"https://www.nobelprize.org/robots.txt","bytes":187,"groups":1},"bots":{"GPTBot":["partial","*",1,1],"ChatGPT-User":["partial","*",1,1],"OAI-SearchBot":["partial","*",1,1],"ClaudeBot":["partial","*",1,1],"Claude-User":["partial","*",1,1],"Claude-SearchBot":["partial","*",1,1],"anthropic-ai":["partial","*",1,1],"PerplexityBot":["partial","*",1,1],"Perplexity-User":["partial","*",1,1],"Google-Extended":["partial","*",1,1],"Applebot-Extended":["partial","*",1,1],"CCBot":["partial","*",1,1],"Bytespider":["partial","*",1,1],"Amazonbot":["partial","*",1,1],"meta-externalagent":["partial","*",1,1],"cohere-ai":["partial","*",1,1],"Diffbot":["partial","*",1,1],"ContextFetchBot":["partial","*",1,1]},"llms":{"status":"missing","http":404,"bytes":145515},"home":{"status":"ok","http":200,"url":"https://www.nobelprize.org/","html_tokens":79809,"markdown_tokens":1840,"pruned_tokens":88}},
{"rank":832,"domain":"bitbucket.org","origin":"https://bitbucket.org","robots":{"status":"ok","http":200,"url":"https://bitbucket.org/robots.txt","bytes":889,"groups":2},"bots":{"GPTBot":["partial","*",38,0],"ChatGPT-User":["partial","*",38,0],"OAI-SearchBot":["partial","*",38,0],"ClaudeBot":["partial","*",38,0],"Claude-User":["partial","*",38,0],"Claude-SearchBot":["partial","*",38,0],"anthropic-ai":["partial","*",38,0],"PerplexityBot":["partial","*",38,0],"Perplexity-User":["partial","*",38,0],"Google-Extended":["partial","*",38,0],"Applebot-Extended":["partial","*",38,0],"CCBot":["partial","*",38,0],"Bytespider":["partial","*",38,0],"Amazonbot":["partial","*",38,0],"meta-externalagent":["partial","*",38,0],"cohere-ai":["partial","*",38,0],"Diffbot":["partial","*",38,0],"ContextFetchBot":["partial","*",38,0]},"llms":{"status":"missing","http":404,"bytes":13872},"home":{"status":"ok","http":200,"url":"https://bitbucket.org/","html_tokens":44584,"markdown_tokens":491,"pruned_tokens":277}},
{"rank":833,"domain":"novell.com","origin":"https://www.novell.com","robots":{"status":"ok","http":200,"url":"https://www.novell.com/robots.txt","bytes":880,"groups":5},"bots":{"GPTBot":["partial","*",25,1],"ChatGPT-User":["partial","*",25,1],"OAI-SearchBot":["partial","*",25,1],"ClaudeBot":["partial","*",25,1],"Claude-User":["partial","*",25,1],"Claude-SearchBot":["partial","*",25,1],"anthropic-ai":["partial","*",25,1],"PerplexityBot":["partial","*",25,1],"Perplexity-User":["partial","*",25,1],"Google-Extended":["partial","*",25,1],"Applebot-Extended":["partial","*",25,1],"CCBot":["partial","*",25,1],"Bytespider":["partial","*",25,1],"Amazonbot":["partial","*",25,1],"meta-externalagent":["partial","*",25,1],"cohere-ai":["partial","*",25,1],"Diffbot":["partial","*",25,1],"ContextFetchBot":["partial","*",25,1]},"llms":{"status":"missing","http":404,"bytes":33686},"home":{"status":"ok","http":200,"url":"https://www.opentext.com/about/brands/microfocus","html_tokens":85606,"markdown_tokens":20,"pruned_tokens":20}},
{"rank":834,"domain":"esa.int","origin":"https://esa.int","robots":{"status":"ok","http":200,"url":"https://www.esa.int/robots.txt","bytes":1062,"groups":2},"bots":{"GPTBot":["partial","*",20,1],"ChatGPT-User":["partial","*",20,1],"OAI-SearchBot":["partial","*",20,1],"ClaudeBot":["partial","*",20,1],"Claude-User":["partial","*",20,1],"Claude-SearchBot":["partial","*",20,1],"anthropic-ai":["partial","*",20,1],"PerplexityBot":["partial","*",20,1],"Perplexity-User":["partial","*",20,1],"Google-Extended":["partial","*",20,1],"Applebot-Extended":["partial","*",20,1],"CCBot":["partial","*",20,1],"Bytespider":["partial","*",20,1],"Amazonbot":["partial","*",20,1],"meta-externalagent":["partial","*",20,1],"cohere-ai":["partial","*",20,1],"Diffbot":["partial","*",20,1],"ContextFetchBot":["partial","*",20,1]},"llms":{"status":"missing","http":404,"bytes":4715},"home":{"status":"ok","http":200,"url":"https://www.esa.int/","html_tokens":13805,"markdown_tokens":42,"pruned_tokens":42}},
{"rank":835,"domain":"gmail.com","origin":"https://gmail.com","robots":{"status":"ok","http":200,"url":"https://gmail.com/robots.txt","bytes":129,"groups":1},"bots":{"GPTBot":["partial","*",3,1],"ChatGPT-User":["partial","*",3,1],"OAI-SearchBot":["partial","*",3,1],"ClaudeBot":["partial","*",3,1],"Claude-User":["partial","*",3,1],"Claude-SearchBot":["partial","*",3,1],"anthropic-ai":["partial","*",3,1],"PerplexityBot":["partial","*",3,1],"Perplexity-User":["partial","*",3,1],"Google-Extended":["partial","*",3,1],"Applebot-Extended":["partial","*",3,1],"CCBot":["partial","*",3,1],"Bytespider":["partial","*",3,1],"Amazonbot":["partial","*",3,1],"meta-externalagent":["partial","*",3,1],"cohere-ai":["partial","*",3,1],"Diffbot":["partial","*",3,1],"ContextFetchBot":["partial","*",3,1]},"llms":{"status":"missing","http":404,"bytes":1569},"home":{"status":"ok","http":200,"url":"https://accounts.google.com/v3/signin/identifier?continue=https://mail.google.com/mail/u/0/&emr=1&followup=https://mail.google.com/mail/u/0/&osid=1&passive=1209600&service=mail&flowName=WebLiteSignIn&flowEntry=ServiceLogin&dsh=S570775338:1792374208030168","html_tokens":236622,"markdown_tokens":33,"pruned_tokens":9}},
{"rank":836,"domain":"e-recht24.de","origin":"https://e-recht24.de","robots":{"status":"ok","http":200,"url":"https://www.e-recht24.de/robots.txt","bytes":1286,"groups":3},"bots":{"GPTBot":["partial","*",10,16],"ChatGPT-User":["partial","*",10,16],"OAI-SearchBot":["partial","*",10,16],"ClaudeBot":["partial","*",10,16],"Claude-User":["partial","*",10,16],"Claude-SearchBot":["partial","*",10,16],"anthropic-ai":["partial","*",10,16],"PerplexityBot":["partial","*",10,16],"Perplexity-User":["partial","*",10,16],"Google-Extended":["partial","*",10,16],"Applebot-Extended":["partial","*",10,16],"CCBot":["partial","*",10,16],"Bytespider":["partial","*",10,16],"Amazonbot":["partial","*",10,16],"meta-externalagent":["partial","*",10,16],"cohere-ai":["partial","*",10,16],"Diffbot":["partial","*",10,16],"ContextFetchBot":["partial","*",10,16]},"llms":{"status":"present","http":200,"bytes":356027},"home":{"status":"ok","http":200,"url":"https://www.e-recht24.de/","html_tokens":38270,"markdown_tokens":1496,"pruned_tokens":50}},
{"rank":837,"domain":"freedesktop.org","origin":"https://freedesktop.org","robots":{"status":"ok","http":200,"url":"https://freedesktop.org/robots.txt","bytes":30,"groups":1},"bots":{"GPTBot":["partial","*",1,0],"ChatGPT-User":["partial","*",1,0],"OAI-SearchBot":["partial","*",1,0],"ClaudeBot":["partial","*",1,0],"Claude-User":["partial","*",1,0],"Claude-SearchBot":["partial","*",1,0],"anthropic-ai":["partial","*",1,0],"PerplexityBot":["partial","*",1,0],"Perplexity-User":["partial","*",1,0],"Google-Extended":["partial","*",1,0],"Applebot-Extended":["partial","*",1,0],"CCBot":["partial","*",1,0],"Bytespider":["partial","*",1,0],"Amazonbot":["partial","*",1,0],"meta-externalagent":["partial","*",1,0],"cohere-ai":["partial","*",1,0],"Diffbot":["partial","*",1,0],"ContextFetchBot":["partial","*",1,0]},"llms":{"status":"missing","http":404,"bytes":306},"home":{"status":"ok","http":200,"url":"https://www.freedesktop.org/wiki/","html_tokens":2150,"markdown_tokens":1172,"pruned_tokens":670}},
{"rank":838,"domain":"alipay.com","origin":"https://alipay.com","robots":{"status":"missing","http":404,"url":"https://alipay.com/robots.txt","bytes":837,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":837},"home":{"status":"disallowed","error":"robots.txt of www.alipay.com not readable (html)"}},
{"rank":839,"domain":"usembassy.gov","origin":"https://usembassy.gov","robots":{"status":"refused","http":403,"url":"https://usembassy.gov/robots.txt","bytes":118,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":840,"domain":"unep.org","origin":"https://unep.org","robots":{"status":"ok","http":200,"url":"https://www.unep.org/robots.txt","bytes":5128,"groups":33},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["partial","*",50,19],"OAI-SearchBot":["partial","*",50,19],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",50,19],"Claude-SearchBot":["partial","*",50,19],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["partial","*",50,19],"Perplexity-User":["partial","*",50,19],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",50,19]},"llms":{"status":"missing","http":404,"bytes":1243},"home":{"status":"http_error","http":403,"url":"https://www.unep.org/en"}},
{"rank":841,"domain":"gotowebinar.com","origin":"https://gotowebinar.com","robots":{"status":"html","http":200,"url":"https://www.goto.com/webinar","bytes":359475,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":842,"domain":"ahrefs.com","origin":"https://ahrefs.com","robots":{"status":"ok","http":200,"url":"https://ahrefs.com/robots.txt","bytes":908,"groups":1},"bots":{"GPTBot":["partial","*",26,4],"ChatGPT-User":["partial","*",26,4],"OAI-SearchBot":["partial","*",26,4],"ClaudeBot":["partial","*",26,4],"Claude-User":["partial","*",26,4],"Claude-SearchBot":["partial","*",26,4],"anthropic-ai":["partial","*",26,4],"PerplexityBot":["partial","*",26,4],"Perplexity-User":["partial","*",26,4],"Google-Extended":["partial","*",26,4],"Applebot-Extended":["partial","*",26,4],"CCBot":["partial","*",26,4],"Bytespider":["partial","*",26,4],"Amazonbot":["partial","*",26,4],"meta-externalagent":["partial","*",26,4],"cohere-ai":["partial","*",26,4],"Diffbot":["partial","*",26,4],"ContextFetchBot":["partial","*",26,4]},"llms":{"status":"missing","http":404,"bytes":507},"home":{"status":"ok","http":200,"url":"https://ahrefs.com/","html_tokens":319404,"markdown_tokens":444,"pruned_tokens":247}},
{"rank":843,"domain":"wpengine.com","origin":"https://wpengine.com","robots":{"status":"ok","http":200,"url":"https://wpengine.com/robots.txt","bytes":1805,"groups":1},"bots":{"GPTBot":["partial","*",37,1],"ChatGPT-User":["partial","*",37,1],"OAI-SearchBot":["partial","*",37,1],"ClaudeBot":["partial","*",37,1],"Claude-User":["partial","*",37,1],"Claude-SearchBot":["partial","*",37,1],"anthropic-ai":["partial","*",37,1],"PerplexityBot":["partial","*",37,1],"Perplexity-User":["partial","*",37,1],"Google-Extended":["partial","*",37,1],"Applebot-Extended":["partial","*",37,1],"CCBot":["partial","*",37,1],"Bytespider":["partial","*",37,1],"Amazonbot":["partial","*",37,1],"meta-externalagent":["partial","*",37,1],"cohere-ai":["partial","*",37,1],"Diffbot":["partial","*",37,1],"ContextFetchBot":["partial","*",37,1]},"llms":{"status":"missing","http":404,"bytes":60},"home":{"status":"ok","http":200,"url":"https://wpengine.com/","html_tokens":90879,"markdown_tokens":372,"pruned_tokens":163}},
{"rank":844,"domain":"perplexity.ai","origin":"https://perplexity.ai","robots":{"status":"ok","http":200,"url":"https://www.perplexity.ai/robots.txt","bytes":448,"groups":2},"bots":{"GPTBot":["partial","*",6,0],"ChatGPT-User":["partial","*",6,0],"OAI-SearchBot":["partial","*",6,0],"ClaudeBot":["partial","*",6,0],"Claude-User":["partial","*",6,0],"Claude-SearchBot":["partial","*",6,0],"anthropic-ai":["partial","*",6,0],"PerplexityBot":["partial","*",6,0],"Perplexity-User":["partial","*",6,0],"Google-Extended":["partial","*",6,0],"Applebot-Extended":["partial","*",6,0],"CCBot":["partial","*",6,0],"Bytespider":["partial","*",6,0],"Amazonbot":["partial","*",6,0],"meta-externalagent":["partial","*",6,0],"cohere-ai":["partial","*",6,0],"Diffbot":["partial","*",6,0],"ContextFetchBot":["partial","*",6,0]},"llms":{"status":"missing","http":404,"bytes":14360},"home":{"status":"ok","http":200,"url":"https://www.perplexity.ai/","html_tokens":4427,"markdown_tokens":0,"pruned_tokens":0}},
{"rank":845,"domain":"wireshark.org","origin":"https://wireshark.org","robots":{"status":"ok","http":200,"url":"https://www.wireshark.org/robots.txt","bytes":74,"groups":1},"bots":{"GPTBot":["partial","*",3,0],"ChatGPT-User":["partial","*",3,0],"OAI-SearchBot":["partial","*",3,0],"ClaudeBot":["partial","*",3,0],"Claude-User":["partial","*",3,0],"Claude-SearchBot":["partial","*",3,0],"anthropic-ai":["partial","*",3,0],"PerplexityBot":["partial","*",3,0],"Perplexity-User":["partial","*",3,0],"Google-Extended":["partial","*",3,0],"Applebot-Extended":["partial","*",3,0],"CCBot":["partial","*",3,0],"Bytespider":["partial","*",3,0],"Amazonbot":["partial","*",3,0],"meta-externalagent":["partial","*",3,0],"cohere-ai":["partial","*",3,0],"Diffbot":["partial","*",3,0],"ContextFetchBot":["partial","*",3,0]},"llms":{"status":"missing","http":404,"bytes":81522},"home":{"status":"ok","http":200,"url":"https://www.wireshark.org/","html_tokens":54111,"markdown_tokens":514,"pruned_tokens":465}},
{"rank":846,"domain":"fbcdn.net","origin":"https://fbcdn.net","robots":{"status":"ok","http":200,"url":"https://fbcdn.net/robots.txt","bytes":3355,"groups":36},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["blocked","*",1,0],"OAI-SearchBot":["blocked","*",1,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["blocked","*",1,0],"Claude-SearchBot":["blocked","*",1,0],"anthropic-ai":["blocked","*",1,0],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["blocked","*",1,0],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","*",1,0],"Bytespider":["blocked","*",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","*",1,0],"cohere-ai":["blocked","*",1,0],"Diffbot":["blocked","*",1,0],"ContextFetchBot":["blocked","*",1,0]},"llms":{"status":"disallowed"},"home":{"status":"disallowed"}},
{"rank":847,"domain":"bu.edu","origin":"https://bu.edu","robots":{"status":"ok","http":200,"url":"https://www.bu.edu/robots.txt","bytes":6365,"groups":7},"bots":{"GPTBot":["partial","named",3,0],"ChatGPT-User":["partial","*",69,0],"OAI-SearchBot":["partial","*",69,0],"ClaudeBot":["partial","named",3,0],"Claude-User":["partial","*",69,0],"Claude-SearchBot":["partial","*",69,0],"anthropic-ai":["partial","*",69,0],"PerplexityBot":["partial","*",69,0],"Perplexity-User":["partial","*",69,0],"Google-Extended":["partial","*",69,0],"Applebot-Extended":["partial","*",69,0],"CCBot":["partial","named",3,0],"Bytespider":["partial","named",3,0],"Amazonbot":["partial","named",3,0],"meta-externalagent":["partial","named",3,0],"cohere-ai":["partial","*",69,0],"Diffbot":["partial","*",69,0],"ContextFetchBot":["partial","*",69,0]},"llms":{"status":"missing","http":404,"bytes":131450},"home":{"status":"ok","http":200,"url":"https://www.bu.edu/","html_tokens":36565,"markdown_tokens":9,"pruned_tokens":9}},
{"rank":848,"domain":"bitnami.com","origin":"https://bitnami.com","robots":{"status":"ok","http":200,"url":"https://bitnami.com/robots.txt","bytes":87,"groups":1},"bots":{"GPTBot":["partial","*",2,0],"ChatGPT-User":["partial","*",2,0],"OAI-SearchBot":["partial","*",2,0],"ClaudeBot":["partial","*",2,0],"Claude-User":["partial","*",2,0],"Claude-SearchBot":["partial","*",2,0],"anthropic-ai":["partial","*",2,0],"PerplexityBot":["partial","*",2,0],"Perplexity-User":["partial","*",2,0],"Google-Extended":["partial","*",2,0],"Applebot-Extended":["partial","*",2,0],"CCBot":["partial","*",2,0],"Bytespider":["partial","*",2,0],"Amazonbot":["partial","*",2,0],"meta-externalagent":["partial","*",2,0],"cohere-ai":["partial","*",2,0],"Diffbot":["partial","*",2,0],"ContextFetchBot":["partial","*",2,0]},"llms":{"status":"missing","http":404,"bytes":79470},"home":{"status":"ok","http":200,"url":"https://bitnami.com/","html_tokens":49750,"markdown_tokens":1584,"pruned_tokens":32}},
{"rank":849,"domain":"wikisource.org","origin":"https://wikisource.org","robots":{"status":"ok","http":200,"url":"https://wikisource.org/robots.txt","bytes":15251,"groups":33},"bots":{"GPTBot":["partial","*",200,4],"ChatGPT-User":["partial","*",200,4],"OAI-SearchBot":["partial","*",200,4],"ClaudeBot":["partial","*",200,4],"Claude-User":["partial","*",200,4],"Claude-SearchBot":["partial","*",200,4],"anthropic-ai":["partial","*",200,4],"PerplexityBot":["partial","*",200,4],"Perplexity-User":["partial","*",200,4],"Google-Extended":["partial","*",200,4],"Applebot-Extended":["partial","*",200,4],"CCBot":["partial","*",200,4],"Bytespider":["partial","*",200,4],"Amazonbot":["partial","*",200,4],"meta-externalagent":["partial","*",200,4],"cohere-ai":["partial","*",200,4],"Diffbot":["partial","*",200,4],"ContextFetchBot":["partial","*",200,4]},"llms":{"status":"missing","http":404,"bytes":1868},"home":{"status":"ok","http":200,"url":"https://wikisource.org/wiki/Wikisource:Main_Page","html_tokens":44190,"markdown_tokens":1470,"pruned_tokens":157}},
{"rank":850,"domain":"osu.edu","origin":"https://osu.edu","robots":{"status":"ok","http":200,"url":"https://www.osu.edu/robots.txt","bytes":126,"groups":1},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"missing","http":404,"bytes":175781},"home":{"status":"ok","http":200,"url":"https://www.osu.edu/","html_tokens":125468,"markdown_tokens":142,"pruned_tokens":142}},
{"rank":851,"domain":"nic.in","origin":"https://www.nic.in","robots":{"status":"refused","http":403,"url":"https://www.nic.in/robots.txt","bytes":378,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":852,"domain":"envato.com","origin":"https://envato.com","robots":{"status":"ok","http":200,"url":"https://www.envato.com/robots.txt","bytes":106,"groups":2},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["blocked","named",1,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"missing","http":404,"bytes":7313},"home":{"status":"disallowed","error":"robots.txt of elements.envato.com not readable (refused)"}},
{"rank":853,"domain":"gesetze-im-internet.de","origin":"https://www.gesetze-im-internet.de","robots":{"status":"ok","http":200,"url":"https://www.gesetze-im-internet.de/robots.txt","bytes":24,"groups":1},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"missing","http":404,"bytes":236},"home":{"status":"ok","http":200,"url":"https://www.gesetze-im-internet.de/","html_tokens":2360,"markdown_tokens":843,"pruned_tokens":210}},
{"rank":854,"domain":"usa.gov","origin":"https://usa.gov","robots":{"status":"ok","http":200,"url":"https://www.usa.gov/robots.txt","bytes":1558,"groups":1},"bots":{"GPTBot":["partial","*",10,32],"ChatGPT-User":["partial","*",10,32],"OAI-SearchBot":["partial","*",10,32],"ClaudeBot":["partial","*",10,32],"Claude-User":["partial","*",10,32],"Claude-SearchBot":["partial","*",10,32],"anthropic-ai":["partial","*",10,32],"PerplexityBot":["partial","*",10,32],"Perplexity-User":["partial","*",10,32],"Google-Extended":["partial","*",10,32],"Applebot-Extended":["partial","*",10,32],"CCBot":["partial","*",10,32],"Bytespider":["partial","*",10,32],"Amazonbot":["partial","*",10,32],"meta-externalagent":["partial","*",10,32],"cohere-ai":["partial","*",10,32],"Diffbot":["partial","*",10,32],"ContextFetchBot":["partial","*",10,32]},"llms":{"status":"missing","http":404,"bytes":35837},"home":{"status":"ok","http":200,"url":"https://www.usa.gov/","html_tokens":12161,"markdown_tokens":702,"pruned_tokens":33}},
{"rank":855,"domain":"speedtest.net","origin":"https://speedtest.net","robots":{"status":"ok","http":200,"url":"https://www.speedtest.net/robots.txt","bytes":247,"groups":2},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":72194},"home":{"status":"ok","http":200,"url":"https://www.speedtest.net/","html_tokens":51645,"markdown_tokens":8,"pruned_tokens":8}},
{"rank":856,"domain":"crunchbase.com","origin":"https://crunchbase.com","robots":{"status":"ok","http":200,"url":"https://www.crunchbase.com/robots.txt","bytes":1874,"groups":3},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["allowed","named",0,1],"OAI-SearchBot":["allowed","named",0,1],"ClaudeBot":["blocked","named",1,0],"Claude-User":["allowed","named",0,1],"Claude-SearchBot":["allowed","named",0,1],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["allowed","named",0,1],"Perplexity-User":["allowed","named",0,1],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["partial","*",17,3],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",17,3]},"llms":{"status":"missing","http":404,"bytes":97157},"home":{"status":"ok","http":200,"url":"https://www.crunchbase.com/","html_tokens":32410,"markdown_tokens":700,"pruned_tokens":362}},
{"rank":857,"domain":"www.gov.br","origin":"https://www.gov.br","robots":{"status":"ok","http":200,"url":"https://www.gov.br/robots.txt","bytes":2102,"groups":2},"bots":{"GPTBot":["partial","*",53,0],"ChatGPT-User":["partial","*",53,0],"OAI-SearchBot":["partial","*",53,0],"ClaudeBot":["partial","*",53,0],"Claude-User":["partial","*",53,0],"Claude-SearchBot":["partial","*",53,0],"anthropic-ai":["partial","*",53,0],"PerplexityBot":["partial","*",53,0],"Perplexity-User":["partial","*",53,0],"Google-Extended":["partial","*",53,0],"Applebot-Extended":["partial","*",53,0],"CCBot":["partial","*",53,0],"Bytespider":["partial","*",53,0],"Amazonbot":["partial","*",53,0],"meta-externalagent":["partial","*",53,0],"cohere-ai":["partial","*",53,0],"Diffbot":["partial","*",53,0],"ContextFetchBot":["partial","*",53,0]},"llms":{"status":"missing","http":404,"bytes":36824},"home":{"status":"ok","http":200,"url":"https://www.gov.br/pt-br","html_tokens":32963,"markdown_tokens":1657,"pruned_tokens":136}},
{"rank":858,"domain":"flic.kr","origin":"https://flic.kr","robots":{"status":"ok","http":200,"url":"https://flic.kr/robots.txt","bytes":33,"groups":1},"bots":{"GPTBot":["partial","*",1,0],"ChatGPT-User":["partial","*",1,0],"OAI-SearchBot":["partial","*",1,0],"ClaudeBot":["partial","*",1,0],"Claude-User":["partial","*",1,0],"Claude-SearchBot":["partial","*",1,0],"anthropic-ai":["partial","*",1,0],"PerplexityBot":["partial","*",1,0],"Perplexity-User":["partial","*",1,0],"Google-Extended":["partial","*",1,0],"Applebot-Extended":["partial","*",1,0],"CCBot":["partial","*",1,0],"Bytespider":["partial","*",1,0],"Amazonbot":["partial","*",1,0],"meta-externalagent":["partial","*",1,0],"cohere-ai":["partial","*",1,0],"Diffbot":["partial","*",1,0],"ContextFetchBot":["partial","*",1,0]},"llms":{"status":"disallowed","error":"robots.txt of www.flickr.com disallows /llms.txt"},"home":{"status":"http_error","http":502,"url":"https://flic.kr/"}},
{"rank":859,"domain":"army.mil","origin":"https://www.army.mil","robots":{"status":"ok","http":200,"url":"https://www.army.mil/robots.txt","bytes":286,"groups":2},"bots":{"GPTBot":["partial","*",7,1],"ChatGPT-User":["partial","*",7,1],"OAI-SearchBot":["partial","*",7,1],"ClaudeBot":["partial","*",7,1],"Claude-User":["partial","*",7,1],"Claude-SearchBot":["partial","*",7,1],"anthropic-ai":["partial","*",7,1],"PerplexityBot":["partial","*",7,1],"Perplexity-User":["partial","*",7,1],"Google-Extended":["partial","*",7,1],"Applebot-Extended":["partial","*",7,1],"CCBot":["partial","*",7,1],"Bytespider":["partial","*",7,1],"Amazonbot":["partial","*",7,1],"meta-externalagent":["partial","*",7,1],"cohere-ai":["partial","*",7,1],"Diffbot":["partial","*",7,1],"ContextFetchBot":["partial","*",7,1]},"llms":{"status":"missing","http":404,"bytes":960},"home":{"status":"ok","http":200,"url":"https://www.army.mil/","html_tokens":43876,"markdown_tokens":79,"pruned_tokens":79}},
{"rank":860,"domain":"1688.com","origin":"https://1688.com","robots":{"status":"ok","http":200,"url":"https://www.1688.com/robots.txt","bytes":8388,"groups":6},"bots":{"GPTBot":["partial","*",69,6],"ChatGPT-User":["partial","*",69,6],"OAI-SearchBot":["partial","*",69,6],"ClaudeBot":["partial","*",69,6],"Claude-User":["partial","*",69,6],"Claude-SearchBot":["partial","*",69,6],"anthropic-ai":["partial","*",69,6],"PerplexityBot":["partial","*",69,6],"Perplexity-User":["partial","*",69,6],"Google-Extended":["partial","*",69,6],"Applebot-Extended":["partial","*",69,6],"CCBot":["partial","*",69,6],"Bytespider":["partial","*",69,6],"Amazonbot":["partial","*",69,6],"meta-externalagent":["partial","*",69,6],"cohere-ai":["partial","*",69,6],"Diffbot":["partial","*",69,6],"ContextFetchBot":["partial","*",69,6]},"llms":{"status":"unreachable","error":"[SSL: CERTIFICATE_VERIFY_FAILED] certificate verify failed: certificate has expired (_ssl.c:1028)"},"home":{"status":"ok","http":200,"url":"https://www.1688.com/","html_tokens":26063,"markdown_tokens":0,"pruned_tokens":0}},
{"rank":861,"domain":"vatican.va","origin":"https://vatican.va","robots":{"status":"ok","http":200,"url":"https://www.vatican.va/robots.txt","bytes":39,"groups":1},"bots":{"GPTBot":["allowed","*",0,1],"ChatGPT-User":["allowed","*",0,1],"OAI-SearchBot":["allowed","*",0,1],"ClaudeBot":["allowed","*",0,1],"Claude-User":["allowed","*",0,1],"Claude-SearchBot":["allowed","*",0,1],"anthropic-ai":["allowed","*",0,1],"PerplexityBot":["allowed","*",0,1],"Perplexity-User":["allowed","*",0,1],"Google-Extended":["allowed","*",0,1],"Applebot-Extended":["allowed","*",0,1],"CCBot":["allowed","*",0,1],"Bytespider":["allowed","*",0,1],"Amazonbot":["allowed","*",0,1],"meta-externalagent":["allowed","*",0,1],"cohere-ai":["allowed","*",0,1],"Diffbot":["allowed","*",0,1],"ContextFetchBot":["allowed","*",0,1]},"llms":{"status":"missing","http":404,"bytes":206},"home":{"status":"ok","http":200,"url":"https://vatican.va/","html_tokens":1093,"markdown_tokens":5,"pruned_tokens":1}},
{"rank":862,"domain":"blogger.googleusercontent.com","origin":"https://blogger.googleusercontent.com","robots":{"status":"ok","http":200,"url":"https://blogger.googleusercontent.com/robots.txt","bytes":572,"groups":1},"bots":{"GPTBot":["partial","*",21,0],"ChatGPT-User":["partial","*",21,0],"OAI-SearchBot":["partial","*",21,0],"ClaudeBot":["partial","*",21,0],"Claude-User":["partial","*",21,0],"Claude-SearchBot":["partial","*",21,0],"anthropic-ai":["partial","*",21,0],"PerplexityBot":["partial","*",21,0],"Perplexity-User":["partial","*",21,0],"Google-Extended":["partial","*",21,0],"Applebot-Extended":["partial","*",21,0],"CCBot":["partial","*",21,0],"Bytespider":["partial","*",21,0],"Amazonbot":["partial","*",21,0],"meta-externalagent":["partial","*",21,0],"cohere-ai":["partial","*",21,0],"Diffbot":["partial","*",21,0],"ContextFetchBot":["partial","*",21,0]},"llms":{"status":"missing","http":404,"bytes":1569},"home":{"status":"http_error","http":404,"url":"https://blogger.googleusercontent.com/"}},
{"rank":863,"domain":"centos.org","origin":"https://centos.org","robots":{"status":"ok","http":200,"url":"https://centos.org/robots.txt","bytes":190,"groups":2},"bots":{"GPTBot":["allowed","*",0,1],"ChatGPT-User":["allowed","*",0,1],"OAI-SearchBot":["allowed","*",0,1],"ClaudeBot":["allowed","*",0,1],"Claude-User":["allowed","*",0,1],"Claude-SearchBot":["allowed","*",0,1],"anthropic-ai":["allowed","*",0,1],"PerplexityBot":["allowed","*",0,1],"Perplexity-User":["allowed","*",0,1],"Google-Extended":["allowed","*",0,1],"Applebot-Extended":["allowed","*",0,1],"CCBot":["allowed","*",0,1],"Bytespider":["allowed","*",0,1],"Amazonbot":["allowed","*",0,1],"meta-externalagent":["allowed","*",0,1],"cohere-ai":["allowed","*",0,1],"Diffbot":["allowed","*",0,1],"ContextFetchBot":["allowed","*",0,1]},"llms":{"status":"missing","http":404,"bytes":8340},"home":{"status":"ok","http":200,"url":"https://centos.org/","html_tokens":10580,"markdown_tokens":207,"pruned_tokens":207}},
{"rank":864,"domain":"dol.gov","origin":"https://dol.gov","robots":{"status":"ok","http":200,"url":"https://www.dol.gov/robots.txt","bytes":2027,"groups":1},"bots":{"GPTBot":["partial","*",34,18],"ChatGPT-User":["partial","*",34,18],"OAI-SearchBot":["partial","*",34,18],"ClaudeBot":["partial","*",34,18],"Claude-User":["partial","*",34,18],"Claude-SearchBot":["partial","*",34,18],"anthropic-ai":["partial","*",34,18],"PerplexityBot":["partial","*",34,18],"Perplexity-User":["partial","*",34,18],"Google-Extended":["partial","*",34,18],"Applebot-Extended":["partial","*",34,18],"CCBot":["partial","*",34,18],"Bytespider":["partial","*",34,18],"Amazonbot":["partial","*",34,18],"meta-externalagent":["partial","*",34,18],"cohere-ai":["partial","*",34,18],"Diffbot":["partial","*",34,18],"ContextFetchBot":["partial","*",34,18]},"llms":{"status":"http_error","http":403,"bytes":153},"home":{"status":"ok","http":200,"url":"https://www.dol.gov/","html_tokens":21716,"markdown_tokens":103,"pruned_tokens":84}},
{"rank":865,"domain":"nasdaq.com","origin":"https://nasdaq.com","robots":{"status":"unreachable","http":null,"url":"https://nasdaq.com/robots.txt","bytes":0,"groups":0,"error":"<StreamReset stream_id:1, error_code:2, remote_reset:True>"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":866,"domain":"or.id","origin":"https://or.id","robots":{"status":"unreachable","http":null,"url":"https://or.id/robots.txt","bytes":0,"groups":0,"error":"cannot resolve or.id"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":867,"domain":"ipcc.ch","origin":"https://ipcc.ch","robots":{"status":"ok","http":200,"url":"https://www.ipcc.ch/robots.txt","bytes":77,"groups":1},"bots":{"GPTBot":["partial","*",1,1],"ChatGPT-User":["partial","*",1,1],"OAI-SearchBot":["partial","*",1,1],"ClaudeBot":["partial","*",1,1],"Claude-User":["partial","*",1,1],"Claude-SearchBot":["partial","*",1,1],"anthropic-ai":["partial","*",1,1],"PerplexityBot":["partial","*",1,1],"Perplexity-User":["partial","*",1,1],"Google-Extended":["partial","*",1,1],"Applebot-Extended":["partial","*",1,1],"CCBot":["partial","*",1,1],"Bytespider":["partial","*",1,1],"Amazonbot":["partial","*",1,1],"meta-externalagent":["partial","*",1,1],"cohere-ai":["partial","*",1,1],"Diffbot":["partial","*",1,1],"ContextFetchBot":["partial","*",1,1]},"llms":{"status":"missing","http":404,"bytes":45914},"home":{"status":"ok","http":200,"url":"https://www.ipcc.ch/","html_tokens":23239,"markdown_tokens":235,"pruned_tokens":235}},
{"rank":868,"domain":"cancer.org","origin":"https://cancer.org","robots":{"status":"ok","http":200,"url":"https://www.cancer.org/robots.txt","bytes":504,"groups":1},"bots":{"GPTBot":["partial","*",8,1],"ChatGPT-User":["partial","*",8,1],"OAI-SearchBot":["partial","*",8,1],"ClaudeBot":["partial","*",8,1],"Claude-User":["partial","*",8,1],"Claude-SearchBot":["partial","*",8,1],"anthropic-ai":["partial","*",8,1],"PerplexityBot":["partial","*",8,1],"Perplexity-User":["partial","*",8,1],"Google-Extended":["partial","*",8,1],"Applebot-Extended":["partial","*",8,1],"CCBot":["partial","*",8,1],"Bytespider":["partial","*",8,1],"Amazonbot":["partial","*",8,1],"meta-externalagent":["partial","*",8,1],"cohere-ai":["partial","*",8,1],"Diffbot":["partial","*",8,1],"ContextFetchBot":["partial","*",8,1]},"llms":{"status":"missing","http":404,"bytes":289411},"home":{"status":"ok","http":200,"url":"https://www.cancer.org/","html_tokens":102317,"markdown_tokens":696,"pruned_tokens":153}},
{"rank":869,"domain":"express.co.uk","origin":"https://express.co.uk","robots":{"status":"ok","http":200,"url":"https://express.co.uk/robots.txt","bytes":1733,"groups":22},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["partial","*",9,0],"OAI-SearchBot":["blocked","named",1,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",9,0],"Claude-SearchBot":["partial","*",9,0],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["partial","*",9,0],"Google-Extended":["partial","*",9,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["partial","*",9,0],"Amazonbot":["partial","*",9,0],"meta-externalagent":["partial","*",9,0],"cohere-ai":["partial","*",9,0],"Diffbot":["partial","*",9,0],"ContextFetchBot":["partial","*",9,0]},"llms":{"status":"missing","http":404,"bytes":67894},"home":{"status":"ok","http":200,"url":"https://www.express.co.uk/","html_tokens":93899,"markdown_tokens":54,"pruned_tokens":54}},
{"rank":870,"domain":"digital.com","origin":"https://digital.com","robots":{"status":"unreachable","http":null,"url":"https://digital.com/robots.txt","bytes":0,"groups":0,"error":"cannot resolve digital.com"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":871,"domain":"greenpeace.org","origin":"https://greenpeace.org","robots":{"status":"ok","http":200,"url":"https://www.greenpeace.org/robots.txt","bytes":23,"groups":1},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"missing","http":404,"bytes":3638},"home":{"status":"ok","http":200,"url":"https://www.greenpeace.org/global/","html_tokens":1175,"markdown_tokens":31,"pruned_tokens":31}},
{"rank":872,"domain":"fiverr.com","origin":"https://fiverr.com","robots":{"status":"ok","http":200,"url":"https://www.fiverr.com/robots.txt","bytes":2049,"groups":2},"bots":{"GPTBot":["partial","*",53,2],"ChatGPT-User":["partial","*",53,2],"OAI-SearchBot":["partial","*",53,2],"ClaudeBot":["partial","*",53,2],"Claude-User":["partial","*",53,2],"Claude-SearchBot":["partial","*",53,2],"anthropic-ai":["partial","*",53,2],"PerplexityBot":["partial","*",53,2],"Perplexity-User":["partial","*",53,2],"Google-Extended":["partial","*",53,2],"Applebot-Extended":["partial","*",53,2],"CCBot":["partial","*",53,2],"Bytespider":["partial","*",53,2],"Amazonbot":["partial","*",53,2],"meta-externalagent":["partial","*",53,2],"cohere-ai":["partial","*",53,2],"Diffbot":["partial","*",53,2],"ContextFetchBot":["partial","*",53,2]},"llms":{"status":"error","error":"unsupported content encoding utf-8"},"home":{"status":"http_error","http":403,"url":"https://www.fiverr.com/"}},
{"rank":873,"domain":"utah.edu","origin":"https://utah.edu","robots":{"status":"ok","http":200,"url":"https://www.utah.edu/robots.txt","bytes":57,"groups":1},"bots":{"GPTBot":["partial","*",2,0],"ChatGPT-User":["partial","*",2,0],"OAI-SearchBot":["partial","*",2,0],"ClaudeBot":["partial","*",2,0],"Claude-User":["partial","*",2,0],"Claude-SearchBot":["partial","*",2,0],"anthropic-ai":["partial","*",2,0],"PerplexityBot":["partial","*",2,0],"Perplexity-User":["partial","*",2,0],"Google-Extended":["partial","*",2,0],"Applebot-Extended":["partial","*",2,0],"CCBot":["partial","*",2,0],"Bytespider":["partial","*",2,0],"Amazonbot":["partial","*",2,0],"meta-externalagent":["partial","*",2,0],"cohere-ai":["partial","*",2,0],"Diffbot":["partial","*",2,0],"ContextFetchBot":["partial","*",2,0]},"llms":{"status":"missing","http":404,"bytes":28957},"home":{"status":"ok","http":200,"url":"https://www.utah.edu/","html_tokens":11530,"markdown_tokens":230,"pruned_tokens":24}},
{"rank":874,"domain":"imageshack.us","origin":"https://imageshack.us","robots":{"status":"ok","http":200,"url":"https://imageshack.us/robots.txt","bytes":2517,"groups":2},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["blocked","named",1,0],"OAI-SearchBot":["blocked","named",1,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["blocked","named",1,0],"Claude-SearchBot":["blocked","named",1,0],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["blocked","named",1,0],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",1,0]},"llms":{"status":"missing","http":404,"bytes":6748},"home":{"status":"ok","http":200,"url":"https://imageshack.com/","html_tokens":1896,"markdown_tokens":26,"pruned_tokens":26}},
{"rank":875,"domain":"dreamhost.com","origin":"https://dreamhost.com","robots":{"status":"ok","http":200,"url":"https://www.dreamhost.com/robots.txt","bytes":422,"groups":1},"bots":{"GPTBot":["partial","*",5,1],"ChatGPT-User":["partial","*",5,1],"OAI-SearchBot":["partial","*",5,1],"ClaudeBot":["partial","*",5,1],"Claude-User":["partial","*",5,1],"Claude-SearchBot":["partial","*",5,1],"anthropic-ai":["partial","*",5,1],"PerplexityBot":["partial","*",5,1],"Perplexity-User":["partial","*",5,1],"Google-Extended":["partial","*",5,1],"Applebot-Extended":["partial","*",5,1],"CCBot":["partial","*",5,1],"Bytespider":["partial","*",5,1],"Amazonbot":["partial","*",5,1],"meta-externalagent":["partial","*",5,1],"cohere-ai":["partial","*",5,1],"Diffbot":["partial","*",5,1],"ContextFetchBot":["partial","*",5,1]},"llms":{"status":"present","http":200,"bytes":18902},"home":{"status":"ok","http":200,"url":"https://www.dreamhost.com/","html_tokens":120595,"markdown_tokens":1221,"pruned_tokens":72}},
{"rank":876,"domain":"xbox.com","origin":"https://xbox.com","robots":{"status":"ok","http":200,"url":"https://www.xbox.com/robots.txt","bytes":289,"groups":1},"bots":{"GPTBot":["partial","*",9,0],"ChatGPT-User":["partial","*",9,0],"OAI-SearchBot":["partial","*",9,0],"ClaudeBot":["partial","*",9,0],"Claude-User":["partial","*",9,0],"Claude-SearchBot":["partial","*",9,0],"anthropic-ai":["partial","*",9,0],"PerplexityBot":["partial","*",9,0],"Perplexity-User":["partial","*",9,0],"Google-Extended":["partial","*",9,0],"Applebot-Extended":["partial","*",9,0],"CCBot":["partial","*",9,0],"Bytespider":["partial","*",9,0],"Amazonbot":["partial","*",9,0],"meta-externalagent":["partial","*",9,0],"cohere-ai":["partial","*",9,0],"Diffbot":["partial","*",9,0],"ContextFetchBot":["partial","*",9,0]},"llms":{"status":"missing","http":404,"bytes":337645},"home":{"status":"ok","http":200,"url":"https://www.xbox.com/en-US/","html_tokens":116165,"markdown_tokens":514,"pruned_tokens":174}},
{"rank":877,"domain":"uk.linkedin.com","origin":"https://uk.linkedin.com","robots":{"status":"ok","http":200,"url":"https://uk.linkedin.com/robots.txt","bytes":120190,"groups":77},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["blocked","named",1,0],"OAI-SearchBot":["partial","named",111,9],"ClaudeBot":["blocked","named",1,0],"Claude-User":["blocked","named",1,0],"Claude-SearchBot":["partial","named",111,9],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["blocked","named",1,0],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","*",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","*",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["blocked","*",1,0]},"llms":{"status":"disallowed"},"home":{"status":"disallowed"}},
{"rank":878,"domain":"federalregister.gov","origin":"https://federalregister.gov","robots":{"status":"ok","http":200,"url":"https://www.federalregister.gov/robots.txt","bytes":320,"groups":1},"bots":{"GPTBot":["partial","*",8,0],"ChatGPT-User":["partial","*",8,0],"OAI-SearchBot":["partial","*",8,0],"ClaudeBot":["partial","*",8,0],"Claude-User":["partial","*",8,0],"Claude-SearchBot":["partial","*",8,0],"anthropic-ai":["partial","*",8,0],"PerplexityBot":["partial","*",8,0],"Perplexity-User":["partial","*",8,0],"Google-Extended":["partial","*",8,0],"Applebot-Extended":["partial","*",8,0],"CCBot":["partial","*",8,0],"Bytespider":["partial","*",8,0],"Amazonbot":["partial","*",8,0],"meta-externalagent":["partial","*",8,0],"cohere-ai":["partial","*",8,0],"Diffbot":["partial","*",8,0],"ContextFetchBot":["partial","*",8,0]},"llms":{"status":"disallowed","error":"robots.txt of unblock.federalregister.gov disallows /"},"home":{"status":"disallowed","error":"robots.txt of unblock.federalregister.gov disallows /"}},
{"rank":879,"domain":"ea.com","origin":"https://ea.com","robots":{"status":"ok","http":200,"url":"https://www.ea.com/robots.txt","bytes":1904,"groups":8},"bots":{"GPTBot":["partial","*",11,0],"ChatGPT-User":["partial","*",11,0],"OAI-SearchBot":["partial","*",11,0],"ClaudeBot":["partial","*",11,0],"Claude-User":["partial","*",11,0],"Claude-SearchBot":["partial","*",11,0],"anthropic-ai":["partial","*",11,0],"PerplexityBot":["partial","*",11,0],"Perplexity-User":["partial","*",11,0],"Google-Extended":["partial","*",11,0],"Applebot-Extended":["partial","*",11,0],"CCBot":["partial","*",11,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["partial","*",11,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["partial","*",11,0],"ContextFetchBot":["partial","*",11,0]},"llms":{"status":"present","http":200,"bytes":12437},"home":{"status":"ok","http":200,"url":"https://www.ea.com/","html_tokens":118812,"markdown_tokens":117,"pruned_tokens":117}},
{"rank":880,"domain":"mystrikingly.com","origin":"https://mystrikingly.com","robots":{"status":"html","http":200,"url":"https://www.strikingly.com","bytes":123810,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":881,"domain":"gumroad.com","origin":"https://gumroad.com","robots":{"status":"ok","http":200,"url":"https://gumroad.com/robots.txt","bytes":32741,"groups":1},"bots":{"GPTBot":["partial","*",1,0],"ChatGPT-User":["partial","*",1,0],"OAI-SearchBot":["partial","*",1,0],"ClaudeBot":["partial","*",1,0],"Claude-User":["partial","*",1,0],"Claude-SearchBot":["partial","*",1,0],"anthropic-ai":["partial","*",1,0],"PerplexityBot":["partial","*",1,0],"Perplexity-User":["partial","*",1,0],"Google-Extended":["partial","*",1,0],"Applebot-Extended":["partial","*",1,0],"CCBot":["partial","*",1,0],"Bytespider":["partial","*",1,0],"Amazonbot":["partial","*",1,0],"meta-externalagent":["partial","*",1,0],"cohere-ai":["partial","*",1,0],"Diffbot":["partial","*",1,0],"ContextFetchBot":["partial","*",1,0]},"llms":{"status":"present","http":200,"bytes":2067},"home":{"status":"ok","http":200,"url":"https://gumroad.com/","html_tokens":32262,"markdown_tokens":584,"pruned_tokens":158}},
{"rank":882,"domain":"rutube.ru","origin":"https://rutube.ru","robots":{"status":"ok","http":200,"url":"https://rutube.ru/robots.txt","bytes":4210,"groups":4},"bots":{"GPTBot":["partial","*",46,0],"ChatGPT-User":["partial","*",46,0],"OAI-SearchBot":["partial","*",46,0],"ClaudeBot":["partial","*",46,0],"Claude-User":["partial","*",46,0],"Claude-SearchBot":["partial","*",46,0],"anthropic-ai":["partial","*",46,0],"PerplexityBot":["partial","*",46,0],"Perplexity-User":["partial","*",46,0],"Google-Extended":["partial","*",46,0],"Applebot-Extended":["partial","*",46,0],"CCBot":["partial","*",46,0],"Bytespider":["partial","*",46,0],"Amazonbot":["partial","*",46,0],"meta-externalagent":["partial","*",46,0],"cohere-ai":["partial","*",46,0],"Diffbot":["partial","*",46,0],"ContextFetchBot":["partial","*",46,0]},"llms":{"status":"missing","http":404,"bytes":289842},"home":{"status":"ok","http":200,"url":"https://rutube.ru/","html_tokens":233276,"markdown_tokens":1445,"pruned_tokens":69}},
{"rank":883,"domain":"douban.com","origin":"https://douban.com","robots":{"status":"ok","http":200,"url":"https://www.douban.com/robots.txt","bytes":895,"groups":3},"bots":{"GPTBot":["partial","*",20,1],"ChatGPT-User":["partial","*",20,1],"OAI-SearchBot":["partial","*",20,1],"ClaudeBot":["partial","*",20,1],"Claude-User":["partial","*",20,1],"Claude-SearchBot":["partial","*",20,1],"anthropic-ai":["partial","*",20,1],"PerplexityBot":["partial","*",20,1],"Perplexity-User":["partial","*",20,1],"Google-Extended":["partial","*",20,1],"Applebot-Extended":["partial","*",20,1],"CCBot":["partial","*",20,1],"Bytespider":["partial","*",20,1],"Amazonbot":["partial","*",20,1],"meta-externalagent":["partial","*",20,1],"cohere-ai":["partial","*",20,1],"Diffbot":["partial","*",20,1],"ContextFetchBot":["partial","*",20,1]},"llms":{"status":"missing","http":404,"bytes":918},"home":{"status":"ok","http":200,"url":"https://www.douban.com/","html_tokens":21428,"markdown_tokens":33,"pruned_tokens":33}},
{"rank":884,"domain":"lua.org","origin":"https://lua.org","robots":{"status":"missing","http":404,"url":"https://lua.org/robots.txt","bytes":153,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":153},"home":{"status":"ok","http":200,"url":"https://lua.org/","html_tokens":509,"markdown_tokens":9,"pruned_tokens":9}},
{"rank":885,"domain":"nhk.or.jp","origin":"https://www.nhk.or.jp","robots":{"status":"ok","http":200,"url":"https://www.nhk.or.jp/robots.txt","bytes":2873,"groups":5},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["blocked","named",1,0],"OAI-SearchBot":["blocked","named",1,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",22,0],"Claude-SearchBot":["partial","*",22,0],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["partial","*",22,0],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["partial","*",22,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["partial","*",22,0],"ContextFetchBot":["partial","*",22,0]},"llms":{"status":"unreachable","error":"<StreamReset stream_id:3, error_code:2, remote_reset:True>"},"home":{"status":"unreachable","error":"<StreamReset stream_id:5, error_code:2, remote_reset:True>"}},
{"rank":886,"domain":"mediawiki.org","origin":"https://mediawiki.org","robots":{"status":"ok","http":200,"url":"https://www.mediawiki.org/robots.txt","bytes":15254,"groups":33},"bots":{"GPTBot":["partial","*",200,4],"ChatGPT-User":["partial","*",200,4],"OAI-SearchBot":["partial","*",200,4],"ClaudeBot":["partial","*",200,4],"Claude-User":["partial","*",200,4],"Claude-SearchBot":["partial","*",200,4],"anthropic-ai":["partial","*",200,4],"PerplexityBot":["partial","*",200,4],"Perplexity-User":["partial","*",200,4],"Google-Extended":["partial","*",200,4],"Applebot-Extended":["partial","*",200,4],"CCBot":["partial","*",200,4],"Bytespider":["partial","*",200,4],"Amazonbot":["partial","*",200,4],"meta-externalagent":["partial","*",200,4],"cohere-ai":["partial","*",200,4],"Diffbot":["partial","*",200,4],"ContextFetchBot":["partial","*",200,4]},"llms":{"status":"missing","http":404,"bytes":1868},"home":{"status":"ok","http":200,"url":"https://www.mediawiki.org/wiki/MediaWiki","html_tokens":22693,"markdown_tokens":444,"pruned_tokens":171}},
{"rank":887,"domain":"ibb.co","origin":"https://ibb.co","robots":{"status":"ok","http":200,"url":"https://ibb.co/robots.txt","bytes":0,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":150},"home":{"status":"ok","http":200,"url":"https://imgbb.com/","html_tokens":10679,"markdown_tokens":85,"pruned_tokens":85}},
{"rank":888,"domain":"go.dev","origin":"https://go.dev","robots":{"status":"ok","http":200,"url":"https://go.dev/robots.txt","bytes":23,"groups":1},"bots":{"GPTBot":["allowed","*",0,1],"ChatGPT-User":["allowed","*",0,1],"OAI-SearchBot":["allowed","*",0,1],"ClaudeBot":["allowed","*",0,1],"Claude-User":["allowed","*",0,1],"Claude-SearchBot":["allowed","*",0,1],"anthropic-ai":["allowed","*",0,1],"PerplexityBot":["allowed","*",0,1],"Perplexity-User":["allowed","*",0,1],"Google-Extended":["allowed","*",0,1],"Applebot-Extended":["allowed","*",0,1],"CCBot":["allowed","*",0,1],"Bytespider":["allowed","*",0,1],"Amazonbot":["allowed","*",0,1],"meta-externalagent":["allowed","*",0,1],"cohere-ai":["allowed","*",0,1],"Diffbot":["allowed","*",0,1],"ContextFetchBot":["allowed","*",0,1]},"llms":{"status":"missing","http":404,"bytes":26974},"home":{"status":"ok","http":200,"url":"https://go.dev/","html_tokens":16039,"markdown_tokens":582,"pruned_tokens":207}},
{"rank":889,"domain":"toutiao.com","origin":"https://toutiao.com","robots":{"status":"ok","http":200,"url":"https://www.toutiao.com/robots.txt","bytes":344,"groups":1},"bots":{"GPTBot":["partial","*",18,0],"ChatGPT-User":["partial","*",18,0],"OAI-SearchBot":["partial","*",18,0],"ClaudeBot":["partial","*",18,0],"Claude-User":["partial","*",18,0],"Claude-SearchBot":["partial","*",18,0],"anthropic-ai":["partial","*",18,0],"PerplexityBot":["partial","*",18,0],"Perplexity-User":["partial","*",18,0],"Google-Extended":["partial","*",18,0],"Applebot-Extended":["partial","*",18,0],"CCBot":["partial","*",18,0],"Bytespider":["partial","*",18,0],"Amazonbot":["partial","*",18,0],"meta-externalagent":["partial","*",18,0],"cohere-ai":["partial","*",18,0],"Diffbot":["partial","*",18,0],"ContextFetchBot":["partial","*",18,0]},"llms":{"status":"timeout","error":"ConnectTimeout"},"home":{"status":"ok","http":200,"url":"https://www.toutiao.com/","html_tokens":1221,"markdown_tokens":6,"pruned_tokens":6}},
{"rank":890,"domain":"hrw.org","origin":"https://hrw.org","robots":{"status":"refused","http":403,"url":"https://hrw.org/robots.txt","bytes":560,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":891,"domain":"investing.com","origin":"https://investing.com","robots":{"status":"refused","http":403,"url":"https://investing.com/robots.txt","bytes":3,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":892,"domain":"panasonic.com","origin":"https://panasonic.com","robots":{"status":"refused","http":403,"url":"https://www.panasonic.com/robots.txt","bytes":385,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":893,"domain":"prezi.com","origin":"https://prezi.com","robots":{"status":"ok","http":200,"url":"https://prezi.com/robots.txt","bytes":833,"groups":5},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["partial","*",4,0],"OAI-SearchBot":["allowed","named",0,1],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",4,0],"Claude-SearchBot":["allowed","named",0,1],"anthropic-ai":["partial","*",4,0],"PerplexityBot":["partial","*",4,0],"Perplexity-User":["partial","*",4,0],"Google-Extended":["partial","*",4,0],"Applebot-Extended":["partial","*",4,0],"CCBot":["partial","*",4,0],"Bytespider":["partial","*",4,0],"Amazonbot":["partial","*",4,0],"meta-externalagent":["partial","*",4,0],"cohere-ai":["partial","*",4,0],"Diffbot":["partial","*",4,0],"ContextFetchBot":["partial","*",4,0]},"llms":{"status":"present","http":200,"bytes":2915},"home":{"status":"ok","http":200,"url":"https://prezi.com/","html_tokens":119044,"markdown_tokens":2053,"pruned_tokens":300}},
{"rank":894,"domain":"nyc.gov","origin":"https://nyc.gov","robots":{"status":"ok","http":200,"url":"https://www.nyc.gov/robots.txt","bytes":36,"groups":1},"bots":{"GPTBot":["partial","*",1,0],"ChatGPT-User":["partial","*",1,0],"OAI-SearchBot":["partial","*",1,0],"ClaudeBot":["partial","*",1,0],"Claude-User":["partial","*",1,0],"Claude-SearchBot":["partial","*",1,0],"anthropic-ai":["partial","*",1,0],"PerplexityBot":["partial","*",1,0],"Perplexity-User":["partial","*",1,0],"Google-Extended":["partial","*",1,0],"Applebot-Extended":["partial","*",1,0],"CCBot":["partial","*",1,0],"Bytespider":["partial","*",1,0],"Amazonbot":["partial","*",1,0],"meta-externalagent":["partial","*",1,0],"cohere-ai":["partial","*",1,0],"Diffbot":["partial","*",1,0],"ContextFetchBot":["partial","*",1,0]},"llms":{"status":"http_error","http":490,"bytes":16870},"home":{"status":"ok","http":200,"url":"https://www.nyc.gov/main","html_tokens":133199,"markdown_tokens":41,"pruned_tokens":41}},
{"rank":895,"domain":"hyatt.com","origin":"https://hyatt.com","robots":{"status":"refused","http":403,"url":"https://www.hyatt.com/robots.txt","bytes":12858,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":896,"domain":"waze.com","origin":"https://waze.com","robots":{"status":"ok","http":200,"url":"https://www.waze.com/robots.txt","bytes":1544,"groups":16},"bots":{"GPTBot":["partial","*",24,2],"ChatGPT-User":["partial","*",24,2],"OAI-SearchBot":["partial","*",24,2],"ClaudeBot":["partial","*",24,2],"Claude-User":["partial","*",24,2],"Claude-SearchBot":["partial","*",24,2],"anthropic-ai":["partial","*",24,2],"PerplexityBot":["partial","*",24,2],"Perplexity-User":["partial","*",24,2],"Google-Extended":["partial","*",24,2],"Applebot-Extended":["partial","*",24,2],"CCBot":["partial","*",24,2],"Bytespider":["partial","*",24,2],"Amazonbot":["partial","*",24,2],"meta-externalagent":["partial","*",24,2],"cohere-ai":["partial","*",24,2],"Diffbot":["partial","*",24,2],"ContextFetchBot":["partial","*",24,2]},"llms":{"status":"missing","http":404,"bytes":30853},"home":{"status":"ok","http":200,"url":"https://www.waze.com/","html_tokens":87847,"markdown_tokens":54,"pruned_tokens":54}},
{"rank":897,"domain":"nintendo.com","origin":"https://nintendo.com","robots":{"status":"ok","http":200,"url":"https://www.nintendo.com/robots.txt","bytes":5596,"groups":26},"bots":{"GPTBot":["partial","named",1,1],"ChatGPT-User":["partial","*",4,1],"OAI-SearchBot":["partial","*",4,1],"ClaudeBot":["partial","named",1,1],"Claude-User":["partial","*",4,1],"Claude-SearchBot":["partial","*",4,1],"anthropic-ai":["partial","*",4,1],"PerplexityBot":["partial","*",4,1],"Perplexity-User":["partial","*",4,1],"Google-Extended":["partial","named",1,1],"Applebot-Extended":["partial","named",1,1],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["partial","*",4,1],"meta-externalagent":["partial","named",1,1],"cohere-ai":["partial","named",1,1],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",4,1]},"llms":{"status":"missing","http":404,"bytes":219466},"home":{"status":"ok","http":200,"url":"https://www.nintendo.com/us/","html_tokens":414540,"markdown_tokens":162,"pruned_tokens":52}},
{"rank":898,"domain":"libsyn.com","origin":"https://libsyn.com","robots":{"status":"ok","http":200,"url":"https://libsyn.com/robots.txt","bytes":168,"groups":1},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"present","http":200,"bytes":8679},"home":{"status":"ok","http":200,"url":"https://libsyn.com/","html_tokens":36128,"markdown_tokens":359,"pruned_tokens":304}},
{"rank":899,"domain":"heyzine.com","origin":"https://heyzine.com","robots":{"status":"ok","http":200,"url":"https://heyzine.com/robots.txt","bytes":33,"groups":1},"bots":{"GPTBot":["partial","*",1,0],"ChatGPT-User":["partial","*",1,0],"OAI-SearchBot":["partial","*",1,0],"ClaudeBot":["partial","*",1,0],"Claude-User":["partial","*",1,0],"Claude-SearchBot":["partial","*",1,0],"anthropic-ai":["partial","*",1,0],"PerplexityBot":["partial","*",1,0],"Perplexity-User":["partial","*",1,0],"Google-Extended":["partial","*",1,0],"Applebot-Extended":["partial","*",1,0],"CCBot":["partial","*",1,0],"Bytespider":["partial","*",1,0],"Amazonbot":["partial","*",1,0],"meta-externalagent":["partial","*",1,0],"cohere-ai":["partial","*",1,0],"Diffbot":["partial","*",1,0],"ContextFetchBot":["partial","*",1,0]},"llms":{"status":"missing","http":404,"bytes":0},"home":{"status":"ok","http":200,"url":"https://heyzine.com/","html_tokens":126170,"markdown_tokens":728,"pruned_tokens":128}},
{"rank":900,"domain":"last.fm","origin":"https://last.fm","robots":{"status":"ok","http":200,"url":"https://www.last.fm/robots.txt","bytes":8727,"groups":3},"bots":{"GPTBot":["partial","*",279,3],"ChatGPT-User":["partial","*",279,3],"OAI-SearchBot":["partial","*",279,3],"ClaudeBot":["partial","*",279,3],"Claude-User":["partial","*",279,3],"Claude-SearchBot":["partial","*",279,3],"anthropic-ai":["partial","*",279,3],"PerplexityBot":["partial","*",279,3],"Perplexity-User":["partial","*",279,3],"Google-Extended":["partial","*",279,3],"Applebot-Extended":["partial","*",279,3],"CCBot":["partial","*",279,3],"Bytespider":["partial","*",279,3],"Amazonbot":["partial","*",279,3],"meta-externalagent":["partial","*",279,3],"cohere-ai":["partial","*",279,3],"Diffbot":["partial","*",279,3],"ContextFetchBot":["partial","*",279,3]},"llms":{"status":"missing","http":404,"bytes":54139},"home":{"status":"ok","http":200,"url":"https://www.last.fm/","html_tokens":15022,"markdown_tokens":176,"pruned_tokens":116}},
{"rank":901,"domain":"gimp.org","origin":"https://gimp.org","robots":{"status":"ok","http":200,"url":"https://www.gimp.org/robots-www.txt","bytes":332,"groups":1},"bots":{"GPTBot":["partial","*",12,0],"ChatGPT-User":["partial","*",12,0],"OAI-SearchBot":["partial","*",12,0],"ClaudeBot":["partial","*",12,0],"Claude-User":["partial","*",12,0],"Claude-SearchBot":["partial","*",12,0],"anthropic-ai":["partial","*",12,0],"PerplexityBot":["partial","*",12,0],"Perplexity-User":["partial","*",12,0],"Google-Extended":["partial","*",12,0],"Applebot-Extended":["partial","*",12,0],"CCBot":["partial","*",12,0],"Bytespider":["partial","*",12,0],"Amazonbot":["partial","*",12,0],"meta-externalagent":["partial","*",12,0],"cohere-ai":["partial","*",12,0],"Diffbot":["partial","*",12,0],"ContextFetchBot":["partial","*",12,0]},"llms":{"status":"missing","http":404,"bytes":10877},"home":{"status":"ok","http":200,"url":"https://www.gimp.org/","html_tokens":5441,"markdown_tokens":255,"pruned_tokens":255}},
{"rank":902,"domain":"cisa.gov","origin":"https://cisa.gov","robots":{"status":"refused","http":403,"url":"https://cisa.gov/robots.txt","bytes":372,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":903,"domain":"asso.fr","origin":"https://asso.fr","robots":{"status":"unreachable","http":null,"url":"https://asso.fr/robots.txt","bytes":0,"groups":0,"error":"cannot resolve asso.fr"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":904,"domain":"ncsu.edu","origin":"https://ncsu.edu","robots":{"status":"html","http":202,"url":"https://www.ncsu.edu/robots.txt","bytes":2435,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":905,"domain":"osha.gov","origin":"https://osha.gov","robots":{"status":"refused","http":403,"url":"https://osha.gov/robots.txt","bytes":919,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":906,"domain":"navy.mil","origin":"https://www.navy.mil","robots":{"status":"refused","http":403,"url":"https://www.navy.mil/robots.txt","bytes":380,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":907,"domain":"b.hatena.ne.jp","origin":"https://b.hatena.ne.jp","robots":{"status":"ok","http":200,"url":"https://b.hatena.ne.jp/robots.txt","bytes":933,"groups":4},"bots":{"GPTBot":["partial","*",12,0],"ChatGPT-User":["partial","*",12,0],"OAI-SearchBot":["partial","*",12,0],"ClaudeBot":["partial","*",12,0],"Claude-User":["partial","*",12,0],"Claude-SearchBot":["partial","*",12,0],"anthropic-ai":["partial","*",12,0],"PerplexityBot":["partial","*",12,0],"Perplexity-User":["partial","*",12,0],"Google-Extended":["partial","*",12,0],"Applebot-Extended":["partial","*",12,0],"CCBot":["partial","*",12,0],"Bytespider":["partial","*",12,0],"Amazonbot":["partial","*",12,0],"meta-externalagent":["partial","*",12,0],"cohere-ai":["partial","*",12,0],"Diffbot":["partial","*",12,0],"ContextFetchBot":["partial","*",12,0]},"llms":{"status":"missing","http":404,"bytes":11899},"home":{"status":"ok","http":200,"url":"https://b.hatena.ne.jp/","html_tokens":141043,"markdown_tokens":100,"pruned_tokens":100}},
{"rank":908,"domain":"lego.com","origin":"https://lego.com","robots":{"status":"ok","http":200,"url":"https://www.lego.com/robots.txt","bytes":784,"groups":1},"bots":{"GPTBot":["partial","*",16,0],"ChatGPT-User":["partial","*",16,0],"OAI-SearchBot":["partial","*",16,0],"ClaudeBot":["partial","*",16,0],"Claude-User":["partial","*",16,0],"Claude-SearchBot":["partial","*",16,0],"anthropic-ai":["partial","*",16,0],"PerplexityBot":["partial","*",16,0],"Perplexity-User":["partial","*",16,0],"Google-Extended":["partial","*",16,0],"Applebot-Extended":["partial","*",16,0],"CCBot":["partial","*",16,0],"Bytespider":["partial","*",16,0],"Amazonbot":["partial","*",16,0],"meta-externalagent":["partial","*",16,0],"cohere-ai":["partial","*",16,0],"Diffbot":["partial","*",16,0],"ContextFetchBot":["partial","*",16,0]},"llms":{"status":"missing","http":404,"bytes":118543},"home":{"status":"http_error","http":403,"url":"https://www.lego.com/"}},
{"rank":909,"domain":"isc.org","origin":"https://isc.org","robots":{"status":"ok","http":200,"url":"https://www.isc.org/robots.txt","bytes":73,"groups":1},"bots":{"GPTBot":["partial","*",1,0],"ChatGPT-User":["partial","*",1,0],"OAI-SearchBot":["partial","*",1,0],"ClaudeBot":["partial","*",1,0],"Claude-User":["partial","*",1,0],"Claude-SearchBot":["partial","*",1,0],"anthropic-ai":["partial","*",1,0],"PerplexityBot":["partial","*",1,0],"Perplexity-User":["partial","*",1,0],"Google-Extended":["partial","*",1,0],"Applebot-Extended":["partial","*",1,0],"CCBot":["partial","*",1,0],"Bytespider":["partial","*",1,0],"Amazonbot":["partial","*",1,0],"meta-externalagent":["partial","*",1,0],"cohere-ai":["partial","*",1,0],"Diffbot":["partial","*",1,0],"ContextFetchBot":["partial","*",1,0]},"llms":{"status":"missing","http":404,"bytes":18832},"home":{"status":"ok","http":200,"url":"https://www.isc.org/","html_tokens":6696,"markdown_tokens":324,"pruned_tokens":39}},
{"rank":910,"domain":"welt.de","origin":"https://welt.de","robots":{"status":"ok","http":200,"url":"https://www.welt.de/robots.txt","bytes":3501,"groups":3},"bots":{"GPTBot":["partial","*",44,0],"ChatGPT-User":["partial","*",44,0],"OAI-SearchBot":["partial","*",44,0],"ClaudeBot":["partial","*",44,0],"Claude-User":["partial","*",44,0],"Claude-SearchBot":["partial","*",44,0],"anthropic-ai":["partial","*",44,0],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["blocked","named",1,0],"Google-Extended":["partial","*",44,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["partial","*",44,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",44,0]},"llms":{"status":"missing","http":404,"bytes":438250},"home":{"status":"ok","http":200,"url":"https://www.welt.de/","html_tokens":351222,"markdown_tokens":91,"pruned_tokens":91}},
{"rank":911,"domain":"amap.com","origin":"https://amap.com","robots":{"status":"ok","http":200,"url":"https://amap.com/robots.txt","bytes":19738,"groups":49},"bots":{"GPTBot":["partial","named",6,11],"ChatGPT-User":["partial","named",6,11],"OAI-SearchBot":["partial","named",6,11],"ClaudeBot":["partial","*",8,11],"Claude-User":["partial","*",8,11],"Claude-SearchBot":["partial","*",8,11],"anthropic-ai":["partial","named",6,11],"PerplexityBot":["partial","named",6,11],"Perplexity-User":["partial","*",8,11],"Google-Extended":["partial","named",6,11],"Applebot-Extended":["partial","*",8,11],"CCBot":["partial","named",6,11],"Bytespider":["partial","named",6,11],"Amazonbot":["partial","named",6,11],"meta-externalagent":["partial","named",6,11],"cohere-ai":["partial","named",6,11],"Diffbot":["partial","*",8,11],"ContextFetchBot":["partial","*",8,11]},"llms":{"status":"disallowed"},"home":{"status":"disallowed"}},
{"rank":912,"domain":"lifehacker.com","origin":"https://lifehacker.com","robots":{"status":"ok","http":200,"url":"https://lifehacker.com/robots.txt","bytes":6427,"groups":3},"bots":{"GPTBot":["partial","named",2,2],"ChatGPT-User":["partial","named",2,2],"OAI-SearchBot":["partial","named",2,2],"ClaudeBot":["partial","named",2,2],"Claude-User":["partial","named",1,2],"Claude-SearchBot":["partial","named",1,2],"anthropic-ai":["partial","named",2,2],"PerplexityBot":["partial","named",2,2],"Perplexity-User":["partial","named",2,2],"Google-Extended":["partial","*",1,1],"Applebot-Extended":["partial","named",2,2],"CCBot":["partial","named",2,2],"Bytespider":["partial","named",2,2],"Amazonbot":["partial","named",2,2],"meta-externalagent":["partial","named",2,2],"cohere-ai":["partial","named",2,2],"Diffbot":["partial","named",2,2],"ContextFetchBot":["partial","*",1,1]},"llms":{"status":"missing","http":404,"bytes":261586},"home":{"status":"ok","http":200,"url":"https://lifehacker.com/","html_tokens":123143,"markdown_tokens":1307,"pruned_tokens":182}},
{"rank":913,"domain":"nba.com","origin":"https://nba.com","robots":{"status":"ok","http":200,"url":"https://www.nba.com/robots.txt","bytes":10036,"groups":13},"bots":{"GPTBot":["partial","named",2,39],"ChatGPT-User":["partial","named",1,30],"OAI-SearchBot":["allowed","named",0,1],"ClaudeBot":["partial","named",1,30],"Claude-User":["partial","*",68,0],"Claude-SearchBot":["partial","*",68,0],"anthropic-ai":["partial","named",1,30],"PerplexityBot":["partial","named",1,30],"Perplexity-User":["partial","*",68,0],"Google-Extended":["partial","named",2,39],"Applebot-Extended":["partial","named",1,30],"CCBot":["partial","named",1,30],"Bytespider":["partial","*",68,0],"Amazonbot":["partial","named",1,30],"meta-externalagent":["partial","*",68,0],"cohere-ai":["partial","*",68,0],"Diffbot":["partial","*",68,0],"ContextFetchBot":["partial","*",68,0]},"llms":{"status":"missing","http":404,"bytes":4810},"home":{"status":"http_error","http":403,"url":"https://www.nba.com/"}},
{"rank":914,"domain":"ucsb.edu","origin":"https://ucsb.edu","robots":{"status":"refused","http":403,"url":"https://ucsb.edu/robots.txt","bytes":560,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":915,"domain":"sfu.ca","origin":"https://www.sfu.ca","robots":{"status":"ok","http":200,"url":"https://www.sfu.ca/robots.txt","bytes":3251,"groups":2},"bots":{"GPTBot":["partial","*",91,4],"ChatGPT-User":["partial","*",91,4],"OAI-SearchBot":["partial","*",91,4],"ClaudeBot":["partial","*",91,4],"Claude-User":["partial","*",91,4],"Claude-SearchBot":["partial","*",91,4],"anthropic-ai":["partial","*",91,4],"PerplexityBot":["partial","*",91,4],"Perplexity-User":["partial","*",91,4],"Google-Extended":["partial","*",91,4],"Applebot-Extended":["partial","*",91,4],"CCBot":["partial","*",91,4],"Bytespider":["partial","*",91,4],"Amazonbot":["partial","*",91,4],"meta-externalagent":["partial","*",91,4],"cohere-ai":["partial","*",91,4],"Diffbot":["partial","*",91,4],"ContextFetchBot":["partial","*",91,4]},"llms":{"status":"missing","http":404,"bytes":2461},"home":{"status":"ok","http":200,"url":"https://www.sfu.ca/","html_tokens":23200,"markdown_tokens":115,"pruned_tokens":62}},
{"rank":916,"domain":"blogspot.co.uk","origin":"https://blogspot.co.uk","robots":{"status":"ok","http":200,"url":"https://blogspot.co.uk/robots.txt","bytes":6624,"groups":4},"bots":{"GPTBot":["partial","*",169,64],"ChatGPT-User":["partial","*",169,64],"OAI-SearchBot":["partial","*",169,64],"ClaudeBot":["partial","*",169,64],"Claude-User":["partial","*",169,64],"Claude-SearchBot":["partial","*",169,64],"anthropic-ai":["partial","*",169,64],"PerplexityBot":["partial","*",169,64],"Perplexity-User":["partial","*",169,64],"Google-Extended":["partial","*",169,64],"Applebot-Extended":["partial","*",169,64],"CCBot":["partial","*",169,64],"Bytespider":["partial","*",169,64],"Amazonbot":["partial","*",169,64],"meta-externalagent":["partial","*",169,64],"cohere-ai":["partial","*",169,64],"Diffbot":["partial","*",169,64],"ContextFetchBot":["partial","*",169,64]},"llms":{"status":"missing","http":404,"bytes":1569},"home":{"status":"ok","http":200,"url":"https://www.google.com/","html_tokens":21311,"markdown_tokens":30,"pruned_tokens":30}},
{"rank":917,"domain":"vkontakte.ru","origin":"https://vkontakte.ru","robots":{"status":"ok","http":200,"url":"https://vkontakte.ru/robots.txt","bytes":55,"groups":1},"bots":{"GPTBot":["partial","*",1,2],"ChatGPT-User":["partial","*",1,2],"OAI-SearchBot":["partial","*",1,2],"ClaudeBot":["partial","*",1,2],"Claude-User":["partial","*",1,2],"Claude-SearchBot":["partial","*",1,2],"anthropic-ai":["partial","*",1,2],"PerplexityBot":["partial","*",1,2],"Perplexity-User":["partial","*",1,2],"Google-Extended":["partial","*",1,2],"Applebot-Extended":["partial","*",1,2],"CCBot":["partial","*",1,2],"Bytespider":["partial","*",1,2],"Amazonbot":["partial","*",1,2],"meta-externalagent":["partial","*",1,2],"cohere-ai":["partial","*",1,2],"Diffbot":["partial","*",1,2],"ContextFetchBot":["partial","*",1,2]},"llms":{"status":"disallowed"},"home":{"status":"disallowed","error":"robots.txt of vk.ru disallows /login?act=vkcomredirect&to="}},
{"rank":918,"domain":"colorlib.com","origin":"https://colorlib.com","robots":{"status":"ok","http":200,"url":"https://colorlib.com/robots.txt","bytes":394,"groups":1},"bots":{"GPTBot":["partial","*",9,0],"ChatGPT-User":["partial","*",9,0],"OAI-SearchBot":["partial","*",9,0],"ClaudeBot":["partial","*",9,0],"Claude-User":["partial","*",9,0],"Claude-SearchBot":["partial","*",9,0],"anthropic-ai":["partial","*",9,0],"PerplexityBot":["partial","*",9,0],"Perplexity-User":["partial","*",9,0],"Google-Extended":["partial","*",9,0],"Applebot-Extended":["partial","*",9,0],"CCBot":["partial","*",9,0],"Bytespider":["partial","*",9,0],"Amazonbot":["partial","*",9,0],"meta-externalagent":["partial","*",9,0],"cohere-ai":["partial","*",9,0],"Diffbot":["partial","*",9,0],"ContextFetchBot":["partial","*",9,0]},"llms":{"status":"present","http":200,"bytes":4875},"home":{"status":"ok","http":200,"url":"https://colorlib.com/","html_tokens":21971,"markdown_tokens":528,"pruned_tokens":228}},
{"rank":919,"domain":"hootsuite.com","origin":"https://hootsuite.com","robots":{"status":"ok","http":200,"url":"https://www.hootsuite.com/robots.txt","bytes":7994,"groups":14},"bots":{"GPTBot":["partial","named",20,1],"ChatGPT-User":["partial","named",20,1],"OAI-SearchBot":["partial","named",20,1],"ClaudeBot":["partial","named",20,1],"Claude-User":["partial","named",20,1],"Claude-SearchBot":["partial","*",20,0],"anthropic-ai":["partial","*",20,0],"PerplexityBot":["partial","named",20,1],"Perplexity-User":["partial","named",20,1],"Google-Extended":["partial","named",20,1],"Applebot-Extended":["partial","named",20,1],"CCBot":["partial","*",20,0],"Bytespider":["partial","*",20,0],"Amazonbot":["partial","named",20,1],"meta-externalagent":["partial","named",20,1],"cohere-ai":["partial","*",20,0],"Diffbot":["partial","*",20,0],"ContextFetchBot":["partial","*",20,0]},"llms":{"status":"present","http":200,"bytes":21369},"home":{"status":"not_html","error":"unsupported content type text/markdown"}},
{"rank":920,"domain":"tesla.com","origin":"https://tesla.com","robots":{"status":"refused","http":403,"url":"https://tesla.com/robots.txt","bytes":373,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":921,"domain":"yp.to","origin":"https://yp.to","robots":{"status":"unreachable","http":null,"url":"https://yp.to/robots.txt","bytes":0,"groups":0,"error":"cannot resolve yp.to"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":922,"domain":"news.com.au","origin":"https://news.com.au","robots":{"status":"ok","http":200,"url":"https://www.news.com.au/robots.txt","bytes":3614,"groups":4},"bots":{"GPTBot":["partial","*",1,2],"ChatGPT-User":["partial","*",1,2],"OAI-SearchBot":["partial","*",1,2],"ClaudeBot":["partial","*",1,2],"Claude-User":["partial","*",1,2],"Claude-SearchBot":["partial","*",1,2],"anthropic-ai":["partial","*",1,2],"PerplexityBot":["partial","*",1,2],"Perplexity-User":["partial","*",1,2],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["partial","*",1,2],"CCBot":["partial","*",1,2],"Bytespider":["partial","*",1,2],"Amazonbot":["partial","*",1,2],"meta-externalagent":["partial","*",1,2],"cohere-ai":["partial","*",1,2],"Diffbot":["partial","*",1,2],"ContextFetchBot":["partial","*",1,2]},"llms":{"status":"disallowed"},"home":{"status":"disallowed"}},
{"rank":923,"domain":"domaintools.com","origin":"https://domaintools.com","robots":{"status":"ok","http":200,"url":"https://www.domaintools.com/robots.txt","bytes":48,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"present","http":200,"bytes":7556},"home":{"status":"ok","http":200,"url":"https://www.domaintools.com/","html_tokens":104919,"markdown_tokens":394,"pruned_tokens":219}},
{"rank":924,"domain":"atlassian.net","origin":"https://atlassian.net","robots":{"status":"ok","http":200,"url":"https://atlassian.net/robots.txt","bytes":347,"groups":2},"bots":{"GPTBot":["partial","*",9,1],"ChatGPT-User":["partial","*",9,1],"OAI-SearchBot":["partial","*",9,1],"ClaudeBot":["partial","*",9,1],"Claude-User":["partial","*",9,1],"Claude-SearchBot":["partial","*",9,1],"anthropic-ai":["partial","*",9,1],"PerplexityBot":["partial","*",9,1],"Perplexity-User":["partial","*",9,1],"Google-Extended":["partial","*",9,1],"Applebot-Extended":["partial","*",9,1],"CCBot":["partial","*",9,1],"Bytespider":["partial","*",9,1],"Amazonbot":["partial","*",9,1],"meta-externalagent":["partial","*",9,1],"cohere-ai":["partial","*",9,1],"Diffbot":["partial","*",9,1],"ContextFetchBot":["partial","*",9,1]},"llms":{"status":"html","http":200,"bytes":1826234},"home":{"status":"ok","http":200,"url":"https://www.atlassian.com/software","html_tokens":456549,"markdown_tokens":151,"pruned_tokens":24}},
{"rank":925,"domain":"target.com","origin":"https://target.com","robots":{"status":"ok","http":200,"url":"https://www.target.com/robots.txt","bytes":3226,"groups":1},"bots":{"GPTBot":["partial","*",114,0],"ChatGPT-User":["partial","*",114,0],"OAI-SearchBot":["partial","*",114,0],"ClaudeBot":["partial","*",114,0],"Claude-User":["partial","*",114,0],"Claude-SearchBot":["partial","*",114,0],"anthropic-ai":["partial","*",114,0],"PerplexityBot":["partial","*",114,0],"Perplexity-User":["partial","*",114,0],"Google-Extended":["partial","*",114,0],"Applebot-Extended":["partial","*",114,0],"CCBot":["partial","*",114,0],"Bytespider":["partial","*",114,0],"Amazonbot":["partial","*",114,0],"meta-externalagent":["partial","*",114,0],"cohere-ai":["partial","*",114,0],"Diffbot":["partial","*",114,0],"ContextFetchBot":["partial","*",114,0]},"llms":{"status":"present","http":200,"bytes":6158},"home":{"status":"ok","http":200,"url":"https://www.target.com/","html_tokens":113046,"markdown_tokens":111,"pruned_tokens":111}},
{"rank":926,"domain":"zimbra.com","origin":"https://zimbra.com","robots":{"status":"refused","http":403,"url":"https://zimbra.com/robots.txt","bytes":560,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":927,"domain":"mlb.com","origin":"https://mlb.com","robots":{"status":"ok","http":200,"url":"https://www.mlb.com/robots.txt","bytes":2224,"groups":1},"bots":{"GPTBot":["partial","*",76,0],"ChatGPT-User":["partial","*",76,0],"OAI-SearchBot":["partial","*",76,0],"ClaudeBot":["partial","*",76,0],"Claude-User":["partial","*",76,0],"Claude-SearchBot":["partial","*",76,0],"anthropic-ai":["partial","*",76,0],"PerplexityBot":["partial","*",76,0],"Perplexity-User":["partial","*",76,0],"Google-Extended":["partial","*",76,0],"Applebot-Extended":["partial","*",76,0],"CCBot":["partial","*",76,0],"Bytespider":["partial","*",76,0],"Amazonbot":["partial","*",76,0],"meta-externalagent":["partial","*",76,0],"cohere-ai":["partial","*",76,0],"Diffbot":["partial","*",76,0],"ContextFetchBot":["partial","*",76,0]},"llms":{"status":"missing","http":404,"bytes":118152},"home":{"status":"ok","http":200,"url":"https://www.mlb.com/","html_tokens":351350,"markdown_tokens":55,"pruned_tokens":55}},
{"rank":928,"domain":"caltech.edu","origin":"https://www.caltech.edu","robots":{"status":"ok","http":200,"url":"https://www.caltech.edu/robots.txt","bytes":265,"groups":3},"bots":{"GPTBot":["partial","*",3,1],"ChatGPT-User":["partial","*",3,1],"OAI-SearchBot":["partial","*",3,1],"ClaudeBot":["partial","*",3,1],"Claude-User":["partial","*",3,1],"Claude-SearchBot":["partial","*",3,1],"anthropic-ai":["partial","*",3,1],"PerplexityBot":["partial","*",3,1],"Perplexity-User":["partial","*",3,1],"Google-Extended":["partial","*",3,1],"Applebot-Extended":["partial","*",3,1],"CCBot":["partial","*",3,1],"Bytespider":["partial","*",3,1],"Amazonbot":["partial","*",3,1],"meta-externalagent":["partial","*",3,1],"cohere-ai":["partial","*",3,1],"Diffbot":["partial","*",3,1],"ContextFetchBot":["partial","*",3,1]},"llms":{"status":"missing","http":404,"bytes":195961},"home":{"status":"ok","http":200,"url":"https://www.caltech.edu/","html_tokens":88596,"markdown_tokens":273,"pruned_tokens":90}},
{"rank":929,"domain":"eventbrite.co.uk","origin":"https://eventbrite.co.uk","robots":{"status":"ok","http":200,"url":"https://eventbrite.co.uk/robots.txt","bytes":7563,"groups":135},"bots":{"GPTBot":["partial","*",51,2],"ChatGPT-User":["partial","*",51,2],"OAI-SearchBot":["partial","*",51,2],"ClaudeBot":["partial","*",51,2],"Claude-User":["partial","*",51,2],"Claude-SearchBot":["partial","*",51,2],"anthropic-ai":["partial","*",51,2],"PerplexityBot":["partial","*",51,2],"Perplexity-User":["partial","*",51,2],"Google-Extended":["partial","*",51,2],"Applebot-Extended":["partial","*",51,2],"CCBot":["blocked","named",1,0],"Bytespider":["partial","*",51,2],"Amazonbot":["partial","*",51,2],"meta-externalagent":["partial","*",51,2],"cohere-ai":["partial","*",51,2],"Diffbot":["partial","*",51,2],"ContextFetchBot":["partial","*",51,2]},"llms":{"status":"missing","http":404,"bytes":79449},"home":{"status":"ok","http":200,"url":"https://www.eventbrite.co.uk","html_tokens":50901,"markdown_tokens":64,"pruned_tokens":64}},
{"rank":930,"domain":"tableau.com","origin":"https://tableau.com","robots":{"status":"refused","http":403,"url":"https://tableau.com/robots.txt","bytes":375,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":931,"domain":"theregister.com","origin":"https://theregister.com","robots":{"status":"ok","http":200,"url":"https://www.theregister.com/robots.txt","bytes":6886,"groups":68},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["allowed","named",0,1],"OAI-SearchBot":["allowed","named",0,1],"ClaudeBot":["blocked","named",1,0],"Claude-User":["allowed","named",0,1],"Claude-SearchBot":["allowed","named",0,1],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["allowed","named",0,1],"Perplexity-User":["partial","*",1,2],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",1,2]},"llms":{"status":"disallowed"},"home":{"status":"disallowed"}},
{"rank":932,"domain":"zapier.com","origin":"https://zapier.com","robots":{"status":"ok","http":200,"url":"https://zapier.com/robots.txt","bytes":1840,"groups":4},"bots":{"GPTBot":["allowed","named",0,1],"ChatGPT-User":["allowed","named",0,1],"OAI-SearchBot":["allowed","named",0,1],"ClaudeBot":["allowed","named",0,1],"Claude-User":["allowed","named",0,1],"Claude-SearchBot":["allowed","named",0,1],"anthropic-ai":["partial","*",24,3],"PerplexityBot":["allowed","named",0,1],"Perplexity-User":["partial","*",24,3],"Google-Extended":["allowed","named",0,1],"Applebot-Extended":["partial","*",24,3],"CCBot":["partial","*",24,3],"Bytespider":["partial","*",24,3],"Amazonbot":["partial","*",24,3],"meta-externalagent":["partial","*",24,3],"cohere-ai":["partial","*",24,3],"Diffbot":["partial","*",24,3],"ContextFetchBot":["partial","*",24,3]},"llms":{"status":"present","http":200,"bytes":12654},"home":{"status":"not_html","error":"unsupported content type text/markdown"}},
{"rank":933,"domain":"iqiyi.com","origin":"https://iqiyi.com","robots":{"status":"ok","http":200,"url":"http://www.iqiyi.com/robots.txt","bytes":697,"groups":14},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["blocked","named",1,0],"OAI-SearchBot":["partial","*",5,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",5,0],"Claude-SearchBot":["partial","*",5,0],"anthropic-ai":["partial","*",5,0],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["partial","*",5,0],"Google-Extended":["partial","*",5,0],"Applebot-Extended":["partial","*",5,0],"CCBot":["partial","*",5,0],"Bytespider":["partial","*",5,0],"Amazonbot":["partial","*",5,0],"meta-externalagent":["partial","*",5,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["partial","*",5,0],"ContextFetchBot":["partial","*",5,0]},"llms":{"status":"missing","http":404,"bytes":180832},"home":{"status":"ok","http":200,"url":"https://www.iqiyi.com/","html_tokens":2044,"markdown_tokens":0,"pruned_tokens":0}},
{"rank":934,"domain":"fool.com","origin":"https://fool.com","robots":{"status":"ok","http":200,"url":"https://www.fool.com/robots.txt","bytes":2311,"groups":3},"bots":{"GPTBot":["partial","*",83,2],"ChatGPT-User":["partial","*",83,2],"OAI-SearchBot":["partial","*",83,2],"ClaudeBot":["partial","*",83,2],"Claude-User":["partial","*",83,2],"Claude-SearchBot":["partial","*",83,2],"anthropic-ai":["partial","*",83,2],"PerplexityBot":["partial","*",83,2],"Perplexity-User":["partial","*",83,2],"Google-Extended":["partial","*",83,2],"Applebot-Extended":["partial","*",83,2],"CCBot":["partial","*",83,2],"Bytespider":["blocked","named",1,0],"Amazonbot":["partial","*",83,2],"meta-externalagent":["partial","*",83,2],"cohere-ai":["partial","*",83,2],"Diffbot":["partial","*",83,2],"ContextFetchBot":["partial","*",83,2]},"llms":{"status":"missing","http":404,"bytes":34623},"home":{"status":"ok","http":200,"url":"https://www.fool.com/","html_tokens":134894,"markdown_tokens":1690,"pruned_tokens":295}},
{"rank":935,"domain":"americanexpress.com","origin":"https://americanexpress.com","robots":{"status":"ok","http":200,"url":"https://www.americanexpress.com/robots.txt","bytes":271,"groups":2},"bots":{"GPTBot":["partial","*",4,0],"ChatGPT-User":["partial","*",4,0],"OAI-SearchBot":["partial","*",4,0],"ClaudeBot":["partial","*",4,0],"Claude-User":["partial","*",4,0],"Claude-SearchBot":["partial","*",4,0],"anthropic-ai":["partial","*",4,0],"PerplexityBot":["partial","*",4,0],"Perplexity-User":["partial","*",4,0],"Google-Extended":["partial","*",4,0],"Applebot-Extended":["partial","*",4,0],"CCBot":["partial","*",4,0],"Bytespider":["partial","*",4,0],"Amazonbot":["partial","*",4,0],"meta-externalagent":["partial","*",4,0],"cohere-ai":["partial","*",4,0],"Diffbot":["partial","*",4,0],"ContextFetchBot":["partial","*",4,0]},"llms":{"status":"present","http":200,"bytes":1043},"home":{"status":"ok","http":200,"url":"https://www.americanexpress.com/","html_tokens":167946,"markdown_tokens":23,"pruned_tokens":23}},
{"rank":936,"domain":"weather.gov","origin":"https://weather.gov","robots":{"status":"missing","http":404,"url":"https://www.weather.gov/robots.txt","bytes":28808,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":28808},"home":{"status":"ok","http":200,"url":"https://www.weather.gov/","html_tokens":30875,"markdown_tokens":101,"pruned_tokens":101}},
{"rank":937,"domain":"ec-lyon.fr","origin":"https://www.ec-lyon.fr","robots":{"status":"ok","http":200,"url":"https://www.ec-lyon.fr/robots.txt","bytes":2141,"groups":1},"bots":{"GPTBot":["partial","*",35,22],"ChatGPT-User":["partial","*",35,22],"OAI-SearchBot":["partial","*",35,22],"ClaudeBot":["partial","*",35,22],"Claude-User":["partial","*",35,22],"Claude-SearchBot":["partial","*",35,22],"anthropic-ai":["partial","*",35,22],"PerplexityBot":["partial","*",35,22],"Perplexity-User":["partial","*",35,22],"Google-Extended":["partial","*",35,22],"Applebot-Extended":["partial","*",35,22],"CCBot":["partial","*",35,22],"Bytespider":["partial","*",35,22],"Amazonbot":["partial","*",35,22],"meta-externalagent":["partial","*",35,22],"cohere-ai":["partial","*",35,22],"Diffbot":["partial","*",35,22],"ContextFetchBot":["partial","*",35,22]},"llms":{"status":"missing","http":404,"bytes":188},"home":{"status":"ok","http":200,"url":"https://www.ec-lyon.fr/","html_tokens":47013,"markdown_tokens":164,"pruned_tokens":151}},
{"rank":938,"domain":"wikibooks.org","origin":"https://wikibooks.org","robots":{"status":"ok","http":200,"url":"https://en.wikibooks.org/robots.txt","bytes":18183,"groups":33},"bots":{"GPTBot":["partial","*",256,4],"ChatGPT-User":["partial","*",256,4],"OAI-SearchBot":["partial","*",256,4],"ClaudeBot":["partial","*",256,4],"Claude-User":["partial","*",256,4],"Claude-SearchBot":["partial","*",256,4],"anthropic-ai":["partial","*",256,4],"PerplexityBot":["partial","*",256,4],"Perplexity-User":["partial","*",256,4],"Google-Extended":["partial","*",256,4],"Applebot-Extended":["partial","*",256,4],"CCBot":["partial","*",256,4],"Bytespider":["partial","*",256,4],"Amazonbot":["partial","*",256,4],"meta-externalagent":["partial","*",256,4],"cohere-ai":["partial","*",256,4],"Diffbot":["partial","*",256,4],"ContextFetchBot":["partial","*",256,4]},"llms":{"status":"missing","http":404,"bytes":1868},"home":{"status":"ok","http":200,"url":"https://www.wikibooks.org/","html_tokens":16663,"markdown_tokens":8,"pruned_tokens":8}},
{"rank":939,"domain":"nymag.com","origin":"https://nymag.com","robots":{"status":"ok","http":200,"url":"https://nymag.com/robots.txt","bytes":8473,"groups":109},"bots":{"GPTBot":["allowed","named",0,1],"ChatGPT-User":["partial","named",1,2],"OAI-SearchBot":["partial","*",24,0],"ClaudeBot":["partial","named",1,2],"Claude-User":["partial","named",1,2],"Claude-SearchBot":["partial","named",1,2],"anthropic-ai":["partial","named",1,2],"PerplexityBot":["partial","named",1,2],"Perplexity-User":["partial","named",1,2],"Google-Extended":["partial","named",1,2],"Applebot-Extended":["partial","named",1,2],"CCBot":["partial","named",1,2],"Bytespider":["partial","named",1,2],"Amazonbot":["allowed","named",0,1],"meta-externalagent":["partial","named",1,2],"cohere-ai":["partial","named",1,2],"Diffbot":["partial","named",1,2],"ContextFetchBot":["partial","*",24,0]},"llms":{"status":"missing","http":404,"bytes":47002},"home":{"status":"ok","http":200,"url":"https://nymag.com/","html_tokens":192320,"markdown_tokens":172,"pruned_tokens":141}},
{"rank":940,"domain":"securityfocus.com","origin":"https://securityfocus.com","robots":{"status":"ok","http":200,"url":"https://securityfocus.com/robots.txt","bytes":70,"groups":1},"bots":{"GPTBot":["allowed","*",0,1],"ChatGPT-User":["allowed","*",0,1],"OAI-SearchBot":["allowed","*",0,1],"ClaudeBot":["allowed","*",0,1],"Claude-User":["allowed","*",0,1],"Claude-SearchBot":["allowed","*",0,1],"anthropic-ai":["allowed","*",0,1],"PerplexityBot":["allowed","*",0,1],"Perplexity-User":["allowed","*",0,1],"Google-Extended":["allowed","*",0,1],"Applebot-Extended":["allowed","*",0,1],"CCBot":["allowed","*",0,1],"Bytespider":["allowed","*",0,1],"Amazonbot":["allowed","*",0,1],"meta-externalagent":["allowed","*",0,1],"cohere-ai":["allowed","*",0,1],"Diffbot":["allowed","*",0,1],"ContextFetchBot":["allowed","*",0,1]},"llms":{"status":"html","http":200,"bytes":11487},"home":{"status":"ok","http":200,"url":"https://securityfocus.com/","html_tokens":2857,"markdown_tokens":259,"pruned_tokens":108}},
{"rank":941,"domain":"ispconfig.org","origin":"https://ispconfig.org","robots":{"status":"ok","http":200,"url":"https://ispconfig.org/robots.txt","bytes":34,"groups":1},"bots":{"GPTBot":["partial","*",1,0],"ChatGPT-User":["partial","*",1,0],"OAI-SearchBot":["partial","*",1,0],"ClaudeBot":["partial","*",1,0],"Claude-User":["partial","*",1,0],"Claude-SearchBot":["partial","*",1,0],"anthropic-ai":["partial","*",1,0],"PerplexityBot":["partial","*",1,0],"Perplexity-User":["partial","*",1,0],"Google-Extended":["partial","*",1,0],"Applebot-Extended":["partial","*",1,0],"CCBot":["partial","*",1,0],"Bytespider":["partial","*",1,0],"Amazonbot":["partial","*",1,0],"meta-externalagent":["partial","*",1,0],"cohere-ai":["partial","*",1,0],"Diffbot":["partial","*",1,0],"ContextFetchBot":["partial","*",1,0]},"llms":{"status":"missing","http":404,"bytes":17756},"home":{"status":"ok","http":200,"url":"https://www.ispconfig.org/","html_tokens":5871,"markdown_tokens":30,"pruned_tokens":30}},
{"rank":942,"domain":"max.ru","origin":"https://max.ru","robots":{"status":"ok","http":200,"url":"https://max.ru/robots.txt","bytes":40,"groups":1},"bots":{"GPTBot":["partial","*",1,1],"ChatGPT-User":["partial","*",1,1],"OAI-SearchBot":["partial","*",1,1],"ClaudeBot":["partial","*",1,1],"Claude-User":["partial","*",1,1],"Claude-SearchBot":["partial","*",1,1],"anthropic-ai":["partial","*",1,1],"PerplexityBot":["partial","*",1,1],"Perplexity-User":["partial","*",1,1],"Google-Extended":["partial","*",1,1],"Applebot-Extended":["partial","*",1,1],"CCBot":["partial","*",1,1],"Bytespider":["partial","*",1,1],"Amazonbot":["partial","*",1,1],"meta-externalagent":["partial","*",1,1],"cohere-ai":["partial","*",1,1],"Diffbot":["partial","*",1,1],"ContextFetchBot":["partial","*",1,1]},"llms":{"status":"missing","http":404,"bytes":1398},"home":{"status":"ok","http":200,"url":"https://max.ru/","html_tokens":10310,"markdown_tokens":218,"pruned_tokens":41}},
{"rank":943,"domain":"consumerreports.org","origin":"https://consumerreports.org","robots":{"status":"ok","http":200,"url":"https://www.consumerreports.org/robots.txt","bytes":827,"groups":11},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["partial","*",13,0],"OAI-SearchBot":["partial","*",13,0],"ClaudeBot":["partial","*",13,0],"Claude-User":["partial","*",13,0],"Claude-SearchBot":["partial","*",13,0],"anthropic-ai":["partial","*",13,0],"PerplexityBot":["partial","*",13,0],"Perplexity-User":["partial","*",13,0],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["partial","*",13,0],"CCBot":["blocked","named",1,0],"Bytespider":["partial","*",13,0],"Amazonbot":["partial","*",13,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["partial","*",13,0],"Diffbot":["partial","*",13,0],"ContextFetchBot":["partial","*",13,0]},"llms":{"status":"missing","http":404,"bytes":59675},"home":{"status":"ok","http":200,"url":"https://www.consumerreports.org/","html_tokens":120014,"markdown_tokens":80,"pruned_tokens":80}},
{"rank":944,"domain":"perl.com","origin":"https://perl.com","robots":{"status":"ok","http":200,"url":"https://www.perl.com/robots.txt","bytes":0,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":17948},"home":{"status":"ok","http":200,"url":"https://www.perl.com/","html_tokens":10552,"markdown_tokens":492,"pruned_tokens":293}},
{"rank":945,"domain":"narod.ru","origin":"https://narod.ru","robots":{"status":"ok","http":200,"url":"https://narod.ru/robots.txt","bytes":1003,"groups":2},"bots":{"GPTBot":["partial","*",27,7],"ChatGPT-User":["partial","*",27,7],"OAI-SearchBot":["partial","*",27,7],"ClaudeBot":["partial","*",27,7],"Claude-User":["partial","*",27,7],"Claude-SearchBot":["partial","*",27,7],"anthropic-ai":["partial","*",27,7],"PerplexityBot":["partial","*",27,7],"Perplexity-User":["partial","*",27,7],"Google-Extended":["partial","*",27,7],"Applebot-Extended":["partial","*",27,7],"CCBot":["partial","*",27,7],"Bytespider":["partial","*",27,7],"Amazonbot":["partial","*",27,7],"meta-externalagent":["partial","*",27,7],"cohere-ai":["partial","*",27,7],"Diffbot":["partial","*",27,7],"ContextFetchBot":["partial","*",27,7]},"llms":{"status":"missing","http":404,"bytes":13159},"home":{"status":"ok","http":200,"url":"https://narod.ru/","html_tokens":9797,"markdown_tokens":543,"pruned_tokens":131}},
{"rank":946,"domain":"digitaltrends.com","origin":"https://digitaltrends.com","robots":{"status":"ok","http":200,"url":"https://www.digitaltrends.com/robots.txt","bytes":3746,"groups":7},"bots":{"GPTBot":["partial","*",27,1],"ChatGPT-User":["partial","*",27,1],"OAI-SearchBot":["partial","named",31,0],"ClaudeBot":["partial","*",27,1],"Claude-User":["partial","*",27,1],"Claude-SearchBot":["partial","*",27,1],"anthropic-ai":["partial","*",27,1],"PerplexityBot":["partial","named",31,0],"Perplexity-User":["partial","named",31,0],"Google-Extended":["partial","*",27,1],"Applebot-Extended":["partial","*",27,1],"CCBot":["partial","*",27,1],"Bytespider":["partial","*",27,1],"Amazonbot":["partial","named",7,0],"meta-externalagent":["partial","named",7,0],"cohere-ai":["partial","*",27,1],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",27,1]},"llms":{"status":"missing","http":404,"bytes":144292},"home":{"status":"ok","http":202,"url":"https://digitaltrends.com/","html_tokens":570,"markdown_tokens":40,"pruned_tokens":40}},
{"rank":947,"domain":"com.com","origin":"https://com.com","robots":{"status":"html","http":200,"url":"https://gen.xyz/","bytes":75217,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":948,"domain":"themegrill.com","origin":"https://themegrill.com","robots":{"status":"ok","http":200,"url":"https://themegrill.com/robots.txt","bytes":172,"groups":1},"bots":{"GPTBot":["partial","*",1,1],"ChatGPT-User":["partial","*",1,1],"OAI-SearchBot":["partial","*",1,1],"ClaudeBot":["partial","*",1,1],"Claude-User":["partial","*",1,1],"Claude-SearchBot":["partial","*",1,1],"anthropic-ai":["partial","*",1,1],"PerplexityBot":["partial","*",1,1],"Perplexity-User":["partial","*",1,1],"Google-Extended":["partial","*",1,1],"Applebot-Extended":["partial","*",1,1],"CCBot":["partial","*",1,1],"Bytespider":["partial","*",1,1],"Amazonbot":["partial","*",1,1],"meta-externalagent":["partial","*",1,1],"cohere-ai":["partial","*",1,1],"Diffbot":["partial","*",1,1],"ContextFetchBot":["partial","*",1,1]},"llms":{"status":"other_text","http":200,"bytes":7320},"home":{"status":"ok","http":200,"url":"https://themegrill.com/","html_tokens":91676,"markdown_tokens":789,"pruned_tokens":132}},
{"rank":949,"domain":"ourworldindata.org","origin":"https://ourworldindata.org","robots":{"status":"ok","http":200,"url":"https://ourworldindata.org/robots.txt","bytes":47,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":22788},"home":{"status":"ok","http":200,"url":"https://ourworldindata.org/","html_tokens":66414,"markdown_tokens":193,"pruned_tokens":158}},
{"rank":950,"domain":"vogue.com","origin":"https://vogue.com","robots":{"status":"ok","http":200,"url":"https://www.vogue.com/robots.txt","bytes":2457,"groups":3},"bots":{"GPTBot":["partial","*",20,17],"ChatGPT-User":["partial","*",20,17],"OAI-SearchBot":["partial","*",20,17],"ClaudeBot":["blocked","named",1,0],"Claude-User":["blocked","named",1,0],"Claude-SearchBot":["blocked","named",1,0],"anthropic-ai":["partial","*",20,17],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["blocked","named",1,0],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",20,17]},"llms":{"status":"missing","http":404,"bytes":791919},"home":{"status":"ok","http":200,"url":"https://www.vogue.com/","html_tokens":532467,"markdown_tokens":29,"pruned_tokens":29}},
{"rank":951,"domain":"ko-fi.com","origin":"https://ko-fi.com","robots":{"status":"ok","http":200,"url":"https://ko-fi.com/robots.txt","bytes":219,"groups":1},"bots":{"GPTBot":["partial","*",7,0],"ChatGPT-User":["partial","*",7,0],"OAI-SearchBot":["partial","*",7,0],"ClaudeBot":["partial","*",7,0],"Claude-User":["partial","*",7,0],"Claude-SearchBot":["partial","*",7,0],"anthropic-ai":["partial","*",7,0],"PerplexityBot":["partial","*",7,0],"Perplexity-User":["partial","*",7,0],"Google-Extended":["partial","*",7,0],"Applebot-Extended":["partial","*",7,0],"CCBot":["partial","*",7,0],"Bytespider":["partial","*",7,0],"Amazonbot":["partial","*",7,0],"meta-externalagent":["partial","*",7,0],"cohere-ai":["partial","*",7,0],"Diffbot":["partial","*",7,0],"ContextFetchBot":["partial","*",7,0]},"llms":{"status":"http_error","http":403,"bytes":5474},"home":{"status":"http_error","http":403,"url":"https://ko-fi.com/"}},
{"rank":952,"domain":"amnesty.org","origin":"https://amnesty.org","robots":{"status":"ok","http":200,"url":"https://www.amnesty.org/robots.txt","bytes":1871,"groups":8},"bots":{"GPTBot":["partial","*",34,17],"ChatGPT-User":["partial","*",34,17],"OAI-SearchBot":["partial","*",34,17],"ClaudeBot":["partial","*",34,17],"Claude-User":["partial","*",34,17],"Claude-SearchBot":["partial","*",34,17],"anthropic-ai":["partial","*",34,17],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["partial","*",34,17],"Google-Extended":["partial","*",34,17],"Applebot-Extended":["partial","*",34,17],"CCBot":["blocked","named",1,0],"Bytespider":["partial","*",34,17],"Amazonbot":["partial","*",34,17],"meta-externalagent":["partial","*",34,17],"cohere-ai":["partial","*",34,17],"Diffbot":["partial","*",34,17],"ContextFetchBot":["partial","*",34,17]},"llms":{"status":"http_error","http":403,"bytes":5542},"home":{"status":"http_error","http":403,"url":"https://amnesty.org/"}},
{"rank":953,"domain":"uk.net","origin":"https://uk.net","robots":{"status":"ok","http":200,"url":"https://uk.net/robots.txt","bytes":0,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":8019},"home":{"status":"ok","http":200,"url":"https://uk.net/","html_tokens":10755,"markdown_tokens":788,"pruned_tokens":772}},
{"rank":954,"domain":"hopkinsmedicine.org","origin":"https://hopkinsmedicine.org","robots":{"status":"refused","http":403,"url":"https://hopkinsmedicine.org/robots.txt","bytes":4552,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":955,"domain":"aka.ms","origin":"https://aka.ms","robots":{"status":"ok","http":200,"url":"https://aka.ms/robots.txt","bytes":98,"groups":1},"bots":{"GPTBot":["partial","*",1,1],"ChatGPT-User":["partial","*",1,1],"OAI-SearchBot":["partial","*",1,1],"ClaudeBot":["partial","*",1,1],"Claude-User":["partial","*",1,1],"Claude-SearchBot":["partial","*",1,1],"anthropic-ai":["partial","*",1,1],"PerplexityBot":["partial","*",1,1],"Perplexity-User":["partial","*",1,1],"Google-Extended":["partial","*",1,1],"Applebot-Extended":["partial","*",1,1],"CCBot":["partial","*",1,1],"Bytespider":["partial","*",1,1],"Amazonbot":["partial","*",1,1],"meta-externalagent":["partial","*",1,1],"cohere-ai":["partial","*",1,1],"Diffbot":["partial","*",1,1],"ContextFetchBot":["partial","*",1,1]},"llms":{"status":"disallowed"},"home":{"status":"disallowed"}},
{"rank":956,"domain":"metmuseum.org","origin":"https://metmuseum.org","robots":{"status":"refused","http":429,"url":"https://metmuseum.org/robots.txt","bytes":34217,"groups":0,"error":"HTTP 429"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":957,"domain":"tollbit-host.com","origin":"https://tollbit-host.com","robots":{"status":"ok","http":200,"url":"https://tollbit-host.com/robots.txt","bytes":26,"groups":1},"bots":{"GPTBot":["blocked","*",1,0],"ChatGPT-User":["blocked","*",1,0],"OAI-SearchBot":["blocked","*",1,0],"ClaudeBot":["blocked","*",1,0],"Claude-User":["blocked","*",1,0],"Claude-SearchBot":["blocked","*",1,0],"anthropic-ai":["blocked","*",1,0],"PerplexityBot":["blocked","*",1,0],"Perplexity-User":["blocked","*",1,0],"Google-Extended":["blocked","*",1,0],"Applebot-Extended":["blocked","*",1,0],"CCBot":["blocked","*",1,0],"Bytespider":["blocked","*",1,0],"Amazonbot":["blocked","*",1,0],"meta-externalagent":["blocked","*",1,0],"cohere-ai":["blocked","*",1,0],"Diffbot":["blocked","*",1,0],"ContextFetchBot":["blocked","*",1,0]},"llms":{"status":"disallowed"},"home":{"status":"disallowed"}},
{"rank":958,"domain":"indiana.edu","origin":"https://indiana.edu","robots":{"status":"html","http":200,"url":"https://bloomington.iu.edu","bytes":318019,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":959,"domain":"georgetown.edu","origin":"https://georgetown.edu","robots":{"status":"refused","http":403,"url":"https://georgetown.edu/robots.txt","bytes":560,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":960,"domain":"tradingview.com","origin":"https://tradingview.com","robots":{"status":"ok","http":200,"url":"https://www.tradingview.com/robots.txt","bytes":1153,"groups":2},"bots":{"GPTBot":["partial","*",15,0],"ChatGPT-User":["partial","named",9,0],"OAI-SearchBot":["partial","named",9,0],"ClaudeBot":["partial","named",9,0],"Claude-User":["partial","*",15,0],"Claude-SearchBot":["partial","*",15,0],"anthropic-ai":["partial","*",15,0],"PerplexityBot":["partial","named",9,0],"Perplexity-User":["partial","*",15,0],"Google-Extended":["partial","named",9,0],"Applebot-Extended":["partial","named",9,0],"CCBot":["partial","*",15,0],"Bytespider":["partial","named",9,0],"Amazonbot":["partial","named",9,0],"meta-externalagent":["partial","named",9,0],"cohere-ai":["partial","*",15,0],"Diffbot":["partial","*",15,0],"ContextFetchBot":["partial","*",15,0]},"llms":{"status":"missing","http":404,"bytes":158148},"home":{"status":"ok","http":200,"url":"https://www.tradingview.com/","html_tokens":315780,"markdown_tokens":139,"pruned_tokens":139}},
{"rank":961,"domain":"iheart.com","origin":"https://iheart.com","robots":{"status":"ok","http":200,"url":"https://www.iheart.com/robots.txt","bytes":589,"groups":5},"bots":{"GPTBot":["partial","*",13,0],"ChatGPT-User":["partial","*",13,0],"OAI-SearchBot":["partial","*",13,0],"ClaudeBot":["partial","*",13,0],"Claude-User":["partial","*",13,0],"Claude-SearchBot":["partial","*",13,0],"anthropic-ai":["partial","*",13,0],"PerplexityBot":["partial","*",13,0],"Perplexity-User":["partial","*",13,0],"Google-Extended":["partial","*",13,0],"Applebot-Extended":["partial","*",13,0],"CCBot":["partial","*",13,0],"Bytespider":["partial","*",13,0],"Amazonbot":["partial","*",13,0],"meta-externalagent":["partial","*",13,0],"cohere-ai":["partial","*",13,0],"Diffbot":["partial","*",13,0],"ContextFetchBot":["partial","*",13,0]},"llms":{"status":"missing","http":404,"bytes":254141},"home":{"status":"ok","http":200,"url":"https://www.iheart.com/","html_tokens":110926,"markdown_tokens":20,"pruned_tokens":20}},
{"rank":962,"domain":"yumpu.com","origin":"https://yumpu.com","robots":{"status":"ok","http":200,"url":"https://yumpu.com/robots.txt","bytes":9093,"groups":20},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["partial","*",62,0],"OAI-SearchBot":["partial","*",62,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",62,0],"Claude-SearchBot":["partial","*",62,0],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["partial","*",62,0],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",62,0]},"llms":{"status":"html","http":200,"bytes":76561},"home":{"status":"ok","http":200,"url":"https://www.yumpu.com/en","html_tokens":107583,"markdown_tokens":917,"pruned_tokens":168}},
{"rank":963,"domain":"upwork.com","origin":"https://upwork.com","robots":{"status":"ok","http":200,"url":"https://www.upwork.com/robots.txt","bytes":9725,"groups":13},"bots":{"GPTBot":["partial","*",259,5],"ChatGPT-User":["allowed","named",0,1],"OAI-SearchBot":["allowed","named",0,1],"ClaudeBot":["partial","*",259,5],"Claude-User":["allowed","named",0,1],"Claude-SearchBot":["allowed","named",0,1],"anthropic-ai":["partial","*",259,5],"PerplexityBot":["allowed","named",0,1],"Perplexity-User":["allowed","named",0,1],"Google-Extended":["allowed","named",0,1],"Applebot-Extended":["allowed","named",0,1],"CCBot":["partial","*",259,5],"Bytespider":["partial","*",259,5],"Amazonbot":["partial","*",259,5],"meta-externalagent":["partial","*",259,5],"cohere-ai":["partial","*",259,5],"Diffbot":["partial","*",259,5],"ContextFetchBot":["partial","*",259,5]},"llms":{"status":"present","http":200,"bytes":5032},"home":{"status":"http_error","http":403,"url":"https://upwork.com/"}},
{"rank":964,"domain":"searchengineland.com","origin":"https://searchengineland.com","robots":{"status":"refused","http":403,"url":"https://searchengineland.com/robots.txt","bytes":5558,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":965,"domain":"medscape.com","origin":"https://medscape.com","robots":{"status":"ok","http":200,"url":"https://www.medscape.com/robots.txt","bytes":902,"groups":9},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["partial","*",20,0],"OAI-SearchBot":["partial","*",20,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",20,0],"Claude-SearchBot":["partial","*",20,0],"anthropic-ai":["partial","*",20,0],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["partial","*",20,0],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["partial","*",20,0],"CCBot":["blocked","named",1,0],"Bytespider":["partial","*",20,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["partial","*",20,0],"cohere-ai":["partial","*",20,0],"Diffbot":["partial","*",20,0],"ContextFetchBot":["partial","*",20,0]},"llms":{"status":"missing","http":404,"bytes":170559},"home":{"status":"ok","http":200,"url":"https://www.medscape.com/","html_tokens":69453,"markdown_tokens":167,"pruned_tokens":127}},
{"rank":966,"domain":"today.com","origin":"https://today.com","robots":{"status":"ok","http":200,"url":"https://www.today.com/robots.txt","bytes":12636,"groups":38},"bots":{"GPTBot":["partial","named",1,8],"ChatGPT-User":["partial","named",1,8],"OAI-SearchBot":["partial","named",1,8],"ClaudeBot":["partial","named",1,8],"Claude-User":["partial","named",1,8],"Claude-SearchBot":["partial","named",1,8],"anthropic-ai":["partial","named",1,8],"PerplexityBot":["partial","named",1,8],"Perplexity-User":["partial","named",1,8],"Google-Extended":["partial","named",1,8],"Applebot-Extended":["partial","named",1,8],"CCBot":["partial","named",1,8],"Bytespider":["partial","named",1,8],"Amazonbot":["partial","named",1,8],"meta-externalagent":["partial","named",1,8],"cohere-ai":["partial","named",1,8],"Diffbot":["partial","named",1,8],"ContextFetchBot":["partial","*",6,0]},"llms":{"status":"missing","http":404,"bytes":197422},"home":{"status":"ok","http":200,"url":"https://www.today.com","html_tokens":262582,"markdown_tokens":398,"pruned_tokens":63}},
{"rank":967,"domain":"aepd.es","origin":"https://aepd.es","robots":{"status":"ok","http":200,"url":"https://www.aepd.es/robots.txt","bytes":2757,"groups":1},"bots":{"GPTBot":["partial","*",49,18],"ChatGPT-User":["partial","*",49,18],"OAI-SearchBot":["partial","*",49,18],"ClaudeBot":["partial","*",49,18],"Claude-User":["partial","*",49,18],"Claude-SearchBot":["partial","*",49,18],"anthropic-ai":["partial","*",49,18],"PerplexityBot":["partial","*",49,18],"Perplexity-User":["partial","*",49,18],"Google-Extended":["partial","*",49,18],"Applebot-Extended":["partial","*",49,18],"CCBot":["partial","*",49,18],"Bytespider":["partial","*",49,18],"Amazonbot":["partial","*",49,18],"meta-externalagent":["partial","*",49,18],"cohere-ai":["partial","*",49,18],"Diffbot":["partial","*",49,18],"ContextFetchBot":["partial","*",49,18]},"llms":{"status":"http_error","http":500,"bytes":1682},"home":{"status":"ok","http":200,"url":"https://www.aepd.es/","html_tokens":57768,"markdown_tokens":847,"pruned_tokens":288}},
{"rank":968,"domain":"nokia.com","origin":"https://nokia.com","robots":{"status":"refused","http":403,"url":"https://www.nokia.com/robots.txt","bytes":381,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":969,"domain":"att.com","origin":"https://att.com","robots":{"status":"ok","http":200,"url":"https://www.att.com/robots.txt","bytes":11245,"groups":3},"bots":{"GPTBot":["partial","*",123,24],"ChatGPT-User":["partial","*",123,24],"OAI-SearchBot":["partial","*",123,24],"ClaudeBot":["partial","*",123,24],"Claude-User":["partial","*",123,24],"Claude-SearchBot":["partial","*",123,24],"anthropic-ai":["partial","*",123,24],"PerplexityBot":["partial","*",123,24],"Perplexity-User":["partial","*",123,24],"Google-Extended":["partial","*",123,24],"Applebot-Extended":["partial","*",123,24],"CCBot":["partial","*",123,24],"Bytespider":["partial","*",123,24],"Amazonbot":["partial","*",123,24],"meta-externalagent":["partial","*",123,24],"cohere-ai":["partial","*",123,24],"Diffbot":["partial","*",123,24],"ContextFetchBot":["partial","*",123,24]},"llms":{"status":"present","http":200,"bytes":13861},"home":{"status":"ok","http":200,"url":"https://www.att.com/","html_tokens":228273,"markdown_tokens":63,"pruned_tokens":63}},
{"rank":970,"domain":"v.youku.com","origin":"https://v.youku.com","robots":{"status":"html","http":200,"url":"https://www.youku.com/ku/webhome","bytes":1030112,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":971,"domain":"metro.co.uk","origin":"https://metro.co.uk","robots":{"status":"ok","http":200,"url":"https://metro.co.uk/robots.txt","bytes":4976,"groups":4},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["blocked","named",1,0],"OAI-SearchBot":["blocked","named",1,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["blocked","named",1,0],"Claude-SearchBot":["blocked","named",1,0],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["blocked","named",1,0],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",12,1]},"llms":{"status":"missing","http":404,"bytes":121286},"home":{"status":"ok","http":200,"url":"https://metro.co.uk/","html_tokens":149829,"markdown_tokens":119,"pruned_tokens":69}},
{"rank":972,"domain":"justia.com","origin":"https://justia.com","robots":{"status":"ok","http":200,"url":"https://www.justia.com/robots.txt","bytes":68,"groups":1},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"missing","http":404,"bytes":45503},"home":{"status":"http_error","http":403,"url":"https://www.justia.com/"}},
{"rank":973,"domain":"bell-labs.com","origin":"https://bell-labs.com","robots":{"status":"missing","http":410,"url":"https://www.bell-labs.com/robots.txt","bytes":0,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":410,"bytes":0},"home":{"status":"disallowed","error":"robots.txt of www.nokia.com not readable (refused)"}},
{"rank":974,"domain":"legislation.gov.uk","origin":"https://legislation.gov.uk","robots":{"status":"ok","http":200,"url":"https://www.legislation.gov.uk/robots.txt","bytes":1735,"groups":1},"bots":{"GPTBot":["partial","*",3,0],"ChatGPT-User":["partial","*",3,0],"OAI-SearchBot":["partial","*",3,0],"ClaudeBot":["partial","*",3,0],"Claude-User":["partial","*",3,0],"Claude-SearchBot":["partial","*",3,0],"anthropic-ai":["partial","*",3,0],"PerplexityBot":["partial","*",3,0],"Perplexity-User":["partial","*",3,0],"Google-Extended":["partial","*",3,0],"Applebot-Extended":["partial","*",3,0],"CCBot":["partial","*",3,0],"Bytespider":["partial","*",3,0],"Amazonbot":["partial","*",3,0],"meta-externalagent":["partial","*",3,0],"cohere-ai":["partial","*",3,0],"Diffbot":["partial","*",3,0],"ContextFetchBot":["partial","*",3,0]},"llms":{"status":"missing","http":404,"bytes":12293},"home":{"status":"ok","http":200,"url":"https://www.legislation.gov.uk/","html_tokens":4608,"markdown_tokens":234,"pruned_tokens":111}},
{"rank":975,"domain":"flaticon.com","origin":"https://flaticon.com","robots":{"status":"ok","http":200,"url":"https://www.flaticon.com/robots.txt","bytes":1725,"groups":3},"bots":{"GPTBot":["partial","*",47,1],"ChatGPT-User":["partial","*",47,1],"OAI-SearchBot":["partial","*",47,1],"ClaudeBot":["partial","*",47,1],"Claude-User":["partial","*",47,1],"Claude-SearchBot":["partial","*",47,1],"anthropic-ai":["partial","*",47,1],"PerplexityBot":["partial","*",47,1],"Perplexity-User":["partial","*",47,1],"Google-Extended":["partial","*",47,1],"Applebot-Extended":["partial","*",47,1],"CCBot":["partial","*",47,1],"Bytespider":["partial","*",47,1],"Amazonbot":["partial","*",47,1],"meta-externalagent":["partial","*",47,1],"cohere-ai":["partial","*",47,1],"Diffbot":["partial","*",47,1],"ContextFetchBot":["partial","*",47,1]},"llms":{"status":"http_error","http":403,"bytes":7952},"home":{"status":"http_error","http":403,"url":"https://www.flaticon.com/"}},
{"rank":976,"domain":"ning.com","origin":"https://ning.com","robots":{"status":"ok","http":200,"url":"https://www.ning.com/robots.txt","bytes":618,"groups":1},"bots":{"GPTBot":["partial","*",16,4],"ChatGPT-User":["partial","*",16,4],"OAI-SearchBot":["partial","*",16,4],"ClaudeBot":["partial","*",16,4],"Claude-User":["partial","*",16,4],"Claude-SearchBot":["partial","*",16,4],"anthropic-ai":["partial","*",16,4],"PerplexityBot":["partial","*",16,4],"Perplexity-User":["partial","*",16,4],"Google-Extended":["partial","*",16,4],"Applebot-Extended":["partial","*",16,4],"CCBot":["partial","*",16,4],"Bytespider":["partial","*",16,4],"Amazonbot":["partial","*",16,4],"meta-externalagent":["partial","*",16,4],"cohere-ai":["partial","*",16,4],"Diffbot":["partial","*",16,4],"ContextFetchBot":["partial","*",16,4]},"llms":{"status":"missing","http":404,"bytes":78610},"home":{"status":"ok","http":200,"url":"https://www.ning.com/","html_tokens":32038,"markdown_tokens":853,"pruned_tokens":375}},
{"rank":977,"domain":"google.ru","origin":"https://google.ru","robots":{"status":"ok","http":200,"url":"https://www.google.ru/robots.txt","bytes":6624,"groups":4},"bots":{"GPTBot":["partial","*",169,64],"ChatGPT-User":["partial","*",169,64],"OAI-SearchBot":["partial","*",169,64],"ClaudeBot":["partial","*",169,64],"Claude-User":["partial","*",169,64],"Claude-SearchBot":["partial","*",169,64],"anthropic-ai":["partial","*",169,64],"PerplexityBot":["partial","*",169,64],"Perplexity-User":["partial","*",169,64],"Google-Extended":["partial","*",169,64],"Applebot-Extended":["partial","*",169,64],"CCBot":["partial","*",169,64],"Bytespider":["partial","*",169,64],"Amazonbot":["partial","*",169,64],"meta-externalagent":["partial","*",169,64],"cohere-ai":["partial","*",169,64],"Diffbot":["partial","*",169,64],"ContextFetchBot":["partial","*",169,64]},"llms":{"status":"missing","http":404,"bytes":1569},"home":{"status":"ok","http":200,"url":"https://www.google.com/","html_tokens":21436,"markdown_tokens":30,"pruned_tokens":30}},
{"rank":978,"domain":"mcgill.ca","origin":"https://mcgill.ca","robots":{"status":"refused","http":429,"url":"https://mcgill.ca/robots.txt","bytes":6559,"groups":0,"error":"HTTP 429"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":979,"domain":"clinicaltrials.gov","origin":"https://clinicaltrials.gov","robots":{"status":"ok","http":200,"url":"https://clinicaltrials.gov/robots.txt","bytes":505,"groups":1},"bots":{"GPTBot":["partial","*",3,2],"ChatGPT-User":["partial","*",3,2],"OAI-SearchBot":["partial","*",3,2],"ClaudeBot":["partial","*",3,2],"Claude-User":["partial","*",3,2],"Claude-SearchBot":["partial","*",3,2],"anthropic-ai":["partial","*",3,2],"PerplexityBot":["partial","*",3,2],"Perplexity-User":["partial","*",3,2],"Google-Extended":["partial","*",3,2],"Applebot-Extended":["partial","*",3,2],"CCBot":["partial","*",3,2],"Bytespider":["partial","*",3,2],"Amazonbot":["partial","*",3,2],"meta-externalagent":["partial","*",3,2],"cohere-ai":["partial","*",3,2],"Diffbot":["partial","*",3,2],"ContextFetchBot":["partial","*",3,2]},"llms":{"status":"html","http":200,"bytes":94247},"home":{"status":"ok","http":200,"url":"https://clinicaltrials.gov/","html_tokens":23562,"markdown_tokens":28,"pruned_tokens":28}},
{"rank":980,"domain":"tomshardware.com","origin":"https://tomshardware.com","robots":{"status":"ok","http":200,"url":"https://www.tomshardware.com/robots.txt","bytes":1885,"groups":5},"bots":{"GPTBot":["partial","*",24,0],"ChatGPT-User":["partial","*",24,0],"OAI-SearchBot":["partial","*",24,0],"ClaudeBot":["partial","*",24,0],"Claude-User":["partial","*",24,0],"Claude-SearchBot":["partial","*",24,0],"anthropic-ai":["partial","*",24,0],"PerplexityBot":["partial","*",24,0],"Perplexity-User":["partial","*",24,0],"Google-Extended":["partial","*",24,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["partial","*",24,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["partial","*",24,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",24,0]},"llms":{"status":"missing","http":404,"bytes":1278445},"home":{"status":"ok","http":200,"url":"https://www.tomshardware.com/","html_tokens":476317,"markdown_tokens":83,"pruned_tokens":83}},
{"rank":981,"domain":"politico.eu","origin":"https://politico.eu","robots":{"status":"ok","http":200,"url":"https://www.politico.eu/robots.txt","bytes":3921,"groups":94},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["blocked","named",1,0],"OAI-SearchBot":["blocked","named",1,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["blocked","named",1,0],"Claude-SearchBot":["blocked","named",1,0],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["blocked","named",1,0],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",2,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",5,1]},"llms":{"status":"missing","http":404,"bytes":256142},"home":{"status":"ok","http":200,"url":"https://www.politico.eu/","html_tokens":115571,"markdown_tokens":329,"pruned_tokens":135}},
{"rank":982,"domain":"xkcd.com","origin":"https://xkcd.com","robots":{"status":"ok","http":200,"url":"https://xkcd.com/robots.txt","bytes":67,"groups":2},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["partial","*",1,0],"OAI-SearchBot":["partial","*",1,0],"ClaudeBot":["partial","*",1,0],"Claude-User":["partial","*",1,0],"Claude-SearchBot":["partial","*",1,0],"anthropic-ai":["partial","*",1,0],"PerplexityBot":["partial","*",1,0],"Perplexity-User":["partial","*",1,0],"Google-Extended":["partial","*",1,0],"Applebot-Extended":["partial","*",1,0],"CCBot":["partial","*",1,0],"Bytespider":["partial","*",1,0],"Amazonbot":["partial","*",1,0],"meta-externalagent":["partial","*",1,0],"cohere-ai":["partial","*",1,0],"Diffbot":["partial","*",1,0],"ContextFetchBot":["partial","*",1,0]},"llms":{"status":"missing","http":404,"bytes":146},"home":{"status":"ok","http":200,"url":"https://xkcd.com/","html_tokens":1805,"markdown_tokens":85,"pruned_tokens":85}},
{"rank":983,"domain":"pitt.edu","origin":"https://pitt.edu","robots":{"status":"ok","http":200,"url":"https://www.pitt.edu//robots.txt","bytes":2027,"groups":1},"bots":{"GPTBot":["partial","*",34,18],"ChatGPT-User":["partial","*",34,18],"OAI-SearchBot":["partial","*",34,18],"ClaudeBot":["partial","*",34,18],"Claude-User":["partial","*",34,18],"Claude-SearchBot":["partial","*",34,18],"anthropic-ai":["partial","*",34,18],"PerplexityBot":["partial","*",34,18],"Perplexity-User":["partial","*",34,18],"Google-Extended":["partial","*",34,18],"Applebot-Extended":["partial","*",34,18],"CCBot":["partial","*",34,18],"Bytespider":["partial","*",34,18],"Amazonbot":["partial","*",34,18],"meta-externalagent":["partial","*",34,18],"cohere-ai":["partial","*",34,18],"Diffbot":["partial","*",34,18],"ContextFetchBot":["partial","*",34,18]},"llms":{"status":"missing","http":404,"bytes":186},"home":{"status":"ok","http":200,"url":"https://www.pitt.edu/","html_tokens":21131,"markdown_tokens":456,"pruned_tokens":73}},
{"rank":984,"domain":"xiaohongshu.com","origin":"https://xiaohongshu.com","robots":{"status":"ok","http":200,"url":"https://www.xiaohongshu.com/robots.txt","bytes":492,"groups":3},"bots":{"GPTBot":["blocked","*",1,0],"ChatGPT-User":["blocked","*",1,0],"OAI-SearchBot":["blocked","*",1,0],"ClaudeBot":["blocked","*",1,0],"Claude-User":["blocked","*",1,0],"Claude-SearchBot":["blocked","*",1,0],"anthropic-ai":["blocked","*",1,0],"PerplexityBot":["blocked","*",1,0],"Perplexity-User":["blocked","*",1,0],"Google-Extended":["blocked","*",1,0],"Applebot-Extended":["blocked","*",1,0],"CCBot":["blocked","*",1,0],"Bytespider":["blocked","*",1,0],"Amazonbot":["blocked","*",1,0],"meta-externalagent":["blocked","*",1,0],"cohere-ai":["blocked","*",1,0],"Diffbot":["blocked","*",1,0],"ContextFetchBot":["blocked","*",1,0]},"llms":{"status":"disallowed"},"home":{"status":"disallowed"}},
{"rank":985,"domain":"zeit.de","origin":"https://zeit.de","robots":{"status":"ok","http":200,"url":"https://www.zeit.de/robots.txt","bytes":2148,"groups":32},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["partial","*",12,1],"OAI-SearchBot":["partial","*",12,1],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",12,1],"Claude-SearchBot":["partial","*",12,1],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["blocked","named",1,0],"Google-Extended":["partial","named",2,2],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["partial","*",12,1],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["partial","*",12,1],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",12,1]},"llms":{"status":"http_error","http":403,"bytes":5439},"home":{"status":"http_error","http":403,"url":"https://www.zeit.de/index"}},
{"rank":986,"domain":"boston.com","origin":"https://boston.com","robots":{"status":"ok","http":200,"url":"https://www.boston.com/robots.txt","bytes":1346,"groups":26},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["partial","*",6,1],"OAI-SearchBot":["partial","*",6,1],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",6,1],"Claude-SearchBot":["partial","*",6,1],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["partial","*",6,1],"Perplexity-User":["partial","*",6,1],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["partial","*",6,1],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",6,1]},"llms":{"status":"missing","http":404,"bytes":180713},"home":{"status":"ok","http":200,"url":"https://www.boston.com/","html_tokens":92716,"markdown_tokens":32,"pruned_tokens":32}},
{"rank":987,"domain":"hbs.edu","origin":"https://hbs.edu","robots":{"status":"ok","http":200,"url":"https://www.hbs.edu/robots.txt","bytes":2855,"groups":2},"bots":{"GPTBot":["partial","*",51,0],"ChatGPT-User":["partial","*",51,0],"OAI-SearchBot":["partial","*",51,0],"ClaudeBot":["partial","*",51,0],"Claude-User":["partial","*",51,0],"Claude-SearchBot":["partial","*",51,0],"anthropic-ai":["partial","*",51,0],"PerplexityBot":["partial","*",51,0],"Perplexity-User":["partial","*",51,0],"Google-Extended":["partial","*",51,0],"Applebot-Extended":["partial","*",51,0],"CCBot":["partial","*",51,0],"Bytespider":["partial","*",51,0],"Amazonbot":["partial","*",51,0],"meta-externalagent":["partial","*",51,0],"cohere-ai":["partial","*",51,0],"Diffbot":["partial","*",51,0],"ContextFetchBot":["partial","*",51,0]},"llms":{"status":"missing","http":404,"bytes":44108},"home":{"status":"ok","http":200,"url":"https://www.hbs.edu/","html_tokens":100835,"markdown_tokens":348,"pruned_tokens":46}},
{"rank":988,"domain":"feedly.com","origin":"https://feedly.com","robots":{"status":"ok","http":200,"url":"https://feedly.com/robots.txt","bytes":636,"groups":1},"bots":{"GPTBot":["partial","*",9,15],"ChatGPT-User":["partial","*",9,15],"OAI-SearchBot":["partial","*",9,15],"ClaudeBot":["partial","*",9,15],"Claude-User":["partial","*",9,15],"Claude-SearchBot":["partial","*",9,15],"anthropic-ai":["partial","*",9,15],"PerplexityBot":["partial","*",9,15],"Perplexity-User":["partial","*",9,15],"Google-Extended":["partial","*",9,15],"Applebot-Extended":["partial","*",9,15],"CCBot":["partial","*",9,15],"Bytespider":["partial","*",9,15],"Amazonbot":["partial","*",9,15],"meta-externalagent":["partial","*",9,15],"cohere-ai":["partial","*",9,15],"Diffbot":["partial","*",9,15],"ContextFetchBot":["partial","*",9,15]},"llms":{"status":"missing","http":404,"bytes":79},"home":{"status":"ok","http":200,"url":"https://feedly.com/","html_tokens":22130,"markdown_tokens":525,"pruned_tokens":59}},
{"rank":989,"domain":"salon.com","origin":"https://salon.com","robots":{"status":"ok","http":200,"url":"https://www.salon.com/robots.txt","bytes":2596,"groups":66},"bots":{"GPTBot":["allowed","named",0,1],"ChatGPT-User":["allowed","named",0,1],"OAI-SearchBot":["allowed","named",0,1],"ClaudeBot":["allowed","named",0,1],"Claude-User":["allowed","named",0,1],"Claude-SearchBot":["allowed","named",0,1],"anthropic-ai":["allowed","named",0,1],"PerplexityBot":["allowed","named",0,1],"Perplexity-User":["allowed","named",0,1],"Google-Extended":["allowed","named",0,1],"Applebot-Extended":["allowed","named",0,1],"CCBot":["allowed","named",0,1],"Bytespider":["allowed","named",0,0],"Amazonbot":["allowed","named",0,1],"meta-externalagent":["allowed","named",0,1],"cohere-ai":["partial","*",1,0],"Diffbot":["partial","*",1,0],"ContextFetchBot":["partial","*",1,0]},"llms":{"status":"missing","http":404,"bytes":32362},"home":{"status":"ok","http":200,"url":"https://www.salon.com/","html_tokens":32605,"markdown_tokens":71,"pruned_tokens":71}},
{"rank":990,"domain":"chron.com","origin":"https://chron.com","robots":{"status":"ok","http":200,"url":"https://www.chron.com/robots.txt","bytes":1084,"groups":3},"bots":{"GPTBot":["partial","*",11,0],"ChatGPT-User":["partial","*",11,0],"OAI-SearchBot":["partial","*",11,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["blocked","named",1,0],"Claude-SearchBot":["partial","*",11,0],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["blocked","named",1,0],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["partial","*",11,0],"meta-externalagent":["partial","*",11,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",11,0]},"llms":{"status":"missing","http":404,"bytes":46817},"home":{"status":"ok","http":200,"url":"https://www.chron.com/","html_tokens":224864,"markdown_tokens":17,"pruned_tokens":17}},
{"rank":991,"domain":"cia.gov","origin":"https://cia.gov","robots":{"status":"ok","http":200,"url":"https://www.cia.gov/robots.txt","bytes":1594,"groups":33},"bots":{"GPTBot":["partial","*",8,0],"ChatGPT-User":["partial","*",8,0],"OAI-SearchBot":["partial","*",8,0],"ClaudeBot":["partial","*",8,0],"Claude-User":["partial","*",8,0],"Claude-SearchBot":["partial","*",8,0],"anthropic-ai":["partial","*",8,0],"PerplexityBot":["partial","*",8,0],"Perplexity-User":["partial","*",8,0],"Google-Extended":["partial","*",8,0],"Applebot-Extended":["partial","*",8,0],"CCBot":["partial","*",8,0],"Bytespider":["partial","*",8,0],"Amazonbot":["partial","*",8,0],"meta-externalagent":["partial","*",8,0],"cohere-ai":["partial","*",8,0],"Diffbot":["partial","*",8,0],"ContextFetchBot":["partial","*",8,0]},"llms":{"status":"missing","http":404,"bytes":509020},"home":{"status":"ok","http":200,"url":"https://www.cia.gov/","html_tokens":137262,"markdown_tokens":201,"pruned_tokens":81}},
{"rank":992,"domain":"codeberg.org","origin":"https://codeberg.org","robots":{"status":"ok","http":200,"url":"https://codeberg.org/robots.txt","bytes":2192,"groups":2},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["blocked","named",1,0],"OAI-SearchBot":["blocked","named",1,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",58,0],"Claude-SearchBot":["partial","*",58,0],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["partial","*",58,0],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",58,0]},"llms":{"status":"missing","http":404,"bytes":16659},"home":{"status":"ok","http":200,"url":"https://codeberg.org/","html_tokens":8615,"markdown_tokens":267,"pruned_tokens":58}},
{"rank":993,"domain":"nydailynews.com","origin":"https://nydailynews.com","robots":{"status":"ok","http":200,"url":"https://www.nydailynews.com/robots.txt","bytes":4484,"groups":68},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["blocked","named",1,0],"OAI-SearchBot":["blocked","named",1,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["blocked","named",1,0],"Claude-SearchBot":["blocked","named",1,0],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["blocked","named",1,0],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["partial","*",8,1],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",8,1]},"llms":{"status":"missing","http":404,"bytes":91319},"home":{"status":"ok","http":200,"url":"https://www.nydailynews.com/","html_tokens":59577,"markdown_tokens":301,"pruned_tokens":161}},
{"rank":994,"domain":"mariadb.org","origin":"https://mariadb.org","robots":{"status":"ok","http":200,"url":"https://mariadb.org/robots.txt","bytes":325,"groups":2},"bots":{"GPTBot":["partial","*",1,0],"ChatGPT-User":["partial","*",1,0],"OAI-SearchBot":["partial","*",1,0],"ClaudeBot":["partial","*",1,0],"Claude-User":["partial","*",1,0],"Claude-SearchBot":["partial","*",1,0],"anthropic-ai":["partial","*",1,0],"PerplexityBot":["partial","*",1,0],"Perplexity-User":["partial","*",1,0],"Google-Extended":["partial","*",1,0],"Applebot-Extended":["partial","*",1,0],"CCBot":["partial","*",1,0],"Bytespider":["partial","*",1,0],"Amazonbot":["partial","*",1,0],"meta-externalagent":["partial","*",1,0],"cohere-ai":["partial","*",1,0],"Diffbot":["partial","*",1,0],"ContextFetchBot":["partial","*",1,0]},"llms":{"status":"missing","http":404,"bytes":100621},"home":{"status":"ok","http":200,"url":"https://mariadb.org/","html_tokens":43233,"markdown_tokens":491,"pruned_tokens":442}},
{"rank":995,"domain":"brandbucket.com","origin":"https://brandbucket.com","robots":{"status":"ok","http":200,"url":"https://www.brandbucket.com/robots.txt","bytes":555,"groups":1},"bots":{"GPTBot":["partial","*",19,0],"ChatGPT-User":["partial","*",19,0],"OAI-SearchBot":["partial","*",19,0],"ClaudeBot":["partial","*",19,0],"Claude-User":["partial","*",19,0],"Claude-SearchBot":["partial","*",19,0],"anthropic-ai":["partial","*",19,0],"PerplexityBot":["partial","*",19,0],"Perplexity-User":["partial","*",19,0],"Google-Extended":["partial","*",19,0],"Applebot-Extended":["partial","*",19,0],"CCBot":["partial","*",19,0],"Bytespider":["partial","*",19,0],"Amazonbot":["partial","*",19,0],"meta-externalagent":["partial","*",19,0],"cohere-ai":["partial","*",19,0],"Diffbot":["partial","*",19,0],"ContextFetchBot":["partial","*",19,0]},"llms":{"status":"html","http":200,"bytes":4195},"home":{"status":"http_error","http":403,"url":"https://brandbucket.com/"}},
{"rank":996,"domain":"trendmicro.com","origin":"https://trendmicro.com","robots":{"status":"ok","http":200,"url":"https://www.trendmicro.com/robots.txt","bytes":2211,"groups":4},"bots":{"GPTBot":["partial","*",18,1],"ChatGPT-User":["partial","named",10,1],"OAI-SearchBot":["partial","named",10,1],"ClaudeBot":["partial","named",10,1],"Claude-User":["partial","named",10,1],"Claude-SearchBot":["partial","named",10,1],"anthropic-ai":["partial","*",18,1],"PerplexityBot":["partial","named",10,1],"Perplexity-User":["partial","named",10,1],"Google-Extended":["partial","named",10,1],"Applebot-Extended":["partial","*",18,1],"CCBot":["partial","named",10,1],"Bytespider":["partial","named",10,1],"Amazonbot":["partial","named",10,1],"meta-externalagent":["partial","named",10,1],"cohere-ai":["partial","named",10,1],"Diffbot":["partial","*",18,1],"ContextFetchBot":["partial","*",18,1]},"llms":{"status":"present","http":200,"bytes":22999},"home":{"status":"ok","http":200,"url":"https://www.trendmicro.com/en_us/business.html","html_tokens":35394,"markdown_tokens":557,"pruned_tokens":234}},
{"rank":997,"domain":"ndtv.com","origin":"https://ndtv.com","robots":{"status":"refused","http":403,"url":"https://www.ndtv.com/robots.txt","bytes":378,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":998,"domain":"fedex.com","origin":"https://fedex.com","robots":{"status":"ok","http":200,"url":"https://www.fedex.com/robots.txt","bytes":909,"groups":1},"bots":{"GPTBot":["partial","*",9,12],"ChatGPT-User":["partial","*",9,12],"OAI-SearchBot":["partial","*",9,12],"ClaudeBot":["partial","*",9,12],"Claude-User":["partial","*",9,12],"Claude-SearchBot":["partial","*",9,12],"anthropic-ai":["partial","*",9,12],"PerplexityBot":["partial","*",9,12],"Perplexity-User":["partial","*",9,12],"Google-Extended":["partial","*",9,12],"Applebot-Extended":["partial","*",9,12],"CCBot":["partial","*",9,12],"Bytespider":["partial","*",9,12],"Amazonbot":["partial","*",9,12],"meta-externalagent":["partial","*",9,12],"cohere-ai":["partial","*",9,12],"Diffbot":["partial","*",9,12],"ContextFetchBot":["partial","*",9,12]},"llms":{"status":"timeout","error":"timed out after 10 s"},"home":{"status":"timeout","error":"timed out after 15 s"}},
{"rank":999,"domain":"eurekalert.org","origin":"https://eurekalert.org","robots":{"status":"ok","http":200,"url":"https://www.eurekalert.org/robots.txt","bytes":2722,"groups":25},"bots":{"GPTBot":["partial","*",22,0],"ChatGPT-User":["partial","*",22,0],"OAI-SearchBot":["partial","*",22,0],"ClaudeBot":["partial","*",22,0],"Claude-User":["partial","*",22,0],"Claude-SearchBot":["partial","*",22,0],"anthropic-ai":["partial","*",22,0],"PerplexityBot":["partial","*",22,0],"Perplexity-User":["partial","*",22,0],"Google-Extended":["partial","*",22,0],"Applebot-Extended":["partial","*",22,0],"CCBot":["partial","*",22,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["partial","*",22,0],"meta-externalagent":["partial","*",22,0],"cohere-ai":["partial","*",22,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",22,0]},"llms":{"status":"html","http":200,"bytes":194},"home":{"status":"ok","http":200,"url":"https://www.eurekalert.org/","html_tokens":29269,"markdown_tokens":124,"pruned_tokens":124}},
{"rank":1000,"domain":"zillow.com","origin":"https://zillow.com","robots":{"status":"ok","http":200,"url":"https://www.zillow.com/robots.txt","bytes":10524,"groups":4},"bots":{"GPTBot":["partial","*",210,40],"ChatGPT-User":["partial","*",210,40],"OAI-SearchBot":["partial","*",210,40],"ClaudeBot":["partial","*",210,40],"Claude-User":["partial","*",210,40],"Claude-SearchBot":["partial","*",210,40],"anthropic-ai":["partial","*",210,40],"PerplexityBot":["partial","*",210,40],"Perplexity-User":["partial","*",210,40],"Google-Extended":["partial","*",210,40],"Applebot-Extended":["partial","named",1,22],"CCBot":["partial","*",210,40],"Bytespider":["partial","*",210,40],"Amazonbot":["partial","named",1,22],"meta-externalagent":["partial","named",1,22],"cohere-ai":["partial","*",210,40],"Diffbot":["partial","*",210,40],"ContextFetchBot":["partial","*",210,40]},"llms":{"status":"missing","http":404,"bytes":445748},"home":{"status":"ok","http":200,"url":"https://www.zillow.com/","html_tokens":117982,"markdown_tokens":1210,"pruned_tokens":172}},
{"rank":1001,"domain":"padlet.com","origin":"https://padlet.com","robots":{"status":"ok","http":200,"url":"https://padlet.com/robots.txt","bytes":175,"groups":1},"bots":{"GPTBot":["allowed","*",0,1],"ChatGPT-User":["allowed","*",0,1],"OAI-SearchBot":["allowed","*",0,1],"ClaudeBot":["allowed","*",0,1],"Claude-User":["allowed","*",0,1],"Claude-SearchBot":["allowed","*",0,1],"anthropic-ai":["allowed","*",0,1],"PerplexityBot":["allowed","*",0,1],"Perplexity-User":["allowed","*",0,1],"Google-Extended":["allowed","*",0,1],"Applebot-Extended":["allowed","*",0,1],"CCBot":["allowed","*",0,1],"Bytespider":["allowed","*",0,1],"Amazonbot":["allowed","*",0,1],"meta-externalagent":["allowed","*",0,1],"cohere-ai":["allowed","*",0,1],"Diffbot":["allowed","*",0,1],"ContextFetchBot":["allowed","*",0,1]},"llms":{"status":"present","http":200,"bytes":2618},"home":{"status":"ok","http":200,"url":"https://padlet.com/","html_tokens":39573,"markdown_tokens":18,"pruned_tokens":18}},
{"rank":1002,"domain":"federalreserve.gov","origin":"https://federalreserve.gov","robots":{"status":"missing","http":404,"url":"https://www.federalreserve.gov/robots.txt","bytes":83301,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":83301},"home":{"status":"ok","http":200,"url":"https://www.federalreserve.gov/","html_tokens":26513,"markdown_tokens":876,"pruned_tokens":106}},
{"rank":1003,"domain":"sueddeutsche.de","origin":"https://sueddeutsche.de","robots":{"status":"ok","http":200,"url":"https://www.sueddeutsche.de/robots.txt","bytes":2273,"groups":33},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["blocked","named",1,0],"OAI-SearchBot":["partial","*",5,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",5,0],"Claude-SearchBot":["partial","*",5,0],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["partial","*",5,0],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["partial","*",5,0],"CCBot":["partial","*",5,0],"Bytespider":["partial","*",5,0],"Amazonbot":["partial","named",1,0],"meta-externalagent":["partial","*",5,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["partial","*",5,0],"ContextFetchBot":["partial","*",5,0]},"llms":{"status":"html","http":200,"bytes":2128297},"home":{"status":"ok","http":200,"url":"https://www.sueddeutsche.de/","html_tokens":531529,"markdown_tokens":66,"pruned_tokens":66}},
{"rank":1004,"domain":"dzen.ru","origin":"https://dzen.ru","robots":{"status":"error","http":null,"url":"https://dzen.ru/robots.txt","bytes":0,"groups":0,"error":"unsupported content encoding aws-chunked"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":1005,"domain":"complianz.io","origin":"https://complianz.io","robots":{"status":"ok","http":200,"url":"https://complianz.io/robots.txt","bytes":171,"groups":1},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"other_text","http":200,"bytes":5512},"home":{"status":"ok","http":200,"url":"https://complianz.io/","html_tokens":33777,"markdown_tokens":67,"pruned_tokens":31}},
{"rank":1006,"domain":"routledge.com","origin":"https://routledge.com","robots":{"status":"ok","http":200,"url":"https://www.routledge.com/robots.txt","bytes":319,"groups":1},"bots":{"GPTBot":["partial","*",10,0],"ChatGPT-User":["partial","*",10,0],"OAI-SearchBot":["partial","*",10,0],"ClaudeBot":["partial","*",10,0],"Claude-User":["partial","*",10,0],"Claude-SearchBot":["partial","*",10,0],"anthropic-ai":["partial","*",10,0],"PerplexityBot":["partial","*",10,0],"Perplexity-User":["partial","*",10,0],"Google-Extended":["partial","*",10,0],"Applebot-Extended":["partial","*",10,0],"CCBot":["partial","*",10,0],"Bytespider":["partial","*",10,0],"Amazonbot":["partial","*",10,0],"meta-externalagent":["partial","*",10,0],"cohere-ai":["partial","*",10,0],"Diffbot":["partial","*",10,0],"ContextFetchBot":["partial","*",10,0]},"llms":{"status":"missing","http":404,"bytes":113083},"home":{"status":"ok","http":200,"url":"https://www.routledge.com/","html_tokens":32691,"markdown_tokens":267,"pruned_tokens":104}},
{"rank":1007,"domain":"rsc.org","origin":"https://rsc.org","robots":{"status":"ok","http":200,"url":"https://www.rsc.org/robots.txt","bytes":2562,"groups":6},"bots":{"GPTBot":["partial","*",22,0],"ChatGPT-User":["partial","*",22,0],"OAI-SearchBot":["partial","*",22,0],"ClaudeBot":["partial","*",22,0],"Claude-User":["partial","*",22,0],"Claude-SearchBot":["partial","*",22,0],"anthropic-ai":["partial","*",22,0],"PerplexityBot":["partial","*",22,0],"Perplexity-User":["partial","*",22,0],"Google-Extended":["partial","*",22,0],"Applebot-Extended":["partial","*",22,0],"CCBot":["partial","*",22,0],"Bytespider":["partial","*",22,0],"Amazonbot":["partial","*",22,0],"meta-externalagent":["partial","*",22,0],"cohere-ai":["partial","*",22,0],"Diffbot":["partial","*",22,0],"ContextFetchBot":["partial","*",22,0]},"llms":{"status":"missing","http":404,"bytes":155266},"home":{"status":"http_error","http":403,"url":"https://rsc.org/"}},
{"rank":1008,"domain":"foodnetwork.com","origin":"https://foodnetwork.com","robots":{"status":"ok","http":200,"url":"https://www.foodnetwork.com/robots.txt","bytes":788,"groups":3},"bots":{"GPTBot":["partial","*",11,5],"ChatGPT-User":["partial","*",11,5],"OAI-SearchBot":["partial","*",11,5],"ClaudeBot":["partial","*",11,5],"Claude-User":["partial","*",11,5],"Claude-SearchBot":["partial","*",11,5],"anthropic-ai":["partial","*",11,5],"PerplexityBot":["partial","*",11,5],"Perplexity-User":["partial","*",11,5],"Google-Extended":["partial","*",11,5],"Applebot-Extended":["partial","*",11,5],"CCBot":["partial","*",11,5],"Bytespider":["partial","*",11,5],"Amazonbot":["partial","*",11,5],"meta-externalagent":["partial","*",11,5],"cohere-ai":["partial","*",11,5],"Diffbot":["partial","*",11,5],"ContextFetchBot":["partial","*",11,5]},"llms":{"status":"missing","http":404,"bytes":319149},"home":{"status":"ok","http":200,"url":"https://www.foodnetwork.com/","html_tokens":83273,"markdown_tokens":142,"pruned_tokens":142}},
{"rank":1009,"domain":"istockphoto.com","origin":"https://istockphoto.com","robots":{"status":"refused","http":403,"url":"https://istockphoto.com/robots.txt","bytes":919,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":1010,"domain":"thedailybeast.com","origin":"https://thedailybeast.com","robots":{"status":"ok","http":200,"url":"https://www.thedailybeast.com/robots.txt","bytes":547,"groups":1},"bots":{"GPTBot":["partial","*",4,1],"ChatGPT-User":["partial","*",4,1],"OAI-SearchBot":["partial","*",4,1],"ClaudeBot":["partial","*",4,1],"Claude-User":["partial","*",4,1],"Claude-SearchBot":["partial","*",4,1],"anthropic-ai":["partial","*",4,1],"PerplexityBot":["partial","*",4,1],"Perplexity-User":["partial","*",4,1],"Google-Extended":["partial","*",4,1],"Applebot-Extended":["partial","*",4,1],"CCBot":["partial","*",4,1],"Bytespider":["partial","*",4,1],"Amazonbot":["partial","*",4,1],"meta-externalagent":["partial","*",4,1],"cohere-ai":["partial","*",4,1],"Diffbot":["partial","*",4,1],"ContextFetchBot":["partial","*",4,1]},"llms":{"status":"missing","http":404,"bytes":146368},"home":{"status":"ok","http":200,"url":"https://www.thedailybeast.com/","html_tokens":141407,"markdown_tokens":61,"pruned_tokens":61}},
{"rank":1011,"domain":"lonelyplanet.com","origin":"https://lonelyplanet.com","robots":{"status":"ok","http":200,"url":"https://www.lonelyplanet.com/robots.txt","bytes":2133,"groups":4},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["partial","*",45,0],"OAI-SearchBot":["partial","*",45,0],"ClaudeBot":["partial","*",45,0],"Claude-User":["partial","*",45,0],"Claude-SearchBot":["partial","*",45,0],"anthropic-ai":["partial","*",45,0],"PerplexityBot":["partial","*",45,0],"Perplexity-User":["partial","*",45,0],"Google-Extended":["partial","*",45,0],"Applebot-Extended":["partial","*",45,0],"CCBot":["partial","*",45,0],"Bytespider":["partial","*",45,0],"Amazonbot":["partial","*",45,0],"meta-externalagent":["partial","*",45,0],"cohere-ai":["partial","*",45,0],"Diffbot":["partial","*",45,0],"ContextFetchBot":["partial","*",45,0]},"llms":{"status":"missing","http":404,"bytes":1699610},"home":{"status":"ok","http":200,"url":"https://www.lonelyplanet.com/","html_tokens":461087,"markdown_tokens":434,"pruned_tokens":4}},
{"rank":1012,"domain":"mhlw.go.jp","origin":"https://www.mhlw.go.jp","robots":{"status":"ok","http":200,"url":"https://www.mhlw.go.jp/robots.txt","bytes":169,"groups":1},"bots":{"GPTBot":["partial","*",3,0],"ChatGPT-User":["partial","*",3,0],"OAI-SearchBot":["partial","*",3,0],"ClaudeBot":["partial","*",3,0],"Claude-User":["partial","*",3,0],"Claude-SearchBot":["partial","*",3,0],"anthropic-ai":["partial","*",3,0],"PerplexityBot":["partial","*",3,0],"Perplexity-User":["partial","*",3,0],"Google-Extended":["partial","*",3,0],"Applebot-Extended":["partial","*",3,0],"CCBot":["partial","*",3,0],"Bytespider":["partial","*",3,0],"Amazonbot":["partial","*",3,0],"meta-externalagent":["partial","*",3,0],"cohere-ai":["partial","*",3,0],"Diffbot":["partial","*",3,0],"ContextFetchBot":["partial","*",3,0]},"llms":{"status":"missing","http":404,"bytes":48956},"home":{"status":"ok","http":200,"url":"https://www.mhlw.go.jp/index.html","html_tokens":18812,"markdown_tokens":7,"pruned_tokens":7}},
{"rank":1013,"domain":"envato.market","origin":"https://envato.market","robots":{"status":"unreachable","http":null,"url":"https://envato.market/robots.txt","bytes":0,"groups":0,"error":"cannot resolve envato.market"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":1014,"domain":"lovable.app","origin":"https://lovable.app","robots":{"status":"refused","http":403,"url":"https://lovable.dev","bytes":5540,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":1015,"domain":"wustl.edu","origin":"https://wustl.edu","robots":{"status":"refused","http":403,"url":"https://wustl.edu/robots.txt","bytes":560,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":1016,"domain":"pastebin.com","origin":"https://pastebin.com","robots":{"status":"ok","http":200,"url":"https://pastebin.com/robots.txt","bytes":307,"groups":2},"bots":{"GPTBot":["partial","*",13,0],"ChatGPT-User":["partial","*",13,0],"OAI-SearchBot":["partial","*",13,0],"ClaudeBot":["partial","*",13,0],"Claude-User":["partial","*",13,0],"Claude-SearchBot":["partial","*",13,0],"anthropic-ai":["partial","*",13,0],"PerplexityBot":["partial","*",13,0],"Perplexity-User":["partial","*",13,0],"Google-Extended":["partial","*",13,0],"Applebot-Extended":["partial","*",13,0],"CCBot":["partial","*",13,0],"Bytespider":["partial","*",13,0],"Amazonbot":["partial","*",13,0],"meta-externalagent":["partial","*",13,0],"cohere-ai":["partial","*",13,0],"Diffbot":["partial","*",13,0],"ContextFetchBot":["partial","*",13,0]},"llms":{"status":"missing","http":404,"bytes":13114},"home":{"status":"ok","http":200,"url":"https://pastebin.com/","html_tokens":10758,"markdown_tokens":61,"pruned_tokens":61}},
{"rank":1017,"domain":"verizon.com","origin":"https://verizon.com","robots":{"status":"ok","http":200,"url":"https://verizon.com/robots.txt","bytes":83,"groups":1},"bots":{"GPTBot":["partial","*",4,0],"ChatGPT-User":["partial","*",4,0],"OAI-SearchBot":["partial","*",4,0],"ClaudeBot":["partial","*",4,0],"Claude-User":["partial","*",4,0],"Claude-SearchBot":["partial","*",4,0],"anthropic-ai":["partial","*",4,0],"PerplexityBot":["partial","*",4,0],"Perplexity-User":["partial","*",4,0],"Google-Extended":["partial","*",4,0],"Applebot-Extended":["partial","*",4,0],"CCBot":["partial","*",4,0],"Bytespider":["partial","*",4,0],"Amazonbot":["partial","*",4,0],"meta-externalagent":["partial","*",4,0],"cohere-ai":["partial","*",4,0],"Diffbot":["partial","*",4,0],"ContextFetchBot":["partial","*",4,0]},"llms":{"status":"html","http":200,"bytes":81484},"home":{"status":"ok","http":200,"url":"https://www.verizon.com","html_tokens":31292,"markdown_tokens":85,"pruned_tokens":49}},
{"rank":1018,"domain":"lse.ac.uk","origin":"https://lse.ac.uk","robots":{"status":"ok","http":200,"url":"https://www.lse.ac.uk/robots.txt","bytes":842,"groups":4},"bots":{"GPTBot":["partial","*",16,0],"ChatGPT-User":["partial","*",16,0],"OAI-SearchBot":["partial","*",16,0],"ClaudeBot":["partial","*",16,0],"Claude-User":["partial","*",16,0],"Claude-SearchBot":["partial","*",16,0],"anthropic-ai":["partial","*",16,0],"PerplexityBot":["partial","*",16,0],"Perplexity-User":["partial","*",16,0],"Google-Extended":["partial","*",16,0],"Applebot-Extended":["partial","*",16,0],"CCBot":["partial","*",16,0],"Bytespider":["partial","*",16,0],"Amazonbot":["partial","*",16,0],"meta-externalagent":["partial","*",16,0],"cohere-ai":["partial","*",16,0],"Diffbot":["partial","*",16,0],"ContextFetchBot":["partial","*",16,0]},"llms":{"status":"missing","http":404,"bytes":14},"home":{"status":"ok","http":200,"url":"https://www.lse.ac.uk/","html_tokens":69632,"markdown_tokens":426,"pruned_tokens":133}},
{"rank":1019,"domain":"pcworld.com","origin":"https://pcworld.com","robots":{"status":"ok","http":200,"url":"https://www.pcworld.com/robots.txt","bytes":2610,"groups":41},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["partial","*",15,0],"OAI-SearchBot":["partial","*",15,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",15,0],"Claude-SearchBot":["partial","*",15,0],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["partial","*",15,0],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["partial","*",15,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",15,0]},"llms":{"status":"missing","http":404,"bytes":213748},"home":{"status":"ok","http":200,"url":"https://www.pcworld.com","html_tokens":66121,"markdown_tokens":630,"pruned_tokens":163}},
{"rank":1020,"domain":"videolan.org","origin":"https://videolan.org","robots":{"status":"refused","http":418,"url":"https://www.videolan.org/robots.txt","bytes":14,"groups":0,"error":"HTTP 418"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":1021,"domain":"hindustantimes.com","origin":"https://hindustantimes.com","robots":{"status":"ok","http":200,"url":"https://www.hindustantimes.com/robots.txt","bytes":2449,"groups":4},"bots":{"GPTBot":["partial","*",33,1],"ChatGPT-User":["partial","*",33,1],"OAI-SearchBot":["partial","*",33,1],"ClaudeBot":["partial","*",33,1],"Claude-User":["partial","*",33,1],"Claude-SearchBot":["partial","*",33,1],"anthropic-ai":["partial","*",33,1],"PerplexityBot":["partial","*",33,1],"Perplexity-User":["partial","*",33,1],"Google-Extended":["partial","named",33,1],"Applebot-Extended":["partial","*",33,1],"CCBot":["partial","*",33,1],"Bytespider":["partial","*",33,1],"Amazonbot":["partial","*",33,1],"meta-externalagent":["partial","*",33,1],"cohere-ai":["partial","*",33,1],"Diffbot":["partial","*",33,1],"ContextFetchBot":["partial","*",33,1]},"llms":{"status":"present","http":200,"bytes":6650},"home":{"status":"ok","http":200,"url":"https://www.hindustantimes.com/","html_tokens":240589,"markdown_tokens":31,"pruned_tokens":31}},
{"rank":1022,"domain":"urbandictionary.com","origin":"https://urbandictionary.com","robots":{"status":"ok","http":200,"url":"https://www.urbandictionary.com/robots.txt","bytes":94,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":46700},"home":{"status":"ok","http":200,"url":"https://www.urbandictionary.com/","html_tokens":58285,"markdown_tokens":305,"pruned_tokens":48}},
{"rank":1023,"domain":"web.app","origin":"https://web.app","robots":{"status":"html","http":200,"url":"https://firebase.google.com/products/hosting/","bytes":510718,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":1024,"domain":"500px.com","origin":"https://500px.com","robots":{"status":"ok","http":200,"url":"https://500px.com/robots.txt","bytes":100,"groups":3},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"html","http":200,"bytes":3337},"home":{"status":"ok","http":200,"url":"https://500px.com/","html_tokens":835,"markdown_tokens":0,"pruned_tokens":0}},
{"rank":1025,"domain":"sba.gov","origin":"https://sba.gov","robots":{"status":"ok","http":200,"url":"https://www.sba.gov/robots.txt","bytes":148,"groups":1},"bots":{"GPTBot":["partial","*",3,1],"ChatGPT-User":["partial","*",3,1],"OAI-SearchBot":["partial","*",3,1],"ClaudeBot":["partial","*",3,1],"Claude-User":["partial","*",3,1],"Claude-SearchBot":["partial","*",3,1],"anthropic-ai":["partial","*",3,1],"PerplexityBot":["partial","*",3,1],"Perplexity-User":["partial","*",3,1],"Google-Extended":["partial","*",3,1],"Applebot-Extended":["partial","*",3,1],"CCBot":["partial","*",3,1],"Bytespider":["partial","*",3,1],"Amazonbot":["partial","*",3,1],"meta-externalagent":["partial","*",3,1],"cohere-ai":["partial","*",3,1],"Diffbot":["partial","*",3,1],"ContextFetchBot":["partial","*",3,1]},"llms":{"status":"missing","http":404,"bytes":133286},"home":{"status":"ok","http":200,"url":"https://www.sba.gov/","html_tokens":37466,"markdown_tokens":88,"pruned_tokens":70}},
{"rank":1026,"domain":"brown.edu","origin":"https://brown.edu","robots":{"status":"ok","http":200,"url":"https://www.brown.edu/robots.txt","bytes":2027,"groups":1},"bots":{"GPTBot":["partial","*",34,18],"ChatGPT-User":["partial","*",34,18],"OAI-SearchBot":["partial","*",34,18],"ClaudeBot":["partial","*",34,18],"Claude-User":["partial","*",34,18],"Claude-SearchBot":["partial","*",34,18],"anthropic-ai":["partial","*",34,18],"PerplexityBot":["partial","*",34,18],"Perplexity-User":["partial","*",34,18],"Google-Extended":["partial","*",34,18],"Applebot-Extended":["partial","*",34,18],"CCBot":["partial","*",34,18],"Bytespider":["partial","*",34,18],"Amazonbot":["partial","*",34,18],"meta-externalagent":["partial","*",34,18],"cohere-ai":["partial","*",34,18],"Diffbot":["partial","*",34,18],"ContextFetchBot":["partial","*",34,18]},"llms":{"status":"missing","http":404,"bytes":187},"home":{"status":"ok","http":200,"url":"https://www.brown.edu/","html_tokens":45968,"markdown_tokens":1958,"pruned_tokens":532}},
{"rank":1027,"domain":"zenodo.org","origin":"https://zenodo.org","robots":{"status":"ok","http":200,"url":"https://zenodo.org/robots.txt","bytes":272,"groups":1},"bots":{"GPTBot":["partial","*",10,1],"ChatGPT-User":["partial","*",10,1],"OAI-SearchBot":["partial","*",10,1],"ClaudeBot":["partial","*",10,1],"Claude-User":["partial","*",10,1],"Claude-SearchBot":["partial","*",10,1],"anthropic-ai":["partial","*",10,1],"PerplexityBot":["partial","*",10,1],"Perplexity-User":["partial","*",10,1],"Google-Extended":["partial","*",10,1],"Applebot-Extended":["partial","*",10,1],"CCBot":["partial","*",10,1],"Bytespider":["partial","*",10,1],"Amazonbot":["partial","*",10,1],"meta-externalagent":["partial","*",10,1],"cohere-ai":["partial","*",10,1],"Diffbot":["partial","*",10,1],"ContextFetchBot":["partial","*",10,1]},"llms":{"status":"missing","http":404,"bytes":14101},"home":{"status":"ok","http":200,"url":"https://zenodo.org/","html_tokens":17138,"markdown_tokens":86,"pruned_tokens":86}},
{"rank":1028,"domain":"dev.to","origin":"https://dev.to","robots":{"status":"ok","http":200,"url":"https://dev.to/robots.txt","bytes":450,"groups":1},"bots":{"GPTBot":["partial","*",16,0],"ChatGPT-User":["partial","*",16,0],"OAI-SearchBot":["partial","*",16,0],"ClaudeBot":["partial","*",16,0],"Claude-User":["partial","*",16,0],"Claude-SearchBot":["partial","*",16,0],"anthropic-ai":["partial","*",16,0],"PerplexityBot":["partial","*",16,0],"Perplexity-User":["partial","*",16,0],"Google-Extended":["partial","*",16,0],"Applebot-Extended":["partial","*",16,0],"CCBot":["partial","*",16,0],"Bytespider":["partial","*",16,0],"Amazonbot":["partial","*",16,0],"meta-externalagent":["partial","*",16,0],"cohere-ai":["partial","*",16,0],"Diffbot":["partial","*",16,0],"ContextFetchBot":["partial","*",16,0]},"llms":{"status":"present","http":200,"bytes":2835},"home":{"status":"ok","http":200,"url":"https://dev.to/","html_tokens":61389,"markdown_tokens":18,"pruned_tokens":18}},
{"rank":1029,"domain":"lg.com","origin":"https://lg.com","robots":{"status":"ok","http":200,"url":"https://www.lg.com/robots.txt","bytes":8054,"groups":4},"bots":{"GPTBot":["partial","*",159,1],"ChatGPT-User":["partial","*",159,1],"OAI-SearchBot":["partial","*",159,1],"ClaudeBot":["partial","*",159,1],"Claude-User":["partial","*",159,1],"Claude-SearchBot":["partial","*",159,1],"anthropic-ai":["partial","*",159,1],"PerplexityBot":["partial","*",159,1],"Perplexity-User":["partial","*",159,1],"Google-Extended":["partial","*",159,1],"Applebot-Extended":["partial","*",159,1],"CCBot":["partial","*",159,1],"Bytespider":["partial","*",159,1],"Amazonbot":["partial","*",159,1],"meta-externalagent":["partial","*",159,1],"cohere-ai":["partial","*",159,1],"Diffbot":["partial","*",159,1],"ContextFetchBot":["partial","*",159,1]},"llms":{"status":"present","http":200,"bytes":2253},"home":{"status":"ok","http":200,"url":"https://www.lg.com/us/","html_tokens":746564,"markdown_tokens":491,"pruned_tokens":80}},
{"rank":1030,"domain":"thefreedictionary.com","origin":"https://thefreedictionary.com","robots":{"status":"ok","http":200,"url":"https://www.thefreedictionary.com/robots.txt","bytes":1146,"groups":4},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["blocked","named",1,0],"OAI-SearchBot":["blocked","named",1,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["blocked","named",1,0],"Claude-SearchBot":["blocked","named",1,0],"anthropic-ai":["partial","*",8,0],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["partial","*",8,0],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["partial","*",8,0],"Diffbot":["partial","*",8,0],"ContextFetchBot":["partial","*",8,0]},"llms":{"status":"html","http":200,"bytes":29742},"home":{"status":"http_error","http":403,"url":"https://thefreedictionary.com/"}},
{"rank":1031,"domain":"usps.com","origin":"https://usps.com","robots":{"status":"ok","http":200,"url":"https://www.usps.com/robots.txt","bytes":435,"groups":1},"bots":{"GPTBot":["partial","*",13,0],"ChatGPT-User":["partial","*",13,0],"OAI-SearchBot":["partial","*",13,0],"ClaudeBot":["partial","*",13,0],"Claude-User":["partial","*",13,0],"Claude-SearchBot":["partial","*",13,0],"anthropic-ai":["partial","*",13,0],"PerplexityBot":["partial","*",13,0],"Perplexity-User":["partial","*",13,0],"Google-Extended":["partial","*",13,0],"Applebot-Extended":["partial","*",13,0],"CCBot":["partial","*",13,0],"Bytespider":["partial","*",13,0],"Amazonbot":["partial","*",13,0],"meta-externalagent":["partial","*",13,0],"cohere-ai":["partial","*",13,0],"Diffbot":["partial","*",13,0],"ContextFetchBot":["partial","*",13,0]},"llms":{"status":"missing","http":404,"bytes":55019},"home":{"status":"ok","http":200,"url":"https://www.usps.com/","html_tokens":22298,"markdown_tokens":105,"pruned_tokens":105}},
{"rank":1032,"domain":"wunderground.com","origin":"https://wunderground.com","robots":{"status":"ok","http":200,"url":"https://www.wunderground.com/robots.txt","bytes":395,"groups":2},"bots":{"GPTBot":["partial","*",2,0],"ChatGPT-User":["partial","*",2,0],"OAI-SearchBot":["partial","*",2,0],"ClaudeBot":["partial","*",2,0],"Claude-User":["partial","*",2,0],"Claude-SearchBot":["partial","*",2,0],"anthropic-ai":["partial","*",2,0],"PerplexityBot":["partial","*",2,0],"Perplexity-User":["partial","*",2,0],"Google-Extended":["partial","*",2,0],"Applebot-Extended":["partial","*",2,0],"CCBot":["partial","*",2,0],"Bytespider":["partial","*",2,0],"Amazonbot":["partial","*",2,0],"meta-externalagent":["partial","*",2,0],"cohere-ai":["partial","*",2,0],"Diffbot":["partial","*",2,0],"ContextFetchBot":["partial","*",2,0]},"llms":{"status":"missing","http":404,"bytes":22245},"home":{"status":"ok","http":200,"url":"https://www.wunderground.com/","html_tokens":9543,"markdown_tokens":72,"pruned_tokens":72}},
{"rank":1033,"domain":"tufts.edu","origin":"https://tufts.edu","robots":{"status":"refused","http":403,"url":"https://tufts.edu/robots.txt","bytes":424,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":1034,"domain":"gamespot.com","origin":"https://gamespot.com","robots":{"status":"refused","http":403,"url":"https://www.gamespot.com/robots.txt","bytes":5575,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":1035,"domain":"mega.nz","origin":"https://mega.nz","robots":{"status":"ok","http":200,"url":"https://mega.nz/robots.txt","bytes":1798,"groups":3},"bots":{"GPTBot":["partial","*",1,97],"ChatGPT-User":["partial","*",1,97],"OAI-SearchBot":["partial","*",1,97],"ClaudeBot":["partial","*",1,97],"Claude-User":["partial","*",1,97],"Claude-SearchBot":["partial","*",1,97],"anthropic-ai":["partial","*",1,97],"PerplexityBot":["partial","*",1,97],"Perplexity-User":["partial","*",1,97],"Google-Extended":["partial","*",1,97],"Applebot-Extended":["partial","*",1,97],"CCBot":["partial","*",1,97],"Bytespider":["partial","*",1,97],"Amazonbot":["partial","*",1,97],"meta-externalagent":["partial","*",1,97],"cohere-ai":["partial","*",1,97],"Diffbot":["partial","*",1,97],"ContextFetchBot":["partial","*",1,97]},"llms":{"status":"disallowed"},"home":{"status":"ok","http":200,"url":"https://mega.nz/","html_tokens":632,"markdown_tokens":0,"pruned_tokens":0}},
{"rank":1036,"domain":"houzz.com","origin":"https://houzz.com","robots":{"status":"ok","http":200,"url":"https://www.houzz.com/robots.txt","bytes":8703,"groups":4},"bots":{"GPTBot":["partial","named",144,13],"ChatGPT-User":["partial","named",144,13],"OAI-SearchBot":["partial","named",144,13],"ClaudeBot":["partial","named",144,13],"Claude-User":["partial","named",144,13],"Claude-SearchBot":["partial","*",144,12],"anthropic-ai":["partial","*",144,12],"PerplexityBot":["partial","named",144,13],"Perplexity-User":["partial","*",144,12],"Google-Extended":["partial","named",144,13],"Applebot-Extended":["partial","*",144,12],"CCBot":["partial","*",144,12],"Bytespider":["partial","*",144,12],"Amazonbot":["partial","*",144,12],"meta-externalagent":["partial","*",144,12],"cohere-ai":["partial","*",144,12],"Diffbot":["partial","*",144,12],"ContextFetchBot":["partial","*",144,12]},"llms":{"status":"missing","http":404,"bytes":902085},"home":{"status":"ok","http":200,"url":"https://www.houzz.com/","html_tokens":233695,"markdown_tokens":62,"pruned_tokens":62}},
{"rank":1037,"domain":"thenextweb.com","origin":"https://thenextweb.com","robots":{"status":"ok","http":200,"url":"https://thenextweb.com/robots.txt","bytes":301,"groups":2},"bots":{"GPTBot":["partial","*",8,0],"ChatGPT-User":["partial","*",8,0],"OAI-SearchBot":["partial","*",8,0],"ClaudeBot":["partial","*",8,0],"Claude-User":["partial","*",8,0],"Claude-SearchBot":["partial","*",8,0],"anthropic-ai":["partial","*",8,0],"PerplexityBot":["partial","*",8,0],"Perplexity-User":["partial","*",8,0],"Google-Extended":["partial","*",8,0],"Applebot-Extended":["partial","*",8,0],"CCBot":["partial","*",8,0],"Bytespider":["partial","*",8,0],"Amazonbot":["partial","*",8,0],"meta-externalagent":["partial","*",8,0],"cohere-ai":["partial","*",8,0],"Diffbot":["partial","*",8,0],"ContextFetchBot":["partial","*",8,0]},"llms":{"status":"missing","http":404,"bytes":112826},"home":{"status":"ok","http":200,"url":"https://thenextweb.com/","html_tokens":61901,"markdown_tokens":119,"pruned_tokens":119}},
{"rank":1038,"domain":"hawaii.edu","origin":"https://hawaii.edu","robots":{"status":"ok","http":200,"url":"https://hawaii.edu/robots.txt","bytes":1688,"groups":3},"bots":{"GPTBot":["partial","*",30,0],"ChatGPT-User":["partial","*",30,0],"OAI-SearchBot":["partial","*",30,0],"ClaudeBot":["partial","*",30,0],"Claude-User":["partial","*",30,0],"Claude-SearchBot":["partial","*",30,0],"anthropic-ai":["partial","*",30,0],"PerplexityBot":["partial","*",30,0],"Perplexity-User":["partial","*",30,0],"Google-Extended":["partial","*",30,0],"Applebot-Extended":["partial","*",30,0],"CCBot":["partial","*",30,0],"Bytespider":["partial","*",30,0],"Amazonbot":["partial","*",30,0],"meta-externalagent":["partial","*",30,0],"cohere-ai":["partial","*",30,0],"Diffbot":["partial","*",30,0],"ContextFetchBot":["partial","*",30,0]},"llms":{"status":"html","http":200,"bytes":128679},"home":{"status":"ok","http":200,"url":"https://hawaii.edu/","html_tokens":32164,"markdown_tokens":38,"pruned_tokens":38}},
{"rank":1039,"domain":"ebay.co.uk","origin":"https://ebay.co.uk","robots":{"status":"ok","http":200,"url":"https://www.ebay.co.uk/robots.txt","bytes":20258,"groups":8},"bots":{"GPTBot":["partial","named",1,3],"ChatGPT-User":["partial","named",192,5],"OAI-SearchBot":["partial","named",192,5],"ClaudeBot":["partial","named",1,3],"Claude-User":["partial","named",192,5],"Claude-SearchBot":["partial","named",192,5],"anthropic-ai":["partial","named",1,3],"PerplexityBot":["partial","named",1,3],"Perplexity-User":["partial","*",197,8],"Google-Extended":["partial","*",197,8],"Applebot-Extended":["partial","named",1,3],"CCBot":["partial","named",1,3],"Bytespider":["partial","named",1,3],"Amazonbot":["partial","named",1,3],"meta-externalagent":["partial","named",1,3],"cohere-ai":["partial","*",197,8],"Diffbot":["partial","*",197,8],"ContextFetchBot":["partial","*",197,8]},"llms":{"status":"missing","http":404,"bytes":77100},"home":{"status":"http_error","http":403,"url":"https://www.ebay.co.uk/"}},
{"rank":1040,"domain":"spglobal.com","origin":"https://spglobal.com","robots":{"status":"refused","http":403,"url":"https://www.spglobal.com/robots.txt","bytes":384,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":1041,"domain":"squareup.com","origin":"https://squareup.com","robots":{"status":"ok","http":200,"url":"https://squareup.com/robots.txt","bytes":2199,"groups":1},"bots":{"GPTBot":["partial","*",39,8],"ChatGPT-User":["partial","*",39,8],"OAI-SearchBot":["partial","*",39,8],"ClaudeBot":["partial","*",39,8],"Claude-User":["partial","*",39,8],"Claude-SearchBot":["partial","*",39,8],"anthropic-ai":["partial","*",39,8],"PerplexityBot":["partial","*",39,8],"Perplexity-User":["partial","*",39,8],"Google-Extended":["partial","*",39,8],"Applebot-Extended":["partial","*",39,8],"CCBot":["partial","*",39,8],"Bytespider":["partial","*",39,8],"Amazonbot":["partial","*",39,8],"meta-externalagent":["partial","*",39,8],"cohere-ai":["partial","*",39,8],"Diffbot":["partial","*",39,8],"ContextFetchBot":["partial","*",39,8]},"llms":{"status":"present","http":200,"bytes":22082},"home":{"status":"ok","http":200,"url":"https://squareup.com/us/en","html_tokens":446299,"markdown_tokens":529,"pruned_tokens":129}},
{"rank":1042,"domain":"jimdosite.com","origin":"https://jimdosite.com","robots":{"status":"unreachable","http":null,"url":"https://jimdosite.com/robots.txt","bytes":0,"groups":0,"error":"cannot resolve jimdosite.com"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":1043,"domain":"sony.com","origin":"https://sony.com","robots":{"status":"refused","http":403,"url":"https://www.sony.com/robots.txt","bytes":184,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":1044,"domain":"space.com","origin":"https://space.com","robots":{"status":"ok","http":200,"url":"https://www.space.com/robots.txt","bytes":1730,"groups":5},"bots":{"GPTBot":["partial","*",22,0],"ChatGPT-User":["partial","*",22,0],"OAI-SearchBot":["partial","*",22,0],"ClaudeBot":["partial","*",22,0],"Claude-User":["partial","*",22,0],"Claude-SearchBot":["partial","*",22,0],"anthropic-ai":["partial","*",22,0],"PerplexityBot":["partial","*",22,0],"Perplexity-User":["partial","*",22,0],"Google-Extended":["partial","*",22,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["partial","*",22,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["partial","*",22,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",22,0]},"llms":{"status":"missing","http":404,"bytes":1195028},"home":{"status":"ok","http":200,"url":"https://www.space.com/","html_tokens":425975,"markdown_tokens":55,"pruned_tokens":30}},
{"rank":1045,"domain":"prestashop.com","origin":"https://prestashop.com","robots":{"status":"ok","http":200,"url":"https://prestashop.com/robots.txt","bytes":1760,"groups":1},"bots":{"GPTBot":["partial","*",51,0],"ChatGPT-User":["partial","*",51,0],"OAI-SearchBot":["partial","*",51,0],"ClaudeBot":["partial","*",51,0],"Claude-User":["partial","*",51,0],"Claude-SearchBot":["partial","*",51,0],"anthropic-ai":["partial","*",51,0],"PerplexityBot":["partial","*",51,0],"Perplexity-User":["partial","*",51,0],"Google-Extended":["partial","*",51,0],"Applebot-Extended":["partial","*",51,0],"CCBot":["partial","*",51,0],"Bytespider":["partial","*",51,0],"Amazonbot":["partial","*",51,0],"meta-externalagent":["partial","*",51,0],"cohere-ai":["partial","*",51,0],"Diffbot":["partial","*",51,0],"ContextFetchBot":["partial","*",51,0]},"llms":{"status":"missing","http":404,"bytes":120447},"home":{"status":"ok","http":200,"url":"https://prestashop.com/","html_tokens":72568,"markdown_tokens":80,"pruned_tokens":80}},
{"rank":1046,"domain":"handle.net","origin":"https://handle.net","robots":{"status":"ok","http":200,"url":"https://handle.net/robots.txt","bytes":56,"groups":1},"bots":{"GPTBot":["partial","*",2,0],"ChatGPT-User":["partial","*",2,0],"OAI-SearchBot":["partial","*",2,0],"ClaudeBot":["partial","*",2,0],"Claude-User":["partial","*",2,0],"Claude-SearchBot":["partial","*",2,0],"anthropic-ai":["partial","*",2,0],"PerplexityBot":["partial","*",2,0],"Perplexity-User":["partial","*",2,0],"Google-Extended":["partial","*",2,0],"Applebot-Extended":["partial","*",2,0],"CCBot":["partial","*",2,0],"Bytespider":["partial","*",2,0],"Amazonbot":["partial","*",2,0],"meta-externalagent":["partial","*",2,0],"cohere-ai":["partial","*",2,0],"Diffbot":["partial","*",2,0],"ContextFetchBot":["partial","*",2,0]},"llms":{"status":"missing","http":404,"bytes":680},"home":{"status":"ok","http":200,"url":"https://handle.net/","html_tokens":1605,"markdown_tokens":546,"pruned_tokens":540}},
{"rank":1047,"domain":"deepseek.com","origin":"https://deepseek.com","robots":{"status":"ok","http":200,"url":"https://deepseek.com/robots.txt","bytes":70,"groups":1},"bots":{"GPTBot":["allowed","*",0,1],"ChatGPT-User":["allowed","*",0,1],"OAI-SearchBot":["allowed","*",0,1],"ClaudeBot":["allowed","*",0,1],"Claude-User":["allowed","*",0,1],"Claude-SearchBot":["allowed","*",0,1],"anthropic-ai":["allowed","*",0,1],"PerplexityBot":["allowed","*",0,1],"Perplexity-User":["allowed","*",0,1],"Google-Extended":["allowed","*",0,1],"Applebot-Extended":["allowed","*",0,1],"CCBot":["allowed","*",0,1],"Bytespider":["allowed","*",0,1],"Amazonbot":["allowed","*",0,1],"meta-externalagent":["allowed","*",0,1],"cohere-ai":["allowed","*",0,1],"Diffbot":["allowed","*",0,1],"ContextFetchBot":["allowed","*",0,1]},"llms":{"status":"missing","http":404,"bytes":115760},"home":{"status":"ok","http":200,"url":"https://deepseek.com/","html_tokens":26493,"markdown_tokens":62,"pruned_tokens":62}},
{"rank":1048,"domain":"ups.com","origin":"https://ups.com","robots":{"status":"html","http":200,"url":"https://www.ups.com/robots.txt","bytes":188,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":1049,"domain":"360.cn","origin":"https://360.cn","robots":{"status":"ok","http":200,"url":"https://360.cn/robots.txt","bytes":502,"groups":2},"bots":{"GPTBot":["partial","*",15,1],"ChatGPT-User":["partial","*",15,1],"OAI-SearchBot":["partial","*",15,1],"ClaudeBot":["partial","*",15,1],"Claude-User":["partial","*",15,1],"Claude-SearchBot":["partial","*",15,1],"anthropic-ai":["partial","*",15,1],"PerplexityBot":["partial","*",15,1],"Perplexity-User":["partial","*",15,1],"Google-Extended":["partial","*",15,1],"Applebot-Extended":["partial","*",15,1],"CCBot":["partial","*",15,1],"Bytespider":["partial","*",15,1],"Amazonbot":["partial","*",15,1],"meta-externalagent":["partial","*",15,1],"cohere-ai":["partial","*",15,1],"Diffbot":["partial","*",15,1],"ContextFetchBot":["partial","*",15,1]},"llms":{"status":"missing","http":404,"bytes":6780},"home":{"status":"ok","http":200,"url":"https://www.360.cn","html_tokens":21076,"markdown_tokens":10,"pruned_tokens":10}},
{"rank":1050,"domain":"is.gd","origin":"https://is.gd","robots":{"status":"ok","http":200,"url":"https://is.gd/robots.txt","bytes":0,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"disallowed","error":"robots.txt of www.sundaymail.co.uk not readable (html)"},"home":{"status":"http_error","http":403,"url":"https://is.gd/"}},
{"rank":1051,"domain":"discogs.com","origin":"https://discogs.com","robots":{"status":"refused","http":403,"url":"https://www.discogs.com/robots.txt","bytes":5596,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":1052,"domain":"twimg.com","origin":"https://twimg.com","robots":{"status":"unreachable","http":null,"url":"https://twimg.com/robots.txt","bytes":0,"groups":0,"error":"cannot resolve twimg.com"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":1053,"domain":"deadline.com","origin":"https://deadline.com","robots":{"status":"ok","http":200,"url":"https://deadline.com/robots.txt","bytes":1285,"groups":26},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["blocked","named",1,0],"OAI-SearchBot":["blocked","named",1,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",11,1],"Claude-SearchBot":["partial","*",11,1],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["partial","*",11,1],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["partial","*",11,1],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",11,1]},"llms":{"status":"missing","http":404,"bytes":678010},"home":{"status":"ok","http":200,"url":"https://deadline.com/","html_tokens":212197,"markdown_tokens":1090,"pruned_tokens":101}},
{"rank":1054,"domain":"zlib.net","origin":"https://zlib.net","robots":{"status":"ok","http":200,"url":"https://zlib.net/robots.txt","bytes":31,"groups":1},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"missing","http":404,"bytes":355},"home":{"status":"ok","http":200,"url":"https://zlib.net/","html_tokens":8002,"markdown_tokens":3009,"pruned_tokens":375}},
{"rank":1055,"domain":"bitly.com","origin":"https://bitly.com","robots":{"status":"ok","http":200,"url":"https://bitly.com/robots.txt","bytes":1175,"groups":2},"bots":{"GPTBot":["allowed","named",0,0],"ChatGPT-User":["allowed","named",0,0],"OAI-SearchBot":["partial","*",28,0],"ClaudeBot":["allowed","named",0,0],"Claude-User":["allowed","named",0,0],"Claude-SearchBot":["allowed","named",0,0],"anthropic-ai":["partial","*",28,0],"PerplexityBot":["partial","*",28,0],"Perplexity-User":["partial","*",28,0],"Google-Extended":["partial","*",28,0],"Applebot-Extended":["partial","*",28,0],"CCBot":["allowed","named",0,0],"Bytespider":["partial","*",28,0],"Amazonbot":["partial","*",28,0],"meta-externalagent":["partial","*",28,0],"cohere-ai":["partial","*",28,0],"Diffbot":["partial","*",28,0],"ContextFetchBot":["partial","*",28,0]},"llms":{"status":"missing","http":404,"bytes":6075},"home":{"status":"ok","http":200,"url":"https://bitly.com/","html_tokens":77218,"markdown_tokens":282,"pruned_tokens":105}},
{"rank":1056,"domain":"cwi.nl","origin":"https://cwi.nl","robots":{"status":"ok","http":200,"url":"https://www.cwi.nl/robots.txt","bytes":44,"groups":1},"bots":{"GPTBot":["partial","*",1,1],"ChatGPT-User":["partial","*",1,1],"OAI-SearchBot":["partial","*",1,1],"ClaudeBot":["partial","*",1,1],"Claude-User":["partial","*",1,1],"Claude-SearchBot":["partial","*",1,1],"anthropic-ai":["partial","*",1,1],"PerplexityBot":["partial","*",1,1],"Perplexity-User":["partial","*",1,1],"Google-Extended":["partial","*",1,1],"Applebot-Extended":["partial","*",1,1],"CCBot":["partial","*",1,1],"Bytespider":["partial","*",1,1],"Amazonbot":["partial","*",1,1],"meta-externalagent":["partial","*",1,1],"cohere-ai":["partial","*",1,1],"Diffbot":["partial","*",1,1],"ContextFetchBot":["partial","*",1,1]},"llms":{"status":"missing","http":404,"bytes":48351},"home":{"status":"ok","http":200,"url":"https://www.cwi.nl/en/","html_tokens":20317,"markdown_tokens":417,"pruned_tokens":67}},
{"rank":1057,"domain":"ssa.gov","origin":"https://www.ssa.gov","robots":{"status":"refused","http":403,"url":"https://www.ssa.gov/robots.txt","bytes":379,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":1058,"domain":"gatech.edu","origin":"https://gatech.edu","robots":{"status":"ok","http":200,"url":"https://www.gatech.edu/robots.txt","bytes":2205,"groups":1},"bots":{"GPTBot":["partial","*",35,22],"ChatGPT-User":["partial","*",35,22],"OAI-SearchBot":["partial","*",35,22],"ClaudeBot":["partial","*",35,22],"Claude-User":["partial","*",35,22],"Claude-SearchBot":["partial","*",35,22],"anthropic-ai":["partial","*",35,22],"PerplexityBot":["partial","*",35,22],"Perplexity-User":["partial","*",35,22],"Google-Extended":["partial","*",35,22],"Applebot-Extended":["partial","*",35,22],"CCBot":["partial","*",35,22],"Bytespider":["partial","*",35,22],"Amazonbot":["partial","*",35,22],"meta-externalagent":["partial","*",35,22],"cohere-ai":["partial","*",35,22],"Diffbot":["partial","*",35,22],"ContextFetchBot":["partial","*",35,22]},"llms":{"status":"missing","http":404,"bytes":188},"home":{"status":"ok","http":200,"url":"https://www.gatech.edu/","html_tokens":28722,"markdown_tokens":121,"pruned_tokens":121}},
{"rank":1059,"domain":"apachelounge.com","origin":"https://apachelounge.com","robots":{"status":"ok","http":200,"url":"https://www.apachelounge.com/robots.txt","bytes":260,"groups":1},"bots":{"GPTBot":["partial","*",10,0],"ChatGPT-User":["partial","*",10,0],"OAI-SearchBot":["partial","*",10,0],"ClaudeBot":["partial","*",10,0],"Claude-User":["partial","*",10,0],"Claude-SearchBot":["partial","*",10,0],"anthropic-ai":["partial","*",10,0],"PerplexityBot":["partial","*",10,0],"Perplexity-User":["partial","*",10,0],"Google-Extended":["partial","*",10,0],"Applebot-Extended":["partial","*",10,0],"CCBot":["partial","*",10,0],"Bytespider":["partial","*",10,0],"Amazonbot":["partial","*",10,0],"meta-externalagent":["partial","*",10,0],"cohere-ai":["partial","*",10,0],"Diffbot":["partial","*",10,0],"ContextFetchBot":["partial","*",10,0]},"llms":{"status":"html","http":200,"bytes":2451},"home":{"status":"ok","http":200,"url":"https://www.apachelounge.com/","html_tokens":5712,"markdown_tokens":847,"pruned_tokens":126}},
{"rank":1060,"domain":"ntp.org","origin":"https://ntp.org","robots":{"status":"ok","http":200,"url":"https://www.ntp.org/robots.txt","bytes":16,"groups":1},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"missing","http":404,"bytes":6601},"home":{"status":"ok","http":200,"url":"https://www.ntp.org/","html_tokens":2279,"markdown_tokens":180,"pruned_tokens":180}},
{"rank":1061,"domain":"billboard.com","origin":"https://billboard.com","robots":{"status":"ok","http":200,"url":"https://www.billboard.com/robots.txt","bytes":2171,"groups":26},"bots":{"GPTBot":["partial","named",1,1],"ChatGPT-User":["partial","named",1,1],"OAI-SearchBot":["partial","named",1,1],"ClaudeBot":["partial","named",1,1],"Claude-User":["partial","*",9,1],"Claude-SearchBot":["partial","*",9,1],"anthropic-ai":["partial","named",1,1],"PerplexityBot":["partial","named",1,1],"Perplexity-User":["partial","*",9,1],"Google-Extended":["partial","named",1,1],"Applebot-Extended":["partial","named",1,1],"CCBot":["partial","named",1,1],"Bytespider":["partial","named",1,1],"Amazonbot":["partial","*",9,1],"meta-externalagent":["partial","named",1,1],"cohere-ai":["partial","named",1,1],"Diffbot":["partial","named",1,1],"ContextFetchBot":["partial","*",9,1]},"llms":{"status":"missing","http":404,"bytes":732519},"home":{"status":"ok","http":200,"url":"https://www.billboard.com/","html_tokens":363511,"markdown_tokens":83,"pruned_tokens":83}},
{"rank":1062,"domain":"voanews.com","origin":"https://voanews.com","robots":{"status":"ok","http":200,"url":"https://www.voanews.com/robots.txt","bytes":444,"groups":3},"bots":{"GPTBot":["partial","*",13,0],"ChatGPT-User":["partial","*",13,0],"OAI-SearchBot":["partial","*",13,0],"ClaudeBot":["partial","*",13,0],"Claude-User":["partial","*",13,0],"Claude-SearchBot":["partial","*",13,0],"anthropic-ai":["partial","*",13,0],"PerplexityBot":["partial","*",13,0],"Perplexity-User":["partial","*",13,0],"Google-Extended":["partial","*",13,0],"Applebot-Extended":["partial","*",13,0],"CCBot":["partial","*",13,0],"Bytespider":["partial","*",13,0],"Amazonbot":["partial","*",13,0],"meta-externalagent":["partial","*",13,0],"cohere-ai":["partial","*",13,0],"Diffbot":["partial","*",13,0],"ContextFetchBot":["partial","*",13,0]},"llms":{"status":"missing","http":404,"bytes":25812},"home":{"status":"ok","http":200,"url":"https://www.voanews.com/","html_tokens":11681,"markdown_tokens":71,"pruned_tokens":71}},
{"rank":1063,"domain":"vt.edu","origin":"https://vt.edu","robots":{"status":"ok","http":200,"url":"https://www.vt.edu/content/dam/vt_edu/robots.txt","bytes":3087,"groups":5},"bots":{"GPTBot":["partial","*",21,0],"ChatGPT-User":["partial","*",21,0],"OAI-SearchBot":["partial","*",21,0],"ClaudeBot":["partial","*",21,0],"Claude-User":["partial","*",21,0],"Claude-SearchBot":["partial","*",21,0],"anthropic-ai":["partial","*",21,0],"PerplexityBot":["partial","*",21,0],"Perplexity-User":["partial","*",21,0],"Google-Extended":["partial","*",21,0],"Applebot-Extended":["partial","*",21,0],"CCBot":["partial","*",21,0],"Bytespider":["partial","*",21,0],"Amazonbot":["partial","*",21,0],"meta-externalagent":["allowed","named",0,1],"cohere-ai":["partial","*",21,0],"Diffbot":["partial","*",21,0],"ContextFetchBot":["partial","*",21,0]},"llms":{"status":"missing","http":404,"bytes":74561},"home":{"status":"ok","http":200,"url":"https://www.vt.edu/","html_tokens":22638,"markdown_tokens":82,"pruned_tokens":82}},
{"rank":1064,"domain":"squid-cache.org","origin":"https://www.squid-cache.org","robots":{"status":"ok","http":200,"url":"https://www.squid-cache.org/robots.txt","bytes":120,"groups":1},"bots":{"GPTBot":["partial","*",5,0],"ChatGPT-User":["partial","*",5,0],"OAI-SearchBot":["partial","*",5,0],"ClaudeBot":["partial","*",5,0],"Claude-User":["partial","*",5,0],"Claude-SearchBot":["partial","*",5,0],"anthropic-ai":["partial","*",5,0],"PerplexityBot":["partial","*",5,0],"Perplexity-User":["partial","*",5,0],"Google-Extended":["partial","*",5,0],"Applebot-Extended":["partial","*",5,0],"CCBot":["partial","*",5,0],"Bytespider":["partial","*",5,0],"Amazonbot":["partial","*",5,0],"meta-externalagent":["partial","*",5,0],"cohere-ai":["partial","*",5,0],"Diffbot":["partial","*",5,0],"ContextFetchBot":["partial","*",5,0]},"llms":{"status":"missing","http":404,"bytes":281},"home":{"status":"ok","http":200,"url":"https://www.squid-cache.org/","html_tokens":1840,"markdown_tokens":525,"pruned_tokens":196}},
{"rank":1065,"domain":"grandviewresearch.com","origin":"https://grandviewresearch.com","robots":{"status":"ok","http":200,"url":"https://www.grandviewresearch.com/robots.txt","bytes":1087,"groups":1},"bots":{"GPTBot":["partial","*",33,0],"ChatGPT-User":["partial","*",33,0],"OAI-SearchBot":["partial","*",33,0],"ClaudeBot":["partial","*",33,0],"Claude-User":["partial","*",33,0],"Claude-SearchBot":["partial","*",33,0],"anthropic-ai":["partial","*",33,0],"PerplexityBot":["partial","*",33,0],"Perplexity-User":["partial","*",33,0],"Google-Extended":["partial","*",33,0],"Applebot-Extended":["partial","*",33,0],"CCBot":["partial","*",33,0],"Bytespider":["partial","*",33,0],"Amazonbot":["partial","*",33,0],"meta-externalagent":["partial","*",33,0],"cohere-ai":["partial","*",33,0],"Diffbot":["partial","*",33,0],"ContextFetchBot":["partial","*",33,0]},"llms":{"status":"present","http":200,"bytes":9417},"home":{"status":"ok","http":200,"url":"https://www.grandviewresearch.com/","html_tokens":53432,"markdown_tokens":798,"pruned_tokens":237}},
{"rank":1066,"domain":"airtable.com","origin":"https://airtable.com","robots":{"status":"ok","http":200,"url":"https://airtable.com/robots.txt","bytes":506,"groups":1},"bots":{"GPTBot":["partial","*",17,2],"ChatGPT-User":["partial","*",17,2],"OAI-SearchBot":["partial","*",17,2],"ClaudeBot":["partial","*",17,2],"Claude-User":["partial","*",17,2],"Claude-SearchBot":["partial","*",17,2],"anthropic-ai":["partial","*",17,2],"PerplexityBot":["partial","*",17,2],"Perplexity-User":["partial","*",17,2],"Google-Extended":["partial","*",17,2],"Applebot-Extended":["partial","*",17,2],"CCBot":["partial","*",17,2],"Bytespider":["partial","*",17,2],"Amazonbot":["partial","*",17,2],"meta-externalagent":["partial","*",17,2],"cohere-ai":["partial","*",17,2],"Diffbot":["partial","*",17,2],"ContextFetchBot":["partial","*",17,2]},"llms":{"status":"missing","http":404,"bytes":306561},"home":{"status":"ok","http":200,"url":"https://www.airtable.com/","html_tokens":95303,"markdown_tokens":302,"pruned_tokens":174}},
{"rank":1067,"domain":"gitbook.io","origin":"https://gitbook.io","robots":{"status":"html","http":200,"url":"https://www.gitbook.com/","bytes":1504022,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":1068,"domain":"dhl.com","origin":"https://dhl.com","robots":{"status":"unreachable","http":null,"url":"https://dhl.com/robots.txt","bytes":0,"groups":0,"error":"<StreamReset stream_id:1, error_code:2, remote_reset:True>"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":1069,"domain":"js.org","origin":"https://js.org","robots":{"status":"missing","http":404,"url":"https://js.org/robots.txt","bytes":11575,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":11575},"home":{"status":"ok","http":200,"url":"https://js.org/","html_tokens":4386,"markdown_tokens":218,"pruned_tokens":50}},
{"rank":1070,"domain":"sch.id","origin":"https://sch.id","robots":{"status":"unreachable","http":null,"url":"https://sch.id/robots.txt","bytes":0,"groups":0,"error":"cannot resolve sch.id"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":1071,"domain":"dictionary.com","origin":"https://dictionary.com","robots":{"status":"ok","http":200,"url":"https://www.dictionary.com/robots.txt","bytes":566,"groups":7},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["partial","*",9,3],"OAI-SearchBot":["partial","*",9,3],"ClaudeBot":["partial","*",9,3],"Claude-User":["partial","*",9,3],"Claude-SearchBot":["partial","*",9,3],"anthropic-ai":["partial","*",9,3],"PerplexityBot":["partial","*",9,3],"Perplexity-User":["partial","*",9,3],"Google-Extended":["partial","*",9,3],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["partial","*",9,3],"Amazonbot":["partial","*",9,3],"meta-externalagent":["partial","*",9,3],"cohere-ai":["partial","*",9,3],"Diffbot":["partial","*",9,3],"ContextFetchBot":["partial","*",9,3]},"llms":{"status":"missing","http":404,"bytes":52216},"home":{"status":"ok","http":200,"url":"https://www.dictionary.com/","html_tokens":16287,"markdown_tokens":69,"pruned_tokens":69}},
{"rank":1072,"domain":"waw.pl","origin":"https://www.waw.pl","robots":{"status":"ok","http":200,"url":"https://www.waw.pl/robots.txt","bytes":49,"groups":1},"bots":{"GPTBot":["allowed","*",0,2],"ChatGPT-User":["allowed","*",0,2],"OAI-SearchBot":["allowed","*",0,2],"ClaudeBot":["allowed","*",0,2],"Claude-User":["allowed","*",0,2],"Claude-SearchBot":["allowed","*",0,2],"anthropic-ai":["allowed","*",0,2],"PerplexityBot":["allowed","*",0,2],"Perplexity-User":["allowed","*",0,2],"Google-Extended":["allowed","*",0,2],"Applebot-Extended":["allowed","*",0,2],"CCBot":["allowed","*",0,2],"Bytespider":["allowed","*",0,2],"Amazonbot":["allowed","*",0,2],"meta-externalagent":["allowed","*",0,2],"cohere-ai":["allowed","*",0,2],"Diffbot":["allowed","*",0,2],"ContextFetchBot":["allowed","*",0,2]},"llms":{"status":"missing","http":404,"bytes":236},"home":{"status":"ok","http":200,"url":"https://www.waw.pl/","html_tokens":651,"markdown_tokens":42,"pruned_tokens":41}},
{"rank":1073,"domain":"over-blog.com","origin":"https://over-blog.com","robots":{"status":"ok","http":200,"url":"https://www.over-blog.com/robots.txt","bytes":23,"groups":1},"bots":{"GPTBot":["allowed","*",0,1],"ChatGPT-User":["allowed","*",0,1],"OAI-SearchBot":["allowed","*",0,1],"ClaudeBot":["allowed","*",0,1],"Claude-User":["allowed","*",0,1],"Claude-SearchBot":["allowed","*",0,1],"anthropic-ai":["allowed","*",0,1],"PerplexityBot":["allowed","*",0,1],"Perplexity-User":["allowed","*",0,1],"Google-Extended":["allowed","*",0,1],"Applebot-Extended":["allowed","*",0,1],"CCBot":["allowed","*",0,1],"Bytespider":["allowed","*",0,1],"Amazonbot":["allowed","*",0,1],"meta-externalagent":["allowed","*",0,1],"cohere-ai":["allowed","*",0,1],"Diffbot":["allowed","*",0,1],"ContextFetchBot":["allowed","*",0,1]},"llms":{"status":"missing","http":404,"bytes":7517},"home":{"status":"ok","http":200,"url":"https://www.over-blog.com/","html_tokens":53646,"markdown_tokens":200,"pruned_tokens":57}},
{"rank":1074,"domain":"sketchfab.com","origin":"https://sketchfab.com","robots":{"status":"html","http":202,"url":"https://sketchfab.com/robots.txt","bytes":2043,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":1075,"domain":"unl.edu","origin":"https://unl.edu","robots":{"status":"ok","http":200,"url":"https://www.unl.edu/robots.txt","bytes":1955,"groups":1},"bots":{"GPTBot":["partial","*",31,18],"ChatGPT-User":["partial","*",31,18],"OAI-SearchBot":["partial","*",31,18],"ClaudeBot":["partial","*",31,18],"Claude-User":["partial","*",31,18],"Claude-SearchBot":["partial","*",31,18],"anthropic-ai":["partial","*",31,18],"PerplexityBot":["partial","*",31,18],"Perplexity-User":["partial","*",31,18],"Google-Extended":["partial","*",31,18],"Applebot-Extended":["partial","*",31,18],"CCBot":["partial","*",31,18],"Bytespider":["partial","*",31,18],"Amazonbot":["partial","*",31,18],"meta-externalagent":["partial","*",31,18],"cohere-ai":["partial","*",31,18],"Diffbot":["partial","*",31,18],"ContextFetchBot":["partial","*",31,18]},"llms":{"status":"missing","http":404,"bytes":185},"home":{"status":"ok","http":200,"url":"https://www.unl.edu/","html_tokens":58414,"markdown_tokens":327,"pruned_tokens":103}},
{"rank":1076,"domain":"computerworld.com","origin":"https://computerworld.com","robots":{"status":"ok","http":200,"url":"https://www.computerworld.com/robots.txt","bytes":3270,"groups":37},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["partial","*",24,0],"OAI-SearchBot":["partial","*",24,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",24,0],"Claude-SearchBot":["partial","*",24,0],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["partial","*",24,0],"Perplexity-User":["partial","*",24,0],"Google-Extended":["partial","*",24,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["partial","*",24,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",24,0]},"llms":{"status":"missing","http":404,"bytes":183624},"home":{"status":"ok","http":200,"url":"https://www.computerworld.com","html_tokens":76726,"markdown_tokens":57,"pruned_tokens":57}},
{"rank":1077,"domain":"gitea.com","origin":"https://gitea.com","robots":{"status":"ok","http":200,"url":"https://gitea.com/robots.txt","bytes":1889,"groups":9},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["blocked","named",1,0],"OAI-SearchBot":["blocked","named",1,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",63,2],"Claude-SearchBot":["partial","*",63,2],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["partial","*",63,2],"Google-Extended":["partial","*",63,2],"Applebot-Extended":["partial","*",63,2],"CCBot":["blocked","named",1,0],"Bytespider":["partial","*",63,2],"Amazonbot":["partial","*",63,2],"meta-externalagent":["partial","*",63,2],"cohere-ai":["partial","*",63,2],"Diffbot":["partial","*",63,2],"ContextFetchBot":["partial","*",63,2]},"llms":{"status":"missing","http":404,"bytes":10330},"home":{"status":"ok","http":200,"url":"https://about.gitea.com","html_tokens":42610,"markdown_tokens":1056,"pruned_tokens":622}},
{"rank":1078,"domain":"epicgames.com","origin":"https://epicgames.com","robots":{"status":"refused","http":403,"url":"https://www.epicgames.com/site/robots.txt","bytes":25375,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":1079,"domain":"unfccc.int","origin":"https://unfccc.int","robots":{"status":"ok","http":200,"url":"https://unfccc.int/robots.txt","bytes":2027,"groups":1},"bots":{"GPTBot":["partial","*",34,18],"ChatGPT-User":["partial","*",34,18],"OAI-SearchBot":["partial","*",34,18],"ClaudeBot":["partial","*",34,18],"Claude-User":["partial","*",34,18],"Claude-SearchBot":["partial","*",34,18],"anthropic-ai":["partial","*",34,18],"PerplexityBot":["partial","*",34,18],"Perplexity-User":["partial","*",34,18],"Google-Extended":["partial","*",34,18],"Applebot-Extended":["partial","*",34,18],"CCBot":["partial","*",34,18],"Bytespider":["partial","*",34,18],"Amazonbot":["partial","*",34,18],"meta-externalagent":["partial","*",34,18],"cohere-ai":["partial","*",34,18],"Diffbot":["partial","*",34,18],"ContextFetchBot":["partial","*",34,18]},"llms":{"status":"timeout","error":"timed out after 10 s"},"home":{"status":"ok","http":200,"url":"https://unfccc.int/","html_tokens":53,"markdown_tokens":0,"pruned_tokens":0}},
{"rank":1080,"domain":"jpost.com","origin":"https://jpost.com","robots":{"status":"ok","http":200,"url":"https://www.jpost.com/robots.txt","bytes":2431,"groups":8},"bots":{"GPTBot":["partial","named",11,1],"ChatGPT-User":["partial","named",22,2],"OAI-SearchBot":["partial","named",11,1],"ClaudeBot":["allowed","none",0,0],"Claude-User":["partial","named",11,1],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["partial","named",11,1],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":108662},"home":{"status":"ok","http":200,"url":"https://www.jpost.com/","html_tokens":194046,"markdown_tokens":628,"pruned_tokens":133}},
{"rank":1081,"domain":"faz.net","origin":"https://faz.net","robots":{"status":"ok","http":200,"url":"https://www.faz.net/robots.txt","bytes":3363,"groups":49},"bots":{"GPTBot":["partial","named",1,1],"ChatGPT-User":["partial","named",1,1],"OAI-SearchBot":["partial","named",1,1],"ClaudeBot":["blocked","named",1,0],"Claude-User":["blocked","named",1,0],"Claude-SearchBot":["blocked","named",1,0],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["partial","*",34,1],"Google-Extended":["partial","named",1,2],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",34,1]},"llms":{"status":"missing","http":404,"bytes":338150},"home":{"status":"ok","http":200,"url":"https://www.faz.net/aktuell/","html_tokens":346241,"markdown_tokens":224,"pruned_tokens":72}},
{"rank":1082,"domain":"termsfeed.com","origin":"https://termsfeed.com","robots":{"status":"ok","http":200,"url":"https://www.termsfeed.com/robots.txt","bytes":2134,"groups":1},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"http_error","http":403,"bytes":5531},"home":{"status":"http_error","http":403,"url":"https://termsfeed.com/"}},
{"rank":1083,"domain":"purl.org","origin":"https://purl.org","robots":{"status":"ok","http":200,"url":"https://purl.org/robots.txt","bytes":29,"groups":1},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"missing","http":404,"bytes":207},"home":{"status":"ok","http":200,"url":"https://purl.archive.org","html_tokens":1342,"markdown_tokens":81,"pruned_tokens":59}},
{"rank":1084,"domain":"aarp.org","origin":"https://aarp.org","robots":{"status":"ok","http":200,"url":"https://www.aarp.org/robots.txt","bytes":1962,"groups":7},"bots":{"GPTBot":["partial","*",21,1],"ChatGPT-User":["partial","*",21,1],"OAI-SearchBot":["partial","*",21,1],"ClaudeBot":["partial","*",21,1],"Claude-User":["partial","*",21,1],"Claude-SearchBot":["partial","*",21,1],"anthropic-ai":["partial","*",21,1],"PerplexityBot":["partial","*",21,1],"Perplexity-User":["partial","*",21,1],"Google-Extended":["partial","*",21,1],"Applebot-Extended":["partial","*",21,1],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["partial","*",21,1],"meta-externalagent":["partial","*",21,1],"cohere-ai":["partial","*",21,1],"Diffbot":["partial","*",21,1],"ContextFetchBot":["partial","*",21,1]},"llms":{"status":"missing","http":404,"bytes":196},"home":{"status":"ok","http":200,"url":"https://www.aarp.org/","html_tokens":276734,"markdown_tokens":254,"pruned_tokens":71}},
{"rank":1085,"domain":"sophos.com","origin":"https://sophos.com","robots":{"status":"ok","http":200,"url":"https://www.sophos.com/robots.txt","bytes":1719,"groups":22},"bots":{"GPTBot":["allowed","named",0,1],"ChatGPT-User":["allowed","named",0,1],"OAI-SearchBot":["allowed","named",0,1],"ClaudeBot":["allowed","named",0,1],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","named",0,1],"PerplexityBot":["allowed","named",0,1],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","named",0,1],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","named",0,1],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"present","http":200,"bytes":25391},"home":{"status":"ok","http":200,"url":"https://www.sophos.com/en-us","html_tokens":248910,"markdown_tokens":318,"pruned_tokens":79}},
{"rank":1086,"domain":"edx.org","origin":"https://edx.org","robots":{"status":"ok","http":200,"url":"https://www.edx.org/robots.txt","bytes":1346,"groups":1},"bots":{"GPTBot":["partial","*",23,32],"ChatGPT-User":["partial","*",23,32],"OAI-SearchBot":["partial","*",23,32],"ClaudeBot":["partial","*",23,32],"Claude-User":["partial","*",23,32],"Claude-SearchBot":["partial","*",23,32],"anthropic-ai":["partial","*",23,32],"PerplexityBot":["partial","*",23,32],"Perplexity-User":["partial","*",23,32],"Google-Extended":["partial","*",23,32],"Applebot-Extended":["partial","*",23,32],"CCBot":["partial","*",23,32],"Bytespider":["partial","*",23,32],"Amazonbot":["partial","*",23,32],"meta-externalagent":["partial","*",23,32],"cohere-ai":["partial","*",23,32],"Diffbot":["partial","*",23,32],"ContextFetchBot":["partial","*",23,32]},"llms":{"status":"present","http":200,"bytes":3645},"home":{"status":"ok","http":200,"url":"https://www.edx.org/","html_tokens":614282,"markdown_tokens":266,"pruned_tokens":203}},
{"rank":1087,"domain":"gob.pe","origin":"https://www.gob.pe","robots":{"status":"ok","http":200,"url":"https://www.gob.pe/robots.txt","bytes":525,"groups":4},"bots":{"GPTBot":["partial","named",3,0],"ChatGPT-User":["partial","*",3,0],"OAI-SearchBot":["partial","named",3,0],"ClaudeBot":["partial","*",3,0],"Claude-User":["partial","*",3,0],"Claude-SearchBot":["partial","*",3,0],"anthropic-ai":["partial","*",3,0],"PerplexityBot":["partial","*",3,0],"Perplexity-User":["partial","*",3,0],"Google-Extended":["partial","*",3,0],"Applebot-Extended":["partial","*",3,0],"CCBot":["partial","*",3,0],"Bytespider":["partial","*",3,0],"Amazonbot":["partial","*",3,0],"meta-externalagent":["partial","*",3,0],"cohere-ai":["partial","*",3,0],"Diffbot":["partial","*",3,0],"ContextFetchBot":["partial","*",3,0]},"llms":{"status":"missing","http":404,"bytes":34833},"home":{"status":"ok","http":200,"url":"https://www.gob.pe/","html_tokens":58545,"markdown_tokens":68,"pruned_tokens":68}},
{"rank":1088,"domain":"vanityfair.com","origin":"https://vanityfair.com","robots":{"status":"ok","http":200,"url":"https://www.vanityfair.com/robots.txt","bytes":2367,"groups":3},"bots":{"GPTBot":["partial","*",16,15],"ChatGPT-User":["partial","*",16,15],"OAI-SearchBot":["partial","*",16,15],"ClaudeBot":["blocked","named",1,0],"Claude-User":["blocked","named",1,0],"Claude-SearchBot":["blocked","named",1,0],"anthropic-ai":["partial","*",16,15],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["blocked","named",1,0],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",16,15]},"llms":{"status":"missing","http":404,"bytes":720753},"home":{"status":"ok","http":200,"url":"https://www.vanityfair.com/","html_tokens":440386,"markdown_tokens":1264,"pruned_tokens":41}},
{"rank":1089,"domain":"virtualmin.com","origin":"https://virtualmin.com","robots":{"status":"error","http":null,"url":"https://virtualmin.com/robots.txt","bytes":0,"groups":0,"error":"more than 5 redirects"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":1090,"domain":"nber.org","origin":"https://nber.org","robots":{"status":"refused","http":403,"url":"https://nber.org/robots.txt","bytes":560,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":1091,"domain":"thestar.com","origin":"https://thestar.com","robots":{"status":"ok","http":200,"url":"https://thestar.com/robots.txt","bytes":13856,"groups":235},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["blocked","named",1,0],"OAI-SearchBot":["blocked","named",1,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["blocked","named",1,0],"Claude-SearchBot":["blocked","named",1,0],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["blocked","named",1,0],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",2,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",68,0]},"llms":{"status":"http_error","http":429,"bytes":64},"home":{"status":"ok","http":200,"url":"https://www.thestar.com/","html_tokens":167470,"markdown_tokens":200,"pruned_tokens":162}},
{"rank":1092,"domain":"heylink.me","origin":"https://heylink.me","robots":{"status":"refused","http":403,"url":"https://heylink.me/robots.txt","bytes":5505,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":1093,"domain":"one.com","origin":"https://one.com","robots":{"status":"ok","http":200,"url":"https://www.one.com/robots.txt","bytes":450,"groups":1},"bots":{"GPTBot":["partial","*",17,0],"ChatGPT-User":["partial","*",17,0],"OAI-SearchBot":["partial","*",17,0],"ClaudeBot":["partial","*",17,0],"Claude-User":["partial","*",17,0],"Claude-SearchBot":["partial","*",17,0],"anthropic-ai":["partial","*",17,0],"PerplexityBot":["partial","*",17,0],"Perplexity-User":["partial","*",17,0],"Google-Extended":["partial","*",17,0],"Applebot-Extended":["partial","*",17,0],"CCBot":["partial","*",17,0],"Bytespider":["partial","*",17,0],"Amazonbot":["partial","*",17,0],"meta-externalagent":["partial","*",17,0],"cohere-ai":["partial","*",17,0],"Diffbot":["partial","*",17,0],"ContextFetchBot":["partial","*",17,0]},"llms":{"status":"present","http":200,"bytes":3764},"home":{"status":"ok","http":200,"url":"https://www.one.com/en-gb/","html_tokens":427507,"markdown_tokens":1310,"pruned_tokens":475}},
{"rank":1094,"domain":"irishtimes.com","origin":"https://irishtimes.com","robots":{"status":"ok","http":200,"url":"https://www.irishtimes.com/robots.txt","bytes":2810,"groups":4},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["blocked","named",1,0],"OAI-SearchBot":["partial","*",50,18],"ClaudeBot":["partial","*",50,18],"Claude-User":["partial","*",50,18],"Claude-SearchBot":["partial","*",50,18],"anthropic-ai":["partial","*",50,18],"PerplexityBot":["partial","*",50,18],"Perplexity-User":["partial","*",50,18],"Google-Extended":["partial","*",50,18],"Applebot-Extended":["partial","*",50,18],"CCBot":["partial","*",50,18],"Bytespider":["partial","*",50,18],"Amazonbot":["partial","*",50,18],"meta-externalagent":["partial","*",50,18],"cohere-ai":["partial","*",50,18],"Diffbot":["partial","*",50,18],"ContextFetchBot":["partial","*",50,18]},"llms":{"status":"missing","http":404,"bytes":237676},"home":{"status":"ok","http":200,"url":"https://www.irishtimes.com/","html_tokens":1056052,"markdown_tokens":61,"pruned_tokens":61}},
{"rank":1095,"domain":"site.com","origin":"https://site.com","robots":{"status":"ok","http":200,"url":"https://site.com/robots.txt","bytes":189,"groups":1},"bots":{"GPTBot":["partial","*",1,1],"ChatGPT-User":["partial","*",1,1],"OAI-SearchBot":["partial","*",1,1],"ClaudeBot":["partial","*",1,1],"Claude-User":["partial","*",1,1],"Claude-SearchBot":["partial","*",1,1],"anthropic-ai":["partial","*",1,1],"PerplexityBot":["partial","*",1,1],"Perplexity-User":["partial","*",1,1],"Google-Extended":["partial","*",1,1],"Applebot-Extended":["partial","*",1,1],"CCBot":["partial","*",1,1],"Bytespider":["partial","*",1,1],"Amazonbot":["partial","*",1,1],"meta-externalagent":["partial","*",1,1],"cohere-ai":["partial","*",1,1],"Diffbot":["partial","*",1,1],"ContextFetchBot":["partial","*",1,1]},"llms":{"status":"html","http":200,"bytes":9942},"home":{"status":"ok","http":200,"url":"https://site.com/","html_tokens":172,"markdown_tokens":0,"pruned_tokens":0}},
{"rank":1096,"domain":"google.co.in","origin":"https://google.co.in","robots":{"status":"ok","http":200,"url":"https://www.google.co.in/robots.txt","bytes":6624,"groups":4},"bots":{"GPTBot":["partial","*",169,64],"ChatGPT-User":["partial","*",169,64],"OAI-SearchBot":["partial","*",169,64],"ClaudeBot":["partial","*",169,64],"Claude-User":["partial","*",169,64],"Claude-SearchBot":["partial","*",169,64],"anthropic-ai":["partial","*",169,64],"PerplexityBot":["partial","*",169,64],"Perplexity-User":["partial","*",169,64],"Google-Extended":["partial","*",169,64],"Applebot-Extended":["partial","*",169,64],"CCBot":["partial","*",169,64],"Bytespider":["partial","*",169,64],"Amazonbot":["partial","*",169,64],"meta-externalagent":["partial","*",169,64],"cohere-ai":["partial","*",169,64],"Diffbot":["partial","*",169,64],"ContextFetchBot":["partial","*",169,64]},"llms":{"status":"missing","http":404,"bytes":1569},"home":{"status":"ok","http":200,"url":"https://www.google.com/","html_tokens":21356,"markdown_tokens":30,"pruned_tokens":30}},
{"rank":1097,"domain":"fraunhofer.de","origin":"https://fraunhofer.de","robots":{"status":"ok","http":200,"url":"https://www.fraunhofer.de/robots.txt","bytes":576,"groups":1},"bots":{"GPTBot":["partial","*",23,0],"ChatGPT-User":["partial","*",23,0],"OAI-SearchBot":["partial","*",23,0],"ClaudeBot":["partial","*",23,0],"Claude-User":["partial","*",23,0],"Claude-SearchBot":["partial","*",23,0],"anthropic-ai":["partial","*",23,0],"PerplexityBot":["partial","*",23,0],"Perplexity-User":["partial","*",23,0],"Google-Extended":["partial","*",23,0],"Applebot-Extended":["partial","*",23,0],"CCBot":["partial","*",23,0],"Bytespider":["partial","*",23,0],"Amazonbot":["partial","*",23,0],"meta-externalagent":["partial","*",23,0],"cohere-ai":["partial","*",23,0],"Diffbot":["partial","*",23,0],"ContextFetchBot":["partial","*",23,0]},"llms":{"status":"missing","http":404,"bytes":75807},"home":{"status":"ok","http":200,"url":"https://www.fraunhofer.de/","html_tokens":42030,"markdown_tokens":133,"pruned_tokens":133}},
{"rank":1098,"domain":"worldcat.org","origin":"https://worldcat.org","robots":{"status":"ok","http":200,"url":"https://search.worldcat.org/robots.txt","bytes":651,"groups":5},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["blocked","named",1,0],"OAI-SearchBot":["partial","*",13,0],"ClaudeBot":["partial","*",13,0],"Claude-User":["partial","*",13,0],"Claude-SearchBot":["partial","*",13,0],"anthropic-ai":["partial","*",13,0],"PerplexityBot":["partial","*",13,0],"Perplexity-User":["partial","*",13,0],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["partial","*",13,0],"CCBot":["blocked","named",1,0],"Bytespider":["partial","*",13,0],"Amazonbot":["partial","*",13,0],"meta-externalagent":["partial","*",13,0],"cohere-ai":["partial","*",13,0],"Diffbot":["partial","*",13,0],"ContextFetchBot":["partial","*",13,0]},"llms":{"status":"missing","http":404,"bytes":191923},"home":{"status":"ok","http":200,"url":"https://search.worldcat.org/","html_tokens":115017,"markdown_tokens":375,"pruned_tokens":272}},
{"rank":1099,"domain":"doxygen.org","origin":"https://doxygen.org","robots":{"status":"ok","http":200,"url":"https://www.doxygen.nl/robots.txt","bytes":1248,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":1618},"home":{"status":"ok","http":200,"url":"https://www.doxygen.nl/","html_tokens":3690,"markdown_tokens":533,"pruned_tokens":353}},
{"rank":1100,"domain":"bostonglobe.com","origin":"https://bostonglobe.com","robots":{"status":"ok","http":200,"url":"https://www.bostonglobe.com/robots.txt","bytes":1373,"groups":25},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":878929},"home":{"status":"ok","http":200,"url":"https://www.bostonglobe.com/","html_tokens":680176,"markdown_tokens":146,"pruned_tokens":146}},
{"rank":1101,"domain":"chrome.com","origin":"https://chrome.com","robots":{"status":"html","http":200,"url":"https://www.google.com/chrome/","bytes":468511,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":1102,"domain":"timeout.com","origin":"https://timeout.com","robots":{"status":"ok","http":200,"url":"https://www.timeout.com/robots.txt","bytes":3151,"groups":9},"bots":{"GPTBot":["partial","*",30,0],"ChatGPT-User":["partial","*",30,0],"OAI-SearchBot":["partial","*",30,0],"ClaudeBot":["partial","*",30,0],"Claude-User":["partial","*",30,0],"Claude-SearchBot":["partial","*",30,0],"anthropic-ai":["partial","*",30,0],"PerplexityBot":["partial","*",30,0],"Perplexity-User":["partial","*",30,0],"Google-Extended":["partial","*",30,0],"Applebot-Extended":["partial","*",30,0],"CCBot":["partial","*",30,0],"Bytespider":["partial","*",30,0],"Amazonbot":["partial","*",30,0],"meta-externalagent":["partial","*",30,0],"cohere-ai":["partial","*",30,0],"Diffbot":["partial","*",30,0],"ContextFetchBot":["partial","*",30,0]},"llms":{"status":"missing","http":404,"bytes":176373},"home":{"status":"ok","http":200,"url":"https://www.timeout.com/","html_tokens":81316,"markdown_tokens":462,"pruned_tokens":174}},
{"rank":1103,"domain":"boutell.com","origin":"https://boutell.com","robots":{"status":"ok","http":200,"url":"https://boutell.com/robots.txt","bytes":35,"groups":1},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"html","http":200,"bytes":17932},"home":{"status":"ok","http":200,"url":"https://boutell.com/","html_tokens":4483,"markdown_tokens":1781,"pruned_tokens":408}},
{"rank":1104,"domain":"thawte.com","origin":"https://thawte.com","robots":{"status":"ok","http":200,"url":"https://www.thawte.com/robots.txt","bytes":286,"groups":1},"bots":{"GPTBot":["partial","*",2,0],"ChatGPT-User":["partial","*",2,0],"OAI-SearchBot":["partial","*",2,0],"ClaudeBot":["partial","*",2,0],"Claude-User":["partial","*",2,0],"Claude-SearchBot":["partial","*",2,0],"anthropic-ai":["partial","*",2,0],"PerplexityBot":["partial","*",2,0],"Perplexity-User":["partial","*",2,0],"Google-Extended":["partial","*",2,0],"Applebot-Extended":["partial","*",2,0],"CCBot":["partial","*",2,0],"Bytespider":["partial","*",2,0],"Amazonbot":["partial","*",2,0],"meta-externalagent":["partial","*",2,0],"cohere-ai":["partial","*",2,0],"Diffbot":["partial","*",2,0],"ContextFetchBot":["partial","*",2,0]},"llms":{"status":"present","http":200,"bytes":3145},"home":{"status":"ok","http":200,"url":"https://www.thawte.com/","html_tokens":13931,"markdown_tokens":486,"pruned_tokens":147}},
{"rank":1105,"domain":"a.co","origin":"https://a.co","robots":{"status":"missing","http":404,"url":"https://a.co/robots.txt","bytes":28,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":28},"home":{"status":"ok","http":200,"url":"https://www.amazon.com/","html_tokens":177640,"markdown_tokens":1135,"pruned_tokens":175}},
{"rank":1106,"domain":"asana.com","origin":"https://asana.com","robots":{"status":"ok","http":200,"url":"https://asana.com/robots.txt","bytes":547,"groups":1},"bots":{"GPTBot":["partial","*",14,4],"ChatGPT-User":["partial","*",14,4],"OAI-SearchBot":["partial","*",14,4],"ClaudeBot":["partial","*",14,4],"Claude-User":["partial","*",14,4],"Claude-SearchBot":["partial","*",14,4],"anthropic-ai":["partial","*",14,4],"PerplexityBot":["partial","*",14,4],"Perplexity-User":["partial","*",14,4],"Google-Extended":["partial","*",14,4],"Applebot-Extended":["partial","*",14,4],"CCBot":["partial","*",14,4],"Bytespider":["partial","*",14,4],"Amazonbot":["partial","*",14,4],"meta-externalagent":["partial","*",14,4],"cohere-ai":["partial","*",14,4],"Diffbot":["partial","*",14,4],"ContextFetchBot":["partial","*",14,4]},"llms":{"status":"present","http":200,"bytes":5691},"home":{"status":"ok","http":200,"url":"https://asana.com/","html_tokens":239974,"markdown_tokens":647,"pruned_tokens":11}},
{"rank":1107,"domain":"symantec.com","origin":"https://symantec.com","robots":{"status":"missing","http":404,"url":"https://www.broadcom.com/404-symantec?sourceURL=http://symantec.com/robots.txt?","bytes":16607,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":16607},"home":{"status":"ok","http":200,"url":"https://www.broadcom.com/products/cybersecurity","html_tokens":4283,"markdown_tokens":0,"pruned_tokens":0}},
{"rank":1108,"domain":"repubblica.it","origin":"https://repubblica.it","robots":{"status":"ok","http":200,"url":"https://www.repubblica.it/robots.txt","bytes":7368,"groups":108},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["partial","*",53,1],"OAI-SearchBot":["partial","*",53,1],"ClaudeBot":["partial","*",53,1],"Claude-User":["partial","*",53,1],"Claude-SearchBot":["partial","*",53,1],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["partial","*",53,1],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["allowed","named",0,1],"meta-externalagent":["partial","*",53,1],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",53,1]},"llms":{"status":"present","http":200,"bytes":2331},"home":{"status":"ok","http":200,"url":"https://www.repubblica.it/","html_tokens":104003,"markdown_tokens":20,"pruned_tokens":20}},
{"rank":1109,"domain":"strava.com","origin":"https://strava.com","robots":{"status":"ok","http":200,"url":"https://www.strava.com/robots.txt","bytes":1510,"groups":2},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["partial","*",43,2],"OAI-SearchBot":["partial","*",43,2],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",43,2],"Claude-SearchBot":["partial","*",43,2],"anthropic-ai":["partial","*",43,2],"PerplexityBot":["partial","*",43,2],"Perplexity-User":["partial","*",43,2],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["partial","*",43,2],"CCBot":["partial","*",43,2],"Bytespider":["partial","*",43,2],"Amazonbot":["partial","*",43,2],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["partial","*",43,2],"Diffbot":["partial","*",43,2],"ContextFetchBot":["partial","*",43,2]},"llms":{"status":"missing","http":404,"bytes":843},"home":{"status":"ok","http":200,"url":"https://www.strava.com/","html_tokens":164450,"markdown_tokens":1149,"pruned_tokens":474}},
{"rank":1110,"domain":"eia.gov","origin":"https://eia.gov","robots":{"status":"ok","http":200,"url":"https://www.eia.gov/robots.txt","bytes":718,"groups":1},"bots":{"GPTBot":["partial","*",24,3],"ChatGPT-User":["partial","*",24,3],"OAI-SearchBot":["partial","*",24,3],"ClaudeBot":["partial","*",24,3],"Claude-User":["partial","*",24,3],"Claude-SearchBot":["partial","*",24,3],"anthropic-ai":["partial","*",24,3],"PerplexityBot":["partial","*",24,3],"Perplexity-User":["partial","*",24,3],"Google-Extended":["partial","*",24,3],"Applebot-Extended":["partial","*",24,3],"CCBot":["partial","*",24,3],"Bytespider":["partial","*",24,3],"Amazonbot":["partial","*",24,3],"meta-externalagent":["partial","*",24,3],"cohere-ai":["partial","*",24,3],"Diffbot":["partial","*",24,3],"ContextFetchBot":["partial","*",24,3]},"llms":{"status":"missing","http":404,"bytes":46403},"home":{"status":"ok","http":200,"url":"https://www.eia.gov/","html_tokens":12096,"markdown_tokens":151,"pruned_tokens":151}},
{"rank":1111,"domain":"bluehost.com","origin":"https://bluehost.com","robots":{"status":"refused","http":403,"url":"https://www.bluehost.com/robots.txt","bytes":447199,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":1112,"domain":"ghost.org","origin":"https://ghost.org","robots":{"status":"ok","http":200,"url":"https://ghost.org/robots.txt","bytes":224,"groups":1},"bots":{"GPTBot":["partial","*",6,1],"ChatGPT-User":["partial","*",6,1],"OAI-SearchBot":["partial","*",6,1],"ClaudeBot":["partial","*",6,1],"Claude-User":["partial","*",6,1],"Claude-SearchBot":["partial","*",6,1],"anthropic-ai":["partial","*",6,1],"PerplexityBot":["partial","*",6,1],"Perplexity-User":["partial","*",6,1],"Google-Extended":["partial","*",6,1],"Applebot-Extended":["partial","*",6,1],"CCBot":["partial","*",6,1],"Bytespider":["partial","*",6,1],"Amazonbot":["partial","*",6,1],"meta-externalagent":["partial","*",6,1],"cohere-ai":["partial","*",6,1],"Diffbot":["partial","*",6,1],"ContextFetchBot":["partial","*",6,1]},"llms":{"status":"missing","http":404,"bytes":52245},"home":{"status":"ok","http":200,"url":"https://ghost.org/","html_tokens":30775,"markdown_tokens":826,"pruned_tokens":119}},
{"rank":1113,"domain":"amazon-adsystem.com","origin":"https://amazon-adsystem.com","robots":{"status":"unreachable","http":null,"url":"https://amazon-adsystem.com/robots.txt","bytes":0,"groups":0,"error":"Server disconnected without sending a response."},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":1114,"domain":"nationalacademies.org","origin":"https://nationalacademies.org","robots":{"status":"ok","http":200,"url":"https://www.nationalacademies.org/robots.txt","bytes":46,"groups":1},"bots":{"GPTBot":["partial","*",1,0],"ChatGPT-User":["partial","*",1,0],"OAI-SearchBot":["partial","*",1,0],"ClaudeBot":["partial","*",1,0],"Claude-User":["partial","*",1,0],"Claude-SearchBot":["partial","*",1,0],"anthropic-ai":["partial","*",1,0],"PerplexityBot":["partial","*",1,0],"Perplexity-User":["partial","*",1,0],"Google-Extended":["partial","*",1,0],"Applebot-Extended":["partial","*",1,0],"CCBot":["partial","*",1,0],"Bytespider":["partial","*",1,0],"Amazonbot":["partial","*",1,0],"meta-externalagent":["partial","*",1,0],"cohere-ai":["partial","*",1,0],"Diffbot":["partial","*",1,0],"ContextFetchBot":["partial","*",1,0]},"llms":{"status":"missing","http":404,"bytes":263068},"home":{"status":"ok","http":200,"url":"https://www.nationalacademies.org/","html_tokens":101363,"markdown_tokens":461,"pruned_tokens":64}},
{"rank":1115,"domain":"cookieadmin.net","origin":"https://cookieadmin.net","robots":{"status":"ok","http":200,"url":"https://cookieadmin.net/robots.txt","bytes":114,"groups":1},"bots":{"GPTBot":["partial","*",1,1],"ChatGPT-User":["partial","*",1,1],"OAI-SearchBot":["partial","*",1,1],"ClaudeBot":["partial","*",1,1],"Claude-User":["partial","*",1,1],"Claude-SearchBot":["partial","*",1,1],"anthropic-ai":["partial","*",1,1],"PerplexityBot":["partial","*",1,1],"Perplexity-User":["partial","*",1,1],"Google-Extended":["partial","*",1,1],"Applebot-Extended":["partial","*",1,1],"CCBot":["partial","*",1,1],"Bytespider":["partial","*",1,1],"Amazonbot":["partial","*",1,1],"meta-externalagent":["partial","*",1,1],"cohere-ai":["partial","*",1,1],"Diffbot":["partial","*",1,1],"ContextFetchBot":["partial","*",1,1]},"llms":{"status":"missing","http":404,"bytes":92285},"home":{"status":"ok","http":200,"url":"https://cookieadmin.net/","html_tokens":64307,"markdown_tokens":765,"pruned_tokens":508}},
{"rank":1116,"domain":"asahi.com","origin":"https://asahi.com","robots":{"status":"ok","http":200,"url":"https://www.asahi.com/robots.txt","bytes":2033,"groups":37},"bots":{"GPTBot":["partial","named",1,1],"ChatGPT-User":["partial","named",1,1],"OAI-SearchBot":["partial","named",1,1],"ClaudeBot":["partial","named",1,1],"Claude-User":["partial","named",1,1],"Claude-SearchBot":["partial","named",1,1],"anthropic-ai":["partial","named",1,1],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["blocked","named",1,0],"Google-Extended":["partial","named",1,1],"Applebot-Extended":["partial","named",1,1],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["partial","named",1,1],"meta-externalagent":["partial","named",1,1],"cohere-ai":["partial","named",1,1],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",8,3]},"llms":{"status":"missing","http":404,"bytes":74234},"home":{"status":"ok","http":200,"url":"https://www.asahi.com/","html_tokens":62502,"markdown_tokens":452,"pruned_tokens":162}},
{"rank":1117,"domain":"penguinrandomhouse.com","origin":"https://penguinrandomhouse.com","robots":{"status":"ok","http":200,"url":"https://www.penguinrandomhouse.com/robots.txt","bytes":321,"groups":1},"bots":{"GPTBot":["partial","*",8,2],"ChatGPT-User":["partial","*",8,2],"OAI-SearchBot":["partial","*",8,2],"ClaudeBot":["partial","*",8,2],"Claude-User":["partial","*",8,2],"Claude-SearchBot":["partial","*",8,2],"anthropic-ai":["partial","*",8,2],"PerplexityBot":["partial","*",8,2],"Perplexity-User":["partial","*",8,2],"Google-Extended":["partial","*",8,2],"Applebot-Extended":["partial","*",8,2],"CCBot":["partial","*",8,2],"Bytespider":["partial","*",8,2],"Amazonbot":["partial","*",8,2],"meta-externalagent":["partial","*",8,2],"cohere-ai":["partial","*",8,2],"Diffbot":["partial","*",8,2],"ContextFetchBot":["partial","*",8,2]},"llms":{"status":"missing","http":404,"bytes":161624},"home":{"status":"ok","http":200,"url":"https://www.penguinrandomhouse.com/","html_tokens":57880,"markdown_tokens":11,"pruned_tokens":11}},
{"rank":1118,"domain":"foursquare.com","origin":"https://foursquare.com","robots":{"status":"ok","http":200,"url":"https://foursquare.com/robots.txt","bytes":1251,"groups":4},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["blocked","named",1,0],"OAI-SearchBot":["partial","*",27,4],"ClaudeBot":["partial","*",27,4],"Claude-User":["partial","*",27,4],"Claude-SearchBot":["partial","*",27,4],"anthropic-ai":["partial","*",27,4],"PerplexityBot":["partial","*",27,4],"Perplexity-User":["partial","*",27,4],"Google-Extended":["partial","*",27,4],"Applebot-Extended":["partial","*",27,4],"CCBot":["partial","named",1,0],"Bytespider":["partial","*",27,4],"Amazonbot":["partial","*",27,4],"meta-externalagent":["partial","*",27,4],"cohere-ai":["partial","*",27,4],"Diffbot":["partial","*",27,4],"ContextFetchBot":["partial","*",27,4]},"llms":{"status":"missing","http":404,"bytes":181168},"home":{"status":"ok","http":200,"url":"https://foursquare.com/","html_tokens":53151,"markdown_tokens":707,"pruned_tokens":158}},
{"rank":1119,"domain":"eu.org","origin":"http://eu.org","robots":{"status":"ok","http":200,"url":"https://nic.eu.org/robots.txt","bytes":132,"groups":1},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":236},"home":{"status":"ok","http":200,"url":"https://nic.eu.org/","html_tokens":889,"markdown_tokens":168,"pruned_tokens":30}},
{"rank":1120,"domain":"libpng.org","origin":"https://libpng.org","robots":{"status":"ok","http":200,"url":"https://libpng.org/robots.txt","bytes":219,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":146},"home":{"status":"ok","http":200,"url":"https://libpng.org/","html_tokens":722,"markdown_tokens":70,"pruned_tokens":70}},
{"rank":1121,"domain":"grammarly.com","origin":"https://grammarly.com","robots":{"status":"ok","http":200,"url":"https://www.grammarly.com/robots.txt","bytes":289,"groups":1},"bots":{"GPTBot":["partial","*",8,0],"ChatGPT-User":["partial","*",8,0],"OAI-SearchBot":["partial","*",8,0],"ClaudeBot":["partial","*",8,0],"Claude-User":["partial","*",8,0],"Claude-SearchBot":["partial","*",8,0],"anthropic-ai":["partial","*",8,0],"PerplexityBot":["partial","*",8,0],"Perplexity-User":["partial","*",8,0],"Google-Extended":["partial","*",8,0],"Applebot-Extended":["partial","*",8,0],"CCBot":["partial","*",8,0],"Bytespider":["partial","*",8,0],"Amazonbot":["partial","*",8,0],"meta-externalagent":["partial","*",8,0],"cohere-ai":["partial","*",8,0],"Diffbot":["partial","*",8,0],"ContextFetchBot":["partial","*",8,0]},"llms":{"status":"missing","http":404,"bytes":474569},"home":{"status":"ok","http":200,"url":"https://www.grammarly.com/","html_tokens":137677,"markdown_tokens":902,"pruned_tokens":354}},
{"rank":1122,"domain":"indianexpress.com","origin":"https://indianexpress.com","robots":{"status":"ok","http":200,"url":"https://indianexpress.com/robots.txt","bytes":4779,"groups":20},"bots":{"GPTBot":["partial","*",120,2],"ChatGPT-User":["partial","*",120,2],"OAI-SearchBot":["partial","*",120,2],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",120,2],"Claude-SearchBot":["partial","*",120,2],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["partial","*",120,2],"Google-Extended":["partial","*",120,2],"Applebot-Extended":["blocked","named",1,0],"CCBot":["partial","*",120,2],"Bytespider":["blocked","named",1,0],"Amazonbot":["partial","*",120,2],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",120,2]},"llms":{"status":"missing","http":404,"bytes":683652},"home":{"status":"ok","http":200,"url":"https://indianexpress.com/","html_tokens":181198,"markdown_tokens":133,"pruned_tokens":133}},
{"rank":1123,"domain":"archlinux.org","origin":"https://archlinux.org","robots":{"status":"ok","http":200,"url":"https://archlinux.org/robots.txt","bytes":149,"groups":1},"bots":{"GPTBot":["partial","*",3,0],"ChatGPT-User":["partial","*",3,0],"OAI-SearchBot":["partial","*",3,0],"ClaudeBot":["partial","*",3,0],"Claude-User":["partial","*",3,0],"Claude-SearchBot":["partial","*",3,0],"anthropic-ai":["partial","*",3,0],"PerplexityBot":["partial","*",3,0],"Perplexity-User":["partial","*",3,0],"Google-Extended":["partial","*",3,0],"Applebot-Extended":["partial","*",3,0],"CCBot":["partial","*",3,0],"Bytespider":["partial","*",3,0],"Amazonbot":["partial","*",3,0],"meta-externalagent":["partial","*",3,0],"cohere-ai":["partial","*",3,0],"Diffbot":["partial","*",3,0],"ContextFetchBot":["partial","*",3,0]},"llms":{"status":"missing","http":404,"bytes":3616},"home":{"status":"ok","http":200,"url":"https://archlinux.org/","html_tokens":6339,"markdown_tokens":766,"pruned_tokens":159}},
{"rank":1124,"domain":"thomsonreuters.com","origin":"https://thomsonreuters.com","robots":{"status":"ok","http":200,"url":"https://www.thomsonreuters.com/robots.txt","bytes":1516,"groups":1},"bots":{"GPTBot":["partial","*",8,2],"ChatGPT-User":["partial","*",8,2],"OAI-SearchBot":["partial","*",8,2],"ClaudeBot":["partial","*",8,2],"Claude-User":["partial","*",8,2],"Claude-SearchBot":["partial","*",8,2],"anthropic-ai":["partial","*",8,2],"PerplexityBot":["partial","*",8,2],"Perplexity-User":["partial","*",8,2],"Google-Extended":["partial","*",8,2],"Applebot-Extended":["partial","*",8,2],"CCBot":["partial","*",8,2],"Bytespider":["partial","*",8,2],"Amazonbot":["partial","*",8,2],"meta-externalagent":["partial","*",8,2],"cohere-ai":["partial","*",8,2],"Diffbot":["partial","*",8,2],"ContextFetchBot":["partial","*",8,2]},"llms":{"status":"missing","http":404,"bytes":301345},"home":{"status":"ok","http":200,"url":"https://www.thomsonreuters.com/en","html_tokens":95080,"markdown_tokens":1645,"pruned_tokens":79}},
{"rank":1125,"domain":"fema.gov","origin":"https://fema.gov","robots":{"status":"refused","http":403,"url":"https://fema.gov/robots.txt","bytes":372,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":1126,"domain":"phpbb.com","origin":"https://phpbb.com","robots":{"status":"refused","http":403,"url":"https://phpbb.com/robots.txt","bytes":5482,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":1127,"domain":"lnk.to","origin":"https://lnk.to","robots":{"status":"ok","http":200,"url":"https://lnk.to/robots.txt","bytes":23,"groups":1},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"http_error","http":403,"bytes":118},"home":{"status":"not_html","error":"unsupported content type text/markdown"}},
{"rank":1128,"domain":"hotjar.com","origin":"https://hotjar.com","robots":{"status":"ok","http":200,"url":"https://www.hotjar.com/robots.txt","bytes":510,"groups":1},"bots":{"GPTBot":["allowed","*",0,1],"ChatGPT-User":["allowed","*",0,1],"OAI-SearchBot":["allowed","*",0,1],"ClaudeBot":["allowed","*",0,1],"Claude-User":["allowed","*",0,1],"Claude-SearchBot":["allowed","*",0,1],"anthropic-ai":["allowed","*",0,1],"PerplexityBot":["allowed","*",0,1],"Perplexity-User":["allowed","*",0,1],"Google-Extended":["allowed","*",0,1],"Applebot-Extended":["allowed","*",0,1],"CCBot":["allowed","*",0,1],"Bytespider":["allowed","*",0,1],"Amazonbot":["allowed","*",0,1],"meta-externalagent":["allowed","*",0,1],"cohere-ai":["allowed","*",0,1],"Diffbot":["allowed","*",0,1],"ContextFetchBot":["allowed","*",0,1]},"llms":{"status":"missing","http":404,"bytes":146315},"home":{"status":"ok","http":200,"url":"https://contentsquare.com/hotjar/?utm_campaign=exp__hj_hp_redirection","html_tokens":114028,"markdown_tokens":403,"pruned_tokens":177}},
{"rank":1129,"domain":"pantheonsite.io","origin":"https://www.pantheonsite.io","robots":{"status":"refused","http":403,"url":"https://www.pantheonsite.io/robots.txt","bytes":560,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":1130,"domain":"miibeian.gov.cn","origin":"https://miibeian.gov.cn","robots":{"status":"timeout","http":null,"url":"https://miibeian.gov.cn/robots.txt","bytes":0,"groups":0,"error":"timed out after 10 s"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":1131,"domain":"hsforms.com","origin":"https://hsforms.com","robots":{"status":"unreachable","http":null,"url":"https://hsforms.com/robots.txt","bytes":0,"groups":0,"error":"cannot resolve hsforms.com"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":1132,"domain":"hpe.com","origin":"https://hpe.com","robots":{"status":"unreachable","http":null,"url":"https://hpe.com/robots.txt","bytes":0,"groups":0,"error":"<StreamReset stream_id:1, error_code:2, remote_reset:True>"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":1133,"domain":"note.com","origin":"https://note.com","robots":{"status":"ok","http":200,"url":"https://note.com/robots.txt","bytes":1197,"groups":6},"bots":{"GPTBot":["partial","*",19,0],"ChatGPT-User":["partial","*",19,0],"OAI-SearchBot":["partial","*",19,0],"ClaudeBot":["partial","*",19,0],"Claude-User":["partial","*",19,0],"Claude-SearchBot":["partial","*",19,0],"anthropic-ai":["partial","*",19,0],"PerplexityBot":["partial","*",19,0],"Perplexity-User":["partial","*",19,0],"Google-Extended":["partial","*",19,0],"Applebot-Extended":["partial","*",19,0],"CCBot":["partial","*",19,0],"Bytespider":["partial","*",19,0],"Amazonbot":["partial","*",19,0],"meta-externalagent":["partial","*",19,0],"cohere-ai":["partial","*",19,0],"Diffbot":["partial","*",19,0],"ContextFetchBot":["partial","*",19,0]},"llms":{"status":"missing","http":404,"bytes":0},"home":{"status":"ok","http":200,"url":"https://note.com/","html_tokens":25025,"markdown_tokens":5,"pruned_tokens":5}},
{"rank":1134,"domain":"unity.com","origin":"https://unity.com","robots":{"status":"ok","http":200,"url":"https://unity.com/robots.txt","bytes":264,"groups":5},"bots":{"GPTBot":["partial","*",3,0],"ChatGPT-User":["partial","*",3,0],"OAI-SearchBot":["partial","*",3,0],"ClaudeBot":["partial","*",3,0],"Claude-User":["partial","*",3,0],"Claude-SearchBot":["partial","*",3,0],"anthropic-ai":["partial","*",3,0],"PerplexityBot":["partial","*",3,0],"Perplexity-User":["partial","*",3,0],"Google-Extended":["partial","*",3,0],"Applebot-Extended":["partial","*",3,0],"CCBot":["partial","*",3,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["partial","*",3,0],"meta-externalagent":["partial","*",3,0],"cohere-ai":["partial","*",3,0],"Diffbot":["partial","*",3,0],"ContextFetchBot":["partial","*",3,0]},"llms":{"status":"present","http":200,"bytes":937665},"home":{"status":"ok","http":200,"url":"https://unity.com/","html_tokens":213310,"markdown_tokens":562,"pruned_tokens":258}},
{"rank":1135,"domain":"japantimes.co.jp","origin":"https://japantimes.co.jp","robots":{"status":"ok","http":200,"url":"https://www.japantimes.co.jp/robots.txt","bytes":349,"groups":1},"bots":{"GPTBot":["partial","*",15,0],"ChatGPT-User":["partial","*",15,0],"OAI-SearchBot":["partial","*",15,0],"ClaudeBot":["partial","*",15,0],"Claude-User":["partial","*",15,0],"Claude-SearchBot":["partial","*",15,0],"anthropic-ai":["partial","*",15,0],"PerplexityBot":["partial","*",15,0],"Perplexity-User":["partial","*",15,0],"Google-Extended":["partial","*",15,0],"Applebot-Extended":["partial","*",15,0],"CCBot":["partial","*",15,0],"Bytespider":["partial","*",15,0],"Amazonbot":["partial","*",15,0],"meta-externalagent":["partial","*",15,0],"cohere-ai":["partial","*",15,0],"Diffbot":["partial","*",15,0],"ContextFetchBot":["partial","*",15,0]},"llms":{"status":"html","http":200,"bytes":411265},"home":{"status":"ok","http":200,"url":"https://www.japantimes.co.jp/","html_tokens":102727,"markdown_tokens":180,"pruned_tokens":17}},
{"rank":1136,"domain":"readthedocs.org","origin":"https://readthedocs.org","robots":{"status":"ok","http":200,"url":"https://app.readthedocs.org/robots.txt","bytes":641,"groups":2},"bots":{"GPTBot":["partial","*",6,0],"ChatGPT-User":["partial","*",6,0],"OAI-SearchBot":["partial","*",6,0],"ClaudeBot":["partial","*",6,0],"Claude-User":["partial","*",6,0],"Claude-SearchBot":["partial","*",6,0],"anthropic-ai":["partial","*",6,0],"PerplexityBot":["partial","*",6,0],"Perplexity-User":["partial","*",6,0],"Google-Extended":["partial","*",6,0],"Applebot-Extended":["partial","*",6,0],"CCBot":["partial","*",6,0],"Bytespider":["partial","*",6,0],"Amazonbot":["partial","*",6,0],"meta-externalagent":["partial","*",6,0],"cohere-ai":["partial","*",6,0],"Diffbot":["partial","*",6,0],"ContextFetchBot":["partial","*",6,0]},"llms":{"status":"missing","http":404,"bytes":23242},"home":{"status":"ok","http":200,"url":"https://about.readthedocs.com/?ref=app.readthedocs.org","html_tokens":8374,"markdown_tokens":694,"pruned_tokens":18}},
{"rank":1137,"domain":"wampserver.com","origin":"https://www.wampserver.com","robots":{"status":"missing","http":404,"url":"https://www.wampserver.com/robots.txt","bytes":146,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":146},"home":{"status":"http_error","http":404,"url":"https://www.wampserver.com/"}},
{"rank":1138,"domain":"sns.qzone.qq.com","origin":"https://sns.qzone.qq.com","robots":{"status":"missing","http":404,"url":"https://sns.qzone.qq.com/robots.txt","bytes":0,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":0},"home":{"status":"http_error","http":404,"url":"https://sns.qzone.qq.com/"}},
{"rank":1139,"domain":"gohugo.io","origin":"https://gohugo.io","robots":{"status":"missing","http":404,"url":"https://gohugo.io/robots.txt","bytes":55829,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":55829},"home":{"status":"ok","http":200,"url":"https://gohugo.io/","html_tokens":22810,"markdown_tokens":237,"pruned_tokens":149}},
{"rank":1140,"domain":"thehindu.com","origin":"https://thehindu.com","robots":{"status":"ok","http":200,"url":"https://www.thehindu.com/robots.txt","bytes":4174,"groups":3},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["blocked","named",1,0],"OAI-SearchBot":["blocked","named",1,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",96,13],"Claude-SearchBot":["partial","*",96,13],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["partial","*",96,13],"Google-Extended":["partial","*",96,13],"Applebot-Extended":["partial","*",96,13],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["partial","*",96,13],"Diffbot":["partial","*",96,13],"ContextFetchBot":["partial","*",96,13]},"llms":{"status":"missing","http":404,"bytes":75302},"home":{"status":"ok","http":200,"url":"https://www.thehindu.com/","html_tokens":82373,"markdown_tokens":175,"pruned_tokens":175}},
{"rank":1141,"domain":"gitee.com","origin":"https://gitee.com","robots":{"status":"ok","http":200,"url":"https://gitee.com/robots.txt","bytes":622,"groups":1},"bots":{"GPTBot":["partial","*",18,0],"ChatGPT-User":["partial","*",18,0],"OAI-SearchBot":["partial","*",18,0],"ClaudeBot":["partial","*",18,0],"Claude-User":["partial","*",18,0],"Claude-SearchBot":["partial","*",18,0],"anthropic-ai":["partial","*",18,0],"PerplexityBot":["partial","*",18,0],"Perplexity-User":["partial","*",18,0],"Google-Extended":["partial","*",18,0],"Applebot-Extended":["partial","*",18,0],"CCBot":["partial","*",18,0],"Bytespider":["partial","*",18,0],"Amazonbot":["partial","*",18,0],"meta-externalagent":["partial","*",18,0],"cohere-ai":["partial","*",18,0],"Diffbot":["partial","*",18,0],"ContextFetchBot":["partial","*",18,0]},"llms":{"status":"present","http":200,"bytes":6487},"home":{"status":"http_error","http":405,"url":"https://gitee.com/"}},
{"rank":1142,"domain":"unhcr.org","origin":"https://unhcr.org","robots":{"status":"refused","http":403,"url":"https://www.unhcr.org/robots.txt","bytes":182571,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":1143,"domain":"gnome.org","origin":"https://gnome.org","robots":{"status":"missing","http":404,"url":"https://www.gnome.org/robots.txt","bytes":3725,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":3725},"home":{"status":"ok","http":200,"url":"https://www.gnome.org/","html_tokens":6620,"markdown_tokens":190,"pruned_tokens":99}},
{"rank":1144,"domain":"uga.edu","origin":"https://uga.edu","robots":{"status":"ok","http":200,"url":"https://www.uga.edu/robots.txt","bytes":309,"groups":1},"bots":{"GPTBot":["partial","*",1,2],"ChatGPT-User":["partial","*",1,2],"OAI-SearchBot":["partial","*",1,2],"ClaudeBot":["partial","*",1,2],"Claude-User":["partial","*",1,2],"Claude-SearchBot":["partial","*",1,2],"anthropic-ai":["partial","*",1,2],"PerplexityBot":["partial","*",1,2],"Perplexity-User":["partial","*",1,2],"Google-Extended":["partial","*",1,2],"Applebot-Extended":["partial","*",1,2],"CCBot":["partial","*",1,2],"Bytespider":["partial","*",1,2],"Amazonbot":["partial","*",1,2],"meta-externalagent":["partial","*",1,2],"cohere-ai":["partial","*",1,2],"Diffbot":["partial","*",1,2],"ContextFetchBot":["partial","*",1,2]},"llms":{"status":"html","http":200,"bytes":3417},"home":{"status":"ok","http":200,"url":"https://www.uga.edu/","html_tokens":855,"markdown_tokens":31,"pruned_tokens":31}},
{"rank":1145,"domain":"buzzsprout.com","origin":"https://buzzsprout.com","robots":{"status":"ok","http":200,"url":"https://www.buzzsprout.com/robots.txt","bytes":36,"groups":1},"bots":{"GPTBot":["partial","*",1,0],"ChatGPT-User":["partial","*",1,0],"OAI-SearchBot":["partial","*",1,0],"ClaudeBot":["partial","*",1,0],"Claude-User":["partial","*",1,0],"Claude-SearchBot":["partial","*",1,0],"anthropic-ai":["partial","*",1,0],"PerplexityBot":["partial","*",1,0],"Perplexity-User":["partial","*",1,0],"Google-Extended":["partial","*",1,0],"Applebot-Extended":["partial","*",1,0],"CCBot":["partial","*",1,0],"Bytespider":["partial","*",1,0],"Amazonbot":["partial","*",1,0],"meta-externalagent":["partial","*",1,0],"cohere-ai":["partial","*",1,0],"Diffbot":["partial","*",1,0],"ContextFetchBot":["partial","*",1,0]},"llms":{"status":"missing","http":404,"bytes":2877},"home":{"status":"ok","http":200,"url":"https://www.buzzsprout.com/","html_tokens":78092,"markdown_tokens":331,"pruned_tokens":183}},
{"rank":1146,"domain":"connect.qq.com","origin":"https://connect.qq.com","robots":{"status":"missing","http":404,"url":"https://connect.qq.com/robots.txt","bytes":153,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":153},"home":{"status":"ok","http":200,"url":"https://connect.qq.com/","html_tokens":661,"markdown_tokens":0,"pruned_tokens":0}},
{"rank":1147,"domain":"timesofisrael.com","origin":"https://timesofisrael.com","robots":{"status":"ok","http":200,"url":"https://www.timesofisrael.com/robots.txt","bytes":1184,"groups":11},"bots":{"GPTBot":["partial","*",19,0],"ChatGPT-User":["partial","*",19,0],"OAI-SearchBot":["partial","*",19,0],"ClaudeBot":["partial","*",19,0],"Claude-User":["partial","*",19,0],"Claude-SearchBot":["partial","*",19,0],"anthropic-ai":["partial","*",19,0],"PerplexityBot":["partial","*",19,0],"Perplexity-User":["partial","*",19,0],"Google-Extended":["partial","*",19,0],"Applebot-Extended":["partial","*",19,0],"CCBot":["partial","*",19,0],"Bytespider":["partial","*",19,0],"Amazonbot":["partial","*",19,0],"meta-externalagent":["partial","*",19,0],"cohere-ai":["partial","*",19,0],"Diffbot":["partial","*",19,0],"ContextFetchBot":["partial","*",19,0]},"llms":{"status":"missing","http":404,"bytes":507},"home":{"status":"http_error","http":403,"url":"https://www.timesofisrael.com/"}},
{"rank":1148,"domain":"accuweather.com","origin":"https://accuweather.com","robots":{"status":"refused","http":403,"url":"https://accuweather.com/robots.txt","bytes":377,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":1149,"domain":"memcached.org","origin":"https://memcached.org","robots":{"status":"ok","http":200,"url":"https://memcached.org/robots.txt","bytes":204,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":169},"home":{"status":"ok","http":200,"url":"https://memcached.org/","html_tokens":1495,"markdown_tokens":256,"pruned_tokens":91}},
{"rank":1150,"domain":"globalnews.ca","origin":"https://globalnews.ca","robots":{"status":"ok","http":200,"url":"https://globalnews.ca/robots.txt","bytes":451,"groups":3},"bots":{"GPTBot":["partial","*",4,1],"ChatGPT-User":["partial","*",4,1],"OAI-SearchBot":["partial","*",4,1],"ClaudeBot":["partial","*",4,1],"Claude-User":["partial","*",4,1],"Claude-SearchBot":["partial","*",4,1],"anthropic-ai":["partial","*",4,1],"PerplexityBot":["partial","*",4,1],"Perplexity-User":["partial","*",4,1],"Google-Extended":["partial","*",4,1],"Applebot-Extended":["partial","*",4,1],"CCBot":["partial","*",4,1],"Bytespider":["partial","*",4,1],"Amazonbot":["partial","*",4,1],"meta-externalagent":["partial","*",4,1],"cohere-ai":["partial","*",4,1],"Diffbot":["partial","*",4,1],"ContextFetchBot":["partial","*",4,1]},"llms":{"status":"missing","http":404,"bytes":34945},"home":{"status":"ok","http":200,"url":"https://globalnews.ca/","html_tokens":114195,"markdown_tokens":19,"pruned_tokens":19}},
{"rank":1151,"domain":"glassdoor.com","origin":"https://glassdoor.com","robots":{"status":"ok","http":200,"url":"https://www.glassdoor.com/robots.txt","bytes":6250,"groups":15},"bots":{"GPTBot":["partial","named",4,4],"ChatGPT-User":["partial","*",141,9],"OAI-SearchBot":["partial","*",141,9],"ClaudeBot":["partial","named",4,4],"Claude-User":["partial","*",141,9],"Claude-SearchBot":["partial","*",141,9],"anthropic-ai":["partial","named",4,4],"PerplexityBot":["partial","*",141,9],"Perplexity-User":["partial","*",141,9],"Google-Extended":["partial","named",4,4],"Applebot-Extended":["partial","named",4,4],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["partial","named",4,4],"meta-externalagent":["partial","*",141,9],"cohere-ai":["partial","named",4,4],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",141,9]},"llms":{"status":"http_error","http":403,"bytes":11626},"home":{"status":"http_error","http":403,"url":"https://glassdoor.com/"}},
{"rank":1152,"domain":"wetransfer.com","origin":"https://wetransfer.com","robots":{"status":"html","http":200,"url":"https://wetransfer.com/account","bytes":28127,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":1153,"domain":"umass.edu","origin":"https://umass.edu","robots":{"status":"ok","http":200,"url":"https://www.umass.edu/robots.txt","bytes":6597,"groups":2},"bots":{"GPTBot":["partial","*",43,20],"ChatGPT-User":["partial","*",43,20],"OAI-SearchBot":["partial","*",43,20],"ClaudeBot":["partial","*",43,20],"Claude-User":["partial","*",43,20],"Claude-SearchBot":["partial","*",43,20],"anthropic-ai":["partial","*",43,20],"PerplexityBot":["partial","*",43,20],"Perplexity-User":["partial","*",43,20],"Google-Extended":["partial","*",43,20],"Applebot-Extended":["partial","*",43,20],"CCBot":["partial","*",43,20],"Bytespider":["partial","*",43,20],"Amazonbot":["partial","*",43,20],"meta-externalagent":["partial","*",43,20],"cohere-ai":["partial","*",43,20],"Diffbot":["partial","*",43,20],"ContextFetchBot":["partial","*",43,20]},"llms":{"status":"missing","http":404,"bytes":173136},"home":{"status":"ok","http":200,"url":"https://www.umass.edu/","html_tokens":59763,"markdown_tokens":487,"pruned_tokens":115}},
{"rank":1154,"domain":"uwaterloo.ca","origin":"https://uwaterloo.ca","robots":{"status":"missing","http":404,"url":"https://uwaterloo.ca/robots.txt","bytes":0,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":279},"home":{"status":"ok","http":200,"url":"https://uwaterloo.ca/","html_tokens":33730,"markdown_tokens":323,"pruned_tokens":168}},
{"rank":1155,"domain":"msdn.com","origin":"https://msdn.com","robots":{"status":"ok","http":200,"url":"https://msdn.microsoft.com/robots.txt","bytes":6716,"groups":1},"bots":{"GPTBot":["partial","*",171,125],"ChatGPT-User":["partial","*",171,125],"OAI-SearchBot":["partial","*",171,125],"ClaudeBot":["partial","*",171,125],"Claude-User":["partial","*",171,125],"Claude-SearchBot":["partial","*",171,125],"anthropic-ai":["partial","*",171,125],"PerplexityBot":["partial","*",171,125],"Perplexity-User":["partial","*",171,125],"Google-Extended":["partial","*",171,125],"Applebot-Extended":["partial","*",171,125],"CCBot":["partial","*",171,125],"Bytespider":["partial","*",171,125],"Amazonbot":["partial","*",171,125],"meta-externalagent":["partial","*",171,125],"cohere-ai":["partial","*",171,125],"Diffbot":["partial","*",171,125],"ContextFetchBot":["partial","*",171,125]},"llms":{"status":"html","http":200,"bytes":26788},"home":{"status":"ok","http":200,"url":"https://learn.microsoft.com/en-us/","html_tokens":6697,"markdown_tokens":105,"pruned_tokens":105}},
{"rank":1156,"domain":"mapquest.com","origin":"https://mapquest.com","robots":{"status":"ok","http":200,"url":"https://www.mapquest.com/robots.txt","bytes":1953,"groups":43},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["blocked","named",2,0],"OAI-SearchBot":["blocked","named",1,0],"ClaudeBot":["blocked","named",2,0],"Claude-User":["partial","*",6,0],"Claude-SearchBot":["partial","*",6,0],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["partial","*",6,0],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",2,0],"cohere-ai":["blocked","named",2,0],"Diffbot":["blocked","named",2,0],"ContextFetchBot":["partial","*",6,0]},"llms":{"status":"html","http":202,"bytes":2471},"home":{"status":"ok","http":202,"url":"https://mapquest.com/","html_tokens":618,"markdown_tokens":40,"pruned_tokens":40}},
{"rank":1157,"domain":"smugmug.com","origin":"https://smugmug.com","robots":{"status":"ok","http":200,"url":"https://www.smugmug.com/robots.txt","bytes":1662,"groups":5},"bots":{"GPTBot":["blocked","*",1,0],"ChatGPT-User":["blocked","*",1,0],"OAI-SearchBot":["blocked","*",1,0],"ClaudeBot":["blocked","*",1,0],"Claude-User":["blocked","*",1,0],"Claude-SearchBot":["blocked","*",1,0],"anthropic-ai":["blocked","*",1,0],"PerplexityBot":["blocked","*",1,0],"Perplexity-User":["blocked","*",1,0],"Google-Extended":["blocked","*",1,0],"Applebot-Extended":["blocked","*",1,0],"CCBot":["blocked","*",1,0],"Bytespider":["blocked","*",1,0],"Amazonbot":["blocked","*",1,0],"meta-externalagent":["blocked","*",1,0],"cohere-ai":["blocked","*",1,0],"Diffbot":["blocked","*",1,0],"ContextFetchBot":["blocked","*",1,0]},"llms":{"status":"disallowed"},"home":{"status":"disallowed"}},
{"rank":1158,"domain":"t-online.de","origin":"https://t-online.de","robots":{"status":"ok","http":200,"url":"https://www.t-online.de/robots.txt","bytes":2206,"groups":6},"bots":{"GPTBot":["partial","named",1,1],"ChatGPT-User":["partial","*",23,0],"OAI-SearchBot":["partial","*",23,0],"ClaudeBot":["partial","named",1,1],"Claude-User":["partial","*",23,0],"Claude-SearchBot":["partial","*",23,0],"anthropic-ai":["partial","*",23,0],"PerplexityBot":["partial","named",1,1],"Perplexity-User":["partial","*",23,0],"Google-Extended":["partial","named",23,1],"Applebot-Extended":["partial","named",1,1],"CCBot":["partial","named",1,1],"Bytespider":["partial","named",1,1],"Amazonbot":["partial","*",23,0],"meta-externalagent":["partial","named",1,1],"cohere-ai":["partial","*",23,0],"Diffbot":["partial","*",23,0],"ContextFetchBot":["partial","*",23,0]},"llms":{"status":"missing","http":404,"bytes":13425},"home":{"status":"ok","http":200,"url":"https://www.t-online.de/","html_tokens":359807,"markdown_tokens":124,"pruned_tokens":35}},
{"rank":1159,"domain":"pcre.org","origin":"https://pcre.org","robots":{"status":"ok","http":200,"url":"https://pcre.org/robots.txt","bytes":24,"groups":1},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"missing","http":404,"bytes":93},"home":{"status":"ok","http":200,"url":"https://www.pcre.org/","html_tokens":1988,"markdown_tokens":1080,"pruned_tokens":543}},
{"rank":1160,"domain":"duolingo.com","origin":"https://duolingo.com","robots":{"status":"ok","http":200,"url":"https://www.duolingo.com/robots.txt","bytes":826,"groups":3},"bots":{"GPTBot":["partial","*",25,0],"ChatGPT-User":["partial","*",25,0],"OAI-SearchBot":["partial","*",25,0],"ClaudeBot":["partial","*",25,0],"Claude-User":["partial","*",25,0],"Claude-SearchBot":["partial","*",25,0],"anthropic-ai":["partial","*",25,0],"PerplexityBot":["partial","*",25,0],"Perplexity-User":["partial","*",25,0],"Google-Extended":["partial","*",25,0],"Applebot-Extended":["partial","*",25,0],"CCBot":["partial","*",25,0],"Bytespider":["partial","*",25,0],"Amazonbot":["partial","*",25,0],"meta-externalagent":["partial","*",25,0],"cohere-ai":["partial","*",25,0],"Diffbot":["partial","*",25,0],"ContextFetchBot":["partial","*",25,0]},"llms":{"status":"html","http":200,"bytes":11579},"home":{"status":"ok","http":200,"url":"https://www.duolingo.com/","html_tokens":2820,"markdown_tokens":0,"pruned_tokens":0}},
{"rank":1161,"domain":"onelink.me","origin":"https://onelink.me","robots":{"status":"html","http":200,"url":"https://www.appsflyer.com/products/deep-linking/","bytes":428465,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":1162,"domain":"jst.go.jp","origin":"https://www.jst.go.jp","robots":{"status":"ok","http":200,"url":"https://www.jst.go.jp/robots.txt","bytes":101,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":1909},"home":{"status":"ok","http":200,"url":"https://www.jst.go.jp/","html_tokens":8724,"markdown_tokens":129,"pruned_tokens":76}},
{"rank":1163,"domain":"amazon.es","origin":"https://amazon.es","robots":{"status":"ok","http":200,"url":"https://www.amazon.es/robots.txt","bytes":7063,"groups":101},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["blocked","named",1,0],"OAI-SearchBot":["blocked","named",1,0],"ClaudeBot":["blocked","named",2,0],"Claude-User":["blocked","named",1,0],"Claude-SearchBot":["blocked","named",1,0],"anthropic-ai":["partial","*",90,8],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["blocked","named",1,0],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["partial","*",90,8],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["partial","*",90,8],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",90,8]},"llms":{"status":"missing","http":404,"bytes":2261},"home":{"status":"ok","http":200,"url":"https://www.amazon.es/-/en/","html_tokens":203692,"markdown_tokens":1011,"pruned_tokens":175}},
{"rank":1164,"domain":"directadmin.com","origin":"https://directadmin.com","robots":{"status":"ok","http":200,"url":"https://directadmin.com/robots.txt","bytes":31,"groups":1},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"crawl_delay","error":"Crawl-delay 300 s"},"home":{"status":"crawl_delay","error":"Crawl-delay 300 s"}},
{"rank":1165,"domain":"oregonstate.edu","origin":"https://oregonstate.edu","robots":{"status":"ok","http":200,"url":"https://oregonstate.edu/robots.txt","bytes":2027,"groups":1},"bots":{"GPTBot":["partial","*",34,18],"ChatGPT-User":["partial","*",34,18],"OAI-SearchBot":["partial","*",34,18],"ClaudeBot":["partial","*",34,18],"Claude-User":["partial","*",34,18],"Claude-SearchBot":["partial","*",34,18],"anthropic-ai":["partial","*",34,18],"PerplexityBot":["partial","*",34,18],"Perplexity-User":["partial","*",34,18],"Google-Extended":["partial","*",34,18],"Applebot-Extended":["partial","*",34,18],"CCBot":["partial","*",34,18],"Bytespider":["partial","*",34,18],"Amazonbot":["partial","*",34,18],"meta-externalagent":["partial","*",34,18],"cohere-ai":["partial","*",34,18],"Diffbot":["partial","*",34,18],"ContextFetchBot":["partial","*",34,18]},"llms":{"status":"missing","http":404,"bytes":189},"home":{"status":"ok","http":200,"url":"https://oregonstate.edu/","html_tokens":16632,"markdown_tokens":582,"pruned_tokens":261}},
{"rank":1166,"domain":"bcg.com","origin":"https://bcg.com","robots":{"status":"ok","http":200,"url":"https://www.bcg.com/robots.txt","bytes":414,"groups":1},"bots":{"GPTBot":["partial","*",5,11],"ChatGPT-User":["partial","*",5,11],"OAI-SearchBot":["partial","*",5,11],"ClaudeBot":["partial","*",5,11],"Claude-User":["partial","*",5,11],"Claude-SearchBot":["partial","*",5,11],"anthropic-ai":["partial","*",5,11],"PerplexityBot":["partial","*",5,11],"Perplexity-User":["partial","*",5,11],"Google-Extended":["partial","*",5,11],"Applebot-Extended":["partial","*",5,11],"CCBot":["partial","*",5,11],"Bytespider":["partial","*",5,11],"Amazonbot":["partial","*",5,11],"meta-externalagent":["partial","*",5,11],"cohere-ai":["partial","*",5,11],"Diffbot":["partial","*",5,11],"ContextFetchBot":["partial","*",5,11]},"llms":{"status":"missing","http":404,"bytes":845355},"home":{"status":"ok","http":200,"url":"https://www.bcg.com/","html_tokens":235829,"markdown_tokens":761,"pruned_tokens":121}},
{"rank":1167,"domain":"wufoo.com","origin":"https://wufoo.com","robots":{"status":"ok","http":200,"url":"https://www.wufoo.com/robots.txt","bytes":171,"groups":1},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"missing","http":404,"bytes":127245},"home":{"status":"ok","http":200,"url":"https://www.wufoo.com/","html_tokens":34554,"markdown_tokens":404,"pruned_tokens":153}},
{"rank":1168,"domain":"redbull.com","origin":"https://redbull.com","robots":{"status":"ok","http":200,"url":"https://www.redbull.com/robots.txt","bytes":540,"groups":2},"bots":{"GPTBot":["partial","*",5,2],"ChatGPT-User":["partial","*",5,2],"OAI-SearchBot":["partial","*",5,2],"ClaudeBot":["partial","*",5,2],"Claude-User":["partial","*",5,2],"Claude-SearchBot":["partial","*",5,2],"anthropic-ai":["partial","*",5,2],"PerplexityBot":["partial","*",5,2],"Perplexity-User":["partial","*",5,2],"Google-Extended":["partial","*",5,2],"Applebot-Extended":["partial","*",5,2],"CCBot":["partial","*",5,2],"Bytespider":["partial","*",5,2],"Amazonbot":["partial","*",5,2],"meta-externalagent":["partial","*",5,2],"cohere-ai":["partial","*",5,2],"Diffbot":["partial","*",5,2],"ContextFetchBot":["partial","*",5,2]},"llms":{"status":"present","http":200,"bytes":958},"home":{"status":"ok","http":200,"url":"https://www.redbull.com/us-en","html_tokens":248118,"markdown_tokens":20,"pruned_tokens":20}},
{"rank":1169,"domain":"arduino.cc","origin":"https://arduino.cc","robots":{"status":"ok","http":200,"url":"https://www.arduino.cc/robots.txt","bytes":275,"groups":1},"bots":{"GPTBot":["allowed","*",0,1],"ChatGPT-User":["allowed","*",0,1],"OAI-SearchBot":["allowed","*",0,1],"ClaudeBot":["allowed","*",0,1],"Claude-User":["allowed","*",0,1],"Claude-SearchBot":["allowed","*",0,1],"anthropic-ai":["allowed","*",0,1],"PerplexityBot":["allowed","*",0,1],"Perplexity-User":["allowed","*",0,1],"Google-Extended":["allowed","*",0,1],"Applebot-Extended":["allowed","*",0,1],"CCBot":["allowed","*",0,1],"Bytespider":["allowed","*",0,1],"Amazonbot":["allowed","*",0,1],"meta-externalagent":["allowed","*",0,1],"cohere-ai":["allowed","*",0,1],"Diffbot":["allowed","*",0,1],"ContextFetchBot":["allowed","*",0,1]},"llms":{"status":"present","http":200,"bytes":5715},"home":{"status":"ok","http":200,"url":"https://www.arduino.cc/","html_tokens":111789,"markdown_tokens":27,"pruned_tokens":27}},
{"rank":1170,"domain":"join.chat","origin":"https://join.chat","robots":{"status":"ok","http":200,"url":"https://join.chat/robots.txt","bytes":667,"groups":2},"bots":{"GPTBot":["partial","*",1,1],"ChatGPT-User":["partial","*",1,1],"OAI-SearchBot":["partial","*",1,1],"ClaudeBot":["partial","*",1,1],"Claude-User":["partial","*",1,1],"Claude-SearchBot":["partial","*",1,1],"anthropic-ai":["partial","*",1,1],"PerplexityBot":["partial","*",1,1],"Perplexity-User":["partial","*",1,1],"Google-Extended":["partial","*",1,1],"Applebot-Extended":["partial","*",1,1],"CCBot":["partial","*",1,1],"Bytespider":["partial","*",1,1],"Amazonbot":["partial","*",1,1],"meta-externalagent":["partial","*",1,1],"cohere-ai":["partial","*",1,1],"Diffbot":["partial","*",1,1],"ContextFetchBot":["partial","*",1,1]},"llms":{"status":"missing","http":404,"bytes":127204},"home":{"status":"ok","http":200,"url":"https://join.chat/en/","html_tokens":42563,"markdown_tokens":768,"pruned_tokens":119}},
{"rank":1171,"domain":"rochester.edu","origin":"https://rochester.edu","robots":{"status":"ok","http":200,"url":"https://www.rochester.edu/robots.txt","bytes":944,"groups":3},"bots":{"GPTBot":["partial","*",28,0],"ChatGPT-User":["partial","*",28,0],"OAI-SearchBot":["partial","*",28,0],"ClaudeBot":["partial","*",28,0],"Claude-User":["partial","*",28,0],"Claude-SearchBot":["partial","*",28,0],"anthropic-ai":["partial","*",28,0],"PerplexityBot":["partial","*",28,0],"Perplexity-User":["partial","*",28,0],"Google-Extended":["partial","*",28,0],"Applebot-Extended":["partial","*",28,0],"CCBot":["partial","*",28,0],"Bytespider":["partial","*",28,0],"Amazonbot":["partial","*",28,0],"meta-externalagent":["partial","*",28,0],"cohere-ai":["partial","*",28,0],"Diffbot":["partial","*",28,0],"ContextFetchBot":["partial","*",28,0]},"llms":{"status":"missing","http":404,"bytes":24901},"home":{"status":"ok","http":200,"url":"https://www.rochester.edu/","html_tokens":11898,"markdown_tokens":240,"pruned_tokens":165}},
{"rank":1172,"domain":"findlaw.com","origin":"https://findlaw.com","robots":{"status":"refused","http":403,"url":"https://findlaw.com/robots.txt","bytes":5643,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":1173,"domain":"snopes.com","origin":"https://snopes.com","robots":{"status":"ok","http":200,"url":"https://www.snopes.com/robots.txt","bytes":2471,"groups":10},"bots":{"GPTBot":["blocked","named",2,0],"ChatGPT-User":["blocked","named",1,0],"OAI-SearchBot":["allowed","*",0,1],"ClaudeBot":["blocked","named",2,0],"Claude-User":["allowed","*",0,1],"Claude-SearchBot":["allowed","*",0,1],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["allowed","*",0,1],"Google-Extended":["allowed","*",0,1],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",2,0],"Bytespider":["blocked","named",2,0],"Amazonbot":["blocked","named",2,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["allowed","*",0,1]},"llms":{"status":"missing","http":404,"bytes":518934},"home":{"status":"ok","http":200,"url":"https://www.snopes.com/","html_tokens":175186,"markdown_tokens":84,"pruned_tokens":60}},
{"rank":1174,"domain":"amazon.in","origin":"https://amazon.in","robots":{"status":"ok","http":200,"url":"https://www.amazon.in/robots.txt","bytes":9320,"groups":103},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["blocked","named",1,0],"OAI-SearchBot":["blocked","named",1,0],"ClaudeBot":["blocked","named",2,0],"Claude-User":["blocked","named",1,0],"Claude-SearchBot":["blocked","named",1,0],"anthropic-ai":["partial","*",82,8],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["blocked","named",1,0],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["partial","*",82,8],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["partial","*",82,8],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",82,8]},"llms":{"status":"missing","http":404,"bytes":2163},"home":{"status":"ok","http":200,"url":"https://www.amazon.in/","html_tokens":152735,"markdown_tokens":100,"pruned_tokens":21}},
{"rank":1175,"domain":"wp.me","origin":"https://wp.me","robots":{"status":"missing","http":404,"url":"https://wp.me/robots.txt","bytes":37,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":37},"home":{"status":"ok","http":200,"url":"https://wordpress.com/","html_tokens":85006,"markdown_tokens":155,"pruned_tokens":155}},
{"rank":1176,"domain":"iso.ch","origin":"https://iso.ch","robots":{"status":"unreachable","http":null,"url":"https://iso.ch/robots.txt","bytes":0,"groups":0,"error":"cannot resolve iso.ch"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":1177,"domain":"google.com.au","origin":"https://google.com.au","robots":{"status":"ok","http":200,"url":"https://www.google.com.au/robots.txt","bytes":6624,"groups":4},"bots":{"GPTBot":["partial","*",169,64],"ChatGPT-User":["partial","*",169,64],"OAI-SearchBot":["partial","*",169,64],"ClaudeBot":["partial","*",169,64],"Claude-User":["partial","*",169,64],"Claude-SearchBot":["partial","*",169,64],"anthropic-ai":["partial","*",169,64],"PerplexityBot":["partial","*",169,64],"Perplexity-User":["partial","*",169,64],"Google-Extended":["partial","*",169,64],"Applebot-Extended":["partial","*",169,64],"CCBot":["partial","*",169,64],"Bytespider":["partial","*",169,64],"Amazonbot":["partial","*",169,64],"meta-externalagent":["partial","*",169,64],"cohere-ai":["partial","*",169,64],"Diffbot":["partial","*",169,64],"ContextFetchBot":["partial","*",169,64]},"llms":{"status":"missing","http":404,"bytes":1569},"home":{"status":"ok","http":200,"url":"https://www.google.com/","html_tokens":21346,"markdown_tokens":30,"pruned_tokens":30}},
{"rank":1178,"domain":"klarna.com","origin":"https://klarna.com","robots":{"status":"ok","http":200,"url":"https://www.klarna.com/robots.txt","bytes":1520,"groups":1},"bots":{"GPTBot":["partial","*",40,1],"ChatGPT-User":["partial","*",40,1],"OAI-SearchBot":["partial","*",40,1],"ClaudeBot":["partial","*",40,1],"Claude-User":["partial","*",40,1],"Claude-SearchBot":["partial","*",40,1],"anthropic-ai":["partial","*",40,1],"PerplexityBot":["partial","*",40,1],"Perplexity-User":["partial","*",40,1],"Google-Extended":["partial","*",40,1],"Applebot-Extended":["partial","*",40,1],"CCBot":["partial","*",40,1],"Bytespider":["partial","*",40,1],"Amazonbot":["partial","*",40,1],"meta-externalagent":["partial","*",40,1],"cohere-ai":["partial","*",40,1],"Diffbot":["partial","*",40,1],"ContextFetchBot":["partial","*",40,1]},"llms":{"status":"present","http":200,"bytes":8503},"home":{"status":"ok","http":200,"url":"https://www.klarna.com/us/?grs=%2F&grr=empty","html_tokens":278882,"markdown_tokens":770,"pruned_tokens":20}},
{"rank":1179,"domain":"namebright.com","origin":"https://namebright.com","robots":{"status":"html","http":200,"url":"https://www.namebright.com/robots.txt","bytes":56082,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":1180,"domain":"pin.it","origin":"https://pin.it","robots":{"status":"html","http":200,"url":"https://www.pinterest.com","bytes":1282576,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":1181,"domain":"buymeacoffee.com","origin":"https://buymeacoffee.com","robots":{"status":"ok","http":200,"url":"https://buymeacoffee.com/robots.txt","bytes":140,"groups":1},"bots":{"GPTBot":["partial","*",1,3],"ChatGPT-User":["partial","*",1,3],"OAI-SearchBot":["partial","*",1,3],"ClaudeBot":["partial","*",1,3],"Claude-User":["partial","*",1,3],"Claude-SearchBot":["partial","*",1,3],"anthropic-ai":["partial","*",1,3],"PerplexityBot":["partial","*",1,3],"Perplexity-User":["partial","*",1,3],"Google-Extended":["partial","*",1,3],"Applebot-Extended":["partial","*",1,3],"CCBot":["partial","*",1,3],"Bytespider":["partial","*",1,3],"Amazonbot":["partial","*",1,3],"meta-externalagent":["partial","*",1,3],"cohere-ai":["partial","*",1,3],"Diffbot":["partial","*",1,3],"ContextFetchBot":["partial","*",1,3]},"llms":{"status":"missing","http":404,"bytes":63951},"home":{"status":"ok","http":200,"url":"https://buymeacoffee.com/","html_tokens":64640,"markdown_tokens":105,"pruned_tokens":105}},
{"rank":1182,"domain":"gzip.org","origin":"https://gzip.org","robots":{"status":"ok","http":200,"url":"https://gzip.org/robots.txt","bytes":31,"groups":1},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"missing","http":404,"bytes":355},"home":{"status":"ok","http":200,"url":"https://gzip.org/","html_tokens":689,"markdown_tokens":424,"pruned_tokens":173}},
{"rank":1183,"domain":"notion.com","origin":"https://notion.com","robots":{"status":"ok","http":200,"url":"https://www.notion.com/robots.txt","bytes":1178,"groups":8},"bots":{"GPTBot":["partial","*",10,1],"ChatGPT-User":["partial","*",10,1],"OAI-SearchBot":["partial","*",10,1],"ClaudeBot":["partial","*",10,1],"Claude-User":["partial","*",10,1],"Claude-SearchBot":["partial","*",10,1],"anthropic-ai":["partial","*",10,1],"PerplexityBot":["partial","*",10,1],"Perplexity-User":["partial","*",10,1],"Google-Extended":["partial","*",10,1],"Applebot-Extended":["partial","*",10,1],"CCBot":["partial","*",10,1],"Bytespider":["partial","*",10,1],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["partial","*",10,1],"Diffbot":["partial","*",10,1],"ContextFetchBot":["partial","*",10,1]},"llms":{"status":"present","http":200,"bytes":6803},"home":{"status":"ok","http":200,"url":"https://www.notion.com/","html_tokens":52540,"markdown_tokens":42,"pruned_tokens":11}},
{"rank":1184,"domain":"coindesk.com","origin":"https://coindesk.com","robots":{"status":"ok","http":200,"url":"https://www.coindesk.com/robots.txt","bytes":4345,"groups":22},"bots":{"GPTBot":["partial","named",1,1],"ChatGPT-User":["partial","*",42,0],"OAI-SearchBot":["partial","*",42,0],"ClaudeBot":["partial","named",1,1],"Claude-User":["partial","*",42,0],"Claude-SearchBot":["partial","*",42,0],"anthropic-ai":["partial","named",1,1],"PerplexityBot":["partial","named",1,1],"Perplexity-User":["partial","*",42,0],"Google-Extended":["partial","named",1,1],"Applebot-Extended":["partial","named",1,1],"CCBot":["partial","named",1,1],"Bytespider":["partial","named",1,1],"Amazonbot":["partial","named",1,1],"meta-externalagent":["partial","named",1,1],"cohere-ai":["partial","named",1,1],"Diffbot":["partial","named",1,1],"ContextFetchBot":["partial","*",42,0]},"llms":{"status":"present","http":200,"bytes":8186},"home":{"status":"ok","http":200,"url":"https://www.coindesk.com/","html_tokens":472197,"markdown_tokens":201,"pruned_tokens":146}},
{"rank":1185,"domain":"logitech.com","origin":"https://logitech.com","robots":{"status":"ok","http":200,"url":"https://www.logitech.com/robots.txt","bytes":413,"groups":1},"bots":{"GPTBot":["partial","*",15,1],"ChatGPT-User":["partial","*",15,1],"OAI-SearchBot":["partial","*",15,1],"ClaudeBot":["partial","*",15,1],"Claude-User":["partial","*",15,1],"Claude-SearchBot":["partial","*",15,1],"anthropic-ai":["partial","*",15,1],"PerplexityBot":["partial","*",15,1],"Perplexity-User":["partial","*",15,1],"Google-Extended":["partial","*",15,1],"Applebot-Extended":["partial","*",15,1],"CCBot":["partial","*",15,1],"Bytespider":["partial","*",15,1],"Amazonbot":["partial","*",15,1],"meta-externalagent":["partial","*",15,1],"cohere-ai":["partial","*",15,1],"Diffbot":["partial","*",15,1],"ContextFetchBot":["partial","*",15,1]},"llms":{"status":"present","http":200,"bytes":5630},"home":{"status":"ok","http":200,"url":"https://www.logitech.com/en-us","html_tokens":91941,"markdown_tokens":331,"pruned_tokens":59}},
{"rank":1186,"domain":"deepl.com","origin":"https://deepl.com","robots":{"status":"ok","http":200,"url":"https://www.deepl.com/robots.txt","bytes":1722,"groups":3},"bots":{"GPTBot":["partial","*",9,1],"ChatGPT-User":["partial","*",9,1],"OAI-SearchBot":["partial","*",9,1],"ClaudeBot":["partial","*",9,1],"Claude-User":["partial","*",9,1],"Claude-SearchBot":["partial","*",9,1],"anthropic-ai":["partial","*",9,1],"PerplexityBot":["partial","*",9,1],"Perplexity-User":["partial","*",9,1],"Google-Extended":["partial","*",9,1],"Applebot-Extended":["partial","*",9,1],"CCBot":["partial","*",9,1],"Bytespider":["partial","*",9,1],"Amazonbot":["partial","*",9,1],"meta-externalagent":["partial","*",9,1],"cohere-ai":["partial","*",9,1],"Diffbot":["partial","*",9,1],"ContextFetchBot":["partial","*",9,1]},"llms":{"status":"missing","http":404,"bytes":46992},"home":{"status":"ok","http":200,"url":"https://www.deepl.com/en","html_tokens":83427,"markdown_tokens":281,"pruned_tokens":228}},
{"rank":1187,"domain":"info.pl","origin":"https://info.pl","robots":{"status":"unreachable","http":null,"url":"https://info.pl/robots.txt","bytes":0,"groups":0,"error":"cannot resolve info.pl"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":1188,"domain":"proquest.com","origin":"https://proquest.com","robots":{"status":"ok","http":200,"url":"https://www.proquest.com/robots.txt","bytes":2491,"groups":4},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["blocked","named",1,0],"OAI-SearchBot":["allowed","named",0,1],"ClaudeBot":["partial","*",77,8],"Claude-User":["partial","*",77,8],"Claude-SearchBot":["partial","*",77,8],"anthropic-ai":["partial","*",77,8],"PerplexityBot":["partial","*",77,8],"Perplexity-User":["partial","*",77,8],"Google-Extended":["partial","*",77,8],"Applebot-Extended":["partial","*",77,8],"CCBot":["partial","*",77,8],"Bytespider":["partial","*",77,8],"Amazonbot":["partial","*",77,8],"meta-externalagent":["partial","*",77,8],"cohere-ai":["partial","*",77,8],"Diffbot":["partial","*",77,8],"ContextFetchBot":["partial","*",77,8]},"llms":{"status":"html","http":200,"bytes":124920},"home":{"status":"ok","http":200,"url":"https://www.proquest.com/","html_tokens":11312,"markdown_tokens":113,"pruned_tokens":113}},
{"rank":1189,"domain":"elmundo.es","origin":"https://elmundo.es","robots":{"status":"ok","http":200,"url":"https://www.elmundo.es/robots.txt","bytes":5156,"groups":117},"bots":{"GPTBot":["partial","named",1,2],"ChatGPT-User":["partial","named",1,2],"OAI-SearchBot":["partial","named",1,2],"ClaudeBot":["partial","*",15,1],"Claude-User":["partial","*",15,1],"Claude-SearchBot":["partial","*",15,1],"anthropic-ai":["partial","named",1,2],"PerplexityBot":["partial","named",1,2],"Perplexity-User":["partial","*",15,1],"Google-Extended":["partial","named",1,2],"Applebot-Extended":["partial","*",15,1],"CCBot":["partial","named",1,2],"Bytespider":["partial","*",15,1],"Amazonbot":["partial","*",15,1],"meta-externalagent":["partial","*",15,1],"cohere-ai":["partial","*",15,1],"Diffbot":["partial","*",15,1],"ContextFetchBot":["partial","*",15,1]},"llms":{"status":"missing","http":404,"bytes":168},"home":{"status":"ok","http":200,"url":"https://www.elmundo.es/","html_tokens":122172,"markdown_tokens":188,"pruned_tokens":59}},
{"rank":1190,"domain":"seattletimes.com","origin":"https://seattletimes.com","robots":{"status":"ok","http":200,"url":"https://www.seattletimes.com/robots.txt","bytes":3168,"groups":52},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["blocked","named",1,0],"OAI-SearchBot":["blocked","named",1,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",18,0],"Claude-SearchBot":["blocked","named",1,0],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["blocked","named",1,0],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",18,0]},"llms":{"status":"missing","http":404,"bytes":104797},"home":{"status":"ok","http":200,"url":"https://www.seattletimes.com/","html_tokens":196499,"markdown_tokens":19,"pruned_tokens":19}},
{"rank":1191,"domain":"boe.es","origin":"https://boe.es","robots":{"status":"ok","http":200,"url":"https://boe.es/robots.txt","bytes":493939,"groups":1},"bots":{"GPTBot":["partial","*",12308,4],"ChatGPT-User":["partial","*",12308,4],"OAI-SearchBot":["partial","*",12308,4],"ClaudeBot":["partial","*",12308,4],"Claude-User":["partial","*",12308,4],"Claude-SearchBot":["partial","*",12308,4],"anthropic-ai":["partial","*",12308,4],"PerplexityBot":["partial","*",12308,4],"Perplexity-User":["partial","*",12308,4],"Google-Extended":["partial","*",12308,4],"Applebot-Extended":["partial","*",12308,4],"CCBot":["partial","*",12308,4],"Bytespider":["partial","*",12308,4],"Amazonbot":["partial","*",12308,4],"meta-externalagent":["partial","*",12308,4],"cohere-ai":["partial","*",12308,4],"Diffbot":["partial","*",12308,4],"ContextFetchBot":["partial","*",12308,4]},"llms":{"status":"missing","http":404,"bytes":12243},"home":{"status":"ok","http":200,"url":"https://boe.es/","html_tokens":5793,"markdown_tokens":21,"pruned_tokens":21}},
{"rank":1192,"domain":"britishcouncil.org","origin":"https://britishcouncil.org","robots":{"status":"unreachable","http":null,"url":"https://britishcouncil.org/robots.txt","bytes":0,"groups":0,"error":"<StreamReset stream_id:1, error_code:2, remote_reset:True>"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":1193,"domain":"searchenginejournal.com","origin":"https://searchenginejournal.com","robots":{"status":"ok","http":200,"url":"https://www.searchenginejournal.com/robots.txt","bytes":2020,"groups":11},"bots":{"GPTBot":["partial","*",36,0],"ChatGPT-User":["partial","*",36,0],"OAI-SearchBot":["partial","*",36,0],"ClaudeBot":["partial","*",36,0],"Claude-User":["partial","*",36,0],"Claude-SearchBot":["partial","*",36,0],"anthropic-ai":["partial","*",36,0],"PerplexityBot":["partial","*",36,0],"Perplexity-User":["partial","*",36,0],"Google-Extended":["partial","*",36,0],"Applebot-Extended":["partial","*",36,0],"CCBot":["partial","*",36,0],"Bytespider":["partial","*",36,0],"Amazonbot":["partial","*",36,0],"meta-externalagent":["partial","*",36,0],"cohere-ai":["partial","*",36,0],"Diffbot":["partial","*",36,0],"ContextFetchBot":["partial","*",36,0]},"llms":{"status":"other_text","http":200,"bytes":36837},"home":{"status":"ok","http":200,"url":"https://www.searchenginejournal.com/","html_tokens":108014,"markdown_tokens":1527,"pruned_tokens":158}},
{"rank":1194,"domain":"siemens.de","origin":"https://siemens.de","robots":{"status":"missing","http":404,"url":"https://www.siemens.de/robots.txt","bytes":6477,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":6477},"home":{"status":"ok","http":200,"url":"https://www.siemens.com/de-de/","html_tokens":73672,"markdown_tokens":198,"pruned_tokens":48}},
{"rank":1195,"domain":"xmlsoft.org","origin":"http://xmlsoft.org","robots":{"status":"missing","http":404,"url":"http://xmlsoft.org/robots.txt","bytes":196,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":196},"home":{"status":"ok","http":200,"url":"https://gitlab.gnome.org/GNOME/libxml2/-/wikis/home","html_tokens":11175,"markdown_tokens":0,"pruned_tokens":0}},
{"rank":1196,"domain":"cuny.edu","origin":"https://cuny.edu","robots":{"status":"ok","http":200,"url":"https://cuny.edu/robots.txt","bytes":964,"groups":5},"bots":{"GPTBot":["blocked","*",1,0],"ChatGPT-User":["blocked","*",1,0],"OAI-SearchBot":["blocked","*",1,0],"ClaudeBot":["blocked","*",1,0],"Claude-User":["blocked","*",1,0],"Claude-SearchBot":["blocked","*",1,0],"anthropic-ai":["blocked","*",1,0],"PerplexityBot":["blocked","*",1,0],"Perplexity-User":["blocked","*",1,0],"Google-Extended":["blocked","*",1,0],"Applebot-Extended":["blocked","*",1,0],"CCBot":["blocked","*",1,0],"Bytespider":["blocked","*",1,0],"Amazonbot":["blocked","*",1,0],"meta-externalagent":["blocked","*",1,0],"cohere-ai":["blocked","*",1,0],"Diffbot":["blocked","*",1,0],"ContextFetchBot":["blocked","*",1,0]},"llms":{"status":"disallowed"},"home":{"status":"disallowed"}},
{"rank":1197,"domain":"theregister.co.uk","origin":"https://theregister.co.uk","robots":{"status":"ok","http":200,"url":"https://www.theregister.com/robots.txt","bytes":6886,"groups":68},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["allowed","named",0,1],"OAI-SearchBot":["allowed","named",0,1],"ClaudeBot":["blocked","named",1,0],"Claude-User":["allowed","named",0,1],"Claude-SearchBot":["allowed","named",0,1],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["allowed","named",0,1],"Perplexity-User":["partial","*",1,2],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",1,2]},"llms":{"status":"disallowed"},"home":{"status":"disallowed"}},
{"rank":1198,"domain":"rbc.ru","origin":"https://rbc.ru","robots":{"status":"ok","http":200,"url":"https://www.rbc.ru/robots.txt","bytes":4696,"groups":3},"bots":{"GPTBot":["partial","*",30,0],"ChatGPT-User":["partial","*",30,0],"OAI-SearchBot":["partial","*",30,0],"ClaudeBot":["partial","*",30,0],"Claude-User":["partial","*",30,0],"Claude-SearchBot":["partial","*",30,0],"anthropic-ai":["partial","*",30,0],"PerplexityBot":["partial","*",30,0],"Perplexity-User":["partial","*",30,0],"Google-Extended":["partial","*",30,0],"Applebot-Extended":["partial","*",30,0],"CCBot":["partial","*",30,0],"Bytespider":["partial","*",30,0],"Amazonbot":["partial","*",30,0],"meta-externalagent":["partial","*",30,0],"cohere-ai":["partial","*",30,0],"Diffbot":["partial","*",30,0],"ContextFetchBot":["partial","*",30,0]},"llms":{"status":"missing","http":404,"bytes":116792},"home":{"status":"http_error","http":401,"url":"https://www.rbc.ru/8c1a1713-f82e-4b8a-97a4-6a2f1cfa20a2/"}},
{"rank":1199,"domain":"cern.ch","origin":"https://cern.ch","robots":{"status":"missing","http":404,"url":"https://cern.ch/robots.txt","bytes":10419,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":10419},"home":{"status":"ok","http":200,"url":"https://home.cern","html_tokens":74662,"markdown_tokens":327,"pruned_tokens":113}},
{"rank":1200,"domain":"disney.com","origin":"https://disney.com","robots":{"status":"ok","http":200,"url":"https://www.disney.com/robots.txt","bytes":2814,"groups":5},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["partial","*",71,0],"OAI-SearchBot":["partial","*",71,0],"ClaudeBot":["partial","*",71,0],"Claude-User":["partial","*",71,0],"Claude-SearchBot":["partial","*",71,0],"anthropic-ai":["partial","*",71,0],"PerplexityBot":["partial","*",71,0],"Perplexity-User":["partial","*",71,0],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["partial","*",71,0],"CCBot":["partial","*",71,0],"Bytespider":["partial","*",71,0],"Amazonbot":["partial","*",71,0],"meta-externalagent":["partial","*",71,0],"cohere-ai":["partial","*",71,0],"Diffbot":["partial","*",71,0],"ContextFetchBot":["partial","*",71,0]},"llms":{"status":"missing","http":404,"bytes":116182},"home":{"status":"ok","http":200,"url":"https://www.disney.com/","html_tokens":108780,"markdown_tokens":158,"pruned_tokens":91}},
{"rank":1201,"domain":"biblegateway.com","origin":"https://biblegateway.com","robots":{"status":"ok","http":200,"url":"https://www.biblegateway.com/robots.txt","bytes":743,"groups":1},"bots":{"GPTBot":["partial","*",26,0],"ChatGPT-User":["partial","*",26,0],"OAI-SearchBot":["partial","*",26,0],"ClaudeBot":["partial","*",26,0],"Claude-User":["partial","*",26,0],"Claude-SearchBot":["partial","*",26,0],"anthropic-ai":["partial","*",26,0],"PerplexityBot":["partial","*",26,0],"Perplexity-User":["partial","*",26,0],"Google-Extended":["partial","*",26,0],"Applebot-Extended":["partial","*",26,0],"CCBot":["partial","*",26,0],"Bytespider":["partial","*",26,0],"Amazonbot":["partial","*",26,0],"meta-externalagent":["partial","*",26,0],"cohere-ai":["partial","*",26,0],"Diffbot":["partial","*",26,0],"ContextFetchBot":["partial","*",26,0]},"llms":{"status":"missing","http":404,"bytes":116419},"home":{"status":"ok","http":200,"url":"https://www.biblegateway.com/","html_tokens":39172,"markdown_tokens":226,"pruned_tokens":111}},
{"rank":1202,"domain":"freetype.org","origin":"https://freetype.org","robots":{"status":"ok","http":200,"url":"https://freetype.org/robots.txt","bytes":24,"groups":1},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"missing","http":404,"bytes":4092},"home":{"status":"ok","http":200,"url":"https://freetype.org/","html_tokens":5541,"markdown_tokens":1849,"pruned_tokens":231}},
{"rank":1203,"domain":"discovery.com","origin":"https://discovery.com","robots":{"status":"ok","http":200,"url":"https://www.discovery.com/robots.txt","bytes":533,"groups":3},"bots":{"GPTBot":["partial","*",6,5],"ChatGPT-User":["partial","*",6,5],"OAI-SearchBot":["partial","*",6,5],"ClaudeBot":["partial","*",6,5],"Claude-User":["partial","*",6,5],"Claude-SearchBot":["partial","*",6,5],"anthropic-ai":["partial","*",6,5],"PerplexityBot":["partial","*",6,5],"Perplexity-User":["partial","*",6,5],"Google-Extended":["partial","*",6,5],"Applebot-Extended":["partial","*",6,5],"CCBot":["partial","*",6,5],"Bytespider":["partial","*",6,5],"Amazonbot":["partial","*",6,5],"meta-externalagent":["partial","*",6,5],"cohere-ai":["partial","*",6,5],"Diffbot":["partial","*",6,5],"ContextFetchBot":["partial","*",6,5]},"llms":{"status":"missing","http":404,"bytes":164547},"home":{"status":"ok","http":200,"url":"https://www.discovery.com/?aid=610757&cid=5963796&crid=351793798","html_tokens":45766,"markdown_tokens":69,"pruned_tokens":69}},
{"rank":1204,"domain":"rottentomatoes.com","origin":"https://rottentomatoes.com","robots":{"status":"ok","http":200,"url":"https://www.rottentomatoes.com/robots.txt","bytes":239,"groups":2},"bots":{"GPTBot":["partial","*",5,0],"ChatGPT-User":["partial","*",5,0],"OAI-SearchBot":["partial","*",5,0],"ClaudeBot":["partial","*",5,0],"Claude-User":["partial","*",5,0],"Claude-SearchBot":["partial","*",5,0],"anthropic-ai":["partial","*",5,0],"PerplexityBot":["partial","*",5,0],"Perplexity-User":["partial","*",5,0],"Google-Extended":["partial","*",5,0],"Applebot-Extended":["partial","*",5,0],"CCBot":["partial","*",5,0],"Bytespider":["partial","*",5,0],"Amazonbot":["partial","*",5,0],"meta-externalagent":["partial","*",5,0],"cohere-ai":["partial","*",5,0],"Diffbot":["partial","*",5,0],"ContextFetchBot":["partial","*",5,0]},"llms":{"status":"missing","http":404,"bytes":42995},"home":{"status":"ok","http":200,"url":"https://www.rottentomatoes.com/","html_tokens":150010,"markdown_tokens":181,"pruned_tokens":103}},
{"rank":1205,"domain":"uefa.com","origin":"https://uefa.com","robots":{"status":"ok","http":200,"url":"https://www.uefa.com/robots.txt","bytes":540,"groups":1},"bots":{"GPTBot":["partial","*",9,0],"ChatGPT-User":["partial","*",9,0],"OAI-SearchBot":["partial","*",9,0],"ClaudeBot":["partial","*",9,0],"Claude-User":["partial","*",9,0],"Claude-SearchBot":["partial","*",9,0],"anthropic-ai":["partial","*",9,0],"PerplexityBot":["partial","*",9,0],"Perplexity-User":["partial","*",9,0],"Google-Extended":["partial","*",9,0],"Applebot-Extended":["partial","*",9,0],"CCBot":["partial","*",9,0],"Bytespider":["partial","*",9,0],"Amazonbot":["partial","*",9,0],"meta-externalagent":["partial","*",9,0],"cohere-ai":["partial","*",9,0],"Diffbot":["partial","*",9,0],"ContextFetchBot":["partial","*",9,0]},"llms":{"status":"missing","http":404,"bytes":105445},"home":{"status":"ok","http":200,"url":"https://www.uefa.com","html_tokens":95443,"markdown_tokens":175,"pruned_tokens":175}},
{"rank":1206,"domain":"ria.ru","origin":"https://ria.ru","robots":{"status":"ok","http":200,"url":"https://ria.ru/robots.txt","bytes":2216,"groups":2},"bots":{"GPTBot":["partial","*",18,2],"ChatGPT-User":["partial","*",18,2],"OAI-SearchBot":["partial","*",18,2],"ClaudeBot":["partial","*",18,2],"Claude-User":["partial","*",18,2],"Claude-SearchBot":["partial","*",18,2],"anthropic-ai":["partial","*",18,2],"PerplexityBot":["partial","*",18,2],"Perplexity-User":["partial","*",18,2],"Google-Extended":["partial","*",18,2],"Applebot-Extended":["partial","*",18,2],"CCBot":["partial","*",18,2],"Bytespider":["partial","*",18,2],"Amazonbot":["partial","*",18,2],"meta-externalagent":["partial","*",18,2],"cohere-ai":["partial","*",18,2],"Diffbot":["partial","*",18,2],"ContextFetchBot":["partial","*",18,2]},"llms":{"status":"missing","http":404,"bytes":99838},"home":{"status":"ok","http":200,"url":"https://ria.ru/","html_tokens":43881,"markdown_tokens":24,"pruned_tokens":24}},
{"rank":1207,"domain":"kadencewp.com","origin":"https://kadencewp.com","robots":{"status":"ok","http":200,"url":"https://www.kadencewp.com/robots.txt","bytes":754,"groups":2},"bots":{"GPTBot":["partial","*",5,0],"ChatGPT-User":["partial","*",5,0],"OAI-SearchBot":["partial","*",5,0],"ClaudeBot":["partial","*",5,0],"Claude-User":["partial","*",5,0],"Claude-SearchBot":["partial","*",5,0],"anthropic-ai":["partial","*",5,0],"PerplexityBot":["partial","*",5,0],"Perplexity-User":["partial","*",5,0],"Google-Extended":["partial","*",5,0],"Applebot-Extended":["partial","*",5,0],"CCBot":["partial","*",5,0],"Bytespider":["partial","*",5,0],"Amazonbot":["partial","*",5,0],"meta-externalagent":["partial","*",5,0],"cohere-ai":["partial","*",5,0],"Diffbot":["partial","*",5,0],"ContextFetchBot":["partial","*",5,0]},"llms":{"status":"missing","http":404,"bytes":232973},"home":{"status":"ok","http":200,"url":"https://www.liquidweb.com/software/kadence/","html_tokens":463742,"markdown_tokens":1302,"pruned_tokens":202}},
{"rank":1208,"domain":"brevo.com","origin":"https://brevo.com","robots":{"status":"ok","http":200,"url":"https://www.brevo.com/robots.txt","bytes":1142,"groups":5},"bots":{"GPTBot":["partial","*",10,0],"ChatGPT-User":["partial","*",10,0],"OAI-SearchBot":["partial","*",10,0],"ClaudeBot":["partial","*",10,0],"Claude-User":["partial","*",10,0],"Claude-SearchBot":["partial","*",10,0],"anthropic-ai":["partial","*",10,0],"PerplexityBot":["partial","*",10,0],"Perplexity-User":["partial","*",10,0],"Google-Extended":["partial","*",10,0],"Applebot-Extended":["partial","*",10,0],"CCBot":["partial","*",10,0],"Bytespider":["partial","*",10,0],"Amazonbot":["partial","*",10,0],"meta-externalagent":["partial","*",10,0],"cohere-ai":["partial","*",10,0],"Diffbot":["partial","*",10,0],"ContextFetchBot":["partial","*",10,0]},"llms":{"status":"present","http":200,"bytes":22165},"home":{"status":"ok","http":200,"url":"https://www.brevo.com/","html_tokens":137677,"markdown_tokens":310,"pruned_tokens":178}},
{"rank":1209,"domain":"ustream.tv","origin":"https://ustream.tv","robots":{"status":"ok","http":200,"url":"https://video.ibm.com/robots.txt","bytes":91,"groups":1},"bots":{"GPTBot":["partial","*",3,1],"ChatGPT-User":["partial","*",3,1],"OAI-SearchBot":["partial","*",3,1],"ClaudeBot":["partial","*",3,1],"Claude-User":["partial","*",3,1],"Claude-SearchBot":["partial","*",3,1],"anthropic-ai":["partial","*",3,1],"PerplexityBot":["partial","*",3,1],"Perplexity-User":["partial","*",3,1],"Google-Extended":["partial","*",3,1],"Applebot-Extended":["partial","*",3,1],"CCBot":["partial","*",3,1],"Bytespider":["partial","*",3,1],"Amazonbot":["partial","*",3,1],"meta-externalagent":["partial","*",3,1],"cohere-ai":["partial","*",3,1],"Diffbot":["partial","*",3,1],"ContextFetchBot":["partial","*",3,1]},"llms":{"status":"missing","http":404,"bytes":22169},"home":{"status":"ok","http":200,"url":"https://video.ibm.com/","html_tokens":2300,"markdown_tokens":0,"pruned_tokens":0}},
{"rank":1210,"domain":"unam.mx","origin":"https://www.unam.mx","robots":{"status":"refused","http":403,"url":"https://www.unam.mx/robots.txt","bytes":4901,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":1211,"domain":"openoffice.org","origin":"https://openoffice.org","robots":{"status":"ok","http":200,"url":"https://www.openoffice.org/robots.txt","bytes":943,"groups":5},"bots":{"GPTBot":["partial","*",35,0],"ChatGPT-User":["partial","*",35,0],"OAI-SearchBot":["partial","*",35,0],"ClaudeBot":["partial","*",35,0],"Claude-User":["partial","*",35,0],"Claude-SearchBot":["partial","*",35,0],"anthropic-ai":["partial","*",35,0],"PerplexityBot":["partial","*",35,0],"Perplexity-User":["partial","*",35,0],"Google-Extended":["partial","*",35,0],"Applebot-Extended":["partial","*",35,0],"CCBot":["partial","*",35,0],"Bytespider":["partial","*",35,0],"Amazonbot":["partial","*",35,0],"meta-externalagent":["partial","*",35,0],"cohere-ai":["partial","*",35,0],"Diffbot":["partial","*",35,0],"ContextFetchBot":["partial","*",35,0]},"llms":{"status":"missing","http":404,"bytes":8994},"home":{"status":"ok","http":200,"url":"https://www.openoffice.org/","html_tokens":10943,"markdown_tokens":2335,"pruned_tokens":1291}},
{"rank":1212,"domain":"faa.gov","origin":"https://faa.gov","robots":{"status":"refused","http":403,"url":"http://www.faa.gov/robots.txt","bytes":379,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":1213,"domain":"ctvnews.ca","origin":"https://ctvnews.ca","robots":{"status":"ok","http":200,"url":"https://www.ctvnews.ca/robots.txt","bytes":1634,"groups":12},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["blocked","named",1,0],"OAI-SearchBot":["blocked","named",1,0],"ClaudeBot":["partial","*",27,1],"Claude-User":["partial","*",27,1],"Claude-SearchBot":["partial","*",27,1],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["partial","*",27,1],"Google-Extended":["partial","*",27,1],"Applebot-Extended":["partial","*",27,1],"CCBot":["blocked","named",1,0],"Bytespider":["partial","*",27,1],"Amazonbot":["partial","*",27,1],"meta-externalagent":["partial","*",27,1],"cohere-ai":["partial","*",27,1],"Diffbot":["partial","*",27,1],"ContextFetchBot":["partial","*",27,1]},"llms":{"status":"missing","http":404,"bytes":140431},"home":{"status":"ok","http":200,"url":"https://www.ctvnews.ca/","html_tokens":137445,"markdown_tokens":98,"pruned_tokens":98}},
{"rank":1214,"domain":"sqlite.org","origin":"https://sqlite.org","robots":{"status":"ok","http":200,"url":"https://sqlite.org/robots.txt","bytes":100,"groups":1},"bots":{"GPTBot":["partial","*",5,0],"ChatGPT-User":["partial","*",5,0],"OAI-SearchBot":["partial","*",5,0],"ClaudeBot":["partial","*",5,0],"Claude-User":["partial","*",5,0],"Claude-SearchBot":["partial","*",5,0],"anthropic-ai":["partial","*",5,0],"PerplexityBot":["partial","*",5,0],"Perplexity-User":["partial","*",5,0],"Google-Extended":["partial","*",5,0],"Applebot-Extended":["partial","*",5,0],"CCBot":["partial","*",5,0],"Bytespider":["partial","*",5,0],"Amazonbot":["partial","*",5,0],"meta-externalagent":["partial","*",5,0],"cohere-ai":["partial","*",5,0],"Diffbot":["partial","*",5,0],"ContextFetchBot":["partial","*",5,0]},"llms":{"status":"missing","http":404,"bytes":161},"home":{"status":"ok","http":200,"url":"https://sqlite.org/","html_tokens":2252,"markdown_tokens":145,"pruned_tokens":145}},
{"rank":1215,"domain":"iastate.edu","origin":"https://iastate.edu","robots":{"status":"ok","http":200,"url":"https://www.iastate.edu/robots.txt","bytes":2141,"groups":1},"bots":{"GPTBot":["partial","*",35,22],"ChatGPT-User":["partial","*",35,22],"OAI-SearchBot":["partial","*",35,22],"ClaudeBot":["partial","*",35,22],"Claude-User":["partial","*",35,22],"Claude-SearchBot":["partial","*",35,22],"anthropic-ai":["partial","*",35,22],"PerplexityBot":["partial","*",35,22],"Perplexity-User":["partial","*",35,22],"Google-Extended":["partial","*",35,22],"Applebot-Extended":["partial","*",35,22],"CCBot":["partial","*",35,22],"Bytespider":["partial","*",35,22],"Amazonbot":["partial","*",35,22],"meta-externalagent":["partial","*",35,22],"cohere-ai":["partial","*",35,22],"Diffbot":["partial","*",35,22],"ContextFetchBot":["partial","*",35,22]},"llms":{"status":"missing","http":404,"bytes":34414},"home":{"status":"ok","http":200,"url":"https://www.iastate.edu/","html_tokens":14897,"markdown_tokens":271,"pruned_tokens":102}},
{"rank":1216,"domain":"redcross.org","origin":"https://redcross.org","robots":{"status":"refused","http":403,"url":"https://www.redcross.org/robots.txt","bytes":380,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":1217,"domain":"se.com","origin":"https://se.com","robots":{"status":"ok","http":200,"url":"https://www.se.com/robots.txt","bytes":46313,"groups":1},"bots":{"GPTBot":["partial","*",69,86],"ChatGPT-User":["partial","*",69,86],"OAI-SearchBot":["partial","*",69,86],"ClaudeBot":["partial","*",69,86],"Claude-User":["partial","*",69,86],"Claude-SearchBot":["partial","*",69,86],"anthropic-ai":["partial","*",69,86],"PerplexityBot":["partial","*",69,86],"Perplexity-User":["partial","*",69,86],"Google-Extended":["partial","*",69,86],"Applebot-Extended":["partial","*",69,86],"CCBot":["partial","*",69,86],"Bytespider":["partial","*",69,86],"Amazonbot":["partial","*",69,86],"meta-externalagent":["partial","*",69,86],"cohere-ai":["partial","*",69,86],"Diffbot":["partial","*",69,86],"ContextFetchBot":["partial","*",69,86]},"llms":{"status":"missing","http":404,"bytes":355},"home":{"status":"ok","http":200,"url":"https://www.se.com/ww/en/","html_tokens":495767,"markdown_tokens":427,"pruned_tokens":69}},
{"rank":1218,"domain":"china.com.cn","origin":"https://www.china.com.cn","robots":{"status":"ok","http":200,"url":"http://www.china.com.cn/robots.txt","bytes":84,"groups":2},"bots":{"GPTBot":["allowed","*",0,1],"ChatGPT-User":["allowed","*",0,1],"OAI-SearchBot":["allowed","*",0,1],"ClaudeBot":["allowed","*",0,1],"Claude-User":["allowed","*",0,1],"Claude-SearchBot":["allowed","*",0,1],"anthropic-ai":["allowed","*",0,1],"PerplexityBot":["allowed","*",0,1],"Perplexity-User":["allowed","*",0,1],"Google-Extended":["allowed","*",0,1],"Applebot-Extended":["allowed","*",0,1],"CCBot":["allowed","*",0,1],"Bytespider":["allowed","*",0,1],"Amazonbot":["allowed","*",0,1],"meta-externalagent":["allowed","*",0,1],"cohere-ai":["allowed","*",0,1],"Diffbot":["allowed","*",0,1],"ContextFetchBot":["allowed","*",0,1]},"llms":{"status":"missing","http":404,"bytes":206},"home":{"status":"ok","http":200,"url":"http://www.china.com.cn/","html_tokens":21601,"markdown_tokens":37,"pruned_tokens":37}},
{"rank":1219,"domain":"rs6.net","origin":"https://rs6.net","robots":{"status":"ok","http":200,"url":"https://rs6.net/robots.txt","bytes":26,"groups":1},"bots":{"GPTBot":["blocked","*",1,0],"ChatGPT-User":["blocked","*",1,0],"OAI-SearchBot":["blocked","*",1,0],"ClaudeBot":["blocked","*",1,0],"Claude-User":["blocked","*",1,0],"Claude-SearchBot":["blocked","*",1,0],"anthropic-ai":["blocked","*",1,0],"PerplexityBot":["blocked","*",1,0],"Perplexity-User":["blocked","*",1,0],"Google-Extended":["blocked","*",1,0],"Applebot-Extended":["blocked","*",1,0],"CCBot":["blocked","*",1,0],"Bytespider":["blocked","*",1,0],"Amazonbot":["blocked","*",1,0],"meta-externalagent":["blocked","*",1,0],"cohere-ai":["blocked","*",1,0],"Diffbot":["blocked","*",1,0],"ContextFetchBot":["blocked","*",1,0]},"llms":{"status":"disallowed"},"home":{"status":"disallowed"}},
{"rank":1220,"domain":"virustotal.com","origin":"https://virustotal.com","robots":{"status":"ok","http":200,"url":"https://virustotal.com/robots.txt","bytes":256,"groups":1},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":4161},"home":{"status":"ok","http":200,"url":"https://www.virustotal.com/gui/","html_tokens":1834,"markdown_tokens":12,"pruned_tokens":12}},
{"rank":1221,"domain":"oxfordjournals.org","origin":"https://oxfordjournals.org","robots":{"status":"ok","http":200,"url":"https://oxfordjournals.org/robots.txt","bytes":4269,"groups":2},"bots":{"GPTBot":["partial","*",61,3],"ChatGPT-User":["partial","*",61,3],"OAI-SearchBot":["partial","*",61,3],"ClaudeBot":["partial","*",61,3],"Claude-User":["partial","*",61,3],"Claude-SearchBot":["partial","*",61,3],"anthropic-ai":["partial","*",61,3],"PerplexityBot":["partial","*",61,3],"Perplexity-User":["partial","*",61,3],"Google-Extended":["partial","*",61,3],"Applebot-Extended":["partial","*",61,3],"CCBot":["partial","*",61,3],"Bytespider":["partial","*",61,3],"Amazonbot":["partial","*",61,3],"meta-externalagent":["partial","*",61,3],"cohere-ai":["partial","*",61,3],"Diffbot":["partial","*",61,3],"ContextFetchBot":["partial","*",61,3]},"llms":{"status":"http_error","http":403,"bytes":5536},"home":{"status":"http_error","http":403,"url":"https://academic.oup.com/[Journals:]"}},
{"rank":1222,"domain":"codepen.io","origin":"https://codepen.io","robots":{"status":"refused","http":403,"url":"https://codepen.io/robots.txt","bytes":5483,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":1223,"domain":"raspberrypi.org","origin":"https://raspberrypi.org","robots":{"status":"ok","http":200,"url":"https://www.raspberrypi.org/robots.txt","bytes":165,"groups":1},"bots":{"GPTBot":["partial","*",3,1],"ChatGPT-User":["partial","*",3,1],"OAI-SearchBot":["partial","*",3,1],"ClaudeBot":["partial","*",3,1],"Claude-User":["partial","*",3,1],"Claude-SearchBot":["partial","*",3,1],"anthropic-ai":["partial","*",3,1],"PerplexityBot":["partial","*",3,1],"Perplexity-User":["partial","*",3,1],"Google-Extended":["partial","*",3,1],"Applebot-Extended":["partial","*",3,1],"CCBot":["partial","*",3,1],"Bytespider":["partial","*",3,1],"Amazonbot":["partial","*",3,1],"meta-externalagent":["partial","*",3,1],"cohere-ai":["partial","*",3,1],"Diffbot":["partial","*",3,1],"ContextFetchBot":["partial","*",3,1]},"llms":{"status":"missing","http":404,"bytes":92530},"home":{"status":"ok","http":200,"url":"https://www.raspberrypi.org/","html_tokens":18259,"markdown_tokens":403,"pruned_tokens":49}},
{"rank":1224,"domain":"habr.com","origin":"https://habr.com","robots":{"status":"ok","http":200,"url":"https://habr.com/robots.txt","bytes":1624,"groups":4},"bots":{"GPTBot":["partial","*",12,0],"ChatGPT-User":["partial","*",12,0],"OAI-SearchBot":["partial","*",12,0],"ClaudeBot":["partial","*",12,0],"Claude-User":["partial","*",12,0],"Claude-SearchBot":["partial","*",12,0],"anthropic-ai":["partial","*",12,0],"PerplexityBot":["partial","*",12,0],"Perplexity-User":["partial","*",12,0],"Google-Extended":["partial","*",12,0],"Applebot-Extended":["partial","*",12,0],"CCBot":["partial","*",12,0],"Bytespider":["partial","*",12,0],"Amazonbot":["partial","*",12,0],"meta-externalagent":["partial","*",12,0],"cohere-ai":["partial","*",12,0],"Diffbot":["partial","*",12,0],"ContextFetchBot":["partial","*",12,0]},"llms":{"status":"timeout","error":"timed out after 10 s"},"home":{"status":"ok","http":200,"url":"https://habr.com/en/feed/","html_tokens":66034,"markdown_tokens":98,"pruned_tokens":98}},
{"rank":1225,"domain":"answers.com","origin":"https://answers.com","robots":{"status":"ok","http":200,"url":"https://www.answers.com/robots.txt","bytes":2224,"groups":11},"bots":{"GPTBot":["partial","*",9,2],"ChatGPT-User":["partial","*",9,2],"OAI-SearchBot":["partial","*",9,2],"ClaudeBot":["partial","*",9,2],"Claude-User":["partial","*",9,2],"Claude-SearchBot":["partial","*",9,2],"anthropic-ai":["partial","*",9,2],"PerplexityBot":["partial","*",9,2],"Perplexity-User":["partial","*",9,2],"Google-Extended":["partial","*",9,2],"Applebot-Extended":["partial","*",9,2],"CCBot":["partial","*",9,2],"Bytespider":["partial","*",9,2],"Amazonbot":["partial","*",9,2],"meta-externalagent":["partial","*",9,2],"cohere-ai":["partial","*",9,2],"Diffbot":["partial","*",9,2],"ContextFetchBot":["partial","*",9,2]},"llms":{"status":"missing","http":404,"bytes":150657},"home":{"status":"ok","http":200,"url":"https://www.answers.com/","html_tokens":95855,"markdown_tokens":1685,"pruned_tokens":129}},
{"rank":1226,"domain":"chromium.org","origin":"https://chromium.org","robots":{"status":"ok","http":200,"url":"https://www.chromium.org/robots.txt","bytes":71,"groups":1},"bots":{"GPTBot":["allowed","*",0,1],"ChatGPT-User":["allowed","*",0,1],"OAI-SearchBot":["allowed","*",0,1],"ClaudeBot":["allowed","*",0,1],"Claude-User":["allowed","*",0,1],"Claude-SearchBot":["allowed","*",0,1],"anthropic-ai":["allowed","*",0,1],"PerplexityBot":["allowed","*",0,1],"Perplexity-User":["allowed","*",0,1],"Google-Extended":["allowed","*",0,1],"Applebot-Extended":["allowed","*",0,1],"CCBot":["allowed","*",0,1],"Bytespider":["allowed","*",0,1],"Amazonbot":["allowed","*",0,1],"meta-externalagent":["allowed","*",0,1],"cohere-ai":["allowed","*",0,1],"Diffbot":["allowed","*",0,1],"ContextFetchBot":["allowed","*",0,1]},"llms":{"status":"missing","http":404,"bytes":21376},"home":{"status":"ok","http":200,"url":"https://www.chromium.org/","html_tokens":31,"markdown_tokens":0,"pruned_tokens":0}},
{"rank":1227,"domain":"onlamp.com","origin":"https://onlamp.com","robots":{"status":"ok","http":200,"url":"https://www.oreilly.com/robots.txt","bytes":3450,"groups":7},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["partial","named",1,1],"OAI-SearchBot":["partial","named",1,1],"ClaudeBot":["partial","named",1,1],"Claude-User":["partial","*",19,0],"Claude-SearchBot":["partial","named",1,1],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["partial","named",1,1],"Perplexity-User":["partial","named",1,1],"Google-Extended":["partial","named",1,1],"Applebot-Extended":["partial","named",1,1],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",19,0]},"llms":{"status":"html","http":200,"bytes":256375},"home":{"status":"ok","http":200,"url":"https://www.oreilly.com/radar/","html_tokens":64085,"markdown_tokens":46,"pruned_tokens":46}},
{"rank":1228,"domain":"buffer.com","origin":"https://buffer.com","robots":{"status":"ok","http":200,"url":"https://buffer.com/robots.txt","bytes":498,"groups":2},"bots":{"GPTBot":["partial","*",5,0],"ChatGPT-User":["partial","*",5,0],"OAI-SearchBot":["partial","*",5,0],"ClaudeBot":["partial","*",5,0],"Claude-User":["partial","*",5,0],"Claude-SearchBot":["partial","*",5,0],"anthropic-ai":["partial","*",5,0],"PerplexityBot":["partial","*",5,0],"Perplexity-User":["partial","*",5,0],"Google-Extended":["partial","*",5,0],"Applebot-Extended":["partial","*",5,0],"CCBot":["partial","*",5,0],"Bytespider":["partial","*",5,0],"Amazonbot":["partial","*",5,0],"meta-externalagent":["partial","*",5,0],"cohere-ai":["partial","*",5,0],"Diffbot":["partial","*",5,0],"ContextFetchBot":["partial","*",5,0]},"llms":{"status":"present","http":200,"bytes":10846},"home":{"status":"ok","http":200,"url":"https://buffer.com/","html_tokens":153803,"markdown_tokens":576,"pruned_tokens":109}},
{"rank":1229,"domain":"unicode.org","origin":"https://unicode.org","robots":{"status":"ok","http":200,"url":"https://unicode.org/robots.txt","bytes":17938,"groups":4},"bots":{"GPTBot":["partial","*",411,3],"ChatGPT-User":["partial","*",411,3],"OAI-SearchBot":["partial","*",411,3],"ClaudeBot":["partial","*",411,3],"Claude-User":["partial","*",411,3],"Claude-SearchBot":["partial","*",411,3],"anthropic-ai":["partial","*",411,3],"PerplexityBot":["partial","*",411,3],"Perplexity-User":["partial","*",411,3],"Google-Extended":["partial","*",411,3],"Applebot-Extended":["partial","*",411,3],"CCBot":["blocked","named",2,0],"Bytespider":["partial","*",411,3],"Amazonbot":["partial","*",411,3],"meta-externalagent":["partial","*",411,3],"cohere-ai":["partial","*",411,3],"Diffbot":["partial","*",411,3],"ContextFetchBot":["partial","*",411,3]},"llms":{"status":"missing","http":404,"bytes":196},"home":{"status":"ok","http":200,"url":"https://unicode.org/","html_tokens":55,"markdown_tokens":11,"pruned_tokens":11}},
{"rank":1230,"domain":"awin1.com","origin":"https://awin1.com","robots":{"status":"ok","http":200,"url":"https://awin1.com/robots.txt","bytes":331,"groups":2},"bots":{"GPTBot":["partial","*",8,0],"ChatGPT-User":["partial","*",8,0],"OAI-SearchBot":["partial","*",8,0],"ClaudeBot":["partial","*",8,0],"Claude-User":["partial","*",8,0],"Claude-SearchBot":["partial","*",8,0],"anthropic-ai":["partial","*",8,0],"PerplexityBot":["partial","*",8,0],"Perplexity-User":["partial","*",8,0],"Google-Extended":["partial","*",8,0],"Applebot-Extended":["partial","*",8,0],"CCBot":["partial","*",8,0],"Bytespider":["partial","*",8,0],"Amazonbot":["partial","*",8,0],"meta-externalagent":["partial","*",8,0],"cohere-ai":["partial","*",8,0],"Diffbot":["partial","*",8,0],"ContextFetchBot":["partial","*",8,0]},"llms":{"status":"timeout","error":"timed out after 10 s"},"home":{"status":"ok","http":200,"url":"https://www.awin.com/","html_tokens":74125,"markdown_tokens":946,"pruned_tokens":106}},
{"rank":1231,"domain":"macrumors.com","origin":"https://macrumors.com","robots":{"status":"ok","http":200,"url":"https://www.macrumors.com/robots.txt","bytes":155,"groups":1},"bots":{"GPTBot":["partial","*",4,0],"ChatGPT-User":["partial","*",4,0],"OAI-SearchBot":["partial","*",4,0],"ClaudeBot":["partial","*",4,0],"Claude-User":["partial","*",4,0],"Claude-SearchBot":["partial","*",4,0],"anthropic-ai":["partial","*",4,0],"PerplexityBot":["partial","*",4,0],"Perplexity-User":["partial","*",4,0],"Google-Extended":["partial","*",4,0],"Applebot-Extended":["partial","*",4,0],"CCBot":["partial","*",4,0],"Bytespider":["partial","*",4,0],"Amazonbot":["partial","*",4,0],"meta-externalagent":["partial","*",4,0],"cohere-ai":["partial","*",4,0],"Diffbot":["partial","*",4,0],"ContextFetchBot":["partial","*",4,0]},"llms":{"status":"missing","http":404,"bytes":104480},"home":{"status":"ok","http":200,"url":"https://www.macrumors.com/","html_tokens":100705,"markdown_tokens":1617,"pruned_tokens":120}},
{"rank":1232,"domain":"blender.org","origin":"https://blender.org","robots":{"status":"ok","http":200,"url":"https://www.blender.org/robots.txt","bytes":116,"groups":1},"bots":{"GPTBot":["partial","*",1,1],"ChatGPT-User":["partial","*",1,1],"OAI-SearchBot":["partial","*",1,1],"ClaudeBot":["partial","*",1,1],"Claude-User":["partial","*",1,1],"Claude-SearchBot":["partial","*",1,1],"anthropic-ai":["partial","*",1,1],"PerplexityBot":["partial","*",1,1],"Perplexity-User":["partial","*",1,1],"Google-Extended":["partial","*",1,1],"Applebot-Extended":["partial","*",1,1],"CCBot":["partial","*",1,1],"Bytespider":["partial","*",1,1],"Amazonbot":["partial","*",1,1],"meta-externalagent":["partial","*",1,1],"cohere-ai":["partial","*",1,1],"Diffbot":["partial","*",1,1],"ContextFetchBot":["partial","*",1,1]},"llms":{"status":"missing","http":404,"bytes":75828},"home":{"status":"ok","http":200,"url":"https://www.blender.org/","html_tokens":29225,"markdown_tokens":709,"pruned_tokens":293}},
{"rank":1233,"domain":"uiowa.edu","origin":"https://uiowa.edu","robots":{"status":"ok","http":200,"url":"https://uiowa.edu/robots.txt","bytes":1900,"groups":1},"bots":{"GPTBot":["partial","*",30,18],"ChatGPT-User":["partial","*",30,18],"OAI-SearchBot":["partial","*",30,18],"ClaudeBot":["partial","*",30,18],"Claude-User":["partial","*",30,18],"Claude-SearchBot":["partial","*",30,18],"anthropic-ai":["partial","*",30,18],"PerplexityBot":["partial","*",30,18],"Perplexity-User":["partial","*",30,18],"Google-Extended":["partial","*",30,18],"Applebot-Extended":["partial","*",30,18],"CCBot":["partial","*",30,18],"Bytespider":["partial","*",30,18],"Amazonbot":["partial","*",30,18],"meta-externalagent":["partial","*",30,18],"cohere-ai":["partial","*",30,18],"Diffbot":["partial","*",30,18],"ContextFetchBot":["partial","*",30,18]},"llms":{"status":"missing","http":404,"bytes":183},"home":{"status":"ok","http":200,"url":"https://uiowa.edu/","html_tokens":25008,"markdown_tokens":99,"pruned_tokens":99}},
{"rank":1234,"domain":"cnrs.fr","origin":"https://cnrs.fr","robots":{"status":"html","http":200,"url":"https://cnrs.fr/robots.txt","bytes":244,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":1235,"domain":"modsecurity.org","origin":"https://modsecurity.org","robots":{"status":"ok","http":200,"url":"https://modsecurity.org/robots.txt","bytes":14,"groups":1},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"missing","http":404,"bytes":9652},"home":{"status":"ok","http":200,"url":"https://modsecurity.org/","html_tokens":5500,"markdown_tokens":562,"pruned_tokens":72}},
{"rank":1236,"domain":"hardened-php.net","origin":"https://hardened-php.net","robots":{"status":"ok","http":200,"url":"http://www.hardened-php.net/robots.txt","bytes":177,"groups":1},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"html","http":200,"bytes":39096},"home":{"status":"ok","http":200,"url":"http://www.hardened-php.net/","html_tokens":9753,"markdown_tokens":1567,"pruned_tokens":503}},
{"rank":1237,"domain":"defense.gov","origin":"https://defense.gov","robots":{"status":"refused","http":403,"url":"https://defense.gov/robots.txt","bytes":375,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":1238,"domain":"enlightenment.org","origin":"https://www.enlightenment.org","robots":{"status":"ok","http":200,"url":"https://www.enlightenment.org/robots.txt","bytes":186,"groups":2},"bots":{"GPTBot":["blocked","*",1,0],"ChatGPT-User":["blocked","*",1,0],"OAI-SearchBot":["blocked","*",1,0],"ClaudeBot":["blocked","*",1,0],"Claude-User":["blocked","*",1,0],"Claude-SearchBot":["blocked","*",1,0],"anthropic-ai":["blocked","*",1,0],"PerplexityBot":["blocked","*",1,0],"Perplexity-User":["blocked","*",1,0],"Google-Extended":["blocked","*",1,0],"Applebot-Extended":["blocked","*",1,0],"CCBot":["blocked","*",1,0],"Bytespider":["blocked","*",1,0],"Amazonbot":["blocked","*",1,0],"meta-externalagent":["blocked","*",1,0],"cohere-ai":["blocked","*",1,0],"Diffbot":["blocked","*",1,0],"ContextFetchBot":["blocked","*",1,0]},"llms":{"status":"disallowed"},"home":{"status":"disallowed"}},
{"rank":1239,"domain":"bild.de","origin":"https://bild.de","robots":{"status":"ok","http":200,"url":"https://www.bild.de/robots.txt","bytes":3053,"groups":3},"bots":{"GPTBot":["partial","*",14,0],"ChatGPT-User":["partial","*",14,0],"OAI-SearchBot":["partial","*",14,0],"ClaudeBot":["partial","*",14,0],"Claude-User":["partial","*",14,0],"Claude-SearchBot":["partial","*",14,0],"anthropic-ai":["partial","*",14,0],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["blocked","named",1,0],"Google-Extended":["partial","*",14,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["partial","*",14,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",14,0]},"llms":{"status":"missing","http":404,"bytes":186876},"home":{"status":"ok","http":200,"url":"https://www.bild.de/","html_tokens":214081,"markdown_tokens":50,"pruned_tokens":50}},
{"rank":1240,"domain":"icio.us","origin":"http://icio.us","robots":{"status":"missing","http":404,"url":"http://icio.us/robots.txt","bytes":269,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":269},"home":{"status":"ok","http":200,"url":"http://icio.us/","html_tokens":2831,"markdown_tokens":1122,"pruned_tokens":111}},
{"rank":1241,"domain":"filezilla-project.org","origin":"https://filezilla-project.org","robots":{"status":"ok","http":200,"url":"https://filezilla-project.org/robots.txt","bytes":104,"groups":1},"bots":{"GPTBot":["partial","*",4,0],"ChatGPT-User":["partial","*",4,0],"OAI-SearchBot":["partial","*",4,0],"ClaudeBot":["partial","*",4,0],"Claude-User":["partial","*",4,0],"Claude-SearchBot":["partial","*",4,0],"anthropic-ai":["partial","*",4,0],"PerplexityBot":["partial","*",4,0],"Perplexity-User":["partial","*",4,0],"Google-Extended":["partial","*",4,0],"Applebot-Extended":["partial","*",4,0],"CCBot":["partial","*",4,0],"Bytespider":["partial","*",4,0],"Amazonbot":["partial","*",4,0],"meta-externalagent":["partial","*",4,0],"cohere-ai":["partial","*",4,0],"Diffbot":["partial","*",4,0],"ContextFetchBot":["partial","*",4,0]},"llms":{"status":"missing","http":404,"bytes":236},"home":{"status":"ok","http":200,"url":"https://filezilla-project.org/","html_tokens":3400,"markdown_tokens":1737,"pruned_tokens":175}},
{"rank":1242,"domain":"webdav.org","origin":"http://webdav.org","robots":{"status":"ok","http":200,"url":"http://webdav.org/robots.txt","bytes":79,"groups":1},"bots":{"GPTBot":["partial","*",3,0],"ChatGPT-User":["partial","*",3,0],"OAI-SearchBot":["partial","*",3,0],"ClaudeBot":["partial","*",3,0],"Claude-User":["partial","*",3,0],"Claude-SearchBot":["partial","*",3,0],"anthropic-ai":["partial","*",3,0],"PerplexityBot":["partial","*",3,0],"Perplexity-User":["partial","*",3,0],"Google-Extended":["partial","*",3,0],"Applebot-Extended":["partial","*",3,0],"CCBot":["partial","*",3,0],"Bytespider":["partial","*",3,0],"Amazonbot":["partial","*",3,0],"meta-externalagent":["partial","*",3,0],"cohere-ai":["partial","*",3,0],"Diffbot":["partial","*",3,0],"ContextFetchBot":["partial","*",3,0]},"llms":{"status":"missing","http":404,"bytes":256},"home":{"status":"ok","http":200,"url":"http://webdav.org/","html_tokens":5299,"markdown_tokens":1887,"pruned_tokens":1261}},
{"rank":1243,"domain":"fastpanel.direct","origin":"https://fastpanel.direct","robots":{"status":"html","http":200,"url":"https://fastpanel.direct/robots.txt","bytes":63655,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":1244,"domain":"standard.co.uk","origin":"https://standard.co.uk","robots":{"status":"ok","http":200,"url":"https://www.standard.co.uk/robots.txt","bytes":422,"groups":3},"bots":{"GPTBot":["partial","*",4,0],"ChatGPT-User":["partial","*",4,0],"OAI-SearchBot":["partial","*",4,0],"ClaudeBot":["partial","*",4,0],"Claude-User":["partial","*",4,0],"Claude-SearchBot":["partial","*",4,0],"anthropic-ai":["partial","*",4,0],"PerplexityBot":["partial","*",4,0],"Perplexity-User":["partial","*",4,0],"Google-Extended":["partial","*",4,0],"Applebot-Extended":["partial","*",4,0],"CCBot":["partial","*",4,0],"Bytespider":["partial","*",4,0],"Amazonbot":["partial","*",4,0],"meta-externalagent":["partial","*",4,0],"cohere-ai":["partial","*",4,0],"Diffbot":["partial","*",4,0],"ContextFetchBot":["partial","*",4,0]},"llms":{"status":"missing","http":404,"bytes":136361},"home":{"status":"ok","http":200,"url":"https://www.standard.co.uk/","html_tokens":113757,"markdown_tokens":57,"pruned_tokens":57}},
{"rank":1245,"domain":"foreignpolicy.com","origin":"https://foreignpolicy.com","robots":{"status":"ok","http":200,"url":"https://foreignpolicy.com/robots.txt","bytes":2787,"groups":4},"bots":{"GPTBot":["partial","named",1,6],"ChatGPT-User":["partial","named",1,6],"OAI-SearchBot":["partial","*",11,3],"ClaudeBot":["partial","named",1,6],"Claude-User":["partial","named",1,6],"Claude-SearchBot":["partial","*",11,3],"anthropic-ai":["partial","*",11,3],"PerplexityBot":["partial","named",1,6],"Perplexity-User":["partial","*",11,3],"Google-Extended":["partial","named",1,6],"Applebot-Extended":["partial","named",1,6],"CCBot":["partial","named",1,6],"Bytespider":["partial","named",1,6],"Amazonbot":["partial","*",11,3],"meta-externalagent":["partial","named",1,6],"cohere-ai":["partial","*",11,3],"Diffbot":["partial","named",1,6],"ContextFetchBot":["partial","*",11,3]},"llms":{"status":"missing","http":404,"bytes":263028},"home":{"status":"ok","http":200,"url":"https://foreignpolicy.com/","html_tokens":139612,"markdown_tokens":74,"pruned_tokens":74}},
{"rank":1246,"domain":"eclipse.org","origin":"https://eclipse.org","robots":{"status":"ok","http":200,"url":"https://www.eclipse.org/robots.txt","bytes":13,"groups":1},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"missing","http":404,"bytes":9535},"home":{"status":"ok","http":200,"url":"https://www.eclipse.org/","html_tokens":10722,"markdown_tokens":387,"pruned_tokens":151}},
{"rank":1247,"domain":"thebalancemoney.com","origin":"https://thebalancemoney.com","robots":{"status":"ok","http":200,"url":"https://www.thebalancemoney.com/robots.txt","bytes":3483,"groups":6},"bots":{"GPTBot":["partial","named",1,0],"ChatGPT-User":["partial","named",1,0],"OAI-SearchBot":["partial","named",1,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["blocked","named",1,0],"Claude-SearchBot":["partial","named",1,2],"anthropic-ai":["partial","named",1,2],"PerplexityBot":["partial","named",1,2],"Perplexity-User":["blocked","named",1,0],"Google-Extended":["partial","named",1,2],"Applebot-Extended":["blocked","named",1,0],"CCBot":["partial","named",1,2],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["partial","named",1,2],"cohere-ai":["partial","named",1,2],"Diffbot":["partial","*",2,0],"ContextFetchBot":["partial","*",2,0]},"llms":{"status":"http_error","http":402,"bytes":612},"home":{"status":"http_error","http":402,"url":"https://www.thebalancemoney.com/"}},
{"rank":1248,"domain":"gettyimages.com","origin":"https://gettyimages.com","robots":{"status":"refused","http":403,"url":"https://gettyimages.com/robots.txt","bytes":919,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":1249,"domain":"mac.com","origin":"https://mac.com","robots":{"status":"html","http":200,"url":"https://www.apple.com/mac/","bytes":1054770,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":1250,"domain":"philips.com","origin":"https://philips.com","robots":{"status":"ok","http":200,"url":"https://www.philips.com/robots.txt","bytes":488,"groups":1},"bots":{"GPTBot":["partial","*",10,0],"ChatGPT-User":["partial","*",10,0],"OAI-SearchBot":["partial","*",10,0],"ClaudeBot":["partial","*",10,0],"Claude-User":["partial","*",10,0],"Claude-SearchBot":["partial","*",10,0],"anthropic-ai":["partial","*",10,0],"PerplexityBot":["partial","*",10,0],"Perplexity-User":["partial","*",10,0],"Google-Extended":["partial","*",10,0],"Applebot-Extended":["partial","*",10,0],"CCBot":["partial","*",10,0],"Bytespider":["partial","*",10,0],"Amazonbot":["partial","*",10,0],"meta-externalagent":["partial","*",10,0],"cohere-ai":["partial","*",10,0],"Diffbot":["partial","*",10,0],"ContextFetchBot":["partial","*",10,0]},"llms":{"status":"missing","http":404,"bytes":132918},"home":{"status":"ok","http":200,"url":"https://www.usa.philips.com/","html_tokens":171046,"markdown_tokens":82,"pruned_tokens":54}},
{"rank":1251,"domain":"kakao.com","origin":"https://kakao.com","robots":{"status":"ok","http":200,"url":"https://kakao.com/robots.txt","bytes":147,"groups":2},"bots":{"GPTBot":["partial","*",1,2],"ChatGPT-User":["partial","*",1,2],"OAI-SearchBot":["partial","*",1,2],"ClaudeBot":["partial","*",1,2],"Claude-User":["partial","*",1,2],"Claude-SearchBot":["partial","*",1,2],"anthropic-ai":["partial","*",1,2],"PerplexityBot":["partial","*",1,2],"Perplexity-User":["partial","*",1,2],"Google-Extended":["partial","*",1,2],"Applebot-Extended":["partial","*",1,2],"CCBot":["partial","*",1,2],"Bytespider":["partial","*",1,2],"Amazonbot":["partial","*",1,2],"meta-externalagent":["partial","*",1,2],"cohere-ai":["partial","*",1,2],"Diffbot":["partial","*",1,2],"ContextFetchBot":["partial","*",1,2]},"llms":{"status":"disallowed"},"home":{"status":"ok","http":200,"url":"https://www.kakaocorp.com/page/","html_tokens":10679,"markdown_tokens":13,"pruned_tokens":13}},
{"rank":1252,"domain":"corriere.it","origin":"https://corriere.it","robots":{"status":"ok","http":200,"url":"https://www.corriere.it/robots.txt","bytes":10125,"groups":28},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["partial","named",1,1],"OAI-SearchBot":["allowed","named",0,1],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",80,27],"Claude-SearchBot":["partial","*",80,27],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["allowed","named",0,1],"Perplexity-User":["partial","*",80,27],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["partial","*",80,27],"Diffbot":["partial","*",80,27],"ContextFetchBot":["partial","*",80,27]},"llms":{"status":"missing","http":404,"bytes":142381},"home":{"status":"ok","http":200,"url":"https://www.corriere.it/","html_tokens":248140,"markdown_tokens":57,"pruned_tokens":57}},
{"rank":1253,"domain":"herokuapp.com","origin":"https://www.herokuapp.com","robots":{"status":"ok","http":200,"url":"https://www.herokuapp.com/robots.txt","bytes":77,"groups":1},"bots":{"GPTBot":["partial","*",1,0],"ChatGPT-User":["partial","*",1,0],"OAI-SearchBot":["partial","*",1,0],"ClaudeBot":["partial","*",1,0],"Claude-User":["partial","*",1,0],"Claude-SearchBot":["partial","*",1,0],"anthropic-ai":["partial","*",1,0],"PerplexityBot":["partial","*",1,0],"Perplexity-User":["partial","*",1,0],"Google-Extended":["partial","*",1,0],"Applebot-Extended":["partial","*",1,0],"CCBot":["partial","*",1,0],"Bytespider":["partial","*",1,0],"Amazonbot":["partial","*",1,0],"meta-externalagent":["partial","*",1,0],"cohere-ai":["partial","*",1,0],"Diffbot":["partial","*",1,0],"ContextFetchBot":["partial","*",1,0]},"llms":{"status":"http_error","http":403,"bytes":55},"home":{"status":"not_html","error":"unsupported content type text/plain"}},
{"rank":1254,"domain":"af.mil","origin":"https://www.af.mil","robots":{"status":"refused","http":403,"url":"https://www.af.mil/robots.txt","bytes":376,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":1255,"domain":"dmca.com","origin":"https://dmca.com","robots":{"status":"ok","http":200,"url":"https://www.dmca.com/robots.txt","bytes":182,"groups":1},"bots":{"GPTBot":["partial","*",4,1],"ChatGPT-User":["partial","*",4,1],"OAI-SearchBot":["partial","*",4,1],"ClaudeBot":["partial","*",4,1],"Claude-User":["partial","*",4,1],"Claude-SearchBot":["partial","*",4,1],"anthropic-ai":["partial","*",4,1],"PerplexityBot":["partial","*",4,1],"Perplexity-User":["partial","*",4,1],"Google-Extended":["partial","*",4,1],"Applebot-Extended":["partial","*",4,1],"CCBot":["partial","*",4,1],"Bytespider":["partial","*",4,1],"Amazonbot":["partial","*",4,1],"meta-externalagent":["partial","*",4,1],"cohere-ai":["partial","*",4,1],"Diffbot":["partial","*",4,1],"ContextFetchBot":["partial","*",4,1]},"llms":{"status":"other_text","http":200,"bytes":10598},"home":{"status":"ok","http":200,"url":"https://www.dmca.com/","html_tokens":129935,"markdown_tokens":2168,"pruned_tokens":921}},
{"rank":1256,"domain":"nsf.gov","origin":"https://nsf.gov","robots":{"status":"ok","http":200,"url":"https://www.nsf.gov/robots.txt","bytes":2803,"groups":1},"bots":{"GPTBot":["partial","*",53,18],"ChatGPT-User":["partial","*",53,18],"OAI-SearchBot":["partial","*",53,18],"ClaudeBot":["partial","*",53,18],"Claude-User":["partial","*",53,18],"Claude-SearchBot":["partial","*",53,18],"anthropic-ai":["partial","*",53,18],"PerplexityBot":["partial","*",53,18],"Perplexity-User":["partial","*",53,18],"Google-Extended":["partial","*",53,18],"Applebot-Extended":["partial","*",53,18],"CCBot":["partial","*",53,18],"Bytespider":["partial","*",53,18],"Amazonbot":["partial","*",53,18],"meta-externalagent":["partial","*",53,18],"cohere-ai":["partial","*",53,18],"Diffbot":["partial","*",53,18],"ContextFetchBot":["partial","*",53,18]},"llms":{"status":"missing","http":404,"bytes":49593},"home":{"status":"ok","http":200,"url":"https://www.nsf.gov/","html_tokens":20360,"markdown_tokens":77,"pruned_tokens":77}},
{"rank":1257,"domain":"del.icio.us","origin":"https://del.icio.us","robots":{"status":"unreachable","http":null,"url":"https://del.icio.us/robots.txt","bytes":0,"groups":0,"error":"[SSL: CERTIFICATE_VERIFY_FAILED] certificate verify failed: unable to get local issuer certificate (_ssl.c:1028)"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":1258,"domain":"qld.gov.au","origin":"https://qld.gov.au","robots":{"status":"ok","http":200,"url":"https://www.qld.gov.au/robots.txt","bytes":293,"groups":1},"bots":{"GPTBot":["partial","*",8,0],"ChatGPT-User":["partial","*",8,0],"OAI-SearchBot":["partial","*",8,0],"ClaudeBot":["partial","*",8,0],"Claude-User":["partial","*",8,0],"Claude-SearchBot":["partial","*",8,0],"anthropic-ai":["partial","*",8,0],"PerplexityBot":["partial","*",8,0],"Perplexity-User":["partial","*",8,0],"Google-Extended":["partial","*",8,0],"Applebot-Extended":["partial","*",8,0],"CCBot":["partial","*",8,0],"Bytespider":["partial","*",8,0],"Amazonbot":["partial","*",8,0],"meta-externalagent":["partial","*",8,0],"cohere-ai":["partial","*",8,0],"Diffbot":["partial","*",8,0],"ContextFetchBot":["partial","*",8,0]},"llms":{"status":"missing","http":404,"bytes":178151},"home":{"status":"ok","http":200,"url":"https://www.qld.gov.au/","html_tokens":55177,"markdown_tokens":199,"pruned_tokens":167}},
{"rank":1259,"domain":"gwu.edu","origin":"https://www.gwu.edu","robots":{"status":"ok","http":200,"url":"https://www.gwu.edu/robots.txt","bytes":2027,"groups":1},"bots":{"GPTBot":["partial","*",34,18],"ChatGPT-User":["partial","*",34,18],"OAI-SearchBot":["partial","*",34,18],"ClaudeBot":["partial","*",34,18],"Claude-User":["partial","*",34,18],"Claude-SearchBot":["partial","*",34,18],"anthropic-ai":["partial","*",34,18],"PerplexityBot":["partial","*",34,18],"Perplexity-User":["partial","*",34,18],"Google-Extended":["partial","*",34,18],"Applebot-Extended":["partial","*",34,18],"CCBot":["partial","*",34,18],"Bytespider":["partial","*",34,18],"Amazonbot":["partial","*",34,18],"meta-externalagent":["partial","*",34,18],"cohere-ai":["partial","*",34,18],"Diffbot":["partial","*",34,18],"ContextFetchBot":["partial","*",34,18]},"llms":{"status":"missing","http":404,"bytes":291},"home":{"status":"ok","http":200,"url":"https://www.gwu.edu/","html_tokens":28341,"markdown_tokens":1223,"pruned_tokens":103}},
{"rank":1260,"domain":"livechat.com","origin":"https://livechat.com","robots":{"status":"ok","http":200,"url":"https://www.livechat.com/robots.txt","bytes":1164,"groups":2},"bots":{"GPTBot":["partial","*",32,1],"ChatGPT-User":["partial","*",32,1],"OAI-SearchBot":["partial","*",32,1],"ClaudeBot":["partial","*",32,1],"Claude-User":["partial","*",32,1],"Claude-SearchBot":["partial","*",32,1],"anthropic-ai":["partial","*",32,1],"PerplexityBot":["partial","*",32,1],"Perplexity-User":["partial","*",32,1],"Google-Extended":["partial","*",32,1],"Applebot-Extended":["partial","*",32,1],"CCBot":["partial","*",32,1],"Bytespider":["partial","*",32,1],"Amazonbot":["partial","*",32,1],"meta-externalagent":["partial","*",32,1],"cohere-ai":["partial","*",32,1],"Diffbot":["partial","*",32,1],"ContextFetchBot":["partial","*",32,1]},"llms":{"status":"present","http":200,"bytes":238040},"home":{"status":"ok","http":200,"url":"https://www.livechat.com/","html_tokens":64537,"markdown_tokens":403,"pruned_tokens":82}},
{"rank":1261,"domain":"uscourts.gov","origin":"https://uscourts.gov","robots":{"status":"ok","http":200,"url":"https://www.uscourts.gov/robots.txt","bytes":1782,"groups":1},"bots":{"GPTBot":["partial","*",28,18],"ChatGPT-User":["partial","*",28,18],"OAI-SearchBot":["partial","*",28,18],"ClaudeBot":["partial","*",28,18],"Claude-User":["partial","*",28,18],"Claude-SearchBot":["partial","*",28,18],"anthropic-ai":["partial","*",28,18],"PerplexityBot":["partial","*",28,18],"Perplexity-User":["partial","*",28,18],"Google-Extended":["partial","*",28,18],"Applebot-Extended":["partial","*",28,18],"CCBot":["partial","*",28,18],"Bytespider":["partial","*",28,18],"Amazonbot":["partial","*",28,18],"meta-externalagent":["partial","*",28,18],"cohere-ai":["partial","*",28,18],"Diffbot":["partial","*",28,18],"ContextFetchBot":["partial","*",28,18]},"llms":{"status":"http_error","http":503,"bytes":1512},"home":{"status":"ok","http":200,"url":"https://www.uscourts.gov/","html_tokens":55607,"markdown_tokens":218,"pruned_tokens":142}},
{"rank":1262,"domain":"ahajournals.org","origin":"https://ahajournals.org","robots":{"status":"ok","http":200,"url":"https://www.ahajournals.org/robots.txt","bytes":2823,"groups":41},"bots":{"GPTBot":["blocked","named",2,0],"ChatGPT-User":["partial","*",18,7],"OAI-SearchBot":["partial","*",18,7],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",18,7],"Claude-SearchBot":["partial","*",18,7],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["partial","*",18,7],"Perplexity-User":["partial","*",18,7],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["partial","*",18,7],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["partial","*",18,7],"ContextFetchBot":["partial","*",18,7]},"llms":{"status":"http_error","http":403,"bytes":5568},"home":{"status":"http_error","http":403,"url":"https://ahajournals.org/"}},
{"rank":1263,"domain":"messenger.com","origin":"https://messenger.com","robots":{"status":"ok","http":200,"url":"https://www.messenger.com/robots.txt","bytes":643,"groups":3},"bots":{"GPTBot":["partial","*",9,0],"ChatGPT-User":["partial","*",9,0],"OAI-SearchBot":["partial","*",9,0],"ClaudeBot":["partial","*",9,0],"Claude-User":["partial","*",9,0],"Claude-SearchBot":["partial","*",9,0],"anthropic-ai":["partial","*",9,0],"PerplexityBot":["partial","*",9,0],"Perplexity-User":["partial","*",9,0],"Google-Extended":["partial","*",9,0],"Applebot-Extended":["partial","*",9,0],"CCBot":["partial","*",9,0],"Bytespider":["partial","*",9,0],"Amazonbot":["partial","*",9,0],"meta-externalagent":["partial","*",9,0],"cohere-ai":["partial","*",9,0],"Diffbot":["partial","*",9,0],"ContextFetchBot":["partial","*",9,0]},"llms":{"status":"missing","http":404,"bytes":1506},"home":{"status":"ok","http":200,"url":"https://www.messenger.com/","html_tokens":27422,"markdown_tokens":39,"pruned_tokens":39}},
{"rank":1264,"domain":"homedepot.com","origin":"https://homedepot.com","robots":{"status":"ok","http":200,"url":"https://www.homedepot.com/robots.txt","bytes":1394,"groups":1},"bots":{"GPTBot":["partial","*",45,1],"ChatGPT-User":["partial","*",45,1],"OAI-SearchBot":["partial","*",45,1],"ClaudeBot":["partial","*",45,1],"Claude-User":["partial","*",45,1],"Claude-SearchBot":["partial","*",45,1],"anthropic-ai":["partial","*",45,1],"PerplexityBot":["partial","*",45,1],"Perplexity-User":["partial","*",45,1],"Google-Extended":["partial","*",45,1],"Applebot-Extended":["partial","*",45,1],"CCBot":["partial","*",45,1],"Bytespider":["partial","*",45,1],"Amazonbot":["partial","*",45,1],"meta-externalagent":["partial","*",45,1],"cohere-ai":["partial","*",45,1],"Diffbot":["partial","*",45,1],"ContextFetchBot":["partial","*",45,1]},"llms":{"status":"missing","http":404,"bytes":2522},"home":{"status":"ok","http":200,"url":"https://www.homedepot.com/","html_tokens":309538,"markdown_tokens":18,"pruned_tokens":18}},
{"rank":1265,"domain":"howtogeek.com","origin":"https://howtogeek.com","robots":{"status":"ok","http":200,"url":"https://www.howtogeek.com/robots.txt","bytes":2039,"groups":2},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["partial","*",22,0],"OAI-SearchBot":["partial","*",22,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",22,0],"Claude-SearchBot":["partial","*",22,0],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["partial","*",22,0],"Perplexity-User":["partial","*",22,0],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",22,0]},"llms":{"status":"missing","http":404,"bytes":112214},"home":{"status":"ok","http":200,"url":"https://www.howtogeek.com/","html_tokens":239937,"markdown_tokens":66,"pruned_tokens":66}},
{"rank":1266,"domain":"biglobe.ne.jp","origin":"https://biglobe.ne.jp","robots":{"status":"missing","http":404,"url":"https://biglobe.ne.jp/robots.txt","bytes":9,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":9},"home":{"status":"ok","http":200,"url":"https://biglobe.ne.jp/","html_tokens":64,"markdown_tokens":0,"pruned_tokens":0}},
{"rank":1267,"domain":"gstatic.com","origin":"https://gstatic.com","robots":{"status":"ok","http":200,"url":"https://www.gstatic.com/robots.txt","bytes":5252,"groups":1},"bots":{"GPTBot":["partial","*",1,196],"ChatGPT-User":["partial","*",1,196],"OAI-SearchBot":["partial","*",1,196],"ClaudeBot":["partial","*",1,196],"Claude-User":["partial","*",1,196],"Claude-SearchBot":["partial","*",1,196],"anthropic-ai":["partial","*",1,196],"PerplexityBot":["partial","*",1,196],"Perplexity-User":["partial","*",1,196],"Google-Extended":["partial","*",1,196],"Applebot-Extended":["partial","*",1,196],"CCBot":["partial","*",1,196],"Bytespider":["partial","*",1,196],"Amazonbot":["partial","*",1,196],"meta-externalagent":["partial","*",1,196],"cohere-ai":["partial","*",1,196],"Diffbot":["partial","*",1,196],"ContextFetchBot":["partial","*",1,196]},"llms":{"status":"disallowed"},"home":{"status":"disallowed"}},
{"rank":1268,"domain":"ap.org","origin":"https://ap.org","robots":{"status":"ok","http":200,"url":"https://ap.org/robots.txt","bytes":2676,"groups":32},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["allowed","*",0,1],"OAI-SearchBot":["allowed","*",0,1],"ClaudeBot":["blocked","named",1,0],"Claude-User":["allowed","*",0,1],"Claude-SearchBot":["allowed","*",0,1],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["allowed","*",0,1],"Perplexity-User":["allowed","*",0,1],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["allowed","*",0,1]},"llms":{"status":"http_error","http":403,"bytes":4539},"home":{"status":"http_error","http":403,"url":"https://ap.org/"}},
{"rank":1269,"domain":"anu.edu.au","origin":"https://anu.edu.au","robots":{"status":"ok","http":200,"url":"https://www.anu.edu.au/robots.txt","bytes":2141,"groups":1},"bots":{"GPTBot":["partial","*",35,22],"ChatGPT-User":["partial","*",35,22],"OAI-SearchBot":["partial","*",35,22],"ClaudeBot":["partial","*",35,22],"Claude-User":["partial","*",35,22],"Claude-SearchBot":["partial","*",35,22],"anthropic-ai":["partial","*",35,22],"PerplexityBot":["partial","*",35,22],"Perplexity-User":["partial","*",35,22],"Google-Extended":["partial","*",35,22],"Applebot-Extended":["partial","*",35,22],"CCBot":["partial","*",35,22],"Bytespider":["partial","*",35,22],"Amazonbot":["partial","*",35,22],"meta-externalagent":["partial","*",35,22],"cohere-ai":["partial","*",35,22],"Diffbot":["partial","*",35,22],"ContextFetchBot":["partial","*",35,22]},"llms":{"status":"missing","http":404,"bytes":146},"home":{"status":"ok","http":200,"url":"https://www.anu.edu.au/","html_tokens":22183,"markdown_tokens":98,"pruned_tokens":63}},
{"rank":1270,"domain":"torproject.org","origin":"https://torproject.org","robots":{"status":"ok","http":200,"url":"https://www.torproject.org/robots.txt","bytes":54,"groups":1},"bots":{"GPTBot":["partial","*",2,0],"ChatGPT-User":["partial","*",2,0],"OAI-SearchBot":["partial","*",2,0],"ClaudeBot":["partial","*",2,0],"Claude-User":["partial","*",2,0],"Claude-SearchBot":["partial","*",2,0],"anthropic-ai":["partial","*",2,0],"PerplexityBot":["partial","*",2,0],"Perplexity-User":["partial","*",2,0],"Google-Extended":["partial","*",2,0],"Applebot-Extended":["partial","*",2,0],"CCBot":["partial","*",2,0],"Bytespider":["partial","*",2,0],"Amazonbot":["partial","*",2,0],"meta-externalagent":["partial","*",2,0],"cohere-ai":["partial","*",2,0],"Diffbot":["partial","*",2,0],"ContextFetchBot":["partial","*",2,0]},"llms":{"status":"missing","http":404,"bytes":305},"home":{"status":"ok","http":200,"url":"https://www.torproject.org/","html_tokens":6371,"markdown_tokens":285,"pruned_tokens":63}},
{"rank":1271,"domain":"producthunt.com","origin":"https://producthunt.com","robots":{"status":"ok","http":200,"url":"https://www.producthunt.com/robots.txt","bytes":1503,"groups":2},"bots":{"GPTBot":["partial","*",14,2],"ChatGPT-User":["partial","*",14,2],"OAI-SearchBot":["partial","*",14,2],"ClaudeBot":["partial","*",14,2],"Claude-User":["partial","*",14,2],"Claude-SearchBot":["partial","*",14,2],"anthropic-ai":["partial","*",14,2],"PerplexityBot":["partial","*",14,2],"Perplexity-User":["partial","*",14,2],"Google-Extended":["partial","*",14,2],"Applebot-Extended":["partial","*",14,2],"CCBot":["partial","*",14,2],"Bytespider":["partial","*",14,2],"Amazonbot":["partial","*",14,2],"meta-externalagent":["partial","*",14,2],"cohere-ai":["partial","*",14,2],"Diffbot":["partial","*",14,2],"ContextFetchBot":["partial","*",14,2]},"llms":{"status":"present","http":200,"bytes":5714},"home":{"status":"ok","http":200,"url":"https://www.producthunt.com/","html_tokens":214330,"markdown_tokens":144,"pruned_tokens":144}},
{"rank":1272,"domain":"blogspot.de","origin":"https://blogspot.de","robots":{"status":"ok","http":200,"url":"https://blogspot.de/robots.txt","bytes":6624,"groups":4},"bots":{"GPTBot":["partial","*",169,64],"ChatGPT-User":["partial","*",169,64],"OAI-SearchBot":["partial","*",169,64],"ClaudeBot":["partial","*",169,64],"Claude-User":["partial","*",169,64],"Claude-SearchBot":["partial","*",169,64],"anthropic-ai":["partial","*",169,64],"PerplexityBot":["partial","*",169,64],"Perplexity-User":["partial","*",169,64],"Google-Extended":["partial","*",169,64],"Applebot-Extended":["partial","*",169,64],"CCBot":["partial","*",169,64],"Bytespider":["partial","*",169,64],"Amazonbot":["partial","*",169,64],"meta-externalagent":["partial","*",169,64],"cohere-ai":["partial","*",169,64],"Diffbot":["partial","*",169,64],"ContextFetchBot":["partial","*",169,64]},"llms":{"status":"missing","http":404,"bytes":1569},"home":{"status":"ok","http":200,"url":"https://www.google.com/","html_tokens":21299,"markdown_tokens":30,"pruned_tokens":30}},
{"rank":1273,"domain":"visa.com","origin":"https://visa.com","robots":{"status":"ok","http":200,"url":"https://www.visa.com/robots.txt","bytes":0,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":160},"home":{"status":"ok","http":200,"url":"https://www.visa.com/en-us","html_tokens":118269,"markdown_tokens":155,"pruned_tokens":52}},
{"rank":1274,"domain":"service-public.fr","origin":"https://service-public.fr","robots":{"status":"ok","http":200,"url":"https://www.service-public.fr/robots.txt","bytes":446,"groups":1},"bots":{"GPTBot":["partial","*",11,2],"ChatGPT-User":["partial","*",11,2],"OAI-SearchBot":["partial","*",11,2],"ClaudeBot":["partial","*",11,2],"Claude-User":["partial","*",11,2],"Claude-SearchBot":["partial","*",11,2],"anthropic-ai":["partial","*",11,2],"PerplexityBot":["partial","*",11,2],"Perplexity-User":["partial","*",11,2],"Google-Extended":["partial","*",11,2],"Applebot-Extended":["partial","*",11,2],"CCBot":["partial","*",11,2],"Bytespider":["partial","*",11,2],"Amazonbot":["partial","*",11,2],"meta-externalagent":["partial","*",11,2],"cohere-ai":["partial","*",11,2],"Diffbot":["partial","*",11,2],"ContextFetchBot":["partial","*",11,2]},"llms":{"status":"missing","http":404,"bytes":16625},"home":{"status":"ok","http":200,"url":"https://www.service-public.gouv.fr/","html_tokens":13249,"markdown_tokens":51,"pruned_tokens":51}},
{"rank":1275,"domain":"cbslocal.com","origin":"https://cbslocal.com","robots":{"status":"missing","http":404,"url":"https://www.cbsnews.com/cbslocal/robots.txt/","bytes":564122,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":564128},"home":{"status":"ok","http":200,"url":"https://www.cbsnews.com/cbslocal/","html_tokens":183137,"markdown_tokens":1491,"pruned_tokens":175}},
{"rank":1276,"domain":"bayern.de","origin":"https://bayern.de","robots":{"status":"ok","http":200,"url":"https://www.bayern.de/robots.txt","bytes":23,"groups":1},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"missing","http":404,"bytes":61940},"home":{"status":"ok","http":200,"url":"https://www.bayern.de/","html_tokens":33758,"markdown_tokens":427,"pruned_tokens":93}},
{"rank":1277,"domain":"app.google","origin":"https://www.app.google","robots":{"status":"refused","http":400,"url":"https://www.app.google/robots.txt","bytes":9258,"groups":0,"error":"HTTP 400"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":1278,"domain":"goethe.de","origin":"https://goethe.de","robots":{"status":"ok","http":200,"url":"https://www.goethe.de/robots.txt","bytes":1199,"groups":2},"bots":{"GPTBot":["partial","*",22,0],"ChatGPT-User":["partial","*",22,0],"OAI-SearchBot":["partial","*",22,0],"ClaudeBot":["partial","*",22,0],"Claude-User":["partial","*",22,0],"Claude-SearchBot":["partial","*",22,0],"anthropic-ai":["partial","*",22,0],"PerplexityBot":["partial","*",22,0],"Perplexity-User":["partial","*",22,0],"Google-Extended":["partial","*",22,0],"Applebot-Extended":["partial","*",22,0],"CCBot":["partial","*",22,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["partial","*",22,0],"meta-externalagent":["partial","*",22,0],"cohere-ai":["partial","*",22,0],"Diffbot":["partial","*",22,0],"ContextFetchBot":["partial","*",22,0]},"llms":{"status":"missing","http":404,"bytes":10},"home":{"status":"http_error","http":403,"url":"https://www.goethe.de/de/index.html"}},
{"rank":1279,"domain":"jp.net","origin":"http://jp.net","robots":{"status":"missing","http":404,"url":"http://www.jp.net/robots.txt","bytes":208,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":206},"home":{"status":"ok","http":200,"url":"http://www.jp.net/","html_tokens":2686,"markdown_tokens":331,"pruned_tokens":331}},
{"rank":1280,"domain":"businessweek.com","origin":"https://businessweek.com","robots":{"status":"html","http":200,"url":"https://www.bloomberg.com/businessweek","bytes":1224876,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":1281,"domain":"website-files.com","origin":"https://website-files.com","robots":{"status":"unreachable","http":null,"url":"https://website-files.com/robots.txt","bytes":0,"groups":0,"error":"cannot resolve website-files.com"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":1282,"domain":"nationalarchives.gov.uk","origin":"https://nationalarchives.gov.uk","robots":{"status":"ok","http":200,"url":"https://www.nationalarchives.gov.uk/robots.txt","bytes":1322,"groups":1},"bots":{"GPTBot":["partial","*",28,0],"ChatGPT-User":["partial","*",28,0],"OAI-SearchBot":["partial","*",28,0],"ClaudeBot":["partial","*",28,0],"Claude-User":["partial","*",28,0],"Claude-SearchBot":["partial","*",28,0],"anthropic-ai":["partial","*",28,0],"PerplexityBot":["partial","*",28,0],"Perplexity-User":["partial","*",28,0],"Google-Extended":["partial","*",28,0],"Applebot-Extended":["partial","*",28,0],"CCBot":["partial","*",28,0],"Bytespider":["partial","*",28,0],"Amazonbot":["partial","*",28,0],"meta-externalagent":["partial","*",28,0],"cohere-ai":["partial","*",28,0],"Diffbot":["partial","*",28,0],"ContextFetchBot":["partial","*",28,0]},"llms":{"status":"missing","http":404,"bytes":34348},"home":{"status":"ok","http":200,"url":"https://www.nationalarchives.gov.uk/","html_tokens":13110,"markdown_tokens":135,"pruned_tokens":84}},
{"rank":1283,"domain":"carrd.co","origin":"https://carrd.co","robots":{"status":"ok","http":200,"url":"https://carrd.com/robots.txt","bytes":29,"groups":1},"bots":{"GPTBot":["partial","*",1,0],"ChatGPT-User":["partial","*",1,0],"OAI-SearchBot":["partial","*",1,0],"ClaudeBot":["partial","*",1,0],"Claude-User":["partial","*",1,0],"Claude-SearchBot":["partial","*",1,0],"anthropic-ai":["partial","*",1,0],"PerplexityBot":["partial","*",1,0],"Perplexity-User":["partial","*",1,0],"Google-Extended":["partial","*",1,0],"Applebot-Extended":["partial","*",1,0],"CCBot":["partial","*",1,0],"Bytespider":["partial","*",1,0],"Amazonbot":["partial","*",1,0],"meta-externalagent":["partial","*",1,0],"cohere-ai":["partial","*",1,0],"Diffbot":["partial","*",1,0],"ContextFetchBot":["partial","*",1,0]},"llms":{"status":"missing","http":404,"bytes":7162},"home":{"status":"ok","http":200,"url":"https://carrd.com/","html_tokens":6386,"markdown_tokens":224,"pruned_tokens":92}},
{"rank":1284,"domain":"cms.gov","origin":"https://cms.gov","robots":{"status":"ok","http":200,"url":"https://www.cms.gov/robots.txt","bytes":1558,"groups":1},"bots":{"GPTBot":["partial","*",14,21],"ChatGPT-User":["partial","*",14,21],"OAI-SearchBot":["partial","*",14,21],"ClaudeBot":["partial","*",14,21],"Claude-User":["partial","*",14,21],"Claude-SearchBot":["partial","*",14,21],"anthropic-ai":["partial","*",14,21],"PerplexityBot":["partial","*",14,21],"Perplexity-User":["partial","*",14,21],"Google-Extended":["partial","*",14,21],"Applebot-Extended":["partial","*",14,21],"CCBot":["partial","*",14,21],"Bytespider":["partial","*",14,21],"Amazonbot":["partial","*",14,21],"meta-externalagent":["partial","*",14,21],"cohere-ai":["partial","*",14,21],"Diffbot":["partial","*",14,21],"ContextFetchBot":["partial","*",14,21]},"llms":{"status":"missing","http":404,"bytes":54916},"home":{"status":"ok","http":200,"url":"https://www.cms.gov/","html_tokens":54857,"markdown_tokens":46,"pruned_tokens":46}},
{"rank":1285,"domain":"craigslist.org","origin":"https://craigslist.org","robots":{"status":"ok","http":200,"url":"https://craigslist.org/robots.txt","bytes":151,"groups":1},"bots":{"GPTBot":["partial","*",8,0],"ChatGPT-User":["partial","*",8,0],"OAI-SearchBot":["partial","*",8,0],"ClaudeBot":["partial","*",8,0],"Claude-User":["partial","*",8,0],"Claude-SearchBot":["partial","*",8,0],"anthropic-ai":["partial","*",8,0],"PerplexityBot":["partial","*",8,0],"Perplexity-User":["partial","*",8,0],"Google-Extended":["partial","*",8,0],"Applebot-Extended":["partial","*",8,0],"CCBot":["partial","*",8,0],"Bytespider":["partial","*",8,0],"Amazonbot":["partial","*",8,0],"meta-externalagent":["partial","*",8,0],"cohere-ai":["partial","*",8,0],"Diffbot":["partial","*",8,0],"ContextFetchBot":["partial","*",8,0]},"llms":{"status":"missing","http":404,"bytes":4544},"home":{"status":"ok","http":200,"url":"https://www.craigslist.org/area/orlando","html_tokens":15071,"markdown_tokens":50,"pruned_tokens":50}},
{"rank":1286,"domain":"bleepingcomputer.com","origin":"https://bleepingcomputer.com","robots":{"status":"ok","http":200,"url":"https://www.bleepingcomputer.com/robots.txt","bytes":3520,"groups":21},"bots":{"GPTBot":["partial","*",60,2],"ChatGPT-User":["partial","*",60,2],"OAI-SearchBot":["partial","*",60,2],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",60,2],"Claude-SearchBot":["partial","*",60,2],"anthropic-ai":["partial","*",60,2],"PerplexityBot":["partial","*",60,2],"Perplexity-User":["partial","*",60,2],"Google-Extended":["partial","*",60,2],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["partial","*",60,2],"Amazonbot":["partial","*",60,2],"meta-externalagent":["partial","*",60,2],"cohere-ai":["partial","*",60,2],"Diffbot":["partial","*",60,2],"ContextFetchBot":["partial","*",60,2]},"llms":{"status":"missing","http":404,"bytes":50607},"home":{"status":"ok","http":200,"url":"https://www.bleepingcomputer.com/","html_tokens":28857,"markdown_tokens":1068,"pruned_tokens":165}},
{"rank":1287,"domain":"counterpane.com","origin":"http://counterpane.com","robots":{"status":"ok","http":200,"url":"https://www.schneier.com/robots.txt","bytes":600,"groups":3},"bots":{"GPTBot":["partial","*",7,1],"ChatGPT-User":["partial","*",7,1],"OAI-SearchBot":["partial","*",7,1],"ClaudeBot":["partial","*",7,1],"Claude-User":["partial","*",7,1],"Claude-SearchBot":["partial","*",7,1],"anthropic-ai":["partial","*",7,1],"PerplexityBot":["partial","*",7,1],"Perplexity-User":["partial","*",7,1],"Google-Extended":["partial","*",7,1],"Applebot-Extended":["partial","*",7,1],"CCBot":["partial","*",7,1],"Bytespider":["partial","*",7,1],"Amazonbot":["partial","*",7,1],"meta-externalagent":["partial","*",7,1],"cohere-ai":["partial","*",7,1],"Diffbot":["partial","*",7,1],"ContextFetchBot":["partial","*",7,1]},"llms":{"status":"missing","http":404,"bytes":42807},"home":{"status":"timeout","error":"ConnectTimeout"}},
{"rank":1288,"domain":"foxbusiness.com","origin":"https://foxbusiness.com","robots":{"status":"ok","http":200,"url":"https://www.foxbusiness.com/robots.txt","bytes":201,"groups":1},"bots":{"GPTBot":["partial","*",4,0],"ChatGPT-User":["partial","*",4,0],"OAI-SearchBot":["partial","*",4,0],"ClaudeBot":["partial","*",4,0],"Claude-User":["partial","*",4,0],"Claude-SearchBot":["partial","*",4,0],"anthropic-ai":["partial","*",4,0],"PerplexityBot":["partial","*",4,0],"Perplexity-User":["partial","*",4,0],"Google-Extended":["partial","*",4,0],"Applebot-Extended":["partial","*",4,0],"CCBot":["partial","*",4,0],"Bytespider":["partial","*",4,0],"Amazonbot":["partial","*",4,0],"meta-externalagent":["partial","*",4,0],"cohere-ai":["partial","*",4,0],"Diffbot":["partial","*",4,0],"ContextFetchBot":["partial","*",4,0]},"llms":{"status":"missing","http":404,"bytes":0},"home":{"status":"ok","http":200,"url":"https://www.foxbusiness.com/","html_tokens":119534,"markdown_tokens":577,"pruned_tokens":155}},
{"rank":1289,"domain":"govinfo.gov","origin":"https://govinfo.gov","robots":{"status":"ok","http":200,"url":"https://www.govinfo.gov/robots.txt","bytes":5092,"groups":1},"bots":{"GPTBot":["partial","*",35,18],"ChatGPT-User":["partial","*",35,18],"OAI-SearchBot":["partial","*",35,18],"ClaudeBot":["partial","*",35,18],"Claude-User":["partial","*",35,18],"Claude-SearchBot":["partial","*",35,18],"anthropic-ai":["partial","*",35,18],"PerplexityBot":["partial","*",35,18],"Perplexity-User":["partial","*",35,18],"Google-Extended":["partial","*",35,18],"Applebot-Extended":["partial","*",35,18],"CCBot":["partial","*",35,18],"Bytespider":["partial","*",35,18],"Amazonbot":["partial","*",35,18],"meta-externalagent":["partial","*",35,18],"cohere-ai":["partial","*",35,18],"Diffbot":["partial","*",35,18],"ContextFetchBot":["partial","*",35,18]},"llms":{"status":"html","http":200,"bytes":65527},"home":{"status":"ok","http":200,"url":"https://www.govinfo.gov/","html_tokens":16382,"markdown_tokens":41,"pruned_tokens":41}},
{"rank":1290,"domain":"ijg.org","origin":"https://ijg.org","robots":{"status":"missing","http":404,"url":"https://ijg.org/robots.txt","bytes":601,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":601},"home":{"status":"ok","http":200,"url":"https://ijg.org/","html_tokens":384,"markdown_tokens":226,"pruned_tokens":71}},
{"rank":1291,"domain":"wikidot.com","origin":"https://wikidot.com","robots":{"status":"missing","http":404,"url":"https://wikidot.com/robots.txt","bytes":0,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":10006},"home":{"status":"ok","http":200,"url":"https://www.wikidot.com/","html_tokens":5577,"markdown_tokens":252,"pruned_tokens":128}},
{"rank":1292,"domain":"lh3.googleusercontent.com","origin":"https://lh3.googleusercontent.com","robots":{"status":"refused","http":400,"url":"https://lh3.googleusercontent.com/robots.txt","bytes":1555,"groups":0,"error":"HTTP 400"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":1293,"domain":"amazon.it","origin":"https://amazon.it","robots":{"status":"ok","http":200,"url":"https://www.amazon.it/robots.txt","bytes":7450,"groups":101},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["blocked","named",1,0],"OAI-SearchBot":["blocked","named",1,0],"ClaudeBot":["blocked","named",2,0],"Claude-User":["blocked","named",1,0],"Claude-SearchBot":["blocked","named",1,0],"anthropic-ai":["partial","*",99,8],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["blocked","named",1,0],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["partial","*",99,8],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["partial","*",99,8],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",99,8]},"llms":{"status":"missing","http":404,"bytes":2233},"home":{"status":"ok","http":200,"url":"https://www.amazon.it/-/en/","html_tokens":195371,"markdown_tokens":1125,"pruned_tokens":78}},
{"rank":1294,"domain":"ow.ly","origin":"https://ow.ly","robots":{"status":"html","http":200,"url":"https://www.origenonline.es/noticias-xii-feria-trufa-soria-cita-mas-importante-espana-para-conocer-diamante-negro-gastronomia/1/10313.html","bytes":277259,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":1295,"domain":"app.link","origin":"https://app.link","robots":{"status":"ok","http":200,"url":"https://app.link/robots.txt","bytes":29529,"groups":1},"bots":{"GPTBot":["partial","*",1,0],"ChatGPT-User":["partial","*",1,0],"OAI-SearchBot":["partial","*",1,0],"ClaudeBot":["partial","*",1,0],"Claude-User":["partial","*",1,0],"Claude-SearchBot":["partial","*",1,0],"anthropic-ai":["partial","*",1,0],"PerplexityBot":["partial","*",1,0],"Perplexity-User":["partial","*",1,0],"Google-Extended":["partial","*",1,0],"Applebot-Extended":["partial","*",1,0],"CCBot":["partial","*",1,0],"Bytespider":["partial","*",1,0],"Amazonbot":["partial","*",1,0],"meta-externalagent":["partial","*",1,0],"cohere-ai":["partial","*",1,0],"Diffbot":["partial","*",1,0],"ContextFetchBot":["partial","*",1,0]},"llms":{"status":"missing","http":404,"bytes":321},"home":{"status":"http_error","http":404,"url":"https://app.link/"}},
{"rank":1296,"domain":"unsw.edu.au","origin":"https://unsw.edu.au","robots":{"status":"refused","http":401,"url":"https://unsw.edu.au/robots.txt","bytes":381,"groups":0,"error":"HTTP 401"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":1297,"domain":"sibforms.com","origin":"https://sibforms.com","robots":{"status":"missing","http":404,"url":"https://sibforms.com/robots.txt","bytes":14,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":14},"home":{"status":"http_error","http":404,"url":"https://sibforms.com/"}},
{"rank":1298,"domain":"rsasecurity.com","origin":"https://rsasecurity.com","robots":{"status":"unreachable","http":null,"url":"https://rsasecurity.com/robots.txt","bytes":0,"groups":0,"error":"cannot resolve rsasecurity.com"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":1299,"domain":"cn.com","origin":"https://cn.com","robots":{"status":"ok","http":200,"url":"https://cn.com/robots.txt","bytes":0,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":7946},"home":{"status":"ok","http":200,"url":"https://cn.com/","html_tokens":10404,"markdown_tokens":928,"pruned_tokens":912}},
{"rank":1300,"domain":"udel.edu","origin":"https://udel.edu","robots":{"status":"ok","http":200,"url":"https://www.udel.edu/content/dam/robots.txt","bytes":71,"groups":1},"bots":{"GPTBot":["partial","*",1,0],"ChatGPT-User":["partial","*",1,0],"OAI-SearchBot":["partial","*",1,0],"ClaudeBot":["partial","*",1,0],"Claude-User":["partial","*",1,0],"Claude-SearchBot":["partial","*",1,0],"anthropic-ai":["partial","*",1,0],"PerplexityBot":["partial","*",1,0],"Perplexity-User":["partial","*",1,0],"Google-Extended":["partial","*",1,0],"Applebot-Extended":["partial","*",1,0],"CCBot":["partial","*",1,0],"Bytespider":["partial","*",1,0],"Amazonbot":["partial","*",1,0],"meta-externalagent":["partial","*",1,0],"cohere-ai":["partial","*",1,0],"Diffbot":["partial","*",1,0],"ContextFetchBot":["partial","*",1,0]},"llms":{"status":"missing","http":404,"bytes":20999},"home":{"status":"ok","http":200,"url":"https://www.udel.edu/","html_tokens":21838,"markdown_tokens":265,"pruned_tokens":175}},
{"rank":1301,"domain":"serverwatch.com","origin":"https://serverwatch.com","robots":{"status":"ok","http":200,"url":"https://www.serverwatch.com/robots.txt","bytes":143,"groups":1},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"present","http":200,"bytes":8169},"home":{"status":"http_error","http":403,"url":"https://www.serverwatch.com/"}},
{"rank":1302,"domain":"business-standard.com","origin":"https://business-standard.com","robots":{"status":"refused","http":403,"url":"https://www.business-standard.com/robots.txt","bytes":395,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":1303,"domain":"icloud.com","origin":"https://icloud.com","robots":{"status":"ok","http":200,"url":"https://www.icloud.com/robots.txt","bytes":1289,"groups":3},"bots":{"GPTBot":["partial","*",39,0],"ChatGPT-User":["partial","*",39,0],"OAI-SearchBot":["partial","*",39,0],"ClaudeBot":["partial","*",39,0],"Claude-User":["partial","*",39,0],"Claude-SearchBot":["partial","*",39,0],"anthropic-ai":["partial","*",39,0],"PerplexityBot":["partial","*",39,0],"Perplexity-User":["partial","*",39,0],"Google-Extended":["partial","*",39,0],"Applebot-Extended":["partial","*",39,0],"CCBot":["partial","*",39,0],"Bytespider":["partial","*",39,0],"Amazonbot":["partial","*",39,0],"meta-externalagent":["partial","*",39,0],"cohere-ai":["partial","*",39,0],"Diffbot":["partial","*",39,0],"ContextFetchBot":["partial","*",39,0]},"llms":{"status":"missing","http":404,"bytes":3250},"home":{"status":"ok","http":200,"url":"https://www.icloud.com/","html_tokens":7832,"markdown_tokens":0,"pruned_tokens":0}},
{"rank":1304,"domain":"uu.nl","origin":"https://uu.nl","robots":{"status":"ok","http":200,"url":"https://www.uu.nl/robots.txt","bytes":2084,"groups":1},"bots":{"GPTBot":["partial","*",37,18],"ChatGPT-User":["partial","*",37,18],"OAI-SearchBot":["partial","*",37,18],"ClaudeBot":["partial","*",37,18],"Claude-User":["partial","*",37,18],"Claude-SearchBot":["partial","*",37,18],"anthropic-ai":["partial","*",37,18],"PerplexityBot":["partial","*",37,18],"Perplexity-User":["partial","*",37,18],"Google-Extended":["partial","*",37,18],"Applebot-Extended":["partial","*",37,18],"CCBot":["partial","*",37,18],"Bytespider":["partial","*",37,18],"Amazonbot":["partial","*",37,18],"meta-externalagent":["partial","*",37,18],"cohere-ai":["partial","*",37,18],"Diffbot":["partial","*",37,18],"ContextFetchBot":["partial","*",37,18]},"llms":{"status":"missing","http":404,"bytes":154},"home":{"status":"ok","http":200,"url":"https://www.uu.nl/","html_tokens":9703,"markdown_tokens":18,"pruned_tokens":18}},
{"rank":1305,"domain":"html5up.net","origin":"https://html5up.net","robots":{"status":"ok","http":200,"url":"https://html5up.net/robots.txt","bytes":36,"groups":1},"bots":{"GPTBot":["partial","*",1,0],"ChatGPT-User":["partial","*",1,0],"OAI-SearchBot":["partial","*",1,0],"ClaudeBot":["partial","*",1,0],"Claude-User":["partial","*",1,0],"Claude-SearchBot":["partial","*",1,0],"anthropic-ai":["partial","*",1,0],"PerplexityBot":["partial","*",1,0],"Perplexity-User":["partial","*",1,0],"Google-Extended":["partial","*",1,0],"Applebot-Extended":["partial","*",1,0],"CCBot":["partial","*",1,0],"Bytespider":["partial","*",1,0],"Amazonbot":["partial","*",1,0],"meta-externalagent":["partial","*",1,0],"cohere-ai":["partial","*",1,0],"Diffbot":["partial","*",1,0],"ContextFetchBot":["partial","*",1,0]},"llms":{"status":"missing","http":404,"bytes":3401},"home":{"status":"ok","http":200,"url":"https://html5up.net/","html_tokens":7487,"markdown_tokens":60,"pruned_tokens":60}},
{"rank":1306,"domain":"gr.com","origin":"https://gr.com","robots":{"status":"unreachable","http":null,"url":"https://gr.com/robots.txt","bytes":0,"groups":0,"error":"[SSL: CERTIFICATE_VERIFY_FAILED] certificate verify failed: Hostname mismatch, certificate is not valid for 'gr.com'. (_ssl.c:1028)"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":1307,"domain":"verywellmind.com","origin":"https://verywellmind.com","robots":{"status":"ok","http":200,"url":"https://www.verywellmind.com/robots.txt","bytes":3477,"groups":6},"bots":{"GPTBot":["partial","named",1,0],"ChatGPT-User":["partial","named",1,0],"OAI-SearchBot":["partial","named",1,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["blocked","named",1,0],"Claude-SearchBot":["partial","named",1,2],"anthropic-ai":["partial","named",1,2],"PerplexityBot":["partial","named",1,2],"Perplexity-User":["blocked","named",1,0],"Google-Extended":["partial","named",1,2],"Applebot-Extended":["blocked","named",1,0],"CCBot":["partial","named",1,2],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["partial","named",1,2],"cohere-ai":["partial","named",1,2],"Diffbot":["partial","*",2,0],"ContextFetchBot":["partial","*",2,0]},"llms":{"status":"http_error","http":402,"bytes":612},"home":{"status":"http_error","http":402,"url":"https://www.verywellmind.com/"}},
{"rank":1308,"domain":"monday.com","origin":"https://monday.com","robots":{"status":"ok","http":200,"url":"https://monday.com/robots.txt","bytes":3091,"groups":7},"bots":{"GPTBot":["partial","*",21,2],"ChatGPT-User":["partial","named",21,0],"OAI-SearchBot":["partial","named",21,0],"ClaudeBot":["partial","named",21,0],"Claude-User":["partial","*",21,2],"Claude-SearchBot":["partial","*",21,2],"anthropic-ai":["partial","*",21,2],"PerplexityBot":["partial","named",21,0],"Perplexity-User":["partial","named",21,0],"Google-Extended":["partial","named",21,1],"Applebot-Extended":["partial","*",21,2],"CCBot":["partial","*",21,2],"Bytespider":["partial","*",21,2],"Amazonbot":["partial","*",21,2],"meta-externalagent":["partial","*",21,2],"cohere-ai":["partial","*",21,2],"Diffbot":["partial","*",21,2],"ContextFetchBot":["partial","*",21,2]},"llms":{"status":"present","http":200,"bytes":9903},"home":{"status":"ok","http":200,"url":"https://monday.com/","html_tokens":156102,"markdown_tokens":199,"pruned_tokens":32}},
{"rank":1309,"domain":"prweb.com","origin":"https://prweb.com","robots":{"status":"ok","http":200,"url":"https://www.prweb.com/robots.txt","bytes":462,"groups":1},"bots":{"GPTBot":["partial","*",10,0],"ChatGPT-User":["partial","*",10,0],"OAI-SearchBot":["partial","*",10,0],"ClaudeBot":["partial","*",10,0],"Claude-User":["partial","*",10,0],"Claude-SearchBot":["partial","*",10,0],"anthropic-ai":["partial","*",10,0],"PerplexityBot":["partial","*",10,0],"Perplexity-User":["partial","*",10,0],"Google-Extended":["partial","*",10,0],"Applebot-Extended":["partial","*",10,0],"CCBot":["partial","*",10,0],"Bytespider":["partial","*",10,0],"Amazonbot":["partial","*",10,0],"meta-externalagent":["partial","*",10,0],"cohere-ai":["partial","*",10,0],"Diffbot":["partial","*",10,0],"ContextFetchBot":["partial","*",10,0]},"llms":{"status":"missing","http":404,"bytes":621},"home":{"status":"ok","http":200,"url":"https://www.prweb.com/","html_tokens":39728,"markdown_tokens":111,"pruned_tokens":111}},
{"rank":1310,"domain":"wassenaar.org","origin":"https://wassenaar.org","robots":{"status":"html","http":200,"url":"https://www.wassenaar.org","bytes":53707,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":1311,"domain":"msnbc.com","origin":"https://msnbc.com","robots":{"status":"ok","http":200,"url":"https://www.ms.now/robots.txt","bytes":2305,"groups":50},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["blocked","named",1,0],"OAI-SearchBot":["blocked","named",1,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["blocked","named",1,0],"Claude-SearchBot":["partial","*",7,0],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["partial","*",7,0],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["partial","named",1,1],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",7,0]},"llms":{"status":"other_text","http":200,"bytes":30103},"home":{"status":"ok","http":200,"url":"https://www.ms.now/","html_tokens":124557,"markdown_tokens":85,"pruned_tokens":85}},
{"rank":1312,"domain":"coinmarketcap.com","origin":"https://coinmarketcap.com","robots":{"status":"ok","http":200,"url":"https://coinmarketcap.com/robots.txt","bytes":1115,"groups":2},"bots":{"GPTBot":["partial","*",12,1],"ChatGPT-User":["partial","*",12,1],"OAI-SearchBot":["partial","*",12,1],"ClaudeBot":["partial","*",12,1],"Claude-User":["partial","*",12,1],"Claude-SearchBot":["partial","*",12,1],"anthropic-ai":["partial","*",12,1],"PerplexityBot":["partial","*",12,1],"Perplexity-User":["partial","*",12,1],"Google-Extended":["partial","*",12,1],"Applebot-Extended":["partial","*",12,1],"CCBot":["partial","*",12,1],"Bytespider":["partial","*",12,1],"Amazonbot":["partial","*",12,1],"meta-externalagent":["partial","*",12,1],"cohere-ai":["partial","*",12,1],"Diffbot":["partial","*",12,1],"ContextFetchBot":["partial","*",12,1]},"llms":{"status":"present","http":200,"bytes":13144},"home":{"status":"ok","http":200,"url":"https://coinmarketcap.com/","html_tokens":235041,"markdown_tokens":2651,"pruned_tokens":84}},
{"rank":1313,"domain":"ama-assn.org","origin":"https://ama-assn.org","robots":{"status":"ok","http":200,"url":"https://www.ama-assn.org/robots.txt","bytes":2096,"groups":1},"bots":{"GPTBot":["partial","*",34,18],"ChatGPT-User":["partial","*",34,18],"OAI-SearchBot":["partial","*",34,18],"ClaudeBot":["partial","*",34,18],"Claude-User":["partial","*",34,18],"Claude-SearchBot":["partial","*",34,18],"anthropic-ai":["partial","*",34,18],"PerplexityBot":["partial","*",34,18],"Perplexity-User":["partial","*",34,18],"Google-Extended":["partial","*",34,18],"Applebot-Extended":["partial","*",34,18],"CCBot":["partial","*",34,18],"Bytespider":["partial","*",34,18],"Amazonbot":["partial","*",34,18],"meta-externalagent":["partial","*",34,18],"cohere-ai":["partial","*",34,18],"Diffbot":["partial","*",34,18],"ContextFetchBot":["partial","*",34,18]},"llms":{"status":"missing","http":404,"bytes":190},"home":{"status":"ok","http":200,"url":"https://www.ama-assn.org/","html_tokens":87147,"markdown_tokens":147,"pruned_tokens":107}},
{"rank":1314,"domain":"gandi.net","origin":"https://gandi.net","robots":{"status":"ok","http":200,"url":"https://www.gandi.net/robots.txt","bytes":9023,"groups":212},"bots":{"GPTBot":["partial","*",30,1],"ChatGPT-User":["partial","*",30,1],"OAI-SearchBot":["partial","*",30,1],"ClaudeBot":["partial","*",30,1],"Claude-User":["partial","*",30,1],"Claude-SearchBot":["partial","*",30,1],"anthropic-ai":["partial","*",30,1],"PerplexityBot":["partial","*",30,1],"Perplexity-User":["partial","*",30,1],"Google-Extended":["partial","*",30,1],"Applebot-Extended":["partial","*",30,1],"CCBot":["partial","*",30,1],"Bytespider":["partial","*",30,1],"Amazonbot":["partial","*",30,1],"meta-externalagent":["partial","*",30,1],"cohere-ai":["partial","*",30,1],"Diffbot":["partial","*",30,1],"ContextFetchBot":["partial","*",30,1]},"llms":{"status":"missing","http":404,"bytes":118289},"home":{"status":"ok","http":200,"url":"https://www.gandi.net/en-US","html_tokens":47357,"markdown_tokens":2288,"pruned_tokens":170}},
{"rank":1315,"domain":"all.accor.com","origin":"https://all.accor.com","robots":{"status":"ok","http":200,"url":"https://all.accor.com/robots.txt","bytes":600,"groups":1},"bots":{"GPTBot":["partial","*",22,0],"ChatGPT-User":["partial","*",22,0],"OAI-SearchBot":["partial","*",22,0],"ClaudeBot":["partial","*",22,0],"Claude-User":["partial","*",22,0],"Claude-SearchBot":["partial","*",22,0],"anthropic-ai":["partial","*",22,0],"PerplexityBot":["partial","*",22,0],"Perplexity-User":["partial","*",22,0],"Google-Extended":["partial","*",22,0],"Applebot-Extended":["partial","*",22,0],"CCBot":["partial","*",22,0],"Bytespider":["partial","*",22,0],"Amazonbot":["partial","*",22,0],"meta-externalagent":["partial","*",22,0],"cohere-ai":["partial","*",22,0],"Diffbot":["partial","*",22,0],"ContextFetchBot":["partial","*",22,0]},"llms":{"status":"present","http":200,"bytes":105718},"home":{"status":"ok","http":200,"url":"https://all.accor.com/a/en.html","html_tokens":286570,"markdown_tokens":14,"pruned_tokens":14}},
{"rank":1316,"domain":"scopus.com","origin":"https://scopus.com","robots":{"status":"refused","http":403,"url":"https://scopus.com/robots.txt","bytes":4543,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":1317,"domain":"or.th","origin":"https://or.th","robots":{"status":"unreachable","http":null,"url":"https://or.th/robots.txt","bytes":0,"groups":0,"error":"cannot resolve or.th"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":1318,"domain":"fastcgi.com","origin":"https://fastcgi.com","robots":{"status":"timeout","http":null,"url":"https://fastcgi.com/robots.txt","bytes":0,"groups":0,"error":"ConnectTimeout"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":1319,"domain":"ora.com","origin":"https://ora.com","robots":{"status":"ok","http":200,"url":"https://www.oreilly.com/robots.txt","bytes":3450,"groups":7},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["partial","named",1,1],"OAI-SearchBot":["partial","named",1,1],"ClaudeBot":["partial","named",1,1],"Claude-User":["partial","*",19,0],"Claude-SearchBot":["partial","named",1,1],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["partial","named",1,1],"Perplexity-User":["partial","named",1,1],"Google-Extended":["partial","named",1,1],"Applebot-Extended":["partial","named",1,1],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",19,0]},"llms":{"status":"present","http":200,"bytes":18762},"home":{"status":"ok","http":200,"url":"https://www.oreilly.com/","html_tokens":79011,"markdown_tokens":458,"pruned_tokens":148}},
{"rank":1320,"domain":"colostate.edu","origin":"https://www.colostate.edu","robots":{"status":"ok","http":200,"url":"https://www.colostate.edu/robots.txt","bytes":191,"groups":1},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"http_error","http":403,"bytes":5570},"home":{"status":"http_error","http":403,"url":"https://www.colostate.edu/"}},
{"rank":1321,"domain":"github.blog","origin":"https://github.blog","robots":{"status":"ok","http":200,"url":"https://github.blog/robots.txt","bytes":269,"groups":1},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"missing","http":404,"bytes":50},"home":{"status":"ok","http":200,"url":"https://github.blog/","html_tokens":72081,"markdown_tokens":619,"pruned_tokens":88}},
{"rank":1322,"domain":"zdf.de","origin":"https://zdf.de","robots":{"status":"ok","http":200,"url":"https://www.zdf.de/robots.txt","bytes":1619,"groups":1},"bots":{"GPTBot":["partial","named",12,0],"ChatGPT-User":["partial","named",12,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["partial","named",12,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["partial","named",12,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":127720},"home":{"status":"ok","http":200,"url":"https://www.zdf.de/","html_tokens":154535,"markdown_tokens":153,"pruned_tokens":31}},
{"rank":1323,"domain":"ebay.de","origin":"https://ebay.de","robots":{"status":"ok","http":200,"url":"https://www.ebay.de/robots.txt","bytes":15980,"groups":7},"bots":{"GPTBot":["partial","named",1,4],"ChatGPT-User":["partial","*",197,8],"OAI-SearchBot":["partial","*",197,8],"ClaudeBot":["partial","named",1,4],"Claude-User":["partial","*",197,8],"Claude-SearchBot":["partial","*",197,8],"anthropic-ai":["partial","named",1,4],"PerplexityBot":["partial","named",1,4],"Perplexity-User":["partial","*",197,8],"Google-Extended":["partial","*",197,8],"Applebot-Extended":["partial","named",1,4],"CCBot":["partial","named",1,4],"Bytespider":["partial","named",1,4],"Amazonbot":["partial","named",1,4],"meta-externalagent":["partial","named",1,4],"cohere-ai":["partial","*",197,8],"Diffbot":["partial","*",197,8],"ContextFetchBot":["partial","*",197,8]},"llms":{"status":"missing","http":404,"bytes":80146},"home":{"status":"http_error","http":403,"url":"https://www.ebay.de/"}},
{"rank":1324,"domain":"adweek.com","origin":"https://adweek.com","robots":{"status":"ok","http":200,"url":"https://www.adweek.com/robots.txt","bytes":9250,"groups":89},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","named",0,1],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":179944},"home":{"status":"ok","http":200,"url":"https://www.adweek.com/","html_tokens":96796,"markdown_tokens":136,"pruned_tokens":82}},
{"rank":1325,"domain":"campaign-archive.com","origin":"https://campaign-archive.com","robots":{"status":"unreachable","http":null,"url":"https://campaign-archive.com/robots.txt","bytes":0,"groups":0,"error":"[SSL: CERTIFICATE_VERIFY_FAILED] certificate verify failed: Hostname mismatch, certificate is not valid for 'campaign-archive.com'. (_ssl.c:1028)"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":1326,"domain":"google.com.br","origin":"https://google.com.br","robots":{"status":"ok","http":200,"url":"https://www.google.com.br/robots.txt","bytes":6624,"groups":4},"bots":{"GPTBot":["partial","*",169,64],"ChatGPT-User":["partial","*",169,64],"OAI-SearchBot":["partial","*",169,64],"ClaudeBot":["partial","*",169,64],"Claude-User":["partial","*",169,64],"Claude-SearchBot":["partial","*",169,64],"anthropic-ai":["partial","*",169,64],"PerplexityBot":["partial","*",169,64],"Perplexity-User":["partial","*",169,64],"Google-Extended":["partial","*",169,64],"Applebot-Extended":["partial","*",169,64],"CCBot":["partial","*",169,64],"Bytespider":["partial","*",169,64],"Amazonbot":["partial","*",169,64],"meta-externalagent":["partial","*",169,64],"cohere-ai":["partial","*",169,64],"Diffbot":["partial","*",169,64],"ContextFetchBot":["partial","*",169,64]},"llms":{"status":"missing","http":404,"bytes":1569},"home":{"status":"ok","http":200,"url":"https://www.google.com/","html_tokens":21306,"markdown_tokens":30,"pruned_tokens":30}},
{"rank":1327,"domain":"straitstimes.com","origin":"https://straitstimes.com","robots":{"status":"ok","http":200,"url":"https://www.straitstimes.com/robots.txt","bytes":3237,"groups":3},"bots":{"GPTBot":["partial","*",87,0],"ChatGPT-User":["partial","*",87,0],"OAI-SearchBot":["partial","*",87,0],"ClaudeBot":["partial","*",87,0],"Claude-User":["partial","*",87,0],"Claude-SearchBot":["partial","*",87,0],"anthropic-ai":["partial","*",87,0],"PerplexityBot":["partial","*",87,0],"Perplexity-User":["partial","*",87,0],"Google-Extended":["partial","named",1,1],"Applebot-Extended":["partial","*",87,0],"CCBot":["partial","*",87,0],"Bytespider":["partial","*",87,0],"Amazonbot":["partial","*",87,0],"meta-externalagent":["partial","*",87,0],"cohere-ai":["partial","*",87,0],"Diffbot":["partial","*",87,0],"ContextFetchBot":["partial","*",87,0]},"llms":{"status":"missing","http":404,"bytes":9},"home":{"status":"ok","http":200,"url":"https://www.straitstimes.com/global","html_tokens":78130,"markdown_tokens":9,"pruned_tokens":9}},
{"rank":1328,"domain":"freetds.org","origin":"https://www.freetds.org","robots":{"status":"missing","http":404,"url":"https://www.freetds.org/robots.txt","bytes":236,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":236},"home":{"status":"ok","http":200,"url":"https://www.freetds.org/","html_tokens":904,"markdown_tokens":266,"pruned_tokens":134}},
{"rank":1329,"domain":"tagesschau.de","origin":"https://tagesschau.de","robots":{"status":"ok","http":200,"url":"https://www.tagesschau.de/robots.txt","bytes":2572,"groups":30},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["partial","*",3,0],"OAI-SearchBot":["partial","*",3,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",3,0],"Claude-SearchBot":["partial","*",3,0],"anthropic-ai":["partial","*",3,0],"PerplexityBot":["partial","*",3,0],"Perplexity-User":["partial","*",3,0],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["partial","*",3,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["partial","*",3,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",3,0]},"llms":{"status":"missing","http":404,"bytes":360834},"home":{"status":"ok","http":200,"url":"https://www.tagesschau.de/","html_tokens":352591,"markdown_tokens":12,"pruned_tokens":12}},
{"rank":1330,"domain":"pdflib.com","origin":"https://pdflib.com","robots":{"status":"ok","http":200,"url":"https://www.pdflib.com/robots.txt","bytes":14,"groups":1},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"missing","http":404,"bytes":77276},"home":{"status":"ok","http":200,"url":"https://www.pdflib.com/","html_tokens":19566,"markdown_tokens":104,"pruned_tokens":104}},
{"rank":1331,"domain":"elle.com","origin":"https://elle.com","robots":{"status":"ok","http":200,"url":"https://elle.com/robots.txt","bytes":953,"groups":3},"bots":{"GPTBot":["partial","named",32,0],"ChatGPT-User":["partial","named",32,0],"OAI-SearchBot":["partial","named",32,0],"ClaudeBot":["partial","*",19,0],"Claude-User":["partial","*",19,0],"Claude-SearchBot":["partial","*",19,0],"anthropic-ai":["partial","*",19,0],"PerplexityBot":["partial","*",19,0],"Perplexity-User":["partial","*",19,0],"Google-Extended":["partial","*",19,0],"Applebot-Extended":["partial","*",19,0],"CCBot":["partial","*",19,0],"Bytespider":["partial","*",19,0],"Amazonbot":["partial","*",19,0],"meta-externalagent":["partial","*",19,0],"cohere-ai":["partial","*",19,0],"Diffbot":["partial","*",19,0],"ContextFetchBot":["partial","*",19,0]},"llms":{"status":"missing","http":404,"bytes":84},"home":{"status":"ok","http":200,"url":"https://www.elle.com/","html_tokens":329528,"markdown_tokens":27,"pruned_tokens":27}},
{"rank":1332,"domain":"tsinghua.edu.cn","origin":"https://www.tsinghua.edu.cn","robots":{"status":"missing","http":404,"url":"https://www.tsinghua.edu.cn/robots.txt","bytes":1693,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":1693},"home":{"status":"ok","http":200,"url":"https://www.tsinghua.edu.cn/","html_tokens":22878,"markdown_tokens":102,"pruned_tokens":102}},
{"rank":1333,"domain":"washingtontimes.com","origin":"https://washingtontimes.com","robots":{"status":"ok","http":200,"url":"https://www.washingtontimes.com/robots.txt","bytes":3694,"groups":16},"bots":{"GPTBot":["partial","*",20,1],"ChatGPT-User":["partial","*",20,1],"OAI-SearchBot":["partial","*",20,1],"ClaudeBot":["partial","*",20,1],"Claude-User":["partial","*",20,1],"Claude-SearchBot":["partial","*",20,1],"anthropic-ai":["partial","*",20,1],"PerplexityBot":["partial","*",20,1],"Perplexity-User":["partial","*",20,1],"Google-Extended":["partial","*",20,1],"Applebot-Extended":["partial","*",20,1],"CCBot":["blocked","named",1,0],"Bytespider":["partial","*",20,1],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["partial","*",20,1],"cohere-ai":["partial","*",20,1],"Diffbot":["partial","*",20,1],"ContextFetchBot":["partial","*",20,1]},"llms":{"status":"http_error","http":403,"bytes":5558},"home":{"status":"ok","http":200,"url":"https://www.washingtontimes.com/","html_tokens":92697,"markdown_tokens":92,"pruned_tokens":92}},
{"rank":1334,"domain":"rfi.fr","origin":"https://rfi.fr","robots":{"status":"ok","http":200,"url":"https://www.rfi.fr/robots.txt","bytes":14105,"groups":328},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["blocked","named",1,0],"OAI-SearchBot":["blocked","named",1,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["blocked","named",1,0],"Claude-SearchBot":["blocked","named",1,0],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["blocked","named",2,0],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["partial","*",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["partial","*",1,0],"Amazonbot":["partial","*",1,0],"meta-externalagent":["partial","*",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["partial","*",1,0],"ContextFetchBot":["partial","*",1,0]},"llms":{"status":"http_error","http":409,"bytes":248},"home":{"status":"ok","http":200,"url":"https://www.rfi.fr/en/","html_tokens":131675,"markdown_tokens":16,"pruned_tokens":16}},
{"rank":1335,"domain":"coinbase.com","origin":"https://coinbase.com","robots":{"status":"ok","http":200,"url":"https://www.coinbase.com/robots.txt","bytes":3282,"groups":9},"bots":{"GPTBot":["partial","*",22,1],"ChatGPT-User":["partial","*",22,1],"OAI-SearchBot":["partial","*",22,1],"ClaudeBot":["partial","*",22,1],"Claude-User":["partial","*",22,1],"Claude-SearchBot":["partial","*",22,1],"anthropic-ai":["partial","*",22,1],"PerplexityBot":["partial","*",22,1],"Perplexity-User":["partial","*",22,1],"Google-Extended":["partial","*",22,1],"Applebot-Extended":["partial","*",22,1],"CCBot":["partial","*",22,1],"Bytespider":["partial","*",22,1],"Amazonbot":["partial","*",22,1],"meta-externalagent":["partial","*",22,1],"cohere-ai":["partial","*",22,1],"Diffbot":["partial","*",22,1],"ContextFetchBot":["partial","*",22,1]},"llms":{"status":"present","http":200,"bytes":7321},"home":{"status":"http_error","http":403,"url":"https://www.coinbase.com/"}},
{"rank":1336,"domain":"hwg.org","origin":"https://hwg.org","robots":{"status":"ok","http":200,"url":"https://hwg.org/robots.txt","bytes":139,"groups":1},"bots":{"GPTBot":["partial","*",1,0],"ChatGPT-User":["partial","*",1,0],"OAI-SearchBot":["partial","*",1,0],"ClaudeBot":["partial","*",1,0],"Claude-User":["partial","*",1,0],"Claude-SearchBot":["partial","*",1,0],"anthropic-ai":["partial","*",1,0],"PerplexityBot":["partial","*",1,0],"Perplexity-User":["partial","*",1,0],"Google-Extended":["partial","*",1,0],"Applebot-Extended":["partial","*",1,0],"CCBot":["partial","*",1,0],"Bytespider":["partial","*",1,0],"Amazonbot":["partial","*",1,0],"meta-externalagent":["partial","*",1,0],"cohere-ai":["partial","*",1,0],"Diffbot":["partial","*",1,0],"ContextFetchBot":["partial","*",1,0]},"llms":{"status":"crawl_delay","error":"Crawl-delay 120 s"},"home":{"status":"crawl_delay","error":"Crawl-delay 120 s"}},
{"rank":1337,"domain":"royalsocietypublishing.org","origin":"https://royalsocietypublishing.org","robots":{"status":"ok","http":200,"url":"https://royalsocietypublishing.org/robots.txt","bytes":1195,"groups":2},"bots":{"GPTBot":["partial","*",36,3],"ChatGPT-User":["partial","*",36,3],"OAI-SearchBot":["partial","*",36,3],"ClaudeBot":["partial","*",36,3],"Claude-User":["partial","*",36,3],"Claude-SearchBot":["partial","*",36,3],"anthropic-ai":["partial","*",36,3],"PerplexityBot":["partial","*",36,3],"Perplexity-User":["partial","*",36,3],"Google-Extended":["partial","*",36,3],"Applebot-Extended":["partial","*",36,3],"CCBot":["partial","*",36,3],"Bytespider":["partial","*",36,3],"Amazonbot":["partial","*",36,3],"meta-externalagent":["partial","*",36,3],"cohere-ai":["partial","*",36,3],"Diffbot":["partial","*",36,3],"ContextFetchBot":["partial","*",36,3]},"llms":{"status":"http_error","http":403,"bytes":5537},"home":{"status":"http_error","http":403,"url":"https://royalsocietypublishing.org/"}},
{"rank":1338,"domain":"flipsnack.com","origin":"https://flipsnack.com","robots":{"status":"ok","http":200,"url":"https://www.flipsnack.com/robots.txt","bytes":375,"groups":3},"bots":{"GPTBot":["partial","*",7,0],"ChatGPT-User":["partial","*",7,0],"OAI-SearchBot":["partial","*",7,0],"ClaudeBot":["partial","*",7,0],"Claude-User":["partial","*",7,0],"Claude-SearchBot":["partial","*",7,0],"anthropic-ai":["partial","*",7,0],"PerplexityBot":["partial","*",7,0],"Perplexity-User":["partial","*",7,0],"Google-Extended":["partial","*",7,0],"Applebot-Extended":["partial","*",7,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["partial","*",7,0],"meta-externalagent":["partial","*",7,0],"cohere-ai":["partial","*",7,0],"Diffbot":["partial","*",7,0],"ContextFetchBot":["partial","*",7,0]},"llms":{"status":"missing","http":404,"bytes":6605},"home":{"status":"ok","http":200,"url":"https://www.flipsnack.com/","html_tokens":52847,"markdown_tokens":1733,"pruned_tokens":954}},
{"rank":1339,"domain":"ti.com","origin":"https://ti.com","robots":{"status":"ok","http":200,"url":"https://www.ti.com/robots.txt","bytes":1284,"groups":1},"bots":{"GPTBot":["partial","*",28,0],"ChatGPT-User":["partial","*",28,0],"OAI-SearchBot":["partial","*",28,0],"ClaudeBot":["partial","*",28,0],"Claude-User":["partial","*",28,0],"Claude-SearchBot":["partial","*",28,0],"anthropic-ai":["partial","*",28,0],"PerplexityBot":["partial","*",28,0],"Perplexity-User":["partial","*",28,0],"Google-Extended":["partial","*",28,0],"Applebot-Extended":["partial","*",28,0],"CCBot":["partial","*",28,0],"Bytespider":["partial","*",28,0],"Amazonbot":["partial","*",28,0],"meta-externalagent":["partial","*",28,0],"cohere-ai":["partial","*",28,0],"Diffbot":["partial","*",28,0],"ContextFetchBot":["partial","*",28,0]},"llms":{"status":"missing","http":404,"bytes":3540},"home":{"status":"ok","http":200,"url":"https://www.ti.com/","html_tokens":16329,"markdown_tokens":97,"pruned_tokens":54}},
{"rank":1340,"domain":"youradchoices.com","origin":"https://youradchoices.com","robots":{"status":"ok","http":200,"url":"https://youradchoices.com/robots.txt","bytes":2713,"groups":1},"bots":{"GPTBot":["partial","*",54,32],"ChatGPT-User":["partial","*",54,32],"OAI-SearchBot":["partial","*",54,32],"ClaudeBot":["partial","*",54,32],"Claude-User":["partial","*",54,32],"Claude-SearchBot":["partial","*",54,32],"anthropic-ai":["partial","*",54,32],"PerplexityBot":["partial","*",54,32],"Perplexity-User":["partial","*",54,32],"Google-Extended":["partial","*",54,32],"Applebot-Extended":["partial","*",54,32],"CCBot":["partial","*",54,32],"Bytespider":["partial","*",54,32],"Amazonbot":["partial","*",54,32],"meta-externalagent":["partial","*",54,32],"cohere-ai":["partial","*",54,32],"Diffbot":["partial","*",54,32],"ContextFetchBot":["partial","*",54,32]},"llms":{"status":"missing","http":404,"bytes":291},"home":{"status":"ok","http":200,"url":"https://youradchoices.com/","html_tokens":8735,"markdown_tokens":44,"pruned_tokens":44}},
{"rank":1341,"domain":"neilpatel.com","origin":"https://neilpatel.com","robots":{"status":"ok","http":200,"url":"https://neilpatel.com/robots.txt","bytes":573,"groups":3},"bots":{"GPTBot":["partial","*",13,1],"ChatGPT-User":["partial","*",13,1],"OAI-SearchBot":["partial","*",13,1],"ClaudeBot":["partial","*",13,1],"Claude-User":["partial","*",13,1],"Claude-SearchBot":["partial","*",13,1],"anthropic-ai":["partial","*",13,1],"PerplexityBot":["partial","*",13,1],"Perplexity-User":["partial","*",13,1],"Google-Extended":["partial","*",13,1],"Applebot-Extended":["partial","*",13,1],"CCBot":["partial","*",13,1],"Bytespider":["partial","*",13,1],"Amazonbot":["partial","*",13,1],"meta-externalagent":["partial","*",13,1],"cohere-ai":["partial","*",13,1],"Diffbot":["partial","*",13,1],"ContextFetchBot":["partial","*",13,1]},"llms":{"status":"other_text","http":200,"bytes":8682},"home":{"status":"http_error","http":403,"url":"https://neilpatel.com/"}},
{"rank":1342,"domain":"dropboxusercontent.com","origin":"https://dropboxusercontent.com","robots":{"status":"unreachable","http":null,"url":"https://dropboxusercontent.com/robots.txt","bytes":0,"groups":0,"error":"cannot resolve dropboxusercontent.com"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":1343,"domain":"pku.edu.cn","origin":"https://www.pku.edu.cn","robots":{"status":"missing","http":404,"url":"https://www.pku.edu.cn/robots.txt","bytes":146,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":146},"home":{"status":"ok","http":200,"url":"https://www.pku.edu.cn/","html_tokens":25167,"markdown_tokens":37,"pruned_tokens":37}},
{"rank":1344,"domain":"opensuse.org","origin":"https://opensuse.org","robots":{"status":"missing","http":404,"url":"https://www.opensuse.org/robots.txt","bytes":146,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":146},"home":{"status":"ok","http":200,"url":"https://www.opensuse.org/","html_tokens":15909,"markdown_tokens":27,"pruned_tokens":27}},
{"rank":1345,"domain":"aap.org","origin":"https://aap.org","robots":{"status":"refused","http":403,"url":"https://www.aap.org/robots.txt","bytes":5506,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":1346,"domain":"u-tokyo.ac.jp","origin":"https://u-tokyo.ac.jp","robots":{"status":"unreachable","http":null,"url":"https://u-tokyo.ac.jp/robots.txt","bytes":0,"groups":0,"error":"cannot resolve u-tokyo.ac.jp"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":1347,"domain":"cnblogs.com","origin":"https://cnblogs.com","robots":{"status":"ok","http":200,"url":"https://www.cnblogs.com/robots.txt","bytes":69,"groups":1},"bots":{"GPTBot":["allowed","*",0,1],"ChatGPT-User":["allowed","*",0,1],"OAI-SearchBot":["allowed","*",0,1],"ClaudeBot":["allowed","*",0,1],"Claude-User":["allowed","*",0,1],"Claude-SearchBot":["allowed","*",0,1],"anthropic-ai":["allowed","*",0,1],"PerplexityBot":["allowed","*",0,1],"Perplexity-User":["allowed","*",0,1],"Google-Extended":["allowed","*",0,1],"Applebot-Extended":["allowed","*",0,1],"CCBot":["allowed","*",0,1],"Bytespider":["allowed","*",0,1],"Amazonbot":["allowed","*",0,1],"meta-externalagent":["allowed","*",0,1],"cohere-ai":["allowed","*",0,1],"Diffbot":["allowed","*",0,1],"ContextFetchBot":["allowed","*",0,1]},"llms":{"status":"missing","http":404,"bytes":1074},"home":{"status":"ok","http":200,"url":"https://www.cnblogs.com/","html_tokens":18919,"markdown_tokens":195,"pruned_tokens":78}},
{"rank":1348,"domain":"apache-ssl.org","origin":"https://apache-ssl.org","robots":{"status":"html","http":200,"url":"https://www.reddit.com/r/DataRecoveryHelp/","bytes":8416,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":1349,"domain":"nielsen.com","origin":"https://nielsen.com","robots":{"status":"ok","http":200,"url":"https://www.nielsen.com/robots.txt","bytes":205,"groups":2},"bots":{"GPTBot":["partial","*",1,0],"ChatGPT-User":["partial","*",1,0],"OAI-SearchBot":["partial","*",1,0],"ClaudeBot":["partial","*",1,0],"Claude-User":["partial","*",1,0],"Claude-SearchBot":["partial","*",1,0],"anthropic-ai":["partial","*",1,0],"PerplexityBot":["partial","*",1,0],"Perplexity-User":["partial","*",1,0],"Google-Extended":["partial","*",1,0],"Applebot-Extended":["partial","*",1,0],"CCBot":["partial","*",1,0],"Bytespider":["partial","*",1,0],"Amazonbot":["partial","*",1,0],"meta-externalagent":["partial","*",1,0],"cohere-ai":["partial","*",1,0],"Diffbot":["partial","*",1,0],"ContextFetchBot":["partial","*",1,0]},"llms":{"status":"present","http":200,"bytes":36822},"home":{"status":"ok","http":200,"url":"https://www.nielsen.com/","html_tokens":196829,"markdown_tokens":180,"pruned_tokens":114}},
{"rank":1350,"domain":"liveinternet.ru","origin":"https://liveinternet.ru","robots":{"status":"ok","http":200,"url":"https://www.liveinternet.ru/robots.txt","bytes":1193,"groups":2},"bots":{"GPTBot":["partial","*",30,0],"ChatGPT-User":["partial","*",30,0],"OAI-SearchBot":["partial","*",30,0],"ClaudeBot":["partial","*",30,0],"Claude-User":["partial","*",30,0],"Claude-SearchBot":["partial","*",30,0],"anthropic-ai":["partial","*",30,0],"PerplexityBot":["partial","*",30,0],"Perplexity-User":["partial","*",30,0],"Google-Extended":["partial","*",30,0],"Applebot-Extended":["partial","*",30,0],"CCBot":["partial","*",30,0],"Bytespider":["partial","*",30,0],"Amazonbot":["partial","*",30,0],"meta-externalagent":["partial","*",30,0],"cohere-ai":["partial","*",30,0],"Diffbot":["partial","*",30,0],"ContextFetchBot":["partial","*",30,0]},"llms":{"status":"missing","http":404,"bytes":2984},"home":{"status":"ok","http":200,"url":"https://www.liveinternet.ru/","html_tokens":8038,"markdown_tokens":47,"pruned_tokens":47}},
{"rank":1351,"domain":"popsci.com","origin":"https://popsci.com","robots":{"status":"ok","http":200,"url":"https://www.popsci.com/robots.txt","bytes":311,"groups":2},"bots":{"GPTBot":["partial","*",6,1],"ChatGPT-User":["partial","*",6,1],"OAI-SearchBot":["partial","*",6,1],"ClaudeBot":["partial","*",6,1],"Claude-User":["partial","*",6,1],"Claude-SearchBot":["partial","*",6,1],"anthropic-ai":["partial","*",6,1],"PerplexityBot":["partial","*",6,1],"Perplexity-User":["partial","*",6,1],"Google-Extended":["partial","*",6,1],"Applebot-Extended":["partial","*",6,1],"CCBot":["partial","*",6,1],"Bytespider":["partial","*",6,1],"Amazonbot":["partial","*",6,1],"meta-externalagent":["partial","*",6,1],"cohere-ai":["partial","*",6,1],"Diffbot":["partial","*",6,1],"ContextFetchBot":["partial","*",6,1]},"llms":{"status":"other_text","http":200,"bytes":2498},"home":{"status":"ok","http":200,"url":"https://www.popsci.com/","html_tokens":36722,"markdown_tokens":211,"pruned_tokens":147}},
{"rank":1352,"domain":"worldometers.info","origin":"https://worldometers.info","robots":{"status":"ok","http":200,"url":"https://www.worldometers.info/robots.txt","bytes":24,"groups":1},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"missing","http":404,"bytes":74729},"home":{"status":"ok","http":200,"url":"https://www.worldometers.info/","html_tokens":58298,"markdown_tokens":317,"pruned_tokens":16}},
{"rank":1353,"domain":"ucsf.edu","origin":"https://ucsf.edu","robots":{"status":"ok","http":200,"url":"https://www.ucsf.edu/robots.txt","bytes":2329,"groups":2},"bots":{"GPTBot":["partial","*",35,18],"ChatGPT-User":["partial","*",35,18],"OAI-SearchBot":["partial","*",35,18],"ClaudeBot":["partial","*",35,18],"Claude-User":["partial","*",35,18],"Claude-SearchBot":["partial","*",35,18],"anthropic-ai":["partial","*",35,18],"PerplexityBot":["partial","*",35,18],"Perplexity-User":["partial","*",35,18],"Google-Extended":["partial","*",35,18],"Applebot-Extended":["partial","*",35,18],"CCBot":["partial","*",35,18],"Bytespider":["partial","*",35,18],"Amazonbot":["partial","*",35,18],"meta-externalagent":["partial","*",35,18],"cohere-ai":["partial","*",35,18],"Diffbot":["partial","*",35,18],"ContextFetchBot":["partial","*",35,18]},"llms":{"status":"present","http":200,"bytes":1898952},"home":{"status":"ok","http":200,"url":"https://www.ucsf.edu/","html_tokens":29200,"markdown_tokens":54,"pruned_tokens":54}},
{"rank":1354,"domain":"force.com","origin":"https://force.com","robots":{"status":"refused","http":403,"url":"https://www.force.com/robots.txt","bytes":280,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":1355,"domain":"imageshack.com","origin":"https://imageshack.com","robots":{"status":"ok","http":200,"url":"https://imageshack.com/robots.txt","bytes":2517,"groups":2},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["blocked","named",1,0],"OAI-SearchBot":["blocked","named",1,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["blocked","named",1,0],"Claude-SearchBot":["blocked","named",1,0],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["blocked","named",1,0],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",1,0]},"llms":{"status":"missing","http":404,"bytes":6748},"home":{"status":"ok","http":200,"url":"https://imageshack.com/","html_tokens":1896,"markdown_tokens":26,"pruned_tokens":26}},
{"rank":1356,"domain":"bestbuy.com","origin":"https://www.bestbuy.com","robots":{"status":"ok","http":200,"url":"https://www.bestbuy.com/robots.txt","bytes":15016,"groups":5},"bots":{"GPTBot":["partial","named",137,2],"ChatGPT-User":["partial","named",137,2],"OAI-SearchBot":["partial","named",137,2],"ClaudeBot":["partial","named",137,2],"Claude-User":["partial","named",137,2],"Claude-SearchBot":["partial","*",182,2],"anthropic-ai":["partial","named",1,0],"PerplexityBot":["partial","named",137,2],"Perplexity-User":["partial","named",137,2],"Google-Extended":["partial","named",137,2],"Applebot-Extended":["partial","*",182,2],"CCBot":["partial","named",1,0],"Bytespider":["partial","named",1,0],"Amazonbot":["partial","named",137,2],"meta-externalagent":["partial","*",182,2],"cohere-ai":["partial","*",182,2],"Diffbot":["partial","*",182,2],"ContextFetchBot":["partial","*",182,2]},"llms":{"status":"missing","http":404,"bytes":287240},"home":{"status":"ok","http":200,"url":"https://www.bestbuy.com/","html_tokens":134573,"markdown_tokens":16,"pruned_tokens":16}},
{"rank":1357,"domain":"vanderbilt.edu","origin":"https://vanderbilt.edu","robots":{"status":"missing","http":404,"url":"https://www.vanderbilt.edu/robots.txt","bytes":108458,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":108458},"home":{"status":"ok","http":200,"url":"https://www.vanderbilt.edu/","html_tokens":42623,"markdown_tokens":343,"pruned_tokens":252}},
{"rank":1358,"domain":"dyndns.org","origin":"https://dyndns.org","robots":{"status":"html","http":200,"url":"https://account.dyn.com/","bytes":28453,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":1359,"domain":"nhtsa.gov","origin":"https://nhtsa.gov","robots":{"status":"refused","http":403,"url":"https://www.nhtsa.gov/robots.txt","bytes":381,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":1360,"domain":"mn.us","origin":"https://mn.us","robots":{"status":"unreachable","http":null,"url":"https://mn.us/robots.txt","bytes":0,"groups":0,"error":"cannot resolve mn.us"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":1361,"domain":"archdaily.com","origin":"https://archdaily.com","robots":{"status":"ok","http":200,"url":"https://www.archdaily.com/robots.txt","bytes":466,"groups":3},"bots":{"GPTBot":["partial","*",15,1],"ChatGPT-User":["partial","*",15,1],"OAI-SearchBot":["partial","*",15,1],"ClaudeBot":["partial","*",15,1],"Claude-User":["partial","*",15,1],"Claude-SearchBot":["partial","*",15,1],"anthropic-ai":["partial","*",15,1],"PerplexityBot":["partial","*",15,1],"Perplexity-User":["partial","*",15,1],"Google-Extended":["partial","*",15,1],"Applebot-Extended":["partial","*",15,1],"CCBot":["partial","*",15,1],"Bytespider":["partial","*",15,1],"Amazonbot":["partial","*",15,1],"meta-externalagent":["partial","*",15,1],"cohere-ai":["partial","*",15,1],"Diffbot":["partial","*",15,1],"ContextFetchBot":["partial","*",15,1]},"llms":{"status":"empty","http":204,"bytes":0},"home":{"status":"ok","http":200,"url":"https://www.archdaily.com/","html_tokens":263699,"markdown_tokens":373,"pruned_tokens":148}},
{"rank":1362,"domain":"sitemaps.org","origin":"https://sitemaps.org","robots":{"status":"ok","http":200,"url":"https://sitemaps.org/robots.txt","bytes":45,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":103},"home":{"status":"ok","http":200,"url":"https://sitemaps.org/","html_tokens":1101,"markdown_tokens":267,"pruned_tokens":112}},
{"rank":1363,"domain":"iu.edu","origin":"https://iu.edu","robots":{"status":"ok","http":200,"url":"https://iu.edu/robots.txt","bytes":7426,"groups":3},"bots":{"GPTBot":["partial","*",193,0],"ChatGPT-User":["partial","*",193,0],"OAI-SearchBot":["partial","*",193,0],"ClaudeBot":["partial","*",193,0],"Claude-User":["partial","*",193,0],"Claude-SearchBot":["partial","*",193,0],"anthropic-ai":["partial","*",193,0],"PerplexityBot":["partial","*",193,0],"Perplexity-User":["partial","*",193,0],"Google-Extended":["partial","*",193,0],"Applebot-Extended":["partial","*",193,0],"CCBot":["partial","*",193,0],"Bytespider":["partial","*",193,0],"Amazonbot":["partial","*",193,0],"meta-externalagent":["partial","*",193,0],"cohere-ai":["partial","*",193,0],"Diffbot":["partial","*",193,0],"ContextFetchBot":["partial","*",193,0]},"llms":{"status":"missing","http":404,"bytes":95881},"home":{"status":"ok","http":200,"url":"https://iu.edu/","html_tokens":30145,"markdown_tokens":524,"pruned_tokens":297}},
{"rank":1364,"domain":"propublica.org","origin":"https://propublica.org","robots":{"status":"ok","http":200,"url":"https://www.propublica.org/robots.txt","bytes":190,"groups":1},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"missing","http":404,"bytes":122687},"home":{"status":"ok","http":200,"url":"https://www.propublica.org/","html_tokens":91802,"markdown_tokens":1039,"pruned_tokens":61}},
{"rank":1365,"domain":"rice.edu","origin":"https://www.rice.edu","robots":{"status":"ok","http":200,"url":"https://www.rice.edu/robots.txt","bytes":1913,"groups":1},"bots":{"GPTBot":["partial","*",31,18],"ChatGPT-User":["partial","*",31,18],"OAI-SearchBot":["partial","*",31,18],"ClaudeBot":["partial","*",31,18],"Claude-User":["partial","*",31,18],"Claude-SearchBot":["partial","*",31,18],"anthropic-ai":["partial","*",31,18],"PerplexityBot":["partial","*",31,18],"Perplexity-User":["partial","*",31,18],"Google-Extended":["partial","*",31,18],"Applebot-Extended":["partial","*",31,18],"CCBot":["partial","*",31,18],"Bytespider":["partial","*",31,18],"Amazonbot":["partial","*",31,18],"meta-externalagent":["partial","*",31,18],"cohere-ai":["partial","*",31,18],"Diffbot":["partial","*",31,18],"ContextFetchBot":["partial","*",31,18]},"llms":{"status":"missing","http":404,"bytes":186},"home":{"status":"ok","http":200,"url":"https://www.rice.edu/","html_tokens":39737,"markdown_tokens":252,"pruned_tokens":220}},
{"rank":1366,"domain":"ualberta.ca","origin":"https://ualberta.ca","robots":{"status":"refused","http":403,"url":"https://www.ualberta.ca/robots.txt","bytes":919,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":1367,"domain":"livestream.com","origin":"https://livestream.com","robots":{"status":"html","http":200,"url":"https://vimeo.com/features/livestreaming","bytes":1416419,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":1368,"domain":"webthing.com","origin":"http://webthing.com","robots":{"status":"ok","http":200,"url":"http://webthing.com/robots.txt","bytes":527,"groups":1},"bots":{"GPTBot":["partial","*",19,0],"ChatGPT-User":["partial","*",19,0],"OAI-SearchBot":["partial","*",19,0],"ClaudeBot":["partial","*",19,0],"Claude-User":["partial","*",19,0],"Claude-SearchBot":["partial","*",19,0],"anthropic-ai":["partial","*",19,0],"PerplexityBot":["partial","*",19,0],"Perplexity-User":["partial","*",19,0],"Google-Extended":["partial","*",19,0],"Applebot-Extended":["partial","*",19,0],"CCBot":["partial","*",19,0],"Bytespider":["partial","*",19,0],"Amazonbot":["partial","*",19,0],"meta-externalagent":["partial","*",19,0],"cohere-ai":["partial","*",19,0],"Diffbot":["partial","*",19,0],"ContextFetchBot":["partial","*",19,0]},"llms":{"status":"missing","http":404,"bytes":268},"home":{"status":"ok","http":200,"url":"http://webthing.com/","html_tokens":1179,"markdown_tokens":454,"pruned_tokens":189}},
{"rank":1369,"domain":"cvent.com","origin":"https://cvent.com","robots":{"status":"ok","http":200,"url":"https://www.cvent.com/robots.txt","bytes":5017,"groups":8},"bots":{"GPTBot":["partial","*",127,18],"ChatGPT-User":["partial","*",127,18],"OAI-SearchBot":["partial","*",127,18],"ClaudeBot":["partial","*",127,18],"Claude-User":["partial","*",127,18],"Claude-SearchBot":["partial","*",127,18],"anthropic-ai":["partial","*",127,18],"PerplexityBot":["partial","*",127,18],"Perplexity-User":["partial","*",127,18],"Google-Extended":["partial","*",127,18],"Applebot-Extended":["partial","*",127,18],"CCBot":["partial","*",127,18],"Bytespider":["partial","*",127,18],"Amazonbot":["partial","*",127,18],"meta-externalagent":["partial","*",127,18],"cohere-ai":["partial","*",127,18],"Diffbot":["partial","*",127,18],"ContextFetchBot":["partial","*",127,18]},"llms":{"status":"present","http":200,"bytes":51453},"home":{"status":"ok","http":200,"url":"https://www.cvent.com/","html_tokens":51301,"markdown_tokens":183,"pruned_tokens":111}},
{"rank":1370,"domain":"forrester.com","origin":"https://forrester.com","robots":{"status":"ok","http":200,"url":"https://www.forrester.com/robots.txt","bytes":1294,"groups":9},"bots":{"GPTBot":["partial","named",1,0],"ChatGPT-User":["partial","*",36,0],"OAI-SearchBot":["partial","*",36,0],"ClaudeBot":["partial","named",1,0],"Claude-User":["partial","named",1,0],"Claude-SearchBot":["partial","named",1,0],"anthropic-ai":["partial","*",36,0],"PerplexityBot":["partial","*",36,0],"Perplexity-User":["partial","*",36,0],"Google-Extended":["partial","named",1,0],"Applebot-Extended":["partial","*",36,0],"CCBot":["partial","named",1,0],"Bytespider":["partial","*",36,0],"Amazonbot":["partial","*",36,0],"meta-externalagent":["partial","*",36,0],"cohere-ai":["partial","*",36,0],"Diffbot":["partial","*",36,0],"ContextFetchBot":["partial","*",36,0]},"llms":{"status":"missing","http":404,"bytes":0},"home":{"status":"ok","http":200,"url":"https://www.forrester.com/bold/","html_tokens":45216,"markdown_tokens":1769,"pruned_tokens":816}},
{"rank":1371,"domain":"nd.edu","origin":"https://nd.edu","robots":{"status":"ok","http":200,"url":"https://www.nd.edu/robots.txt","bytes":183,"groups":1},"bots":{"GPTBot":["partial","*",6,0],"ChatGPT-User":["partial","*",6,0],"OAI-SearchBot":["partial","*",6,0],"ClaudeBot":["partial","*",6,0],"Claude-User":["partial","*",6,0],"Claude-SearchBot":["partial","*",6,0],"anthropic-ai":["partial","*",6,0],"PerplexityBot":["partial","*",6,0],"Perplexity-User":["partial","*",6,0],"Google-Extended":["partial","*",6,0],"Applebot-Extended":["partial","*",6,0],"CCBot":["partial","*",6,0],"Bytespider":["partial","*",6,0],"Amazonbot":["partial","*",6,0],"meta-externalagent":["partial","*",6,0],"cohere-ai":["partial","*",6,0],"Diffbot":["partial","*",6,0],"ContextFetchBot":["partial","*",6,0]},"llms":{"status":"missing","http":404,"bytes":58599},"home":{"status":"ok","http":200,"url":"https://www.nd.edu/","html_tokens":21362,"markdown_tokens":472,"pruned_tokens":81}},
{"rank":1372,"domain":"esquire.com","origin":"https://esquire.com","robots":{"status":"ok","http":200,"url":"https://esquire.com/robots.txt","bytes":959,"groups":3},"bots":{"GPTBot":["partial","named",32,0],"ChatGPT-User":["partial","named",32,0],"OAI-SearchBot":["partial","named",32,0],"ClaudeBot":["partial","*",19,0],"Claude-User":["partial","*",19,0],"Claude-SearchBot":["partial","*",19,0],"anthropic-ai":["partial","*",19,0],"PerplexityBot":["partial","*",19,0],"Perplexity-User":["partial","*",19,0],"Google-Extended":["partial","*",19,0],"Applebot-Extended":["partial","*",19,0],"CCBot":["partial","*",19,0],"Bytespider":["partial","*",19,0],"Amazonbot":["partial","*",19,0],"meta-externalagent":["partial","*",19,0],"cohere-ai":["partial","*",19,0],"Diffbot":["partial","*",19,0],"ContextFetchBot":["partial","*",19,0]},"llms":{"status":"missing","http":404,"bytes":84},"home":{"status":"ok","http":200,"url":"https://www.esquire.com/","html_tokens":232736,"markdown_tokens":117,"pruned_tokens":117}},
{"rank":1373,"domain":"lothar.com","origin":"https://lothar.com","robots":{"status":"missing","http":404,"url":"https://lothar.com/robots.txt","bytes":167,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":167},"home":{"status":"http_error","http":500,"url":"https://lothar.com/"}},
{"rank":1374,"domain":"biorxiv.org","origin":"https://biorxiv.org","robots":{"status":"ok","http":200,"url":"https://www.biorxiv.org/robots.txt","bytes":3636,"groups":43},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["blocked","named",1,0],"OAI-SearchBot":["partial","*",37,32],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",37,32],"Claude-SearchBot":["partial","*",37,32],"anthropic-ai":["partial","*",37,32],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["partial","*",37,32],"Google-Extended":["partial","*",37,32],"Applebot-Extended":["partial","*",37,32],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["partial","*",37,32],"cohere-ai":["partial","*",37,32],"Diffbot":["partial","*",37,32],"ContextFetchBot":["partial","*",37,32]},"llms":{"status":"missing","http":404,"bytes":22422},"home":{"status":"ok","http":200,"url":"https://www.biorxiv.org/","html_tokens":8042,"markdown_tokens":5,"pruned_tokens":5}},
{"rank":1375,"domain":"google.com.hk","origin":"https://google.com.hk","robots":{"status":"ok","http":200,"url":"https://www.google.com.hk/robots.txt","bytes":6624,"groups":4},"bots":{"GPTBot":["partial","*",169,64],"ChatGPT-User":["partial","*",169,64],"OAI-SearchBot":["partial","*",169,64],"ClaudeBot":["partial","*",169,64],"Claude-User":["partial","*",169,64],"Claude-SearchBot":["partial","*",169,64],"anthropic-ai":["partial","*",169,64],"PerplexityBot":["partial","*",169,64],"Perplexity-User":["partial","*",169,64],"Google-Extended":["partial","*",169,64],"Applebot-Extended":["partial","*",169,64],"CCBot":["partial","*",169,64],"Bytespider":["partial","*",169,64],"Amazonbot":["partial","*",169,64],"meta-externalagent":["partial","*",169,64],"cohere-ai":["partial","*",169,64],"Diffbot":["partial","*",169,64],"ContextFetchBot":["partial","*",169,64]},"llms":{"status":"missing","http":404,"bytes":1569},"home":{"status":"ok","http":200,"url":"https://www.google.com.hk/","html_tokens":21519,"markdown_tokens":38,"pruned_tokens":38}},
{"rank":1376,"domain":"reverbnation.com","origin":"https://reverbnation.com","robots":{"status":"ok","http":200,"url":"https://www.reverbnation.com/robots.txt","bytes":99,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":5201},"home":{"status":"ok","http":200,"url":"https://www.reverbnation.com/","html_tokens":12419,"markdown_tokens":264,"pruned_tokens":128}},
{"rank":1377,"domain":"chinanews.com.cn","origin":"https://www.chinanews.com.cn","robots":{"status":"ok","http":200,"url":"https://www.chinanews.com.cn/robots.txt","bytes":858,"groups":1},"bots":{"GPTBot":["partial","*",38,1],"ChatGPT-User":["partial","*",38,1],"OAI-SearchBot":["partial","*",38,1],"ClaudeBot":["partial","*",38,1],"Claude-User":["partial","*",38,1],"Claude-SearchBot":["partial","*",38,1],"anthropic-ai":["partial","*",38,1],"PerplexityBot":["partial","*",38,1],"Perplexity-User":["partial","*",38,1],"Google-Extended":["partial","*",38,1],"Applebot-Extended":["partial","*",38,1],"CCBot":["partial","*",38,1],"Bytespider":["partial","*",38,1],"Amazonbot":["partial","*",38,1],"meta-externalagent":["partial","*",38,1],"cohere-ai":["partial","*",38,1],"Diffbot":["partial","*",38,1],"ContextFetchBot":["partial","*",38,1]},"llms":{"status":"missing","http":404,"bytes":6124},"home":{"status":"ok","http":200,"url":"https://www.chinanews.com.cn/","html_tokens":47568,"markdown_tokens":53,"pruned_tokens":14}},
{"rank":1378,"domain":"seekingalpha.com","origin":"https://seekingalpha.com","robots":{"status":"ok","http":200,"url":"https://seekingalpha.com/robots.txt","bytes":5081,"groups":106},"bots":{"GPTBot":["partial","*",21,0],"ChatGPT-User":["blocked","named",1,0],"OAI-SearchBot":["partial","*",21,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["blocked","named",1,0],"Claude-SearchBot":["partial","*",21,0],"anthropic-ai":["partial","*",21,0],"PerplexityBot":["partial","*",21,0],"Perplexity-User":["blocked","named",1,0],"Google-Extended":["partial","*",21,0],"Applebot-Extended":["partial","*",21,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["partial","*",21,0],"meta-externalagent":["blocked","named",2,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",21,0]},"llms":{"status":"missing","http":404,"bytes":0},"home":{"status":"ok","http":200,"url":"https://seekingalpha.com/","html_tokens":153116,"markdown_tokens":1396,"pruned_tokens":127}},
{"rank":1379,"domain":"mcafee.com","origin":"https://mcafee.com","robots":{"status":"unreachable","http":null,"url":"https://mcafee.com/robots.txt","bytes":0,"groups":0,"error":"<StreamReset stream_id:1, error_code:2, remote_reset:True>"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":1380,"domain":"ampproject.org","origin":"https://ampproject.org","robots":{"status":"missing","http":404,"url":"https://amp.dev/robots.txt/?referrer=ampproject.org","bytes":77664,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":77664},"home":{"status":"ok","http":200,"url":"https://amp.dev/","html_tokens":34397,"markdown_tokens":65,"pruned_tokens":65}},
{"rank":1381,"domain":"ru.com","origin":"https://ru.com","robots":{"status":"unreachable","http":null,"url":"https://ru.com/robots.txt","bytes":0,"groups":0,"error":"cannot resolve ru.com"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":1382,"domain":"lovable.dev","origin":"https://lovable.dev","robots":{"status":"ok","http":200,"url":"https://lovable.dev/robots.txt","bytes":1784,"groups":4},"bots":{"GPTBot":["partial","named",15,1],"ChatGPT-User":["partial","named",7,1],"OAI-SearchBot":["partial","named",15,1],"ClaudeBot":["partial","named",15,1],"Claude-User":["partial","named",7,1],"Claude-SearchBot":["partial","*",14,1],"anthropic-ai":["partial","*",14,1],"PerplexityBot":["partial","named",15,1],"Perplexity-User":["partial","*",14,1],"Google-Extended":["partial","named",15,1],"Applebot-Extended":["partial","named",15,1],"CCBot":["partial","*",14,1],"Bytespider":["partial","*",14,1],"Amazonbot":["partial","*",14,1],"meta-externalagent":["partial","*",14,1],"cohere-ai":["partial","*",14,1],"Diffbot":["partial","*",14,1],"ContextFetchBot":["partial","*",14,1]},"llms":{"status":"present","http":200,"bytes":4747},"home":{"status":"http_error","http":403,"url":"https://lovable.dev/"}},
{"rank":1383,"domain":"freshdesk.com","origin":"https://freshdesk.com","robots":{"status":"ok","http":200,"url":"https://freshdesk.com/robots.txt","bytes":240,"groups":1},"bots":{"GPTBot":["partial","*",3,1],"ChatGPT-User":["partial","*",3,1],"OAI-SearchBot":["partial","*",3,1],"ClaudeBot":["partial","*",3,1],"Claude-User":["partial","*",3,1],"Claude-SearchBot":["partial","*",3,1],"anthropic-ai":["partial","*",3,1],"PerplexityBot":["partial","*",3,1],"Perplexity-User":["partial","*",3,1],"Google-Extended":["partial","*",3,1],"Applebot-Extended":["partial","*",3,1],"CCBot":["partial","*",3,1],"Bytespider":["partial","*",3,1],"Amazonbot":["partial","*",3,1],"meta-externalagent":["partial","*",3,1],"cohere-ai":["partial","*",3,1],"Diffbot":["partial","*",3,1],"ContextFetchBot":["partial","*",3,1]},"llms":{"status":"missing","http":404,"bytes":235894},"home":{"status":"ok","http":200,"url":"https://www.freshworks.com/freshdesk/","html_tokens":251892,"markdown_tokens":169,"pruned_tokens":169}},
{"rank":1384,"domain":"aps.org","origin":"https://aps.org","robots":{"status":"ok","http":200,"url":"https://www.aps.org/robots.txt","bytes":2669,"groups":1},"bots":{"GPTBot":["partial","*",39,0],"ChatGPT-User":["partial","*",39,0],"OAI-SearchBot":["partial","*",39,0],"ClaudeBot":["partial","*",39,0],"Claude-User":["partial","*",39,0],"Claude-SearchBot":["partial","*",39,0],"anthropic-ai":["partial","*",39,0],"PerplexityBot":["partial","*",39,0],"Perplexity-User":["partial","*",39,0],"Google-Extended":["partial","*",39,0],"Applebot-Extended":["partial","*",39,0],"CCBot":["partial","*",39,0],"Bytespider":["partial","*",39,0],"Amazonbot":["partial","*",39,0],"meta-externalagent":["partial","*",39,0],"cohere-ai":["partial","*",39,0],"Diffbot":["partial","*",39,0],"ContextFetchBot":["partial","*",39,0]},"llms":{"status":"http_error","http":403,"bytes":5522},"home":{"status":"http_error","http":403,"url":"https://www.aps.org/"}},
{"rank":1385,"domain":"powerbi.com","origin":"https://powerbi.com","robots":{"status":"html","http":200,"url":"https://app.powerbi.com/robots.txt","bytes":22775,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":1386,"domain":"univie.ac.at","origin":"https://univie.ac.at","robots":{"status":"ok","http":200,"url":"https://www.univie.ac.at/robots.txt","bytes":70,"groups":1},"bots":{"GPTBot":["partial","*",3,0],"ChatGPT-User":["partial","*",3,0],"OAI-SearchBot":["partial","*",3,0],"ClaudeBot":["partial","*",3,0],"Claude-User":["partial","*",3,0],"Claude-SearchBot":["partial","*",3,0],"anthropic-ai":["partial","*",3,0],"PerplexityBot":["partial","*",3,0],"Perplexity-User":["partial","*",3,0],"Google-Extended":["partial","*",3,0],"Applebot-Extended":["partial","*",3,0],"CCBot":["partial","*",3,0],"Bytespider":["partial","*",3,0],"Amazonbot":["partial","*",3,0],"meta-externalagent":["partial","*",3,0],"cohere-ai":["partial","*",3,0],"Diffbot":["partial","*",3,0],"ContextFetchBot":["partial","*",3,0]},"llms":{"status":"missing","http":404,"bytes":295663},"home":{"status":"ok","http":200,"url":"https://www.univie.ac.at/","html_tokens":107404,"markdown_tokens":1122,"pruned_tokens":168}},
{"rank":1387,"domain":"abb.com","origin":"https://abb.com","robots":{"status":"unreachable","http":null,"url":"https://abb.com/robots.txt","bytes":0,"groups":0,"error":"<StreamReset stream_id:1, error_code:2, remote_reset:True>"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":1388,"domain":"wordfence.com","origin":"https://wordfence.com","robots":{"status":"ok","http":200,"url":"https://www.wordfence.com/robots.txt","bytes":55,"groups":1},"bots":{"GPTBot":["partial","*",2,0],"ChatGPT-User":["partial","*",2,0],"OAI-SearchBot":["partial","*",2,0],"ClaudeBot":["partial","*",2,0],"Claude-User":["partial","*",2,0],"Claude-SearchBot":["partial","*",2,0],"anthropic-ai":["partial","*",2,0],"PerplexityBot":["partial","*",2,0],"Perplexity-User":["partial","*",2,0],"Google-Extended":["partial","*",2,0],"Applebot-Extended":["partial","*",2,0],"CCBot":["partial","*",2,0],"Bytespider":["partial","*",2,0],"Amazonbot":["partial","*",2,0],"meta-externalagent":["partial","*",2,0],"cohere-ai":["partial","*",2,0],"Diffbot":["partial","*",2,0],"ContextFetchBot":["partial","*",2,0]},"llms":{"status":"html","http":202,"bytes":2275},"home":{"status":"ok","http":202,"url":"https://www.wordfence.com/","html_tokens":569,"markdown_tokens":40,"pruned_tokens":40}},
{"rank":1389,"domain":"i2.wp.com","origin":"https://i2.wp.com","robots":{"status":"ok","http":200,"url":"https://i2.wp.com/robots.txt","bytes":343,"groups":10},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["blocked","named",1,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"http_error","http":400,"bytes":49},"home":{"status":"http_error","http":400,"url":"https://i2.wp.com/"}},
{"rank":1390,"domain":"discordapp.com","origin":"https://discordapp.com","robots":{"status":"ok","http":200,"url":"https://discord.com/robots.txt","bytes":1774,"groups":3},"bots":{"GPTBot":["partial","*",24,10],"ChatGPT-User":["partial","*",24,10],"OAI-SearchBot":["partial","*",24,10],"ClaudeBot":["partial","*",24,10],"Claude-User":["partial","*",24,10],"Claude-SearchBot":["partial","*",24,10],"anthropic-ai":["partial","*",24,10],"PerplexityBot":["partial","*",24,10],"Perplexity-User":["partial","*",24,10],"Google-Extended":["partial","*",24,10],"Applebot-Extended":["partial","*",24,10],"CCBot":["partial","*",24,10],"Bytespider":["partial","*",24,10],"Amazonbot":["partial","*",24,10],"meta-externalagent":["partial","*",24,10],"cohere-ai":["partial","*",24,10],"Diffbot":["partial","*",24,10],"ContextFetchBot":["partial","*",24,10]},"llms":{"status":"missing","http":404,"bytes":44855},"home":{"status":"ok","http":200,"url":"https://discord.com/","html_tokens":42531,"markdown_tokens":367,"pruned_tokens":48}},
{"rank":1391,"domain":"thingiverse.com","origin":"https://thingiverse.com","robots":{"status":"refused","http":403,"url":"https://www.thingiverse.com/robots.txt","bytes":5586,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":1392,"domain":"atlasobscura.com","origin":"https://atlasobscura.com","robots":{"status":"refused","http":403,"url":"https://atlasobscura.com/robots.txt","bytes":4910,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":1393,"domain":"moma.org","origin":"https://moma.org","robots":{"status":"ok","http":200,"url":"https://www.moma.org/robots.txt","bytes":870,"groups":1},"bots":{"GPTBot":["partial","*",20,0],"ChatGPT-User":["partial","*",20,0],"OAI-SearchBot":["partial","*",20,0],"ClaudeBot":["partial","*",20,0],"Claude-User":["partial","*",20,0],"Claude-SearchBot":["partial","*",20,0],"anthropic-ai":["partial","*",20,0],"PerplexityBot":["partial","*",20,0],"Perplexity-User":["partial","*",20,0],"Google-Extended":["partial","*",20,0],"Applebot-Extended":["partial","*",20,0],"CCBot":["partial","*",20,0],"Bytespider":["partial","*",20,0],"Amazonbot":["partial","*",20,0],"meta-externalagent":["partial","*",20,0],"cohere-ai":["partial","*",20,0],"Diffbot":["partial","*",20,0],"ContextFetchBot":["partial","*",20,0]},"llms":{"status":"missing","http":404,"bytes":148454},"home":{"status":"http_error","http":403,"url":"https://moma.org/"}},
{"rank":1394,"domain":"copyright.gov","origin":"https://copyright.gov","robots":{"status":"missing","http":404,"url":"https://copyright.gov/robots.txt","bytes":26585,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":26585},"home":{"status":"ok","http":200,"url":"https://copyright.gov/","html_tokens":13688,"markdown_tokens":545,"pruned_tokens":135}},
{"rank":1395,"domain":"golux.com","origin":"https://golux.com","robots":{"status":"unreachable","http":null,"url":"https://golux.com/robots.txt","bytes":0,"groups":0,"error":"All connection attempts failed"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":1396,"domain":"spaceship.com","origin":"https://spaceship.com","robots":{"status":"ok","http":200,"url":"https://www.spaceship.com/robots.txt","bytes":46,"groups":1},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"http_error","http":403,"bytes":5570},"home":{"status":"http_error","http":403,"url":"https://www.spaceship.com/"}},
{"rank":1397,"domain":"virtualbox.org","origin":"https://virtualbox.org","robots":{"status":"html","http":200,"url":"https://www.virtualbox.org/","bytes":20306,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":1398,"domain":"rwth-aachen.de","origin":"https://rwth-aachen.de","robots":{"status":"ok","http":200,"url":"https://rwth-aachen.de/robots.txt","bytes":36,"groups":1},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"missing","http":404,"bytes":4425},"home":{"status":"ok","http":200,"url":"https://www.rwth-aachen.de/cms/~a/root/?lidx=1","html_tokens":24662,"markdown_tokens":248,"pruned_tokens":75}},
{"rank":1399,"domain":"arte.tv","origin":"https://arte.tv","robots":{"status":"ok","http":200,"url":"https://arte.tv/robots.txt","bytes":103,"groups":1},"bots":{"GPTBot":["blocked","*",1,0],"ChatGPT-User":["blocked","*",1,0],"OAI-SearchBot":["blocked","*",1,0],"ClaudeBot":["blocked","*",1,0],"Claude-User":["blocked","*",1,0],"Claude-SearchBot":["blocked","*",1,0],"anthropic-ai":["blocked","*",1,0],"PerplexityBot":["blocked","*",1,0],"Perplexity-User":["blocked","*",1,0],"Google-Extended":["blocked","*",1,0],"Applebot-Extended":["blocked","*",1,0],"CCBot":["blocked","*",1,0],"Bytespider":["blocked","*",1,0],"Amazonbot":["blocked","*",1,0],"meta-externalagent":["blocked","*",1,0],"cohere-ai":["blocked","*",1,0],"Diffbot":["blocked","*",1,0],"ContextFetchBot":["blocked","*",1,0]},"llms":{"status":"disallowed"},"home":{"status":"disallowed"}},
{"rank":1400,"domain":"akamaihd.net","origin":"https://akamaihd.net","robots":{"status":"unreachable","http":null,"url":"https://akamaihd.net/robots.txt","bytes":0,"groups":0,"error":"cannot resolve akamaihd.net"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":1401,"domain":"popularmechanics.com","origin":"https://popularmechanics.com","robots":{"status":"ok","http":200,"url":"https://popularmechanics.com/robots.txt","bytes":977,"groups":3},"bots":{"GPTBot":["partial","named",32,0],"ChatGPT-User":["partial","named",32,0],"OAI-SearchBot":["partial","named",32,0],"ClaudeBot":["partial","*",19,0],"Claude-User":["partial","*",19,0],"Claude-SearchBot":["partial","*",19,0],"anthropic-ai":["partial","*",19,0],"PerplexityBot":["partial","*",19,0],"Perplexity-User":["partial","*",19,0],"Google-Extended":["partial","*",19,0],"Applebot-Extended":["partial","*",19,0],"CCBot":["partial","*",19,0],"Bytespider":["partial","*",19,0],"Amazonbot":["partial","*",19,0],"meta-externalagent":["partial","*",19,0],"cohere-ai":["partial","*",19,0],"Diffbot":["partial","*",19,0],"ContextFetchBot":["partial","*",19,0]},"llms":{"status":"missing","http":404,"bytes":84},"home":{"status":"ok","http":200,"url":"https://www.popularmechanics.com/","html_tokens":229842,"markdown_tokens":81,"pruned_tokens":81}},
{"rank":1402,"domain":"avast.com","origin":"https://avast.com","robots":{"status":"refused","http":403,"url":"https://www.avast.com/robots.txt","bytes":379,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":1403,"domain":"wto.org","origin":"https://wto.org","robots":{"status":"ok","http":200,"url":"https://www.wto.org/robots.txt","bytes":872,"groups":1},"bots":{"GPTBot":["partial","*",9,1],"ChatGPT-User":["partial","*",9,1],"OAI-SearchBot":["partial","*",9,1],"ClaudeBot":["partial","*",9,1],"Claude-User":["partial","*",9,1],"Claude-SearchBot":["partial","*",9,1],"anthropic-ai":["partial","*",9,1],"PerplexityBot":["partial","*",9,1],"Perplexity-User":["partial","*",9,1],"Google-Extended":["partial","*",9,1],"Applebot-Extended":["partial","*",9,1],"CCBot":["partial","*",9,1],"Bytespider":["partial","*",9,1],"Amazonbot":["partial","*",9,1],"meta-externalagent":["partial","*",9,1],"cohere-ai":["partial","*",9,1],"Diffbot":["partial","*",9,1],"ContextFetchBot":["partial","*",9,1]},"llms":{"status":"other_text","http":200,"bytes":19907},"home":{"status":"not_html","error":"unsupported content type text/markdown"}},
{"rank":1404,"domain":"l.ink","origin":"https://l.ink","robots":{"status":"html","http":200,"url":"https://porkbun.com/products/webhosting/managedLinks","bytes":92637,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":1405,"domain":"brave.com","origin":"https://brave.com","robots":{"status":"ok","http":200,"url":"https://brave.com/robots.txt","bytes":53,"groups":1},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"missing","http":404,"bytes":54560},"home":{"status":"ok","http":200,"url":"https://brave.com/","html_tokens":35557,"markdown_tokens":1730,"pruned_tokens":895}},
{"rank":1406,"domain":"apachetutor.org","origin":"https://apachetutor.org","robots":{"status":"unreachable","http":null,"url":"https://apachetutor.org/robots.txt","bytes":0,"groups":0,"error":"cannot resolve apachetutor.org"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":1407,"domain":"g2.com","origin":"https://g2.com","robots":{"status":"ok","http":200,"url":"https://www.g2.com/robots.txt","bytes":7309,"groups":8},"bots":{"GPTBot":["partial","named",101,0],"ChatGPT-User":["partial","*",88,0],"OAI-SearchBot":["partial","*",88,0],"ClaudeBot":["partial","named",101,0],"Claude-User":["partial","*",88,0],"Claude-SearchBot":["partial","*",88,0],"anthropic-ai":["partial","*",88,0],"PerplexityBot":["partial","*",88,0],"Perplexity-User":["partial","*",88,0],"Google-Extended":["partial","*",88,0],"Applebot-Extended":["partial","named",101,0],"CCBot":["partial","named",101,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["partial","named",101,0],"meta-externalagent":["partial","named",101,0],"cohere-ai":["partial","*",88,0],"Diffbot":["partial","named",1,0],"ContextFetchBot":["partial","*",88,0]},"llms":{"status":"present","http":200,"bytes":4456},"home":{"status":"http_error","http":403,"url":"https://www.g2.com/"}},
{"rank":1408,"domain":"hindawi.com","origin":"https://hindawi.com","robots":{"status":"ok","http":200,"url":"https://www.hindawi.com/robots.txt","bytes":962,"groups":4},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["partial","*",21,8],"OAI-SearchBot":["partial","*",21,8],"ClaudeBot":["partial","*",21,8],"Claude-User":["partial","*",21,8],"Claude-SearchBot":["partial","*",21,8],"anthropic-ai":["partial","*",21,8],"PerplexityBot":["partial","*",21,8],"Perplexity-User":["partial","*",21,8],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["partial","*",21,8],"CCBot":["partial","*",21,8],"Bytespider":["partial","*",21,8],"Amazonbot":["partial","*",21,8],"meta-externalagent":["partial","*",21,8],"cohere-ai":["partial","*",21,8],"Diffbot":["partial","*",21,8],"ContextFetchBot":["partial","*",21,8]},"llms":{"status":"http_error","http":403,"bytes":5756},"home":{"status":"http_error","http":403,"url":"https://hindawi.com/"}},
{"rank":1409,"domain":"citrix.com","origin":"https://citrix.com","robots":{"status":"ok","http":200,"url":"https://www.citrix.com/robots.txt","bytes":1015,"groups":1},"bots":{"GPTBot":["partial","*",34,0],"ChatGPT-User":["partial","*",34,0],"OAI-SearchBot":["partial","*",34,0],"ClaudeBot":["partial","*",34,0],"Claude-User":["partial","*",34,0],"Claude-SearchBot":["partial","*",34,0],"anthropic-ai":["partial","*",34,0],"PerplexityBot":["partial","*",34,0],"Perplexity-User":["partial","*",34,0],"Google-Extended":["partial","*",34,0],"Applebot-Extended":["partial","*",34,0],"CCBot":["partial","*",34,0],"Bytespider":["partial","*",34,0],"Amazonbot":["partial","*",34,0],"meta-externalagent":["partial","*",34,0],"cohere-ai":["partial","*",34,0],"Diffbot":["partial","*",34,0],"ContextFetchBot":["partial","*",34,0]},"llms":{"status":"missing","http":404,"bytes":72796},"home":{"status":"ok","http":200,"url":"https://www.citrix.com/","html_tokens":30367,"markdown_tokens":317,"pruned_tokens":193}},
{"rank":1410,"domain":"leparisien.fr","origin":"https://leparisien.fr","robots":{"status":"ok","http":200,"url":"https://www.leparisien.fr/robots.txt","bytes":14536,"groups":216},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["partial","*",175,2],"OAI-SearchBot":["partial","*",175,2],"ClaudeBot":["partial","named",1,1],"Claude-User":["partial","*",175,2],"Claude-SearchBot":["partial","*",175,2],"anthropic-ai":["partial","named",1,1],"PerplexityBot":["partial","named",1,1],"Perplexity-User":["partial","*",175,2],"Google-Extended":["partial","*",175,2],"Applebot-Extended":["partial","*",175,2],"CCBot":["blocked","named",1,0],"Bytespider":["partial","*",175,2],"Amazonbot":["partial","named",1,1],"meta-externalagent":["partial","*",175,2],"cohere-ai":["partial","named",1,1],"Diffbot":["partial","*",175,2],"ContextFetchBot":["partial","*",175,2]},"llms":{"status":"missing","http":404,"bytes":207639},"home":{"status":"http_error","http":403,"url":"https://www.leparisien.fr/"}},
{"rank":1411,"domain":"miamiherald.com","origin":"https://miamiherald.com","robots":{"status":"unreachable","http":null,"url":"https://miamiherald.com/robots.txt","bytes":0,"groups":0,"error":"<StreamReset stream_id:1, error_code:2, remote_reset:True>"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":1412,"domain":"wsu.edu","origin":"https://wsu.edu","robots":{"status":"ok","http":200,"url":"https://wsu.edu/robots.txt","bytes":145,"groups":1},"bots":{"GPTBot":["partial","*",1,1],"ChatGPT-User":["partial","*",1,1],"OAI-SearchBot":["partial","*",1,1],"ClaudeBot":["partial","*",1,1],"Claude-User":["partial","*",1,1],"Claude-SearchBot":["partial","*",1,1],"anthropic-ai":["partial","*",1,1],"PerplexityBot":["partial","*",1,1],"Perplexity-User":["partial","*",1,1],"Google-Extended":["partial","*",1,1],"Applebot-Extended":["partial","*",1,1],"CCBot":["partial","*",1,1],"Bytespider":["partial","*",1,1],"Amazonbot":["partial","*",1,1],"meta-externalagent":["partial","*",1,1],"cohere-ai":["partial","*",1,1],"Diffbot":["partial","*",1,1],"ContextFetchBot":["partial","*",1,1]},"llms":{"status":"missing","http":404,"bytes":77792},"home":{"status":"ok","http":200,"url":"https://wsu.edu/","html_tokens":27715,"markdown_tokens":163,"pruned_tokens":163}},
{"rank":1413,"domain":"cronolog.org","origin":"http://cronolog.org","robots":{"status":"html","http":200,"url":"http://cronolog.org/robots.txt","bytes":4297,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":1414,"domain":"yolasite.com","origin":"https://yolasite.com","robots":{"status":"refused","http":403,"url":"https://yolasite.com/robots.txt","bytes":5507,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":1415,"domain":"csrc.nist.gov","origin":"https://csrc.nist.gov","robots":{"status":"html","http":200,"url":"https://csrc.nist.gov/","bytes":48666,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":1416,"domain":"pbase.com","origin":"https://pbase.com","robots":{"status":"html","http":200,"url":"https://pbase.com/robots.txt","bytes":3701,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":1417,"domain":"7-zip.org","origin":"https://7-zip.org","robots":{"status":"ok","http":200,"url":"https://7-zip.org/robots.txt","bytes":0,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":162},"home":{"status":"ok","http":200,"url":"https://7-zip.org/","html_tokens":1697,"markdown_tokens":373,"pruned_tokens":247}},
{"rank":1418,"domain":"manchester.ac.uk","origin":"https://manchester.ac.uk","robots":{"status":"ok","http":200,"url":"https://www.manchester.ac.uk/robots.txt","bytes":434,"groups":1},"bots":{"GPTBot":["partial","*",9,1],"ChatGPT-User":["partial","*",9,1],"OAI-SearchBot":["partial","*",9,1],"ClaudeBot":["partial","*",9,1],"Claude-User":["partial","*",9,1],"Claude-SearchBot":["partial","*",9,1],"anthropic-ai":["partial","*",9,1],"PerplexityBot":["partial","*",9,1],"Perplexity-User":["partial","*",9,1],"Google-Extended":["partial","*",9,1],"Applebot-Extended":["partial","*",9,1],"CCBot":["partial","*",9,1],"Bytespider":["partial","*",9,1],"Amazonbot":["partial","*",9,1],"meta-externalagent":["partial","*",9,1],"cohere-ai":["partial","*",9,1],"Diffbot":["partial","*",9,1],"ContextFetchBot":["partial","*",9,1]},"llms":{"status":"missing","http":404,"bytes":45239},"home":{"status":"ok","http":200,"url":"https://www.manchester.ac.uk/","html_tokens":17576,"markdown_tokens":102,"pruned_tokens":102}},
{"rank":1419,"domain":"phpmyadmin.net","origin":"https://phpmyadmin.net","robots":{"status":"ok","http":200,"url":"https://www.phpmyadmin.net/robots.txt","bytes":71,"groups":1},"bots":{"GPTBot":["allowed","*",0,1],"ChatGPT-User":["allowed","*",0,1],"OAI-SearchBot":["allowed","*",0,1],"ClaudeBot":["allowed","*",0,1],"Claude-User":["allowed","*",0,1],"Claude-SearchBot":["allowed","*",0,1],"anthropic-ai":["allowed","*",0,1],"PerplexityBot":["allowed","*",0,1],"Perplexity-User":["allowed","*",0,1],"Google-Extended":["allowed","*",0,1],"Applebot-Extended":["allowed","*",0,1],"CCBot":["allowed","*",0,1],"Bytespider":["allowed","*",0,1],"Amazonbot":["allowed","*",0,1],"meta-externalagent":["allowed","*",0,1],"cohere-ai":["allowed","*",0,1],"Diffbot":["allowed","*",0,1],"ContextFetchBot":["allowed","*",0,1]},"llms":{"status":"missing","http":404,"bytes":15917},"home":{"status":"ok","http":200,"url":"https://www.phpmyadmin.net/","html_tokens":7992,"markdown_tokens":474,"pruned_tokens":199}},
{"rank":1420,"domain":"virginia.gov","origin":"https://www.virginia.gov","robots":{"status":"ok","http":200,"url":"https://www.virginia.gov/robots.txt","bytes":387,"groups":2},"bots":{"GPTBot":["partial","*",15,0],"ChatGPT-User":["partial","*",15,0],"OAI-SearchBot":["partial","*",15,0],"ClaudeBot":["partial","*",15,0],"Claude-User":["partial","*",15,0],"Claude-SearchBot":["partial","*",15,0],"anthropic-ai":["partial","*",15,0],"PerplexityBot":["partial","*",15,0],"Perplexity-User":["partial","*",15,0],"Google-Extended":["partial","*",15,0],"Applebot-Extended":["partial","*",15,0],"CCBot":["partial","*",15,0],"Bytespider":["partial","*",15,0],"Amazonbot":["partial","*",15,0],"meta-externalagent":["partial","*",15,0],"cohere-ai":["partial","*",15,0],"Diffbot":["partial","*",15,0],"ContextFetchBot":["partial","*",15,0]},"llms":{"status":"missing","http":404,"bytes":325},"home":{"status":"ok","http":200,"url":"https://www.virginia.gov/","html_tokens":16065,"markdown_tokens":17,"pruned_tokens":17}},
{"rank":1421,"domain":"alexa.com","origin":"https://alexa.com","robots":{"status":"html","http":200,"url":"https://www.amazon.com/ap/signin?clientContext=263-6461375-1107100&openid.pape.max_auth_age=3600&openid.return_to=https%3A%2F%2Falexa.amazon.com%2Fhome&openid.identity=http%3A%2F%2Fspecs.openid.net%2Fauth%2F2.0%2Fidentifier_select&openid.assoc_handle=metis_alexa_amzn_us&openid.mode=checkid_setup&siteState=263-6461375-1107100&marketPlaceId=ATVPDKIKX0DER&language=en_US&openid.claimed_id=http%3A%2F%2Fspecs.openid.net%2Fauth%2F2.0%2Fidentifier_select&openid.ns=http%3A%2F%2Fspecs.openid.net%2Fauth%2F2.0","bytes":29381,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":1422,"domain":"imperial.ac.uk","origin":"https://imperial.ac.uk","robots":{"status":"ok","http":200,"url":"https://www.imperial.ac.uk/robots.txt","bytes":12713,"groups":2},"bots":{"GPTBot":["partial","*",93,18],"ChatGPT-User":["partial","*",93,18],"OAI-SearchBot":["partial","*",93,18],"ClaudeBot":["partial","*",93,18],"Claude-User":["partial","*",93,18],"Claude-SearchBot":["partial","*",93,18],"anthropic-ai":["partial","*",93,18],"PerplexityBot":["partial","*",93,18],"Perplexity-User":["partial","*",93,18],"Google-Extended":["partial","*",93,18],"Applebot-Extended":["partial","*",93,18],"CCBot":["partial","*",93,18],"Bytespider":["partial","*",93,18],"Amazonbot":["partial","*",93,18],"meta-externalagent":["partial","*",93,18],"cohere-ai":["partial","*",93,18],"Diffbot":["partial","*",93,18],"ContextFetchBot":["partial","*",93,18]},"llms":{"status":"present","http":200,"bytes":4228},"home":{"status":"ok","http":200,"url":"https://www.imperial.ac.uk/","html_tokens":14244,"markdown_tokens":99,"pruned_tokens":62}},
{"rank":1423,"domain":"aclu.org","origin":"https://aclu.org","robots":{"status":"ok","http":200,"url":"https://aclu.org/robots.txt","bytes":1370,"groups":1},"bots":{"GPTBot":["partial","*",50,0],"ChatGPT-User":["partial","*",50,0],"OAI-SearchBot":["partial","*",50,0],"ClaudeBot":["partial","*",50,0],"Claude-User":["partial","*",50,0],"Claude-SearchBot":["partial","*",50,0],"anthropic-ai":["partial","*",50,0],"PerplexityBot":["partial","*",50,0],"Perplexity-User":["partial","*",50,0],"Google-Extended":["partial","*",50,0],"Applebot-Extended":["partial","*",50,0],"CCBot":["partial","*",50,0],"Bytespider":["partial","*",50,0],"Amazonbot":["partial","*",50,0],"meta-externalagent":["partial","*",50,0],"cohere-ai":["partial","*",50,0],"Diffbot":["partial","*",50,0],"ContextFetchBot":["partial","*",50,0]},"llms":{"status":"missing","http":404,"bytes":18122},"home":{"status":"ok","http":200,"url":"https://www.aclu.org/","html_tokens":19740,"markdown_tokens":704,"pruned_tokens":54}},
{"rank":1424,"domain":"miro.com","origin":"https://miro.com","robots":{"status":"ok","http":200,"url":"https://miro.com/robots.txt","bytes":1703,"groups":4},"bots":{"GPTBot":["partial","*",64,0],"ChatGPT-User":["partial","*",64,0],"OAI-SearchBot":["partial","*",64,0],"ClaudeBot":["partial","*",64,0],"Claude-User":["partial","*",64,0],"Claude-SearchBot":["partial","*",64,0],"anthropic-ai":["partial","*",64,0],"PerplexityBot":["partial","*",64,0],"Perplexity-User":["partial","*",64,0],"Google-Extended":["partial","*",64,0],"Applebot-Extended":["partial","*",64,0],"CCBot":["partial","*",64,0],"Bytespider":["partial","*",64,0],"Amazonbot":["partial","*",64,0],"meta-externalagent":["partial","*",64,0],"cohere-ai":["partial","*",64,0],"Diffbot":["partial","*",64,0],"ContextFetchBot":["partial","*",64,0]},"llms":{"status":"missing","http":404,"bytes":166411},"home":{"status":"ok","http":200,"url":"https://miro.com/","html_tokens":257215,"markdown_tokens":577,"pruned_tokens":289}},
{"rank":1425,"domain":"chinaz.com","origin":"https://chinaz.com","robots":{"status":"ok","http":200,"url":"https://www.chinaz.com/robots.txt","bytes":13,"groups":1},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"html","http":200,"bytes":53543},"home":{"status":"ok","http":200,"url":"https://www.chinaz.com/","html_tokens":57129,"markdown_tokens":25,"pruned_tokens":25}},
{"rank":1426,"domain":"upi.com","origin":"https://upi.com","robots":{"status":"ok","http":200,"url":"https://www.upi.com/robots.txt","bytes":540,"groups":9},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["partial","*",10,1],"OAI-SearchBot":["partial","*",10,1],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",10,1],"Claude-SearchBot":["partial","*",10,1],"anthropic-ai":["partial","*",10,1],"PerplexityBot":["partial","*",10,1],"Perplexity-User":["partial","*",10,1],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["partial","*",10,1],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["partial","*",10,1],"Diffbot":["partial","*",10,1],"ContextFetchBot":["partial","*",10,1]},"llms":{"status":"missing","http":404,"bytes":230965},"home":{"status":"ok","http":200,"url":"https://www.upi.com/","html_tokens":86673,"markdown_tokens":66,"pruned_tokens":42}},
{"rank":1427,"domain":"libreoffice.org","origin":"https://libreoffice.org","robots":{"status":"missing","http":404,"url":"https://www.libreoffice.org/robots.txt","bytes":146,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":146},"home":{"status":"ok","http":200,"url":"https://www.libreoffice.org/","html_tokens":10785,"markdown_tokens":103,"pruned_tokens":94}},
{"rank":1428,"domain":"page.link","origin":"https://page.link","robots":{"status":"refused","http":400,"url":"https://page.link/robots.txt","bytes":9258,"groups":0,"error":"HTTP 400"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":1429,"domain":"apple.co","origin":"https://apple.co","robots":{"status":"ok","http":200,"url":"https://apple.co/robots.txt","bytes":123,"groups":1},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"html","http":200,"bytes":327072},"home":{"status":"ok","http":200,"url":"https://www.apple.com/","html_tokens":81664,"markdown_tokens":27,"pruned_tokens":27}},
{"rank":1430,"domain":"csmonitor.com","origin":"https://csmonitor.com","robots":{"status":"ok","http":200,"url":"https://www.csmonitor.com/robots.txt","bytes":2351,"groups":34},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["partial","*",32,0],"OAI-SearchBot":["partial","*",32,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",32,0],"Claude-SearchBot":["partial","*",32,0],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["partial","*",32,0],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["partial","*",32,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",32,0]},"llms":{"status":"missing","http":404,"bytes":26364},"home":{"status":"ok","http":200,"url":"https://www.csmonitor.com/","html_tokens":28529,"markdown_tokens":66,"pruned_tokens":66}},
{"rank":1431,"domain":"spreaker.com","origin":"https://spreaker.com","robots":{"status":"ok","http":200,"url":"https://www.spreaker.com/robots.txt","bytes":194,"groups":1},"bots":{"GPTBot":["partial","*",8,0],"ChatGPT-User":["partial","*",8,0],"OAI-SearchBot":["partial","*",8,0],"ClaudeBot":["partial","*",8,0],"Claude-User":["partial","*",8,0],"Claude-SearchBot":["partial","*",8,0],"anthropic-ai":["partial","*",8,0],"PerplexityBot":["partial","*",8,0],"Perplexity-User":["partial","*",8,0],"Google-Extended":["partial","*",8,0],"Applebot-Extended":["partial","*",8,0],"CCBot":["partial","*",8,0],"Bytespider":["partial","*",8,0],"Amazonbot":["partial","*",8,0],"meta-externalagent":["partial","*",8,0],"cohere-ai":["partial","*",8,0],"Diffbot":["partial","*",8,0],"ContextFetchBot":["partial","*",8,0]},"llms":{"status":"missing","http":404,"bytes":276649},"home":{"status":"ok","http":200,"url":"https://www.spreaker.com/","html_tokens":257719,"markdown_tokens":61,"pruned_tokens":47}},
{"rank":1432,"domain":"rtve.es","origin":"https://rtve.es","robots":{"status":"ok","http":200,"url":"https://www.rtve.es/akamai/robots.txt","bytes":10301,"groups":2},"bots":{"GPTBot":["partial","*",161,28],"ChatGPT-User":["partial","*",161,28],"OAI-SearchBot":["partial","*",161,28],"ClaudeBot":["partial","*",161,28],"Claude-User":["partial","*",161,28],"Claude-SearchBot":["partial","*",161,28],"anthropic-ai":["partial","*",161,28],"PerplexityBot":["partial","*",161,28],"Perplexity-User":["partial","*",161,28],"Google-Extended":["partial","*",161,28],"Applebot-Extended":["partial","*",161,28],"CCBot":["partial","*",161,28],"Bytespider":["partial","*",161,28],"Amazonbot":["partial","*",161,28],"meta-externalagent":["partial","*",161,28],"cohere-ai":["partial","*",161,28],"Diffbot":["partial","*",161,28],"ContextFetchBot":["partial","*",161,28]},"llms":{"status":"other_text","http":200,"bytes":2},"home":{"status":"ok","http":200,"url":"https://www.rtve.es/","html_tokens":353920,"markdown_tokens":69,"pruned_tokens":69}},
{"rank":1433,"domain":"checkpoint.com","origin":"https://checkpoint.com","robots":{"status":"html","http":202,"url":"https://checkpoint.com/robots.txt","bytes":2070,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":1434,"domain":"draft.blogger.com","origin":"https://draft.blogger.com","robots":{"status":"ok","http":200,"url":"https://draft.blogger.com/robots.txt","bytes":572,"groups":1},"bots":{"GPTBot":["partial","*",21,0],"ChatGPT-User":["partial","*",21,0],"OAI-SearchBot":["partial","*",21,0],"ClaudeBot":["partial","*",21,0],"Claude-User":["partial","*",21,0],"Claude-SearchBot":["partial","*",21,0],"anthropic-ai":["partial","*",21,0],"PerplexityBot":["partial","*",21,0],"Perplexity-User":["partial","*",21,0],"Google-Extended":["partial","*",21,0],"Applebot-Extended":["partial","*",21,0],"CCBot":["partial","*",21,0],"Bytespider":["partial","*",21,0],"Amazonbot":["partial","*",21,0],"meta-externalagent":["partial","*",21,0],"cohere-ai":["partial","*",21,0],"Diffbot":["partial","*",21,0],"ContextFetchBot":["partial","*",21,0]},"llms":{"status":"missing","http":404,"bytes":168},"home":{"status":"ok","http":200,"url":"https://draft.blogger.com/about/?bpli=1","html_tokens":42097,"markdown_tokens":271,"pruned_tokens":145}},
{"rank":1435,"domain":"emarketer.com","origin":"https://emarketer.com","robots":{"status":"ok","http":200,"url":"https://www.emarketer.com/robots.txt","bytes":3776,"groups":2},"bots":{"GPTBot":["partial","*",127,1],"ChatGPT-User":["partial","*",127,1],"OAI-SearchBot":["partial","*",127,1],"ClaudeBot":["partial","*",127,1],"Claude-User":["partial","*",127,1],"Claude-SearchBot":["partial","*",127,1],"anthropic-ai":["partial","*",127,1],"PerplexityBot":["partial","*",127,1],"Perplexity-User":["partial","*",127,1],"Google-Extended":["partial","named",2,0],"Applebot-Extended":["partial","*",127,1],"CCBot":["partial","*",127,1],"Bytespider":["partial","*",127,1],"Amazonbot":["partial","*",127,1],"meta-externalagent":["partial","*",127,1],"cohere-ai":["partial","*",127,1],"Diffbot":["partial","*",127,1],"ContextFetchBot":["partial","*",127,1]},"llms":{"status":"missing","http":404,"bytes":105164},"home":{"status":"http_error","http":403,"url":"https://www.emarketer.com/"}},
{"rank":1436,"domain":"thinkwithgoogle.com","origin":"https://thinkwithgoogle.com","robots":{"status":"ok","http":200,"url":"https://www.thinkwithgoogle.com/robots.txt","bytes":175,"groups":2},"bots":{"GPTBot":["partial","*",1,1],"ChatGPT-User":["partial","*",1,1],"OAI-SearchBot":["partial","*",1,1],"ClaudeBot":["partial","*",1,1],"Claude-User":["partial","*",1,1],"Claude-SearchBot":["partial","*",1,1],"anthropic-ai":["partial","*",1,1],"PerplexityBot":["partial","*",1,1],"Perplexity-User":["partial","*",1,1],"Google-Extended":["partial","*",1,1],"Applebot-Extended":["partial","*",1,1],"CCBot":["partial","*",1,1],"Bytespider":["partial","*",1,1],"Amazonbot":["partial","*",1,1],"meta-externalagent":["partial","*",1,1],"cohere-ai":["partial","*",1,1],"Diffbot":["partial","*",1,1],"ContextFetchBot":["partial","*",1,1]},"llms":{"status":"html","http":200,"bytes":223505},"home":{"status":"ok","http":200,"url":"https://business.google.com/us/think/","html_tokens":55827,"markdown_tokens":10,"pruned_tokens":10}},
{"rank":1437,"domain":"python.ca","origin":"https://python.ca","robots":{"status":"ok","http":200,"url":"https://python.ca/robots.txt","bytes":32,"groups":1},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":146},"home":{"status":"ok","http":200,"url":"https://www.python.org/","html_tokens":12940,"markdown_tokens":254,"pruned_tokens":58}},
{"rank":1438,"domain":"lavanguardia.com","origin":"https://lavanguardia.com","robots":{"status":"ok","http":200,"url":"https://www.lavanguardia.com/robots.txt","bytes":12984,"groups":35},"bots":{"GPTBot":["partial","named",1,2],"ChatGPT-User":["partial","*",241,71],"OAI-SearchBot":["partial","*",241,71],"ClaudeBot":["partial","named",1,2],"Claude-User":["partial","*",241,71],"Claude-SearchBot":["partial","*",241,71],"anthropic-ai":["partial","*",241,71],"PerplexityBot":["partial","*",241,71],"Perplexity-User":["partial","*",241,71],"Google-Extended":["partial","*",241,71],"Applebot-Extended":["partial","*",241,71],"CCBot":["partial","named",1,2],"Bytespider":["partial","named",1,2],"Amazonbot":["partial","*",241,71],"meta-externalagent":["partial","*",241,71],"cohere-ai":["partial","*",241,71],"Diffbot":["partial","*",241,71],"ContextFetchBot":["partial","*",241,71]},"llms":{"status":"present","http":200,"bytes":3656},"home":{"status":"ok","http":200,"url":"https://www.lavanguardia.com/","html_tokens":179999,"markdown_tokens":301,"pruned_tokens":147}},
{"rank":1439,"domain":"uscis.gov","origin":"https://uscis.gov","robots":{"status":"refused","http":403,"url":"https://uscis.gov/robots.txt","bytes":373,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":1440,"domain":"franceinfo.fr","origin":"https://franceinfo.fr","robots":{"status":"ok","http":200,"url":"https://www.franceinfo.fr/robots.txt","bytes":3692,"groups":6},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["blocked","named",1,0],"OAI-SearchBot":["blocked","named",1,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["blocked","named",1,0],"Claude-SearchBot":["blocked","named",1,0],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["blocked","named",1,0],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",36,0]},"llms":{"status":"missing","http":404,"bytes":285146},"home":{"status":"ok","http":200,"url":"https://www.franceinfo.fr/","html_tokens":401168,"markdown_tokens":228,"pruned_tokens":175}},
{"rank":1441,"domain":"laravel-news.com","origin":"https://laravel-news.com","robots":{"status":"ok","http":200,"url":"https://laravel-news.com/robots.txt","bytes":144,"groups":1},"bots":{"GPTBot":["partial","*",2,0],"ChatGPT-User":["partial","*",2,0],"OAI-SearchBot":["partial","*",2,0],"ClaudeBot":["partial","*",2,0],"Claude-User":["partial","*",2,0],"Claude-SearchBot":["partial","*",2,0],"anthropic-ai":["partial","*",2,0],"PerplexityBot":["partial","*",2,0],"Perplexity-User":["partial","*",2,0],"Google-Extended":["partial","*",2,0],"Applebot-Extended":["partial","*",2,0],"CCBot":["partial","*",2,0],"Bytespider":["partial","*",2,0],"Amazonbot":["partial","*",2,0],"meta-externalagent":["partial","*",2,0],"cohere-ai":["partial","*",2,0],"Diffbot":["partial","*",2,0],"ContextFetchBot":["partial","*",2,0]},"llms":{"status":"missing","http":404,"bytes":38802},"home":{"status":"ok","http":200,"url":"https://laravel-news.com/","html_tokens":30035,"markdown_tokens":165,"pruned_tokens":159}},
{"rank":1442,"domain":"packagist.org","origin":"https://packagist.org","robots":{"status":"ok","http":200,"url":"https://packagist.org/robots.txt","bytes":154,"groups":1},"bots":{"GPTBot":["partial","*",3,0],"ChatGPT-User":["partial","*",3,0],"OAI-SearchBot":["partial","*",3,0],"ClaudeBot":["partial","*",3,0],"Claude-User":["partial","*",3,0],"Claude-SearchBot":["partial","*",3,0],"anthropic-ai":["partial","*",3,0],"PerplexityBot":["partial","*",3,0],"Perplexity-User":["partial","*",3,0],"Google-Extended":["partial","*",3,0],"Applebot-Extended":["partial","*",3,0],"CCBot":["partial","*",3,0],"Bytespider":["partial","*",3,0],"Amazonbot":["partial","*",3,0],"meta-externalagent":["partial","*",3,0],"cohere-ai":["partial","*",3,0],"Diffbot":["partial","*",3,0],"ContextFetchBot":["partial","*",3,0]},"llms":{"status":"missing","http":404,"bytes":12022},"home":{"status":"ok","http":200,"url":"https://packagist.org/","html_tokens":4505,"markdown_tokens":850,"pruned_tokens":170}},
{"rank":1443,"domain":"rand.org","origin":"https://rand.org","robots":{"status":"refused","http":403,"url":"https://www.rand.org/robots.txt","bytes":919,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":1444,"domain":"hcaptcha.com","origin":"https://hcaptcha.com","robots":{"status":"ok","http":200,"url":"https://www.hcaptcha.com/robots.txt","bytes":109,"groups":1},"bots":{"GPTBot":["partial","*",2,1],"ChatGPT-User":["partial","*",2,1],"OAI-SearchBot":["partial","*",2,1],"ClaudeBot":["partial","*",2,1],"Claude-User":["partial","*",2,1],"Claude-SearchBot":["partial","*",2,1],"anthropic-ai":["partial","*",2,1],"PerplexityBot":["partial","*",2,1],"Perplexity-User":["partial","*",2,1],"Google-Extended":["partial","*",2,1],"Applebot-Extended":["partial","*",2,1],"CCBot":["partial","*",2,1],"Bytespider":["partial","*",2,1],"Amazonbot":["partial","*",2,1],"meta-externalagent":["partial","*",2,1],"cohere-ai":["partial","*",2,1],"Diffbot":["partial","*",2,1],"ContextFetchBot":["partial","*",2,1]},"llms":{"status":"present","http":200,"bytes":13865},"home":{"status":"ok","http":200,"url":"https://www.hcaptcha.com/","html_tokens":25572,"markdown_tokens":137,"pruned_tokens":76}},
{"rank":1445,"domain":"artstation.com","origin":"https://artstation.com","robots":{"status":"ok","http":200,"url":"https://www.artstation.com/robots.txt","bytes":927,"groups":1},"bots":{"GPTBot":["partial","*",10,0],"ChatGPT-User":["partial","*",10,0],"OAI-SearchBot":["partial","*",10,0],"ClaudeBot":["partial","*",10,0],"Claude-User":["partial","*",10,0],"Claude-SearchBot":["partial","*",10,0],"anthropic-ai":["partial","*",10,0],"PerplexityBot":["partial","*",10,0],"Perplexity-User":["partial","*",10,0],"Google-Extended":["partial","*",10,0],"Applebot-Extended":["partial","*",10,0],"CCBot":["partial","*",10,0],"Bytespider":["partial","*",10,0],"Amazonbot":["partial","*",10,0],"meta-externalagent":["partial","*",10,0],"cohere-ai":["partial","*",10,0],"Diffbot":["partial","*",10,0],"ContextFetchBot":["partial","*",10,0]},"llms":{"status":"html","http":200,"bytes":91611},"home":{"status":"http_error","http":403,"url":"https://artstation.com/"}},
{"rank":1446,"domain":"athemes.com","origin":"https://athemes.com","robots":{"status":"ok","http":200,"url":"https://athemes.com/robots.txt","bytes":422,"groups":1},"bots":{"GPTBot":["partial","*",11,1],"ChatGPT-User":["partial","*",11,1],"OAI-SearchBot":["partial","*",11,1],"ClaudeBot":["partial","*",11,1],"Claude-User":["partial","*",11,1],"Claude-SearchBot":["partial","*",11,1],"anthropic-ai":["partial","*",11,1],"PerplexityBot":["partial","*",11,1],"Perplexity-User":["partial","*",11,1],"Google-Extended":["partial","*",11,1],"Applebot-Extended":["partial","*",11,1],"CCBot":["partial","*",11,1],"Bytespider":["partial","*",11,1],"Amazonbot":["partial","*",11,1],"meta-externalagent":["partial","*",11,1],"cohere-ai":["partial","*",11,1],"Diffbot":["partial","*",11,1],"ContextFetchBot":["partial","*",11,1]},"llms":{"status":"other_text","http":200,"bytes":93338},"home":{"status":"ok","http":200,"url":"https://athemes.com/","html_tokens":44122,"markdown_tokens":374,"pruned_tokens":74}},
{"rank":1447,"domain":"everydayhealth.com","origin":"https://everydayhealth.com","robots":{"status":"ok","http":200,"url":"https://www.everydayhealth.com/robots.txt","bytes":183249,"groups":4},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["blocked","named",1,0],"OAI-SearchBot":["blocked","named",1,0],"ClaudeBot":["partial","named",1,2],"Claude-User":["partial","named",1,2],"Claude-SearchBot":["partial","named",1,2],"anthropic-ai":["partial","named",1,2],"PerplexityBot":["partial","named",1,2],"Perplexity-User":["partial","named",1,2],"Google-Extended":["partial","named",2764,0],"Applebot-Extended":["partial","named",1,2],"CCBot":["partial","named",1,2],"Bytespider":["partial","named",1,2],"Amazonbot":["partial","named",1,2],"meta-externalagent":["partial","named",1,2],"cohere-ai":["partial","named",1,2],"Diffbot":["partial","named",1,2],"ContextFetchBot":["partial","*",14,0]},"llms":{"status":"missing","http":404,"bytes":126902},"home":{"status":"ok","http":200,"url":"https://www.everydayhealth.com/","html_tokens":74512,"markdown_tokens":660,"pruned_tokens":164}},
{"rank":1448,"domain":"kommersant.ru","origin":"https://kommersant.ru","robots":{"status":"ok","http":200,"url":"https://www.kommersant.ru/robots.txt","bytes":4026,"groups":6},"bots":{"GPTBot":["partial","*",73,1],"ChatGPT-User":["partial","*",73,1],"OAI-SearchBot":["partial","*",73,1],"ClaudeBot":["partial","*",73,1],"Claude-User":["partial","*",73,1],"Claude-SearchBot":["partial","*",73,1],"anthropic-ai":["partial","*",73,1],"PerplexityBot":["partial","*",73,1],"Perplexity-User":["partial","*",73,1],"Google-Extended":["partial","*",73,1],"Applebot-Extended":["partial","*",73,1],"CCBot":["partial","*",73,1],"Bytespider":["partial","*",73,1],"Amazonbot":["partial","*",73,1],"meta-externalagent":["partial","*",73,1],"cohere-ai":["partial","*",73,1],"Diffbot":["partial","*",73,1],"ContextFetchBot":["partial","*",73,1]},"llms":{"status":"missing","http":404,"bytes":343085},"home":{"status":"ok","http":200,"url":"https://www.kommersant.ru/","html_tokens":188392,"markdown_tokens":1028,"pruned_tokens":173}},
{"rank":1449,"domain":"elementor.com","origin":"https://elementor.com","robots":{"status":"ok","http":200,"url":"https://elementor.com/robots.txt","bytes":24,"groups":1},"bots":{"GPTBot":["allowed","*",0,1],"ChatGPT-User":["allowed","*",0,1],"OAI-SearchBot":["allowed","*",0,1],"ClaudeBot":["allowed","*",0,1],"Claude-User":["allowed","*",0,1],"Claude-SearchBot":["allowed","*",0,1],"anthropic-ai":["allowed","*",0,1],"PerplexityBot":["allowed","*",0,1],"Perplexity-User":["allowed","*",0,1],"Google-Extended":["allowed","*",0,1],"Applebot-Extended":["allowed","*",0,1],"CCBot":["allowed","*",0,1],"Bytespider":["allowed","*",0,1],"Amazonbot":["allowed","*",0,1],"meta-externalagent":["allowed","*",0,1],"cohere-ai":["allowed","*",0,1],"Diffbot":["allowed","*",0,1],"ContextFetchBot":["allowed","*",0,1]},"llms":{"status":"present","http":200,"bytes":13563},"home":{"status":"ok","http":200,"url":"https://elementor.com/","html_tokens":155022,"markdown_tokens":689,"pruned_tokens":77}},
{"rank":1450,"domain":"tripadvisor.co.uk","origin":"https://tripadvisor.co.uk","robots":{"status":"ok","http":200,"url":"https://www.tripadvisor.co.uk/robots.txt","bytes":22931,"groups":9},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["partial","named",623,8],"OAI-SearchBot":["partial","named",623,8],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",621,8],"Claude-SearchBot":["partial","*",621,8],"anthropic-ai":["partial","*",621,8],"PerplexityBot":["partial","named",625,8],"Perplexity-User":["partial","*",621,8],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["partial","*",621,8],"ContextFetchBot":["partial","*",621,8]},"llms":{"status":"http_error","http":403,"bytes":777},"home":{"status":"http_error","http":403,"url":"https://www.tripadvisor.co.uk/"}},
{"rank":1451,"domain":"dartmouth.edu","origin":"https://dartmouth.edu","robots":{"status":"ok","http":200,"url":"http://www.dartmouth.edu/robots.txt","bytes":151,"groups":2},"bots":{"GPTBot":["partial","*",3,0],"ChatGPT-User":["partial","*",3,0],"OAI-SearchBot":["partial","*",3,0],"ClaudeBot":["partial","*",3,0],"Claude-User":["partial","*",3,0],"Claude-SearchBot":["partial","*",3,0],"anthropic-ai":["partial","*",3,0],"PerplexityBot":["partial","*",3,0],"Perplexity-User":["partial","*",3,0],"Google-Extended":["partial","*",3,0],"Applebot-Extended":["partial","*",3,0],"CCBot":["partial","*",3,0],"Bytespider":["partial","*",3,0],"Amazonbot":["partial","*",3,0],"meta-externalagent":["partial","*",3,0],"cohere-ai":["partial","*",3,0],"Diffbot":["partial","*",3,0],"ContextFetchBot":["partial","*",3,0]},"llms":{"status":"missing","http":404,"bytes":3596},"home":{"status":"ok","http":200,"url":"https://home.dartmouth.edu/","html_tokens":32756,"markdown_tokens":350,"pruned_tokens":158}},
{"rank":1452,"domain":"focus.de","origin":"https://focus.de","robots":{"status":"refused","http":403,"url":"https://focus.de/robots.txt","bytes":372,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":1453,"domain":"ons.gov.uk","origin":"https://ons.gov.uk","robots":{"status":"missing","http":404,"url":"https://www.ons.gov.uk/robots.txt","bytes":102334,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":102324},"home":{"status":"ok","http":200,"url":"https://www.ons.gov.uk/","html_tokens":20214,"markdown_tokens":16,"pruned_tokens":16}},
{"rank":1454,"domain":"garanteprivacy.it","origin":"https://garanteprivacy.it","robots":{"status":"timeout","http":null,"url":"https://garanteprivacy.it/robots.txt","bytes":0,"groups":0,"error":"ConnectTimeout"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":1455,"domain":"teepublic.com","origin":"https://teepublic.com","robots":{"status":"refused","http":403,"url":"https://www.teepublic.com/robots.txt","bytes":5576,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":1456,"domain":"cosmopolitan.com","origin":"https://cosmopolitan.com","robots":{"status":"ok","http":200,"url":"https://cosmopolitan.com/robots.txt","bytes":1093,"groups":3},"bots":{"GPTBot":["partial","named",32,0],"ChatGPT-User":["partial","named",32,0],"OAI-SearchBot":["partial","named",32,0],"ClaudeBot":["partial","*",19,0],"Claude-User":["partial","*",19,0],"Claude-SearchBot":["partial","*",19,0],"anthropic-ai":["partial","*",19,0],"PerplexityBot":["partial","*",19,0],"Perplexity-User":["partial","*",19,0],"Google-Extended":["partial","*",19,0],"Applebot-Extended":["partial","*",19,0],"CCBot":["partial","*",19,0],"Bytespider":["partial","*",19,0],"Amazonbot":["partial","*",19,0],"meta-externalagent":["partial","*",19,0],"cohere-ai":["partial","*",19,0],"Diffbot":["partial","*",19,0],"ContextFetchBot":["partial","*",19,0]},"llms":{"status":"missing","http":404,"bytes":84},"home":{"status":"ok","http":200,"url":"https://www.cosmopolitan.com/","html_tokens":172952,"markdown_tokens":202,"pruned_tokens":44}},
{"rank":1457,"domain":"fujitsu.com","origin":"https://www.fujitsu.com","robots":{"status":"refused","http":429,"url":"https://global.fujitsu/en-global","bytes":34223,"groups":0,"error":"HTTP 429"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":1458,"domain":"mongodb.com","origin":"https://mongodb.com","robots":{"status":"ok","http":200,"url":"https://www.mongodb.com/robots.txt","bytes":2859,"groups":7},"bots":{"GPTBot":["partial","*",63,0],"ChatGPT-User":["partial","*",63,0],"OAI-SearchBot":["partial","*",63,0],"ClaudeBot":["partial","*",63,0],"Claude-User":["partial","*",63,0],"Claude-SearchBot":["partial","*",63,0],"anthropic-ai":["partial","*",63,0],"PerplexityBot":["partial","*",63,0],"Perplexity-User":["partial","*",63,0],"Google-Extended":["partial","*",63,0],"Applebot-Extended":["partial","*",63,0],"CCBot":["partial","*",63,0],"Bytespider":["partial","*",63,0],"Amazonbot":["partial","*",63,0],"meta-externalagent":["partial","*",63,0],"cohere-ai":["partial","*",63,0],"Diffbot":["partial","*",63,0],"ContextFetchBot":["partial","*",63,0]},"llms":{"status":"present","http":200,"bytes":10385},"home":{"status":"ok","http":200,"url":"https://www.mongodb.com/","html_tokens":250186,"markdown_tokens":726,"pruned_tokens":386}},
{"rank":1459,"domain":"imagemagick.com","origin":"http://imagemagick.com","robots":{"status":"missing","http":404,"url":"http://imagemagick.com/robots.txt","bytes":236,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":236},"home":{"status":"ok","http":200,"url":"https://imagemagick.org","html_tokens":6550,"markdown_tokens":133,"pruned_tokens":101}},
{"rank":1460,"domain":"unimelb.edu.au","origin":"https://unimelb.edu.au","robots":{"status":"refused","http":403,"url":"https://unimelb.edu.au/robots.txt","bytes":5530,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":1461,"domain":"webflow.com","origin":"https://webflow.com","robots":{"status":"ok","http":200,"url":"https://webflow.com/robots.txt","bytes":279,"groups":1},"bots":{"GPTBot":["partial","*",5,2],"ChatGPT-User":["partial","*",5,2],"OAI-SearchBot":["partial","*",5,2],"ClaudeBot":["partial","*",5,2],"Claude-User":["partial","*",5,2],"Claude-SearchBot":["partial","*",5,2],"anthropic-ai":["partial","*",5,2],"PerplexityBot":["partial","*",5,2],"Perplexity-User":["partial","*",5,2],"Google-Extended":["partial","*",5,2],"Applebot-Extended":["partial","*",5,2],"CCBot":["partial","*",5,2],"Bytespider":["partial","*",5,2],"Amazonbot":["partial","*",5,2],"meta-externalagent":["partial","*",5,2],"cohere-ai":["partial","*",5,2],"Diffbot":["partial","*",5,2],"ContextFetchBot":["partial","*",5,2]},"llms":{"status":"present","http":200,"bytes":13293},"home":{"status":"ok","http":200,"url":"https://webflow.com/","html_tokens":131130,"markdown_tokens":2498,"pruned_tokens":600}},
{"rank":1462,"domain":"homestead.com","origin":"https://homestead.com","robots":{"status":"refused","http":403,"url":"https://homestead.com/robots.txt","bytes":5572,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":1463,"domain":"usf.edu","origin":"https://www.usf.edu","robots":{"status":"ok","http":200,"url":"https://www.usf.edu/robots.txt","bytes":996,"groups":3},"bots":{"GPTBot":["partial","*",25,0],"ChatGPT-User":["partial","*",25,0],"OAI-SearchBot":["partial","*",25,0],"ClaudeBot":["partial","*",25,0],"Claude-User":["partial","*",25,0],"Claude-SearchBot":["partial","*",25,0],"anthropic-ai":["partial","*",25,0],"PerplexityBot":["partial","*",25,0],"Perplexity-User":["partial","*",25,0],"Google-Extended":["partial","*",25,0],"Applebot-Extended":["partial","*",25,0],"CCBot":["partial","*",25,0],"Bytespider":["partial","*",25,0],"Amazonbot":["partial","*",25,0],"meta-externalagent":["partial","*",25,0],"cohere-ai":["partial","*",25,0],"Diffbot":["partial","*",25,0],"ContextFetchBot":["partial","*",25,0]},"llms":{"status":"disallowed"},"home":{"status":"ok","http":200,"url":"https://www.usf.edu/","html_tokens":34598,"markdown_tokens":57,"pruned_tokens":57}},
{"rank":1464,"domain":"meti.go.jp","origin":"https://www.meti.go.jp","robots":{"status":"refused","http":403,"url":"https://www.meti.go.jp/robots.txt","bytes":5045,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":1465,"domain":"gb.net","origin":"https://gb.net","robots":{"status":"ok","http":200,"url":"https://gb.net/robots.txt","bytes":0,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":8004},"home":{"status":"ok","http":200,"url":"https://gb.net/","html_tokens":9822,"markdown_tokens":820,"pruned_tokens":746}},
{"rank":1466,"domain":"wikiquote.org","origin":"https://wikiquote.org","robots":{"status":"ok","http":200,"url":"https://en.wikiquote.org/robots.txt","bytes":15253,"groups":33},"bots":{"GPTBot":["partial","*",200,4],"ChatGPT-User":["partial","*",200,4],"OAI-SearchBot":["partial","*",200,4],"ClaudeBot":["partial","*",200,4],"Claude-User":["partial","*",200,4],"Claude-SearchBot":["partial","*",200,4],"anthropic-ai":["partial","*",200,4],"PerplexityBot":["partial","*",200,4],"Perplexity-User":["partial","*",200,4],"Google-Extended":["partial","*",200,4],"Applebot-Extended":["partial","*",200,4],"CCBot":["partial","*",200,4],"Bytespider":["partial","*",200,4],"Amazonbot":["partial","*",200,4],"meta-externalagent":["partial","*",200,4],"cohere-ai":["partial","*",200,4],"Diffbot":["partial","*",200,4],"ContextFetchBot":["partial","*",200,4]},"llms":{"status":"missing","http":404,"bytes":1868},"home":{"status":"ok","http":200,"url":"https://www.wikiquote.org/","html_tokens":16872,"markdown_tokens":10,"pruned_tokens":10}},
{"rank":1467,"domain":"codeplex.com","origin":"https://codeplex.com","robots":{"status":"html","http":200,"url":"https://www.microsoft.com/en-us","bytes":283550,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":1468,"domain":"google.pl","origin":"https://google.pl","robots":{"status":"ok","http":200,"url":"https://www.google.pl/robots.txt","bytes":6624,"groups":4},"bots":{"GPTBot":["partial","*",169,64],"ChatGPT-User":["partial","*",169,64],"OAI-SearchBot":["partial","*",169,64],"ClaudeBot":["partial","*",169,64],"Claude-User":["partial","*",169,64],"Claude-SearchBot":["partial","*",169,64],"anthropic-ai":["partial","*",169,64],"PerplexityBot":["partial","*",169,64],"Perplexity-User":["partial","*",169,64],"Google-Extended":["partial","*",169,64],"Applebot-Extended":["partial","*",169,64],"CCBot":["partial","*",169,64],"Bytespider":["partial","*",169,64],"Amazonbot":["partial","*",169,64],"meta-externalagent":["partial","*",169,64],"cohere-ai":["partial","*",169,64],"Diffbot":["partial","*",169,64],"ContextFetchBot":["partial","*",169,64]},"llms":{"status":"missing","http":404,"bytes":1569},"home":{"status":"ok","http":200,"url":"https://www.google.com/","html_tokens":21339,"markdown_tokens":30,"pruned_tokens":30}},
{"rank":1469,"domain":"pinimg.com","origin":"https://pinimg.com","robots":{"status":"ok","http":200,"url":"https://www.pinterest.com/robots.txt","bytes":43558,"groups":8},"bots":{"GPTBot":["blocked","*",1,0],"ChatGPT-User":["blocked","*",1,0],"OAI-SearchBot":["blocked","*",1,0],"ClaudeBot":["blocked","*",1,0],"Claude-User":["blocked","*",1,0],"Claude-SearchBot":["blocked","*",1,0],"anthropic-ai":["blocked","*",1,0],"PerplexityBot":["blocked","*",1,0],"Perplexity-User":["blocked","*",1,0],"Google-Extended":["blocked","*",1,0],"Applebot-Extended":["blocked","*",1,0],"CCBot":["blocked","*",1,0],"Bytespider":["blocked","*",1,0],"Amazonbot":["blocked","*",1,0],"meta-externalagent":["blocked","*",1,0],"cohere-ai":["blocked","*",1,0],"Diffbot":["blocked","*",1,0],"ContextFetchBot":["blocked","*",1,0]},"llms":{"status":"disallowed"},"home":{"status":"disallowed"}},
{"rank":1470,"domain":"fsu.edu","origin":"https://fsu.edu","robots":{"status":"refused","http":403,"url":"https://www.fsu.edu/robots.txt","bytes":560,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":1471,"domain":"flipboard.com","origin":"https://flipboard.com","robots":{"status":"ok","http":200,"url":"https://flipboard.com/robots.txt","bytes":5261,"groups":3},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["partial","*",18,0],"OAI-SearchBot":["blocked","named",1,0],"ClaudeBot":["partial","*",18,0],"Claude-User":["partial","*",18,0],"Claude-SearchBot":["partial","*",18,0],"anthropic-ai":["partial","*",18,0],"PerplexityBot":["partial","*",18,0],"Perplexity-User":["partial","*",18,0],"Google-Extended":["partial","*",18,0],"Applebot-Extended":["partial","*",18,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["partial","*",18,0],"cohere-ai":["partial","*",18,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",18,0]},"llms":{"status":"missing","http":404,"bytes":67114},"home":{"status":"ok","http":200,"url":"https://flipboard.com/","html_tokens":326786,"markdown_tokens":14,"pruned_tokens":14}},
{"rank":1472,"domain":"iplanet.com","origin":"https://iplanet.com","robots":{"status":"timeout","http":null,"url":"https://iplanet.com/robots.txt","bytes":0,"groups":0,"error":"ConnectTimeout"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":1473,"domain":"nato.int","origin":"https://nato.int","robots":{"status":"ok","http":200,"url":"https://nato.int/robots.txt","bytes":80,"groups":1},"bots":{"GPTBot":["partial","*",1,0],"ChatGPT-User":["partial","*",1,0],"OAI-SearchBot":["partial","*",1,0],"ClaudeBot":["partial","*",1,0],"Claude-User":["partial","*",1,0],"Claude-SearchBot":["partial","*",1,0],"anthropic-ai":["partial","*",1,0],"PerplexityBot":["partial","*",1,0],"Perplexity-User":["partial","*",1,0],"Google-Extended":["partial","*",1,0],"Applebot-Extended":["partial","*",1,0],"CCBot":["partial","*",1,0],"Bytespider":["partial","*",1,0],"Amazonbot":["partial","*",1,0],"meta-externalagent":["partial","*",1,0],"cohere-ai":["partial","*",1,0],"Diffbot":["partial","*",1,0],"ContextFetchBot":["partial","*",1,0]},"llms":{"status":"missing","http":404,"bytes":287164},"home":{"status":"ok","http":200,"url":"https://nato.int/en","html_tokens":84705,"markdown_tokens":92,"pruned_tokens":88}},
{"rank":1474,"domain":"aboutamazon.com","origin":"https://aboutamazon.com","robots":{"status":"ok","http":200,"url":"https://www.aboutamazon.com/robots.txt","bytes":78,"groups":1},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"missing","http":404,"bytes":36791},"home":{"status":"ok","http":200,"url":"https://www.aboutamazon.com/","html_tokens":88195,"markdown_tokens":559,"pruned_tokens":159}},
{"rank":1475,"domain":"mass.gov","origin":"https://mass.gov","robots":{"status":"refused","http":403,"url":"https://www.mass.gov/robots.txt","bytes":14061,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":1476,"domain":"lastpass.com","origin":"https://lastpass.com","robots":{"status":"ok","http":200,"url":"https://www.lastpass.com/robots.txt","bytes":1078,"groups":10},"bots":{"GPTBot":["allowed","named",0,1],"ChatGPT-User":["allowed","named",0,1],"OAI-SearchBot":["allowed","named",0,1],"ClaudeBot":["allowed","named",0,1],"Claude-User":["allowed","named",0,1],"Claude-SearchBot":["allowed","named",0,1],"anthropic-ai":["partial","*",27,0],"PerplexityBot":["allowed","named",0,1],"Perplexity-User":["allowed","named",0,1],"Google-Extended":["allowed","named",0,1],"Applebot-Extended":["partial","*",27,0],"CCBot":["partial","*",27,0],"Bytespider":["partial","*",27,0],"Amazonbot":["partial","*",27,0],"meta-externalagent":["partial","*",27,0],"cohere-ai":["partial","*",27,0],"Diffbot":["partial","*",27,0],"ContextFetchBot":["partial","*",27,0]},"llms":{"status":"missing","http":404,"bytes":0},"home":{"status":"ok","http":200,"url":"https://www.lastpass.com/","html_tokens":64893,"markdown_tokens":4039,"pruned_tokens":265}},
{"rank":1477,"domain":"sports.yahoo.com","origin":"https://sports.yahoo.com","robots":{"status":"ok","http":200,"url":"https://sports.yahoo.com/robots.txt","bytes":2167,"groups":44},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["blocked","named",1,0],"OAI-SearchBot":["partial","*",20,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",20,0],"Claude-SearchBot":["partial","*",20,0],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["partial","*",20,0],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["partial","*",20,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["partial","*",20,0],"meta-externalagent":["partial","*",20,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",20,0]},"llms":{"status":"missing","http":404,"bytes":618934},"home":{"status":"ok","http":200,"url":"https://sports.yahoo.com/","html_tokens":234909,"markdown_tokens":33,"pruned_tokens":33}},
{"rank":1478,"domain":"social-plugins.line.me","origin":"https://social-plugins.line.me","robots":{"status":"html","http":200,"url":"https://developers.line.biz/?r=/docs/line-social-plugins/","bytes":42909,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":1479,"domain":"stadtgame.com","origin":"https://stadtgame.com","robots":{"status":"refused","http":429,"url":"https://stadtgame.com/robots.txt","bytes":17,"groups":0,"error":"HTTP 429"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":1480,"domain":"getcomposer.org","origin":"https://getcomposer.org","robots":{"status":"ok","http":200,"url":"https://getcomposer.org/robots.txt","bytes":15,"groups":1},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"missing","http":404,"bytes":993},"home":{"status":"ok","http":200,"url":"https://getcomposer.org/","html_tokens":1133,"markdown_tokens":16,"pruned_tokens":16}},
{"rank":1481,"domain":"francetvinfo.fr","origin":"https://francetvinfo.fr","robots":{"status":"ok","http":200,"url":"https://www.franceinfo.fr/robots.txt","bytes":3692,"groups":6},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["blocked","named",1,0],"OAI-SearchBot":["blocked","named",1,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["blocked","named",1,0],"Claude-SearchBot":["blocked","named",1,0],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["blocked","named",1,0],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",36,0]},"llms":{"status":"missing","http":404,"bytes":317674},"home":{"status":"ok","http":200,"url":"https://www.franceinfo.fr/","html_tokens":401168,"markdown_tokens":228,"pruned_tokens":175}},
{"rank":1482,"domain":"techrepublic.com","origin":"https://techrepublic.com","robots":{"status":"ok","http":200,"url":"https://www.techrepublic.com/robots.txt","bytes":671,"groups":2},"bots":{"GPTBot":["partial","*",20,1],"ChatGPT-User":["partial","*",20,1],"OAI-SearchBot":["partial","*",20,1],"ClaudeBot":["partial","*",20,1],"Claude-User":["partial","*",20,1],"Claude-SearchBot":["partial","*",20,1],"anthropic-ai":["partial","*",20,1],"PerplexityBot":["partial","*",20,1],"Perplexity-User":["partial","*",20,1],"Google-Extended":["partial","*",20,1],"Applebot-Extended":["partial","*",20,1],"CCBot":["partial","*",20,1],"Bytespider":["partial","*",20,1],"Amazonbot":["partial","*",20,1],"meta-externalagent":["partial","*",20,1],"cohere-ai":["partial","*",20,1],"Diffbot":["partial","*",20,1],"ContextFetchBot":["partial","*",20,1]},"llms":{"status":"present","http":200,"bytes":14765},"home":{"status":"http_error","http":403,"url":"https://www.techrepublic.com/"}},
{"rank":1483,"domain":"peatix.com","origin":"https://peatix.com","robots":{"status":"ok","http":200,"url":"https://peatix.com/robots.txt","bytes":1349,"groups":2},"bots":{"GPTBot":["partial","*",30,9],"ChatGPT-User":["partial","*",30,9],"OAI-SearchBot":["partial","*",30,9],"ClaudeBot":["partial","*",30,9],"Claude-User":["partial","*",30,9],"Claude-SearchBot":["partial","*",30,9],"anthropic-ai":["partial","*",30,9],"PerplexityBot":["partial","*",30,9],"Perplexity-User":["partial","*",30,9],"Google-Extended":["partial","*",30,9],"Applebot-Extended":["partial","*",30,9],"CCBot":["partial","*",30,9],"Bytespider":["partial","*",30,9],"Amazonbot":["partial","*",30,9],"meta-externalagent":["partial","*",30,9],"cohere-ai":["partial","*",30,9],"Diffbot":["partial","*",30,9],"ContextFetchBot":["partial","*",30,9]},"llms":{"status":"html","http":200,"bytes":21168},"home":{"status":"ok","http":200,"url":"https://peatix.com/us/","html_tokens":5292,"markdown_tokens":0,"pruned_tokens":0}},
{"rank":1484,"domain":"test.com","origin":"https://www.test.com","robots":{"status":"refused","http":403,"url":"https://www.test.com/robots.txt","bytes":5507,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":1485,"domain":"myportfolio.com","origin":"https://myportfolio.com","robots":{"status":"ok","http":200,"url":"https://portfolio.adobe.com/robots.txt","bytes":32,"groups":1},"bots":{"GPTBot":["partial","*",1,0],"ChatGPT-User":["partial","*",1,0],"OAI-SearchBot":["partial","*",1,0],"ClaudeBot":["partial","*",1,0],"Claude-User":["partial","*",1,0],"Claude-SearchBot":["partial","*",1,0],"anthropic-ai":["partial","*",1,0],"PerplexityBot":["partial","*",1,0],"Perplexity-User":["partial","*",1,0],"Google-Extended":["partial","*",1,0],"Applebot-Extended":["partial","*",1,0],"CCBot":["partial","*",1,0],"Bytespider":["partial","*",1,0],"Amazonbot":["partial","*",1,0],"meta-externalagent":["partial","*",1,0],"cohere-ai":["partial","*",1,0],"Diffbot":["partial","*",1,0],"ContextFetchBot":["partial","*",1,0]},"llms":{"status":"html","http":200,"bytes":20668},"home":{"status":"ok","http":200,"url":"https://portfolio.adobe.com/","html_tokens":5167,"markdown_tokens":0,"pruned_tokens":0}},
{"rank":1486,"domain":"ebsco.com","origin":"https://ebsco.com","robots":{"status":"ok","http":200,"url":"https://www.ebsco.com/robots.txt","bytes":14338,"groups":23},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["partial","named",3,1],"OAI-SearchBot":["partial","named",3,1],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","named",3,1],"Claude-SearchBot":["partial","named",3,1],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["partial","named",3,1],"Perplexity-User":["partial","named",3,1],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",3,1]},"llms":{"status":"missing","http":404,"bytes":63894},"home":{"status":"ok","http":200,"url":"https://www.ebsco.com/","html_tokens":27389,"markdown_tokens":595,"pruned_tokens":62}},
{"rank":1487,"domain":"digitaloceanspaces.com","origin":"https://digitaloceanspaces.com","robots":{"status":"html","http":200,"url":"https://www.digitalocean.com","bytes":297763,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":1488,"domain":"norton.com","origin":"https://norton.com","robots":{"status":"refused","http":403,"url":"https://us.norton.com/robots.txt","bytes":379,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":1489,"domain":"byu.edu","origin":"https://byu.edu","robots":{"status":"ok","http":200,"url":"https://www.byu.edu/robots.txt","bytes":29,"groups":1},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"timeout","error":"timed out after 10 s"},"home":{"status":"disallowed","error":"robots.txt of www.byu.edu not readable (timeout)"}},
{"rank":1490,"domain":"handelsblatt.com","origin":"https://handelsblatt.com","robots":{"status":"ok","http":200,"url":"https://www.handelsblatt.com/robots.txt","bytes":1728,"groups":2},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["partial","*",32,1],"OAI-SearchBot":["partial","*",32,1],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",32,1],"Claude-SearchBot":["blocked","named",1,0],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["partial","*",32,1],"Perplexity-User":["partial","*",32,1],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",32,1]},"llms":{"status":"missing","http":404,"bytes":321},"home":{"status":"ok","http":200,"url":"https://www.handelsblatt.com/","html_tokens":456938,"markdown_tokens":61,"pruned_tokens":61}},
{"rank":1491,"domain":"smashingmagazine.com","origin":"https://smashingmagazine.com","robots":{"status":"ok","http":200,"url":"https://www.smashingmagazine.com/robots.txt","bytes":13053,"groups":1},"bots":{"GPTBot":["partial","*",190,1],"ChatGPT-User":["partial","*",190,1],"OAI-SearchBot":["partial","*",190,1],"ClaudeBot":["partial","*",190,1],"Claude-User":["partial","*",190,1],"Claude-SearchBot":["partial","*",190,1],"anthropic-ai":["partial","*",190,1],"PerplexityBot":["partial","*",190,1],"Perplexity-User":["partial","*",190,1],"Google-Extended":["partial","*",190,1],"Applebot-Extended":["partial","*",190,1],"CCBot":["partial","*",190,1],"Bytespider":["partial","*",190,1],"Amazonbot":["partial","*",190,1],"meta-externalagent":["partial","*",190,1],"cohere-ai":["partial","*",190,1],"Diffbot":["partial","*",190,1],"ContextFetchBot":["partial","*",190,1]},"llms":{"status":"missing","http":404,"bytes":148176},"home":{"status":"ok","http":200,"url":"https://www.smashingmagazine.com/","html_tokens":72195,"markdown_tokens":419,"pruned_tokens":77}},
{"rank":1492,"domain":"uoregon.edu","origin":"https://uoregon.edu","robots":{"status":"ok","http":200,"url":"https://www.uoregon.edu/robots.txt","bytes":60,"groups":1},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"missing","http":404,"bytes":66630},"home":{"status":"ok","http":200,"url":"https://www.uoregon.edu/","html_tokens":24507,"markdown_tokens":159,"pruned_tokens":159}},
{"rank":1493,"domain":"spec.org","origin":"https://spec.org","robots":{"status":"ok","http":200,"url":"https://spec.org/robots.txt","bytes":659,"groups":1},"bots":{"GPTBot":["partial","*",7,0],"ChatGPT-User":["partial","*",7,0],"OAI-SearchBot":["partial","*",7,0],"ClaudeBot":["partial","*",7,0],"Claude-User":["partial","*",7,0],"Claude-SearchBot":["partial","*",7,0],"anthropic-ai":["partial","*",7,0],"PerplexityBot":["partial","*",7,0],"Perplexity-User":["partial","*",7,0],"Google-Extended":["partial","*",7,0],"Applebot-Extended":["partial","*",7,0],"CCBot":["partial","*",7,0],"Bytespider":["partial","*",7,0],"Amazonbot":["partial","*",7,0],"meta-externalagent":["partial","*",7,0],"cohere-ai":["partial","*",7,0],"Diffbot":["partial","*",7,0],"ContextFetchBot":["partial","*",7,0]},"llms":{"status":"missing","http":404,"bytes":32901},"home":{"status":"ok","http":200,"url":"https://spec.org/","html_tokens":11100,"markdown_tokens":875,"pruned_tokens":541}},
{"rank":1494,"domain":"gla.ac.uk","origin":"https://gla.ac.uk","robots":{"status":"ok","http":200,"url":"https://www.gla.ac.uk/robots.txt","bytes":7027,"groups":1},"bots":{"GPTBot":["partial","*",209,0],"ChatGPT-User":["partial","*",209,0],"OAI-SearchBot":["partial","*",209,0],"ClaudeBot":["partial","*",209,0],"Claude-User":["partial","*",209,0],"Claude-SearchBot":["partial","*",209,0],"anthropic-ai":["partial","*",209,0],"PerplexityBot":["partial","*",209,0],"Perplexity-User":["partial","*",209,0],"Google-Extended":["partial","*",209,0],"Applebot-Extended":["partial","*",209,0],"CCBot":["partial","*",209,0],"Bytespider":["partial","*",209,0],"Amazonbot":["partial","*",209,0],"meta-externalagent":["partial","*",209,0],"cohere-ai":["partial","*",209,0],"Diffbot":["partial","*",209,0],"ContextFetchBot":["partial","*",209,0]},"llms":{"status":"html","http":200,"bytes":43568},"home":{"status":"ok","http":200,"url":"https://www.gla.ac.uk/","html_tokens":15836,"markdown_tokens":92,"pruned_tokens":92}},
{"rank":1495,"domain":"blogspot.ca","origin":"https://blogspot.ca","robots":{"status":"ok","http":200,"url":"https://blogspot.ca/robots.txt","bytes":6624,"groups":4},"bots":{"GPTBot":["partial","*",169,64],"ChatGPT-User":["partial","*",169,64],"OAI-SearchBot":["partial","*",169,64],"ClaudeBot":["partial","*",169,64],"Claude-User":["partial","*",169,64],"Claude-SearchBot":["partial","*",169,64],"anthropic-ai":["partial","*",169,64],"PerplexityBot":["partial","*",169,64],"Perplexity-User":["partial","*",169,64],"Google-Extended":["partial","*",169,64],"Applebot-Extended":["partial","*",169,64],"CCBot":["partial","*",169,64],"Bytespider":["partial","*",169,64],"Amazonbot":["partial","*",169,64],"meta-externalagent":["partial","*",169,64],"cohere-ai":["partial","*",169,64],"Diffbot":["partial","*",169,64],"ContextFetchBot":["partial","*",169,64]},"llms":{"status":"missing","http":404,"bytes":1569},"home":{"status":"ok","http":200,"url":"https://www.google.com/","html_tokens":21286,"markdown_tokens":30,"pruned_tokens":30}},
{"rank":1496,"domain":"cloudinary.com","origin":"https://cloudinary.com","robots":{"status":"ok","http":200,"url":"https://cloudinary.com/robots.txt","bytes":673,"groups":4},"bots":{"GPTBot":["partial","*",9,0],"ChatGPT-User":["partial","*",9,0],"OAI-SearchBot":["partial","*",9,0],"ClaudeBot":["partial","*",9,0],"Claude-User":["partial","*",9,0],"Claude-SearchBot":["partial","*",9,0],"anthropic-ai":["partial","*",9,0],"PerplexityBot":["partial","*",9,0],"Perplexity-User":["partial","*",9,0],"Google-Extended":["partial","*",9,0],"Applebot-Extended":["partial","*",9,0],"CCBot":["partial","*",9,0],"Bytespider":["partial","*",9,0],"Amazonbot":["partial","*",9,0],"meta-externalagent":["partial","*",9,0],"cohere-ai":["partial","*",9,0],"Diffbot":["partial","*",9,0],"ContextFetchBot":["partial","*",9,0]},"llms":{"status":"present","http":200,"bytes":15371},"home":{"status":"ok","http":200,"url":"https://cloudinary.com/","html_tokens":93570,"markdown_tokens":141,"pruned_tokens":66}},
{"rank":1497,"domain":"genius.com","origin":"https://genius.com","robots":{"status":"ok","http":200,"url":"https://genius.com/robots.txt","bytes":1149,"groups":2},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["blocked","named",1,0],"OAI-SearchBot":["partial","*",18,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",18,0],"Claude-SearchBot":["partial","*",18,0],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["partial","*",18,0],"Perplexity-User":["partial","*",18,0],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["partial","*",18,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",18,0]},"llms":{"status":"missing","http":404,"bytes":0},"home":{"status":"ok","http":200,"url":"https://genius.com/","html_tokens":76446,"markdown_tokens":41,"pruned_tokens":41}},
{"rank":1498,"domain":"igvita.com","origin":"https://igvita.com","robots":{"status":"ok","http":200,"url":"https://www.igvita.com/robots.txt","bytes":108,"groups":1},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"missing","http":404,"bytes":14381},"home":{"status":"ok","http":200,"url":"https://ilya.grigorik.com/","html_tokens":26625,"markdown_tokens":343,"pruned_tokens":150}},
{"rank":1499,"domain":"apachetoday.com","origin":"https://apachetoday.com","robots":{"status":"html","http":200,"url":"https://apachetoday.com/","bytes":61413,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":1500,"domain":"ecfr.gov","origin":"https://ecfr.gov","robots":{"status":"ok","http":200,"url":"https://www.ecfr.gov/robots.txt","bytes":435,"groups":1},"bots":{"GPTBot":["partial","*",9,0],"ChatGPT-User":["partial","*",9,0],"OAI-SearchBot":["partial","*",9,0],"ClaudeBot":["partial","*",9,0],"Claude-User":["partial","*",9,0],"Claude-SearchBot":["partial","*",9,0],"anthropic-ai":["partial","*",9,0],"PerplexityBot":["partial","*",9,0],"Perplexity-User":["partial","*",9,0],"Google-Extended":["partial","*",9,0],"Applebot-Extended":["partial","*",9,0],"CCBot":["partial","*",9,0],"Bytespider":["partial","*",9,0],"Amazonbot":["partial","*",9,0],"meta-externalagent":["partial","*",9,0],"cohere-ai":["partial","*",9,0],"Diffbot":["partial","*",9,0],"ContextFetchBot":["partial","*",9,0]},"llms":{"status":"disallowed","error":"robots.txt of unblock.federalregister.gov disallows /"},"home":{"status":"disallowed","error":"robots.txt of unblock.federalregister.gov disallows /"}},
{"rank":1501,"domain":"usp.br","origin":"https://usp.br","robots":{"status":"server_error","http":500,"url":"https://usp.br/robots.txt","bytes":39108,"groups":0,"error":"HTTP 500"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":1502,"domain":"cbp.gov","origin":"https://cbp.gov","robots":{"status":"refused","http":403,"url":"https://cbp.gov/robots.txt","bytes":371,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":1503,"domain":"chinanews.com","origin":"https://chinanews.com","robots":{"status":"ok","http":200,"url":"https://www.chinanews.com.cn/robots.txt","bytes":858,"groups":1},"bots":{"GPTBot":["partial","*",38,1],"ChatGPT-User":["partial","*",38,1],"OAI-SearchBot":["partial","*",38,1],"ClaudeBot":["partial","*",38,1],"Claude-User":["partial","*",38,1],"Claude-SearchBot":["partial","*",38,1],"anthropic-ai":["partial","*",38,1],"PerplexityBot":["partial","*",38,1],"Perplexity-User":["partial","*",38,1],"Google-Extended":["partial","*",38,1],"Applebot-Extended":["partial","*",38,1],"CCBot":["partial","*",38,1],"Bytespider":["partial","*",38,1],"Amazonbot":["partial","*",38,1],"meta-externalagent":["partial","*",38,1],"cohere-ai":["partial","*",38,1],"Diffbot":["partial","*",38,1],"ContextFetchBot":["partial","*",38,1]},"llms":{"status":"missing","http":404,"bytes":6124},"home":{"status":"ok","http":200,"url":"https://www.chinanews.com.cn/","html_tokens":47568,"markdown_tokens":53,"pruned_tokens":14}},
{"rank":1504,"domain":"nj.gov","origin":"https://nj.gov","robots":{"status":"ok","http":200,"url":"https://nj.gov/robots.txt","bytes":377,"groups":1},"bots":{"GPTBot":["partial","*",8,4],"ChatGPT-User":["partial","*",8,4],"OAI-SearchBot":["partial","*",8,4],"ClaudeBot":["partial","*",8,4],"Claude-User":["partial","*",8,4],"Claude-SearchBot":["partial","*",8,4],"anthropic-ai":["partial","*",8,4],"PerplexityBot":["partial","*",8,4],"Perplexity-User":["partial","*",8,4],"Google-Extended":["partial","*",8,4],"Applebot-Extended":["partial","*",8,4],"CCBot":["partial","*",8,4],"Bytespider":["partial","*",8,4],"Amazonbot":["partial","*",8,4],"meta-externalagent":["partial","*",8,4],"cohere-ai":["partial","*",8,4],"Diffbot":["partial","*",8,4],"ContextFetchBot":["partial","*",8,4]},"llms":{"status":"missing","http":404,"bytes":11444},"home":{"status":"ok","http":200,"url":"https://nj.gov/","html_tokens":12568,"markdown_tokens":39,"pruned_tokens":39}},
{"rank":1505,"domain":"barrons.com","origin":"https://barrons.com","robots":{"status":"ok","http":200,"url":"https://www.barrons.com/robots.txt","bytes":3585,"groups":3},"bots":{"GPTBot":["partial","named",8,1],"ChatGPT-User":["partial","named",8,1],"OAI-SearchBot":["partial","named",8,1],"ClaudeBot":["blocked","*",1,0],"Claude-User":["blocked","*",1,0],"Claude-SearchBot":["blocked","*",1,0],"anthropic-ai":["blocked","*",1,0],"PerplexityBot":["blocked","*",1,0],"Perplexity-User":["blocked","*",1,0],"Google-Extended":["blocked","*",1,0],"Applebot-Extended":["blocked","*",1,0],"CCBot":["blocked","*",1,0],"Bytespider":["blocked","*",1,0],"Amazonbot":["blocked","*",1,0],"meta-externalagent":["blocked","*",1,0],"cohere-ai":["blocked","*",1,0],"Diffbot":["blocked","*",1,0],"ContextFetchBot":["blocked","*",1,0]},"llms":{"status":"disallowed"},"home":{"status":"disallowed"}},
{"rank":1506,"domain":"ms.now","origin":"https://ms.now","robots":{"status":"ok","http":200,"url":"https://www.ms.now/robots.txt","bytes":2305,"groups":50},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["blocked","named",1,0],"OAI-SearchBot":["blocked","named",1,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["blocked","named",1,0],"Claude-SearchBot":["partial","*",7,0],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["partial","*",7,0],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["partial","named",1,1],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",7,0]},"llms":{"status":"other_text","http":200,"bytes":30103},"home":{"status":"ok","http":200,"url":"https://www.ms.now/","html_tokens":124557,"markdown_tokens":85,"pruned_tokens":85}},
{"rank":1507,"domain":"item.taobao.com","origin":"https://item.taobao.com","robots":{"status":"ok","http":200,"url":"https://item.taobao.com/robots.txt","bytes":97,"groups":3},"bots":{"GPTBot":["blocked","*",1,0],"ChatGPT-User":["blocked","*",1,0],"OAI-SearchBot":["blocked","*",1,0],"ClaudeBot":["blocked","*",1,0],"Claude-User":["blocked","*",1,0],"Claude-SearchBot":["blocked","*",1,0],"anthropic-ai":["blocked","*",1,0],"PerplexityBot":["blocked","*",1,0],"Perplexity-User":["blocked","*",1,0],"Google-Extended":["blocked","*",1,0],"Applebot-Extended":["blocked","*",1,0],"CCBot":["blocked","*",1,0],"Bytespider":["blocked","*",1,0],"Amazonbot":["blocked","*",1,0],"meta-externalagent":["blocked","*",1,0],"cohere-ai":["blocked","*",1,0],"Diffbot":["blocked","*",1,0],"ContextFetchBot":["blocked","*",1,0]},"llms":{"status":"disallowed"},"home":{"status":"disallowed"}},
{"rank":1508,"domain":"bankrate.com","origin":"https://bankrate.com","robots":{"status":"ok","http":200,"url":"https://www.bankrate.com/robots.txt","bytes":1277,"groups":3},"bots":{"GPTBot":["partial","*",42,5],"ChatGPT-User":["partial","*",42,5],"OAI-SearchBot":["partial","*",42,5],"ClaudeBot":["partial","*",42,5],"Claude-User":["partial","*",42,5],"Claude-SearchBot":["partial","*",42,5],"anthropic-ai":["partial","*",42,5],"PerplexityBot":["partial","*",42,5],"Perplexity-User":["partial","*",42,5],"Google-Extended":["partial","*",42,5],"Applebot-Extended":["partial","*",42,5],"CCBot":["partial","*",42,5],"Bytespider":["partial","*",42,5],"Amazonbot":["partial","*",42,5],"meta-externalagent":["partial","*",42,5],"cohere-ai":["partial","*",42,5],"Diffbot":["partial","*",42,5],"ContextFetchBot":["partial","*",42,5]},"llms":{"status":"present","http":200,"bytes":10231},"home":{"status":"ok","http":200,"url":"https://www.bankrate.com/","html_tokens":322271,"markdown_tokens":691,"pruned_tokens":162}},
{"rank":1509,"domain":"itv.com","origin":"https://itv.com","robots":{"status":"unreachable","http":null,"url":"https://itv.com/robots.txt","bytes":0,"groups":0,"error":"<StreamReset stream_id:1, error_code:2, remote_reset:True>"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":1510,"domain":"morningstar.com","origin":"https://morningstar.com","robots":{"status":"ok","http":200,"url":"https://www.morningstar.com/robots.txt","bytes":423,"groups":1},"bots":{"GPTBot":["partial","*",12,0],"ChatGPT-User":["partial","*",12,0],"OAI-SearchBot":["partial","*",12,0],"ClaudeBot":["partial","*",12,0],"Claude-User":["partial","*",12,0],"Claude-SearchBot":["partial","*",12,0],"anthropic-ai":["partial","*",12,0],"PerplexityBot":["partial","*",12,0],"Perplexity-User":["partial","*",12,0],"Google-Extended":["partial","*",12,0],"Applebot-Extended":["partial","*",12,0],"CCBot":["partial","*",12,0],"Bytespider":["partial","*",12,0],"Amazonbot":["partial","*",12,0],"meta-externalagent":["partial","*",12,0],"cohere-ai":["partial","*",12,0],"Diffbot":["partial","*",12,0],"ContextFetchBot":["partial","*",12,0]},"llms":{"status":"missing","http":404,"bytes":0},"home":{"status":"http_error","http":403,"url":"https://www.morningstar.com/"}},
{"rank":1511,"domain":"leavenworth-net.com","origin":"https://leavenworth-net.com","robots":{"status":"ok","http":200,"url":"https://leavenworth-net.com/robots.txt","bytes":77,"groups":1},"bots":{"GPTBot":["allowed","*",0,1],"ChatGPT-User":["allowed","*",0,1],"OAI-SearchBot":["allowed","*",0,1],"ClaudeBot":["allowed","*",0,1],"Claude-User":["allowed","*",0,1],"Claude-SearchBot":["allowed","*",0,1],"anthropic-ai":["allowed","*",0,1],"PerplexityBot":["allowed","*",0,1],"Perplexity-User":["allowed","*",0,1],"Google-Extended":["allowed","*",0,1],"Applebot-Extended":["allowed","*",0,1],"CCBot":["allowed","*",0,1],"Bytespider":["allowed","*",0,1],"Amazonbot":["allowed","*",0,1],"meta-externalagent":["allowed","*",0,1],"cohere-ai":["allowed","*",0,1],"Diffbot":["allowed","*",0,1],"ContextFetchBot":["allowed","*",0,1]},"llms":{"status":"missing","http":404,"bytes":513},"home":{"status":"ok","http":200,"url":"https://leavenworth-net.com/","html_tokens":5537,"markdown_tokens":158,"pruned_tokens":158}},
{"rank":1512,"domain":"nerdwallet.com","origin":"https://nerdwallet.com","robots":{"status":"ok","http":200,"url":"https://www.nerdwallet.com/robots.txt","bytes":1720,"groups":3},"bots":{"GPTBot":["partial","*",37,5],"ChatGPT-User":["partial","*",37,5],"OAI-SearchBot":["partial","*",37,5],"ClaudeBot":["partial","*",37,5],"Claude-User":["partial","*",37,5],"Claude-SearchBot":["partial","*",37,5],"anthropic-ai":["partial","*",37,5],"PerplexityBot":["partial","*",37,5],"Perplexity-User":["partial","*",37,5],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["partial","*",37,5],"CCBot":["partial","*",37,5],"Bytespider":["partial","*",37,5],"Amazonbot":["partial","*",37,5],"meta-externalagent":["partial","*",37,5],"cohere-ai":["partial","*",37,5],"Diffbot":["partial","*",37,5],"ContextFetchBot":["partial","*",37,5]},"llms":{"status":"missing","http":404,"bytes":624381},"home":{"status":"ok","http":200,"url":"https://www.nerdwallet.com/","html_tokens":326718,"markdown_tokens":439,"pruned_tokens":57}},
{"rank":1513,"domain":"geeksforgeeks.org","origin":"https://geeksforgeeks.org","robots":{"status":"ok","http":200,"url":"https://www.geeksforgeeks.org/robots.txt","bytes":681,"groups":6},"bots":{"GPTBot":["partial","*",6,0],"ChatGPT-User":["partial","*",6,0],"OAI-SearchBot":["partial","*",6,0],"ClaudeBot":["partial","*",6,0],"Claude-User":["partial","*",6,0],"Claude-SearchBot":["partial","*",6,0],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["partial","*",6,0],"Perplexity-User":["partial","*",6,0],"Google-Extended":["partial","*",6,0],"Applebot-Extended":["partial","*",6,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["partial","*",6,0],"meta-externalagent":["partial","*",6,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["partial","*",6,0],"ContextFetchBot":["partial","*",6,0]},"llms":{"status":"html","http":200,"bytes":88972},"home":{"status":"ok","http":200,"url":"https://www.geeksforgeeks.org/","html_tokens":31139,"markdown_tokens":352,"pruned_tokens":175}},
{"rank":1514,"domain":"tx.us","origin":"https://tx.us","robots":{"status":"unreachable","http":null,"url":"https://tx.us/robots.txt","bytes":0,"groups":0,"error":"cannot resolve tx.us"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":1515,"domain":"mercurynews.com","origin":"https://mercurynews.com","robots":{"status":"ok","http":200,"url":"https://www.mercurynews.com/robots.txt","bytes":4559,"groups":68},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["blocked","named",1,0],"OAI-SearchBot":["blocked","named",1,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["blocked","named",1,0],"Claude-SearchBot":["blocked","named",1,0],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["blocked","named",1,0],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["partial","*",12,1],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",12,1]},"llms":{"status":"missing","http":404,"bytes":90412},"home":{"status":"ok","http":200,"url":"https://www.mercurynews.com/","html_tokens":73016,"markdown_tokens":291,"pruned_tokens":124}},
{"rank":1516,"domain":"bnf.fr","origin":"https://bnf.fr","robots":{"status":"ok","http":200,"url":"https://www.bnf.fr/robots.txt","bytes":1594,"groups":1},"bots":{"GPTBot":["partial","*",22,18],"ChatGPT-User":["partial","*",22,18],"OAI-SearchBot":["partial","*",22,18],"ClaudeBot":["partial","*",22,18],"Claude-User":["partial","*",22,18],"Claude-SearchBot":["partial","*",22,18],"anthropic-ai":["partial","*",22,18],"PerplexityBot":["partial","*",22,18],"Perplexity-User":["partial","*",22,18],"Google-Extended":["partial","*",22,18],"Applebot-Extended":["partial","*",22,18],"CCBot":["partial","*",22,18],"Bytespider":["partial","*",22,18],"Amazonbot":["partial","*",22,18],"meta-externalagent":["partial","*",22,18],"cohere-ai":["partial","*",22,18],"Diffbot":["partial","*",22,18],"ContextFetchBot":["partial","*",22,18]},"llms":{"status":"missing","http":404,"bytes":184},"home":{"status":"ok","http":200,"url":"https://www.bnf.fr/fr","html_tokens":54371,"markdown_tokens":307,"pruned_tokens":74}},
{"rank":1517,"domain":"nrw.de","origin":"http://nrw.de","robots":{"status":"html","http":200,"url":"https://www.land.nrw/","bytes":161590,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":1518,"domain":"lulu.com","origin":"https://lulu.com","robots":{"status":"ok","http":200,"url":"https://www.lulu.com/robots.txt","bytes":1777,"groups":4},"bots":{"GPTBot":["partial","*",14,0],"ChatGPT-User":["partial","*",14,0],"OAI-SearchBot":["partial","*",14,0],"ClaudeBot":["partial","*",14,0],"Claude-User":["partial","*",14,0],"Claude-SearchBot":["partial","*",14,0],"anthropic-ai":["partial","*",14,0],"PerplexityBot":["partial","*",14,0],"Perplexity-User":["partial","*",14,0],"Google-Extended":["partial","*",14,0],"Applebot-Extended":["partial","*",14,0],"CCBot":["partial","*",14,0],"Bytespider":["partial","*",14,0],"Amazonbot":["partial","*",14,0],"meta-externalagent":["partial","*",14,0],"cohere-ai":["partial","*",14,0],"Diffbot":["partial","*",14,0],"ContextFetchBot":["partial","*",14,0]},"llms":{"status":"missing","http":404,"bytes":172941},"home":{"status":"ok","http":200,"url":"https://www.lulu.com/","html_tokens":73788,"markdown_tokens":97,"pruned_tokens":97}},
{"rank":1519,"domain":"vuejs.org","origin":"https://vuejs.org","robots":{"status":"ok","http":200,"url":"https://vuejs.org/robots.txt","bytes":119,"groups":1},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"present","http":200,"bytes":7415},"home":{"status":"ok","http":200,"url":"https://vuejs.org/","html_tokens":20898,"markdown_tokens":119,"pruned_tokens":119}},
{"rank":1520,"domain":"pressreader.com","origin":"https://pressreader.com","robots":{"status":"ok","http":200,"url":"https://www.pressreader.com/robots.txt","bytes":36448,"groups":2},"bots":{"GPTBot":["partial","named",3,1],"ChatGPT-User":["partial","named",3,1],"OAI-SearchBot":["partial","named",3,1],"ClaudeBot":["partial","named",3,1],"Claude-User":["partial","named",3,1],"Claude-SearchBot":["partial","named",3,1],"anthropic-ai":["partial","named",3,1],"PerplexityBot":["partial","named",3,1],"Perplexity-User":["partial","named",3,1],"Google-Extended":["partial","named",3,1],"Applebot-Extended":["partial","named",3,1],"CCBot":["partial","named",3,1],"Bytespider":["allowed","none",0,0],"Amazonbot":["partial","named",3,1],"meta-externalagent":["partial","named",3,1],"cohere-ai":["partial","named",3,1],"Diffbot":["partial","named",3,1],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"http_error","http":403,"bytes":4548},"home":{"status":"http_error","http":403,"url":"https://www.pressreader.com/"}},
{"rank":1521,"domain":"abc.es","origin":"https://abc.es","robots":{"status":"ok","http":200,"url":"https://www.abc.es/robots.txt","bytes":3178,"groups":4},"bots":{"GPTBot":["partial","named",2,3],"ChatGPT-User":["partial","named",2,3],"OAI-SearchBot":["partial","named",2,3],"ClaudeBot":["partial","named",2,3],"Claude-User":["partial","named",2,3],"Claude-SearchBot":["partial","named",2,3],"anthropic-ai":["partial","*",81,0],"PerplexityBot":["partial","named",2,3],"Perplexity-User":["partial","named",2,3],"Google-Extended":["partial","*",81,0],"Applebot-Extended":["partial","named",2,3],"CCBot":["partial","named",2,3],"Bytespider":["partial","named",2,3],"Amazonbot":["partial","named",2,3],"meta-externalagent":["partial","named",2,3],"cohere-ai":["partial","named",2,3],"Diffbot":["partial","*",81,0],"ContextFetchBot":["partial","*",81,0]},"llms":{"status":"missing","http":404,"bytes":245900},"home":{"status":"ok","http":200,"url":"https://www.abc.es/","html_tokens":85736,"markdown_tokens":815,"pruned_tokens":57}},
{"rank":1522,"domain":"patch.com","origin":"https://patch.com","robots":{"status":"ok","http":200,"url":"https://patch.com/robots.txt","bytes":4240,"groups":35},"bots":{"GPTBot":["partial","*",73,32],"ChatGPT-User":["partial","*",73,32],"OAI-SearchBot":["partial","*",73,32],"ClaudeBot":["partial","*",73,32],"Claude-User":["partial","*",73,32],"Claude-SearchBot":["partial","*",73,32],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["partial","*",73,32],"Perplexity-User":["partial","*",73,32],"Google-Extended":["partial","*",73,32],"Applebot-Extended":["partial","*",73,32],"CCBot":["blocked","named",3,0],"Bytespider":["partial","*",73,32],"Amazonbot":["partial","*",73,32],"meta-externalagent":["partial","*",73,32],"cohere-ai":["blocked","named",1,0],"Diffbot":["partial","*",73,32],"ContextFetchBot":["partial","*",73,32]},"llms":{"status":"present","http":200,"bytes":3840},"home":{"status":"ok","http":200,"url":"https://patch.com/","html_tokens":90841,"markdown_tokens":1191,"pruned_tokens":64}},
{"rank":1523,"domain":"cdninstagram.com","origin":"https://cdninstagram.com","robots":{"status":"unreachable","http":null,"url":"https://cdninstagram.com/robots.txt","bytes":0,"groups":0,"error":"cannot resolve cdninstagram.com"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":1524,"domain":"xe.com","origin":"https://xe.com","robots":{"status":"ok","http":200,"url":"https://www.xe.com/robots.txt","bytes":2036,"groups":2},"bots":{"GPTBot":["partial","*",5,0],"ChatGPT-User":["partial","*",5,0],"OAI-SearchBot":["partial","*",5,0],"ClaudeBot":["partial","*",5,0],"Claude-User":["partial","*",5,0],"Claude-SearchBot":["partial","*",5,0],"anthropic-ai":["partial","*",5,0],"PerplexityBot":["partial","*",5,0],"Perplexity-User":["partial","*",5,0],"Google-Extended":["partial","*",5,0],"Applebot-Extended":["partial","*",5,0],"CCBot":["partial","*",5,0],"Bytespider":["partial","*",5,0],"Amazonbot":["partial","*",5,0],"meta-externalagent":["partial","*",5,0],"cohere-ai":["partial","*",5,0],"Diffbot":["partial","*",5,0],"ContextFetchBot":["partial","*",5,0]},"llms":{"status":"present","http":200,"bytes":5162},"home":{"status":"ok","http":200,"url":"https://www.xe.com/","html_tokens":108686,"markdown_tokens":426,"pruned_tokens":106}},
{"rank":1525,"domain":"prtimes.jp","origin":"https://prtimes.jp","robots":{"status":"ok","http":200,"url":"https://prtimes.jp/robots.txt","bytes":254,"groups":1},"bots":{"GPTBot":["partial","*",3,1],"ChatGPT-User":["partial","*",3,1],"OAI-SearchBot":["partial","*",3,1],"ClaudeBot":["partial","*",3,1],"Claude-User":["partial","*",3,1],"Claude-SearchBot":["partial","*",3,1],"anthropic-ai":["partial","*",3,1],"PerplexityBot":["partial","*",3,1],"Perplexity-User":["partial","*",3,1],"Google-Extended":["partial","*",3,1],"Applebot-Extended":["partial","*",3,1],"CCBot":["partial","*",3,1],"Bytespider":["partial","*",3,1],"Amazonbot":["partial","*",3,1],"meta-externalagent":["partial","*",3,1],"cohere-ai":["partial","*",3,1],"Diffbot":["partial","*",3,1],"ContextFetchBot":["partial","*",3,1]},"llms":{"status":"missing","http":404,"bytes":128584},"home":{"status":"ok","http":200,"url":"https://prtimes.jp/","html_tokens":93947,"markdown_tokens":112,"pruned_tokens":112}},
{"rank":1526,"domain":"berlin.de","origin":"https://berlin.de","robots":{"status":"ok","http":200,"url":"https://berlin.de/robots.txt","bytes":24,"groups":1},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"missing","http":410,"bytes":189},"home":{"status":"ok","http":200,"url":"https://www.berlin.de/","html_tokens":9117,"markdown_tokens":69,"pruned_tokens":67}},
{"rank":1527,"domain":"verywellhealth.com","origin":"https://verywellhealth.com","robots":{"status":"ok","http":200,"url":"https://www.verywellhealth.com/robots.txt","bytes":3481,"groups":6},"bots":{"GPTBot":["partial","named",1,0],"ChatGPT-User":["partial","named",1,0],"OAI-SearchBot":["partial","named",1,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["blocked","named",1,0],"Claude-SearchBot":["partial","named",1,2],"anthropic-ai":["partial","named",1,2],"PerplexityBot":["partial","named",1,2],"Perplexity-User":["blocked","named",1,0],"Google-Extended":["partial","named",1,2],"Applebot-Extended":["blocked","named",1,0],"CCBot":["partial","named",1,2],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["partial","named",1,2],"cohere-ai":["partial","named",1,2],"Diffbot":["partial","*",2,0],"ContextFetchBot":["partial","*",2,0]},"llms":{"status":"http_error","http":402,"bytes":612},"home":{"status":"http_error","http":402,"url":"https://www.verywellhealth.com/"}},
{"rank":1528,"domain":"ew.com","origin":"https://ew.com","robots":{"status":"ok","http":200,"url":"https://ew.com/robots.txt","bytes":3449,"groups":6},"bots":{"GPTBot":["partial","named",1,0],"ChatGPT-User":["partial","named",1,0],"OAI-SearchBot":["partial","named",1,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["blocked","named",1,0],"Claude-SearchBot":["partial","named",1,2],"anthropic-ai":["partial","named",1,2],"PerplexityBot":["partial","named",1,2],"Perplexity-User":["blocked","named",1,0],"Google-Extended":["partial","named",1,2],"Applebot-Extended":["blocked","named",1,0],"CCBot":["partial","named",1,2],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["partial","named",1,2],"cohere-ai":["partial","named",1,2],"Diffbot":["partial","*",2,0],"ContextFetchBot":["partial","*",2,0]},"llms":{"status":"http_error","http":402,"bytes":612},"home":{"status":"http_error","http":402,"url":"https://ew.com/"}},
{"rank":1529,"domain":"livemint.com","origin":"https://livemint.com","robots":{"status":"ok","http":200,"url":"https://www.livemint.com/robots.txt","bytes":1875,"groups":6},"bots":{"GPTBot":["partial","*",27,1],"ChatGPT-User":["partial","*",27,1],"OAI-SearchBot":["partial","*",27,1],"ClaudeBot":["partial","*",27,1],"Claude-User":["partial","*",27,1],"Claude-SearchBot":["partial","*",27,1],"anthropic-ai":["partial","*",27,1],"PerplexityBot":["partial","*",27,1],"Perplexity-User":["partial","*",27,1],"Google-Extended":["partial","named",27,1],"Applebot-Extended":["partial","*",27,1],"CCBot":["partial","*",27,1],"Bytespider":["partial","*",27,1],"Amazonbot":["partial","*",27,1],"meta-externalagent":["partial","*",27,1],"cohere-ai":["partial","*",27,1],"Diffbot":["partial","*",27,1],"ContextFetchBot":["partial","*",27,1]},"llms":{"status":"present","http":200,"bytes":11369},"home":{"status":"ok","http":200,"url":"https://www.livemint.com/","html_tokens":100531,"markdown_tokens":754,"pruned_tokens":131}},
{"rank":1530,"domain":"qodeinteractive.com","origin":"https://qodeinteractive.com","robots":{"status":"ok","http":200,"url":"https://qodeinteractive.com/robots.txt","bytes":170,"groups":1},"bots":{"GPTBot":["partial","*",1,1],"ChatGPT-User":["partial","*",1,1],"OAI-SearchBot":["partial","*",1,1],"ClaudeBot":["partial","*",1,1],"Claude-User":["partial","*",1,1],"Claude-SearchBot":["partial","*",1,1],"anthropic-ai":["partial","*",1,1],"PerplexityBot":["partial","*",1,1],"Perplexity-User":["partial","*",1,1],"Google-Extended":["partial","*",1,1],"Applebot-Extended":["partial","*",1,1],"CCBot":["partial","*",1,1],"Bytespider":["partial","*",1,1],"Amazonbot":["partial","*",1,1],"meta-externalagent":["partial","*",1,1],"cohere-ai":["partial","*",1,1],"Diffbot":["partial","*",1,1],"ContextFetchBot":["partial","*",1,1]},"llms":{"status":"missing","http":404,"bytes":707355},"home":{"status":"ok","http":200,"url":"https://qodeinteractive.com/","html_tokens":221185,"markdown_tokens":1431,"pruned_tokens":553}},
{"rank":1531,"domain":"cointelegraph.com","origin":"https://cointelegraph.com","robots":{"status":"ok","http":200,"url":"https://cointelegraph.com/robots.txt","bytes":384,"groups":1},"bots":{"GPTBot":["partial","*",15,0],"ChatGPT-User":["partial","*",15,0],"OAI-SearchBot":["partial","*",15,0],"ClaudeBot":["partial","*",15,0],"Claude-User":["partial","*",15,0],"Claude-SearchBot":["partial","*",15,0],"anthropic-ai":["partial","*",15,0],"PerplexityBot":["partial","*",15,0],"Perplexity-User":["partial","*",15,0],"Google-Extended":["partial","*",15,0],"Applebot-Extended":["partial","*",15,0],"CCBot":["partial","*",15,0],"Bytespider":["partial","*",15,0],"Amazonbot":["partial","*",15,0],"meta-externalagent":["partial","*",15,0],"cohere-ai":["partial","*",15,0],"Diffbot":["partial","*",15,0],"ContextFetchBot":["partial","*",15,0]},"llms":{"status":"missing","http":404,"bytes":17077},"home":{"status":"ok","http":200,"url":"https://cointelegraph.com/","html_tokens":103874,"markdown_tokens":79,"pruned_tokens":79}},
{"rank":1532,"domain":"elastic.co","origin":"https://elastic.co","robots":{"status":"ok","http":200,"url":"https://www.elastic.co/robots.txt","bytes":37560,"groups":1},"bots":{"GPTBot":["partial","*",26,890],"ChatGPT-User":["partial","*",26,890],"OAI-SearchBot":["partial","*",26,890],"ClaudeBot":["partial","*",26,890],"Claude-User":["partial","*",26,890],"Claude-SearchBot":["partial","*",26,890],"anthropic-ai":["partial","*",26,890],"PerplexityBot":["partial","*",26,890],"Perplexity-User":["partial","*",26,890],"Google-Extended":["partial","*",26,890],"Applebot-Extended":["partial","*",26,890],"CCBot":["partial","*",26,890],"Bytespider":["partial","*",26,890],"Amazonbot":["partial","*",26,890],"meta-externalagent":["partial","*",26,890],"cohere-ai":["partial","*",26,890],"Diffbot":["partial","*",26,890],"ContextFetchBot":["partial","*",26,890]},"llms":{"status":"missing","http":404,"bytes":76250},"home":{"status":"ok","http":200,"url":"https://www.elastic.co/","html_tokens":225060,"markdown_tokens":1837,"pruned_tokens":429}},
{"rank":1533,"domain":"tradingeconomics.com","origin":"https://tradingeconomics.com","robots":{"status":"ok","http":200,"url":"https://tradingeconomics.com/robots.txt","bytes":49,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"html","http":200,"bytes":2165},"home":{"status":"ok","http":200,"url":"https://tradingeconomics.com/","html_tokens":87648,"markdown_tokens":270,"pruned_tokens":158}},
{"rank":1534,"domain":"gmu.edu","origin":"https://gmu.edu","robots":{"status":"ok","http":200,"url":"https://www.gmu.edu/robots.txt","bytes":2027,"groups":1},"bots":{"GPTBot":["partial","*",34,18],"ChatGPT-User":["partial","*",34,18],"OAI-SearchBot":["partial","*",34,18],"ClaudeBot":["partial","*",34,18],"Claude-User":["partial","*",34,18],"Claude-SearchBot":["partial","*",34,18],"anthropic-ai":["partial","*",34,18],"PerplexityBot":["partial","*",34,18],"Perplexity-User":["partial","*",34,18],"Google-Extended":["partial","*",34,18],"Applebot-Extended":["partial","*",34,18],"CCBot":["partial","*",34,18],"Bytespider":["partial","*",34,18],"Amazonbot":["partial","*",34,18],"meta-externalagent":["partial","*",34,18],"cohere-ai":["partial","*",34,18],"Diffbot":["partial","*",34,18],"ContextFetchBot":["partial","*",34,18]},"llms":{"status":"missing","http":404,"bytes":185},"home":{"status":"ok","http":200,"url":"https://www.gmu.edu/","html_tokens":27141,"markdown_tokens":661,"pruned_tokens":163}},
{"rank":1535,"domain":"ibtimes.com","origin":"https://ibtimes.com","robots":{"status":"ok","http":200,"url":"https://www.ibtimes.com/robots.txt","bytes":2467,"groups":2},"bots":{"GPTBot":["partial","*",12,0],"ChatGPT-User":["partial","*",12,0],"OAI-SearchBot":["partial","*",12,0],"ClaudeBot":["partial","*",12,0],"Claude-User":["partial","*",12,0],"Claude-SearchBot":["partial","*",12,0],"anthropic-ai":["partial","*",12,0],"PerplexityBot":["partial","*",12,0],"Perplexity-User":["partial","*",12,0],"Google-Extended":["partial","*",12,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["partial","*",12,0],"ContextFetchBot":["partial","*",12,0]},"llms":{"status":"missing","http":404,"bytes":119474},"home":{"status":"ok","http":200,"url":"https://www.ibtimes.com/","html_tokens":47361,"markdown_tokens":607,"pruned_tokens":155}},
{"rank":1536,"domain":"altibox.no","origin":"https://altibox.no","robots":{"status":"ok","http":200,"url":"https://www.altibox.no/robots.txt","bytes":107,"groups":1},"bots":{"GPTBot":["partial","*",5,0],"ChatGPT-User":["partial","*",5,0],"OAI-SearchBot":["partial","*",5,0],"ClaudeBot":["partial","*",5,0],"Claude-User":["partial","*",5,0],"Claude-SearchBot":["partial","*",5,0],"anthropic-ai":["partial","*",5,0],"PerplexityBot":["partial","*",5,0],"Perplexity-User":["partial","*",5,0],"Google-Extended":["partial","*",5,0],"Applebot-Extended":["partial","*",5,0],"CCBot":["partial","*",5,0],"Bytespider":["partial","*",5,0],"Amazonbot":["partial","*",5,0],"meta-externalagent":["partial","*",5,0],"cohere-ai":["partial","*",5,0],"Diffbot":["partial","*",5,0],"ContextFetchBot":["partial","*",5,0]},"llms":{"status":"missing","http":404,"bytes":30634},"home":{"status":"ok","http":200,"url":"https://www.altibox.no/","html_tokens":19907,"markdown_tokens":199,"pruned_tokens":142}},
{"rank":1537,"domain":"workers.dev","origin":"https://workers.dev","robots":{"status":"html","http":200,"url":"https://www.cloudflare.com/products/workers/","bytes":757728,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":1538,"domain":"yougov.com","origin":"https://yougov.com","robots":{"status":"ok","http":200,"url":"https://yougov.com/robots.txt","bytes":2499,"groups":2},"bots":{"GPTBot":["partial","*",22,1],"ChatGPT-User":["partial","*",22,1],"OAI-SearchBot":["partial","*",22,1],"ClaudeBot":["partial","*",22,1],"Claude-User":["partial","*",22,1],"Claude-SearchBot":["partial","*",22,1],"anthropic-ai":["partial","*",22,1],"PerplexityBot":["partial","*",22,1],"Perplexity-User":["partial","*",22,1],"Google-Extended":["partial","*",22,1],"Applebot-Extended":["partial","*",22,1],"CCBot":["partial","*",22,1],"Bytespider":["partial","*",22,1],"Amazonbot":["partial","*",22,1],"meta-externalagent":["partial","*",22,1],"cohere-ai":["partial","*",22,1],"Diffbot":["partial","*",22,1],"ContextFetchBot":["partial","*",22,1]},"llms":{"status":"missing","http":404,"bytes":403151},"home":{"status":"ok","http":200,"url":"https://yougov.com/","html_tokens":185161,"markdown_tokens":45,"pruned_tokens":45}},
{"rank":1539,"domain":"michigan.gov","origin":"https://michigan.gov","robots":{"status":"refused","http":403,"url":"https://michigan.gov/robots.txt","bytes":374,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":1540,"domain":"boingboing.net","origin":"https://boingboing.net","robots":{"status":"ok","http":200,"url":"https://boingboing.net/robots.txt","bytes":171,"groups":1},"bots":{"GPTBot":["partial","*",5,0],"ChatGPT-User":["partial","*",5,0],"OAI-SearchBot":["partial","*",5,0],"ClaudeBot":["partial","*",5,0],"Claude-User":["partial","*",5,0],"Claude-SearchBot":["partial","*",5,0],"anthropic-ai":["partial","*",5,0],"PerplexityBot":["partial","*",5,0],"Perplexity-User":["partial","*",5,0],"Google-Extended":["partial","*",5,0],"Applebot-Extended":["partial","*",5,0],"CCBot":["partial","*",5,0],"Bytespider":["partial","*",5,0],"Amazonbot":["partial","*",5,0],"meta-externalagent":["partial","*",5,0],"cohere-ai":["partial","*",5,0],"Diffbot":["partial","*",5,0],"ContextFetchBot":["partial","*",5,0]},"llms":{"status":"present","http":200,"bytes":925},"home":{"status":"ok","http":200,"url":"https://boingboing.net/","html_tokens":26290,"markdown_tokens":317,"pruned_tokens":62}},
{"rank":1541,"domain":"threebit.net","origin":"https://threebit.net","robots":{"status":"html","http":200,"url":"https://leavenworth-net.com/","bytes":26703,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":1542,"domain":"br.com","origin":"https://br.com","robots":{"status":"ok","http":200,"url":"https://br.com/robots.txt","bytes":0,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":7899},"home":{"status":"ok","http":200,"url":"https://br.com/","html_tokens":10293,"markdown_tokens":972,"pruned_tokens":958}},
{"rank":1543,"domain":"tinypic.com","origin":"https://tinypic.com","robots":{"status":"refused","http":403,"url":"https://tinypic.com/robots.txt","bytes":40,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":1544,"domain":"unixtools.org","origin":"https://unixtools.org","robots":{"status":"ok","http":200,"url":"https://www.unixtools.org/robots.txt","bytes":86,"groups":1},"bots":{"GPTBot":["partial","*",1,0],"ChatGPT-User":["partial","*",1,0],"OAI-SearchBot":["partial","*",1,0],"ClaudeBot":["partial","*",1,0],"Claude-User":["partial","*",1,0],"Claude-SearchBot":["partial","*",1,0],"anthropic-ai":["partial","*",1,0],"PerplexityBot":["partial","*",1,0],"Perplexity-User":["partial","*",1,0],"Google-Extended":["partial","*",1,0],"Applebot-Extended":["partial","*",1,0],"CCBot":["partial","*",1,0],"Bytespider":["partial","*",1,0],"Amazonbot":["partial","*",1,0],"meta-externalagent":["partial","*",1,0],"cohere-ai":["partial","*",1,0],"Diffbot":["partial","*",1,0],"ContextFetchBot":["partial","*",1,0]},"llms":{"status":"http_error","http":403,"bytes":111},"home":{"status":"ok","http":200,"url":"https://www.unixtools.org/","html_tokens":4924,"markdown_tokens":661,"pruned_tokens":135}},
{"rank":1545,"domain":"windy.com","origin":"https://windy.com","robots":{"status":"ok","http":200,"url":"https://www.windy.com/robots.txt","bytes":0,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"html","http":200,"bytes":8011},"home":{"status":"ok","http":200,"url":"https://www.windy.com/","html_tokens":2003,"markdown_tokens":7,"pruned_tokens":7}},
{"rank":1546,"domain":"tagesspiegel.de","origin":"https://tagesspiegel.de","robots":{"status":"ok","http":200,"url":"https://www.tagesspiegel.de/robots.txt","bytes":3790,"groups":20},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["partial","*",2,0],"OAI-SearchBot":["partial","*",2,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",2,0],"Claude-SearchBot":["partial","*",2,0],"anthropic-ai":["partial","*",2,0],"PerplexityBot":["partial","*",2,0],"Perplexity-User":["partial","*",2,0],"Google-Extended":["partial","named",2,2],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["partial","*",2,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["partial","*",2,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",2,0]},"llms":{"status":"missing","http":404,"bytes":353954},"home":{"status":"ok","http":200,"url":"https://www.tagesspiegel.de/","html_tokens":486857,"markdown_tokens":5151,"pruned_tokens":93}},
{"rank":1547,"domain":"luma.com","origin":"https://luma.com","robots":{"status":"ok","http":200,"url":"https://luma.com/robots.txt","bytes":150,"groups":1},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":27821},"home":{"status":"ok","http":200,"url":"https://luma.com/","html_tokens":11280,"markdown_tokens":91,"pruned_tokens":91}},
{"rank":1548,"domain":"sleepfoundation.org","origin":"https://sleepfoundation.org","robots":{"status":"ok","http":200,"url":"https://www.sleepfoundation.org/robots.txt","bytes":219,"groups":1},"bots":{"GPTBot":["partial","*",4,2],"ChatGPT-User":["partial","*",4,2],"OAI-SearchBot":["partial","*",4,2],"ClaudeBot":["partial","*",4,2],"Claude-User":["partial","*",4,2],"Claude-SearchBot":["partial","*",4,2],"anthropic-ai":["partial","*",4,2],"PerplexityBot":["partial","*",4,2],"Perplexity-User":["partial","*",4,2],"Google-Extended":["partial","*",4,2],"Applebot-Extended":["partial","*",4,2],"CCBot":["partial","*",4,2],"Bytespider":["partial","*",4,2],"Amazonbot":["partial","*",4,2],"meta-externalagent":["partial","*",4,2],"cohere-ai":["partial","*",4,2],"Diffbot":["partial","*",4,2],"ContextFetchBot":["partial","*",4,2]},"llms":{"status":"missing","http":404,"bytes":214317},"home":{"status":"ok","http":200,"url":"https://www.sleepfoundation.org/","html_tokens":72135,"markdown_tokens":304,"pruned_tokens":65}},
{"rank":1549,"domain":"midjourney.com","origin":"https://midjourney.com","robots":{"status":"refused","http":403,"url":"https://midjourney.com/robots.txt","bytes":5552,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":1550,"domain":"buffalo.edu","origin":"https://buffalo.edu","robots":{"status":"ok","http":200,"url":"https://www.buffalo.edu/robots.txt","bytes":743,"groups":5},"bots":{"GPTBot":["partial","*",14,0],"ChatGPT-User":["partial","*",14,0],"OAI-SearchBot":["partial","*",14,0],"ClaudeBot":["partial","*",14,0],"Claude-User":["partial","*",14,0],"Claude-SearchBot":["partial","*",14,0],"anthropic-ai":["partial","*",14,0],"PerplexityBot":["partial","*",14,0],"Perplexity-User":["partial","*",14,0],"Google-Extended":["partial","*",14,0],"Applebot-Extended":["partial","*",14,0],"CCBot":["partial","*",14,0],"Bytespider":["partial","*",14,0],"Amazonbot":["partial","*",14,0],"meta-externalagent":["partial","*",14,0],"cohere-ai":["partial","*",14,0],"Diffbot":["partial","*",14,0],"ContextFetchBot":["partial","*",14,0]},"llms":{"status":"missing","http":404,"bytes":7347},"home":{"status":"ok","http":200,"url":"https://www.buffalo.edu/","html_tokens":70987,"markdown_tokens":105,"pruned_tokens":105}},
{"rank":1551,"domain":"helsinki.fi","origin":"https://helsinki.fi","robots":{"status":"ok","http":200,"url":"https://helsinki.fi/robots.txt","bytes":72,"groups":1},"bots":{"GPTBot":["blocked","*",1,0],"ChatGPT-User":["blocked","*",1,0],"OAI-SearchBot":["blocked","*",1,0],"ClaudeBot":["blocked","*",1,0],"Claude-User":["blocked","*",1,0],"Claude-SearchBot":["blocked","*",1,0],"anthropic-ai":["blocked","*",1,0],"PerplexityBot":["blocked","*",1,0],"Perplexity-User":["blocked","*",1,0],"Google-Extended":["blocked","*",1,0],"Applebot-Extended":["blocked","*",1,0],"CCBot":["blocked","*",1,0],"Bytespider":["blocked","*",1,0],"Amazonbot":["blocked","*",1,0],"meta-externalagent":["blocked","*",1,0],"cohere-ai":["blocked","*",1,0],"Diffbot":["blocked","*",1,0],"ContextFetchBot":["blocked","*",1,0]},"llms":{"status":"disallowed"},"home":{"status":"disallowed"}},
{"rank":1552,"domain":"degruyter.com","origin":"https://degruyter.com","robots":{"status":"ok","http":200,"url":"https://www.degruyterbrill.com/robots.txt","bytes":777,"groups":6},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["partial","*",15,0],"OAI-SearchBot":["partial","*",15,0],"ClaudeBot":["partial","*",15,0],"Claude-User":["partial","*",15,0],"Claude-SearchBot":["partial","*",15,0],"anthropic-ai":["partial","*",15,0],"PerplexityBot":["partial","*",15,0],"Perplexity-User":["partial","*",15,0],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["partial","*",15,0],"CCBot":["blocked","named",1,0],"Bytespider":["partial","*",15,0],"Amazonbot":["partial","*",15,0],"meta-externalagent":["partial","*",15,0],"cohere-ai":["partial","*",15,0],"Diffbot":["partial","*",15,0],"ContextFetchBot":["partial","*",15,0]},"llms":{"status":"missing","http":404,"bytes":76435},"home":{"status":"ok","http":200,"url":"https://www.degruyterbrill.com/","html_tokens":29390,"markdown_tokens":289,"pruned_tokens":109}},
{"rank":1553,"domain":"dp.ua","origin":"https://www.dp.ua","robots":{"status":"ok","http":200,"url":"https://nic.dp.ua/robots.txt","bytes":71,"groups":1},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"missing","http":404,"bytes":236},"home":{"status":"ok","http":200,"url":"https://nic.dp.ua/index.html","html_tokens":1468,"markdown_tokens":502,"pruned_tokens":416}},
{"rank":1554,"domain":"uq.edu.au","origin":"https://uq.edu.au","robots":{"status":"ok","http":200,"url":"https://uq.edu.au/robots.txt","bytes":2155,"groups":1},"bots":{"GPTBot":["partial","*",34,18],"ChatGPT-User":["partial","*",34,18],"OAI-SearchBot":["partial","*",34,18],"ClaudeBot":["partial","*",34,18],"Claude-User":["partial","*",34,18],"Claude-SearchBot":["partial","*",34,18],"anthropic-ai":["partial","*",34,18],"PerplexityBot":["partial","*",34,18],"Perplexity-User":["partial","*",34,18],"Google-Extended":["partial","*",34,18],"Applebot-Extended":["partial","*",34,18],"CCBot":["partial","*",34,18],"Bytespider":["partial","*",34,18],"Amazonbot":["partial","*",34,18],"meta-externalagent":["partial","*",34,18],"cohere-ai":["partial","*",34,18],"Diffbot":["partial","*",34,18],"ContextFetchBot":["partial","*",34,18]},"llms":{"status":"missing","http":404,"bytes":78426},"home":{"status":"ok","http":200,"url":"https://uq.edu.au/","html_tokens":23969,"markdown_tokens":161,"pruned_tokens":161}},
{"rank":1555,"domain":"kubernetes.io","origin":"https://kubernetes.io","robots":{"status":"ok","http":200,"url":"https://kubernetes.io/robots.txt","bytes":148,"groups":1},"bots":{"GPTBot":["partial","*",5,0],"ChatGPT-User":["partial","*",5,0],"OAI-SearchBot":["partial","*",5,0],"ClaudeBot":["partial","*",5,0],"Claude-User":["partial","*",5,0],"Claude-SearchBot":["partial","*",5,0],"anthropic-ai":["partial","*",5,0],"PerplexityBot":["partial","*",5,0],"Perplexity-User":["partial","*",5,0],"Google-Extended":["partial","*",5,0],"Applebot-Extended":["partial","*",5,0],"CCBot":["partial","*",5,0],"Bytespider":["partial","*",5,0],"Amazonbot":["partial","*",5,0],"meta-externalagent":["partial","*",5,0],"cohere-ai":["partial","*",5,0],"Diffbot":["partial","*",5,0],"ContextFetchBot":["partial","*",5,0]},"llms":{"status":"missing","http":404,"bytes":32890},"home":{"status":"ok","http":200,"url":"https://kubernetes.io/","html_tokens":10383,"markdown_tokens":429,"pruned_tokens":299}},
{"rank":1556,"domain":"stlouisfed.org","origin":"https://stlouisfed.org","robots":{"status":"ok","http":200,"url":"https://www.stlouisfed.org/robots.txt","bytes":386,"groups":5},"bots":{"GPTBot":["allowed","named",0,1],"ChatGPT-User":["partial","*",3,0],"OAI-SearchBot":["partial","*",3,0],"ClaudeBot":["allowed","named",0,1],"Claude-User":["partial","*",3,0],"Claude-SearchBot":["partial","*",3,0],"anthropic-ai":["partial","*",3,0],"PerplexityBot":["allowed","named",0,1],"Perplexity-User":["partial","*",3,0],"Google-Extended":["allowed","named",0,1],"Applebot-Extended":["partial","*",3,0],"CCBot":["partial","*",3,0],"Bytespider":["partial","*",3,0],"Amazonbot":["partial","*",3,0],"meta-externalagent":["partial","*",3,0],"cohere-ai":["partial","*",3,0],"Diffbot":["partial","*",3,0],"ContextFetchBot":["partial","*",3,0]},"llms":{"status":"present","http":200,"bytes":4315},"home":{"status":"ok","http":200,"url":"https://www.stlouisfed.org/","html_tokens":278473,"markdown_tokens":65,"pruned_tokens":65}},
{"rank":1557,"domain":"nghttp2.org","origin":"https://nghttp2.org","robots":{"status":"ok","http":200,"url":"https://nghttp2.org/robots.txt","bytes":62,"groups":1},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"missing","http":404,"bytes":9},"home":{"status":"ok","http":200,"url":"https://nghttp2.org/","html_tokens":1628,"markdown_tokens":192,"pruned_tokens":148}},
{"rank":1558,"domain":"cfr.org","origin":"https://cfr.org","robots":{"status":"ok","http":200,"url":"https://www.cfr.org/robots.txt","bytes":2675,"groups":1},"bots":{"GPTBot":["partial","*",53,18],"ChatGPT-User":["partial","*",53,18],"OAI-SearchBot":["partial","*",53,18],"ClaudeBot":["partial","*",53,18],"Claude-User":["partial","*",53,18],"Claude-SearchBot":["partial","*",53,18],"anthropic-ai":["partial","*",53,18],"PerplexityBot":["partial","*",53,18],"Perplexity-User":["partial","*",53,18],"Google-Extended":["partial","*",53,18],"Applebot-Extended":["partial","*",53,18],"CCBot":["partial","*",53,18],"Bytespider":["partial","*",53,18],"Amazonbot":["partial","*",53,18],"meta-externalagent":["partial","*",53,18],"cohere-ai":["partial","*",53,18],"Diffbot":["partial","*",53,18],"ContextFetchBot":["partial","*",53,18]},"llms":{"status":"missing","http":404,"bytes":244519},"home":{"status":"ok","http":200,"url":"https://www.cfr.org/","html_tokens":233075,"markdown_tokens":272,"pruned_tokens":144}},
{"rank":1559,"domain":"nfl.com","origin":"https://nfl.com","robots":{"status":"ok","http":200,"url":"https://www.nfl.com/robots.txt","bytes":343,"groups":2},"bots":{"GPTBot":["partial","*",10,0],"ChatGPT-User":["partial","*",10,0],"OAI-SearchBot":["partial","*",10,0],"ClaudeBot":["partial","*",10,0],"Claude-User":["partial","*",10,0],"Claude-SearchBot":["partial","*",10,0],"anthropic-ai":["partial","*",10,0],"PerplexityBot":["partial","*",10,0],"Perplexity-User":["partial","*",10,0],"Google-Extended":["partial","*",10,0],"Applebot-Extended":["partial","*",10,0],"CCBot":["partial","*",10,0],"Bytespider":["partial","*",10,0],"Amazonbot":["partial","*",10,0],"meta-externalagent":["partial","*",10,0],"cohere-ai":["partial","*",10,0],"Diffbot":["partial","*",10,0],"ContextFetchBot":["partial","*",10,0]},"llms":{"status":"missing","http":404,"bytes":205300},"home":{"status":"ok","http":200,"url":"https://www.nfl.com/","html_tokens":954836,"markdown_tokens":21,"pruned_tokens":21}},
{"rank":1560,"domain":"unwomen.org","origin":"https://unwomen.org","robots":{"status":"refused","http":403,"url":"https://unwomen.org/robots.txt","bytes":560,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":1561,"domain":"aa.com.tr","origin":"https://aa.com.tr","robots":{"status":"ok","http":200,"url":"https://aa.com.tr/robots.txt","bytes":706,"groups":1},"bots":{"GPTBot":["partial","*",3,1],"ChatGPT-User":["partial","*",3,1],"OAI-SearchBot":["partial","*",3,1],"ClaudeBot":["partial","*",3,1],"Claude-User":["partial","*",3,1],"Claude-SearchBot":["partial","*",3,1],"anthropic-ai":["partial","*",3,1],"PerplexityBot":["partial","*",3,1],"Perplexity-User":["partial","*",3,1],"Google-Extended":["partial","*",3,1],"Applebot-Extended":["partial","*",3,1],"CCBot":["partial","*",3,1],"Bytespider":["partial","*",3,1],"Amazonbot":["partial","*",3,1],"meta-externalagent":["partial","*",3,1],"cohere-ai":["partial","*",3,1],"Diffbot":["partial","*",3,1],"ContextFetchBot":["partial","*",3,1]},"llms":{"status":"html","http":200,"bytes":56373},"home":{"status":"ok","http":200,"url":"https://aa.com.tr/","html_tokens":8429,"markdown_tokens":0,"pruned_tokens":0}},
{"rank":1562,"domain":"tass.ru","origin":"https://tass.ru","robots":{"status":"refused","http":403,"url":"https://tass.ru/robots.txt","bytes":1293,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":1563,"domain":"whiterabbitpress.com","origin":"https://whiterabbitpress.com","robots":{"status":"ok","http":200,"url":"https://whiterabbitpress.com/robots.txt","bytes":64,"groups":1},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"missing","http":404,"bytes":1583},"home":{"status":"ok","http":200,"url":"https://whiterabbitpress.com/","html_tokens":11642,"markdown_tokens":618,"pruned_tokens":124}},
{"rank":1564,"domain":"shrm.org","origin":"https://shrm.org","robots":{"status":"ok","http":200,"url":"https://www.shrm.org/robots.txt","bytes":938,"groups":3},"bots":{"GPTBot":["partial","*",15,0],"ChatGPT-User":["partial","*",15,0],"OAI-SearchBot":["partial","*",15,0],"ClaudeBot":["partial","*",15,0],"Claude-User":["partial","*",15,0],"Claude-SearchBot":["partial","*",15,0],"anthropic-ai":["partial","*",15,0],"PerplexityBot":["partial","*",15,0],"Perplexity-User":["partial","*",15,0],"Google-Extended":["partial","*",15,0],"Applebot-Extended":["partial","*",15,0],"CCBot":["partial","*",15,0],"Bytespider":["partial","*",15,0],"Amazonbot":["partial","*",15,0],"meta-externalagent":["partial","*",15,0],"cohere-ai":["partial","*",15,0],"Diffbot":["partial","*",15,0],"ContextFetchBot":["partial","*",15,0]},"llms":{"status":"missing","http":404,"bytes":476992},"home":{"status":"ok","http":200,"url":"https://www.shrm.org/","html_tokens":118564,"markdown_tokens":349,"pruned_tokens":332}},
{"rank":1565,"domain":"nationalpost.com","origin":"https://nationalpost.com","robots":{"status":"ok","http":200,"url":"https://nationalpost.com/robots.txt","bytes":1227,"groups":15},"bots":{"GPTBot":["partial","*",3,6],"ChatGPT-User":["partial","*",3,6],"OAI-SearchBot":["partial","*",3,6],"ClaudeBot":["partial","*",3,6],"Claude-User":["partial","*",3,6],"Claude-SearchBot":["partial","*",3,6],"anthropic-ai":["partial","*",3,6],"PerplexityBot":["partial","*",3,6],"Perplexity-User":["partial","*",3,6],"Google-Extended":["partial","*",3,6],"Applebot-Extended":["partial","*",3,6],"CCBot":["partial","*",3,6],"Bytespider":["partial","*",3,6],"Amazonbot":["partial","*",3,6],"meta-externalagent":["partial","*",3,6],"cohere-ai":["partial","*",3,6],"Diffbot":["partial","*",3,6],"ContextFetchBot":["partial","*",3,6]},"llms":{"status":"missing","http":404,"bytes":304764},"home":{"status":"ok","http":200,"url":"https://nationalpost.com/","html_tokens":150221,"markdown_tokens":165,"pruned_tokens":165}},
{"rank":1566,"domain":"minecraft.net","origin":"https://minecraft.net","robots":{"status":"unreachable","http":null,"url":"https://minecraft.net/robots.txt","bytes":0,"groups":0,"error":"<StreamReset stream_id:1, error_code:2, remote_reset:True>"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":1567,"domain":"alltrails.com","origin":"https://alltrails.com","robots":{"status":"ok","http":200,"url":"https://www.alltrails.com/robots.txt","bytes":5408,"groups":4},"bots":{"GPTBot":["partial","named",1,78],"ChatGPT-User":["partial","named",1,78],"OAI-SearchBot":["partial","named",1,78],"ClaudeBot":["partial","named",1,78],"Claude-User":["partial","named",1,78],"Claude-SearchBot":["partial","named",1,78],"anthropic-ai":["partial","*",25,1],"PerplexityBot":["partial","named",1,78],"Perplexity-User":["partial","named",1,78],"Google-Extended":["partial","named",1,78],"Applebot-Extended":["partial","named",1,78],"CCBot":["partial","named",1,78],"Bytespider":["partial","*",25,1],"Amazonbot":["partial","*",25,1],"meta-externalagent":["partial","named",1,78],"cohere-ai":["partial","*",25,1],"Diffbot":["partial","*",25,1],"ContextFetchBot":["partial","*",25,1]},"llms":{"status":"http_error","http":403,"bytes":773},"home":{"status":"ok","http":200,"url":"https://www.alltrails.com/","html_tokens":138265,"markdown_tokens":32,"pruned_tokens":32}},
{"rank":1568,"domain":"ticketmaster.com","origin":"https://ticketmaster.com","robots":{"status":"ok","http":200,"url":"https://www.ticketmaster.com/robots.txt","bytes":2546,"groups":4},"bots":{"GPTBot":["partial","*",70,0],"ChatGPT-User":["partial","*",70,0],"OAI-SearchBot":["partial","*",70,0],"ClaudeBot":["partial","*",70,0],"Claude-User":["partial","*",70,0],"Claude-SearchBot":["partial","*",70,0],"anthropic-ai":["partial","*",70,0],"PerplexityBot":["partial","*",70,0],"Perplexity-User":["partial","*",70,0],"Google-Extended":["partial","*",70,0],"Applebot-Extended":["partial","*",70,0],"CCBot":["partial","*",70,0],"Bytespider":["partial","*",70,0],"Amazonbot":["partial","*",70,0],"meta-externalagent":["partial","*",70,0],"cohere-ai":["partial","*",70,0],"Diffbot":["partial","*",70,0],"ContextFetchBot":["partial","*",70,0]},"llms":{"status":"missing","http":404,"bytes":168720},"home":{"status":"ok","http":200,"url":"https://www.ticketmaster.com/","html_tokens":140863,"markdown_tokens":35,"pruned_tokens":35}},
{"rank":1569,"domain":"redbubble.com","origin":"https://redbubble.com","robots":{"status":"refused","http":403,"url":"https://redbubble.com/robots.txt","bytes":5530,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":1570,"domain":"gao.gov","origin":"https://gao.gov","robots":{"status":"ok","http":200,"url":"https://www.gao.gov/robots.txt","bytes":2806,"groups":3},"bots":{"GPTBot":["partial","*",48,18],"ChatGPT-User":["partial","*",48,18],"OAI-SearchBot":["partial","*",48,18],"ClaudeBot":["partial","*",48,18],"Claude-User":["partial","*",48,18],"Claude-SearchBot":["partial","*",48,18],"anthropic-ai":["partial","*",48,18],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["partial","*",48,18],"Google-Extended":["partial","*",48,18],"Applebot-Extended":["partial","*",48,18],"CCBot":["partial","*",48,18],"Bytespider":["blocked","named",1,0],"Amazonbot":["partial","*",48,18],"meta-externalagent":["partial","*",48,18],"cohere-ai":["partial","*",48,18],"Diffbot":["partial","*",48,18],"ContextFetchBot":["partial","*",48,18]},"llms":{"status":"crawl_delay","error":"Crawl-delay 420 s"},"home":{"status":"crawl_delay","error":"Crawl-delay 420 s"}},
{"rank":1571,"domain":"hu.net","origin":"https://hu.net","robots":{"status":"ok","http":200,"url":"https://hu.net/robots.txt","bytes":0,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":7737},"home":{"status":"ok","http":200,"url":"https://hu.net/","html_tokens":9034,"markdown_tokens":870,"pruned_tokens":857}},
{"rank":1572,"domain":"reference.com","origin":"https://reference.com","robots":{"status":"missing","http":404,"url":"https://www.reference.com/robots.txt","bytes":2353,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":2353},"home":{"status":"ok","http":200,"url":"https://www.reference.com/","html_tokens":837,"markdown_tokens":159,"pruned_tokens":64}},
{"rank":1573,"domain":"haproxy.org","origin":"https://haproxy.org","robots":{"status":"ok","http":200,"url":"https://www.haproxy.org/robots.txt","bytes":49,"groups":1},"bots":{"GPTBot":["partial","*",2,0],"ChatGPT-User":["partial","*",2,0],"OAI-SearchBot":["partial","*",2,0],"ClaudeBot":["partial","*",2,0],"Claude-User":["partial","*",2,0],"Claude-SearchBot":["partial","*",2,0],"anthropic-ai":["partial","*",2,0],"PerplexityBot":["partial","*",2,0],"Perplexity-User":["partial","*",2,0],"Google-Extended":["partial","*",2,0],"Applebot-Extended":["partial","*",2,0],"CCBot":["partial","*",2,0],"Bytespider":["partial","*",2,0],"Amazonbot":["partial","*",2,0],"meta-externalagent":["partial","*",2,0],"cohere-ai":["partial","*",2,0],"Diffbot":["partial","*",2,0],"ContextFetchBot":["partial","*",2,0]},"llms":{"status":"missing","http":404,"bytes":206},"home":{"status":"ok","http":200,"url":"https://www.haproxy.org/","html_tokens":21331,"markdown_tokens":2778,"pruned_tokens":649}},
{"rank":1574,"domain":"si.com","origin":"https://si.com","robots":{"status":"ok","http":200,"url":"https://www.si.com/robots.txt","bytes":1423,"groups":14},"bots":{"GPTBot":["partial","*",23,2],"ChatGPT-User":["partial","*",23,2],"OAI-SearchBot":["partial","*",23,2],"ClaudeBot":["partial","*",23,2],"Claude-User":["partial","*",23,2],"Claude-SearchBot":["partial","*",23,2],"anthropic-ai":["partial","*",23,2],"PerplexityBot":["partial","*",23,2],"Perplexity-User":["partial","*",23,2],"Google-Extended":["partial","*",23,2],"Applebot-Extended":["partial","*",23,2],"CCBot":["partial","*",23,2],"Bytespider":["partial","*",23,2],"Amazonbot":["partial","*",23,2],"meta-externalagent":["partial","*",23,2],"cohere-ai":["partial","*",23,2],"Diffbot":["partial","*",23,2],"ContextFetchBot":["partial","*",23,2]},"llms":{"status":"present","http":200,"bytes":12169},"home":{"status":"ok","http":200,"url":"https://www.si.com/","html_tokens":527760,"markdown_tokens":738,"pruned_tokens":116}},
{"rank":1575,"domain":"staticflickr.com","origin":"https://staticflickr.com","robots":{"status":"unreachable","http":null,"url":"https://staticflickr.com/robots.txt","bytes":0,"groups":0,"error":"cannot resolve staticflickr.com"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":1576,"domain":"outlookindia.com","origin":"https://outlookindia.com","robots":{"status":"ok","http":200,"url":"https://www.outlookindia.com/robots.txt","bytes":2673,"groups":11},"bots":{"GPTBot":["allowed","named",0,1],"ChatGPT-User":["allowed","named",0,1],"OAI-SearchBot":["partial","*",64,1],"ClaudeBot":["partial","*",64,1],"Claude-User":["partial","*",64,1],"Claude-SearchBot":["partial","*",64,1],"anthropic-ai":["partial","*",64,1],"PerplexityBot":["allowed","named",0,1],"Perplexity-User":["partial","*",64,1],"Google-Extended":["allowed","named",0,1],"Applebot-Extended":["partial","*",64,1],"CCBot":["partial","*",64,1],"Bytespider":["partial","*",64,1],"Amazonbot":["partial","*",64,1],"meta-externalagent":["allowed","named",0,1],"cohere-ai":["partial","*",64,1],"Diffbot":["partial","*",64,1],"ContextFetchBot":["partial","*",64,1]},"llms":{"status":"present","http":200,"bytes":4359},"home":{"status":"ok","http":200,"url":"https://www.outlookindia.com/","html_tokens":60545,"markdown_tokens":844,"pruned_tokens":158}},
{"rank":1577,"domain":"9to5mac.com","origin":"https://9to5mac.com","robots":{"status":"ok","http":200,"url":"https://9to5mac.com/robots.txt","bytes":911,"groups":4},"bots":{"GPTBot":["partial","*",4,0],"ChatGPT-User":["partial","*",4,0],"OAI-SearchBot":["partial","*",4,0],"ClaudeBot":["partial","*",4,0],"Claude-User":["partial","*",4,0],"Claude-SearchBot":["partial","*",4,0],"anthropic-ai":["partial","*",4,0],"PerplexityBot":["partial","*",4,0],"Perplexity-User":["partial","*",4,0],"Google-Extended":["partial","*",4,0],"Applebot-Extended":["partial","*",4,0],"CCBot":["partial","*",4,0],"Bytespider":["partial","*",4,0],"Amazonbot":["partial","*",4,0],"meta-externalagent":["partial","*",4,0],"cohere-ai":["partial","*",4,0],"Diffbot":["partial","*",4,0],"ContextFetchBot":["partial","*",4,0]},"llms":{"status":"missing","http":404,"bytes":114186},"home":{"status":"ok","http":200,"url":"https://9to5mac.com/","html_tokens":68866,"markdown_tokens":1692,"pruned_tokens":201}},
{"rank":1578,"domain":"sciencealert.com","origin":"https://sciencealert.com","robots":{"status":"ok","http":200,"url":"https://sciencealert.com/robots.txt","bytes":697,"groups":2},"bots":{"GPTBot":["partial","*",19,1],"ChatGPT-User":["partial","*",19,1],"OAI-SearchBot":["partial","*",19,1],"ClaudeBot":["partial","*",19,1],"Claude-User":["partial","*",19,1],"Claude-SearchBot":["partial","*",19,1],"anthropic-ai":["partial","*",19,1],"PerplexityBot":["partial","*",19,1],"Perplexity-User":["partial","*",19,1],"Google-Extended":["partial","*",19,1],"Applebot-Extended":["partial","*",19,1],"CCBot":["partial","*",19,1],"Bytespider":["partial","*",19,1],"Amazonbot":["partial","*",19,1],"meta-externalagent":["partial","*",19,1],"cohere-ai":["partial","*",19,1],"Diffbot":["partial","*",19,1],"ContextFetchBot":["partial","*",19,1]},"llms":{"status":"missing","http":404,"bytes":139621},"home":{"status":"ok","http":200,"url":"https://www.sciencealert.com/","html_tokens":49436,"markdown_tokens":7,"pruned_tokens":7}},
{"rank":1579,"domain":"tilda.ws","origin":"https://tilda.ws","robots":{"status":"missing","http":404,"url":"https://tilda.ws/robots.txt","bytes":574,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":574},"home":{"status":"ok","http":200,"url":"https://tilda.cc","html_tokens":117491,"markdown_tokens":191,"pruned_tokens":27}},
{"rank":1580,"domain":"akamai.com","origin":"https://akamai.com","robots":{"status":"refused","http":403,"url":"https://www.akamai.com/robots.txt","bytes":380,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":1581,"domain":"kremlin.ru","origin":"http://kremlin.ru","robots":{"status":"ok","http":200,"url":"http://kremlin.ru/robots.txt","bytes":629,"groups":1},"bots":{"GPTBot":["partial","*",18,0],"ChatGPT-User":["partial","*",18,0],"OAI-SearchBot":["partial","*",18,0],"ClaudeBot":["partial","*",18,0],"Claude-User":["partial","*",18,0],"Claude-SearchBot":["partial","*",18,0],"anthropic-ai":["partial","*",18,0],"PerplexityBot":["partial","*",18,0],"Perplexity-User":["partial","*",18,0],"Google-Extended":["partial","*",18,0],"Applebot-Extended":["partial","*",18,0],"CCBot":["partial","*",18,0],"Bytespider":["partial","*",18,0],"Amazonbot":["partial","*",18,0],"meta-externalagent":["partial","*",18,0],"cohere-ai":["partial","*",18,0],"Diffbot":["partial","*",18,0],"ContextFetchBot":["partial","*",18,0]},"llms":{"status":"missing","http":404,"bytes":27752},"home":{"status":"ok","http":200,"url":"http://kremlin.ru/","html_tokens":13147,"markdown_tokens":1416,"pruned_tokens":173}},
{"rank":1582,"domain":"in.gov","origin":"https://in.gov","robots":{"status":"ok","http":200,"url":"https://www.in.gov/robots.txt","bytes":57,"groups":1},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"missing","http":404,"bytes":52161},"home":{"status":"ok","http":200,"url":"https://www.in.gov/","html_tokens":60,"markdown_tokens":0,"pruned_tokens":0}},
{"rank":1583,"domain":"expiredwixdomain.com","origin":"https://expiredwixdomain.com","robots":{"status":"ok","http":200,"url":"https://expiredwixdomain.com/robots.txt","bytes":498,"groups":4},"bots":{"GPTBot":["partial","*",1,1],"ChatGPT-User":["partial","*",1,1],"OAI-SearchBot":["partial","*",1,1],"ClaudeBot":["partial","*",1,1],"Claude-User":["partial","*",1,1],"Claude-SearchBot":["partial","*",1,1],"anthropic-ai":["partial","*",1,1],"PerplexityBot":["partial","*",1,1],"Perplexity-User":["partial","*",1,1],"Google-Extended":["partial","*",1,1],"Applebot-Extended":["partial","*",1,1],"CCBot":["partial","*",1,1],"Bytespider":["partial","*",1,1],"Amazonbot":["partial","*",1,1],"meta-externalagent":["partial","*",1,1],"cohere-ai":["partial","*",1,1],"Diffbot":["partial","*",1,1],"ContextFetchBot":["partial","*",1,1]},"llms":{"status":"missing","http":404,"bytes":2239},"home":{"status":"http_error","http":404,"url":"https://www.expiredwixdomain.com/"}},
{"rank":1584,"domain":"symfony.com","origin":"https://symfony.com","robots":{"status":"ok","http":200,"url":"https://symfony.com/robots.txt","bytes":191,"groups":2},"bots":{"GPTBot":["partial","*",3,2],"ChatGPT-User":["partial","*",3,2],"OAI-SearchBot":["partial","*",3,2],"ClaudeBot":["partial","*",3,2],"Claude-User":["partial","*",3,2],"Claude-SearchBot":["partial","*",3,2],"anthropic-ai":["partial","*",3,2],"PerplexityBot":["partial","*",3,2],"Perplexity-User":["partial","*",3,2],"Google-Extended":["partial","*",3,2],"Applebot-Extended":["partial","*",3,2],"CCBot":["partial","*",3,2],"Bytespider":["partial","*",3,2],"Amazonbot":["partial","*",3,2],"meta-externalagent":["partial","*",3,2],"cohere-ai":["partial","*",3,2],"Diffbot":["partial","*",3,2],"ContextFetchBot":["partial","*",3,2]},"llms":{"status":"missing","http":404,"bytes":124121},"home":{"status":"ok","http":200,"url":"https://symfony.com/","html_tokens":38370,"markdown_tokens":560,"pruned_tokens":320}},
{"rank":1585,"domain":"idc.com","origin":"https://idc.com","robots":{"status":"ok","http":200,"url":"https://www.idc.com/robots.txt","bytes":169,"groups":1},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"missing","http":404,"bytes":58319},"home":{"status":"ok","http":200,"url":"https://www.idc.com/","html_tokens":35511,"markdown_tokens":222,"pruned_tokens":58}},
{"rank":1586,"domain":"haaretz.com","origin":"https://haaretz.com","robots":{"status":"refused","http":403,"url":"https://www.haaretz.com/robots.txt","bytes":424,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":1587,"domain":"tmz.com","origin":"https://tmz.com","robots":{"status":"ok","http":200,"url":"https://www.tmz.com/robots.txt","bytes":262,"groups":3},"bots":{"GPTBot":["partial","*",4,0],"ChatGPT-User":["partial","*",4,0],"OAI-SearchBot":["partial","*",4,0],"ClaudeBot":["partial","*",4,0],"Claude-User":["partial","*",4,0],"Claude-SearchBot":["partial","*",4,0],"anthropic-ai":["partial","*",4,0],"PerplexityBot":["partial","*",4,0],"Perplexity-User":["partial","*",4,0],"Google-Extended":["partial","*",4,0],"Applebot-Extended":["partial","*",4,0],"CCBot":["partial","*",4,0],"Bytespider":["partial","*",4,0],"Amazonbot":["partial","*",4,0],"meta-externalagent":["partial","*",4,0],"cohere-ai":["partial","*",4,0],"Diffbot":["partial","*",4,0],"ContextFetchBot":["partial","*",4,0]},"llms":{"status":"missing","http":404,"bytes":61942},"home":{"status":"ok","http":200,"url":"https://www.tmz.com/","html_tokens":149452,"markdown_tokens":129,"pruned_tokens":129}},
{"rank":1588,"domain":"degruyterbrill.com","origin":"https://degruyterbrill.com","robots":{"status":"ok","http":200,"url":"https://www.degruyterbrill.com/robots.txt","bytes":777,"groups":6},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["partial","*",15,0],"OAI-SearchBot":["partial","*",15,0],"ClaudeBot":["partial","*",15,0],"Claude-User":["partial","*",15,0],"Claude-SearchBot":["partial","*",15,0],"anthropic-ai":["partial","*",15,0],"PerplexityBot":["partial","*",15,0],"Perplexity-User":["partial","*",15,0],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["partial","*",15,0],"CCBot":["blocked","named",1,0],"Bytespider":["partial","*",15,0],"Amazonbot":["partial","*",15,0],"meta-externalagent":["partial","*",15,0],"cohere-ai":["partial","*",15,0],"Diffbot":["partial","*",15,0],"ContextFetchBot":["partial","*",15,0]},"llms":{"status":"missing","http":404,"bytes":76435},"home":{"status":"ok","http":200,"url":"https://www.degruyterbrill.com/","html_tokens":29390,"markdown_tokens":289,"pruned_tokens":109}},
{"rank":1589,"domain":"httpwg.org","origin":"https://httpwg.org","robots":{"status":"missing","http":404,"url":"https://httpwg.org/robots.txt","bytes":9379,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":9379},"home":{"status":"ok","http":200,"url":"https://httpwg.org/","html_tokens":2097,"markdown_tokens":34,"pruned_tokens":34}},
{"rank":1590,"domain":"guinnessworldrecords.com","origin":"https://guinnessworldrecords.com","robots":{"status":"ok","http":200,"url":"https://www.guinnessworldrecords.com/robots.txt","bytes":1014,"groups":1},"bots":{"GPTBot":["partial","*",35,0],"ChatGPT-User":["partial","*",35,0],"OAI-SearchBot":["partial","*",35,0],"ClaudeBot":["partial","*",35,0],"Claude-User":["partial","*",35,0],"Claude-SearchBot":["partial","*",35,0],"anthropic-ai":["partial","*",35,0],"PerplexityBot":["partial","*",35,0],"Perplexity-User":["partial","*",35,0],"Google-Extended":["partial","*",35,0],"Applebot-Extended":["partial","*",35,0],"CCBot":["partial","*",35,0],"Bytespider":["partial","*",35,0],"Amazonbot":["partial","*",35,0],"meta-externalagent":["partial","*",35,0],"cohere-ai":["partial","*",35,0],"Diffbot":["partial","*",35,0],"ContextFetchBot":["partial","*",35,0]},"llms":{"status":"missing","http":404,"bytes":405},"home":{"status":"ok","http":200,"url":"https://www.guinnessworldrecords.com/","html_tokens":10566,"markdown_tokens":63,"pruned_tokens":63}},
{"rank":1591,"domain":"fu-berlin.de","origin":"https://fu-berlin.de","robots":{"status":"ok","http":200,"url":"https://www.fu-berlin.de/robots.txt","bytes":787,"groups":1},"bots":{"GPTBot":["partial","*",13,0],"ChatGPT-User":["partial","*",13,0],"OAI-SearchBot":["partial","*",13,0],"ClaudeBot":["partial","*",13,0],"Claude-User":["partial","*",13,0],"Claude-SearchBot":["partial","*",13,0],"anthropic-ai":["partial","*",13,0],"PerplexityBot":["partial","*",13,0],"Perplexity-User":["partial","*",13,0],"Google-Extended":["partial","*",13,0],"Applebot-Extended":["partial","*",13,0],"CCBot":["partial","*",13,0],"Bytespider":["partial","*",13,0],"Amazonbot":["partial","*",13,0],"meta-externalagent":["partial","*",13,0],"cohere-ai":["partial","*",13,0],"Diffbot":["partial","*",13,0],"ContextFetchBot":["partial","*",13,0]},"llms":{"status":"missing","http":404,"bytes":2320},"home":{"status":"ok","http":200,"url":"https://www.fu-berlin.de/","html_tokens":14653,"markdown_tokens":321,"pruned_tokens":132}},
{"rank":1592,"domain":"java.net","origin":"https://java.net","robots":{"status":"timeout","http":null,"url":"https://java.net/robots.txt","bytes":0,"groups":0,"error":"ConnectTimeout"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":1593,"domain":"expedia.com","origin":"https://expedia.com","robots":{"status":"ok","http":200,"url":"https://www.expedia.com/robots.txt","bytes":2210,"groups":4},"bots":{"GPTBot":["partial","*",52,0],"ChatGPT-User":["partial","named",6,1],"OAI-SearchBot":["partial","named",6,1],"ClaudeBot":["partial","*",52,0],"Claude-User":["partial","named",6,1],"Claude-SearchBot":["partial","named",6,1],"anthropic-ai":["partial","*",52,0],"PerplexityBot":["partial","named",6,1],"Perplexity-User":["partial","named",6,1],"Google-Extended":["partial","*",52,0],"Applebot-Extended":["partial","*",52,0],"CCBot":["partial","*",52,0],"Bytespider":["partial","*",52,0],"Amazonbot":["partial","*",52,0],"meta-externalagent":["partial","*",52,0],"cohere-ai":["partial","*",52,0],"Diffbot":["partial","*",52,0],"ContextFetchBot":["partial","*",52,0]},"llms":{"status":"present","http":200,"bytes":7011},"home":{"status":"http_error","http":429,"url":"https://www.expedia.com/"}},
{"rank":1594,"domain":"rumble.com","origin":"https://rumble.com","robots":{"status":"refused","http":403,"url":"https://rumble.com/robots.txt","bytes":5505,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":1595,"domain":"wistia.com","origin":"https://wistia.com","robots":{"status":"ok","http":200,"url":"https://wistia.com/robots.txt","bytes":162,"groups":1},"bots":{"GPTBot":["partial","*",2,0],"ChatGPT-User":["partial","*",2,0],"OAI-SearchBot":["partial","*",2,0],"ClaudeBot":["partial","*",2,0],"Claude-User":["partial","*",2,0],"Claude-SearchBot":["partial","*",2,0],"anthropic-ai":["partial","*",2,0],"PerplexityBot":["partial","*",2,0],"Perplexity-User":["partial","*",2,0],"Google-Extended":["partial","*",2,0],"Applebot-Extended":["partial","*",2,0],"CCBot":["partial","*",2,0],"Bytespider":["partial","*",2,0],"Amazonbot":["partial","*",2,0],"meta-externalagent":["partial","*",2,0],"cohere-ai":["partial","*",2,0],"Diffbot":["partial","*",2,0],"ContextFetchBot":["partial","*",2,0]},"llms":{"status":"missing","http":404,"bytes":113000},"home":{"status":"ok","http":200,"url":"https://wistia.com/","html_tokens":68189,"markdown_tokens":17,"pruned_tokens":4}},
{"rank":1596,"domain":"honeywell.com","origin":"https://honeywell.com","robots":{"status":"ok","http":200,"url":"https://www.honeywell.com/robots.txt","bytes":1859,"groups":1},"bots":{"GPTBot":["partial","*",11,0],"ChatGPT-User":["partial","*",11,0],"OAI-SearchBot":["partial","*",11,0],"ClaudeBot":["partial","*",11,0],"Claude-User":["partial","*",11,0],"Claude-SearchBot":["partial","*",11,0],"anthropic-ai":["partial","*",11,0],"PerplexityBot":["partial","*",11,0],"Perplexity-User":["partial","*",11,0],"Google-Extended":["partial","*",11,0],"Applebot-Extended":["partial","*",11,0],"CCBot":["partial","*",11,0],"Bytespider":["partial","*",11,0],"Amazonbot":["partial","*",11,0],"meta-externalagent":["partial","*",11,0],"cohere-ai":["partial","*",11,0],"Diffbot":["partial","*",11,0],"ContextFetchBot":["partial","*",11,0]},"llms":{"status":"missing","http":404,"bytes":109526},"home":{"status":"ok","http":200,"url":"https://www.honeywell.com/us/en","html_tokens":40274,"markdown_tokens":229,"pruned_tokens":179}},
{"rank":1597,"domain":"mcdonalds.com","origin":"https://mcdonalds.com","robots":{"status":"refused","http":403,"url":"https://www.mcdonalds.com/robots.txt","bytes":383,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":1598,"domain":"goodhousekeeping.com","origin":"https://goodhousekeeping.com","robots":{"status":"ok","http":200,"url":"https://goodhousekeeping.com/robots.txt","bytes":977,"groups":3},"bots":{"GPTBot":["partial","named",32,0],"ChatGPT-User":["partial","named",32,0],"OAI-SearchBot":["partial","named",32,0],"ClaudeBot":["partial","*",19,0],"Claude-User":["partial","*",19,0],"Claude-SearchBot":["partial","*",19,0],"anthropic-ai":["partial","*",19,0],"PerplexityBot":["partial","*",19,0],"Perplexity-User":["partial","*",19,0],"Google-Extended":["partial","*",19,0],"Applebot-Extended":["partial","*",19,0],"CCBot":["partial","*",19,0],"Bytespider":["partial","*",19,0],"Amazonbot":["partial","*",19,0],"meta-externalagent":["partial","*",19,0],"cohere-ai":["partial","*",19,0],"Diffbot":["partial","*",19,0],"ContextFetchBot":["partial","*",19,0]},"llms":{"status":"present","http":200,"bytes":628350},"home":{"status":"ok","http":200,"url":"https://www.goodhousekeeping.com/","html_tokens":215385,"markdown_tokens":163,"pruned_tokens":163}},
{"rank":1599,"domain":"tum.de","origin":"https://tum.de","robots":{"status":"ok","http":200,"url":"https://www.tum.de/robots.txt","bytes":92,"groups":1},"bots":{"GPTBot":["partial","*",2,0],"ChatGPT-User":["partial","*",2,0],"OAI-SearchBot":["partial","*",2,0],"ClaudeBot":["partial","*",2,0],"Claude-User":["partial","*",2,0],"Claude-SearchBot":["partial","*",2,0],"anthropic-ai":["partial","*",2,0],"PerplexityBot":["partial","*",2,0],"Perplexity-User":["partial","*",2,0],"Google-Extended":["partial","*",2,0],"Applebot-Extended":["partial","*",2,0],"CCBot":["partial","*",2,0],"Bytespider":["partial","*",2,0],"Amazonbot":["partial","*",2,0],"meta-externalagent":["partial","*",2,0],"cohere-ai":["partial","*",2,0],"Diffbot":["partial","*",2,0],"ContextFetchBot":["partial","*",2,0]},"llms":{"status":"present","http":200,"bytes":5404},"home":{"status":"ok","http":200,"url":"https://www.tum.de/","html_tokens":117280,"markdown_tokens":401,"pruned_tokens":118}},
{"rank":1600,"domain":"kcl.ac.uk","origin":"https://kcl.ac.uk","robots":{"status":"ok","http":200,"url":"https://www.kcl.ac.uk/robots.txt","bytes":3169,"groups":7},"bots":{"GPTBot":["partial","*",65,0],"ChatGPT-User":["partial","*",65,0],"OAI-SearchBot":["partial","*",65,0],"ClaudeBot":["partial","*",65,0],"Claude-User":["partial","*",65,0],"Claude-SearchBot":["partial","*",65,0],"anthropic-ai":["partial","*",65,0],"PerplexityBot":["partial","*",65,0],"Perplexity-User":["partial","*",65,0],"Google-Extended":["partial","*",65,0],"Applebot-Extended":["partial","*",65,0],"CCBot":["partial","*",65,0],"Bytespider":["partial","*",65,0],"Amazonbot":["partial","*",65,0],"meta-externalagent":["partial","*",65,0],"cohere-ai":["partial","*",65,0],"Diffbot":["partial","*",65,0],"ContextFetchBot":["partial","*",65,0]},"llms":{"status":"missing","http":404,"bytes":14},"home":{"status":"ok","http":200,"url":"https://www.kcl.ac.uk/","html_tokens":24774,"markdown_tokens":80,"pruned_tokens":80}},
{"rank":1601,"domain":"softpedia.com","origin":"https://softpedia.com","robots":{"status":"ok","http":200,"url":"https://www.softpedia.com/robots.txt","bytes":1651,"groups":37},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["partial","*",11,0],"OAI-SearchBot":["partial","*",11,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",11,0],"Claude-SearchBot":["partial","*",11,0],"anthropic-ai":["partial","*",11,0],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["partial","*",11,0],"Google-Extended":["partial","*",11,0],"Applebot-Extended":["partial","*",11,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["partial","*",11,0],"cohere-ai":["partial","*",11,0],"Diffbot":["partial","*",11,0],"ContextFetchBot":["partial","*",11,0]},"llms":{"status":"missing","http":404,"bytes":21666},"home":{"status":"http_error","http":403,"url":"https://www.softpedia.com/"}},
{"rank":1602,"domain":"wikidata.org","origin":"https://wikidata.org","robots":{"status":"ok","http":200,"url":"https://www.wikidata.org/robots.txt","bytes":15896,"groups":33},"bots":{"GPTBot":["partial","*",212,5],"ChatGPT-User":["partial","*",212,5],"OAI-SearchBot":["partial","*",212,5],"ClaudeBot":["partial","*",212,5],"Claude-User":["partial","*",212,5],"Claude-SearchBot":["partial","*",212,5],"anthropic-ai":["partial","*",212,5],"PerplexityBot":["partial","*",212,5],"Perplexity-User":["partial","*",212,5],"Google-Extended":["partial","*",212,5],"Applebot-Extended":["partial","*",212,5],"CCBot":["partial","*",212,5],"Bytespider":["partial","*",212,5],"Amazonbot":["partial","*",212,5],"meta-externalagent":["partial","*",212,5],"cohere-ai":["partial","*",212,5],"Diffbot":["partial","*",212,5],"ContextFetchBot":["partial","*",212,5]},"llms":{"status":"missing","http":404,"bytes":1868},"home":{"status":"ok","http":200,"url":"https://www.wikidata.org/wiki/Wikidata:Main_Page","html_tokens":51568,"markdown_tokens":876,"pruned_tokens":323}},
{"rank":1603,"domain":"wmo.int","origin":"https://wmo.int","robots":{"status":"refused","http":403,"url":"https://wmo.int/robots.txt","bytes":560,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":1604,"domain":"iaea.org","origin":"https://iaea.org","robots":{"status":"ok","http":200,"url":"https://www.iaea.org/robots.txt","bytes":3502,"groups":1},"bots":{"GPTBot":["partial","*",76,18],"ChatGPT-User":["partial","*",76,18],"OAI-SearchBot":["partial","*",76,18],"ClaudeBot":["partial","*",76,18],"Claude-User":["partial","*",76,18],"Claude-SearchBot":["partial","*",76,18],"anthropic-ai":["partial","*",76,18],"PerplexityBot":["partial","*",76,18],"Perplexity-User":["partial","*",76,18],"Google-Extended":["partial","*",76,18],"Applebot-Extended":["partial","*",76,18],"CCBot":["partial","*",76,18],"Bytespider":["partial","*",76,18],"Amazonbot":["partial","*",76,18],"meta-externalagent":["partial","*",76,18],"cohere-ai":["partial","*",76,18],"Diffbot":["partial","*",76,18],"ContextFetchBot":["partial","*",76,18]},"llms":{"status":"missing","http":404,"bytes":291},"home":{"status":"http_error","http":403,"url":"https://iaea.org/"}},
{"rank":1605,"domain":"dallasnews.com","origin":"https://dallasnews.com","robots":{"status":"ok","http":200,"url":"https://www.dallasnews.com/robots.txt","bytes":1205,"groups":3},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["blocked","named",1,0],"OAI-SearchBot":["partial","*",7,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["blocked","named",1,0],"Claude-SearchBot":["partial","*",7,0],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["blocked","named",1,0],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["partial","*",7,0],"meta-externalagent":["partial","*",7,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",7,0]},"llms":{"status":"missing","http":404,"bytes":64743},"home":{"status":"ok","http":200,"url":"https://www.dallasnews.com/","html_tokens":228648,"markdown_tokens":19,"pruned_tokens":19}},
{"rank":1606,"domain":"aip.org","origin":"https://aip.org","robots":{"status":"ok","http":200,"url":"https://www.aip.org/robots.txt","bytes":89,"groups":2},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":127128},"home":{"status":"ok","http":200,"url":"https://www.aip.org/","html_tokens":75374,"markdown_tokens":115,"pruned_tokens":115}},
{"rank":1607,"domain":"samhsa.gov","origin":"https://samhsa.gov","robots":{"status":"ok","http":200,"url":"https://www.samhsa.gov/robots.txt","bytes":2997,"groups":1},"bots":{"GPTBot":["partial","*",49,18],"ChatGPT-User":["partial","*",49,18],"OAI-SearchBot":["partial","*",49,18],"ClaudeBot":["partial","*",49,18],"Claude-User":["partial","*",49,18],"Claude-SearchBot":["partial","*",49,18],"anthropic-ai":["partial","*",49,18],"PerplexityBot":["partial","*",49,18],"Perplexity-User":["partial","*",49,18],"Google-Extended":["partial","*",49,18],"Applebot-Extended":["partial","*",49,18],"CCBot":["partial","*",49,18],"Bytespider":["partial","*",49,18],"Amazonbot":["partial","*",49,18],"meta-externalagent":["partial","*",49,18],"cohere-ai":["partial","*",49,18],"Diffbot":["partial","*",49,18],"ContextFetchBot":["partial","*",49,18]},"llms":{"status":"missing","http":404,"bytes":188},"home":{"status":"ok","http":200,"url":"https://www.samhsa.gov/","html_tokens":176058,"markdown_tokens":264,"pruned_tokens":87}},
{"rank":1608,"domain":"filesusr.com","origin":"https://www.filesusr.com","robots":{"status":"refused","http":403,"url":"https://www.filesusr.com/robots.txt","bytes":9,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":1609,"domain":"kaggle.com","origin":"https://kaggle.com","robots":{"status":"missing","http":404,"url":"https://www.kaggle.com/robots.txt","bytes":5490,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"present","http":200,"bytes":2248},"home":{"status":"ok","http":200,"url":"https://www.kaggle.com/","html_tokens":1490,"markdown_tokens":0,"pruned_tokens":0}},
{"rank":1610,"domain":"infobae.com","origin":"https://infobae.com","robots":{"status":"ok","http":200,"url":"https://www.infobae.com/robots.txt","bytes":1360,"groups":1},"bots":{"GPTBot":["partial","*",1,0],"ChatGPT-User":["partial","*",1,0],"OAI-SearchBot":["partial","*",1,0],"ClaudeBot":["partial","*",1,0],"Claude-User":["partial","*",1,0],"Claude-SearchBot":["partial","*",1,0],"anthropic-ai":["partial","*",1,0],"PerplexityBot":["partial","*",1,0],"Perplexity-User":["partial","*",1,0],"Google-Extended":["partial","*",1,0],"Applebot-Extended":["partial","*",1,0],"CCBot":["partial","*",1,0],"Bytespider":["partial","*",1,0],"Amazonbot":["partial","*",1,0],"meta-externalagent":["partial","*",1,0],"cohere-ai":["partial","*",1,0],"Diffbot":["partial","*",1,0],"ContextFetchBot":["partial","*",1,0]},"llms":{"status":"missing","http":404,"bytes":18612},"home":{"status":"ok","http":200,"url":"https://www.infobae.com/estados-unidos/","html_tokens":126815,"markdown_tokens":25,"pruned_tokens":25}},
{"rank":1611,"domain":"epfl.ch","origin":"https://epfl.ch","robots":{"status":"ok","http":200,"url":"https://www.epfl.ch/robots.txt","bytes":0,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":48822},"home":{"status":"ok","http":200,"url":"https://www.epfl.ch/en/","html_tokens":19557,"markdown_tokens":213,"pruned_tokens":203}},
{"rank":1612,"domain":"blogspot.com.es","origin":"https://blogspot.com.es","robots":{"status":"missing","http":404,"url":"https://blogspot.com.es/robots.txt","bytes":1571,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":1569},"home":{"status":"http_error","http":404,"url":"https://blogspot.com.es/"}},
{"rank":1613,"domain":"hubpages.com","origin":"https://hubpages.com","robots":{"status":"ok","http":200,"url":"https://hubpages.com/robots.txt","bytes":1253,"groups":9},"bots":{"GPTBot":["partial","*",16,0],"ChatGPT-User":["partial","*",16,0],"OAI-SearchBot":["partial","*",16,0],"ClaudeBot":["partial","*",16,0],"Claude-User":["partial","*",16,0],"Claude-SearchBot":["partial","*",16,0],"anthropic-ai":["partial","*",16,0],"PerplexityBot":["partial","*",16,0],"Perplexity-User":["partial","*",16,0],"Google-Extended":["partial","*",16,0],"Applebot-Extended":["partial","*",16,0],"CCBot":["partial","*",16,0],"Bytespider":["partial","*",16,0],"Amazonbot":["partial","*",16,0],"meta-externalagent":["partial","*",16,0],"cohere-ai":["partial","*",16,0],"Diffbot":["partial","*",16,0],"ContextFetchBot":["partial","*",16,0]},"llms":{"status":"missing","http":404,"bytes":31755},"home":{"status":"http_error","http":403,"url":"https://discover.hubpages.com"}},
{"rank":1614,"domain":"demon.co.uk","origin":"https://demon.co.uk","robots":{"status":"unreachable","http":null,"url":"https://demon.co.uk/robots.txt","bytes":0,"groups":0,"error":"cannot resolve demon.co.uk"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":1615,"domain":"pinterest.co.uk","origin":"https://pinterest.co.uk","robots":{"status":"ok","http":200,"url":"https://uk.pinterest.com/robots.txt","bytes":29429,"groups":8},"bots":{"GPTBot":["blocked","*",1,0],"ChatGPT-User":["blocked","*",1,0],"OAI-SearchBot":["blocked","*",1,0],"ClaudeBot":["blocked","*",1,0],"Claude-User":["blocked","*",1,0],"Claude-SearchBot":["blocked","*",1,0],"anthropic-ai":["blocked","*",1,0],"PerplexityBot":["blocked","*",1,0],"Perplexity-User":["blocked","*",1,0],"Google-Extended":["blocked","*",1,0],"Applebot-Extended":["blocked","*",1,0],"CCBot":["blocked","*",1,0],"Bytespider":["blocked","*",1,0],"Amazonbot":["blocked","*",1,0],"meta-externalagent":["blocked","*",1,0],"cohere-ai":["blocked","*",1,0],"Diffbot":["blocked","*",1,0],"ContextFetchBot":["blocked","*",1,0]},"llms":{"status":"disallowed"},"home":{"status":"disallowed"}},
{"rank":1616,"domain":"wa.gov.au","origin":"https://wa.gov.au","robots":{"status":"ok","http":200,"url":"https://www.wa.gov.au/robots.txt","bytes":2173,"groups":1},"bots":{"GPTBot":["partial","*",37,18],"ChatGPT-User":["partial","*",37,18],"OAI-SearchBot":["partial","*",37,18],"ClaudeBot":["partial","*",37,18],"Claude-User":["partial","*",37,18],"Claude-SearchBot":["partial","*",37,18],"anthropic-ai":["partial","*",37,18],"PerplexityBot":["partial","*",37,18],"Perplexity-User":["partial","*",37,18],"Google-Extended":["partial","*",37,18],"Applebot-Extended":["partial","*",37,18],"CCBot":["partial","*",37,18],"Bytespider":["partial","*",37,18],"Amazonbot":["partial","*",37,18],"meta-externalagent":["partial","*",37,18],"cohere-ai":["partial","*",37,18],"Diffbot":["partial","*",37,18],"ContextFetchBot":["partial","*",37,18]},"llms":{"status":"missing","http":404,"bytes":187},"home":{"status":"ok","http":200,"url":"https://www.wa.gov.au/","html_tokens":19718,"markdown_tokens":51,"pruned_tokens":51}},
{"rank":1617,"domain":"cookieyes.com","origin":"https://cookieyes.com","robots":{"status":"ok","http":200,"url":"https://www.cookieyes.com/robots.txt","bytes":4331,"groups":19},"bots":{"GPTBot":["partial","named",4,2],"ChatGPT-User":["partial","named",4,2],"OAI-SearchBot":["partial","named",4,2],"ClaudeBot":["partial","named",4,2],"Claude-User":["partial","named",4,2],"Claude-SearchBot":["partial","named",4,2],"anthropic-ai":["partial","*",4,2],"PerplexityBot":["partial","named",4,2],"Perplexity-User":["partial","named",4,2],"Google-Extended":["partial","named",4,2],"Applebot-Extended":["partial","named",4,2],"CCBot":["partial","named",4,2],"Bytespider":["partial","*",4,2],"Amazonbot":["partial","*",4,2],"meta-externalagent":["partial","*",4,2],"cohere-ai":["partial","*",4,2],"Diffbot":["partial","*",4,2],"ContextFetchBot":["partial","*",4,2]},"llms":{"status":"present","http":200,"bytes":6096},"home":{"status":"not_html","error":"unsupported content type text/markdown"}},
{"rank":1618,"domain":"orf.at","origin":"https://orf.at","robots":{"status":"ok","http":200,"url":"https://orf.at/robots.txt","bytes":667,"groups":6},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["partial","*",2,0],"OAI-SearchBot":["partial","*",2,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",2,0],"Claude-SearchBot":["partial","*",2,0],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["partial","*",2,0],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["partial","*",2,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["partial","*",2,0],"meta-externalagent":["partial","*",2,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",2,0]},"llms":{"status":"missing","http":404,"bytes":15604},"home":{"status":"ok","http":200,"url":"https://orf.at/","html_tokens":19295,"markdown_tokens":294,"pruned_tokens":156}},
{"rank":1619,"domain":"instructure.com","origin":"https://instructure.com","robots":{"status":"ok","http":200,"url":"https://www.instructure.com/robots.txt","bytes":2102,"groups":1},"bots":{"GPTBot":["partial","*",38,18],"ChatGPT-User":["partial","*",38,18],"OAI-SearchBot":["partial","*",38,18],"ClaudeBot":["partial","*",38,18],"Claude-User":["partial","*",38,18],"Claude-SearchBot":["partial","*",38,18],"anthropic-ai":["partial","*",38,18],"PerplexityBot":["partial","*",38,18],"Perplexity-User":["partial","*",38,18],"Google-Extended":["partial","*",38,18],"Applebot-Extended":["partial","*",38,18],"CCBot":["partial","*",38,18],"Bytespider":["partial","*",38,18],"Amazonbot":["partial","*",38,18],"meta-externalagent":["partial","*",38,18],"cohere-ai":["partial","*",38,18],"Diffbot":["partial","*",38,18],"ContextFetchBot":["partial","*",38,18]},"llms":{"status":"present","http":200,"bytes":576732},"home":{"status":"ok","http":200,"url":"https://www.instructure.com/","html_tokens":55282,"markdown_tokens":446,"pruned_tokens":190}},
{"rank":1620,"domain":"bfmtv.com","origin":"https://bfmtv.com","robots":{"status":"ok","http":200,"url":"https://www.bfmtv.com/robots.txt","bytes":3768,"groups":68},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["partial","named",1,14],"OAI-SearchBot":["partial","named",1,14],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","named",1,14],"Claude-SearchBot":["partial","named",1,14],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["partial","named",1,14],"Google-Extended":["partial","named",1,14],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",2,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",2,0]},"llms":{"status":"http_error","http":400,"bytes":55123},"home":{"status":"ok","http":200,"url":"https://www.bfmtv.com/","html_tokens":248209,"markdown_tokens":1364,"pruned_tokens":120}},
{"rank":1621,"domain":"knowyourmeme.com","origin":"https://knowyourmeme.com","robots":{"status":"ok","http":200,"url":"https://knowyourmeme.com/robots.txt","bytes":924,"groups":1},"bots":{"GPTBot":["partial","*",30,0],"ChatGPT-User":["partial","*",30,0],"OAI-SearchBot":["partial","*",30,0],"ClaudeBot":["partial","*",30,0],"Claude-User":["partial","*",30,0],"Claude-SearchBot":["partial","*",30,0],"anthropic-ai":["partial","*",30,0],"PerplexityBot":["partial","*",30,0],"Perplexity-User":["partial","*",30,0],"Google-Extended":["partial","*",30,0],"Applebot-Extended":["partial","*",30,0],"CCBot":["partial","*",30,0],"Bytespider":["partial","*",30,0],"Amazonbot":["partial","*",30,0],"meta-externalagent":["partial","*",30,0],"cohere-ai":["partial","*",30,0],"Diffbot":["partial","*",30,0],"ContextFetchBot":["partial","*",30,0]},"llms":{"status":"missing","http":404,"bytes":5410},"home":{"status":"ok","http":200,"url":"https://knowyourmeme.com/","html_tokens":50978,"markdown_tokens":372,"pruned_tokens":108}},
{"rank":1622,"domain":"ouest-france.fr","origin":"https://ouest-france.fr","robots":{"status":"ok","http":200,"url":"https://www.ouest-france.fr/robots.txt","bytes":9822,"groups":174},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["partial","named",3,26],"OAI-SearchBot":["partial","named",3,26],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","named",3,26],"Claude-SearchBot":["partial","named",3,26],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["partial","named",3,26],"Perplexity-User":["partial","named",3,26],"Google-Extended":["partial","named",3,26],"Applebot-Extended":["partial","named",3,26],"CCBot":["blocked","named",1,0],"Bytespider":["partial","named",3,26],"Amazonbot":["partial","named",4,26],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["blocked","*",1,0]},"llms":{"status":"disallowed"},"home":{"status":"disallowed"}},
{"rank":1623,"domain":"radiofrance.fr","origin":"https://radiofrance.fr","robots":{"status":"ok","http":200,"url":"https://www.radiofrance.fr/robots.txt","bytes":4390,"groups":73},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["blocked","named",1,0],"OAI-SearchBot":["blocked","named",1,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",10,0],"Claude-SearchBot":["partial","*",10,0],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["partial","*",10,0],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["partial","*",10,0],"ContextFetchBot":["partial","*",10,0]},"llms":{"status":"missing","http":404,"bytes":153724},"home":{"status":"ok","http":200,"url":"https://www.radiofrance.fr/","html_tokens":537524,"markdown_tokens":1208,"pruned_tokens":168}},
{"rank":1624,"domain":"justgiving.com","origin":"https://justgiving.com","robots":{"status":"ok","http":200,"url":"https://justgiving.com/robots.txt","bytes":1343,"groups":9},"bots":{"GPTBot":["partial","*",5,0],"ChatGPT-User":["partial","*",5,0],"OAI-SearchBot":["partial","*",5,0],"ClaudeBot":["partial","*",5,0],"Claude-User":["partial","*",5,0],"Claude-SearchBot":["partial","*",5,0],"anthropic-ai":["partial","*",5,0],"PerplexityBot":["partial","*",5,0],"Perplexity-User":["partial","*",5,0],"Google-Extended":["partial","*",5,0],"Applebot-Extended":["partial","*",5,0],"CCBot":["partial","*",5,0],"Bytespider":["partial","*",5,0],"Amazonbot":["partial","*",5,0],"meta-externalagent":["partial","*",5,0],"cohere-ai":["partial","*",5,0],"Diffbot":["partial","*",5,0],"ContextFetchBot":["partial","*",5,0]},"llms":{"status":"present","http":200,"bytes":1358},"home":{"status":"ok","http":200,"url":"https://justgiving.com/","html_tokens":1320,"markdown_tokens":0,"pruned_tokens":0}},
{"rank":1625,"domain":"realtor.com","origin":"https://realtor.com","robots":{"status":"ok","http":200,"url":"https://www.realtor.com/robots.txt","bytes":17447,"groups":16},"bots":{"GPTBot":["partial","*",285,5],"ChatGPT-User":["partial","*",285,5],"OAI-SearchBot":["partial","*",285,5],"ClaudeBot":["partial","*",285,5],"Claude-User":["partial","*",285,5],"Claude-SearchBot":["partial","*",285,5],"anthropic-ai":["partial","*",285,5],"PerplexityBot":["partial","*",285,5],"Perplexity-User":["partial","*",285,5],"Google-Extended":["partial","*",285,5],"Applebot-Extended":["partial","*",285,5],"CCBot":["partial","*",285,5],"Bytespider":["partial","*",285,5],"Amazonbot":["partial","*",285,5],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["partial","*",285,5],"Diffbot":["partial","*",285,5],"ContextFetchBot":["partial","*",285,5]},"llms":{"status":"missing","http":404,"bytes":559},"home":{"status":"http_error","http":429,"url":"https://www.realtor.com/"}},
{"rank":1626,"domain":"meta.wikimedia.org","origin":"https://meta.wikimedia.org","robots":{"status":"ok","http":200,"url":"https://meta.wikimedia.org/robots.txt","bytes":18164,"groups":34},"bots":{"GPTBot":["partial","*",252,4],"ChatGPT-User":["partial","*",252,4],"OAI-SearchBot":["partial","*",252,4],"ClaudeBot":["partial","*",252,4],"Claude-User":["partial","*",252,4],"Claude-SearchBot":["partial","*",252,4],"anthropic-ai":["partial","*",252,4],"PerplexityBot":["partial","*",252,4],"Perplexity-User":["partial","*",252,4],"Google-Extended":["partial","*",252,4],"Applebot-Extended":["partial","*",252,4],"CCBot":["partial","*",252,4],"Bytespider":["partial","*",252,4],"Amazonbot":["partial","*",252,4],"meta-externalagent":["partial","*",252,4],"cohere-ai":["partial","*",252,4],"Diffbot":["partial","*",252,4],"ContextFetchBot":["partial","*",252,4]},"llms":{"status":"missing","http":404,"bytes":1868},"home":{"status":"disallowed","error":"robots.txt of meta.wikimedia.org disallows /wiki/Special:MyLanguage/Main_Page"}},
{"rank":1627,"domain":"xda-developers.com","origin":"https://xda-developers.com","robots":{"status":"ok","http":200,"url":"https://www.xda-developers.com/robots.txt","bytes":2049,"groups":2},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["partial","*",22,0],"OAI-SearchBot":["partial","*",22,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",22,0],"Claude-SearchBot":["partial","*",22,0],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["partial","*",22,0],"Perplexity-User":["partial","*",22,0],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",22,0]},"llms":{"status":"missing","http":404,"bytes":109257},"home":{"status":"ok","http":200,"url":"https://www.xda-developers.com/","html_tokens":238771,"markdown_tokens":25,"pruned_tokens":25}},
{"rank":1628,"domain":"audible.com","origin":"https://audible.com","robots":{"status":"refused","http":403,"url":"https://www.audible.com/robots.txt","bytes":919,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":1629,"domain":"archive.is","origin":"https://archive.is","robots":{"status":"ok","http":200,"url":"https://archive.is/robots.txt","bytes":439,"groups":1},"bots":{"GPTBot":["partial","*",14,2],"ChatGPT-User":["partial","*",14,2],"OAI-SearchBot":["partial","*",14,2],"ClaudeBot":["partial","*",14,2],"Claude-User":["partial","*",14,2],"Claude-SearchBot":["partial","*",14,2],"anthropic-ai":["partial","*",14,2],"PerplexityBot":["partial","*",14,2],"Perplexity-User":["partial","*",14,2],"Google-Extended":["partial","*",14,2],"Applebot-Extended":["partial","*",14,2],"CCBot":["partial","*",14,2],"Bytespider":["partial","*",14,2],"Amazonbot":["partial","*",14,2],"meta-externalagent":["partial","*",14,2],"cohere-ai":["partial","*",14,2],"Diffbot":["partial","*",14,2],"ContextFetchBot":["partial","*",14,2]},"llms":{"status":"missing","http":404,"bytes":13},"home":{"status":"ok","http":200,"url":"https://archive.is/","html_tokens":3546,"markdown_tokens":154,"pruned_tokens":38}},
{"rank":1630,"domain":"supremecourt.gov","origin":"https://supremecourt.gov","robots":{"status":"ok","http":200,"url":"https://www.supremecourt.gov/robots.txt","bytes":155,"groups":3},"bots":{"GPTBot":["partial","*",3,0],"ChatGPT-User":["partial","*",3,0],"OAI-SearchBot":["partial","*",3,0],"ClaudeBot":["partial","*",3,0],"Claude-User":["partial","*",3,0],"Claude-SearchBot":["partial","*",3,0],"anthropic-ai":["partial","*",3,0],"PerplexityBot":["partial","*",3,0],"Perplexity-User":["partial","*",3,0],"Google-Extended":["partial","*",3,0],"Applebot-Extended":["partial","*",3,0],"CCBot":["partial","*",3,0],"Bytespider":["partial","*",3,0],"Amazonbot":["partial","*",3,0],"meta-externalagent":["partial","*",3,0],"cohere-ai":["partial","*",3,0],"Diffbot":["partial","*",3,0],"ContextFetchBot":["partial","*",3,0]},"llms":{"status":"missing","http":404,"bytes":1175},"home":{"status":"ok","http":200,"url":"https://www.supremecourt.gov/","html_tokens":34305,"markdown_tokens":286,"pruned_tokens":172}},
{"rank":1631,"domain":"infogram.com","origin":"https://infogram.com","robots":{"status":"ok","http":200,"url":"https://infogram.com/robots.txt","bytes":97,"groups":1},"bots":{"GPTBot":["partial","*",2,0],"ChatGPT-User":["partial","*",2,0],"OAI-SearchBot":["partial","*",2,0],"ClaudeBot":["partial","*",2,0],"Claude-User":["partial","*",2,0],"Claude-SearchBot":["partial","*",2,0],"anthropic-ai":["partial","*",2,0],"PerplexityBot":["partial","*",2,0],"Perplexity-User":["partial","*",2,0],"Google-Extended":["partial","*",2,0],"Applebot-Extended":["partial","*",2,0],"CCBot":["partial","*",2,0],"Bytespider":["partial","*",2,0],"Amazonbot":["partial","*",2,0],"meta-externalagent":["partial","*",2,0],"cohere-ai":["partial","*",2,0],"Diffbot":["partial","*",2,0],"ContextFetchBot":["partial","*",2,0]},"llms":{"status":"missing","http":404,"bytes":410278},"home":{"status":"ok","http":200,"url":"https://infogram.com/","html_tokens":126754,"markdown_tokens":320,"pruned_tokens":142}},
{"rank":1632,"domain":"ipsos.com","origin":"https://ipsos.com","robots":{"status":"html","http":200,"url":"https://www.ipsos.com/en-us","bytes":54897,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":1633,"domain":"pa.gov","origin":"https://pa.gov","robots":{"status":"ok","http":200,"url":"https://www.pa.gov/robots.txt","bytes":1208,"groups":1},"bots":{"GPTBot":["partial","*",3,0],"ChatGPT-User":["partial","*",3,0],"OAI-SearchBot":["partial","*",3,0],"ClaudeBot":["partial","*",3,0],"Claude-User":["partial","*",3,0],"Claude-SearchBot":["partial","*",3,0],"anthropic-ai":["partial","*",3,0],"PerplexityBot":["partial","*",3,0],"Perplexity-User":["partial","*",3,0],"Google-Extended":["partial","*",3,0],"Applebot-Extended":["partial","*",3,0],"CCBot":["partial","*",3,0],"Bytespider":["partial","*",3,0],"Amazonbot":["partial","*",3,0],"meta-externalagent":["partial","*",3,0],"cohere-ai":["partial","*",3,0],"Diffbot":["partial","*",3,0],"ContextFetchBot":["partial","*",3,0]},"llms":{"status":"missing","http":404,"bytes":75571},"home":{"status":"ok","http":200,"url":"https://www.pa.gov/","html_tokens":24694,"markdown_tokens":39,"pruned_tokens":39}},
{"rank":1634,"domain":"blogspot.fr","origin":"https://blogspot.fr","robots":{"status":"ok","http":200,"url":"https://blogspot.fr/robots.txt","bytes":6624,"groups":4},"bots":{"GPTBot":["partial","*",169,64],"ChatGPT-User":["partial","*",169,64],"OAI-SearchBot":["partial","*",169,64],"ClaudeBot":["partial","*",169,64],"Claude-User":["partial","*",169,64],"Claude-SearchBot":["partial","*",169,64],"anthropic-ai":["partial","*",169,64],"PerplexityBot":["partial","*",169,64],"Perplexity-User":["partial","*",169,64],"Google-Extended":["partial","*",169,64],"Applebot-Extended":["partial","*",169,64],"CCBot":["partial","*",169,64],"Bytespider":["partial","*",169,64],"Amazonbot":["partial","*",169,64],"meta-externalagent":["partial","*",169,64],"cohere-ai":["partial","*",169,64],"Diffbot":["partial","*",169,64],"ContextFetchBot":["partial","*",169,64]},"llms":{"status":"missing","http":404,"bytes":1569},"home":{"status":"ok","http":200,"url":"https://www.google.com/","html_tokens":21482,"markdown_tokens":30,"pruned_tokens":30}},
{"rank":1635,"domain":"mlit.go.jp","origin":"https://www.mlit.go.jp","robots":{"status":"missing","http":404,"url":"https://www.mlit.go.jp/robots.txt","bytes":234,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":234},"home":{"status":"ok","http":200,"url":"https://www.mlit.go.jp/","html_tokens":12501,"markdown_tokens":61,"pruned_tokens":61}},
{"rank":1636,"domain":"ford.com","origin":"https://www.ford.com","robots":{"status":"ok","http":200,"url":"https://www.ford.com/robots.txt","bytes":1243,"groups":3},"bots":{"GPTBot":["partial","*",19,3],"ChatGPT-User":["partial","*",19,3],"OAI-SearchBot":["partial","*",19,3],"ClaudeBot":["partial","*",19,3],"Claude-User":["partial","*",19,3],"Claude-SearchBot":["partial","*",19,3],"anthropic-ai":["partial","*",19,3],"PerplexityBot":["partial","*",19,3],"Perplexity-User":["partial","*",19,3],"Google-Extended":["partial","*",19,3],"Applebot-Extended":["partial","*",19,3],"CCBot":["partial","*",19,3],"Bytespider":["partial","named",1,0],"Amazonbot":["partial","*",19,3],"meta-externalagent":["partial","*",19,3],"cohere-ai":["partial","*",19,3],"Diffbot":["partial","*",19,3],"ContextFetchBot":["partial","*",19,3]},"llms":{"status":"present","http":200,"bytes":788},"home":{"status":"ok","http":200,"url":"https://www.ford.com/","html_tokens":498422,"markdown_tokens":301,"pruned_tokens":104}},
{"rank":1637,"domain":"michelin.com","origin":"https://michelin.com","robots":{"status":"server_error","http":500,"url":"https://michelin.com/robots.txt","bytes":54,"groups":0,"error":"HTTP 500"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":1638,"domain":"hexun.com","origin":"https://hexun.com","robots":{"status":"missing","http":404,"url":"https://i.hexun.com/robots.txt","bytes":61,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":61},"home":{"status":"disallowed","error":"robots.txt of www.hexun.com not readable (html)"}},
{"rank":1639,"domain":"united-domains.de","origin":"https://united-domains.de","robots":{"status":"ok","http":200,"url":"https://www.united-domains.de/robots.txt","bytes":919,"groups":2},"bots":{"GPTBot":["partial","*",29,0],"ChatGPT-User":["partial","*",29,0],"OAI-SearchBot":["partial","*",29,0],"ClaudeBot":["partial","*",29,0],"Claude-User":["partial","*",29,0],"Claude-SearchBot":["partial","*",29,0],"anthropic-ai":["partial","*",29,0],"PerplexityBot":["partial","*",29,0],"Perplexity-User":["partial","*",29,0],"Google-Extended":["partial","*",29,0],"Applebot-Extended":["partial","*",29,0],"CCBot":["partial","*",29,0],"Bytespider":["partial","*",29,0],"Amazonbot":["partial","*",29,0],"meta-externalagent":["partial","*",29,0],"cohere-ai":["partial","*",29,0],"Diffbot":["partial","*",29,0],"ContextFetchBot":["partial","*",29,0]},"llms":{"status":"missing","http":404,"bytes":84932},"home":{"status":"ok","http":200,"url":"https://www.united-domains.de/","html_tokens":52312,"markdown_tokens":1194,"pruned_tokens":173}},
{"rank":1640,"domain":"cio.com","origin":"https://cio.com","robots":{"status":"ok","http":200,"url":"https://www.cio.com/robots.txt","bytes":4341,"groups":37},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["partial","*",24,0],"OAI-SearchBot":["partial","*",24,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",24,0],"Claude-SearchBot":["partial","*",24,0],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["partial","*",24,0],"Perplexity-User":["partial","*",24,0],"Google-Extended":["partial","*",24,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["partial","*",24,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",24,0]},"llms":{"status":"missing","http":404,"bytes":196781},"home":{"status":"ok","http":200,"url":"https://www.cio.com/","html_tokens":75034,"markdown_tokens":84,"pruned_tokens":84}},
{"rank":1641,"domain":"disneyplus.com","origin":"https://disneyplus.com","robots":{"status":"ok","http":200,"url":"https://www.disneyplus.com/robots.txt","bytes":2794,"groups":2},"bots":{"GPTBot":["partial","named",19,1],"ChatGPT-User":["partial","named",19,1],"OAI-SearchBot":["partial","named",19,1],"ClaudeBot":["partial","named",19,1],"Claude-User":["partial","named",19,1],"Claude-SearchBot":["partial","named",19,1],"anthropic-ai":["blocked","*",1,0],"PerplexityBot":["partial","named",19,1],"Perplexity-User":["blocked","*",1,0],"Google-Extended":["partial","named",19,1],"Applebot-Extended":["partial","named",19,1],"CCBot":["partial","named",19,1],"Bytespider":["partial","named",19,1],"Amazonbot":["partial","named",19,1],"meta-externalagent":["partial","named",19,1],"cohere-ai":["blocked","*",1,0],"Diffbot":["blocked","*",1,0],"ContextFetchBot":["blocked","*",1,0]},"llms":{"status":"disallowed"},"home":{"status":"disallowed"}},
{"rank":1642,"domain":"grigorik.com","origin":"https://grigorik.com","robots":{"status":"timeout","http":null,"url":"https://grigorik.com/robots.txt","bytes":0,"groups":0,"error":"ConnectTimeout"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":1643,"domain":"ca.us","origin":"https://ca.us","robots":{"status":"unreachable","http":null,"url":"https://ca.us/robots.txt","bytes":0,"groups":0,"error":"cannot resolve ca.us"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":1644,"domain":"bundestag.de","origin":"https://bundestag.de","robots":{"status":"ok","http":200,"url":"https://www.bundestag.de/robots.txt","bytes":504,"groups":1},"bots":{"GPTBot":["partial","*",7,1],"ChatGPT-User":["partial","*",7,1],"OAI-SearchBot":["partial","*",7,1],"ClaudeBot":["partial","*",7,1],"Claude-User":["partial","*",7,1],"Claude-SearchBot":["partial","*",7,1],"anthropic-ai":["partial","*",7,1],"PerplexityBot":["partial","*",7,1],"Perplexity-User":["partial","*",7,1],"Google-Extended":["partial","*",7,1],"Applebot-Extended":["partial","*",7,1],"CCBot":["partial","*",7,1],"Bytespider":["partial","*",7,1],"Amazonbot":["partial","*",7,1],"meta-externalagent":["partial","*",7,1],"cohere-ai":["partial","*",7,1],"Diffbot":["partial","*",7,1],"ContextFetchBot":["partial","*",7,1]},"llms":{"status":"missing","http":404,"bytes":42893},"home":{"status":"ok","http":200,"url":"https://www.bundestag.de/","html_tokens":80725,"markdown_tokens":4,"pruned_tokens":4}},
{"rank":1645,"domain":"americanbar.org","origin":"https://americanbar.org","robots":{"status":"refused","http":403,"url":"https://americanbar.org/robots.txt","bytes":5595,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":1646,"domain":"nzherald.co.nz","origin":"https://nzherald.co.nz","robots":{"status":"ok","http":200,"url":"https://www.nzherald.co.nz/robots.txt","bytes":44665,"groups":381},"bots":{"GPTBot":["partial","named",20,1],"ChatGPT-User":["partial","named",20,1],"OAI-SearchBot":["partial","named",20,1],"ClaudeBot":["partial","named",20,1],"Claude-User":["partial","named",20,1],"Claude-SearchBot":["partial","named",20,1],"anthropic-ai":["partial","named",20,1],"PerplexityBot":["partial","named",20,1],"Perplexity-User":["partial","named",20,1],"Google-Extended":["partial","named",20,1],"Applebot-Extended":["partial","named",20,1],"CCBot":["partial","named",1,1],"Bytespider":["partial","named",20,1],"Amazonbot":["partial","named",1,1],"meta-externalagent":["partial","named",20,1],"cohere-ai":["partial","named",20,1],"Diffbot":["partial","named",1,1],"ContextFetchBot":["partial","*",19,0]},"llms":{"status":"missing","http":404,"bytes":712720},"home":{"status":"ok","http":200,"url":"https://www.nzherald.co.nz/","html_tokens":490562,"markdown_tokens":20,"pruned_tokens":20}},
{"rank":1647,"domain":"fsf.org","origin":"https://fsf.org","robots":{"status":"ok","http":200,"url":"https://www.fsf.org/robots.txt","bytes":3037,"groups":46},"bots":{"GPTBot":["partial","*",10,0],"ChatGPT-User":["partial","*",10,0],"OAI-SearchBot":["partial","*",10,0],"ClaudeBot":["partial","*",10,0],"Claude-User":["partial","*",10,0],"Claude-SearchBot":["partial","*",10,0],"anthropic-ai":["partial","*",10,0],"PerplexityBot":["partial","*",10,0],"Perplexity-User":["partial","*",10,0],"Google-Extended":["partial","*",10,0],"Applebot-Extended":["partial","*",10,0],"CCBot":["partial","*",10,0],"Bytespider":["partial","*",10,0],"Amazonbot":["partial","*",10,0],"meta-externalagent":["partial","*",10,0],"cohere-ai":["partial","*",10,0],"Diffbot":["partial","*",10,0],"ContextFetchBot":["partial","*",10,0]},"llms":{"status":"missing","http":404,"bytes":43882},"home":{"status":"ok","http":200,"url":"https://www.fsf.org/","html_tokens":16097,"markdown_tokens":443,"pruned_tokens":27}},
{"rank":1648,"domain":"hm.com","origin":"https://hm.com","robots":{"status":"refused","http":403,"url":"https://hm.com/robots.txt","bytes":370,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":1649,"domain":"we.tl","origin":"https://we.tl","robots":{"status":"missing","http":404,"url":"https://www.wetransfer.com/redirect/error","bytes":26781,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"disallowed","error":"robots.txt of www.wetransfer.com not readable (html)"},"home":{"status":"disallowed","error":"robots.txt of wetransfer.com not readable (html)"}},
{"rank":1650,"domain":"r-project.org","origin":"https://r-project.org","robots":{"status":"missing","http":404,"url":"https://www.r-project.org/robots.txt","bytes":991,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":991},"home":{"status":"ok","http":200,"url":"https://www.r-project.org/","html_tokens":1863,"markdown_tokens":126,"pruned_tokens":126}},
{"rank":1651,"domain":"warwick.ac.uk","origin":"https://warwick.ac.uk","robots":{"status":"ok","http":200,"url":"https://warwick.ac.uk/robots.txt","bytes":1424,"groups":3},"bots":{"GPTBot":["partial","*",4,6],"ChatGPT-User":["partial","*",4,6],"OAI-SearchBot":["partial","*",4,6],"ClaudeBot":["partial","*",4,6],"Claude-User":["partial","*",4,6],"Claude-SearchBot":["partial","*",4,6],"anthropic-ai":["partial","*",4,6],"PerplexityBot":["partial","*",4,6],"Perplexity-User":["partial","*",4,6],"Google-Extended":["partial","*",4,6],"Applebot-Extended":["partial","*",4,6],"CCBot":["partial","*",4,6],"Bytespider":["partial","*",4,6],"Amazonbot":["partial","*",4,6],"meta-externalagent":["partial","*",4,6],"cohere-ai":["partial","*",4,6],"Diffbot":["partial","*",4,6],"ContextFetchBot":["partial","*",4,6]},"llms":{"status":"present","http":200,"bytes":1992},"home":{"status":"ok","http":200,"url":"https://warwick.ac.uk/","html_tokens":25521,"markdown_tokens":666,"pruned_tokens":85}},
{"rank":1652,"domain":"uky.edu","origin":"https://uky.edu","robots":{"status":"ok","http":200,"url":"https://www.uky.edu/robots.txt","bytes":2027,"groups":1},"bots":{"GPTBot":["partial","*",34,18],"ChatGPT-User":["partial","*",34,18],"OAI-SearchBot":["partial","*",34,18],"ClaudeBot":["partial","*",34,18],"Claude-User":["partial","*",34,18],"Claude-SearchBot":["partial","*",34,18],"anthropic-ai":["partial","*",34,18],"PerplexityBot":["partial","*",34,18],"Perplexity-User":["partial","*",34,18],"Google-Extended":["partial","*",34,18],"Applebot-Extended":["partial","*",34,18],"CCBot":["partial","*",34,18],"Bytespider":["partial","*",34,18],"Amazonbot":["partial","*",34,18],"meta-externalagent":["partial","*",34,18],"cohere-ai":["partial","*",34,18],"Diffbot":["partial","*",34,18],"ContextFetchBot":["partial","*",34,18]},"llms":{"status":"missing","http":404,"bytes":31824},"home":{"status":"ok","http":200,"url":"https://www.uky.edu/","html_tokens":26354,"markdown_tokens":126,"pruned_tokens":126}},
{"rank":1653,"domain":"maps.google.de","origin":"https://maps.google.de","robots":{"status":"ok","http":200,"url":"https://maps.google.de/robots.txt","bytes":6624,"groups":4},"bots":{"GPTBot":["partial","*",169,64],"ChatGPT-User":["partial","*",169,64],"OAI-SearchBot":["partial","*",169,64],"ClaudeBot":["partial","*",169,64],"Claude-User":["partial","*",169,64],"Claude-SearchBot":["partial","*",169,64],"anthropic-ai":["partial","*",169,64],"PerplexityBot":["partial","*",169,64],"Perplexity-User":["partial","*",169,64],"Google-Extended":["partial","*",169,64],"Applebot-Extended":["partial","*",169,64],"CCBot":["partial","*",169,64],"Bytespider":["partial","*",169,64],"Amazonbot":["partial","*",169,64],"meta-externalagent":["partial","*",169,64],"cohere-ai":["partial","*",169,64],"Diffbot":["partial","*",169,64],"ContextFetchBot":["partial","*",169,64]},"llms":{"status":"missing","http":404,"bytes":1569},"home":{"status":"ok","http":200,"url":"https://www.google.de/maps","html_tokens":54321,"markdown_tokens":30,"pruned_tokens":30}},
{"rank":1654,"domain":"ct.gov","origin":"https://ct.gov","robots":{"status":"html","http":200,"url":"https://ct.gov/robots.txt","bytes":239,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":1655,"domain":"emory.edu","origin":"https://emory.edu","robots":{"status":"ok","http":200,"url":"https://www.emory.edu/robots.txt","bytes":2005,"groups":1},"bots":{"GPTBot":["partial","*",33,18],"ChatGPT-User":["partial","*",33,18],"OAI-SearchBot":["partial","*",33,18],"ClaudeBot":["partial","*",33,18],"Claude-User":["partial","*",33,18],"Claude-SearchBot":["partial","*",33,18],"anthropic-ai":["partial","*",33,18],"PerplexityBot":["partial","*",33,18],"Perplexity-User":["partial","*",33,18],"Google-Extended":["partial","*",33,18],"Applebot-Extended":["partial","*",33,18],"CCBot":["partial","*",33,18],"Bytespider":["partial","*",33,18],"Amazonbot":["partial","*",33,18],"meta-externalagent":["partial","*",33,18],"cohere-ai":["partial","*",33,18],"Diffbot":["partial","*",33,18],"ContextFetchBot":["partial","*",33,18]},"llms":{"status":"missing","http":404,"bytes":187},"home":{"status":"ok","http":200,"url":"https://www.emory.edu/","html_tokens":45639,"markdown_tokens":132,"pruned_tokens":132}},
{"rank":1656,"domain":"schema.org","origin":"https://schema.org","robots":{"status":"ok","http":200,"url":"https://schema.org/robots.txt","bytes":96,"groups":1},"bots":{"GPTBot":["partial","*",1,0],"ChatGPT-User":["partial","*",1,0],"OAI-SearchBot":["partial","*",1,0],"ClaudeBot":["partial","*",1,0],"Claude-User":["partial","*",1,0],"Claude-SearchBot":["partial","*",1,0],"anthropic-ai":["partial","*",1,0],"PerplexityBot":["partial","*",1,0],"Perplexity-User":["partial","*",1,0],"Google-Extended":["partial","*",1,0],"Applebot-Extended":["partial","*",1,0],"CCBot":["partial","*",1,0],"Bytespider":["partial","*",1,0],"Amazonbot":["partial","*",1,0],"meta-externalagent":["partial","*",1,0],"cohere-ai":["partial","*",1,0],"Diffbot":["partial","*",1,0],"ContextFetchBot":["partial","*",1,0]},"llms":{"status":"missing","http":404,"bytes":285},"home":{"status":"ok","http":200,"url":"https://schema.org/","html_tokens":2057,"markdown_tokens":314,"pruned_tokens":314}},
{"rank":1657,"domain":"nifty.com","origin":"https://nifty.com","robots":{"status":"ok","http":200,"url":"https://www.nifty.com/robots.txt","bytes":93,"groups":1},"bots":{"GPTBot":["partial","*",4,0],"ChatGPT-User":["partial","*",4,0],"OAI-SearchBot":["partial","*",4,0],"ClaudeBot":["partial","*",4,0],"Claude-User":["partial","*",4,0],"Claude-SearchBot":["partial","*",4,0],"anthropic-ai":["partial","*",4,0],"PerplexityBot":["partial","*",4,0],"Perplexity-User":["partial","*",4,0],"Google-Extended":["partial","*",4,0],"Applebot-Extended":["partial","*",4,0],"CCBot":["partial","*",4,0],"Bytespider":["partial","*",4,0],"Amazonbot":["partial","*",4,0],"meta-externalagent":["partial","*",4,0],"cohere-ai":["partial","*",4,0],"Diffbot":["partial","*",4,0],"ContextFetchBot":["partial","*",4,0]},"llms":{"status":"missing","http":404,"bytes":22042},"home":{"status":"ok","http":200,"url":"https://www.nifty.com/","html_tokens":27455,"markdown_tokens":51,"pruned_tokens":51}},
{"rank":1658,"domain":"hostnet.nl","origin":"https://hostnet.nl","robots":{"status":"ok","http":200,"url":"https://www.hostnet.nl/robots.txt","bytes":6669,"groups":21},"bots":{"GPTBot":["partial","named",11,2],"ChatGPT-User":["partial","named",11,2],"OAI-SearchBot":["partial","named",11,2],"ClaudeBot":["partial","named",11,2],"Claude-User":["partial","named",11,2],"Claude-SearchBot":["partial","named",11,2],"anthropic-ai":["partial","*",11,2],"PerplexityBot":["partial","named",11,2],"Perplexity-User":["partial","named",11,2],"Google-Extended":["partial","named",11,2],"Applebot-Extended":["partial","named",11,2],"CCBot":["partial","named",11,2],"Bytespider":["partial","named",11,2],"Amazonbot":["partial","named",11,2],"meta-externalagent":["partial","named",11,2],"cohere-ai":["partial","*",11,2],"Diffbot":["partial","*",11,2],"ContextFetchBot":["partial","*",11,2]},"llms":{"status":"present","http":200,"bytes":4294},"home":{"status":"ok","http":200,"url":"https://www.hostnet.nl/","html_tokens":316428,"markdown_tokens":216,"pruned_tokens":28}},
{"rank":1659,"domain":"sleepycat.com","origin":"http://sleepycat.com","robots":{"status":"refused","http":403,"url":"https://www.oracle.com/us/products/database/overview/index.html","bytes":1448,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":1660,"domain":"mtv.com","origin":"https://mtv.com","robots":{"status":"ok","http":200,"url":"https://www.mtv.com/robots.txt","bytes":209,"groups":2},"bots":{"GPTBot":["partial","*",1,0],"ChatGPT-User":["partial","*",1,0],"OAI-SearchBot":["partial","*",1,0],"ClaudeBot":["partial","*",1,0],"Claude-User":["partial","*",1,0],"Claude-SearchBot":["partial","*",1,0],"anthropic-ai":["partial","*",1,0],"PerplexityBot":["partial","*",1,0],"Perplexity-User":["partial","*",1,0],"Google-Extended":["partial","*",1,0],"Applebot-Extended":["partial","*",1,0],"CCBot":["partial","*",1,0],"Bytespider":["partial","*",1,0],"Amazonbot":["partial","*",1,0],"meta-externalagent":["partial","*",1,0],"cohere-ai":["partial","*",1,0],"Diffbot":["partial","*",1,0],"ContextFetchBot":["partial","*",1,0]},"llms":{"status":"missing","http":404,"bytes":935060},"home":{"status":"ok","http":200,"url":"https://www.mtv.com","html_tokens":518943,"markdown_tokens":73,"pruned_tokens":73}},
{"rank":1661,"domain":"wpzoom.com","origin":"https://wpzoom.com","robots":{"status":"ok","http":200,"url":"https://wpzoom.com/robots.txt","bytes":148,"groups":1},"bots":{"GPTBot":["partial","*",2,1],"ChatGPT-User":["partial","*",2,1],"OAI-SearchBot":["partial","*",2,1],"ClaudeBot":["partial","*",2,1],"Claude-User":["partial","*",2,1],"Claude-SearchBot":["partial","*",2,1],"anthropic-ai":["partial","*",2,1],"PerplexityBot":["partial","*",2,1],"Perplexity-User":["partial","*",2,1],"Google-Extended":["partial","*",2,1],"Applebot-Extended":["partial","*",2,1],"CCBot":["partial","*",2,1],"Bytespider":["partial","*",2,1],"Amazonbot":["partial","*",2,1],"meta-externalagent":["partial","*",2,1],"cohere-ai":["partial","*",2,1],"Diffbot":["partial","*",2,1],"ContextFetchBot":["partial","*",2,1]},"llms":{"status":"other_text","http":200,"bytes":6990},"home":{"status":"ok","http":200,"url":"https://www.wpzoom.com/","html_tokens":118510,"markdown_tokens":899,"pruned_tokens":502}},
{"rank":1662,"domain":"twilio.com","origin":"https://twilio.com","robots":{"status":"ok","http":200,"url":"https://www.twilio.com/robots.txt","bytes":976,"groups":2},"bots":{"GPTBot":["partial","*",13,6],"ChatGPT-User":["partial","*",13,6],"OAI-SearchBot":["partial","*",13,6],"ClaudeBot":["partial","*",13,6],"Claude-User":["partial","*",13,6],"Claude-SearchBot":["partial","*",13,6],"anthropic-ai":["partial","*",13,6],"PerplexityBot":["partial","*",13,6],"Perplexity-User":["partial","*",13,6],"Google-Extended":["partial","*",13,6],"Applebot-Extended":["partial","*",13,6],"CCBot":["partial","*",13,6],"Bytespider":["partial","*",13,6],"Amazonbot":["partial","*",13,6],"meta-externalagent":["partial","*",13,6],"cohere-ai":["partial","*",13,6],"Diffbot":["partial","*",13,6],"ContextFetchBot":["partial","*",13,6]},"llms":{"status":"other_text","http":200,"bytes":2354712},"home":{"status":"ok","http":200,"url":"https://www.twilio.com/en-us","html_tokens":181364,"markdown_tokens":248,"pruned_tokens":168}},
{"rank":1663,"domain":"ndr.de","origin":"https://ndr.de","robots":{"status":"ok","http":200,"url":"https://www.ndr.de/robots.txt","bytes":2495,"groups":21},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["partial","*",8,0],"OAI-SearchBot":["partial","*",8,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",8,0],"Claude-SearchBot":["partial","*",8,0],"anthropic-ai":["partial","*",8,0],"PerplexityBot":["partial","*",8,0],"Perplexity-User":["partial","*",8,0],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["partial","*",8,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["partial","*",8,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",8,0]},"llms":{"status":"missing","http":404,"bytes":45931},"home":{"status":"ok","http":200,"url":"https://www.ndr.de/","html_tokens":81654,"markdown_tokens":255,"pruned_tokens":167}},
{"rank":1664,"domain":"kotaku.com","origin":"https://kotaku.com","robots":{"status":"ok","http":200,"url":"https://kotaku.com/robots.txt","bytes":1220,"groups":18},"bots":{"GPTBot":["partial","*",16,1],"ChatGPT-User":["partial","*",16,1],"OAI-SearchBot":["partial","*",16,1],"ClaudeBot":["partial","*",16,1],"Claude-User":["partial","*",16,1],"Claude-SearchBot":["partial","*",16,1],"anthropic-ai":["partial","*",16,1],"PerplexityBot":["partial","*",16,1],"Perplexity-User":["partial","*",16,1],"Google-Extended":["partial","*",16,1],"Applebot-Extended":["partial","*",16,1],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["partial","*",16,1],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",16,1]},"llms":{"status":"missing","http":404,"bytes":67900},"home":{"status":"ok","http":200,"url":"https://kotaku.com/","html_tokens":45512,"markdown_tokens":63,"pruned_tokens":63}},
{"rank":1665,"domain":"startribune.com","origin":"https://startribune.com","robots":{"status":"refused","http":429,"url":"https://startribune.com/robots.txt","bytes":34228,"groups":0,"error":"HTTP 429"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":1666,"domain":"fortinet.com","origin":"https://fortinet.com","robots":{"status":"ok","http":200,"url":"https://www.fortinet.com/robots.txt","bytes":3110,"groups":4},"bots":{"GPTBot":["partial","named",38,0],"ChatGPT-User":["partial","named",38,0],"OAI-SearchBot":["partial","named",38,0],"ClaudeBot":["blocked","*",1,0],"Claude-User":["partial","named",38,0],"Claude-SearchBot":["partial","named",38,0],"anthropic-ai":["blocked","*",1,0],"PerplexityBot":["partial","named",38,0],"Perplexity-User":["blocked","*",1,0],"Google-Extended":["partial","named",38,0],"Applebot-Extended":["blocked","*",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","*",1,0],"Amazonbot":["blocked","*",1,0],"meta-externalagent":["blocked","*",1,0],"cohere-ai":["blocked","*",1,0],"Diffbot":["blocked","*",1,0],"ContextFetchBot":["blocked","*",1,0]},"llms":{"status":"disallowed"},"home":{"status":"disallowed"}},
{"rank":1667,"domain":"nngroup.com","origin":"https://nngroup.com","robots":{"status":"ok","http":200,"url":"https://www.nngroup.com/robots.txt","bytes":690,"groups":15},"bots":{"GPTBot":["partial","*",6,0],"ChatGPT-User":["partial","*",6,0],"OAI-SearchBot":["partial","*",6,0],"ClaudeBot":["partial","*",6,0],"Claude-User":["partial","*",6,0],"Claude-SearchBot":["partial","*",6,0],"anthropic-ai":["partial","*",6,0],"PerplexityBot":["partial","*",6,0],"Perplexity-User":["partial","*",6,0],"Google-Extended":["allowed","named",0,1],"Applebot-Extended":["partial","*",6,0],"CCBot":["blocked","named",1,0],"Bytespider":["partial","*",6,0],"Amazonbot":["partial","*",6,0],"meta-externalagent":["partial","*",6,0],"cohere-ai":["partial","*",6,0],"Diffbot":["partial","*",6,0],"ContextFetchBot":["partial","*",6,0]},"llms":{"status":"missing","http":404,"bytes":93388},"home":{"status":"ok","http":200,"url":"https://www.nngroup.com/","html_tokens":27781,"markdown_tokens":270,"pruned_tokens":116}},
{"rank":1668,"domain":"neocities.org","origin":"https://neocities.org","robots":{"status":"ok","http":200,"url":"https://neocities.org/robots.txt","bytes":66,"groups":1},"bots":{"GPTBot":["allowed","*",0,1],"ChatGPT-User":["allowed","*",0,1],"OAI-SearchBot":["allowed","*",0,1],"ClaudeBot":["allowed","*",0,1],"Claude-User":["allowed","*",0,1],"Claude-SearchBot":["allowed","*",0,1],"anthropic-ai":["allowed","*",0,1],"PerplexityBot":["allowed","*",0,1],"Perplexity-User":["allowed","*",0,1],"Google-Extended":["allowed","*",0,1],"Applebot-Extended":["allowed","*",0,1],"CCBot":["allowed","*",0,1],"Bytespider":["allowed","*",0,1],"Amazonbot":["allowed","*",0,1],"meta-externalagent":["allowed","*",0,1],"cohere-ai":["allowed","*",0,1],"Diffbot":["allowed","*",0,1],"ContextFetchBot":["allowed","*",0,1]},"llms":{"status":"missing","http":404,"bytes":6114},"home":{"status":"ok","http":200,"url":"https://neocities.org/","html_tokens":5131,"markdown_tokens":276,"pruned_tokens":123}},
{"rank":1669,"domain":"motherjones.com","origin":"https://motherjones.com","robots":{"status":"ok","http":200,"url":"https://www.motherjones.com/robots.txt","bytes":906,"groups":15},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["partial","named",2,2],"OAI-SearchBot":["partial","named",1,2],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","named",1,2],"Claude-SearchBot":["partial","named",1,2],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["blocked","named",1,0],"Google-Extended":["partial","*",1,1],"Applebot-Extended":["blocked","named",1,0],"CCBot":["partial","*",1,1],"Bytespider":["partial","*",1,1],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["partial","*",1,1],"ContextFetchBot":["partial","*",1,1]},"llms":{"status":"missing","http":404,"bytes":132681},"home":{"status":"ok","http":200,"url":"https://www.motherjones.com/","html_tokens":69503,"markdown_tokens":345,"pruned_tokens":38}},
{"rank":1670,"domain":"news-medical.net","origin":"https://news-medical.net","robots":{"status":"ok","http":200,"url":"https://www.news-medical.net/robots.txt","bytes":156,"groups":2},"bots":{"GPTBot":["partial","*",5,0],"ChatGPT-User":["blocked","named",1,0],"OAI-SearchBot":["partial","*",5,0],"ClaudeBot":["partial","*",5,0],"Claude-User":["partial","*",5,0],"Claude-SearchBot":["partial","*",5,0],"anthropic-ai":["partial","*",5,0],"PerplexityBot":["partial","*",5,0],"Perplexity-User":["partial","*",5,0],"Google-Extended":["partial","*",5,0],"Applebot-Extended":["partial","*",5,0],"CCBot":["partial","*",5,0],"Bytespider":["partial","*",5,0],"Amazonbot":["partial","*",5,0],"meta-externalagent":["partial","*",5,0],"cohere-ai":["partial","*",5,0],"Diffbot":["partial","*",5,0],"ContextFetchBot":["partial","*",5,0]},"llms":{"status":"missing","http":404,"bytes":159415},"home":{"status":"ok","http":200,"url":"https://www.news-medical.net/","html_tokens":43066,"markdown_tokens":181,"pruned_tokens":123}},
{"rank":1671,"domain":"fas.org","origin":"https://fas.org","robots":{"status":"ok","http":200,"url":"https://fas.org/robots.txt","bytes":165,"groups":1},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"missing","http":404,"bytes":54603},"home":{"status":"ok","http":200,"url":"https://fas.org/","html_tokens":28025,"markdown_tokens":254,"pruned_tokens":124}},
{"rank":1672,"domain":"rambler.ru","origin":"https://rambler.ru","robots":{"status":"ok","http":200,"url":"https://www.rambler.ru/robots.txt","bytes":2015,"groups":6},"bots":{"GPTBot":["allowed","named",0,1],"ChatGPT-User":["partial","*",18,3],"OAI-SearchBot":["partial","*",18,3],"ClaudeBot":["partial","*",18,3],"Claude-User":["partial","*",18,3],"Claude-SearchBot":["partial","*",18,3],"anthropic-ai":["partial","*",18,3],"PerplexityBot":["partial","*",18,3],"Perplexity-User":["partial","*",18,3],"Google-Extended":["partial","*",18,3],"Applebot-Extended":["partial","*",18,3],"CCBot":["partial","*",18,3],"Bytespider":["partial","*",18,3],"Amazonbot":["partial","*",18,3],"meta-externalagent":["partial","*",18,3],"cohere-ai":["partial","*",18,3],"Diffbot":["partial","*",18,3],"ContextFetchBot":["partial","*",18,3]},"llms":{"status":"missing","http":404,"bytes":17880},"home":{"status":"ok","http":200,"url":"https://www.rambler.ru/","html_tokens":45172,"markdown_tokens":25,"pruned_tokens":25}},
{"rank":1673,"domain":"linksynergy.com","origin":"https://linksynergy.com","robots":{"status":"unreachable","http":null,"url":"https://linksynergy.com/robots.txt","bytes":0,"groups":0,"error":"cannot resolve linksynergy.com"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":1674,"domain":"makeuseof.com","origin":"https://makeuseof.com","robots":{"status":"ok","http":200,"url":"https://www.makeuseof.com/robots.txt","bytes":2039,"groups":2},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["partial","*",22,0],"OAI-SearchBot":["partial","*",22,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",22,0],"Claude-SearchBot":["partial","*",22,0],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["partial","*",22,0],"Perplexity-User":["partial","*",22,0],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",22,0]},"llms":{"status":"missing","http":404,"bytes":106026},"home":{"status":"ok","http":200,"url":"https://www.makeuseof.com/","html_tokens":232666,"markdown_tokens":40,"pruned_tokens":40}},
{"rank":1675,"domain":"uni-heidelberg.de","origin":"https://uni-heidelberg.de","robots":{"status":"ok","http":200,"url":"https://www.uni-heidelberg.de/robots.txt","bytes":1773,"groups":1},"bots":{"GPTBot":["partial","*",22,18],"ChatGPT-User":["partial","*",22,18],"OAI-SearchBot":["partial","*",22,18],"ClaudeBot":["partial","*",22,18],"Claude-User":["partial","*",22,18],"Claude-SearchBot":["partial","*",22,18],"anthropic-ai":["partial","*",22,18],"PerplexityBot":["partial","*",22,18],"Perplexity-User":["partial","*",22,18],"Google-Extended":["partial","*",22,18],"Applebot-Extended":["partial","*",22,18],"CCBot":["partial","*",22,18],"Bytespider":["partial","*",22,18],"Amazonbot":["partial","*",22,18],"meta-externalagent":["partial","*",22,18],"cohere-ai":["partial","*",22,18],"Diffbot":["partial","*",22,18],"ContextFetchBot":["partial","*",22,18]},"llms":{"status":"missing","http":404,"bytes":7186},"home":{"status":"ok","http":200,"url":"https://www.uni-heidelberg.de/en","html_tokens":73337,"markdown_tokens":525,"pruned_tokens":101}},
{"rank":1676,"domain":"ubereats.com","origin":"https://ubereats.com","robots":{"status":"refused","http":403,"url":"https://www.ubereats.com/robots.txt","bytes":5583,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":1677,"domain":"channel4.com","origin":"https://channel4.com","robots":{"status":"ok","http":200,"url":"https://www.channel4.com/robots.txt","bytes":1742,"groups":28},"bots":{"GPTBot":["partial","named",1,1],"ChatGPT-User":["partial","named",1,1],"OAI-SearchBot":["partial","named",1,1],"ClaudeBot":["partial","named",1,1],"Claude-User":["partial","*",4,0],"Claude-SearchBot":["partial","*",4,0],"anthropic-ai":["partial","named",1,1],"PerplexityBot":["partial","named",1,1],"Perplexity-User":["partial","*",4,0],"Google-Extended":["partial","named",1,1],"Applebot-Extended":["partial","named",1,1],"CCBot":["partial","named",1,1],"Bytespider":["partial","named",1,1],"Amazonbot":["partial","*",4,0],"meta-externalagent":["partial","named",1,1],"cohere-ai":["partial","named",1,1],"Diffbot":["partial","*",4,0],"ContextFetchBot":["partial","*",4,0]},"llms":{"status":"missing","http":404,"bytes":588471},"home":{"status":"ok","http":200,"url":"https://www.channel4.com/","html_tokens":3496,"markdown_tokens":3,"pruned_tokens":3}},
{"rank":1678,"domain":"bookshop.org","origin":"https://bookshop.org","robots":{"status":"ok","http":200,"url":"https://bookshop.org/robots.txt","bytes":1120,"groups":4},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["partial","named",1,11],"OAI-SearchBot":["partial","named",1,11],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","named",1,11],"Claude-SearchBot":["partial","named",1,11],"anthropic-ai":["partial","*",14,1],"PerplexityBot":["partial","named",1,11],"Perplexity-User":["partial","*",14,1],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",14,1]},"llms":{"status":"missing","http":404,"bytes":1008},"home":{"status":"http_error","http":403,"url":"https://bookshop.org/"}},
{"rank":1679,"domain":"uic.edu","origin":"https://uic.edu","robots":{"status":"unreachable","http":null,"url":"https://uic.edu/robots.txt","bytes":0,"groups":0,"error":"[SSL: CERTIFICATE_VERIFY_FAILED] certificate verify failed: unable to get local issuer certificate (_ssl.c:1028)"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":1680,"domain":"in.th","origin":"https://in.th","robots":{"status":"unreachable","http":null,"url":"https://in.th/robots.txt","bytes":0,"groups":0,"error":"cannot resolve in.th"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":1681,"domain":"pandora.com","origin":"https://pandora.com","robots":{"status":"ok","http":200,"url":"https://pandora.com/robots.txt","bytes":1822,"groups":4},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["partial","*",4,0],"OAI-SearchBot":["partial","*",4,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",4,0],"Claude-SearchBot":["partial","*",4,0],"anthropic-ai":["partial","*",4,0],"PerplexityBot":["partial","*",4,0],"Perplexity-User":["partial","*",4,0],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["partial","*",4,0],"Diffbot":["partial","*",4,0],"ContextFetchBot":["partial","*",4,0]},"llms":{"status":"missing","http":404,"bytes":7936},"home":{"status":"ok","http":200,"url":"https://www.pandora.com","html_tokens":5419,"markdown_tokens":94,"pruned_tokens":94}},
{"rank":1682,"domain":"nus.edu.sg","origin":"https://nus.edu.sg","robots":{"status":"html","http":200,"url":"https://nus.edu.sg/robots.txt","bytes":2825,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":1683,"domain":"uva.nl","origin":"https://uva.nl","robots":{"status":"ok","http":200,"url":"https://www.uva.nl/robots.txt","bytes":0,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":125706},"home":{"status":"ok","http":200,"url":"https://www.uva.nl/","html_tokens":42259,"markdown_tokens":252,"pruned_tokens":42}},
{"rank":1684,"domain":"ardmediathek.de","origin":"https://ardmediathek.de","robots":{"status":"ok","http":200,"url":"https://www.ardmediathek.de/robots.txt","bytes":2426,"groups":21},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["partial","*",7,0],"OAI-SearchBot":["partial","*",7,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",7,0],"Claude-SearchBot":["partial","*",7,0],"anthropic-ai":["partial","*",7,0],"PerplexityBot":["partial","*",7,0],"Perplexity-User":["partial","*",7,0],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["partial","*",7,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["partial","*",7,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",7,0]},"llms":{"status":"missing","http":410,"bytes":4},"home":{"status":"ok","http":200,"url":"https://www.ardmediathek.de/","html_tokens":84850,"markdown_tokens":37,"pruned_tokens":37}},
{"rank":1685,"domain":"wolfram.com","origin":"https://wolfram.com","robots":{"status":"ok","http":200,"url":"https://wolfram.com/robots.txt","bytes":1354,"groups":14},"bots":{"GPTBot":["partial","named",1,0],"ChatGPT-User":["partial","*",18,0],"OAI-SearchBot":["partial","*",18,0],"ClaudeBot":["partial","*",18,0],"Claude-User":["partial","*",18,0],"Claude-SearchBot":["partial","*",18,0],"anthropic-ai":["partial","*",18,0],"PerplexityBot":["partial","*",18,0],"Perplexity-User":["partial","*",18,0],"Google-Extended":["partial","*",18,0],"Applebot-Extended":["partial","*",18,0],"CCBot":["partial","named",1,0],"Bytespider":["partial","*",18,0],"Amazonbot":["partial","named",1,0],"meta-externalagent":["partial","*",18,0],"cohere-ai":["partial","*",18,0],"Diffbot":["partial","*",18,0],"ContextFetchBot":["partial","*",18,0]},"llms":{"status":"present","http":200,"bytes":648197},"home":{"status":"ok","http":200,"url":"https://wolfram.com/","html_tokens":77208,"markdown_tokens":554,"pruned_tokens":104}},
{"rank":1686,"domain":"vistaprint.com","origin":"https://vistaprint.com","robots":{"status":"ok","http":200,"url":"https://www.vistaprint.com/robots.txt","bytes":4144,"groups":4},"bots":{"GPTBot":["partial","*",103,0],"ChatGPT-User":["partial","*",103,0],"OAI-SearchBot":["partial","*",103,0],"ClaudeBot":["partial","*",103,0],"Claude-User":["partial","*",103,0],"Claude-SearchBot":["partial","*",103,0],"anthropic-ai":["partial","*",103,0],"PerplexityBot":["partial","*",103,0],"Perplexity-User":["partial","*",103,0],"Google-Extended":["partial","*",103,0],"Applebot-Extended":["partial","*",103,0],"CCBot":["partial","*",103,0],"Bytespider":["partial","*",103,0],"Amazonbot":["partial","*",103,0],"meta-externalagent":["partial","*",103,0],"cohere-ai":["partial","*",103,0],"Diffbot":["partial","*",103,0],"ContextFetchBot":["partial","*",103,0]},"llms":{"status":"missing","http":404,"bytes":228774},"home":{"status":"ok","http":200,"url":"https://www.vistaprint.com/","html_tokens":131985,"markdown_tokens":116,"pruned_tokens":116}},
{"rank":1687,"domain":"joinmastodon.org","origin":"https://joinmastodon.org","robots":{"status":"missing","http":404,"url":"https://joinmastodon.org/robots.txt","bytes":41029,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":41029},"home":{"status":"ok","http":200,"url":"https://joinmastodon.org/","html_tokens":31721,"markdown_tokens":465,"pruned_tokens":170}},
{"rank":1688,"domain":"timeshighereducation.com","origin":"https://timeshighereducation.com","robots":{"status":"ok","http":200,"url":"https://www.timeshighereducation.com/robots.txt","bytes":4104,"groups":1},"bots":{"GPTBot":["partial","*",89,1],"ChatGPT-User":["partial","*",89,1],"OAI-SearchBot":["partial","*",89,1],"ClaudeBot":["partial","*",89,1],"Claude-User":["partial","*",89,1],"Claude-SearchBot":["partial","*",89,1],"anthropic-ai":["partial","*",89,1],"PerplexityBot":["partial","*",89,1],"Perplexity-User":["partial","*",89,1],"Google-Extended":["partial","*",89,1],"Applebot-Extended":["partial","*",89,1],"CCBot":["partial","*",89,1],"Bytespider":["partial","*",89,1],"Amazonbot":["partial","*",89,1],"meta-externalagent":["partial","*",89,1],"cohere-ai":["partial","*",89,1],"Diffbot":["partial","*",89,1],"ContextFetchBot":["partial","*",89,1]},"llms":{"status":"http_error","http":403,"bytes":146},"home":{"status":"ok","http":200,"url":"https://www.timeshighereducation.com/","html_tokens":75965,"markdown_tokens":83,"pruned_tokens":83}},
{"rank":1689,"domain":"tomsguide.com","origin":"https://tomsguide.com","robots":{"status":"ok","http":200,"url":"https://www.tomsguide.com/robots.txt","bytes":1991,"groups":6},"bots":{"GPTBot":["partial","*",22,0],"ChatGPT-User":["partial","*",22,0],"OAI-SearchBot":["partial","*",22,0],"ClaudeBot":["partial","*",22,0],"Claude-User":["partial","*",22,0],"Claude-SearchBot":["partial","*",22,0],"anthropic-ai":["partial","*",22,0],"PerplexityBot":["partial","*",22,0],"Perplexity-User":["partial","*",22,0],"Google-Extended":["partial","*",22,0],"Applebot-Extended":["blocked","named",2,0],"CCBot":["partial","*",22,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["partial","*",22,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",22,0]},"llms":{"status":"missing","http":404,"bytes":1410021},"home":{"status":"ok","http":200,"url":"https://www.tomsguide.com/","html_tokens":687290,"markdown_tokens":3310,"pruned_tokens":146}},
{"rank":1690,"domain":"my.id","origin":"https://my.id","robots":{"status":"unreachable","http":null,"url":"https://my.id/robots.txt","bytes":0,"groups":0,"error":"cannot resolve my.id"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":1691,"domain":"emerald.com","origin":"https://emerald.com","robots":{"status":"ok","http":200,"url":"https://www.emerald.com/robots.txt","bytes":1952,"groups":7},"bots":{"GPTBot":["partial","*",52,3],"ChatGPT-User":["allowed","named",0,1],"OAI-SearchBot":["allowed","named",0,1],"ClaudeBot":["partial","*",52,3],"Claude-User":["allowed","named",0,1],"Claude-SearchBot":["partial","*",52,3],"anthropic-ai":["partial","*",52,3],"PerplexityBot":["partial","*",52,3],"Perplexity-User":["allowed","named",0,1],"Google-Extended":["partial","*",52,3],"Applebot-Extended":["partial","*",52,3],"CCBot":["partial","*",52,3],"Bytespider":["partial","*",52,3],"Amazonbot":["partial","*",52,3],"meta-externalagent":["partial","*",52,3],"cohere-ai":["partial","*",52,3],"Diffbot":["partial","*",52,3],"ContextFetchBot":["partial","*",52,3]},"llms":{"status":"http_error","http":403,"bytes":5526},"home":{"status":"http_error","http":403,"url":"https://www.emerald.com/"}},
{"rank":1692,"domain":"j.map.baidu.com","origin":"https://j.map.baidu.com","robots":{"status":"html","http":200,"url":"https://map.baidu.com","bytes":301698,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":1693,"domain":"elements.envato.com","origin":"https://elements.envato.com","robots":{"status":"refused","http":403,"url":"https://elements.envato.com/robots.txt","bytes":5578,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":1694,"domain":"xs4all.nl","origin":"https://xs4all.nl","robots":{"status":"html","http":200,"url":"https://www.xs4all.nl/robots.txt","bytes":23210,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":1695,"domain":"euractiv.com","origin":"https://euractiv.com","robots":{"status":"ok","http":200,"url":"https://www.rapporteur.com/robots.txt","bytes":1997,"groups":2},"bots":{"GPTBot":["partial","*",72,1],"ChatGPT-User":["partial","*",72,1],"OAI-SearchBot":["partial","*",72,1],"ClaudeBot":["partial","*",72,1],"Claude-User":["partial","*",72,1],"Claude-SearchBot":["partial","*",72,1],"anthropic-ai":["partial","*",72,1],"PerplexityBot":["partial","*",72,1],"Perplexity-User":["partial","*",72,1],"Google-Extended":["partial","*",72,1],"Applebot-Extended":["partial","*",72,1],"CCBot":["partial","*",72,1],"Bytespider":["partial","*",72,1],"Amazonbot":["partial","*",72,1],"meta-externalagent":["partial","*",72,1],"cohere-ai":["partial","*",72,1],"Diffbot":["partial","*",72,1],"ContextFetchBot":["partial","*",72,1]},"llms":{"status":"http_error","http":403,"bytes":4547},"home":{"status":"http_error","http":403,"url":"https://www.rapporteur.com/"}},
{"rank":1696,"domain":"iata.org","origin":"https://iata.org","robots":{"status":"ok","http":200,"url":"https://www.iata.org/robots.txt","bytes":213,"groups":1},"bots":{"GPTBot":["partial","*",6,0],"ChatGPT-User":["partial","*",6,0],"OAI-SearchBot":["partial","*",6,0],"ClaudeBot":["partial","*",6,0],"Claude-User":["partial","*",6,0],"Claude-SearchBot":["partial","*",6,0],"anthropic-ai":["partial","*",6,0],"PerplexityBot":["partial","*",6,0],"Perplexity-User":["partial","*",6,0],"Google-Extended":["partial","*",6,0],"Applebot-Extended":["partial","*",6,0],"CCBot":["partial","*",6,0],"Bytespider":["partial","*",6,0],"Amazonbot":["partial","*",6,0],"meta-externalagent":["partial","*",6,0],"cohere-ai":["partial","*",6,0],"Diffbot":["partial","*",6,0],"ContextFetchBot":["partial","*",6,0]},"llms":{"status":"missing","http":404,"bytes":91456},"home":{"status":"ok","http":200,"url":"https://www.iata.org/","html_tokens":34129,"markdown_tokens":71,"pruned_tokens":71}},
{"rank":1697,"domain":"ozon.ru","origin":"https://ozon.ru","robots":{"status":"ok","http":200,"url":"https://www.ozon.ru/robots.txt?__rr=1","bytes":12940,"groups":3},"bots":{"GPTBot":["partial","*",65,2],"ChatGPT-User":["partial","*",65,2],"OAI-SearchBot":["partial","*",65,2],"ClaudeBot":["partial","*",65,2],"Claude-User":["partial","*",65,2],"Claude-SearchBot":["partial","*",65,2],"anthropic-ai":["partial","*",65,2],"PerplexityBot":["partial","*",65,2],"Perplexity-User":["partial","*",65,2],"Google-Extended":["partial","*",65,2],"Applebot-Extended":["partial","*",65,2],"CCBot":["partial","*",65,2],"Bytespider":["partial","*",65,2],"Amazonbot":["partial","*",65,2],"meta-externalagent":["partial","*",65,2],"cohere-ai":["partial","*",65,2],"Diffbot":["partial","*",65,2],"ContextFetchBot":["partial","*",65,2]},"llms":{"status":"http_error","http":403,"bytes":9646},"home":{"status":"http_error","http":403,"url":"https://ozon.ru/?__rr=1"}},
{"rank":1698,"domain":"core.ac.uk","origin":"https://core.ac.uk","robots":{"status":"ok","http":200,"url":"https://core.ac.uk/robots.txt","bytes":1329,"groups":1},"bots":{"GPTBot":["partial","*",1,0],"ChatGPT-User":["partial","*",1,0],"OAI-SearchBot":["partial","*",1,0],"ClaudeBot":["partial","*",1,0],"Claude-User":["partial","*",1,0],"Claude-SearchBot":["partial","*",1,0],"anthropic-ai":["partial","*",1,0],"PerplexityBot":["partial","*",1,0],"Perplexity-User":["partial","*",1,0],"Google-Extended":["partial","*",1,0],"Applebot-Extended":["partial","*",1,0],"CCBot":["partial","*",1,0],"Bytespider":["partial","*",1,0],"Amazonbot":["partial","*",1,0],"meta-externalagent":["partial","*",1,0],"cohere-ai":["partial","*",1,0],"Diffbot":["partial","*",1,0],"ContextFetchBot":["partial","*",1,0]},"llms":{"status":"missing","http":404,"bytes":55077},"home":{"status":"ok","http":200,"url":"https://core.ac.uk/","html_tokens":22904,"markdown_tokens":824,"pruned_tokens":74}},
{"rank":1699,"domain":"liberation.fr","origin":"https://liberation.fr","robots":{"status":"ok","http":200,"url":"https://www.liberation.fr/robots.txt","bytes":1799,"groups":3},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["blocked","named",1,0],"OAI-SearchBot":["blocked","named",1,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",13,0],"Claude-SearchBot":["partial","*",13,0],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["partial","*",13,0],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",13,0]},"llms":{"status":"http_error","http":403,"bytes":773},"home":{"status":"http_error","http":403,"url":"https://www.liberation.fr/"}},
{"rank":1700,"domain":"eastmoney.com","origin":"https://eastmoney.com","robots":{"status":"ok","http":200,"url":"https://www.eastmoney.com/","bytes":349560,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"html","http":200,"bytes":349560},"home":{"status":"ok","http":200,"url":"https://www.eastmoney.com/","html_tokens":81887,"markdown_tokens":75,"pruned_tokens":75}},
{"rank":1701,"domain":"mcusercontent.com","origin":"https://mcusercontent.com","robots":{"status":"ok","http":200,"url":"https://mcusercontent.com/robots.txt","bytes":138,"groups":4},"bots":{"GPTBot":["blocked","*",1,0],"ChatGPT-User":["blocked","*",1,0],"OAI-SearchBot":["blocked","*",1,0],"ClaudeBot":["blocked","*",1,0],"Claude-User":["blocked","*",1,0],"Claude-SearchBot":["blocked","*",1,0],"anthropic-ai":["blocked","*",1,0],"PerplexityBot":["blocked","*",1,0],"Perplexity-User":["blocked","*",1,0],"Google-Extended":["blocked","*",1,0],"Applebot-Extended":["blocked","*",1,0],"CCBot":["blocked","*",1,0],"Bytespider":["blocked","*",1,0],"Amazonbot":["blocked","*",1,0],"meta-externalagent":["blocked","*",1,0],"cohere-ai":["blocked","*",1,0],"Diffbot":["blocked","*",1,0],"ContextFetchBot":["blocked","*",1,0]},"llms":{"status":"disallowed"},"home":{"status":"disallowed"}},
{"rank":1702,"domain":"3m.com","origin":"https://3m.com","robots":{"status":"unreachable","http":null,"url":"https://3m.com/robots.txt","bytes":0,"groups":0,"error":"<StreamReset stream_id:1, error_code:2, remote_reset:True>"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":1703,"domain":"thetimes.com","origin":"https://thetimes.com","robots":{"status":"ok","http":200,"url":"https://www.thetimes.co.uk/robots.txt","bytes":1700,"groups":2},"bots":{"GPTBot":["partial","named",8,1],"ChatGPT-User":["partial","named",8,1],"OAI-SearchBot":["partial","named",8,1],"ClaudeBot":["blocked","*",1,0],"Claude-User":["blocked","*",1,0],"Claude-SearchBot":["blocked","*",1,0],"anthropic-ai":["blocked","*",1,0],"PerplexityBot":["blocked","*",1,0],"Perplexity-User":["blocked","*",1,0],"Google-Extended":["blocked","*",1,0],"Applebot-Extended":["blocked","*",1,0],"CCBot":["blocked","*",1,0],"Bytespider":["blocked","*",1,0],"Amazonbot":["blocked","*",1,0],"meta-externalagent":["blocked","*",1,0],"cohere-ai":["blocked","*",1,0],"Diffbot":["blocked","*",1,0],"ContextFetchBot":["blocked","*",1,0]},"llms":{"status":"disallowed"},"home":{"status":"disallowed"}},
{"rank":1704,"domain":"doaj.org","origin":"https://doaj.org","robots":{"status":"ok","http":200,"url":"https://doaj.org/robots.txt","bytes":793,"groups":1},"bots":{"GPTBot":["partial","*",13,1],"ChatGPT-User":["partial","*",13,1],"OAI-SearchBot":["partial","*",13,1],"ClaudeBot":["partial","*",13,1],"Claude-User":["partial","*",13,1],"Claude-SearchBot":["partial","*",13,1],"anthropic-ai":["partial","*",13,1],"PerplexityBot":["partial","*",13,1],"Perplexity-User":["partial","*",13,1],"Google-Extended":["partial","*",13,1],"Applebot-Extended":["partial","*",13,1],"CCBot":["partial","*",13,1],"Bytespider":["partial","*",13,1],"Amazonbot":["partial","*",13,1],"meta-externalagent":["partial","*",13,1],"cohere-ai":["partial","*",13,1],"Diffbot":["partial","*",13,1],"ContextFetchBot":["partial","*",13,1]},"llms":{"status":"missing","http":404,"bytes":54048},"home":{"status":"http_error","http":403,"url":"https://doaj.org/"}},
{"rank":1705,"domain":"unctad.org","origin":"https://unctad.org","robots":{"status":"refused","http":403,"url":"https://unctad.org/robots.txt","bytes":5398,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":1706,"domain":"leeds.ac.uk","origin":"https://leeds.ac.uk","robots":{"status":"ok","http":200,"url":"https://leeds.ac.uk/robots.txt","bytes":0,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":93991},"home":{"status":"ok","http":200,"url":"https://www.leeds.ac.uk/","html_tokens":18065,"markdown_tokens":82,"pruned_tokens":34}},
{"rank":1707,"domain":"missouri.edu","origin":"https://missouri.edu","robots":{"status":"ok","http":200,"url":"https://missouri.edu/robots.txt","bytes":2005,"groups":1},"bots":{"GPTBot":["partial","*",33,18],"ChatGPT-User":["partial","*",33,18],"OAI-SearchBot":["partial","*",33,18],"ClaudeBot":["partial","*",33,18],"Claude-User":["partial","*",33,18],"Claude-SearchBot":["partial","*",33,18],"anthropic-ai":["partial","*",33,18],"PerplexityBot":["partial","*",33,18],"Perplexity-User":["partial","*",33,18],"Google-Extended":["partial","*",33,18],"Applebot-Extended":["partial","*",33,18],"CCBot":["partial","*",33,18],"Bytespider":["partial","*",33,18],"Amazonbot":["partial","*",33,18],"meta-externalagent":["partial","*",33,18],"cohere-ai":["partial","*",33,18],"Diffbot":["partial","*",33,18],"ContextFetchBot":["partial","*",33,18]},"llms":{"status":"missing","http":404,"bytes":186},"home":{"status":"ok","http":200,"url":"https://missouri.edu/","html_tokens":22260,"markdown_tokens":617,"pruned_tokens":152}},
{"rank":1708,"domain":"infoworld.com","origin":"https://infoworld.com","robots":{"status":"ok","http":200,"url":"https://www.infoworld.com/robots.txt","bytes":2912,"groups":37},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["partial","*",24,0],"OAI-SearchBot":["partial","*",24,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",24,0],"Claude-SearchBot":["partial","*",24,0],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["partial","*",24,0],"Perplexity-User":["partial","*",24,0],"Google-Extended":["partial","*",24,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["partial","*",24,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",24,0]},"llms":{"status":"missing","http":404,"bytes":167210},"home":{"status":"ok","http":200,"url":"https://www.infoworld.com","html_tokens":72036,"markdown_tokens":249,"pruned_tokens":120}},
{"rank":1709,"domain":"altavista.com","origin":"https://altavista.com","robots":{"status":"ok","http":200,"url":"http://www.altavista.com/robots.txt","bytes":2177,"groups":47},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["blocked","named",1,0],"OAI-SearchBot":["blocked","named",1,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",24,1],"Claude-SearchBot":["partial","*",24,1],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["partial","*",24,1],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["partial","*",24,1],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["partial","*",24,1],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",24,1]},"llms":{"status":"unreachable","error":"Server disconnected"},"home":{"status":"unreachable","error":"Server disconnected"}},
{"rank":1710,"domain":"hulu.com","origin":"https://hulu.com","robots":{"status":"ok","http":200,"url":"https://www.hulu.com/robots.txt","bytes":3093,"groups":2},"bots":{"GPTBot":["partial","named",34,1],"ChatGPT-User":["partial","named",34,1],"OAI-SearchBot":["partial","named",34,1],"ClaudeBot":["partial","named",34,1],"Claude-User":["partial","named",34,1],"Claude-SearchBot":["partial","named",34,1],"anthropic-ai":["blocked","*",1,0],"PerplexityBot":["partial","named",34,1],"Perplexity-User":["blocked","*",1,0],"Google-Extended":["partial","named",34,1],"Applebot-Extended":["partial","named",34,1],"CCBot":["partial","named",34,1],"Bytespider":["partial","named",34,1],"Amazonbot":["partial","named",34,1],"meta-externalagent":["partial","named",34,1],"cohere-ai":["blocked","*",1,0],"Diffbot":["blocked","*",1,0],"ContextFetchBot":["blocked","*",1,0]},"llms":{"status":"disallowed"},"home":{"status":"disallowed"}},
{"rank":1711,"domain":"medrxiv.org","origin":"https://medrxiv.org","robots":{"status":"refused","http":403,"url":"https://medrxiv.org/robots.txt","bytes":5549,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":1712,"domain":"ucr.edu","origin":"https://ucr.edu","robots":{"status":"refused","http":403,"url":"https://www.ucr.edu","bytes":560,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":1713,"domain":"click.linksynergy.com","origin":"https://click.linksynergy.com","robots":{"status":"ok","http":200,"url":"https://click.linksynergy.com/robots.txt","bytes":26,"groups":1},"bots":{"GPTBot":["blocked","*",1,0],"ChatGPT-User":["blocked","*",1,0],"OAI-SearchBot":["blocked","*",1,0],"ClaudeBot":["blocked","*",1,0],"Claude-User":["blocked","*",1,0],"Claude-SearchBot":["blocked","*",1,0],"anthropic-ai":["blocked","*",1,0],"PerplexityBot":["blocked","*",1,0],"Perplexity-User":["blocked","*",1,0],"Google-Extended":["blocked","*",1,0],"Applebot-Extended":["blocked","*",1,0],"CCBot":["blocked","*",1,0],"Bytespider":["blocked","*",1,0],"Amazonbot":["blocked","*",1,0],"meta-externalagent":["blocked","*",1,0],"cohere-ai":["blocked","*",1,0],"Diffbot":["blocked","*",1,0],"ContextFetchBot":["blocked","*",1,0]},"llms":{"status":"disallowed"},"home":{"status":"disallowed"}},
{"rank":1714,"domain":"nhl.com","origin":"https://nhl.com","robots":{"status":"ok","http":200,"url":"https://www.nhl.com/robots.txt","bytes":255,"groups":1},"bots":{"GPTBot":["partial","*",10,1],"ChatGPT-User":["partial","*",10,1],"OAI-SearchBot":["partial","*",10,1],"ClaudeBot":["partial","*",10,1],"Claude-User":["partial","*",10,1],"Claude-SearchBot":["partial","*",10,1],"anthropic-ai":["partial","*",10,1],"PerplexityBot":["partial","*",10,1],"Perplexity-User":["partial","*",10,1],"Google-Extended":["partial","*",10,1],"Applebot-Extended":["partial","*",10,1],"CCBot":["partial","*",10,1],"Bytespider":["partial","*",10,1],"Amazonbot":["partial","*",10,1],"meta-externalagent":["partial","*",10,1],"cohere-ai":["partial","*",10,1],"Diffbot":["partial","*",10,1],"ContextFetchBot":["partial","*",10,1]},"llms":{"status":"disallowed","error":"robots.txt of www.nhl.com disallows /errors/not-found"},"home":{"status":"ok","http":200,"url":"https://www.nhl.com/","html_tokens":87154,"markdown_tokens":79,"pruned_tokens":79}},
{"rank":1715,"domain":"softonic.com","origin":"https://softonic.com","robots":{"status":"refused","http":412,"url":"https://softonic.com/robots.txt","bytes":0,"groups":0,"error":"HTTP 412"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":1716,"domain":"uio.no","origin":"https://uio.no","robots":{"status":"ok","http":200,"url":"https://www.uio.no/robots.txt","bytes":6606,"groups":2},"bots":{"GPTBot":["partial","*",56,4],"ChatGPT-User":["partial","*",56,4],"OAI-SearchBot":["partial","*",56,4],"ClaudeBot":["partial","*",56,4],"Claude-User":["partial","*",56,4],"Claude-SearchBot":["partial","*",56,4],"anthropic-ai":["partial","*",56,4],"PerplexityBot":["partial","*",56,4],"Perplexity-User":["partial","*",56,4],"Google-Extended":["partial","*",56,4],"Applebot-Extended":["partial","*",56,4],"CCBot":["partial","*",56,4],"Bytespider":["partial","*",56,4],"Amazonbot":["partial","*",56,4],"meta-externalagent":["partial","*",56,4],"cohere-ai":["partial","*",56,4],"Diffbot":["partial","*",56,4],"ContextFetchBot":["partial","*",56,4]},"llms":{"status":"missing","http":404,"bytes":4929},"home":{"status":"ok","http":200,"url":"https://www.uio.no/","html_tokens":9957,"markdown_tokens":33,"pruned_tokens":33}},
{"rank":1717,"domain":"springeropen.com","origin":"https://springeropen.com","robots":{"status":"missing","http":404,"url":"https://www.springeropen.com/robots.txt","bytes":8190,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"html","http":200,"bytes":265322},"home":{"status":"ok","http":200,"url":"https://link.springer.com/brands/springer","html_tokens":66326,"markdown_tokens":673,"pruned_tokens":128}},
{"rank":1718,"domain":"wired.co.uk","origin":"https://wired.co.uk","robots":{"status":"ok","http":200,"url":"https://www.wired.com/robots.txt","bytes":2245,"groups":3},"bots":{"GPTBot":["partial","*",14,17],"ChatGPT-User":["partial","*",14,17],"OAI-SearchBot":["partial","*",14,17],"ClaudeBot":["blocked","named",1,0],"Claude-User":["blocked","named",1,0],"Claude-SearchBot":["blocked","named",1,0],"anthropic-ai":["partial","*",14,17],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["blocked","named",1,0],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",14,17]},"llms":{"status":"missing","http":404,"bytes":866157},"home":{"status":"ok","http":200,"url":"https://www.wired.com/","html_tokens":390211,"markdown_tokens":57,"pruned_tokens":57}},
{"rank":1719,"domain":"tiny.cc","origin":"https://tiny.cc","robots":{"status":"ok","http":200,"url":"https://tiny.cc/robots.txt","bytes":104,"groups":1},"bots":{"GPTBot":["partial","*",2,0],"ChatGPT-User":["partial","*",2,0],"OAI-SearchBot":["partial","*",2,0],"ClaudeBot":["partial","*",2,0],"Claude-User":["partial","*",2,0],"Claude-SearchBot":["partial","*",2,0],"anthropic-ai":["partial","*",2,0],"PerplexityBot":["partial","*",2,0],"Perplexity-User":["partial","*",2,0],"Google-Extended":["partial","*",2,0],"Applebot-Extended":["partial","*",2,0],"CCBot":["partial","*",2,0],"Bytespider":["partial","*",2,0],"Amazonbot":["partial","*",2,0],"meta-externalagent":["partial","*",2,0],"cohere-ai":["partial","*",2,0],"Diffbot":["partial","*",2,0],"ContextFetchBot":["partial","*",2,0]},"llms":{"status":"missing","http":404,"bytes":6682},"home":{"status":"ok","http":200,"url":"https://tiny.cc/","html_tokens":4442,"markdown_tokens":107,"pruned_tokens":62}},
{"rank":1720,"domain":"insider.com","origin":"https://insider.com","robots":{"status":"ok","http":200,"url":"https://www.businessinsider.com/robots.txt","bytes":2231,"groups":13},"bots":{"GPTBot":["partial","*",31,0],"ChatGPT-User":["partial","*",31,0],"OAI-SearchBot":["partial","*",31,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",31,0],"Claude-SearchBot":["partial","*",31,0],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["partial","*",31,0],"Perplexity-User":["partial","*",31,0],"Google-Extended":["partial","*",31,0],"Applebot-Extended":["partial","*",31,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["partial","*",31,0],"meta-externalagent":["partial","*",31,0],"cohere-ai":["partial","*",31,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",31,0]},"llms":{"status":"present","http":200,"bytes":1066},"home":{"status":"ok","http":200,"url":"https://www.businessinsider.com/","html_tokens":151111,"markdown_tokens":157,"pruned_tokens":30}},
{"rank":1721,"domain":"hatenablog.com","origin":"https://hatenablog.com","robots":{"status":"ok","http":200,"url":"https://hatena.blog/robots.txt","bytes":96,"groups":1},"bots":{"GPTBot":["partial","*",2,2],"ChatGPT-User":["partial","*",2,2],"OAI-SearchBot":["partial","*",2,2],"ClaudeBot":["partial","*",2,2],"Claude-User":["partial","*",2,2],"Claude-SearchBot":["partial","*",2,2],"anthropic-ai":["partial","*",2,2],"PerplexityBot":["partial","*",2,2],"Perplexity-User":["partial","*",2,2],"Google-Extended":["partial","*",2,2],"Applebot-Extended":["partial","*",2,2],"CCBot":["partial","*",2,2],"Bytespider":["partial","*",2,2],"Amazonbot":["partial","*",2,2],"meta-externalagent":["partial","*",2,2],"cohere-ai":["partial","*",2,2],"Diffbot":["partial","*",2,2],"ContextFetchBot":["partial","*",2,2]},"llms":{"status":"missing","http":404,"bytes":12085},"home":{"status":"ok","http":200,"url":"https://hatena.blog/","html_tokens":78321,"markdown_tokens":1778,"pruned_tokens":145}},
{"rank":1722,"domain":"channelnewsasia.com","origin":"https://channelnewsasia.com","robots":{"status":"ok","http":200,"url":"https://www.channelnewsasia.com/robots.txt","bytes":2388,"groups":1},"bots":{"GPTBot":["partial","*",29,25],"ChatGPT-User":["partial","*",29,25],"OAI-SearchBot":["partial","*",29,25],"ClaudeBot":["partial","*",29,25],"Claude-User":["partial","*",29,25],"Claude-SearchBot":["partial","*",29,25],"anthropic-ai":["partial","*",29,25],"PerplexityBot":["partial","*",29,25],"Perplexity-User":["partial","*",29,25],"Google-Extended":["partial","*",29,25],"Applebot-Extended":["partial","*",29,25],"CCBot":["partial","*",29,25],"Bytespider":["partial","*",29,25],"Amazonbot":["partial","*",29,25],"meta-externalagent":["partial","*",29,25],"cohere-ai":["partial","*",29,25],"Diffbot":["partial","*",29,25],"ContextFetchBot":["partial","*",29,25]},"llms":{"status":"missing","http":404,"bytes":409},"home":{"status":"ok","http":200,"url":"https://www.channelnewsasia.com/","html_tokens":163743,"markdown_tokens":146,"pruned_tokens":146}},
{"rank":1723,"domain":"ucsc.edu","origin":"https://ucsc.edu","robots":{"status":"refused","http":403,"url":"https://www.ucsc.edu/robots.txt","bytes":5571,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":1724,"domain":"cdn.prod.website-files.com","origin":"https://cdn.prod.website-files.com","robots":{"status":"refused","http":403,"url":"https://cdn.prod.website-files.com/robots.txt","bytes":243,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":1725,"domain":"unodc.org","origin":"https://unodc.org","robots":{"status":"ok","http":200,"url":"https://www.unodc.org/robots.txt","bytes":268,"groups":1},"bots":{"GPTBot":["partial","*",10,0],"ChatGPT-User":["partial","*",10,0],"OAI-SearchBot":["partial","*",10,0],"ClaudeBot":["partial","*",10,0],"Claude-User":["partial","*",10,0],"Claude-SearchBot":["partial","*",10,0],"anthropic-ai":["partial","*",10,0],"PerplexityBot":["partial","*",10,0],"Perplexity-User":["partial","*",10,0],"Google-Extended":["partial","*",10,0],"Applebot-Extended":["partial","*",10,0],"CCBot":["partial","*",10,0],"Bytespider":["partial","*",10,0],"Amazonbot":["partial","*",10,0],"meta-externalagent":["partial","*",10,0],"cohere-ai":["partial","*",10,0],"Diffbot":["partial","*",10,0],"ContextFetchBot":["partial","*",10,0]},"llms":{"status":"missing","http":404,"bytes":236},"home":{"status":"ok","http":200,"url":"https://www.unodc.org/","html_tokens":14120,"markdown_tokens":201,"pruned_tokens":175}},
{"rank":1726,"domain":"radissonhotels.com","origin":"https://radissonhotels.com","robots":{"status":"refused","http":403,"url":"https://www.radissonhotels.com/robots.txt","bytes":15515,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":1727,"domain":"lbl.gov","origin":"https://lbl.gov","robots":{"status":"ok","http":200,"url":"https://www.lbl.gov/robots.txt","bytes":59,"groups":1},"bots":{"GPTBot":["partial","*",2,0],"ChatGPT-User":["partial","*",2,0],"OAI-SearchBot":["partial","*",2,0],"ClaudeBot":["partial","*",2,0],"Claude-User":["partial","*",2,0],"Claude-SearchBot":["partial","*",2,0],"anthropic-ai":["partial","*",2,0],"PerplexityBot":["partial","*",2,0],"Perplexity-User":["partial","*",2,0],"Google-Extended":["partial","*",2,0],"Applebot-Extended":["partial","*",2,0],"CCBot":["partial","*",2,0],"Bytespider":["partial","*",2,0],"Amazonbot":["partial","*",2,0],"meta-externalagent":["partial","*",2,0],"cohere-ai":["partial","*",2,0],"Diffbot":["partial","*",2,0],"ContextFetchBot":["partial","*",2,0]},"llms":{"status":"missing","http":404,"bytes":60},"home":{"status":"ok","http":200,"url":"https://www.lbl.gov/","html_tokens":17597,"markdown_tokens":94,"pruned_tokens":94}},
{"rank":1728,"domain":"suntimes.com","origin":"https://suntimes.com","robots":{"status":"ok","http":200,"url":"https://chicago.suntimes.com/robots.txt","bytes":1985,"groups":11},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["partial","*",5,0],"OAI-SearchBot":["partial","*",5,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",5,0],"Claude-SearchBot":["partial","*",5,0],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["partial","*",5,0],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["partial","*",5,0],"meta-externalagent":["allowed","named",0,1],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",5,0]},"llms":{"status":"missing","http":404,"bytes":607016},"home":{"status":"ok","http":200,"url":"https://chicago.suntimes.com/","html_tokens":297601,"markdown_tokens":709,"pruned_tokens":126}},
{"rank":1729,"domain":"4shared.com","origin":"https://4shared.com","robots":{"status":"ok","http":200,"url":"https://www.4shared.com/robots.txt","bytes":79,"groups":1},"bots":{"GPTBot":["partial","*",1,0],"ChatGPT-User":["partial","*",1,0],"OAI-SearchBot":["partial","*",1,0],"ClaudeBot":["partial","*",1,0],"Claude-User":["partial","*",1,0],"Claude-SearchBot":["partial","*",1,0],"anthropic-ai":["partial","*",1,0],"PerplexityBot":["partial","*",1,0],"Perplexity-User":["partial","*",1,0],"Google-Extended":["partial","*",1,0],"Applebot-Extended":["partial","*",1,0],"CCBot":["partial","*",1,0],"Bytespider":["partial","*",1,0],"Amazonbot":["partial","*",1,0],"meta-externalagent":["partial","*",1,0],"cohere-ai":["partial","*",1,0],"Diffbot":["partial","*",1,0],"ContextFetchBot":["partial","*",1,0]},"llms":{"status":"html","http":200,"bytes":54187},"home":{"status":"ok","http":200,"url":"https://www.4shared.com","html_tokens":13397,"markdown_tokens":71,"pruned_tokens":71}},
{"rank":1730,"domain":"utah.gov","origin":"https://utah.gov","robots":{"status":"html","http":200,"url":"https://www.utah.gov/index.html","bytes":15767,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":1731,"domain":"ubisoft.com","origin":"https://ubisoft.com","robots":{"status":"ok","http":200,"url":"https://www.ubisoft.com/robots.txt","bytes":318,"groups":1},"bots":{"GPTBot":["partial","*",5,0],"ChatGPT-User":["partial","*",5,0],"OAI-SearchBot":["partial","*",5,0],"ClaudeBot":["partial","*",5,0],"Claude-User":["partial","*",5,0],"Claude-SearchBot":["partial","*",5,0],"anthropic-ai":["partial","*",5,0],"PerplexityBot":["partial","*",5,0],"Perplexity-User":["partial","*",5,0],"Google-Extended":["partial","*",5,0],"Applebot-Extended":["partial","*",5,0],"CCBot":["partial","*",5,0],"Bytespider":["partial","*",5,0],"Amazonbot":["partial","*",5,0],"meta-externalagent":["partial","*",5,0],"cohere-ai":["partial","*",5,0],"Diffbot":["partial","*",5,0],"ContextFetchBot":["partial","*",5,0]},"llms":{"status":"html","http":200,"bytes":548368},"home":{"status":"ok","http":200,"url":"https://www.ubisoft.com/en-us","html_tokens":136986,"markdown_tokens":447,"pruned_tokens":22}},
{"rank":1732,"domain":"nbc.com","origin":"https://nbc.com","robots":{"status":"ok","http":200,"url":"https://www.nbc.com/robots.txt","bytes":6732,"groups":1},"bots":{"GPTBot":["partial","*",198,0],"ChatGPT-User":["partial","*",198,0],"OAI-SearchBot":["partial","*",198,0],"ClaudeBot":["partial","*",198,0],"Claude-User":["partial","*",198,0],"Claude-SearchBot":["partial","*",198,0],"anthropic-ai":["partial","*",198,0],"PerplexityBot":["partial","*",198,0],"Perplexity-User":["partial","*",198,0],"Google-Extended":["partial","*",198,0],"Applebot-Extended":["partial","*",198,0],"CCBot":["partial","*",198,0],"Bytespider":["partial","*",198,0],"Amazonbot":["partial","*",198,0],"meta-externalagent":["partial","*",198,0],"cohere-ai":["partial","*",198,0],"Diffbot":["partial","*",198,0],"ContextFetchBot":["partial","*",198,0]},"llms":{"status":"http_error","http":503,"bytes":893405},"home":{"status":"ok","http":200,"url":"https://www.nbc.com/","html_tokens":128207,"markdown_tokens":82,"pruned_tokens":82}},
{"rank":1733,"domain":"tutsplus.com","origin":"https://tutsplus.com","robots":{"status":"ok","http":200,"url":"https://tutsplus.com/robots.txt","bytes":355,"groups":2},"bots":{"GPTBot":["partial","*",9,0],"ChatGPT-User":["partial","*",9,0],"OAI-SearchBot":["partial","*",9,0],"ClaudeBot":["partial","*",9,0],"Claude-User":["partial","*",9,0],"Claude-SearchBot":["partial","*",9,0],"anthropic-ai":["partial","*",9,0],"PerplexityBot":["partial","*",9,0],"Perplexity-User":["partial","*",9,0],"Google-Extended":["partial","*",9,0],"Applebot-Extended":["partial","*",9,0],"CCBot":["blocked","named",1,0],"Bytespider":["partial","*",9,0],"Amazonbot":["partial","*",9,0],"meta-externalagent":["partial","*",9,0],"cohere-ai":["partial","*",9,0],"Diffbot":["partial","*",9,0],"ContextFetchBot":["partial","*",9,0]},"llms":{"status":"html","http":200,"bytes":159830},"home":{"status":"ok","http":200,"url":"https://tutsplus.com/","html_tokens":39940,"markdown_tokens":424,"pruned_tokens":130}},
{"rank":1734,"domain":"dynadot.com","origin":"https://dynadot.com","robots":{"status":"ok","http":200,"url":"https://www.dynadot.com/robots.txt","bytes":30426,"groups":1},"bots":{"GPTBot":["partial","*",807,0],"ChatGPT-User":["partial","*",807,0],"OAI-SearchBot":["partial","*",807,0],"ClaudeBot":["partial","*",807,0],"Claude-User":["partial","*",807,0],"Claude-SearchBot":["partial","*",807,0],"anthropic-ai":["partial","*",807,0],"PerplexityBot":["partial","*",807,0],"Perplexity-User":["partial","*",807,0],"Google-Extended":["partial","*",807,0],"Applebot-Extended":["partial","*",807,0],"CCBot":["partial","*",807,0],"Bytespider":["partial","*",807,0],"Amazonbot":["partial","*",807,0],"meta-externalagent":["partial","*",807,0],"cohere-ai":["partial","*",807,0],"Diffbot":["partial","*",807,0],"ContextFetchBot":["partial","*",807,0]},"llms":{"status":"present","http":200,"bytes":6460},"home":{"status":"ok","http":200,"url":"https://www.dynadot.com/","html_tokens":84628,"markdown_tokens":1739,"pruned_tokens":319}},
{"rank":1735,"domain":"consumerfinance.gov","origin":"https://consumerfinance.gov","robots":{"status":"ok","http":200,"url":"https://www.consumerfinance.gov/robots.txt","bytes":589,"groups":1},"bots":{"GPTBot":["partial","*",14,0],"ChatGPT-User":["partial","*",14,0],"OAI-SearchBot":["partial","*",14,0],"ClaudeBot":["partial","*",14,0],"Claude-User":["partial","*",14,0],"Claude-SearchBot":["partial","*",14,0],"anthropic-ai":["partial","*",14,0],"PerplexityBot":["partial","*",14,0],"Perplexity-User":["partial","*",14,0],"Google-Extended":["partial","*",14,0],"Applebot-Extended":["partial","*",14,0],"CCBot":["partial","*",14,0],"Bytespider":["partial","*",14,0],"Amazonbot":["partial","*",14,0],"meta-externalagent":["partial","*",14,0],"cohere-ai":["partial","*",14,0],"Diffbot":["partial","*",14,0],"ContextFetchBot":["partial","*",14,0]},"llms":{"status":"missing","http":404,"bytes":136177},"home":{"status":"ok","http":200,"url":"https://www.consumerfinance.gov/","html_tokens":36660,"markdown_tokens":179,"pruned_tokens":53}},
{"rank":1736,"domain":"hotels.com","origin":"https://hotels.com","robots":{"status":"unreachable","http":null,"url":"https://hotels.com/robots.txt","bytes":0,"groups":0,"error":"<StreamReset stream_id:1, error_code:2, remote_reset:True>"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":1737,"domain":"coca-cola.com","origin":"https://coca-cola.com","robots":{"status":"ok","http":200,"url":"https://www.coca-cola.com/robots.txt","bytes":184,"groups":1},"bots":{"GPTBot":["partial","*",4,0],"ChatGPT-User":["partial","*",4,0],"OAI-SearchBot":["partial","*",4,0],"ClaudeBot":["partial","*",4,0],"Claude-User":["partial","*",4,0],"Claude-SearchBot":["partial","*",4,0],"anthropic-ai":["partial","*",4,0],"PerplexityBot":["partial","*",4,0],"Perplexity-User":["partial","*",4,0],"Google-Extended":["partial","*",4,0],"Applebot-Extended":["partial","*",4,0],"CCBot":["partial","*",4,0],"Bytespider":["partial","*",4,0],"Amazonbot":["partial","*",4,0],"meta-externalagent":["partial","*",4,0],"cohere-ai":["partial","*",4,0],"Diffbot":["partial","*",4,0],"ContextFetchBot":["partial","*",4,0]},"llms":{"status":"missing","http":404,"bytes":77290},"home":{"status":"ok","http":200,"url":"https://www.coca-cola.com/us/en","html_tokens":44223,"markdown_tokens":243,"pruned_tokens":105}},
{"rank":1738,"domain":"proton.me","origin":"https://proton.me","robots":{"status":"ok","http":200,"url":"https://proton.me/robots.txt","bytes":181,"groups":1},"bots":{"GPTBot":["partial","*",1,3],"ChatGPT-User":["partial","*",1,3],"OAI-SearchBot":["partial","*",1,3],"ClaudeBot":["partial","*",1,3],"Claude-User":["partial","*",1,3],"Claude-SearchBot":["partial","*",1,3],"anthropic-ai":["partial","*",1,3],"PerplexityBot":["partial","*",1,3],"Perplexity-User":["partial","*",1,3],"Google-Extended":["partial","*",1,3],"Applebot-Extended":["partial","*",1,3],"CCBot":["partial","*",1,3],"Bytespider":["partial","*",1,3],"Amazonbot":["partial","*",1,3],"meta-externalagent":["partial","*",1,3],"cohere-ai":["partial","*",1,3],"Diffbot":["partial","*",1,3],"ContextFetchBot":["partial","*",1,3]},"llms":{"status":"missing","http":404,"bytes":240245},"home":{"status":"ok","http":200,"url":"https://proton.me/","html_tokens":106584,"markdown_tokens":863,"pruned_tokens":423}},
{"rank":1739,"domain":"allrecipes.com","origin":"https://allrecipes.com","robots":{"status":"ok","http":200,"url":"https://www.allrecipes.com/robots.txt","bytes":3490,"groups":6},"bots":{"GPTBot":["partial","named",1,0],"ChatGPT-User":["partial","named",1,0],"OAI-SearchBot":["partial","named",1,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["blocked","named",1,0],"Claude-SearchBot":["partial","named",1,2],"anthropic-ai":["partial","named",1,2],"PerplexityBot":["partial","named",1,2],"Perplexity-User":["blocked","named",1,0],"Google-Extended":["partial","named",1,2],"Applebot-Extended":["blocked","named",1,0],"CCBot":["partial","named",1,2],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["partial","named",1,2],"cohere-ai":["partial","named",1,2],"Diffbot":["partial","*",3,0],"ContextFetchBot":["partial","*",3,0]},"llms":{"status":"http_error","http":402,"bytes":612},"home":{"status":"http_error","http":402,"url":"https://www.allrecipes.com/"}},
{"rank":1740,"domain":"buzzfeednews.com","origin":"https://buzzfeednews.com","robots":{"status":"ok","http":200,"url":"https://www.buzzfeednews.com/robots.txt","bytes":3219,"groups":38},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["blocked","named",1,0],"OAI-SearchBot":["blocked","named",1,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["blocked","named",1,0],"Claude-SearchBot":["blocked","named",1,0],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["blocked","named",1,0],"Google-Extended":["blocked","named",2,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",20,0]},"llms":{"status":"missing","http":404,"bytes":7029},"home":{"status":"ok","http":200,"url":"https://www.buzzfeednews.com/","html_tokens":127170,"markdown_tokens":805,"pruned_tokens":163}},
{"rank":1741,"domain":"stuff.co.nz","origin":"https://stuff.co.nz","robots":{"status":"ok","http":200,"url":"https://www.stuff.co.nz/robots.txt","bytes":1620,"groups":23},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["blocked","named",1,0],"OAI-SearchBot":["blocked","named",1,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",12,0],"Claude-SearchBot":["partial","*",12,0],"anthropic-ai":["partial","*",12,0],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["partial","*",12,0],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["partial","*",12,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",12,0]},"llms":{"status":"html","http":200,"bytes":64887},"home":{"status":"ok","http":200,"url":"https://www.stuff.co.nz/","html_tokens":16438,"markdown_tokens":0,"pruned_tokens":0}},
{"rank":1742,"domain":"fl.us","origin":"https://fl.us","robots":{"status":"unreachable","http":null,"url":"https://fl.us/robots.txt","bytes":0,"groups":0,"error":"cannot resolve fl.us"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":1743,"domain":"swissinfo.ch","origin":"https://swissinfo.ch","robots":{"status":"ok","http":200,"url":"https://www.swissinfo.ch/robots.txt","bytes":17673,"groups":1},"bots":{"GPTBot":["partial","*",2,0],"ChatGPT-User":["partial","*",2,0],"OAI-SearchBot":["partial","*",2,0],"ClaudeBot":["partial","*",2,0],"Claude-User":["partial","*",2,0],"Claude-SearchBot":["partial","*",2,0],"anthropic-ai":["partial","*",2,0],"PerplexityBot":["partial","*",2,0],"Perplexity-User":["partial","*",2,0],"Google-Extended":["partial","*",2,0],"Applebot-Extended":["partial","*",2,0],"CCBot":["partial","*",2,0],"Bytespider":["partial","*",2,0],"Amazonbot":["partial","*",2,0],"meta-externalagent":["partial","*",2,0],"cohere-ai":["partial","*",2,0],"Diffbot":["partial","*",2,0],"ContextFetchBot":["partial","*",2,0]},"llms":{"status":"missing","http":404,"bytes":1965711},"home":{"status":"ok","http":200,"url":"https://www.swissinfo.ch/eng/","html_tokens":560410,"markdown_tokens":1141,"pruned_tokens":158}},
{"rank":1744,"domain":"alicdn.com","origin":"https://alicdn.com","robots":{"status":"timeout","http":null,"url":"https://alicdn.com/robots.txt","bytes":0,"groups":0,"error":"ConnectTimeout"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":1745,"domain":"wolframalpha.com","origin":"https://wolframalpha.com","robots":{"status":"ok","http":200,"url":"https://www.wolframalpha.com/robots.txt","bytes":142,"groups":4},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["blocked","named",1,0],"OAI-SearchBot":["partial","*",1,0],"ClaudeBot":["partial","*",1,0],"Claude-User":["partial","*",1,0],"Claude-SearchBot":["partial","*",1,0],"anthropic-ai":["partial","*",1,0],"PerplexityBot":["partial","*",1,0],"Perplexity-User":["partial","*",1,0],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["partial","*",1,0],"CCBot":["partial","*",1,0],"Bytespider":["partial","*",1,0],"Amazonbot":["partial","*",1,0],"meta-externalagent":["partial","*",1,0],"cohere-ai":["partial","*",1,0],"Diffbot":["partial","*",1,0],"ContextFetchBot":["partial","*",1,0]},"llms":{"status":"missing","http":404,"bytes":72647},"home":{"status":"ok","http":200,"url":"https://www.wolframalpha.com/","html_tokens":45314,"markdown_tokens":51,"pruned_tokens":51}},
{"rank":1746,"domain":"blackberry.com","origin":"https://blackberry.com","robots":{"status":"ok","http":200,"url":"https://www.blackberry.com/robots.txt","bytes":910,"groups":2},"bots":{"GPTBot":["partial","*",21,0],"ChatGPT-User":["partial","*",21,0],"OAI-SearchBot":["partial","*",21,0],"ClaudeBot":["partial","*",21,0],"Claude-User":["partial","*",21,0],"Claude-SearchBot":["partial","*",21,0],"anthropic-ai":["partial","*",21,0],"PerplexityBot":["partial","*",21,0],"Perplexity-User":["partial","*",21,0],"Google-Extended":["partial","*",21,0],"Applebot-Extended":["partial","*",21,0],"CCBot":["partial","*",21,0],"Bytespider":["partial","*",21,0],"Amazonbot":["partial","*",21,0],"meta-externalagent":["partial","*",21,0],"cohere-ai":["partial","*",21,0],"Diffbot":["partial","*",21,0],"ContextFetchBot":["partial","*",21,0]},"llms":{"status":"missing","http":404,"bytes":98},"home":{"status":"disallowed","error":"robots.txt of web.blackberry.com not readable (html)"}},
{"rank":1747,"domain":"gambleaware.org","origin":"https://gambleaware.org","robots":{"status":"ok","http":200,"url":"https://www.gambleaware.org/robots.txt/","bytes":71,"groups":1},"bots":{"GPTBot":["allowed","*",0,1],"ChatGPT-User":["allowed","*",0,1],"OAI-SearchBot":["allowed","*",0,1],"ClaudeBot":["allowed","*",0,1],"Claude-User":["allowed","*",0,1],"Claude-SearchBot":["allowed","*",0,1],"anthropic-ai":["allowed","*",0,1],"PerplexityBot":["allowed","*",0,1],"Perplexity-User":["allowed","*",0,1],"Google-Extended":["allowed","*",0,1],"Applebot-Extended":["allowed","*",0,1],"CCBot":["allowed","*",0,1],"Bytespider":["allowed","*",0,1],"Amazonbot":["allowed","*",0,1],"meta-externalagent":["allowed","*",0,1],"cohere-ai":["allowed","*",0,1],"Diffbot":["allowed","*",0,1],"ContextFetchBot":["allowed","*",0,1]},"llms":{"status":"missing","http":404,"bytes":27242},"home":{"status":"ok","http":200,"url":"https://www.gambleaware.org/","html_tokens":11228,"markdown_tokens":281,"pruned_tokens":130}},
{"rank":1748,"domain":"starbucks.com","origin":"https://starbucks.com","robots":{"status":"ok","http":200,"url":"https://www.starbucks.com/robots.txt","bytes":186,"groups":1},"bots":{"GPTBot":["allowed","*",0,5],"ChatGPT-User":["allowed","*",0,5],"OAI-SearchBot":["allowed","*",0,5],"ClaudeBot":["allowed","*",0,5],"Claude-User":["allowed","*",0,5],"Claude-SearchBot":["allowed","*",0,5],"anthropic-ai":["allowed","*",0,5],"PerplexityBot":["allowed","*",0,5],"Perplexity-User":["allowed","*",0,5],"Google-Extended":["allowed","*",0,5],"Applebot-Extended":["allowed","*",0,5],"CCBot":["allowed","*",0,5],"Bytespider":["allowed","*",0,5],"Amazonbot":["allowed","*",0,5],"meta-externalagent":["allowed","*",0,5],"cohere-ai":["allowed","*",0,5],"Diffbot":["allowed","*",0,5],"ContextFetchBot":["allowed","*",0,5]},"llms":{"status":"missing","http":404,"bytes":23997},"home":{"status":"ok","http":200,"url":"https://www.starbucks.com/","html_tokens":8554,"markdown_tokens":52,"pruned_tokens":52}},
{"rank":1749,"domain":"paris.fr","origin":"https://paris.fr","robots":{"status":"ok","http":200,"url":"https://www.paris.fr/robots.txt","bytes":143,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":153},"home":{"status":"ok","http":200,"url":"https://www.paris.fr/","html_tokens":87394,"markdown_tokens":468,"pruned_tokens":167}},
{"rank":1750,"domain":"documentcloud.org","origin":"https://documentcloud.org","robots":{"status":"refused","http":403,"url":"https://documentcloud.org/robots.txt","bytes":5197,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":1751,"domain":"maryland.gov","origin":"https://maryland.gov","robots":{"status":"ok","http":200,"url":"https://www.maryland.gov/robots.txt","bytes":2005,"groups":1},"bots":{"GPTBot":["partial","*",33,18],"ChatGPT-User":["partial","*",33,18],"OAI-SearchBot":["partial","*",33,18],"ClaudeBot":["partial","*",33,18],"Claude-User":["partial","*",33,18],"Claude-SearchBot":["partial","*",33,18],"anthropic-ai":["partial","*",33,18],"PerplexityBot":["partial","*",33,18],"Perplexity-User":["partial","*",33,18],"Google-Extended":["partial","*",33,18],"Applebot-Extended":["partial","*",33,18],"CCBot":["partial","*",33,18],"Bytespider":["partial","*",33,18],"Amazonbot":["partial","*",33,18],"meta-externalagent":["partial","*",33,18],"cohere-ai":["partial","*",33,18],"Diffbot":["partial","*",33,18],"ContextFetchBot":["partial","*",33,18]},"llms":{"status":"present","http":200,"bytes":3216},"home":{"status":"ok","http":200,"url":"https://www.maryland.gov/","html_tokens":27002,"markdown_tokens":241,"pruned_tokens":92}},
{"rank":1752,"domain":"theintercept.com","origin":"https://theintercept.com","robots":{"status":"ok","http":200,"url":"https://theintercept.com/robots.txt","bytes":517,"groups":9},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["blocked","named",1,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["blocked","named",1,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"missing","http":404,"bytes":10768},"home":{"status":"ok","http":200,"url":"https://theintercept.com/","html_tokens":36790,"markdown_tokens":32,"pruned_tokens":32}},
{"rank":1753,"domain":"www.gov.pl","origin":"https://www.gov.pl","robots":{"status":"ok","http":200,"url":"https://www.gov.pl/static/code/robots.txt","bytes":23,"groups":1},"bots":{"GPTBot":["allowed","*",0,1],"ChatGPT-User":["allowed","*",0,1],"OAI-SearchBot":["allowed","*",0,1],"ClaudeBot":["allowed","*",0,1],"Claude-User":["allowed","*",0,1],"Claude-SearchBot":["allowed","*",0,1],"anthropic-ai":["allowed","*",0,1],"PerplexityBot":["allowed","*",0,1],"Perplexity-User":["allowed","*",0,1],"Google-Extended":["allowed","*",0,1],"Applebot-Extended":["allowed","*",0,1],"CCBot":["allowed","*",0,1],"Bytespider":["allowed","*",0,1],"Amazonbot":["allowed","*",0,1],"meta-externalagent":["allowed","*",0,1],"cohere-ai":["allowed","*",0,1],"Diffbot":["allowed","*",0,1],"ContextFetchBot":["allowed","*",0,1]},"llms":{"status":"html","http":200,"bytes":40564},"home":{"status":"ok","http":200,"url":"https://www.gov.pl/","html_tokens":10038,"markdown_tokens":1205,"pruned_tokens":175}},
{"rank":1754,"domain":"louvre.fr","origin":"https://louvre.fr","robots":{"status":"ok","http":200,"url":"https://www.louvre.fr/robots.txt","bytes":1067,"groups":11},"bots":{"GPTBot":["partial","named",4,0],"ChatGPT-User":["partial","named",4,0],"OAI-SearchBot":["partial","*",2,0],"ClaudeBot":["partial","named",4,0],"Claude-User":["partial","*",2,0],"Claude-SearchBot":["partial","*",2,0],"anthropic-ai":["partial","named",4,0],"PerplexityBot":["partial","*",2,0],"Perplexity-User":["partial","*",2,0],"Google-Extended":["partial","named",4,0],"Applebot-Extended":["partial","*",2,0],"CCBot":["partial","named",4,0],"Bytespider":["partial","named",4,0],"Amazonbot":["partial","named",4,0],"meta-externalagent":["partial","*",2,0],"cohere-ai":["partial","*",2,0],"Diffbot":["partial","*",2,0],"ContextFetchBot":["partial","*",2,0]},"llms":{"status":"missing","http":404,"bytes":118851},"home":{"status":"ok","http":200,"url":"https://www.louvre.fr/","html_tokens":98408,"markdown_tokens":119,"pruned_tokens":57}},
{"rank":1755,"domain":"porsche.com","origin":"https://porsche.com","robots":{"status":"ok","http":200,"url":"https://www.porsche.com/robots.txt","bytes":4639,"groups":8},"bots":{"GPTBot":["partial","*",64,0],"ChatGPT-User":["partial","*",64,0],"OAI-SearchBot":["partial","*",64,0],"ClaudeBot":["partial","*",64,0],"Claude-User":["partial","*",64,0],"Claude-SearchBot":["partial","*",64,0],"anthropic-ai":["partial","*",64,0],"PerplexityBot":["partial","*",64,0],"Perplexity-User":["partial","*",64,0],"Google-Extended":["partial","*",64,0],"Applebot-Extended":["partial","*",64,0],"CCBot":["partial","*",64,0],"Bytespider":["partial","*",64,0],"Amazonbot":["partial","*",64,0],"meta-externalagent":["partial","*",64,0],"cohere-ai":["partial","*",64,0],"Diffbot":["partial","*",64,0],"ContextFetchBot":["partial","*",64,0]},"llms":{"status":"missing","http":404,"bytes":93219},"home":{"status":"ok","http":200,"url":"https://www.porsche.com/usa/?cs_redirect=1","html_tokens":88751,"markdown_tokens":83,"pruned_tokens":83}},
{"rank":1756,"domain":"cnki.net","origin":"https://cnki.net","robots":{"status":"ok","http":200,"url":"https://cnki.net/robots.txt","bytes":27,"groups":1},"bots":{"GPTBot":["blocked","*",1,0],"ChatGPT-User":["blocked","*",1,0],"OAI-SearchBot":["blocked","*",1,0],"ClaudeBot":["blocked","*",1,0],"Claude-User":["blocked","*",1,0],"Claude-SearchBot":["blocked","*",1,0],"anthropic-ai":["blocked","*",1,0],"PerplexityBot":["blocked","*",1,0],"Perplexity-User":["blocked","*",1,0],"Google-Extended":["blocked","*",1,0],"Applebot-Extended":["blocked","*",1,0],"CCBot":["blocked","*",1,0],"Bytespider":["blocked","*",1,0],"Amazonbot":["blocked","*",1,0],"meta-externalagent":["blocked","*",1,0],"cohere-ai":["blocked","*",1,0],"Diffbot":["blocked","*",1,0],"ContextFetchBot":["blocked","*",1,0]},"llms":{"status":"disallowed"},"home":{"status":"disallowed"}},
{"rank":1757,"domain":"themeansar.com","origin":"https://themeansar.com","robots":{"status":"ok","http":200,"url":"https://themeansar.com/robots.txt","bytes":193,"groups":2},"bots":{"GPTBot":["partial","*",2,1],"ChatGPT-User":["partial","*",2,1],"OAI-SearchBot":["partial","*",2,1],"ClaudeBot":["partial","*",2,1],"Claude-User":["partial","*",2,1],"Claude-SearchBot":["partial","*",2,1],"anthropic-ai":["partial","*",2,1],"PerplexityBot":["partial","*",2,1],"Perplexity-User":["partial","*",2,1],"Google-Extended":["partial","*",2,1],"Applebot-Extended":["partial","*",2,1],"CCBot":["partial","*",2,1],"Bytespider":["partial","*",2,1],"Amazonbot":["partial","*",2,1],"meta-externalagent":["partial","*",2,1],"cohere-ai":["partial","*",2,1],"Diffbot":["partial","*",2,1],"ContextFetchBot":["partial","*",2,1]},"llms":{"status":"other_text","http":200,"bytes":2324},"home":{"status":"ok","http":200,"url":"https://themeansar.com/","html_tokens":86803,"markdown_tokens":229,"pruned_tokens":98}},
{"rank":1758,"domain":"blizzard.com","origin":"https://blizzard.com","robots":{"status":"refused","http":403,"url":"https://blizzard.com/robots.txt","bytes":118,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":1759,"domain":"thenation.com","origin":"https://thenation.com","robots":{"status":"ok","http":200,"url":"https://www.thenation.com/robots.txt","bytes":560,"groups":1},"bots":{"GPTBot":["partial","*",14,0],"ChatGPT-User":["partial","*",14,0],"OAI-SearchBot":["partial","*",14,0],"ClaudeBot":["partial","*",14,0],"Claude-User":["partial","*",14,0],"Claude-SearchBot":["partial","*",14,0],"anthropic-ai":["partial","*",14,0],"PerplexityBot":["partial","*",14,0],"Perplexity-User":["partial","*",14,0],"Google-Extended":["partial","*",14,0],"Applebot-Extended":["partial","*",14,0],"CCBot":["partial","*",14,0],"Bytespider":["partial","*",14,0],"Amazonbot":["partial","*",14,0],"meta-externalagent":["partial","*",14,0],"cohere-ai":["partial","*",14,0],"Diffbot":["partial","*",14,0],"ContextFetchBot":["partial","*",14,0]},"llms":{"status":"timeout","error":"timed out after 10 s"},"home":{"status":"ok","http":200,"url":"https://www.thenation.com/","html_tokens":71220,"markdown_tokens":898,"pruned_tokens":140}},
{"rank":1760,"domain":"openlibrary.org","origin":"https://openlibrary.org","robots":{"status":"ok","http":200,"url":"https://openlibrary.org/robots.txt","bytes":870,"groups":3},"bots":{"GPTBot":["partial","*",16,0],"ChatGPT-User":["partial","*",16,0],"OAI-SearchBot":["partial","*",16,0],"ClaudeBot":["allowed","named",0,0],"Claude-User":["partial","*",16,0],"Claude-SearchBot":["partial","*",16,0],"anthropic-ai":["allowed","named",0,0],"PerplexityBot":["partial","*",16,0],"Perplexity-User":["partial","*",16,0],"Google-Extended":["partial","*",16,0],"Applebot-Extended":["partial","*",16,0],"CCBot":["partial","*",16,0],"Bytespider":["partial","*",16,0],"Amazonbot":["partial","*",16,0],"meta-externalagent":["allowed","named",0,0],"cohere-ai":["partial","*",16,0],"Diffbot":["partial","*",16,0],"ContextFetchBot":["partial","*",16,0]},"llms":{"status":"missing","http":404,"bytes":42630},"home":{"status":"ok","http":200,"url":"https://openlibrary.org/","html_tokens":23237,"markdown_tokens":211,"pruned_tokens":157}},
{"rank":1761,"domain":"greenend.org.uk","origin":"https://www.greenend.org.uk","robots":{"status":"ok","http":200,"url":"https://www.greenend.org.uk/robots.txt","bytes":32,"groups":1},"bots":{"GPTBot":["partial","*",1,0],"ChatGPT-User":["partial","*",1,0],"OAI-SearchBot":["partial","*",1,0],"ClaudeBot":["partial","*",1,0],"Claude-User":["partial","*",1,0],"Claude-SearchBot":["partial","*",1,0],"anthropic-ai":["partial","*",1,0],"PerplexityBot":["partial","*",1,0],"Perplexity-User":["partial","*",1,0],"Google-Extended":["partial","*",1,0],"Applebot-Extended":["partial","*",1,0],"CCBot":["partial","*",1,0],"Bytespider":["partial","*",1,0],"Amazonbot":["partial","*",1,0],"meta-externalagent":["partial","*",1,0],"cohere-ai":["partial","*",1,0],"Diffbot":["partial","*",1,0],"ContextFetchBot":["partial","*",1,0]},"llms":{"status":"missing","http":404,"bytes":2325},"home":{"status":"ok","http":200,"url":"https://www.greenend.org.uk/","html_tokens":1498,"markdown_tokens":140,"pruned_tokens":103}},
{"rank":1762,"domain":"vercel.com","origin":"https://vercel.com","robots":{"status":"ok","http":200,"url":"https://vercel.com/robots.txt","bytes":629,"groups":1},"bots":{"GPTBot":["partial","*",10,6],"ChatGPT-User":["partial","*",10,6],"OAI-SearchBot":["partial","*",10,6],"ClaudeBot":["partial","*",10,6],"Claude-User":["partial","*",10,6],"Claude-SearchBot":["partial","*",10,6],"anthropic-ai":["partial","*",10,6],"PerplexityBot":["partial","*",10,6],"Perplexity-User":["partial","*",10,6],"Google-Extended":["partial","*",10,6],"Applebot-Extended":["partial","*",10,6],"CCBot":["partial","*",10,6],"Bytespider":["partial","*",10,6],"Amazonbot":["partial","*",10,6],"meta-externalagent":["partial","*",10,6],"cohere-ai":["partial","*",10,6],"Diffbot":["partial","*",10,6],"ContextFetchBot":["partial","*",10,6]},"llms":{"status":"present","http":200,"bytes":5240},"home":{"status":"ok","http":200,"url":"https://vercel.com/","html_tokens":156084,"markdown_tokens":163,"pruned_tokens":163}},
{"rank":1763,"domain":"godaddysites.com","origin":"https://godaddysites.com","robots":{"status":"missing","http":404,"url":"https://godaddysites.com/robots.txt","bytes":189878,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":189878},"home":{"status":"http_error","http":404,"url":"https://godaddysites.com/"}},
{"rank":1764,"domain":"gnupg.org","origin":"https://gnupg.org","robots":{"status":"ok","http":200,"url":"https://gnupg.org/robots.txt","bytes":53,"groups":1},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"missing","http":404,"bytes":312},"home":{"status":"ok","http":200,"url":"https://gnupg.org/","html_tokens":4372,"markdown_tokens":1976,"pruned_tokens":352}},
{"rank":1765,"domain":"tudelft.nl","origin":"https://tudelft.nl","robots":{"status":"ok","http":200,"url":"https://www.tudelft.nl/robots.txt","bytes":195,"groups":1},"bots":{"GPTBot":["partial","*",7,1],"ChatGPT-User":["partial","*",7,1],"OAI-SearchBot":["partial","*",7,1],"ClaudeBot":["partial","*",7,1],"Claude-User":["partial","*",7,1],"Claude-SearchBot":["partial","*",7,1],"anthropic-ai":["partial","*",7,1],"PerplexityBot":["partial","*",7,1],"Perplexity-User":["partial","*",7,1],"Google-Extended":["partial","*",7,1],"Applebot-Extended":["partial","*",7,1],"CCBot":["partial","*",7,1],"Bytespider":["partial","*",7,1],"Amazonbot":["partial","*",7,1],"meta-externalagent":["partial","*",7,1],"cohere-ai":["partial","*",7,1],"Diffbot":["partial","*",7,1],"ContextFetchBot":["partial","*",7,1]},"llms":{"status":"missing","http":404,"bytes":99121},"home":{"status":"ok","http":200,"url":"https://www.tudelft.nl/","html_tokens":48782,"markdown_tokens":97,"pruned_tokens":18}},
{"rank":1766,"domain":"thoughtco.com","origin":"https://thoughtco.com","robots":{"status":"ok","http":200,"url":"https://www.thoughtco.com/robots.txt","bytes":3471,"groups":6},"bots":{"GPTBot":["partial","named",1,0],"ChatGPT-User":["partial","named",1,0],"OAI-SearchBot":["partial","named",1,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["blocked","named",1,0],"Claude-SearchBot":["partial","named",1,2],"anthropic-ai":["partial","named",1,2],"PerplexityBot":["partial","named",1,2],"Perplexity-User":["blocked","named",1,0],"Google-Extended":["partial","named",1,2],"Applebot-Extended":["blocked","named",1,0],"CCBot":["partial","named",1,2],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["partial","named",1,2],"cohere-ai":["partial","named",1,2],"Diffbot":["partial","*",2,0],"ContextFetchBot":["partial","*",2,0]},"llms":{"status":"http_error","http":402,"bytes":612},"home":{"status":"http_error","http":402,"url":"https://www.thoughtco.com/"}},
{"rank":1767,"domain":"gmw.cn","origin":"https://gmw.cn","robots":{"status":"ok","http":200,"url":"https://www.gmw.cn/robots.txt","bytes":2345,"groups":1},"bots":{"GPTBot":["partial","*",14,1],"ChatGPT-User":["partial","*",14,1],"OAI-SearchBot":["partial","*",14,1],"ClaudeBot":["partial","*",14,1],"Claude-User":["partial","*",14,1],"Claude-SearchBot":["partial","*",14,1],"anthropic-ai":["partial","*",14,1],"PerplexityBot":["partial","*",14,1],"Perplexity-User":["partial","*",14,1],"Google-Extended":["partial","*",14,1],"Applebot-Extended":["partial","*",14,1],"CCBot":["partial","*",14,1],"Bytespider":["partial","*",14,1],"Amazonbot":["partial","*",14,1],"meta-externalagent":["partial","*",14,1],"cohere-ai":["partial","*",14,1],"Diffbot":["partial","*",14,1],"ContextFetchBot":["partial","*",14,1]},"llms":{"status":"present","http":200,"bytes":19954},"home":{"status":"ok","http":200,"url":"https://www.gmw.cn/","html_tokens":39213,"markdown_tokens":17,"pruned_tokens":17}},
{"rank":1768,"domain":"kit.com","origin":"https://kit.com","robots":{"status":"ok","http":200,"url":"https://kit.com/robots.txt","bytes":2720,"groups":1},"bots":{"GPTBot":["partial","*",5,0],"ChatGPT-User":["partial","*",5,0],"OAI-SearchBot":["partial","*",5,0],"ClaudeBot":["partial","*",5,0],"Claude-User":["partial","*",5,0],"Claude-SearchBot":["partial","*",5,0],"anthropic-ai":["partial","*",5,0],"PerplexityBot":["partial","*",5,0],"Perplexity-User":["partial","*",5,0],"Google-Extended":["partial","*",5,0],"Applebot-Extended":["partial","*",5,0],"CCBot":["partial","*",5,0],"Bytespider":["partial","*",5,0],"Amazonbot":["partial","*",5,0],"meta-externalagent":["partial","*",5,0],"cohere-ai":["partial","*",5,0],"Diffbot":["partial","*",5,0],"ContextFetchBot":["partial","*",5,0]},"llms":{"status":"missing","http":404,"bytes":101340},"home":{"status":"http_error","http":403,"url":"https://kit.com/"}},
{"rank":1769,"domain":"indiatoday.in","origin":"https://indiatoday.in","robots":{"status":"ok","http":200,"url":"https://www.indiatoday.in/robots.txt","bytes":3936,"groups":3},"bots":{"GPTBot":["partial","*",60,1],"ChatGPT-User":["partial","*",60,1],"OAI-SearchBot":["partial","*",60,1],"ClaudeBot":["partial","*",60,1],"Claude-User":["partial","*",60,1],"Claude-SearchBot":["partial","*",60,1],"anthropic-ai":["partial","*",60,1],"PerplexityBot":["partial","*",60,1],"Perplexity-User":["partial","*",60,1],"Google-Extended":["partial","*",60,1],"Applebot-Extended":["partial","*",60,1],"CCBot":["partial","*",60,1],"Bytespider":["partial","*",60,1],"Amazonbot":["partial","*",60,1],"meta-externalagent":["partial","*",60,1],"cohere-ai":["partial","*",60,1],"Diffbot":["partial","*",60,1],"ContextFetchBot":["partial","*",60,1]},"llms":{"status":"missing","http":404,"bytes":581532},"home":{"status":"ok","http":200,"url":"https://www.indiatoday.in/","html_tokens":323637,"markdown_tokens":1547,"pruned_tokens":71}},
{"rank":1770,"domain":"colorado.gov","origin":"https://colorado.gov","robots":{"status":"ok","http":200,"url":"https://www.colorado.gov/robots.txt","bytes":1775,"groups":1},"bots":{"GPTBot":["partial","*",45,0],"ChatGPT-User":["partial","*",45,0],"OAI-SearchBot":["partial","*",45,0],"ClaudeBot":["partial","*",45,0],"Claude-User":["partial","*",45,0],"Claude-SearchBot":["partial","*",45,0],"anthropic-ai":["partial","*",45,0],"PerplexityBot":["partial","*",45,0],"Perplexity-User":["partial","*",45,0],"Google-Extended":["partial","*",45,0],"Applebot-Extended":["partial","*",45,0],"CCBot":["partial","*",45,0],"Bytespider":["partial","*",45,0],"Amazonbot":["partial","*",45,0],"meta-externalagent":["partial","*",45,0],"cohere-ai":["partial","*",45,0],"Diffbot":["partial","*",45,0],"ContextFetchBot":["partial","*",45,0]},"llms":{"status":"http_error","http":403,"bytes":919},"home":{"status":"http_error","http":403,"url":"https://www.colorado.gov/"}},
{"rank":1771,"domain":"britishmuseum.org","origin":"https://britishmuseum.org","robots":{"status":"ok","http":200,"url":"https://britishmuseum.org/robots.txt","bytes":2676,"groups":32},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["allowed","*",0,1],"OAI-SearchBot":["allowed","*",0,1],"ClaudeBot":["blocked","named",1,0],"Claude-User":["allowed","*",0,1],"Claude-SearchBot":["allowed","*",0,1],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["allowed","*",0,1],"Perplexity-User":["allowed","*",0,1],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["allowed","*",0,1]},"llms":{"status":"missing","http":404,"bytes":195},"home":{"status":"timeout","error":"timed out after 15 s"}},
{"rank":1772,"domain":"youradchoices.ca","origin":"https://youradchoices.ca","robots":{"status":"ok","http":200,"url":"https://youradchoices.ca/robots.txt","bytes":1507,"groups":1},"bots":{"GPTBot":["partial","named",27,1],"ChatGPT-User":["partial","*",27,1],"OAI-SearchBot":["partial","*",27,1],"ClaudeBot":["partial","named",27,1],"Claude-User":["partial","*",27,1],"Claude-SearchBot":["partial","*",27,1],"anthropic-ai":["partial","named",27,1],"PerplexityBot":["partial","*",27,1],"Perplexity-User":["partial","*",27,1],"Google-Extended":["partial","named",27,1],"Applebot-Extended":["partial","named",27,1],"CCBot":["partial","named",27,1],"Bytespider":["partial","named",27,1],"Amazonbot":["partial","named",27,1],"meta-externalagent":["partial","named",27,1],"cohere-ai":["partial","named",27,1],"Diffbot":["partial","*",27,1],"ContextFetchBot":["partial","*",27,1]},"llms":{"status":"missing","http":404,"bytes":0},"home":{"status":"ok","http":200,"url":"https://youradchoices.ca/","html_tokens":52113,"markdown_tokens":117,"pruned_tokens":117}},
{"rank":1773,"domain":"catchthemes.com","origin":"https://catchthemes.com","robots":{"status":"ok","http":200,"url":"https://catchthemes.com/robots.txt","bytes":244,"groups":1},"bots":{"GPTBot":["partial","*",2,1],"ChatGPT-User":["partial","*",2,1],"OAI-SearchBot":["partial","*",2,1],"ClaudeBot":["partial","*",2,1],"Claude-User":["partial","*",2,1],"Claude-SearchBot":["partial","*",2,1],"anthropic-ai":["partial","*",2,1],"PerplexityBot":["partial","*",2,1],"Perplexity-User":["partial","*",2,1],"Google-Extended":["partial","*",2,1],"Applebot-Extended":["partial","*",2,1],"CCBot":["partial","*",2,1],"Bytespider":["partial","*",2,1],"Amazonbot":["partial","*",2,1],"meta-externalagent":["partial","*",2,1],"cohere-ai":["partial","*",2,1],"Diffbot":["partial","*",2,1],"ContextFetchBot":["partial","*",2,1]},"llms":{"status":"other_text","http":200,"bytes":7193},"home":{"status":"ok","http":200,"url":"https://catchthemes.com/","html_tokens":43010,"markdown_tokens":1037,"pruned_tokens":112}},
{"rank":1774,"domain":"postimg.cc","origin":"https://postimg.cc","robots":{"status":"ok","http":200,"url":"https://postimg.cc/robots.txt","bytes":0,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":28645},"home":{"status":"ok","http":200,"url":"https://postimages.org/","html_tokens":10448,"markdown_tokens":198,"pruned_tokens":25}},
{"rank":1775,"domain":"googleadservices.com","origin":"https://googleadservices.com","robots":{"status":"missing","http":404,"url":"https://googleadservices.com/robots.txt","bytes":1571,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":1569},"home":{"status":"http_error","http":404,"url":"https://googleadservices.com/"}},
{"rank":1776,"domain":"impact.com","origin":"https://impact.com","robots":{"status":"ok","http":200,"url":"https://impact.com/robots.txt","bytes":519,"groups":9},"bots":{"GPTBot":["partial","*",4,1],"ChatGPT-User":["partial","*",4,1],"OAI-SearchBot":["partial","*",4,1],"ClaudeBot":["partial","*",4,1],"Claude-User":["partial","*",4,1],"Claude-SearchBot":["partial","*",4,1],"anthropic-ai":["partial","*",4,1],"PerplexityBot":["partial","*",4,1],"Perplexity-User":["partial","*",4,1],"Google-Extended":["partial","*",4,1],"Applebot-Extended":["partial","*",4,1],"CCBot":["partial","*",4,1],"Bytespider":["blocked","named",1,0],"Amazonbot":["partial","*",4,1],"meta-externalagent":["partial","*",4,1],"cohere-ai":["partial","*",4,1],"Diffbot":["partial","*",4,1],"ContextFetchBot":["partial","*",4,1]},"llms":{"status":"present","http":200,"bytes":1230},"home":{"status":"ok","http":200,"url":"https://impact.com/","html_tokens":209630,"markdown_tokens":1247,"pruned_tokens":597}},
{"rank":1777,"domain":"made-in-china.com","origin":"https://made-in-china.com","robots":{"status":"ok","http":200,"url":"https://www.made-in-china.com/robots.txt","bytes":2114,"groups":5},"bots":{"GPTBot":["partial","named",1,12],"ChatGPT-User":["partial","named",1,12],"OAI-SearchBot":["partial","named",1,12],"ClaudeBot":["partial","named",1,12],"Claude-User":["partial","named",1,12],"Claude-SearchBot":["partial","named",1,12],"anthropic-ai":["partial","*",33,1],"PerplexityBot":["partial","named",1,12],"Perplexity-User":["partial","named",1,12],"Google-Extended":["partial","named",1,12],"Applebot-Extended":["partial","named",1,12],"CCBot":["partial","named",1,12],"Bytespider":["partial","*",33,1],"Amazonbot":["partial","*",33,1],"meta-externalagent":["partial","*",33,1],"cohere-ai":["partial","*",33,1],"Diffbot":["partial","*",33,1],"ContextFetchBot":["partial","*",33,1]},"llms":{"status":"other_text","http":200,"bytes":99808},"home":{"status":"ok","http":200,"url":"https://www.made-in-china.com/","html_tokens":100125,"markdown_tokens":57,"pruned_tokens":57}},
{"rank":1778,"domain":"thestreet.com","origin":"https://thestreet.com","robots":{"status":"ok","http":200,"url":"https://www.thestreet.com/robots.txt","bytes":1060,"groups":1},"bots":{"GPTBot":["partial","*",44,5],"ChatGPT-User":["partial","*",44,5],"OAI-SearchBot":["partial","*",44,5],"ClaudeBot":["partial","*",44,5],"Claude-User":["partial","*",44,5],"Claude-SearchBot":["partial","*",44,5],"anthropic-ai":["partial","*",44,5],"PerplexityBot":["partial","*",44,5],"Perplexity-User":["partial","*",44,5],"Google-Extended":["partial","*",44,5],"Applebot-Extended":["partial","*",44,5],"CCBot":["partial","*",44,5],"Bytespider":["partial","*",44,5],"Amazonbot":["partial","*",44,5],"meta-externalagent":["partial","*",44,5],"cohere-ai":["partial","*",44,5],"Diffbot":["partial","*",44,5],"ContextFetchBot":["partial","*",44,5]},"llms":{"status":"http_error","http":403,"bytes":773},"home":{"status":"http_error","http":403,"url":"https://thestreet.com/"}},
{"rank":1779,"domain":"gpo.gov","origin":"https://gpo.gov","robots":{"status":"ok","http":200,"url":"https://www.gpo.gov/robots.txt","bytes":124,"groups":1},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"missing","http":404,"bytes":60908},"home":{"status":"ok","http":200,"url":"https://www.gpo.gov/","html_tokens":23755,"markdown_tokens":529,"pruned_tokens":94}},
{"rank":1780,"domain":"azure.com","origin":"https://azure.com","robots":{"status":"html","http":200,"url":"https://azure.microsoft.com/en-us","bytes":463128,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":1781,"domain":"superbthemes.com","origin":"https://superbthemes.com","robots":{"status":"ok","http":200,"url":"https://superbthemes.com/robots.txt","bytes":704,"groups":2},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","named",0,7],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"other_text","http":200,"bytes":4180},"home":{"status":"ok","http":200,"url":"https://superbthemes.com/","html_tokens":56880,"markdown_tokens":2002,"pruned_tokens":330}},
{"rank":1782,"domain":"leadconnectorhq.com","origin":"https://leadconnectorhq.com","robots":{"status":"ok","http":200,"url":"https://www.leadconnectorhq.com/robots.txt","bytes":0,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"empty","http":200,"bytes":0},"home":{"status":"ok","http":200,"url":"https://www.leadconnectorhq.com/","html_tokens":50736,"markdown_tokens":214,"pruned_tokens":49}},
{"rank":1783,"domain":"nextcloud.com","origin":"https://nextcloud.com","robots":{"status":"ok","http":200,"url":"https://nextcloud.com/robots.txt","bytes":171,"groups":1},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"missing","http":404,"bytes":103787},"home":{"status":"ok","http":200,"url":"https://nextcloud.com/","html_tokens":59547,"markdown_tokens":800,"pruned_tokens":539}},
{"rank":1784,"domain":"warnerbros.com","origin":"https://warnerbros.com","robots":{"status":"html","http":200,"url":"https://www.warnerbros.com/","bytes":388387,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":1785,"domain":"chosun.com","origin":"https://chosun.com","robots":{"status":"ok","http":200,"url":"https://www.chosun.com/robots.txt","bytes":479,"groups":4},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["partial","*",8,0],"OAI-SearchBot":["partial","*",8,0],"ClaudeBot":["partial","*",8,0],"Claude-User":["partial","*",8,0],"Claude-SearchBot":["partial","*",8,0],"anthropic-ai":["partial","*",8,0],"PerplexityBot":["partial","*",8,0],"Perplexity-User":["partial","*",8,0],"Google-Extended":["partial","*",8,0],"Applebot-Extended":["partial","*",8,0],"CCBot":["partial","*",8,0],"Bytespider":["partial","*",8,0],"Amazonbot":["partial","*",8,0],"meta-externalagent":["partial","*",8,0],"cohere-ai":["partial","*",8,0],"Diffbot":["partial","*",8,0],"ContextFetchBot":["partial","*",8,0]},"llms":{"status":"missing","http":404,"bytes":325},"home":{"status":"ok","http":200,"url":"https://www.chosun.com/","html_tokens":757311,"markdown_tokens":39,"pruned_tokens":39}},
{"rank":1786,"domain":"adp.com","origin":"https://adp.com","robots":{"status":"ok","http":200,"url":"https://www.adp.com/robots.txt","bytes":2570,"groups":1},"bots":{"GPTBot":["partial","*",55,4],"ChatGPT-User":["partial","*",55,4],"OAI-SearchBot":["partial","*",55,4],"ClaudeBot":["partial","*",55,4],"Claude-User":["partial","*",55,4],"Claude-SearchBot":["partial","*",55,4],"anthropic-ai":["partial","*",55,4],"PerplexityBot":["partial","*",55,4],"Perplexity-User":["partial","*",55,4],"Google-Extended":["partial","*",55,4],"Applebot-Extended":["partial","*",55,4],"CCBot":["partial","*",55,4],"Bytespider":["partial","*",55,4],"Amazonbot":["partial","*",55,4],"meta-externalagent":["partial","*",55,4],"cohere-ai":["partial","*",55,4],"Diffbot":["partial","*",55,4],"ContextFetchBot":["partial","*",55,4]},"llms":{"status":"present","http":200,"bytes":6147},"home":{"status":"ok","http":200,"url":"https://www.adp.com/","html_tokens":59890,"markdown_tokens":529,"pruned_tokens":188}},
{"rank":1787,"domain":"dezeen.com","origin":"https://dezeen.com","robots":{"status":"ok","http":200,"url":"https://www.dezeen.com/robots.txt","bytes":114,"groups":1},"bots":{"GPTBot":["partial","*",1,1],"ChatGPT-User":["partial","*",1,1],"OAI-SearchBot":["partial","*",1,1],"ClaudeBot":["partial","*",1,1],"Claude-User":["partial","*",1,1],"Claude-SearchBot":["partial","*",1,1],"anthropic-ai":["partial","*",1,1],"PerplexityBot":["partial","*",1,1],"Perplexity-User":["partial","*",1,1],"Google-Extended":["partial","*",1,1],"Applebot-Extended":["partial","*",1,1],"CCBot":["partial","*",1,1],"Bytespider":["partial","*",1,1],"Amazonbot":["partial","*",1,1],"meta-externalagent":["partial","*",1,1],"cohere-ai":["partial","*",1,1],"Diffbot":["partial","*",1,1],"ContextFetchBot":["partial","*",1,1]},"llms":{"status":"missing","http":404,"bytes":171379},"home":{"status":"http_error","http":403,"url":"https://dezeen.com/"}},
{"rank":1788,"domain":"monster.com","origin":"https://monster.com","robots":{"status":"ok","http":200,"url":"https://www.monster.com/robots.txt","bytes":1804,"groups":1},"bots":{"GPTBot":["partial","*",57,1],"ChatGPT-User":["partial","*",57,1],"OAI-SearchBot":["partial","*",57,1],"ClaudeBot":["partial","*",57,1],"Claude-User":["partial","*",57,1],"Claude-SearchBot":["partial","*",57,1],"anthropic-ai":["partial","*",57,1],"PerplexityBot":["partial","*",57,1],"Perplexity-User":["partial","*",57,1],"Google-Extended":["partial","*",57,1],"Applebot-Extended":["partial","*",57,1],"CCBot":["partial","*",57,1],"Bytespider":["partial","*",57,1],"Amazonbot":["partial","*",57,1],"meta-externalagent":["partial","*",57,1],"cohere-ai":["partial","*",57,1],"Diffbot":["partial","*",57,1],"ContextFetchBot":["partial","*",57,1]},"llms":{"status":"present","http":200,"bytes":14627},"home":{"status":"http_error","http":403,"url":"https://www.monster.com/"}},
{"rank":1789,"domain":"diigo.com","origin":"https://diigo.com","robots":{"status":"refused","http":403,"url":"https://diigo.com/robots.txt","bytes":5525,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":1790,"domain":"thenationalnews.com","origin":"https://thenationalnews.com","robots":{"status":"ok","http":200,"url":"https://www.thenationalnews.com/robots.txt","bytes":3124,"groups":2},"bots":{"GPTBot":["partial","*",26,1],"ChatGPT-User":["partial","*",26,1],"OAI-SearchBot":["partial","*",26,1],"ClaudeBot":["partial","*",26,1],"Claude-User":["partial","*",26,1],"Claude-SearchBot":["partial","*",26,1],"anthropic-ai":["partial","*",26,1],"PerplexityBot":["partial","*",26,1],"Perplexity-User":["partial","*",26,1],"Google-Extended":["partial","*",26,1],"Applebot-Extended":["partial","*",26,1],"CCBot":["partial","*",26,1],"Bytespider":["partial","*",26,1],"Amazonbot":["partial","*",26,1],"meta-externalagent":["partial","*",26,1],"cohere-ai":["partial","*",26,1],"Diffbot":["partial","*",26,1],"ContextFetchBot":["partial","*",26,1]},"llms":{"status":"missing","http":404,"bytes":473781},"home":{"status":"ok","http":200,"url":"https://www.thenationalnews.com/","html_tokens":450301,"markdown_tokens":67,"pruned_tokens":67}},
{"rank":1791,"domain":"emptyhammock.com","origin":"https://emptyhammock.com","robots":{"status":"ok","http":200,"url":"https://emptyhammock.com/robots.txt","bytes":38,"groups":1},"bots":{"GPTBot":["partial","*",1,0],"ChatGPT-User":["partial","*",1,0],"OAI-SearchBot":["partial","*",1,0],"ClaudeBot":["partial","*",1,0],"Claude-User":["partial","*",1,0],"Claude-SearchBot":["partial","*",1,0],"anthropic-ai":["partial","*",1,0],"PerplexityBot":["partial","*",1,0],"Perplexity-User":["partial","*",1,0],"Google-Extended":["partial","*",1,0],"Applebot-Extended":["partial","*",1,0],"CCBot":["partial","*",1,0],"Bytespider":["partial","*",1,0],"Amazonbot":["partial","*",1,0],"meta-externalagent":["partial","*",1,0],"cohere-ai":["partial","*",1,0],"Diffbot":["partial","*",1,0],"ContextFetchBot":["partial","*",1,0]},"llms":{"status":"missing","http":404,"bytes":4846},"home":{"status":"ok","http":200,"url":"https://emptyhammock.com/","html_tokens":1840,"markdown_tokens":331,"pruned_tokens":156}},
{"rank":1792,"domain":"tate.org.uk","origin":"https://tate.org.uk","robots":{"status":"ok","http":200,"url":"https://www.tate.org.uk/robots.txt","bytes":176,"groups":1},"bots":{"GPTBot":["partial","*",4,1],"ChatGPT-User":["partial","*",4,1],"OAI-SearchBot":["partial","*",4,1],"ClaudeBot":["partial","*",4,1],"Claude-User":["partial","*",4,1],"Claude-SearchBot":["partial","*",4,1],"anthropic-ai":["partial","*",4,1],"PerplexityBot":["partial","*",4,1],"Perplexity-User":["partial","*",4,1],"Google-Extended":["partial","*",4,1],"Applebot-Extended":["partial","*",4,1],"CCBot":["partial","*",4,1],"Bytespider":["partial","*",4,1],"Amazonbot":["partial","*",4,1],"meta-externalagent":["partial","*",4,1],"cohere-ai":["partial","*",4,1],"Diffbot":["partial","*",4,1],"ContextFetchBot":["partial","*",4,1]},"llms":{"status":"missing","http":404,"bytes":20057},"home":{"status":"ok","http":200,"url":"https://www.tate.org.uk/","html_tokens":32940,"markdown_tokens":37,"pruned_tokens":37}},
{"rank":1793,"domain":"pcgamer.com","origin":"https://pcgamer.com","robots":{"status":"ok","http":200,"url":"https://www.pcgamer.com/robots.txt","bytes":1831,"groups":4},"bots":{"GPTBot":["partial","*",18,0],"ChatGPT-User":["partial","*",18,0],"OAI-SearchBot":["partial","*",18,0],"ClaudeBot":["partial","*",18,0],"Claude-User":["partial","*",18,0],"Claude-SearchBot":["partial","*",18,0],"anthropic-ai":["partial","*",18,0],"PerplexityBot":["partial","*",18,0],"Perplexity-User":["partial","*",18,0],"Google-Extended":["partial","*",18,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["partial","*",18,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["partial","*",18,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",18,0]},"llms":{"status":"missing","http":404,"bytes":1260993},"home":{"status":"ok","http":200,"url":"https://www.pcgamer.com/","html_tokens":493688,"markdown_tokens":46,"pruned_tokens":46}},
{"rank":1794,"domain":"hud.gov","origin":"https://hud.gov","robots":{"status":"ok","http":200,"url":"https://www.hud.gov/robots.txt","bytes":2027,"groups":1},"bots":{"GPTBot":["partial","*",34,18],"ChatGPT-User":["partial","*",34,18],"OAI-SearchBot":["partial","*",34,18],"ClaudeBot":["partial","*",34,18],"Claude-User":["partial","*",34,18],"Claude-SearchBot":["partial","*",34,18],"anthropic-ai":["partial","*",34,18],"PerplexityBot":["partial","*",34,18],"Perplexity-User":["partial","*",34,18],"Google-Extended":["partial","*",34,18],"Applebot-Extended":["partial","*",34,18],"CCBot":["partial","*",34,18],"Bytespider":["partial","*",34,18],"Amazonbot":["partial","*",34,18],"meta-externalagent":["partial","*",34,18],"cohere-ai":["partial","*",34,18],"Diffbot":["partial","*",34,18],"ContextFetchBot":["partial","*",34,18]},"llms":{"status":"missing","http":404,"bytes":545},"home":{"status":"ok","http":200,"url":"https://www.hud.gov/","html_tokens":40010,"markdown_tokens":141,"pruned_tokens":141}},
{"rank":1795,"domain":"ibiblio.org","origin":"https://ibiblio.org","robots":{"status":"ok","http":200,"url":"https://ibiblio.org/robots.txt","bytes":702,"groups":6},"bots":{"GPTBot":["partial","*",10,0],"ChatGPT-User":["partial","*",10,0],"OAI-SearchBot":["partial","*",10,0],"ClaudeBot":["partial","*",10,0],"Claude-User":["partial","*",10,0],"Claude-SearchBot":["partial","*",10,0],"anthropic-ai":["partial","*",10,0],"PerplexityBot":["partial","*",10,0],"Perplexity-User":["partial","*",10,0],"Google-Extended":["partial","*",10,0],"Applebot-Extended":["partial","*",10,0],"CCBot":["partial","*",10,0],"Bytespider":["partial","*",10,0],"Amazonbot":["partial","*",10,0],"meta-externalagent":["partial","*",10,0],"cohere-ai":["partial","*",10,0],"Diffbot":["partial","*",10,0],"ContextFetchBot":["partial","*",10,0]},"llms":{"status":"missing","http":404,"bytes":32970},"home":{"status":"ok","http":200,"url":"https://ibiblio.org/","html_tokens":1106,"markdown_tokens":269,"pruned_tokens":153}},
{"rank":1796,"domain":"google.at","origin":"https://google.at","robots":{"status":"ok","http":200,"url":"https://www.google.at/robots.txt","bytes":6624,"groups":4},"bots":{"GPTBot":["partial","*",169,64],"ChatGPT-User":["partial","*",169,64],"OAI-SearchBot":["partial","*",169,64],"ClaudeBot":["partial","*",169,64],"Claude-User":["partial","*",169,64],"Claude-SearchBot":["partial","*",169,64],"anthropic-ai":["partial","*",169,64],"PerplexityBot":["partial","*",169,64],"Perplexity-User":["partial","*",169,64],"Google-Extended":["partial","*",169,64],"Applebot-Extended":["partial","*",169,64],"CCBot":["partial","*",169,64],"Bytespider":["partial","*",169,64],"Amazonbot":["partial","*",169,64],"meta-externalagent":["partial","*",169,64],"cohere-ai":["partial","*",169,64],"Diffbot":["partial","*",169,64],"ContextFetchBot":["partial","*",169,64]},"llms":{"status":"missing","http":404,"bytes":1569},"home":{"status":"ok","http":200,"url":"https://www.google.com/","html_tokens":21303,"markdown_tokens":30,"pruned_tokens":30}},
{"rank":1797,"domain":"mathworks.com","origin":"https://mathworks.com","robots":{"status":"refused","http":403,"url":"https://www.mathworks.com/robots.txt","bytes":379,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":1798,"domain":"similarweb.com","origin":"https://similarweb.com","robots":{"status":"ok","http":200,"url":"https://www.similarweb.com/robots.txt","bytes":1026,"groups":1},"bots":{"GPTBot":["partial","*",10,2],"ChatGPT-User":["partial","*",10,2],"OAI-SearchBot":["partial","*",10,2],"ClaudeBot":["partial","*",10,2],"Claude-User":["partial","*",10,2],"Claude-SearchBot":["partial","*",10,2],"anthropic-ai":["partial","*",10,2],"PerplexityBot":["partial","*",10,2],"Perplexity-User":["partial","*",10,2],"Google-Extended":["partial","*",10,2],"Applebot-Extended":["partial","*",10,2],"CCBot":["partial","*",10,2],"Bytespider":["partial","*",10,2],"Amazonbot":["partial","*",10,2],"meta-externalagent":["partial","*",10,2],"cohere-ai":["partial","*",10,2],"Diffbot":["partial","*",10,2],"ContextFetchBot":["partial","*",10,2]},"llms":{"status":"present","http":200,"bytes":19987},"home":{"status":"ok","http":200,"url":"https://www.similarweb.com/","html_tokens":229364,"markdown_tokens":140,"pruned_tokens":77}},
{"rank":1799,"domain":"malwarebytes.com","origin":"https://malwarebytes.com","robots":{"status":"ok","http":200,"url":"https://www.malwarebytes.com/robots.txt","bytes":1816,"groups":1},"bots":{"GPTBot":["partial","named",42,1],"ChatGPT-User":["partial","*",42,1],"OAI-SearchBot":["partial","named",42,1],"ClaudeBot":["partial","named",42,1],"Claude-User":["partial","*",42,1],"Claude-SearchBot":["partial","*",42,1],"anthropic-ai":["partial","*",42,1],"PerplexityBot":["partial","named",42,1],"Perplexity-User":["partial","*",42,1],"Google-Extended":["partial","*",42,1],"Applebot-Extended":["partial","*",42,1],"CCBot":["partial","*",42,1],"Bytespider":["partial","*",42,1],"Amazonbot":["partial","*",42,1],"meta-externalagent":["partial","*",42,1],"cohere-ai":["partial","*",42,1],"Diffbot":["partial","*",42,1],"ContextFetchBot":["partial","*",42,1]},"llms":{"status":"present","http":200,"bytes":7001},"home":{"status":"ok","http":200,"url":"https://www.malwarebytes.com","html_tokens":256868,"markdown_tokens":1111,"pruned_tokens":268}},
{"rank":1800,"domain":"consultant.ru","origin":"https://consultant.ru","robots":{"status":"ok","http":200,"url":"https://www.consultant.ru/robots.txt","bytes":1176,"groups":1},"bots":{"GPTBot":["partial","*",27,0],"ChatGPT-User":["partial","*",27,0],"OAI-SearchBot":["partial","*",27,0],"ClaudeBot":["partial","*",27,0],"Claude-User":["partial","*",27,0],"Claude-SearchBot":["partial","*",27,0],"anthropic-ai":["partial","*",27,0],"PerplexityBot":["partial","*",27,0],"Perplexity-User":["partial","*",27,0],"Google-Extended":["partial","*",27,0],"Applebot-Extended":["partial","*",27,0],"CCBot":["partial","*",27,0],"Bytespider":["partial","*",27,0],"Amazonbot":["partial","*",27,0],"meta-externalagent":["partial","*",27,0],"cohere-ai":["partial","*",27,0],"Diffbot":["partial","*",27,0],"ContextFetchBot":["partial","*",27,0]},"llms":{"status":"missing","http":404,"bytes":146},"home":{"status":"ok","http":200,"url":"https://www.consultant.ru/","html_tokens":9172,"markdown_tokens":12,"pruned_tokens":12}},
{"rank":1801,"domain":"bankofamerica.com","origin":"https://bankofamerica.com","robots":{"status":"ok","http":200,"url":"https://www.bankofamerica.com/robots.txt","bytes":3312,"groups":4},"bots":{"GPTBot":["partial","*",32,0],"ChatGPT-User":["partial","*",32,0],"OAI-SearchBot":["partial","*",32,0],"ClaudeBot":["partial","*",32,0],"Claude-User":["partial","*",32,0],"Claude-SearchBot":["partial","*",32,0],"anthropic-ai":["partial","*",32,0],"PerplexityBot":["partial","*",32,0],"Perplexity-User":["partial","*",32,0],"Google-Extended":["partial","*",32,0],"Applebot-Extended":["partial","*",32,0],"CCBot":["partial","*",32,0],"Bytespider":["partial","*",32,0],"Amazonbot":["partial","*",32,0],"meta-externalagent":["partial","*",32,0],"cohere-ai":["partial","*",32,0],"Diffbot":["partial","*",32,0],"ContextFetchBot":["partial","*",32,0]},"llms":{"status":"missing","http":404,"bytes":5872},"home":{"status":"ok","http":200,"url":"https://www.bankofamerica.com/","html_tokens":80923,"markdown_tokens":225,"pruned_tokens":66}},
{"rank":1802,"domain":"br.de","origin":"https://br.de","robots":{"status":"ok","http":200,"url":"https://br.de/robots.txt","bytes":3013,"groups":21},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["partial","*",24,0],"OAI-SearchBot":["partial","*",24,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",24,0],"Claude-SearchBot":["partial","*",24,0],"anthropic-ai":["partial","*",24,0],"PerplexityBot":["partial","*",24,0],"Perplexity-User":["partial","*",24,0],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["partial","*",24,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["partial","*",24,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",24,0]},"llms":{"status":"missing","http":404,"bytes":76836},"home":{"status":"ok","http":200,"url":"https://www.br.de/index.html","html_tokens":51792,"markdown_tokens":83,"pruned_tokens":83}},
{"rank":1803,"domain":"google.ch","origin":"https://google.ch","robots":{"status":"ok","http":200,"url":"https://www.google.ch/robots.txt","bytes":6624,"groups":4},"bots":{"GPTBot":["partial","*",169,64],"ChatGPT-User":["partial","*",169,64],"OAI-SearchBot":["partial","*",169,64],"ClaudeBot":["partial","*",169,64],"Claude-User":["partial","*",169,64],"Claude-SearchBot":["partial","*",169,64],"anthropic-ai":["partial","*",169,64],"PerplexityBot":["partial","*",169,64],"Perplexity-User":["partial","*",169,64],"Google-Extended":["partial","*",169,64],"Applebot-Extended":["partial","*",169,64],"CCBot":["partial","*",169,64],"Bytespider":["partial","*",169,64],"Amazonbot":["partial","*",169,64],"meta-externalagent":["partial","*",169,64],"cohere-ai":["partial","*",169,64],"Diffbot":["partial","*",169,64],"ContextFetchBot":["partial","*",169,64]},"llms":{"status":"missing","http":404,"bytes":1569},"home":{"status":"ok","http":200,"url":"https://www.google.com/","html_tokens":21351,"markdown_tokens":30,"pruned_tokens":30}},
{"rank":1804,"domain":"hostingersite.com","origin":"https://www.hostingersite.com","robots":{"status":"ok","http":200,"url":"https://www.hostingersite.com/robots.txt","bytes":58,"groups":2},"bots":{"GPTBot":["allowed","*",0,1],"ChatGPT-User":["allowed","*",0,1],"OAI-SearchBot":["allowed","*",0,1],"ClaudeBot":["allowed","*",0,1],"Claude-User":["allowed","*",0,1],"Claude-SearchBot":["allowed","*",0,1],"anthropic-ai":["allowed","*",0,1],"PerplexityBot":["allowed","*",0,1],"Perplexity-User":["allowed","*",0,1],"Google-Extended":["allowed","*",0,1],"Applebot-Extended":["allowed","*",0,1],"CCBot":["allowed","*",0,1],"Bytespider":["allowed","*",0,1],"Amazonbot":["allowed","*",0,1],"meta-externalagent":["allowed","*",0,1],"cohere-ai":["allowed","*",0,1],"Diffbot":["allowed","*",0,1],"ContextFetchBot":["allowed","*",0,1]},"llms":{"status":"unreachable","error":"<StreamReset stream_id:3, error_code:1, remote_reset:True>"},"home":{"status":"unreachable","error":"<StreamReset stream_id:5, error_code:1, remote_reset:True>"}},
{"rank":1805,"domain":"avg.com","origin":"https://avg.com","robots":{"status":"refused","http":403,"url":"http://www.avg.com/robots.txt","bytes":379,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":1806,"domain":"whatwg.org","origin":"https://whatwg.org","robots":{"status":"ok","http":200,"url":"https://whatwg.org/robots.txt","bytes":277,"groups":4},"bots":{"GPTBot":["partial","*",5,0],"ChatGPT-User":["partial","*",5,0],"OAI-SearchBot":["partial","*",5,0],"ClaudeBot":["partial","*",5,0],"Claude-User":["partial","*",5,0],"Claude-SearchBot":["partial","*",5,0],"anthropic-ai":["partial","*",5,0],"PerplexityBot":["partial","*",5,0],"Perplexity-User":["partial","*",5,0],"Google-Extended":["partial","*",5,0],"Applebot-Extended":["partial","*",5,0],"CCBot":["partial","*",5,0],"Bytespider":["partial","*",5,0],"Amazonbot":["partial","*",5,0],"meta-externalagent":["partial","*",5,0],"cohere-ai":["partial","*",5,0],"Diffbot":["partial","*",5,0],"ContextFetchBot":["partial","*",5,0]},"llms":{"status":"missing","http":404,"bytes":1391},"home":{"status":"ok","http":200,"url":"https://whatwg.org/","html_tokens":1096,"markdown_tokens":17,"pruned_tokens":17}},
{"rank":1807,"domain":"asm.org","origin":"https://asm.org","robots":{"status":"ok","http":200,"url":"https://asm.org/robots.txt","bytes":477,"groups":1},"bots":{"GPTBot":["partial","*",17,1],"ChatGPT-User":["partial","*",17,1],"OAI-SearchBot":["partial","*",17,1],"ClaudeBot":["partial","*",17,1],"Claude-User":["partial","*",17,1],"Claude-SearchBot":["partial","*",17,1],"anthropic-ai":["partial","*",17,1],"PerplexityBot":["partial","*",17,1],"Perplexity-User":["partial","*",17,1],"Google-Extended":["partial","*",17,1],"Applebot-Extended":["partial","*",17,1],"CCBot":["partial","*",17,1],"Bytespider":["partial","*",17,1],"Amazonbot":["partial","*",17,1],"meta-externalagent":["partial","*",17,1],"cohere-ai":["partial","*",17,1],"Diffbot":["partial","*",17,1],"ContextFetchBot":["partial","*",17,1]},"llms":{"status":"missing","http":404,"bytes":5714},"home":{"status":"ok","http":200,"url":"https://asm.org/","html_tokens":15848,"markdown_tokens":284,"pruned_tokens":92}},
{"rank":1808,"domain":"ansa.it","origin":"https://ansa.it","robots":{"status":"ok","http":200,"url":"https://www.ansa.it/robots.txt","bytes":1248,"groups":21},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["blocked","named",1,0],"OAI-SearchBot":["partial","*",14,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",14,0],"Claude-SearchBot":["partial","*",14,0],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["partial","*",14,0],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["partial","*",14,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",14,0]},"llms":{"status":"missing","http":404,"bytes":30272},"home":{"status":"ok","http":200,"url":"https://www.ansa.it/","html_tokens":204154,"markdown_tokens":241,"pruned_tokens":78}},
{"rank":1809,"domain":"od.ua","origin":"https://od.ua","robots":{"status":"unreachable","http":null,"url":"https://od.ua/robots.txt","bytes":0,"groups":0,"error":"cannot resolve od.ua"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":1810,"domain":"treehugger.com","origin":"https://treehugger.com","robots":{"status":"ok","http":200,"url":"https://www.treehugger.com/robots.txt","bytes":3473,"groups":6},"bots":{"GPTBot":["partial","named",1,0],"ChatGPT-User":["partial","named",1,0],"OAI-SearchBot":["partial","named",1,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["blocked","named",1,0],"Claude-SearchBot":["partial","named",1,2],"anthropic-ai":["partial","named",1,2],"PerplexityBot":["partial","named",1,2],"Perplexity-User":["blocked","named",1,0],"Google-Extended":["partial","named",1,2],"Applebot-Extended":["blocked","named",1,0],"CCBot":["partial","named",1,2],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["partial","named",1,2],"cohere-ai":["partial","named",1,2],"Diffbot":["partial","*",2,0],"ContextFetchBot":["partial","*",2,0]},"llms":{"status":"http_error","http":402,"bytes":612},"home":{"status":"http_error","http":402,"url":"https://www.treehugger.com/"}},
{"rank":1811,"domain":"freewebs.com","origin":"https://freewebs.com","robots":{"status":"html","http":200,"url":"https://www.vistaprint.com/digital-marketing/webs-shutdown","bytes":173973,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":1812,"domain":"feishu.cn","origin":"https://feishu.cn","robots":{"status":"ok","http":200,"url":"https://www.feishu.cn/robots.txt","bytes":496,"groups":1},"bots":{"GPTBot":["partial","*",11,1],"ChatGPT-User":["partial","*",11,1],"OAI-SearchBot":["partial","*",11,1],"ClaudeBot":["partial","*",11,1],"Claude-User":["partial","*",11,1],"Claude-SearchBot":["partial","*",11,1],"anthropic-ai":["partial","*",11,1],"PerplexityBot":["partial","*",11,1],"Perplexity-User":["partial","*",11,1],"Google-Extended":["partial","*",11,1],"Applebot-Extended":["partial","*",11,1],"CCBot":["partial","*",11,1],"Bytespider":["partial","*",11,1],"Amazonbot":["partial","*",11,1],"meta-externalagent":["partial","*",11,1],"cohere-ai":["partial","*",11,1],"Diffbot":["partial","*",11,1],"ContextFetchBot":["partial","*",11,1]},"llms":{"status":"missing","http":404,"bytes":4514},"home":{"status":"ok","http":200,"url":"https://www.larksuite.com/en_sg/?from_site=feishu","html_tokens":107354,"markdown_tokens":0,"pruned_tokens":0}},
{"rank":1813,"domain":"specbench.org","origin":"https://www.specbench.org","robots":{"status":"ok","http":200,"url":"https://www.spec.org/robots.txt","bytes":659,"groups":1},"bots":{"GPTBot":["partial","*",7,0],"ChatGPT-User":["partial","*",7,0],"OAI-SearchBot":["partial","*",7,0],"ClaudeBot":["partial","*",7,0],"Claude-User":["partial","*",7,0],"Claude-SearchBot":["partial","*",7,0],"anthropic-ai":["partial","*",7,0],"PerplexityBot":["partial","*",7,0],"Perplexity-User":["partial","*",7,0],"Google-Extended":["partial","*",7,0],"Applebot-Extended":["partial","*",7,0],"CCBot":["partial","*",7,0],"Bytespider":["partial","*",7,0],"Amazonbot":["partial","*",7,0],"meta-externalagent":["partial","*",7,0],"cohere-ai":["partial","*",7,0],"Diffbot":["partial","*",7,0],"ContextFetchBot":["partial","*",7,0]},"llms":{"status":"missing","http":404,"bytes":32901},"home":{"status":"ok","http":200,"url":"https://www.spec.org/","html_tokens":11100,"markdown_tokens":875,"pruned_tokens":139}},
{"rank":1814,"domain":"denverpost.com","origin":"https://denverpost.com","robots":{"status":"ok","http":200,"url":"https://www.denverpost.com/robots.txt","bytes":4480,"groups":68},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["blocked","named",1,0],"OAI-SearchBot":["blocked","named",1,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["blocked","named",1,0],"Claude-SearchBot":["blocked","named",1,0],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["blocked","named",1,0],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["partial","*",8,1],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",8,1]},"llms":{"status":"missing","http":404,"bytes":95229},"home":{"status":"ok","http":200,"url":"https://www.denverpost.com/","html_tokens":75043,"markdown_tokens":324,"pruned_tokens":105}},
{"rank":1815,"domain":"web.id","origin":"https://web.id","robots":{"status":"unreachable","http":null,"url":"https://web.id/robots.txt","bytes":0,"groups":0,"error":"cannot resolve web.id"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":1816,"domain":"nj.com","origin":"https://nj.com","robots":{"status":"ok","http":200,"url":"https://www.nj.com/robots.txt","bytes":2794,"groups":4},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["partial","named",1,11],"OAI-SearchBot":["partial","named",1,11],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","named",1,11],"Claude-SearchBot":["partial","named",1,11],"anthropic-ai":["partial","*",30,3],"PerplexityBot":["partial","named",1,11],"Perplexity-User":["partial","named",1,11],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["partial","*",30,3],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",30,3]},"llms":{"status":"missing","http":404,"bytes":324608},"home":{"status":"http_error","http":403,"url":"https://www.nj.com/"}},
{"rank":1817,"domain":"tally.so","origin":"https://tally.so","robots":{"status":"ok","http":200,"url":"https://tally.so/robots.txt","bytes":62,"groups":1},"bots":{"GPTBot":["allowed","*",0,1],"ChatGPT-User":["allowed","*",0,1],"OAI-SearchBot":["allowed","*",0,1],"ClaudeBot":["allowed","*",0,1],"Claude-User":["allowed","*",0,1],"Claude-SearchBot":["allowed","*",0,1],"anthropic-ai":["allowed","*",0,1],"PerplexityBot":["allowed","*",0,1],"Perplexity-User":["allowed","*",0,1],"Google-Extended":["allowed","*",0,1],"Applebot-Extended":["allowed","*",0,1],"CCBot":["allowed","*",0,1],"Bytespider":["allowed","*",0,1],"Amazonbot":["allowed","*",0,1],"meta-externalagent":["allowed","*",0,1],"cohere-ai":["allowed","*",0,1],"Diffbot":["allowed","*",0,1],"ContextFetchBot":["allowed","*",0,1]},"llms":{"status":"present","http":200,"bytes":4324},"home":{"status":"ok","http":200,"url":"https://tally.so/","html_tokens":74238,"markdown_tokens":899,"pruned_tokens":213}},
{"rank":1818,"domain":"sproutsocial.com","origin":"https://sproutsocial.com","robots":{"status":"ok","http":200,"url":"https://sproutsocial.com/robots.txt","bytes":383,"groups":1},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"present","http":200,"bytes":15219},"home":{"status":"ok","http":200,"url":"https://sproutsocial.com/","html_tokens":46642,"markdown_tokens":364,"pruned_tokens":90}},
{"rank":1819,"domain":"us.org","origin":"https://us.org","robots":{"status":"ok","http":200,"url":"https://us.org/robots.txt","bytes":0,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":7806},"home":{"status":"ok","http":200,"url":"https://us.org/","html_tokens":9891,"markdown_tokens":895,"pruned_tokens":751}},
{"rank":1820,"domain":"liebertpub.com","origin":"https://liebertpub.com","robots":{"status":"ok","http":200,"url":"https://journals.sagepub.com/robots.txt?cf-mal-redirected=true&","bytes":1200,"groups":6},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["partial","*",18,10],"OAI-SearchBot":["partial","*",18,10],"ClaudeBot":["partial","*",18,10],"Claude-User":["partial","*",18,10],"Claude-SearchBot":["partial","*",18,10],"anthropic-ai":["partial","*",18,10],"PerplexityBot":["partial","*",18,10],"Perplexity-User":["partial","*",18,10],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["partial","*",18,10],"CCBot":["blocked","named",1,0],"Bytespider":["partial","*",18,10],"Amazonbot":["partial","*",18,10],"meta-externalagent":["partial","*",18,10],"cohere-ai":["partial","*",18,10],"Diffbot":["partial","*",18,10],"ContextFetchBot":["partial","*",18,10]},"llms":{"status":"http_error","http":403,"bytes":5697},"home":{"status":"http_error","http":403,"url":"https://journals.sagepub.com/?cf-mal-redirected=true&"}},
{"rank":1821,"domain":"eset.com","origin":"https://eset.com","robots":{"status":"refused","http":403,"url":"https://www.eset.com/robots.txt","bytes":266,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":1822,"domain":"lesechos.fr","origin":"https://lesechos.fr","robots":{"status":"ok","http":200,"url":"https://www.lesechos.fr/robots.txt","bytes":15868,"groups":421},"bots":{"GPTBot":["partial","named",1,1],"ChatGPT-User":["partial","named",1,1],"OAI-SearchBot":["partial","named",1,1],"ClaudeBot":["blocked","named",1,0],"Claude-User":["blocked","named",1,0],"Claude-SearchBot":["blocked","named",1,0],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["partial","named",1,1],"Perplexity-User":["partial","named",1,1],"Google-Extended":["partial","named",1,1],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",2,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["partial","*",2,0],"ContextFetchBot":["partial","*",2,0]},"llms":{"status":"http_error","http":403,"bytes":381},"home":{"status":"http_error","http":403,"url":"https://www.lesechos.fr/"}},
{"rank":1823,"domain":"sourceforge.io","origin":"https://sourceforge.io","robots":{"status":"html","http":200,"url":"https://sourceforge.net/","bytes":124158,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":1824,"domain":"oregon.gov","origin":"https://oregon.gov","robots":{"status":"ok","http":200,"url":"https://www.oregon.gov/robots.txt","bytes":7226,"groups":4},"bots":{"GPTBot":["partial","*",66,63],"ChatGPT-User":["partial","*",66,63],"OAI-SearchBot":["partial","*",66,63],"ClaudeBot":["partial","*",66,63],"Claude-User":["partial","*",66,63],"Claude-SearchBot":["partial","*",66,63],"anthropic-ai":["partial","*",66,63],"PerplexityBot":["partial","*",66,63],"Perplexity-User":["partial","*",66,63],"Google-Extended":["partial","*",66,63],"Applebot-Extended":["partial","*",66,63],"CCBot":["partial","*",66,63],"Bytespider":["partial","*",66,63],"Amazonbot":["partial","*",66,63],"meta-externalagent":["partial","*",66,63],"cohere-ai":["partial","*",66,63],"Diffbot":["partial","*",66,63],"ContextFetchBot":["partial","*",66,63]},"llms":{"status":"missing","http":404,"bytes":91261},"home":{"status":"ok","http":200,"url":"https://www.oregon.gov/Pages/index.aspx","html_tokens":23452,"markdown_tokens":8,"pruned_tokens":8}},
{"rank":1825,"domain":"aaa.com","origin":"https://aaa.com","robots":{"status":"ok","http":200,"url":"https://www.aaa.com/robots.txt","bytes":357,"groups":1},"bots":{"GPTBot":["partial","*",7,1],"ChatGPT-User":["partial","*",7,1],"OAI-SearchBot":["partial","*",7,1],"ClaudeBot":["partial","*",7,1],"Claude-User":["partial","*",7,1],"Claude-SearchBot":["partial","*",7,1],"anthropic-ai":["partial","*",7,1],"PerplexityBot":["partial","*",7,1],"Perplexity-User":["partial","*",7,1],"Google-Extended":["partial","*",7,1],"Applebot-Extended":["partial","*",7,1],"CCBot":["partial","*",7,1],"Bytespider":["partial","*",7,1],"Amazonbot":["partial","*",7,1],"meta-externalagent":["partial","*",7,1],"cohere-ai":["partial","*",7,1],"Diffbot":["partial","*",7,1],"ContextFetchBot":["partial","*",7,1]},"llms":{"status":"html","http":200,"bytes":3650},"home":{"status":"ok","http":200,"url":"https://www.acg.aaa.com?zip=32826&stateprov=fl&city=orlando&devicecd=PC","html_tokens":85782,"markdown_tokens":4207,"pruned_tokens":155}},
{"rank":1826,"domain":"backlinko.com","origin":"https://backlinko.com","robots":{"status":"ok","http":200,"url":"https://backlinko.com/robots.txt","bytes":418,"groups":3},"bots":{"GPTBot":["partial","*",4,0],"ChatGPT-User":["partial","*",4,0],"OAI-SearchBot":["partial","*",4,0],"ClaudeBot":["partial","*",4,0],"Claude-User":["partial","*",4,0],"Claude-SearchBot":["partial","*",4,0],"anthropic-ai":["partial","*",4,0],"PerplexityBot":["partial","*",4,0],"Perplexity-User":["partial","*",4,0],"Google-Extended":["partial","*",4,0],"Applebot-Extended":["partial","*",4,0],"CCBot":["partial","*",4,0],"Bytespider":["partial","*",4,0],"Amazonbot":["partial","*",4,0],"meta-externalagent":["partial","*",4,0],"cohere-ai":["partial","*",4,0],"Diffbot":["partial","*",4,0],"ContextFetchBot":["partial","*",4,0]},"llms":{"status":"missing","http":404,"bytes":224230},"home":{"status":"ok","http":200,"url":"https://backlinko.com/","html_tokens":81275,"markdown_tokens":108,"pruned_tokens":34}},
{"rank":1827,"domain":"home.blog","origin":"https://home.blog","robots":{"status":"ok","http":200,"url":"https://home.blog/robots.txt","bytes":88,"groups":1},"bots":{"GPTBot":["blocked","*",1,0],"ChatGPT-User":["blocked","*",1,0],"OAI-SearchBot":["blocked","*",1,0],"ClaudeBot":["blocked","*",1,0],"Claude-User":["blocked","*",1,0],"Claude-SearchBot":["blocked","*",1,0],"anthropic-ai":["blocked","*",1,0],"PerplexityBot":["blocked","*",1,0],"Perplexity-User":["blocked","*",1,0],"Google-Extended":["blocked","*",1,0],"Applebot-Extended":["blocked","*",1,0],"CCBot":["blocked","*",1,0],"Bytespider":["blocked","*",1,0],"Amazonbot":["blocked","*",1,0],"meta-externalagent":["blocked","*",1,0],"cohere-ai":["blocked","*",1,0],"Diffbot":["blocked","*",1,0],"ContextFetchBot":["blocked","*",1,0]},"llms":{"status":"disallowed"},"home":{"status":"disallowed"}},
{"rank":1828,"domain":"mynavi.jp","origin":"https://mynavi.jp","robots":{"status":"ok","http":200,"url":"https://mynavi.jp/robots.txt","bytes":104,"groups":1},"bots":{"GPTBot":["partial","*",2,1],"ChatGPT-User":["partial","*",2,1],"OAI-SearchBot":["partial","*",2,1],"ClaudeBot":["partial","*",2,1],"Claude-User":["partial","*",2,1],"Claude-SearchBot":["partial","*",2,1],"anthropic-ai":["partial","*",2,1],"PerplexityBot":["partial","*",2,1],"Perplexity-User":["partial","*",2,1],"Google-Extended":["partial","*",2,1],"Applebot-Extended":["partial","*",2,1],"CCBot":["partial","*",2,1],"Bytespider":["partial","*",2,1],"Amazonbot":["partial","*",2,1],"meta-externalagent":["partial","*",2,1],"cohere-ai":["partial","*",2,1],"Diffbot":["partial","*",2,1],"ContextFetchBot":["partial","*",2,1]},"llms":{"status":"missing","http":404,"bytes":26164},"home":{"status":"ok","http":200,"url":"https://www.mynavi.jp/","html_tokens":9482,"markdown_tokens":22,"pruned_tokens":22}},
{"rank":1829,"domain":"polygon.com","origin":"https://polygon.com","robots":{"status":"ok","http":200,"url":"https://www.polygon.com/robots.txt","bytes":2116,"groups":21},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["blocked","named",1,0],"OAI-SearchBot":["partial","*",18,1],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",18,1],"Claude-SearchBot":["partial","*",18,1],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["partial","*",18,1],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["partial","*",18,1],"meta-externalagent":["partial","*",18,1],"cohere-ai":["partial","*",18,1],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",18,1]},"llms":{"status":"missing","http":404,"bytes":119400},"home":{"status":"ok","http":200,"url":"https://www.polygon.com/","html_tokens":295594,"markdown_tokens":101,"pruned_tokens":101}},
{"rank":1830,"domain":"moe.gov.cn","origin":"https://www.moe.gov.cn","robots":{"status":"missing","http":404,"url":"http://www.moe.gov.cn/robots.txt","bytes":12,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":12},"home":{"status":"ok","http":200,"url":"http://www.moe.gov.cn/","html_tokens":21721,"markdown_tokens":7,"pruned_tokens":7}},
{"rank":1831,"domain":"uconn.edu","origin":"https://uconn.edu","robots":{"status":"ok","http":200,"url":"https://uconn.edu/robots.txt","bytes":294,"groups":1},"bots":{"GPTBot":["partial","*",7,1],"ChatGPT-User":["partial","*",7,1],"OAI-SearchBot":["partial","*",7,1],"ClaudeBot":["partial","*",7,1],"Claude-User":["partial","*",7,1],"Claude-SearchBot":["partial","*",7,1],"anthropic-ai":["partial","*",7,1],"PerplexityBot":["partial","*",7,1],"Perplexity-User":["partial","*",7,1],"Google-Extended":["partial","*",7,1],"Applebot-Extended":["partial","*",7,1],"CCBot":["partial","*",7,1],"Bytespider":["partial","*",7,1],"Amazonbot":["partial","*",7,1],"meta-externalagent":["partial","*",7,1],"cohere-ai":["partial","*",7,1],"Diffbot":["partial","*",7,1],"ContextFetchBot":["partial","*",7,1]},"llms":{"status":"missing","http":404,"bytes":92708},"home":{"status":"ok","http":200,"url":"https://uconn.edu/","html_tokens":36257,"markdown_tokens":384,"pruned_tokens":135}},
{"rank":1832,"domain":"thespruce.com","origin":"https://thespruce.com","robots":{"status":"ok","http":200,"url":"https://www.thespruce.com/robots.txt","bytes":3471,"groups":6},"bots":{"GPTBot":["partial","named",1,0],"ChatGPT-User":["partial","named",1,0],"OAI-SearchBot":["partial","named",1,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["blocked","named",1,0],"Claude-SearchBot":["partial","named",1,2],"anthropic-ai":["partial","named",1,2],"PerplexityBot":["partial","named",1,2],"Perplexity-User":["blocked","named",1,0],"Google-Extended":["partial","named",1,2],"Applebot-Extended":["blocked","named",1,0],"CCBot":["partial","named",1,2],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["partial","named",1,2],"cohere-ai":["partial","named",1,2],"Diffbot":["partial","*",2,0],"ContextFetchBot":["partial","*",2,0]},"llms":{"status":"http_error","http":402,"bytes":612},"home":{"status":"http_error","http":402,"url":"https://www.thespruce.com/"}},
{"rank":1833,"domain":"mofcom.gov.cn","origin":"https://www.mofcom.gov.cn","robots":{"status":"missing","http":404,"url":"https://www.mofcom.gov.cn/robots.txt","bytes":150,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":150},"home":{"status":"ok","http":200,"url":"https://www.mofcom.gov.cn/","html_tokens":8468,"markdown_tokens":134,"pruned_tokens":134}},
{"rank":1834,"domain":"myworkdayjobs.com","origin":"https://myworkdayjobs.com","robots":{"status":"unreachable","http":null,"url":"https://myworkdayjobs.com/robots.txt","bytes":0,"groups":0,"error":"myworkdayjobs.com is not a public address"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":1835,"domain":"scielo.br","origin":"https://scielo.br","robots":{"status":"ok","http":200,"url":"https://www.scielo.br/robots.txt","bytes":105,"groups":2},"bots":{"GPTBot":["partial","*",1,1],"ChatGPT-User":["partial","*",1,1],"OAI-SearchBot":["partial","*",1,1],"ClaudeBot":["partial","*",1,1],"Claude-User":["partial","*",1,1],"Claude-SearchBot":["partial","*",1,1],"anthropic-ai":["partial","*",1,1],"PerplexityBot":["partial","*",1,1],"Perplexity-User":["partial","*",1,1],"Google-Extended":["partial","*",1,1],"Applebot-Extended":["partial","*",1,1],"CCBot":["partial","*",1,1],"Bytespider":["partial","*",1,1],"Amazonbot":["partial","*",1,1],"meta-externalagent":["partial","*",1,1],"cohere-ai":["partial","*",1,1],"Diffbot":["partial","*",1,1],"ContextFetchBot":["partial","*",1,1]},"llms":{"status":"timeout","error":"timed out after 10 s"},"home":{"status":"ok","http":200,"url":"https://www.scielo.br/","html_tokens":16483,"markdown_tokens":17,"pruned_tokens":17}},
{"rank":1836,"domain":"bahn.de","origin":"https://bahn.de","robots":{"status":"ok","http":200,"url":"https://www.bahn.de/robots.txt","bytes":118,"groups":2},"bots":{"GPTBot":["partial","*",2,0],"ChatGPT-User":["partial","*",2,0],"OAI-SearchBot":["partial","*",2,0],"ClaudeBot":["partial","*",2,0],"Claude-User":["partial","*",2,0],"Claude-SearchBot":["partial","*",2,0],"anthropic-ai":["partial","*",2,0],"PerplexityBot":["partial","*",2,0],"Perplexity-User":["partial","*",2,0],"Google-Extended":["partial","*",2,0],"Applebot-Extended":["partial","*",2,0],"CCBot":["partial","*",2,0],"Bytespider":["partial","*",2,0],"Amazonbot":["partial","*",2,0],"meta-externalagent":["partial","*",2,0],"cohere-ai":["partial","*",2,0],"Diffbot":["partial","*",2,0],"ContextFetchBot":["partial","*",2,0]},"llms":{"status":"missing","http":404,"bytes":369},"home":{"status":"ok","http":200,"url":"https://www.bahn.de/","html_tokens":14834,"markdown_tokens":14,"pruned_tokens":14}},
{"rank":1837,"domain":"theage.com.au","origin":"https://theage.com.au","robots":{"status":"ok","http":200,"url":"https://www.theage.com.au/robots.txt","bytes":4074,"groups":52},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["blocked","named",1,0],"OAI-SearchBot":["blocked","named",1,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",5,1],"Claude-SearchBot":["partial","*",5,1],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["partial","*",5,1],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",5,1]},"llms":{"status":"missing","http":404,"bytes":170698},"home":{"status":"ok","http":200,"url":"https://www.theage.com.au/","html_tokens":304889,"markdown_tokens":1266,"pruned_tokens":151}},
{"rank":1838,"domain":"ohio.gov","origin":"https://ohio.gov","robots":{"status":"missing","http":404,"url":"https://ohio.gov/robots.txt","bytes":5140,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":5140},"home":{"status":"http_error","http":404,"url":"https://ohio.gov/"}},
{"rank":1839,"domain":"wdr.de","origin":"https://wdr.de","robots":{"status":"ok","http":200,"url":"https://www.wdr.de/robots.txt","bytes":2567,"groups":22},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["partial","*",9,5],"OAI-SearchBot":["partial","*",9,5],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",9,5],"Claude-SearchBot":["partial","*",9,5],"anthropic-ai":["partial","*",9,5],"PerplexityBot":["partial","*",9,5],"Perplexity-User":["partial","*",9,5],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["partial","*",9,5],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["partial","*",9,5],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",9,5]},"llms":{"status":"missing","http":410,"bytes":28144},"home":{"status":"ok","http":200,"url":"https://www1.wdr.de/index.html","html_tokens":104787,"markdown_tokens":1416,"pruned_tokens":56}},
{"rank":1840,"domain":"openedition.org","origin":"https://openedition.org","robots":{"status":"ok","http":200,"url":"https://www.openedition.org/robots.txt","bytes":135,"groups":1},"bots":{"GPTBot":["partial","*",3,0],"ChatGPT-User":["partial","*",3,0],"OAI-SearchBot":["partial","*",3,0],"ClaudeBot":["partial","*",3,0],"Claude-User":["partial","*",3,0],"Claude-SearchBot":["partial","*",3,0],"anthropic-ai":["partial","*",3,0],"PerplexityBot":["partial","*",3,0],"Perplexity-User":["partial","*",3,0],"Google-Extended":["partial","*",3,0],"Applebot-Extended":["partial","*",3,0],"CCBot":["partial","*",3,0],"Bytespider":["partial","*",3,0],"Amazonbot":["partial","*",3,0],"meta-externalagent":["partial","*",3,0],"cohere-ai":["partial","*",3,0],"Diffbot":["partial","*",3,0],"ContextFetchBot":["partial","*",3,0]},"llms":{"status":"missing","http":404,"bytes":0},"home":{"status":"ok","http":200,"url":"https://www.openedition.org/","html_tokens":40647,"markdown_tokens":1702,"pruned_tokens":66}},
{"rank":1841,"domain":"marca.com","origin":"https://marca.com","robots":{"status":"ok","http":200,"url":"https://www.marca.com/robots.txt","bytes":5011,"groups":116},"bots":{"GPTBot":["partial","named",1,2],"ChatGPT-User":["partial","named",1,2],"OAI-SearchBot":["partial","named",1,2],"ClaudeBot":["partial","*",13,0],"Claude-User":["partial","*",13,0],"Claude-SearchBot":["partial","*",13,0],"anthropic-ai":["partial","named",1,2],"PerplexityBot":["partial","named",1,2],"Perplexity-User":["partial","*",13,0],"Google-Extended":["partial","named",1,2],"Applebot-Extended":["partial","*",13,0],"CCBot":["partial","named",1,2],"Bytespider":["partial","*",13,0],"Amazonbot":["partial","*",13,0],"meta-externalagent":["partial","*",13,0],"cohere-ai":["partial","*",13,0],"Diffbot":["partial","*",13,0],"ContextFetchBot":["partial","*",13,0]},"llms":{"status":"missing","http":404,"bytes":168},"home":{"status":"ok","http":200,"url":"https://www.marca.com/","html_tokens":191093,"markdown_tokens":7,"pruned_tokens":7}},
{"rank":1842,"domain":"uh.edu","origin":"https://uh.edu","robots":{"status":"ok","http":200,"url":"https://www.uh.edu/robots.txt","bytes":964,"groups":3},"bots":{"GPTBot":["partial","*",27,0],"ChatGPT-User":["partial","*",27,0],"OAI-SearchBot":["partial","*",27,0],"ClaudeBot":["partial","*",27,0],"Claude-User":["partial","*",27,0],"Claude-SearchBot":["partial","*",27,0],"anthropic-ai":["partial","*",27,0],"PerplexityBot":["partial","*",27,0],"Perplexity-User":["partial","*",27,0],"Google-Extended":["partial","*",27,0],"Applebot-Extended":["partial","*",27,0],"CCBot":["partial","*",27,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["partial","*",27,0],"meta-externalagent":["partial","*",27,0],"cohere-ai":["partial","*",27,0],"Diffbot":["partial","*",27,0],"ContextFetchBot":["partial","*",27,0]},"llms":{"status":"missing","http":404,"bytes":24065},"home":{"status":"ok","http":200,"url":"https://www.uh.edu/","html_tokens":45996,"markdown_tokens":499,"pruned_tokens":80}},
{"rank":1843,"domain":"huffingtonpost.co.uk","origin":"https://huffingtonpost.co.uk","robots":{"status":"ok","http":200,"url":"https://www.huffingtonpost.co.uk/robots.txt","bytes":2055,"groups":24},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["blocked","named",1,0],"OAI-SearchBot":["blocked","named",1,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["blocked","named",1,0],"Claude-SearchBot":["blocked","named",1,0],"anthropic-ai":["partial","*",5,0],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["blocked","named",1,0],"Google-Extended":["partial","*",5,0],"Applebot-Extended":["partial","*",5,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["partial","*",5,0],"Diffbot":["partial","*",5,0],"ContextFetchBot":["partial","*",5,0]},"llms":{"status":"missing","http":404,"bytes":115790},"home":{"status":"ok","http":200,"url":"https://www.huffingtonpost.co.uk/","html_tokens":137754,"markdown_tokens":19,"pruned_tokens":19}},
{"rank":1844,"domain":"codecademy.com","origin":"https://codecademy.com","robots":{"status":"ok","http":200,"url":"https://www.codecademy.com/robots.txt","bytes":198,"groups":1},"bots":{"GPTBot":["partial","*",6,0],"ChatGPT-User":["partial","*",6,0],"OAI-SearchBot":["partial","*",6,0],"ClaudeBot":["partial","*",6,0],"Claude-User":["partial","*",6,0],"Claude-SearchBot":["partial","*",6,0],"anthropic-ai":["partial","*",6,0],"PerplexityBot":["partial","*",6,0],"Perplexity-User":["partial","*",6,0],"Google-Extended":["partial","*",6,0],"Applebot-Extended":["partial","*",6,0],"CCBot":["partial","*",6,0],"Bytespider":["partial","*",6,0],"Amazonbot":["partial","*",6,0],"meta-externalagent":["partial","*",6,0],"cohere-ai":["partial","*",6,0],"Diffbot":["partial","*",6,0],"ContextFetchBot":["partial","*",6,0]},"llms":{"status":"missing","http":404,"bytes":0},"home":{"status":"ok","http":200,"url":"https://www.codecademy.com/","html_tokens":109877,"markdown_tokens":577,"pruned_tokens":263}},
{"rank":1845,"domain":"fws.gov","origin":"https://fws.gov","robots":{"status":"ok","http":200,"url":"https://www.fws.gov/robots.txt","bytes":2243,"groups":1},"bots":{"GPTBot":["partial","*",38,18],"ChatGPT-User":["partial","*",38,18],"OAI-SearchBot":["partial","*",38,18],"ClaudeBot":["partial","*",38,18],"Claude-User":["partial","*",38,18],"Claude-SearchBot":["partial","*",38,18],"anthropic-ai":["partial","*",38,18],"PerplexityBot":["partial","*",38,18],"Perplexity-User":["partial","*",38,18],"Google-Extended":["partial","*",38,18],"Applebot-Extended":["partial","*",38,18],"CCBot":["partial","*",38,18],"Bytespider":["partial","*",38,18],"Amazonbot":["partial","*",38,18],"meta-externalagent":["partial","*",38,18],"cohere-ai":["partial","*",38,18],"Diffbot":["partial","*",38,18],"ContextFetchBot":["partial","*",38,18]},"llms":{"status":"missing","http":404,"bytes":708},"home":{"status":"ok","http":200,"url":"https://www.fws.gov/","html_tokens":56745,"markdown_tokens":246,"pruned_tokens":88}},
{"rank":1846,"domain":"mo.gov","origin":"https://mo.gov","robots":{"status":"ok","http":200,"url":"https://mo.gov/robots.txt","bytes":1593,"groups":1},"bots":{"GPTBot":["partial","*",22,18],"ChatGPT-User":["partial","*",22,18],"OAI-SearchBot":["partial","*",22,18],"ClaudeBot":["partial","*",22,18],"Claude-User":["partial","*",22,18],"Claude-SearchBot":["partial","*",22,18],"anthropic-ai":["partial","*",22,18],"PerplexityBot":["partial","*",22,18],"Perplexity-User":["partial","*",22,18],"Google-Extended":["partial","*",22,18],"Applebot-Extended":["partial","*",22,18],"CCBot":["partial","*",22,18],"Bytespider":["partial","*",22,18],"Amazonbot":["partial","*",22,18],"meta-externalagent":["partial","*",22,18],"cohere-ai":["partial","*",22,18],"Diffbot":["partial","*",22,18],"ContextFetchBot":["partial","*",22,18]},"llms":{"status":"missing","http":404,"bytes":313},"home":{"status":"ok","http":200,"url":"https://mo.gov/","html_tokens":18882,"markdown_tokens":213,"pruned_tokens":164}},
{"rank":1847,"domain":"mordorintelligence.com","origin":"https://mordorintelligence.com","robots":{"status":"ok","http":200,"url":"https://mordorintelligence.com/robots.txt","bytes":156,"groups":1},"bots":{"GPTBot":["partial","*",4,0],"ChatGPT-User":["partial","*",4,0],"OAI-SearchBot":["partial","*",4,0],"ClaudeBot":["partial","*",4,0],"Claude-User":["partial","*",4,0],"Claude-SearchBot":["partial","*",4,0],"anthropic-ai":["partial","*",4,0],"PerplexityBot":["partial","*",4,0],"Perplexity-User":["partial","*",4,0],"Google-Extended":["partial","*",4,0],"Applebot-Extended":["partial","*",4,0],"CCBot":["partial","*",4,0],"Bytespider":["partial","*",4,0],"Amazonbot":["partial","*",4,0],"meta-externalagent":["partial","*",4,0],"cohere-ai":["partial","*",4,0],"Diffbot":["partial","*",4,0],"ContextFetchBot":["partial","*",4,0]},"llms":{"status":"missing","http":404,"bytes":6708},"home":{"status":"ok","http":200,"url":"https://www.mordorintelligence.com/","html_tokens":17634,"markdown_tokens":2361,"pruned_tokens":416}},
{"rank":1848,"domain":"bundesregierung.de","origin":"https://bundesregierung.de","robots":{"status":"ok","http":200,"url":"https://www.bundesregierung.de/robots.txt","bytes":1935,"groups":44},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":114941},"home":{"status":"ok","http":200,"url":"https://www.bundesregierung.de/breg-de","html_tokens":48366,"markdown_tokens":311,"pruned_tokens":60}},
{"rank":1849,"domain":"csis.org","origin":"https://csis.org","robots":{"status":"ok","http":200,"url":"https://csis.org/robots.txt","bytes":2120,"groups":2},"bots":{"GPTBot":["partial","*",36,19],"ChatGPT-User":["partial","*",36,19],"OAI-SearchBot":["partial","*",36,19],"ClaudeBot":["partial","*",36,19],"Claude-User":["partial","*",36,19],"Claude-SearchBot":["partial","*",36,19],"anthropic-ai":["partial","*",36,19],"PerplexityBot":["partial","*",36,19],"Perplexity-User":["partial","*",36,19],"Google-Extended":["partial","*",36,19],"Applebot-Extended":["partial","*",36,19],"CCBot":["partial","*",36,19],"Bytespider":["blocked","named",1,0],"Amazonbot":["partial","*",36,19],"meta-externalagent":["partial","*",36,19],"cohere-ai":["partial","*",36,19],"Diffbot":["partial","*",36,19],"ContextFetchBot":["partial","*",36,19]},"llms":{"status":"missing","http":404,"bytes":547},"home":{"status":"ok","http":200,"url":"https://www.csis.org/","html_tokens":84754,"markdown_tokens":167,"pruned_tokens":4}},
{"rank":1850,"domain":"flippingbook.com","origin":"https://flippingbook.com","robots":{"status":"ok","http":200,"url":"https://flippingbook.com/robots.txt","bytes":1842,"groups":3},"bots":{"GPTBot":["partial","*",20,2],"ChatGPT-User":["partial","*",20,2],"OAI-SearchBot":["partial","*",20,2],"ClaudeBot":["partial","*",20,2],"Claude-User":["partial","*",20,2],"Claude-SearchBot":["partial","*",20,2],"anthropic-ai":["partial","*",20,2],"PerplexityBot":["partial","*",20,2],"Perplexity-User":["partial","*",20,2],"Google-Extended":["partial","*",20,2],"Applebot-Extended":["partial","*",20,2],"CCBot":["partial","*",20,2],"Bytespider":["partial","*",20,2],"Amazonbot":["partial","*",20,2],"meta-externalagent":["partial","*",20,2],"cohere-ai":["partial","*",20,2],"Diffbot":["partial","*",20,2],"ContextFetchBot":["partial","*",20,2]},"llms":{"status":"missing","http":404,"bytes":38248},"home":{"status":"ok","http":200,"url":"https://flippingbook.com/","html_tokens":62792,"markdown_tokens":1815,"pruned_tokens":826}},
{"rank":1851,"domain":"gq.com","origin":"https://gq.com","robots":{"status":"ok","http":200,"url":"https://www.gq.com/robots.txt","bytes":2157,"groups":3},"bots":{"GPTBot":["partial","*",13,15],"ChatGPT-User":["partial","*",13,15],"OAI-SearchBot":["partial","*",13,15],"ClaudeBot":["blocked","named",1,0],"Claude-User":["blocked","named",1,0],"Claude-SearchBot":["blocked","named",1,0],"anthropic-ai":["partial","*",13,15],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["blocked","named",1,0],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",13,15]},"llms":{"status":"missing","http":404,"bytes":788966},"home":{"status":"ok","http":200,"url":"https://www.gq.com/","html_tokens":612517,"markdown_tokens":1329,"pruned_tokens":7}},
{"rank":1852,"domain":"bitcoin.org","origin":"https://bitcoin.org","robots":{"status":"ok","http":200,"url":"https://bitcoin.org/robots.txt","bytes":43,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":513},"home":{"status":"ok","http":200,"url":"https://bitcoin.org/en/","html_tokens":8491,"markdown_tokens":129,"pruned_tokens":129}},
{"rank":1853,"domain":"rug.nl","origin":"https://rug.nl","robots":{"status":"ok","http":200,"url":"https://www.rug.nl/robots.txt","bytes":715,"groups":2},"bots":{"GPTBot":["partial","*",22,0],"ChatGPT-User":["partial","*",22,0],"OAI-SearchBot":["partial","*",22,0],"ClaudeBot":["partial","*",22,0],"Claude-User":["partial","*",22,0],"Claude-SearchBot":["partial","*",22,0],"anthropic-ai":["partial","*",22,0],"PerplexityBot":["partial","*",22,0],"Perplexity-User":["partial","*",22,0],"Google-Extended":["partial","*",22,0],"Applebot-Extended":["partial","*",22,0],"CCBot":["partial","*",22,0],"Bytespider":["partial","*",22,0],"Amazonbot":["partial","*",22,0],"meta-externalagent":["partial","*",22,0],"cohere-ai":["partial","*",22,0],"Diffbot":["partial","*",22,0],"ContextFetchBot":["partial","*",22,0]},"llms":{"status":"missing","http":404,"bytes":37704},"home":{"status":"ok","http":200,"url":"https://www.rug.nl/","html_tokens":12279,"markdown_tokens":185,"pruned_tokens":169}},
{"rank":1854,"domain":"inria.fr","origin":"https://inria.fr","robots":{"status":"ok","http":200,"url":"https://inria.fr/robots.txt","bytes":1594,"groups":1},"bots":{"GPTBot":["partial","*",22,18],"ChatGPT-User":["partial","*",22,18],"OAI-SearchBot":["partial","*",22,18],"ClaudeBot":["partial","*",22,18],"Claude-User":["partial","*",22,18],"Claude-SearchBot":["partial","*",22,18],"anthropic-ai":["partial","*",22,18],"PerplexityBot":["partial","*",22,18],"Perplexity-User":["partial","*",22,18],"Google-Extended":["partial","*",22,18],"Applebot-Extended":["partial","*",22,18],"CCBot":["partial","*",22,18],"Bytespider":["partial","*",22,18],"Amazonbot":["partial","*",22,18],"meta-externalagent":["partial","*",22,18],"cohere-ai":["partial","*",22,18],"Diffbot":["partial","*",22,18],"ContextFetchBot":["partial","*",22,18]},"llms":{"status":"missing","http":404,"bytes":182},"home":{"status":"ok","http":200,"url":"https://inria.fr/fr","html_tokens":35340,"markdown_tokens":8,"pruned_tokens":8}},
{"rank":1855,"domain":"ucalgary.ca","origin":"https://ucalgary.ca","robots":{"status":"ok","http":200,"url":"https://ucalgary.ca/robots.txt","bytes":1817,"groups":1},"bots":{"GPTBot":["partial","*",34,9],"ChatGPT-User":["partial","*",34,9],"OAI-SearchBot":["partial","*",34,9],"ClaudeBot":["partial","*",34,9],"Claude-User":["partial","*",34,9],"Claude-SearchBot":["partial","*",34,9],"anthropic-ai":["partial","*",34,9],"PerplexityBot":["partial","*",34,9],"Perplexity-User":["partial","*",34,9],"Google-Extended":["partial","*",34,9],"Applebot-Extended":["partial","*",34,9],"CCBot":["partial","*",34,9],"Bytespider":["partial","*",34,9],"Amazonbot":["partial","*",34,9],"meta-externalagent":["partial","*",34,9],"cohere-ai":["partial","*",34,9],"Diffbot":["partial","*",34,9],"ContextFetchBot":["partial","*",34,9]},"llms":{"status":"missing","http":404,"bytes":83765},"home":{"status":"ok","http":200,"url":"https://ucalgary.ca/","html_tokens":30996,"markdown_tokens":96,"pruned_tokens":96}},
{"rank":1856,"domain":"google.be","origin":"https://google.be","robots":{"status":"ok","http":200,"url":"https://www.google.be/robots.txt","bytes":6624,"groups":4},"bots":{"GPTBot":["partial","*",169,64],"ChatGPT-User":["partial","*",169,64],"OAI-SearchBot":["partial","*",169,64],"ClaudeBot":["partial","*",169,64],"Claude-User":["partial","*",169,64],"Claude-SearchBot":["partial","*",169,64],"anthropic-ai":["partial","*",169,64],"PerplexityBot":["partial","*",169,64],"Perplexity-User":["partial","*",169,64],"Google-Extended":["partial","*",169,64],"Applebot-Extended":["partial","*",169,64],"CCBot":["partial","*",169,64],"Bytespider":["partial","*",169,64],"Amazonbot":["partial","*",169,64],"meta-externalagent":["partial","*",169,64],"cohere-ai":["partial","*",169,64],"Diffbot":["partial","*",169,64],"ContextFetchBot":["partial","*",169,64]},"llms":{"status":"missing","http":404,"bytes":1569},"home":{"status":"ok","http":200,"url":"https://www.google.com/","html_tokens":21340,"markdown_tokens":30,"pruned_tokens":30}},
{"rank":1857,"domain":"gingerall.com","origin":"https://gingerall.com","robots":{"status":"timeout","http":null,"url":"https://gingerall.com/robots.txt","bytes":0,"groups":0,"error":"ConnectTimeout"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":1858,"domain":"ebscohost.com","origin":"https://ebscohost.com","robots":{"status":"refused","http":403,"url":"https://ebscohost.com/robots.txt","bytes":560,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":1859,"domain":"gitea.io","origin":"https://gitea.io","robots":{"status":"ok","http":200,"url":"https://gitea.io/robots.txt","bytes":2676,"groups":32},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["allowed","*",0,1],"OAI-SearchBot":["allowed","*",0,1],"ClaudeBot":["blocked","named",1,0],"Claude-User":["allowed","*",0,1],"Claude-SearchBot":["allowed","*",0,1],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["allowed","*",0,1],"Perplexity-User":["allowed","*",0,1],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["allowed","*",0,1]},"llms":{"status":"missing","http":404,"bytes":10330},"home":{"status":"ok","http":200,"url":"https://about.gitea.com","html_tokens":42610,"markdown_tokens":1056,"pruned_tokens":398}},
{"rank":1860,"domain":"gitlab.io","origin":"https://gitlab.io","robots":{"status":"html","http":200,"url":"https://about.gitlab.com/","bytes":217137,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":1861,"domain":"poynter.org","origin":"https://poynter.org","robots":{"status":"ok","http":200,"url":"https://poynter.org/robots.txt","bytes":113,"groups":2},"bots":{"GPTBot":["partial","*",2,0],"ChatGPT-User":["partial","*",2,0],"OAI-SearchBot":["partial","*",2,0],"ClaudeBot":["partial","*",2,0],"Claude-User":["partial","*",2,0],"Claude-SearchBot":["partial","*",2,0],"anthropic-ai":["partial","*",2,0],"PerplexityBot":["partial","*",2,0],"Perplexity-User":["partial","*",2,0],"Google-Extended":["partial","*",2,0],"Applebot-Extended":["partial","*",2,0],"CCBot":["partial","*",2,0],"Bytespider":["partial","*",2,0],"Amazonbot":["partial","*",2,0],"meta-externalagent":["partial","*",2,0],"cohere-ai":["partial","*",2,0],"Diffbot":["partial","*",2,0],"ContextFetchBot":["partial","*",2,0]},"llms":{"status":"crawl_delay","error":"Crawl-delay 600 s"},"home":{"status":"crawl_delay","error":"Crawl-delay 600 s"}},
{"rank":1862,"domain":"travelandleisure.com","origin":"https://travelandleisure.com","robots":{"status":"ok","http":200,"url":"https://www.travelandleisure.com/robots.txt","bytes":3485,"groups":6},"bots":{"GPTBot":["partial","named",1,0],"ChatGPT-User":["partial","named",1,0],"OAI-SearchBot":["partial","named",1,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["blocked","named",1,0],"Claude-SearchBot":["partial","named",1,2],"anthropic-ai":["partial","named",1,2],"PerplexityBot":["partial","named",1,2],"Perplexity-User":["blocked","named",1,0],"Google-Extended":["partial","named",1,2],"Applebot-Extended":["blocked","named",1,0],"CCBot":["partial","named",1,2],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["partial","named",1,2],"cohere-ai":["partial","named",1,2],"Diffbot":["partial","*",2,0],"ContextFetchBot":["partial","*",2,0]},"llms":{"status":"http_error","http":402,"bytes":612},"home":{"status":"http_error","http":402,"url":"https://www.travelandleisure.com/"}},
{"rank":1863,"domain":"golang.org","origin":"https://golang.org","robots":{"status":"ok","http":200,"url":"https://go.dev/robots.txt","bytes":23,"groups":1},"bots":{"GPTBot":["allowed","*",0,1],"ChatGPT-User":["allowed","*",0,1],"OAI-SearchBot":["allowed","*",0,1],"ClaudeBot":["allowed","*",0,1],"Claude-User":["allowed","*",0,1],"Claude-SearchBot":["allowed","*",0,1],"anthropic-ai":["allowed","*",0,1],"PerplexityBot":["allowed","*",0,1],"Perplexity-User":["allowed","*",0,1],"Google-Extended":["allowed","*",0,1],"Applebot-Extended":["allowed","*",0,1],"CCBot":["allowed","*",0,1],"Bytespider":["allowed","*",0,1],"Amazonbot":["allowed","*",0,1],"meta-externalagent":["allowed","*",0,1],"cohere-ai":["allowed","*",0,1],"Diffbot":["allowed","*",0,1],"ContextFetchBot":["allowed","*",0,1]},"llms":{"status":"missing","http":404,"bytes":26974},"home":{"status":"ok","http":200,"url":"https://go.dev/","html_tokens":16039,"markdown_tokens":582,"pruned_tokens":119}},
{"rank":1864,"domain":"as.com","origin":"https://as.com","robots":{"status":"ok","http":200,"url":"https://as.com/robots.txt","bytes":4616,"groups":101},"bots":{"GPTBot":["partial","*",27,0],"ChatGPT-User":["partial","*",27,0],"OAI-SearchBot":["partial","*",27,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["blocked","named",1,0],"Claude-SearchBot":["blocked","named",1,0],"anthropic-ai":["partial","*",27,0],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["blocked","named",1,0],"Google-Extended":["partial","*",27,0],"Applebot-Extended":["partial","*",27,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["partial","*",27,0],"cohere-ai":["partial","*",27,0],"Diffbot":["partial","*",27,0],"ContextFetchBot":["partial","*",27,0]},"llms":{"status":"missing","http":404,"bytes":209152},"home":{"status":"http_error","http":403,"url":"https://as.com/"}},
{"rank":1865,"domain":"yomiuri.co.jp","origin":"https://yomiuri.co.jp","robots":{"status":"ok","http":200,"url":"https://www.yomiuri.co.jp/robots.txt","bytes":3619,"groups":10},"bots":{"GPTBot":["partial","named",1,2],"ChatGPT-User":["partial","named",1,2],"OAI-SearchBot":["partial","named",1,4],"ClaudeBot":["partial","named",1,2],"Claude-User":["partial","named",1,2],"Claude-SearchBot":["partial","named",1,2],"anthropic-ai":["partial","named",1,2],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["blocked","named",1,0],"Google-Extended":["partial","named",55,4],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["partial","named",1,4],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",3,0]},"llms":{"status":"missing","http":404,"bytes":82431},"home":{"status":"ok","http":200,"url":"https://www.yomiuri.co.jp/","html_tokens":104666,"markdown_tokens":3478,"pruned_tokens":175}},
{"rank":1866,"domain":"observer.com","origin":"https://observer.com","robots":{"status":"ok","http":200,"url":"https://observer.com/robots.txt","bytes":174,"groups":2},"bots":{"GPTBot":["partial","*",2,1],"ChatGPT-User":["partial","*",2,1],"OAI-SearchBot":["partial","*",2,1],"ClaudeBot":["partial","*",2,1],"Claude-User":["partial","*",2,1],"Claude-SearchBot":["partial","*",2,1],"anthropic-ai":["partial","*",2,1],"PerplexityBot":["partial","*",2,1],"Perplexity-User":["partial","*",2,1],"Google-Extended":["partial","*",2,1],"Applebot-Extended":["partial","*",2,1],"CCBot":["partial","*",2,1],"Bytespider":["partial","*",2,1],"Amazonbot":["partial","*",2,1],"meta-externalagent":["partial","*",2,1],"cohere-ai":["partial","*",2,1],"Diffbot":["partial","*",2,1],"ContextFetchBot":["partial","*",2,1]},"llms":{"status":"missing","http":404,"bytes":67522},"home":{"status":"ok","http":200,"url":"https://observer.com/","html_tokens":91823,"markdown_tokens":75,"pruned_tokens":75}},
{"rank":1867,"domain":"qualcomm.com","origin":"https://qualcomm.com","robots":{"status":"ok","http":200,"url":"https://www.qualcomm.com/robots.txt","bytes":492,"groups":6},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"present","http":200,"bytes":1269},"home":{"status":"ok","http":200,"url":"https://www.qualcomm.com","html_tokens":2497,"markdown_tokens":12,"pruned_tokens":12}},
{"rank":1868,"domain":"asp.net","origin":"https://asp.net","robots":{"status":"html","http":200,"url":"https://dotnet.microsoft.com/en-us/apps/aspnet","bytes":239170,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":1869,"domain":"inquirer.com","origin":"https://inquirer.com","robots":{"status":"ok","http":200,"url":"https://www.inquirer.com/robots.txt","bytes":547,"groups":1},"bots":{"GPTBot":["partial","*",10,0],"ChatGPT-User":["partial","*",10,0],"OAI-SearchBot":["partial","*",10,0],"ClaudeBot":["partial","*",10,0],"Claude-User":["partial","*",10,0],"Claude-SearchBot":["partial","*",10,0],"anthropic-ai":["partial","*",10,0],"PerplexityBot":["partial","*",10,0],"Perplexity-User":["partial","*",10,0],"Google-Extended":["partial","*",10,0],"Applebot-Extended":["partial","*",10,0],"CCBot":["partial","*",10,0],"Bytespider":["partial","*",10,0],"Amazonbot":["partial","*",10,0],"meta-externalagent":["partial","*",10,0],"cohere-ai":["partial","*",10,0],"Diffbot":["partial","*",10,0],"ContextFetchBot":["partial","*",10,0]},"llms":{"status":"missing","http":404,"bytes":324910},"home":{"status":"ok","http":200,"url":"https://www.inquirer.com/","html_tokens":790016,"markdown_tokens":3,"pruned_tokens":3}},
{"rank":1870,"domain":"timesonline.co.uk","origin":"http://timesonline.co.uk","robots":{"status":"html","http":200,"url":"https://www.thetimes.com/","bytes":1426840,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":1871,"domain":"windows.com","origin":"https://windows.com","robots":{"status":"html","http":200,"url":"https://www.microsoft.com/en-us/windows/","bytes":508451,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":1872,"domain":"make.wordpress.org","origin":"https://make.wordpress.org","robots":{"status":"ok","http":200,"url":"https://make.wordpress.org/robots.txt","bytes":2603,"groups":1},"bots":{"GPTBot":["partial","*",1,3],"ChatGPT-User":["partial","*",1,3],"OAI-SearchBot":["partial","*",1,3],"ClaudeBot":["partial","*",1,3],"Claude-User":["partial","*",1,3],"Claude-SearchBot":["partial","*",1,3],"anthropic-ai":["partial","*",1,3],"PerplexityBot":["partial","*",1,3],"Perplexity-User":["partial","*",1,3],"Google-Extended":["partial","*",1,3],"Applebot-Extended":["partial","*",1,3],"CCBot":["partial","*",1,3],"Bytespider":["partial","*",1,3],"Amazonbot":["partial","*",1,3],"meta-externalagent":["partial","*",1,3],"cohere-ai":["partial","*",1,3],"Diffbot":["partial","*",1,3],"ContextFetchBot":["partial","*",1,3]},"llms":{"status":"missing","http":404,"bytes":148771},"home":{"status":"ok","http":200,"url":"https://make.wordpress.org/","html_tokens":56195,"markdown_tokens":81,"pruned_tokens":81}},
{"rank":1873,"domain":"djangoproject.com","origin":"https://djangoproject.com","robots":{"status":"ok","http":200,"url":"https://www.djangoproject.com/robots.txt","bytes":53,"groups":1},"bots":{"GPTBot":["partial","*",2,0],"ChatGPT-User":["partial","*",2,0],"OAI-SearchBot":["partial","*",2,0],"ClaudeBot":["partial","*",2,0],"Claude-User":["partial","*",2,0],"Claude-SearchBot":["partial","*",2,0],"anthropic-ai":["partial","*",2,0],"PerplexityBot":["partial","*",2,0],"Perplexity-User":["partial","*",2,0],"Google-Extended":["partial","*",2,0],"Applebot-Extended":["partial","*",2,0],"CCBot":["partial","*",2,0],"Bytespider":["partial","*",2,0],"Amazonbot":["partial","*",2,0],"meta-externalagent":["partial","*",2,0],"cohere-ai":["partial","*",2,0],"Diffbot":["partial","*",2,0],"ContextFetchBot":["partial","*",2,0]},"llms":{"status":"missing","http":404,"bytes":14421},"home":{"status":"ok","http":200,"url":"https://www.djangoproject.com/","html_tokens":5155,"markdown_tokens":160,"pruned_tokens":160}},
{"rank":1874,"domain":"uni-muenchen.de","origin":"https://www.uni-muenchen.de","robots":{"status":"ok","http":200,"url":"https://www.lmu.de/robots.txt","bytes":3293,"groups":1},"bots":{"GPTBot":["partial","*",44,0],"ChatGPT-User":["partial","*",44,0],"OAI-SearchBot":["partial","*",44,0],"ClaudeBot":["partial","*",44,0],"Claude-User":["partial","*",44,0],"Claude-SearchBot":["partial","*",44,0],"anthropic-ai":["partial","*",44,0],"PerplexityBot":["partial","*",44,0],"Perplexity-User":["partial","*",44,0],"Google-Extended":["partial","*",44,0],"Applebot-Extended":["partial","*",44,0],"CCBot":["partial","*",44,0],"Bytespider":["partial","*",44,0],"Amazonbot":["partial","*",44,0],"meta-externalagent":["partial","*",44,0],"cohere-ai":["partial","*",44,0],"Diffbot":["partial","*",44,0],"ContextFetchBot":["partial","*",44,0]},"llms":{"status":"missing","http":404,"bytes":18469},"home":{"status":"ok","http":200,"url":"https://www.lmu.de/de/index.html","html_tokens":10638,"markdown_tokens":414,"pruned_tokens":130}},
{"rank":1875,"domain":"yorku.ca","origin":"https://yorku.ca","robots":{"status":"ok","http":200,"url":"https://www.yorku.ca/robots.txt","bytes":4974,"groups":1},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"missing","http":404,"bytes":80636},"home":{"status":"ok","http":200,"url":"https://www.yorku.ca/","html_tokens":40958,"markdown_tokens":250,"pruned_tokens":197}},
{"rank":1876,"domain":"schneier.com","origin":"https://schneier.com","robots":{"status":"ok","http":200,"url":"https://www.schneier.com/robots.txt","bytes":600,"groups":3},"bots":{"GPTBot":["partial","*",7,1],"ChatGPT-User":["partial","*",7,1],"OAI-SearchBot":["partial","*",7,1],"ClaudeBot":["partial","*",7,1],"Claude-User":["partial","*",7,1],"Claude-SearchBot":["partial","*",7,1],"anthropic-ai":["partial","*",7,1],"PerplexityBot":["partial","*",7,1],"Perplexity-User":["partial","*",7,1],"Google-Extended":["partial","*",7,1],"Applebot-Extended":["partial","*",7,1],"CCBot":["partial","*",7,1],"Bytespider":["partial","*",7,1],"Amazonbot":["partial","*",7,1],"meta-externalagent":["partial","*",7,1],"cohere-ai":["partial","*",7,1],"Diffbot":["partial","*",7,1],"ContextFetchBot":["partial","*",7,1]},"llms":{"status":"missing","http":404,"bytes":42733},"home":{"status":"ok","http":200,"url":"https://www.schneier.com/","html_tokens":20690,"markdown_tokens":4844,"pruned_tokens":12}},
{"rank":1877,"domain":"nfpa.org","origin":"https://nfpa.org","robots":{"status":"ok","http":200,"url":"https://www.nfpa.org/robots.txt","bytes":467,"groups":4},"bots":{"GPTBot":["partial","*",5,3],"ChatGPT-User":["partial","*",5,3],"OAI-SearchBot":["partial","*",5,3],"ClaudeBot":["partial","*",5,3],"Claude-User":["partial","*",5,3],"Claude-SearchBot":["partial","*",5,3],"anthropic-ai":["partial","*",5,3],"PerplexityBot":["partial","*",5,3],"Perplexity-User":["partial","*",5,3],"Google-Extended":["partial","*",5,3],"Applebot-Extended":["partial","*",5,3],"CCBot":["partial","*",5,3],"Bytespider":["partial","*",5,3],"Amazonbot":["partial","*",5,3],"meta-externalagent":["partial","*",5,3],"cohere-ai":["partial","*",5,3],"Diffbot":["partial","*",5,3],"ContextFetchBot":["partial","*",5,3]},"llms":{"status":"missing","http":404,"bytes":221891},"home":{"status":"ok","http":200,"url":"https://www.nfpa.org/","html_tokens":63697,"markdown_tokens":0,"pruned_tokens":0}},
{"rank":1878,"domain":"azcentral.com","origin":"https://azcentral.com","robots":{"status":"ok","http":200,"url":"https://www.azcentral.com/robots.txt","bytes":12851,"groups":288},"bots":{"GPTBot":["partial","named",1,4],"ChatGPT-User":["partial","named",1,4],"OAI-SearchBot":["partial","named",1,4],"ClaudeBot":["blocked","named",1,0],"Claude-User":["blocked","named",1,0],"Claude-SearchBot":["blocked","named",1,0],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["partial","named",1,4],"Perplexity-User":["blocked","named",1,0],"Google-Extended":["partial","named",1,4],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",2,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",34,0]},"llms":{"status":"http_error","http":402,"bytes":254},"home":{"status":"http_error","http":402,"url":"https://www.azcentral.com/"}},
{"rank":1879,"domain":"clck.ru","origin":"https://clck.ru","robots":{"status":"ok","http":200,"url":"https://clck.ru/robots.txt","bytes":35,"groups":1},"bots":{"GPTBot":["partial","*",1,1],"ChatGPT-User":["partial","*",1,1],"OAI-SearchBot":["partial","*",1,1],"ClaudeBot":["partial","*",1,1],"Claude-User":["partial","*",1,1],"Claude-SearchBot":["partial","*",1,1],"anthropic-ai":["partial","*",1,1],"PerplexityBot":["partial","*",1,1],"Perplexity-User":["partial","*",1,1],"Google-Extended":["partial","*",1,1],"Applebot-Extended":["partial","*",1,1],"CCBot":["partial","*",1,1],"Bytespider":["partial","*",1,1],"Amazonbot":["partial","*",1,1],"meta-externalagent":["partial","*",1,1],"cohere-ai":["partial","*",1,1],"Diffbot":["partial","*",1,1],"ContextFetchBot":["partial","*",1,1]},"llms":{"status":"disallowed"},"home":{"status":"ok","http":200,"url":"https://clck.ru/","html_tokens":2160,"markdown_tokens":19,"pruned_tokens":19}},
{"rank":1880,"domain":"reliefweb.int","origin":"https://reliefweb.int","robots":{"status":"ok","http":200,"url":"https://reliefweb.int/robots.txt","bytes":2323,"groups":1},"bots":{"GPTBot":["partial","*",38,22],"ChatGPT-User":["partial","*",38,22],"OAI-SearchBot":["partial","*",38,22],"ClaudeBot":["partial","*",38,22],"Claude-User":["partial","*",38,22],"Claude-SearchBot":["partial","*",38,22],"anthropic-ai":["partial","*",38,22],"PerplexityBot":["partial","*",38,22],"Perplexity-User":["partial","*",38,22],"Google-Extended":["partial","*",38,22],"Applebot-Extended":["partial","*",38,22],"CCBot":["partial","*",38,22],"Bytespider":["partial","*",38,22],"Amazonbot":["partial","*",38,22],"meta-externalagent":["partial","*",38,22],"cohere-ai":["partial","*",38,22],"Diffbot":["partial","*",38,22],"ContextFetchBot":["partial","*",38,22]},"llms":{"status":"missing","http":404,"bytes":146},"home":{"status":"ok","http":200,"url":"https://reliefweb.int/","html_tokens":34005,"markdown_tokens":501,"pruned_tokens":101}},
{"rank":1881,"domain":"delta.com","origin":"https://www.delta.com","robots":{"status":"ok","http":200,"url":"https://www.delta.com/robots.txt","bytes":6163,"groups":1},"bots":{"GPTBot":["partial","*",156,37],"ChatGPT-User":["partial","*",156,37],"OAI-SearchBot":["partial","*",156,37],"ClaudeBot":["partial","*",156,37],"Claude-User":["partial","*",156,37],"Claude-SearchBot":["partial","*",156,37],"anthropic-ai":["partial","*",156,37],"PerplexityBot":["partial","*",156,37],"Perplexity-User":["partial","*",156,37],"Google-Extended":["partial","*",156,37],"Applebot-Extended":["partial","*",156,37],"CCBot":["partial","*",156,37],"Bytespider":["partial","*",156,37],"Amazonbot":["partial","*",156,37],"meta-externalagent":["partial","*",156,37],"cohere-ai":["partial","*",156,37],"Diffbot":["partial","*",156,37],"ContextFetchBot":["partial","*",156,37]},"llms":{"status":"missing","http":404,"bytes":48652},"home":{"status":"ok","http":200,"url":"https://www.delta.com/","html_tokens":2089,"markdown_tokens":0,"pruned_tokens":0}},
{"rank":1882,"domain":"deepmind.google","origin":"https://deepmind.google","robots":{"status":"ok","http":200,"url":"https://deepmind.google/robots.txt","bytes":69,"groups":1},"bots":{"GPTBot":["allowed","*",0,1],"ChatGPT-User":["allowed","*",0,1],"OAI-SearchBot":["allowed","*",0,1],"ClaudeBot":["allowed","*",0,1],"Claude-User":["allowed","*",0,1],"Claude-SearchBot":["allowed","*",0,1],"anthropic-ai":["allowed","*",0,1],"PerplexityBot":["allowed","*",0,1],"Perplexity-User":["allowed","*",0,1],"Google-Extended":["allowed","*",0,1],"Applebot-Extended":["allowed","*",0,1],"CCBot":["allowed","*",0,1],"Bytespider":["allowed","*",0,1],"Amazonbot":["allowed","*",0,1],"meta-externalagent":["allowed","*",0,1],"cohere-ai":["allowed","*",0,1],"Diffbot":["allowed","*",0,1],"ContextFetchBot":["allowed","*",0,1]},"llms":{"status":"present","http":200,"bytes":6928},"home":{"status":"ok","http":200,"url":"https://deepmind.google/","html_tokens":66637,"markdown_tokens":227,"pruned_tokens":146}},
{"rank":1883,"domain":"newrepublic.com","origin":"https://newrepublic.com","robots":{"status":"ok","http":200,"url":"https://newrepublic.com/robots.txt","bytes":561,"groups":3},"bots":{"GPTBot":["partial","*",3,0],"ChatGPT-User":["partial","*",3,0],"OAI-SearchBot":["partial","*",3,0],"ClaudeBot":["partial","*",3,0],"Claude-User":["partial","*",3,0],"Claude-SearchBot":["partial","*",3,0],"anthropic-ai":["partial","*",3,0],"PerplexityBot":["partial","*",3,0],"Perplexity-User":["partial","*",3,0],"Google-Extended":["partial","*",3,0],"Applebot-Extended":["partial","*",3,0],"CCBot":["partial","*",3,0],"Bytespider":["partial","*",3,0],"Amazonbot":["partial","*",3,0],"meta-externalagent":["partial","*",3,0],"cohere-ai":["partial","*",3,0],"Diffbot":["partial","*",3,0],"ContextFetchBot":["partial","*",3,0]},"llms":{"status":"missing","http":404,"bytes":98935},"home":{"status":"ok","http":200,"url":"https://newrepublic.com/","html_tokens":74864,"markdown_tokens":16,"pruned_tokens":16}},
{"rank":1884,"domain":"sbs.com.au","origin":"https://sbs.com.au","robots":{"status":"ok","http":200,"url":"https://www.sbs.com.au/robots.txt","bytes":7524,"groups":48},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["partial","named",5,1],"OAI-SearchBot":["partial","named",5,1],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","named",5,1],"Claude-SearchBot":["partial","named",5,1],"anthropic-ai":["partial","*",5,0],"PerplexityBot":["partial","named",5,1],"Perplexity-User":["partial","named",5,1],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",5,0]},"llms":{"status":"missing","http":404,"bytes":97688},"home":{"status":"ok","http":200,"url":"https://www.sbs.com.au/","html_tokens":316728,"markdown_tokens":393,"pruned_tokens":100}},
{"rank":1885,"domain":"tieba.baidu.com","origin":"https://tieba.baidu.com","robots":{"status":"ok","http":200,"url":"https://tieba.baidu.com/robots.txt","bytes":1628,"groups":20},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"http_error","http":403,"bytes":2475},"home":{"status":"http_error","http":403,"url":"https://tieba.baidu.com/"}},
{"rank":1886,"domain":"uw.edu","origin":"https://uw.edu","robots":{"status":"ok","http":200,"url":"https://www.washington.edu/robots.txt","bytes":76,"groups":1},"bots":{"GPTBot":["partial","*",3,0],"ChatGPT-User":["partial","*",3,0],"OAI-SearchBot":["partial","*",3,0],"ClaudeBot":["partial","*",3,0],"Claude-User":["partial","*",3,0],"Claude-SearchBot":["partial","*",3,0],"anthropic-ai":["partial","*",3,0],"PerplexityBot":["partial","*",3,0],"Perplexity-User":["partial","*",3,0],"Google-Extended":["partial","*",3,0],"Applebot-Extended":["partial","*",3,0],"CCBot":["partial","*",3,0],"Bytespider":["partial","*",3,0],"Amazonbot":["partial","*",3,0],"meta-externalagent":["partial","*",3,0],"cohere-ai":["partial","*",3,0],"Diffbot":["partial","*",3,0],"ContextFetchBot":["partial","*",3,0]},"llms":{"status":"missing","http":404,"bytes":19665},"home":{"status":"ok","http":200,"url":"https://www.washington.edu/","html_tokens":15928,"markdown_tokens":367,"pruned_tokens":78}},
{"rank":1887,"domain":"adage.com","origin":"https://adage.com","robots":{"status":"refused","http":403,"url":"https://adage.com/robots.txt","bytes":371,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":1888,"domain":"repec.org","origin":"http://repec.org","robots":{"status":"missing","http":404,"url":"http://repec.org/robots.txt","bytes":408,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":408},"home":{"status":"ok","http":200,"url":"http://repec.org/","html_tokens":3748,"markdown_tokens":1092,"pruned_tokens":350}},
{"rank":1889,"domain":"or.at","origin":"https://or.at","robots":{"status":"unreachable","http":null,"url":"https://or.at/robots.txt","bytes":0,"groups":0,"error":"cannot resolve or.at"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":1890,"domain":"goo.ne.jp","origin":"https://goo.ne.jp","robots":{"status":"unreachable","http":null,"url":"https://goo.ne.jp/robots.txt","bytes":0,"groups":0,"error":"cannot resolve goo.ne.jp"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":1891,"domain":"newatlas.com","origin":"https://newatlas.com","robots":{"status":"ok","http":200,"url":"https://newatlas.com/robots.txt","bytes":357,"groups":3},"bots":{"GPTBot":["partial","*",3,0],"ChatGPT-User":["partial","*",3,0],"OAI-SearchBot":["partial","*",3,0],"ClaudeBot":["partial","*",3,0],"Claude-User":["partial","*",3,0],"Claude-SearchBot":["partial","*",3,0],"anthropic-ai":["partial","*",3,0],"PerplexityBot":["partial","*",3,0],"Perplexity-User":["partial","*",3,0],"Google-Extended":["partial","*",3,0],"Applebot-Extended":["partial","*",3,0],"CCBot":["partial","*",3,0],"Bytespider":["partial","*",3,0],"Amazonbot":["partial","*",3,0],"meta-externalagent":["partial","*",3,0],"cohere-ai":["partial","*",3,0],"Diffbot":["partial","*",3,0],"ContextFetchBot":["partial","*",3,0]},"llms":{"status":"missing","http":404,"bytes":97328},"home":{"status":"ok","http":200,"url":"https://newatlas.com/","html_tokens":53726,"markdown_tokens":835,"pruned_tokens":151}},
{"rank":1892,"domain":"futurism.com","origin":"https://futurism.com","robots":{"status":"ok","http":200,"url":"https://futurism.com/robots.txt","bytes":360,"groups":2},"bots":{"GPTBot":["partial","*",6,1],"ChatGPT-User":["partial","*",6,1],"OAI-SearchBot":["partial","*",6,1],"ClaudeBot":["partial","*",6,1],"Claude-User":["partial","*",6,1],"Claude-SearchBot":["partial","*",6,1],"anthropic-ai":["partial","*",6,1],"PerplexityBot":["partial","*",6,1],"Perplexity-User":["partial","*",6,1],"Google-Extended":["partial","*",6,1],"Applebot-Extended":["partial","*",6,1],"CCBot":["partial","*",6,1],"Bytespider":["partial","*",6,1],"Amazonbot":["partial","*",6,1],"meta-externalagent":["partial","*",6,1],"cohere-ai":["partial","*",6,1],"Diffbot":["partial","*",6,1],"ContextFetchBot":["partial","*",6,1]},"llms":{"status":"missing","http":404,"bytes":102954},"home":{"status":"ok","http":200,"url":"https://futurism.com/","html_tokens":43395,"markdown_tokens":137,"pruned_tokens":137}},
{"rank":1893,"domain":"opaque.net","origin":"https://opaque.net","robots":{"status":"html","http":200,"url":"https://opaque.net/robots.txt","bytes":586,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":1894,"domain":"netperf.org","origin":"https://netperf.org","robots":{"status":"timeout","http":null,"url":"https://netperf.org/robots.txt","bytes":0,"groups":0,"error":"ConnectTimeout"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":1895,"domain":"rg.ru","origin":"https://rg.ru","robots":{"status":"ok","http":200,"url":"https://rg.ru/robots.txt","bytes":2848,"groups":4},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["partial","*",26,0],"OAI-SearchBot":["partial","*",26,0],"ClaudeBot":["partial","*",26,0],"Claude-User":["partial","*",26,0],"Claude-SearchBot":["partial","*",26,0],"anthropic-ai":["partial","*",26,0],"PerplexityBot":["partial","*",26,0],"Perplexity-User":["partial","*",26,0],"Google-Extended":["partial","*",26,0],"Applebot-Extended":["partial","*",26,0],"CCBot":["partial","*",26,0],"Bytespider":["partial","*",26,0],"Amazonbot":["partial","*",26,0],"meta-externalagent":["partial","*",26,0],"cohere-ai":["partial","*",26,0],"Diffbot":["partial","*",26,0],"ContextFetchBot":["partial","*",26,0]},"llms":{"status":"missing","http":404,"bytes":17061},"home":{"status":"ok","http":200,"url":"https://rg.ru/","html_tokens":39080,"markdown_tokens":95,"pruned_tokens":95}},
{"rank":1896,"domain":"sbnation.com","origin":"https://sbnation.com","robots":{"status":"ok","http":200,"url":"https://sbnation.com/robots.txt","bytes":5650,"groups":106},"bots":{"GPTBot":["allowed","named",0,1],"ChatGPT-User":["partial","named",1,1],"OAI-SearchBot":["partial","*",15,2],"ClaudeBot":["partial","named",1,1],"Claude-User":["partial","named",1,1],"Claude-SearchBot":["partial","named",1,1],"anthropic-ai":["partial","named",1,1],"PerplexityBot":["partial","named",1,1],"Perplexity-User":["partial","named",1,1],"Google-Extended":["partial","named",1,1],"Applebot-Extended":["partial","named",1,1],"CCBot":["partial","named",1,1],"Bytespider":["partial","named",1,1],"Amazonbot":["allowed","named",0,1],"meta-externalagent":["partial","named",1,1],"cohere-ai":["partial","named",1,1],"Diffbot":["partial","named",1,1],"ContextFetchBot":["partial","*",15,2]},"llms":{"status":"missing","http":404,"bytes":129384},"home":{"status":"ok","http":200,"url":"https://www.sbnation.com/","html_tokens":207533,"markdown_tokens":20,"pruned_tokens":20}},
{"rank":1897,"domain":"gulfnews.com","origin":"https://gulfnews.com","robots":{"status":"ok","http":200,"url":"https://gulfnews.com/robots.txt","bytes":1951,"groups":4},"bots":{"GPTBot":["partial","*",25,0],"ChatGPT-User":["partial","*",25,0],"OAI-SearchBot":["partial","*",25,0],"ClaudeBot":["partial","*",25,0],"Claude-User":["partial","*",25,0],"Claude-SearchBot":["partial","*",25,0],"anthropic-ai":["partial","*",25,0],"PerplexityBot":["partial","*",25,0],"Perplexity-User":["partial","*",25,0],"Google-Extended":["partial","*",25,0],"Applebot-Extended":["partial","*",25,0],"CCBot":["partial","*",25,0],"Bytespider":["partial","*",25,0],"Amazonbot":["partial","*",25,0],"meta-externalagent":["partial","*",25,0],"cohere-ai":["partial","*",25,0],"Diffbot":["partial","*",25,0],"ContextFetchBot":["partial","*",25,0]},"llms":{"status":"missing","http":404,"bytes":987259},"home":{"status":"ok","http":200,"url":"https://gulfnews.com/","html_tokens":323376,"markdown_tokens":581,"pruned_tokens":175}},
{"rank":1898,"domain":"haveibeenpwned.com","origin":"https://haveibeenpwned.com","robots":{"status":"ok","http":200,"url":"https://haveibeenpwned.com/robots.txt","bytes":555,"groups":1},"bots":{"GPTBot":["partial","*",16,0],"ChatGPT-User":["partial","*",16,0],"OAI-SearchBot":["partial","*",16,0],"ClaudeBot":["partial","*",16,0],"Claude-User":["partial","*",16,0],"Claude-SearchBot":["partial","*",16,0],"anthropic-ai":["partial","*",16,0],"PerplexityBot":["partial","*",16,0],"Perplexity-User":["partial","*",16,0],"Google-Extended":["partial","*",16,0],"Applebot-Extended":["partial","*",16,0],"CCBot":["partial","*",16,0],"Bytespider":["partial","*",16,0],"Amazonbot":["partial","*",16,0],"meta-externalagent":["partial","*",16,0],"cohere-ai":["partial","*",16,0],"Diffbot":["partial","*",16,0],"ContextFetchBot":["partial","*",16,0]},"llms":{"status":"missing","http":404,"bytes":14068},"home":{"status":"ok","http":200,"url":"https://haveibeenpwned.com/","html_tokens":6918,"markdown_tokens":35,"pruned_tokens":35}},
{"rank":1899,"domain":"acast.com","origin":"https://acast.com","robots":{"status":"ok","http":200,"url":"https://www.acast.com/robots.txt","bytes":42,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":50983},"home":{"status":"ok","http":200,"url":"https://www.acast.com/en","html_tokens":29385,"markdown_tokens":168,"pruned_tokens":60}},
{"rank":1900,"domain":"goldmansachs.com","origin":"https://goldmansachs.com","robots":{"status":"refused","http":403,"url":"https://goldmansachs.com/robots.txt","bytes":380,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":1901,"domain":"auctollo.com","origin":"https://auctollo.com","robots":{"status":"ok","http":200,"url":"https://auctollo.com/robots.txt","bytes":516,"groups":2},"bots":{"GPTBot":["partial","*",6,1],"ChatGPT-User":["partial","*",6,1],"OAI-SearchBot":["partial","*",6,1],"ClaudeBot":["partial","*",6,1],"Claude-User":["partial","*",6,1],"Claude-SearchBot":["partial","*",6,1],"anthropic-ai":["partial","*",6,1],"PerplexityBot":["partial","*",6,1],"Perplexity-User":["partial","*",6,1],"Google-Extended":["partial","*",6,1],"Applebot-Extended":["partial","*",6,1],"CCBot":["partial","*",6,1],"Bytespider":["partial","*",6,1],"Amazonbot":["partial","*",6,1],"meta-externalagent":["partial","*",6,1],"cohere-ai":["partial","*",6,1],"Diffbot":["partial","*",6,1],"ContextFetchBot":["partial","*",6,1]},"llms":{"status":"missing","http":404,"bytes":13},"home":{"status":"ok","http":200,"url":"https://auctollo.com/","html_tokens":18106,"markdown_tokens":745,"pruned_tokens":166}},
{"rank":1902,"domain":"cpsc.gov","origin":"https://cpsc.gov","robots":{"status":"ok","http":200,"url":"https://www.cpsc.gov/robots.txt","bytes":3254,"groups":1},"bots":{"GPTBot":["partial","*",55,30],"ChatGPT-User":["partial","*",55,30],"OAI-SearchBot":["partial","*",55,30],"ClaudeBot":["partial","*",55,30],"Claude-User":["partial","*",55,30],"Claude-SearchBot":["partial","*",55,30],"anthropic-ai":["partial","*",55,30],"PerplexityBot":["partial","*",55,30],"Perplexity-User":["partial","*",55,30],"Google-Extended":["partial","*",55,30],"Applebot-Extended":["partial","*",55,30],"CCBot":["partial","*",55,30],"Bytespider":["partial","*",55,30],"Amazonbot":["partial","*",55,30],"meta-externalagent":["partial","*",55,30],"cohere-ai":["partial","*",55,30],"Diffbot":["partial","*",55,30],"ContextFetchBot":["partial","*",55,30]},"llms":{"status":"missing","http":404,"bytes":186},"home":{"status":"ok","http":200,"url":"https://www.cpsc.gov/","html_tokens":17277,"markdown_tokens":128,"pruned_tokens":128}},
{"rank":1903,"domain":"wallpapers.com","origin":"https://wallpapers.com","robots":{"status":"ok","http":200,"url":"https://wallpapers.com/robots.txt","bytes":809,"groups":1},"bots":{"GPTBot":["partial","*",29,2],"ChatGPT-User":["partial","*",29,2],"OAI-SearchBot":["partial","*",29,2],"ClaudeBot":["partial","*",29,2],"Claude-User":["partial","*",29,2],"Claude-SearchBot":["partial","*",29,2],"anthropic-ai":["partial","*",29,2],"PerplexityBot":["partial","*",29,2],"Perplexity-User":["partial","*",29,2],"Google-Extended":["partial","*",29,2],"Applebot-Extended":["partial","*",29,2],"CCBot":["partial","*",29,2],"Bytespider":["partial","*",29,2],"Amazonbot":["partial","*",29,2],"meta-externalagent":["partial","*",29,2],"cohere-ai":["partial","*",29,2],"Diffbot":["partial","*",29,2],"ContextFetchBot":["partial","*",29,2]},"llms":{"status":"missing","http":404,"bytes":1830},"home":{"status":"ok","http":200,"url":"https://wallpapers.com/","html_tokens":117049,"markdown_tokens":282,"pruned_tokens":103}},
{"rank":1904,"domain":"oregonlive.com","origin":"https://oregonlive.com","robots":{"status":"ok","http":200,"url":"https://www.oregonlive.com/robots.txt","bytes":2842,"groups":4},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["partial","named",1,11],"OAI-SearchBot":["partial","named",1,11],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","named",1,11],"Claude-SearchBot":["partial","named",1,11],"anthropic-ai":["partial","*",30,3],"PerplexityBot":["partial","named",1,11],"Perplexity-User":["partial","named",1,11],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["partial","*",30,3],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",30,3]},"llms":{"status":"missing","http":404,"bytes":311386},"home":{"status":"http_error","http":403,"url":"https://www.oregonlive.com/"}},
{"rank":1905,"domain":"brightcove.net","origin":"https://brightcove.net","robots":{"status":"unreachable","http":null,"url":"https://brightcove.net/robots.txt","bytes":0,"groups":0,"error":"cannot resolve brightcove.net"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":1906,"domain":"destatis.de","origin":"https://destatis.de","robots":{"status":"ok","http":200,"url":"https://www.destatis.de/robots.txt","bytes":374,"groups":1},"bots":{"GPTBot":["partial","*",2,6],"ChatGPT-User":["partial","*",2,6],"OAI-SearchBot":["partial","*",2,6],"ClaudeBot":["partial","*",2,6],"Claude-User":["partial","*",2,6],"Claude-SearchBot":["partial","*",2,6],"anthropic-ai":["partial","*",2,6],"PerplexityBot":["partial","*",2,6],"Perplexity-User":["partial","*",2,6],"Google-Extended":["partial","*",2,6],"Applebot-Extended":["partial","*",2,6],"CCBot":["partial","*",2,6],"Bytespider":["partial","*",2,6],"Amazonbot":["partial","*",2,6],"meta-externalagent":["partial","*",2,6],"cohere-ai":["partial","*",2,6],"Diffbot":["partial","*",2,6],"ContextFetchBot":["partial","*",2,6]},"llms":{"status":"missing","http":404,"bytes":64945},"home":{"status":"timeout","error":"timed out after 15 s"}},
{"rank":1907,"domain":"harpersbazaar.com","origin":"https://harpersbazaar.com","robots":{"status":"ok","http":200,"url":"https://harpersbazaar.com/robots.txt","bytes":971,"groups":3},"bots":{"GPTBot":["partial","named",32,0],"ChatGPT-User":["partial","named",32,0],"OAI-SearchBot":["partial","named",32,0],"ClaudeBot":["partial","*",19,0],"Claude-User":["partial","*",19,0],"Claude-SearchBot":["partial","*",19,0],"anthropic-ai":["partial","*",19,0],"PerplexityBot":["partial","*",19,0],"Perplexity-User":["partial","*",19,0],"Google-Extended":["partial","*",19,0],"Applebot-Extended":["partial","*",19,0],"CCBot":["partial","*",19,0],"Bytespider":["partial","*",19,0],"Amazonbot":["partial","*",19,0],"meta-externalagent":["partial","*",19,0],"cohere-ai":["partial","*",19,0],"Diffbot":["partial","*",19,0],"ContextFetchBot":["partial","*",19,0]},"llms":{"status":"missing","http":404,"bytes":84},"home":{"status":"ok","http":200,"url":"https://www.harpersbazaar.com/","html_tokens":227709,"markdown_tokens":68,"pruned_tokens":68}},
{"rank":1908,"domain":"toptal.com","origin":"https://toptal.com","robots":{"status":"ok","http":200,"url":"https://www.toptal.com/robots.txt","bytes":513,"groups":2},"bots":{"GPTBot":["partial","*",15,0],"ChatGPT-User":["partial","*",15,0],"OAI-SearchBot":["partial","*",15,0],"ClaudeBot":["partial","*",15,0],"Claude-User":["partial","*",15,0],"Claude-SearchBot":["partial","*",15,0],"anthropic-ai":["partial","*",15,0],"PerplexityBot":["partial","*",15,0],"Perplexity-User":["partial","*",15,0],"Google-Extended":["partial","*",15,0],"Applebot-Extended":["partial","*",15,0],"CCBot":["partial","*",15,0],"Bytespider":["partial","*",15,0],"Amazonbot":["partial","*",15,0],"meta-externalagent":["partial","*",15,0],"cohere-ai":["partial","*",15,0],"Diffbot":["partial","*",15,0],"ContextFetchBot":["partial","*",15,0]},"llms":{"status":"present","http":200,"bytes":26858},"home":{"status":"ok","http":200,"url":"https://www.toptal.com/","html_tokens":149945,"markdown_tokens":1495,"pruned_tokens":103}},
{"rank":1909,"domain":"list-manage1.com","origin":"https://list-manage1.com","robots":{"status":"timeout","http":null,"url":"https://list-manage1.com/robots.txt","bytes":0,"groups":0,"error":"ConnectTimeout"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":1910,"domain":"cryoutcreations.eu","origin":"https://cryoutcreations.eu","robots":{"status":"ok","http":200,"url":"https://www.cryoutcreations.eu/robots.txt","bytes":23,"groups":1},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"missing","http":404,"bytes":117227},"home":{"status":"ok","http":200,"url":"https://www.cryoutcreations.eu/","html_tokens":29323,"markdown_tokens":457,"pruned_tokens":122}},
{"rank":1911,"domain":"videojs.com","origin":"https://videojs.com","robots":{"status":"ok","http":200,"url":"https://videojs.org/robots.txt","bytes":62,"groups":1},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"present","http":200,"bytes":2814},"home":{"status":"ok","http":200,"url":"https://videojs.org/","html_tokens":38858,"markdown_tokens":216,"pruned_tokens":95}},
{"rank":1912,"domain":"sendgrid.net","origin":"https://sendgrid.net","robots":{"status":"missing","http":404,"url":"https://sendgrid.net/robots.txt","bytes":0,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":146},"home":{"status":"http_error","http":404,"url":"https://sendgrid.net/"}},
{"rank":1913,"domain":"cursor.com","origin":"https://cursor.com","robots":{"status":"ok","http":200,"url":"https://cursor.com/robots.txt","bytes":1047,"groups":1},"bots":{"GPTBot":["partial","*",36,3],"ChatGPT-User":["partial","*",36,3],"OAI-SearchBot":["partial","*",36,3],"ClaudeBot":["partial","*",36,3],"Claude-User":["partial","*",36,3],"Claude-SearchBot":["partial","*",36,3],"anthropic-ai":["partial","*",36,3],"PerplexityBot":["partial","*",36,3],"Perplexity-User":["partial","*",36,3],"Google-Extended":["partial","*",36,3],"Applebot-Extended":["partial","*",36,3],"CCBot":["partial","*",36,3],"Bytespider":["partial","*",36,3],"Amazonbot":["partial","*",36,3],"meta-externalagent":["partial","*",36,3],"cohere-ai":["partial","*",36,3],"Diffbot":["partial","*",36,3],"ContextFetchBot":["partial","*",36,3]},"llms":{"status":"present","http":200,"bytes":24384},"home":{"status":"ok","http":200,"url":"https://cursor.com/","html_tokens":154585,"markdown_tokens":1335,"pruned_tokens":350}},
{"rank":1914,"domain":"utk.edu","origin":"https://utk.edu","robots":{"status":"ok","http":200,"url":"http://www.utk.edu/robots.txt","bytes":242,"groups":1},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"http_error","http":403,"bytes":5538},"home":{"status":"http_error","http":403,"url":"https://utk.edu/"}},
{"rank":1915,"domain":"mentalfloss.com","origin":"https://mentalfloss.com","robots":{"status":"ok","http":200,"url":"https://www.mentalfloss.com/robots.txt","bytes":3434,"groups":27},"bots":{"GPTBot":["allowed","named",0,1],"ChatGPT-User":["allowed","named",0,1],"OAI-SearchBot":["allowed","named",0,1],"ClaudeBot":["allowed","named",0,1],"Claude-User":["partial","*",85,2],"Claude-SearchBot":["partial","*",85,2],"anthropic-ai":["partial","*",85,2],"PerplexityBot":["allowed","named",0,1],"Perplexity-User":["partial","*",85,2],"Google-Extended":["allowed","named",0,1],"Applebot-Extended":["partial","*",85,2],"CCBot":["partial","*",85,2],"Bytespider":["partial","*",85,2],"Amazonbot":["partial","*",85,2],"meta-externalagent":["partial","*",85,2],"cohere-ai":["partial","*",85,2],"Diffbot":["partial","*",85,2],"ContextFetchBot":["partial","*",85,2]},"llms":{"status":"present","http":200,"bytes":12194},"home":{"status":"ok","http":200,"url":"https://www.mentalfloss.com/","html_tokens":333385,"markdown_tokens":91,"pruned_tokens":91}},
{"rank":1916,"domain":"sf.net","origin":"https://sf.net","robots":{"status":"ok","http":200,"url":"https://sourceforge.net/robots.txt","bytes":5837,"groups":3},"bots":{"GPTBot":["partial","*",164,12],"ChatGPT-User":["partial","*",164,12],"OAI-SearchBot":["partial","*",164,12],"ClaudeBot":["partial","*",164,12],"Claude-User":["partial","*",164,12],"Claude-SearchBot":["partial","*",164,12],"anthropic-ai":["partial","*",164,12],"PerplexityBot":["partial","*",164,12],"Perplexity-User":["partial","*",164,12],"Google-Extended":["partial","*",164,12],"Applebot-Extended":["partial","*",164,12],"CCBot":["partial","*",164,12],"Bytespider":["partial","*",164,12],"Amazonbot":["partial","*",164,12],"meta-externalagent":["partial","*",164,12],"cohere-ai":["partial","*",164,12],"Diffbot":["partial","*",164,12],"ContextFetchBot":["partial","*",164,12]},"llms":{"status":"present","http":200,"bytes":3334905},"home":{"status":"ok","http":200,"url":"https://sourceforge.net/","html_tokens":30992,"markdown_tokens":166,"pruned_tokens":166}},
{"rank":1917,"domain":"msk.ru","origin":"https://www.msk.ru","robots":{"status":"html","http":200,"url":"https://www.mos.ru/","bytes":360670,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":1918,"domain":"freecodecamp.org","origin":"https://freecodecamp.org","robots":{"status":"ok","http":200,"url":"https://www.freecodecamp.org/robots.txt","bytes":79,"groups":1},"bots":{"GPTBot":["allowed","*",0,1],"ChatGPT-User":["allowed","*",0,1],"OAI-SearchBot":["allowed","*",0,1],"ClaudeBot":["allowed","*",0,1],"Claude-User":["allowed","*",0,1],"Claude-SearchBot":["allowed","*",0,1],"anthropic-ai":["allowed","*",0,1],"PerplexityBot":["allowed","*",0,1],"Perplexity-User":["allowed","*",0,1],"Google-Extended":["allowed","*",0,1],"Applebot-Extended":["allowed","*",0,1],"CCBot":["allowed","*",0,1],"Bytespider":["allowed","*",0,1],"Amazonbot":["allowed","*",0,1],"meta-externalagent":["allowed","*",0,1],"cohere-ai":["allowed","*",0,1],"Diffbot":["allowed","*",0,1],"ContextFetchBot":["allowed","*",0,1]},"llms":{"status":"missing","http":404,"bytes":501540},"home":{"status":"ok","http":200,"url":"https://www.freecodecamp.org/","html_tokens":125383,"markdown_tokens":0,"pruned_tokens":0}},
{"rank":1919,"domain":"webaim.org","origin":"https://webaim.org","robots":{"status":"ok","http":200,"url":"https://webaim.org/robots.txt","bytes":110,"groups":1},"bots":{"GPTBot":["partial","*",3,0],"ChatGPT-User":["partial","*",3,0],"OAI-SearchBot":["partial","*",3,0],"ClaudeBot":["partial","*",3,0],"Claude-User":["partial","*",3,0],"Claude-SearchBot":["partial","*",3,0],"anthropic-ai":["partial","*",3,0],"PerplexityBot":["partial","*",3,0],"Perplexity-User":["partial","*",3,0],"Google-Extended":["partial","*",3,0],"Applebot-Extended":["partial","*",3,0],"CCBot":["partial","*",3,0],"Bytespider":["partial","*",3,0],"Amazonbot":["partial","*",3,0],"meta-externalagent":["partial","*",3,0],"cohere-ai":["partial","*",3,0],"Diffbot":["partial","*",3,0],"ContextFetchBot":["partial","*",3,0]},"llms":{"status":"missing","http":404,"bytes":5683},"home":{"status":"ok","http":200,"url":"https://webaim.org/","html_tokens":2031,"markdown_tokens":262,"pruned_tokens":83}},
{"rank":1920,"domain":"jpmorgan.com","origin":"https://jpmorgan.com","robots":{"status":"ok","http":200,"url":"https://www.jpmorgan.com/robots.txt","bytes":432,"groups":1},"bots":{"GPTBot":["partial","*",6,1],"ChatGPT-User":["partial","*",6,1],"OAI-SearchBot":["partial","*",6,1],"ClaudeBot":["partial","*",6,1],"Claude-User":["partial","*",6,1],"Claude-SearchBot":["partial","*",6,1],"anthropic-ai":["partial","*",6,1],"PerplexityBot":["partial","*",6,1],"Perplexity-User":["partial","*",6,1],"Google-Extended":["partial","*",6,1],"Applebot-Extended":["partial","*",6,1],"CCBot":["partial","*",6,1],"Bytespider":["partial","*",6,1],"Amazonbot":["partial","*",6,1],"meta-externalagent":["partial","*",6,1],"cohere-ai":["partial","*",6,1],"Diffbot":["partial","*",6,1],"ContextFetchBot":["partial","*",6,1]},"llms":{"status":"missing","http":404,"bytes":6997},"home":{"status":"ok","http":200,"url":"https://www.jpmorgan.com/global","html_tokens":51694,"markdown_tokens":292,"pruned_tokens":75}},
{"rank":1921,"domain":"collegeboard.org","origin":"https://collegeboard.org","robots":{"status":"ok","http":200,"url":"https://www.collegeboard.org/robots.txt","bytes":1777,"groups":1},"bots":{"GPTBot":["partial","*",26,18],"ChatGPT-User":["partial","*",26,18],"OAI-SearchBot":["partial","*",26,18],"ClaudeBot":["partial","*",26,18],"Claude-User":["partial","*",26,18],"Claude-SearchBot":["partial","*",26,18],"anthropic-ai":["partial","*",26,18],"PerplexityBot":["partial","*",26,18],"Perplexity-User":["partial","*",26,18],"Google-Extended":["partial","*",26,18],"Applebot-Extended":["partial","*",26,18],"CCBot":["partial","*",26,18],"Bytespider":["partial","*",26,18],"Amazonbot":["partial","*",26,18],"meta-externalagent":["partial","*",26,18],"cohere-ai":["partial","*",26,18],"Diffbot":["partial","*",26,18],"ContextFetchBot":["partial","*",26,18]},"llms":{"status":"missing","http":404,"bytes":194},"home":{"status":"ok","http":200,"url":"https://www.collegeboard.org/","html_tokens":10369,"markdown_tokens":38,"pruned_tokens":38}},
{"rank":1922,"domain":"ctfassets.net","origin":"https://ctfassets.net","robots":{"status":"unreachable","http":null,"url":"https://ctfassets.net/robots.txt","bytes":0,"groups":0,"error":"cannot resolve ctfassets.net"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":1923,"domain":"comcast.net","origin":"https://comcast.net","robots":{"status":"refused","http":403,"url":"https://www.xfinity.com/","bytes":369,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":1924,"domain":"oecd-ilibrary.org","origin":"https://oecd-ilibrary.org","robots":{"status":"refused","http":403,"url":"https://oecd-ilibrary.org/robots.txt","bytes":0,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":1925,"domain":"t.ly","origin":"https://t.ly","robots":{"status":"ok","http":200,"url":"https://t.ly/robots.txt","bytes":57,"groups":1},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"http_error","http":403,"bytes":5514},"home":{"status":"http_error","http":403,"url":"https://t.ly/"}},
{"rank":1926,"domain":"freep.com","origin":"https://freep.com","robots":{"status":"ok","http":200,"url":"https://www.freep.com/robots.txt","bytes":12811,"groups":288},"bots":{"GPTBot":["partial","named",1,4],"ChatGPT-User":["partial","named",1,4],"OAI-SearchBot":["partial","named",1,4],"ClaudeBot":["blocked","named",1,0],"Claude-User":["blocked","named",1,0],"Claude-SearchBot":["blocked","named",1,0],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["partial","named",1,4],"Perplexity-User":["blocked","named",1,0],"Google-Extended":["partial","named",1,4],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",2,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",33,0]},"llms":{"status":"http_error","http":402,"bytes":254},"home":{"status":"http_error","http":402,"url":"https://www.freep.com/"}},
{"rank":1927,"domain":"hackernoon.com","origin":"https://hackernoon.com","robots":{"status":"ok","http":200,"url":"https://hackernoon.com/robots.txt","bytes":4959,"groups":44},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["allowed","named",0,2],"OAI-SearchBot":["allowed","named",0,1],"ClaudeBot":["blocked","named",1,0],"Claude-User":["allowed","named",0,1],"Claude-SearchBot":["allowed","named",0,1],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["allowed","named",0,1],"Perplexity-User":["blocked","*",1,0],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["blocked","*",1,0]},"llms":{"status":"disallowed"},"home":{"status":"disallowed"}},
{"rank":1928,"domain":"encyclopedia.com","origin":"https://encyclopedia.com","robots":{"status":"ok","http":200,"url":"https://www.encyclopedia.com/robots.txt","bytes":1635,"groups":2},"bots":{"GPTBot":["partial","*",22,18],"ChatGPT-User":["partial","*",22,18],"OAI-SearchBot":["partial","*",22,18],"ClaudeBot":["partial","*",22,18],"Claude-User":["partial","*",22,18],"Claude-SearchBot":["partial","*",22,18],"anthropic-ai":["partial","*",22,18],"PerplexityBot":["partial","*",22,18],"Perplexity-User":["partial","*",22,18],"Google-Extended":["partial","*",22,18],"Applebot-Extended":["partial","*",22,18],"CCBot":["partial","*",22,18],"Bytespider":["partial","*",22,18],"Amazonbot":["partial","*",22,18],"meta-externalagent":["partial","*",22,18],"cohere-ai":["partial","*",22,18],"Diffbot":["partial","*",22,18],"ContextFetchBot":["partial","*",22,18]},"llms":{"status":"missing","http":404,"bytes":24928},"home":{"status":"ok","http":200,"url":"https://www.encyclopedia.com/","html_tokens":9148,"markdown_tokens":64,"pruned_tokens":64}},
{"rank":1929,"domain":"re.kr","origin":"https://re.kr","robots":{"status":"unreachable","http":null,"url":"https://re.kr/robots.txt","bytes":0,"groups":0,"error":"cannot resolve re.kr"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":1930,"domain":"cri.cn","origin":"https://cri.cn","robots":{"status":"ok","http":200,"url":"https://cri.cn/robots.txt","bytes":29,"groups":1},"bots":{"GPTBot":["partial","*",1,0],"ChatGPT-User":["partial","*",1,0],"OAI-SearchBot":["partial","*",1,0],"ClaudeBot":["partial","*",1,0],"Claude-User":["partial","*",1,0],"Claude-SearchBot":["partial","*",1,0],"anthropic-ai":["partial","*",1,0],"PerplexityBot":["partial","*",1,0],"Perplexity-User":["partial","*",1,0],"Google-Extended":["partial","*",1,0],"Applebot-Extended":["partial","*",1,0],"CCBot":["partial","*",1,0],"Bytespider":["partial","*",1,0],"Amazonbot":["partial","*",1,0],"meta-externalagent":["partial","*",1,0],"cohere-ai":["partial","*",1,0],"Diffbot":["partial","*",1,0],"ContextFetchBot":["partial","*",1,0]},"llms":{"status":"missing","http":404,"bytes":46463},"home":{"status":"ok","http":200,"url":"https://cri.cn/","html_tokens":40259,"markdown_tokens":292,"pruned_tokens":120}},
{"rank":1931,"domain":"inkscape.org","origin":"https://inkscape.org","robots":{"status":"ok","http":200,"url":"https://inkscape.org/robots.txt","bytes":219,"groups":4},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["partial","*",4,0],"OAI-SearchBot":["partial","*",4,0],"ClaudeBot":["partial","*",4,0],"Claude-User":["partial","*",4,0],"Claude-SearchBot":["partial","*",4,0],"anthropic-ai":["partial","*",4,0],"PerplexityBot":["partial","*",4,0],"Perplexity-User":["partial","*",4,0],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["partial","*",4,0],"Bytespider":["partial","*",4,0],"Amazonbot":["partial","*",4,0],"meta-externalagent":["partial","*",4,0],"cohere-ai":["partial","*",4,0],"Diffbot":["partial","*",4,0],"ContextFetchBot":["partial","*",4,0]},"llms":{"status":"crawl_delay","error":"Crawl-delay 86400 s"},"home":{"status":"crawl_delay","error":"Crawl-delay 86400 s"}},
{"rank":1932,"domain":"rte.ie","origin":"https://rte.ie","robots":{"status":"ok","http":200,"url":"https://www.rte.ie/robots.txt","bytes":1695,"groups":19},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["blocked","named",1,0],"OAI-SearchBot":["partial","*",36,0],"ClaudeBot":["partial","*",36,0],"Claude-User":["partial","*",36,0],"Claude-SearchBot":["partial","*",36,0],"anthropic-ai":["partial","*",36,0],"PerplexityBot":["partial","*",36,0],"Perplexity-User":["partial","*",36,0],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["partial","*",36,0],"CCBot":["blocked","named",1,0],"Bytespider":["partial","*",36,0],"Amazonbot":["partial","*",36,0],"meta-externalagent":["partial","*",36,0],"cohere-ai":["partial","*",36,0],"Diffbot":["partial","*",36,0],"ContextFetchBot":["partial","*",36,0]},"llms":{"status":"missing","http":404,"bytes":81319},"home":{"status":"ok","http":200,"url":"https://www.rte.ie/","html_tokens":89255,"markdown_tokens":45,"pruned_tokens":45}},
{"rank":1933,"domain":"caddyserver.com","origin":"https://caddyserver.com","robots":{"status":"missing","http":404,"url":"https://caddyserver.com/robots.txt","bytes":0,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":0},"home":{"status":"ok","http":200,"url":"https://caddyserver.com/","html_tokens":18155,"markdown_tokens":3729,"pruned_tokens":575}},
{"rank":1934,"domain":"mofa.go.jp","origin":"https://www.mofa.go.jp","robots":{"status":"refused","http":403,"url":"https://www.mofa.go.jp/robots.txt","bytes":386,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":1935,"domain":"eater.com","origin":"https://eater.com","robots":{"status":"ok","http":200,"url":"https://eater.com/robots.txt","bytes":5644,"groups":106},"bots":{"GPTBot":["allowed","named",0,1],"ChatGPT-User":["partial","named",1,1],"OAI-SearchBot":["partial","*",15,2],"ClaudeBot":["partial","named",1,1],"Claude-User":["partial","named",1,1],"Claude-SearchBot":["partial","named",1,1],"anthropic-ai":["partial","named",1,1],"PerplexityBot":["partial","named",1,1],"Perplexity-User":["partial","named",1,1],"Google-Extended":["partial","named",1,1],"Applebot-Extended":["partial","named",1,1],"CCBot":["partial","named",1,1],"Bytespider":["partial","named",1,1],"Amazonbot":["allowed","named",0,1],"meta-externalagent":["partial","named",1,1],"cohere-ai":["partial","named",1,1],"Diffbot":["partial","named",1,1],"ContextFetchBot":["partial","*",15,2]},"llms":{"status":"missing","http":404,"bytes":69968},"home":{"status":"ok","http":200,"url":"https://www.eater.com/","html_tokens":269741,"markdown_tokens":10,"pruned_tokens":10}},
{"rank":1936,"domain":"ilsole24ore.com","origin":"https://ilsole24ore.com","robots":{"status":"ok","http":200,"url":"https://www.ilsole24ore.com/robots.txt","bytes":831,"groups":18},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["blocked","named",1,0],"OAI-SearchBot":["blocked","named",1,0],"ClaudeBot":["allowed","*",0,1],"Claude-User":["allowed","*",0,1],"Claude-SearchBot":["allowed","*",0,1],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["blocked","named",1,0],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["allowed","*",0,1],"CCBot":["blocked","named",1,0],"Bytespider":["allowed","*",0,1],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["allowed","*",0,1],"Diffbot":["allowed","*",0,1],"ContextFetchBot":["allowed","*",0,1]},"llms":{"status":"missing","http":404,"bytes":936246},"home":{"status":"ok","http":200,"url":"https://www.ilsole24ore.com/","html_tokens":245071,"markdown_tokens":52,"pruned_tokens":52}},
{"rank":1937,"domain":"nice.org.uk","origin":"https://nice.org.uk","robots":{"status":"ok","http":200,"url":"https://www.nice.org.uk/robots.txt","bytes":124,"groups":1},"bots":{"GPTBot":["partial","*",2,1],"ChatGPT-User":["partial","*",2,1],"OAI-SearchBot":["partial","*",2,1],"ClaudeBot":["partial","*",2,1],"Claude-User":["partial","*",2,1],"Claude-SearchBot":["partial","*",2,1],"anthropic-ai":["partial","*",2,1],"PerplexityBot":["partial","*",2,1],"Perplexity-User":["partial","*",2,1],"Google-Extended":["partial","*",2,1],"Applebot-Extended":["partial","*",2,1],"CCBot":["partial","*",2,1],"Bytespider":["partial","*",2,1],"Amazonbot":["partial","*",2,1],"meta-externalagent":["partial","*",2,1],"cohere-ai":["partial","*",2,1],"Diffbot":["partial","*",2,1],"ContextFetchBot":["partial","*",2,1]},"llms":{"status":"missing","http":404,"bytes":10438},"home":{"status":"ok","http":200,"url":"https://www.nice.org.uk/","html_tokens":27586,"markdown_tokens":365,"pruned_tokens":50}},
{"rank":1938,"domain":"paloaltonetworks.com","origin":"https://paloaltonetworks.com","robots":{"status":"ok","http":200,"url":"https://www.paloaltonetworks.com/robots.txt","bytes":7701,"groups":3},"bots":{"GPTBot":["partial","*",83,18],"ChatGPT-User":["partial","named",83,19],"OAI-SearchBot":["partial","named",83,19],"ClaudeBot":["partial","*",83,18],"Claude-User":["partial","named",83,19],"Claude-SearchBot":["partial","named",83,19],"anthropic-ai":["partial","*",83,18],"PerplexityBot":["partial","named",83,19],"Perplexity-User":["partial","*",83,18],"Google-Extended":["partial","*",83,18],"Applebot-Extended":["partial","*",83,18],"CCBot":["partial","*",83,18],"Bytespider":["partial","*",83,18],"Amazonbot":["partial","*",83,18],"meta-externalagent":["partial","*",83,18],"cohere-ai":["partial","*",83,18],"Diffbot":["partial","*",83,18],"ContextFetchBot":["partial","*",83,18]},"llms":{"status":"present","http":200,"bytes":17903},"home":{"status":"ok","http":200,"url":"https://www.paloaltonetworks.com","html_tokens":156670,"markdown_tokens":580,"pruned_tokens":164}},
{"rank":1939,"domain":"haproxy.com","origin":"https://haproxy.com","robots":{"status":"ok","http":200,"url":"https://www.haproxy.com/robots.txt","bytes":356,"groups":2},"bots":{"GPTBot":["partial","*",7,0],"ChatGPT-User":["partial","*",7,0],"OAI-SearchBot":["partial","*",7,0],"ClaudeBot":["partial","*",7,0],"Claude-User":["partial","*",7,0],"Claude-SearchBot":["partial","*",7,0],"anthropic-ai":["partial","*",7,0],"PerplexityBot":["partial","*",7,0],"Perplexity-User":["partial","*",7,0],"Google-Extended":["partial","*",7,0],"Applebot-Extended":["partial","*",7,0],"CCBot":["partial","*",7,0],"Bytespider":["partial","*",7,0],"Amazonbot":["partial","*",7,0],"meta-externalagent":["partial","*",7,0],"cohere-ai":["partial","*",7,0],"Diffbot":["partial","*",7,0],"ContextFetchBot":["partial","*",7,0]},"llms":{"status":"missing","http":404,"bytes":338677},"home":{"status":"ok","http":200,"url":"https://www.haproxy.com/","html_tokens":150785,"markdown_tokens":438,"pruned_tokens":222}},
{"rank":1940,"domain":"www.fs.usda.gov","origin":"https://www.fs.usda.gov","robots":{"status":"ok","http":200,"url":"https://www.fs.usda.gov/robots.txt","bytes":10158,"groups":6},"bots":{"GPTBot":["partial","*",36,36],"ChatGPT-User":["partial","*",36,36],"OAI-SearchBot":["partial","*",36,36],"ClaudeBot":["partial","*",36,36],"Claude-User":["partial","*",36,36],"Claude-SearchBot":["partial","*",36,36],"anthropic-ai":["partial","*",36,36],"PerplexityBot":["partial","*",36,36],"Perplexity-User":["partial","*",36,36],"Google-Extended":["partial","*",36,36],"Applebot-Extended":["partial","*",36,36],"CCBot":["partial","*",36,36],"Bytespider":["partial","*",36,36],"Amazonbot":["partial","*",36,36],"meta-externalagent":["partial","*",36,36],"cohere-ai":["partial","*",36,36],"Diffbot":["partial","*",36,36],"ContextFetchBot":["partial","*",36,36]},"llms":{"status":"http_error","http":504,"bytes":1602},"home":{"status":"ok","http":200,"url":"https://www.fs.usda.gov/","html_tokens":24114,"markdown_tokens":87,"pruned_tokens":69}},
{"rank":1941,"domain":"codecanyon.net","origin":"https://codecanyon.net","robots":{"status":"ok","http":200,"url":"https://codecanyon.net/robots.txt","bytes":871,"groups":2},"bots":{"GPTBot":["partial","*",29,0],"ChatGPT-User":["partial","*",29,0],"OAI-SearchBot":["partial","*",29,0],"ClaudeBot":["partial","*",29,0],"Claude-User":["partial","*",29,0],"Claude-SearchBot":["partial","*",29,0],"anthropic-ai":["partial","*",29,0],"PerplexityBot":["partial","*",29,0],"Perplexity-User":["partial","*",29,0],"Google-Extended":["partial","*",29,0],"Applebot-Extended":["partial","*",29,0],"CCBot":["blocked","named",1,0],"Bytespider":["partial","*",29,0],"Amazonbot":["partial","*",29,0],"meta-externalagent":["partial","*",29,0],"cohere-ai":["partial","*",29,0],"Diffbot":["partial","*",29,0],"ContextFetchBot":["partial","*",29,0]},"llms":{"status":"missing","http":404,"bytes":21},"home":{"status":"ok","http":200,"url":"https://codecanyon.net/","html_tokens":107173,"markdown_tokens":84,"pruned_tokens":84}},
{"rank":1942,"domain":"europepmc.org","origin":"https://europepmc.org","robots":{"status":"refused","http":403,"url":"https://europepmc.org/robots.txt","bytes":5572,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":1943,"domain":"bain.com","origin":"https://bain.com","robots":{"status":"ok","http":200,"url":"https://www.bain.com/robots.txt","bytes":735,"groups":1},"bots":{"GPTBot":["partial","*",5,0],"ChatGPT-User":["partial","*",5,0],"OAI-SearchBot":["partial","*",5,0],"ClaudeBot":["partial","*",5,0],"Claude-User":["partial","*",5,0],"Claude-SearchBot":["partial","*",5,0],"anthropic-ai":["partial","*",5,0],"PerplexityBot":["partial","*",5,0],"Perplexity-User":["partial","*",5,0],"Google-Extended":["partial","*",5,0],"Applebot-Extended":["partial","*",5,0],"CCBot":["partial","*",5,0],"Bytespider":["partial","*",5,0],"Amazonbot":["partial","*",5,0],"meta-externalagent":["partial","*",5,0],"cohere-ai":["partial","*",5,0],"Diffbot":["partial","*",5,0],"ContextFetchBot":["partial","*",5,0]},"llms":{"status":"present","http":200,"bytes":14444},"home":{"status":"ok","http":200,"url":"https://www.bain.com/","html_tokens":54705,"markdown_tokens":26,"pruned_tokens":26}},
{"rank":1944,"domain":"csiro.au","origin":"https://www.csiro.au","robots":{"status":"ok","http":200,"url":"https://www.csiro.au/robots.txt","bytes":58,"groups":1},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"missing","http":404,"bytes":1245},"home":{"status":"ok","http":200,"url":"https://www.csiro.au/","html_tokens":31372,"markdown_tokens":84,"pruned_tokens":53}},
{"rank":1945,"domain":"uni-bonn.de","origin":"https://uni-bonn.de","robots":{"status":"ok","http":200,"url":"https://www.uni-bonn.de/robots.txt","bytes":134486,"groups":3},"bots":{"GPTBot":["partial","*",1,0],"ChatGPT-User":["partial","*",1,0],"OAI-SearchBot":["partial","*",1,0],"ClaudeBot":["partial","*",1,0],"Claude-User":["partial","*",1,0],"Claude-SearchBot":["partial","*",1,0],"anthropic-ai":["partial","*",1,0],"PerplexityBot":["partial","*",1,0],"Perplexity-User":["partial","*",1,0],"Google-Extended":["partial","*",1,0],"Applebot-Extended":["partial","*",1,0],"CCBot":["partial","*",1,0],"Bytespider":["partial","*",1,0],"Amazonbot":["partial","*",1,0],"meta-externalagent":["partial","*",1,0],"cohere-ai":["partial","*",1,0],"Diffbot":["partial","*",1,0],"ContextFetchBot":["partial","*",1,0]},"llms":{"status":"missing","http":404,"bytes":1261},"home":{"status":"ok","http":200,"url":"https://www.uni-bonn.de/de","html_tokens":40623,"markdown_tokens":529,"pruned_tokens":172}},
{"rank":1946,"domain":"poetryfoundation.org","origin":"https://poetryfoundation.org","robots":{"status":"ok","http":200,"url":"https://www.poetryfoundation.org/robots.txt","bytes":2922,"groups":33},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["partial","*",8,1],"OAI-SearchBot":["partial","*",8,1],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",8,1],"Claude-SearchBot":["partial","*",8,1],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["partial","*",8,1],"Perplexity-User":["partial","*",8,1],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",8,1]},"llms":{"status":"missing","http":404,"bytes":224676},"home":{"status":"ok","http":200,"url":"https://www.poetryfoundation.org/","html_tokens":198125,"markdown_tokens":250,"pruned_tokens":51}},
{"rank":1947,"domain":"fca.org.uk","origin":"https://fca.org.uk","robots":{"status":"ok","http":200,"url":"https://www.fca.org.uk/robots.txt","bytes":2522,"groups":3},"bots":{"GPTBot":["partial","*",36,21],"ChatGPT-User":["partial","*",36,21],"OAI-SearchBot":["partial","*",36,21],"ClaudeBot":["partial","*",36,21],"Claude-User":["partial","*",36,21],"Claude-SearchBot":["partial","*",36,21],"anthropic-ai":["partial","*",36,21],"PerplexityBot":["partial","*",36,21],"Perplexity-User":["partial","*",36,21],"Google-Extended":["partial","*",36,21],"Applebot-Extended":["partial","*",36,21],"CCBot":["partial","*",36,21],"Bytespider":["partial","*",36,21],"Amazonbot":["partial","*",36,21],"meta-externalagent":["partial","*",36,21],"cohere-ai":["partial","*",36,21],"Diffbot":["partial","*",36,21],"ContextFetchBot":["partial","*",36,21]},"llms":{"status":"missing","http":404,"bytes":188},"home":{"status":"ok","http":200,"url":"https://www.fca.org.uk/","html_tokens":45657,"markdown_tokens":117,"pruned_tokens":45}},
{"rank":1948,"domain":"ucoz.ru","origin":"https://ucoz.ru","robots":{"status":"ok","http":200,"url":"https://ucoz.ru/robots.txt","bytes":121,"groups":1},"bots":{"GPTBot":["partial","*",5,0],"ChatGPT-User":["partial","*",5,0],"OAI-SearchBot":["partial","*",5,0],"ClaudeBot":["partial","*",5,0],"Claude-User":["partial","*",5,0],"Claude-SearchBot":["partial","*",5,0],"anthropic-ai":["partial","*",5,0],"PerplexityBot":["partial","*",5,0],"Perplexity-User":["partial","*",5,0],"Google-Extended":["partial","*",5,0],"Applebot-Extended":["partial","*",5,0],"CCBot":["partial","*",5,0],"Bytespider":["partial","*",5,0],"Amazonbot":["partial","*",5,0],"meta-externalagent":["partial","*",5,0],"cohere-ai":["partial","*",5,0],"Diffbot":["partial","*",5,0],"ContextFetchBot":["partial","*",5,0]},"llms":{"status":"present","http":200,"bytes":33987},"home":{"status":"ok","http":200,"url":"https://www.ucoz.ru/","html_tokens":13583,"markdown_tokens":183,"pruned_tokens":133}},
{"rank":1949,"domain":"vrbo.com","origin":"https://vrbo.com","robots":{"status":"ok","http":200,"url":"https://www.vrbo.com/robots.txt","bytes":2046,"groups":4},"bots":{"GPTBot":["partial","*",51,0],"ChatGPT-User":["partial","named",5,1],"OAI-SearchBot":["partial","named",5,1],"ClaudeBot":["partial","*",51,0],"Claude-User":["partial","named",5,1],"Claude-SearchBot":["partial","named",5,1],"anthropic-ai":["partial","*",51,0],"PerplexityBot":["partial","named",5,1],"Perplexity-User":["partial","named",5,1],"Google-Extended":["partial","*",51,0],"Applebot-Extended":["partial","*",51,0],"CCBot":["partial","*",51,0],"Bytespider":["partial","*",51,0],"Amazonbot":["partial","*",51,0],"meta-externalagent":["partial","*",51,0],"cohere-ai":["partial","*",51,0],"Diffbot":["partial","*",51,0],"ContextFetchBot":["partial","*",51,0]},"llms":{"status":"http_error","http":403,"bytes":243},"home":{"status":"http_error","http":429,"url":"https://www.vrbo.com/"}},
{"rank":1950,"domain":"tunein.com","origin":"https://tunein.com","robots":{"status":"ok","http":200,"url":"https://tunein.com/robots.txt","bytes":774,"groups":1},"bots":{"GPTBot":["partial","*",33,0],"ChatGPT-User":["partial","*",33,0],"OAI-SearchBot":["partial","*",33,0],"ClaudeBot":["partial","*",33,0],"Claude-User":["partial","*",33,0],"Claude-SearchBot":["partial","*",33,0],"anthropic-ai":["partial","*",33,0],"PerplexityBot":["partial","*",33,0],"Perplexity-User":["partial","*",33,0],"Google-Extended":["partial","*",33,0],"Applebot-Extended":["partial","*",33,0],"CCBot":["partial","*",33,0],"Bytespider":["partial","*",33,0],"Amazonbot":["partial","*",33,0],"meta-externalagent":["partial","*",33,0],"cohere-ai":["partial","*",33,0],"Diffbot":["partial","*",33,0],"ContextFetchBot":["partial","*",33,0]},"llms":{"status":"missing","http":404,"bytes":327198},"home":{"status":"ok","http":200,"url":"https://tunein.com/","html_tokens":103783,"markdown_tokens":169,"pruned_tokens":76}},
{"rank":1951,"domain":"osf.io","origin":"https://osf.io","robots":{"status":"ok","http":200,"url":"https://osf.io/robots.txt","bytes":254,"groups":2},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["partial","*",2,0],"OAI-SearchBot":["partial","*",2,0],"ClaudeBot":["partial","*",2,0],"Claude-User":["partial","*",2,0],"Claude-SearchBot":["partial","*",2,0],"anthropic-ai":["partial","*",2,0],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["partial","*",2,0],"Google-Extended":["partial","*",2,0],"Applebot-Extended":["partial","*",2,0],"CCBot":["partial","*",2,0],"Bytespider":["partial","*",2,0],"Amazonbot":["partial","*",2,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["partial","*",2,0],"Diffbot":["partial","*",2,0],"ContextFetchBot":["partial","*",2,0]},"llms":{"status":"html","http":200,"bytes":4207},"home":{"status":"ok","http":200,"url":"https://osf.io/","html_tokens":1052,"markdown_tokens":0,"pruned_tokens":0}},
{"rank":1952,"domain":"worldwildlife.org","origin":"https://worldwildlife.org","robots":{"status":"ok","http":200,"url":"https://www.worldwildlife.org/robots.txt","bytes":94,"groups":1},"bots":{"GPTBot":["partial","*",1,1],"ChatGPT-User":["partial","*",1,1],"OAI-SearchBot":["partial","*",1,1],"ClaudeBot":["partial","*",1,1],"Claude-User":["partial","*",1,1],"Claude-SearchBot":["partial","*",1,1],"anthropic-ai":["partial","*",1,1],"PerplexityBot":["partial","*",1,1],"Perplexity-User":["partial","*",1,1],"Google-Extended":["partial","*",1,1],"Applebot-Extended":["partial","*",1,1],"CCBot":["partial","*",1,1],"Bytespider":["partial","*",1,1],"Amazonbot":["partial","*",1,1],"meta-externalagent":["partial","*",1,1],"cohere-ai":["partial","*",1,1],"Diffbot":["partial","*",1,1],"ContextFetchBot":["partial","*",1,1]},"llms":{"status":"missing","http":404,"bytes":146052},"home":{"status":"http_error","http":403,"url":"https://worldwildlife.org/"}},
{"rank":1953,"domain":"famethemes.com","origin":"https://famethemes.com","robots":{"status":"ok","http":200,"url":"https://www.famethemes.com/robots.txt","bytes":176,"groups":1},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"missing","http":404,"bytes":41532},"home":{"status":"ok","http":200,"url":"https://www.famethemes.com/","html_tokens":17641,"markdown_tokens":109,"pruned_tokens":109}},
{"rank":1954,"domain":"sapo.pt","origin":"https://sapo.pt","robots":{"status":"ok","http":200,"url":"https://sapo.pt/robots.txt","bytes":2139,"groups":47},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["blocked","named",1,0],"OAI-SearchBot":["blocked","named",1,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["blocked","named",1,0],"Claude-SearchBot":["blocked","named",1,0],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["blocked","named",1,0],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",10,0]},"llms":{"status":"missing","http":404,"bytes":45232},"home":{"status":"ok","http":200,"url":"https://sapo.pt/","html_tokens":310842,"markdown_tokens":34,"pruned_tokens":34}},
{"rank":1955,"domain":"hearstnetworks.com","origin":"https://hearstnetworks.com","robots":{"status":"ok","http":200,"url":"https://www.hearstnetworks.com/robots.txt","bytes":2229,"groups":3},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["partial","*",34,18],"OAI-SearchBot":["partial","*",34,18],"ClaudeBot":["partial","*",34,18],"Claude-User":["partial","*",34,18],"Claude-SearchBot":["partial","*",34,18],"anthropic-ai":["partial","*",34,18],"PerplexityBot":["partial","*",34,18],"Perplexity-User":["partial","*",34,18],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["partial","*",34,18],"CCBot":["partial","*",34,18],"Bytespider":["partial","*",34,18],"Amazonbot":["partial","*",34,18],"meta-externalagent":["partial","*",34,18],"cohere-ai":["partial","*",34,18],"Diffbot":["partial","*",34,18],"ContextFetchBot":["partial","*",34,18]},"llms":{"status":"missing","http":404,"bytes":196},"home":{"status":"ok","http":200,"url":"https://www.hearstnetworks.com/","html_tokens":10662,"markdown_tokens":736,"pruned_tokens":145}},
{"rank":1956,"domain":"vaticannews.va","origin":"https://vaticannews.va","robots":{"status":"ok","http":200,"url":"https://www.vaticannews.va/robots.txt","bytes":61,"groups":1},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"missing","http":404,"bytes":196},"home":{"status":"ok","http":200,"url":"https://www.vaticannews.va/","html_tokens":381,"markdown_tokens":0,"pruned_tokens":0}},
{"rank":1957,"domain":"blogs.com","origin":"https://blogs.com","robots":{"status":"refused","http":403,"url":"https://www.networksolutions.com/typepad?siteID=100&channelID=P99C100S653N0B5A1D0E0000V111","bytes":5026,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":1958,"domain":"yoast.com","origin":"https://yoast.com","robots":{"status":"ok","http":200,"url":"https://yoast.com/robots.txt","bytes":927,"groups":2},"bots":{"GPTBot":["partial","*",5,0],"ChatGPT-User":["partial","*",5,0],"OAI-SearchBot":["partial","*",5,0],"ClaudeBot":["partial","*",5,0],"Claude-User":["partial","*",5,0],"Claude-SearchBot":["partial","*",5,0],"anthropic-ai":["partial","*",5,0],"PerplexityBot":["partial","*",5,0],"Perplexity-User":["partial","*",5,0],"Google-Extended":["partial","*",5,0],"Applebot-Extended":["partial","*",5,0],"CCBot":["partial","*",5,0],"Bytespider":["partial","*",5,0],"Amazonbot":["partial","*",5,0],"meta-externalagent":["partial","*",5,0],"cohere-ai":["partial","*",5,0],"Diffbot":["partial","*",5,0],"ContextFetchBot":["partial","*",5,0]},"llms":{"status":"present","http":200,"bytes":7682},"home":{"status":"ok","http":200,"url":"https://yoast.com/","html_tokens":56003,"markdown_tokens":831,"pruned_tokens":427}},
{"rank":1959,"domain":"abc.com","origin":"https://abc.com","robots":{"status":"ok","http":200,"url":"https://abc.com/robots.txt","bytes":794,"groups":6},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["partial","*",15,0],"OAI-SearchBot":["partial","*",15,0],"ClaudeBot":["partial","*",15,0],"Claude-User":["partial","*",15,0],"Claude-SearchBot":["partial","*",15,0],"anthropic-ai":["partial","*",15,0],"PerplexityBot":["partial","*",15,0],"Perplexity-User":["partial","*",15,0],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["partial","*",15,0],"CCBot":["partial","*",15,0],"Bytespider":["partial","*",15,0],"Amazonbot":["partial","*",15,0],"meta-externalagent":["partial","*",15,0],"cohere-ai":["partial","*",15,0],"Diffbot":["partial","*",15,0],"ContextFetchBot":["partial","*",15,0]},"llms":{"status":"missing","http":404,"bytes":1},"home":{"status":"ok","http":200,"url":"https://abc.com/","html_tokens":524029,"markdown_tokens":137,"pruned_tokens":36}},
{"rank":1960,"domain":"rentry.co","origin":"https://rentry.co","robots":{"status":"ok","http":200,"url":"https://rentry.co/robots.txt","bytes":152,"groups":1},"bots":{"GPTBot":["partial","*",7,0],"ChatGPT-User":["partial","*",7,0],"OAI-SearchBot":["partial","*",7,0],"ClaudeBot":["partial","*",7,0],"Claude-User":["partial","*",7,0],"Claude-SearchBot":["partial","*",7,0],"anthropic-ai":["partial","*",7,0],"PerplexityBot":["partial","*",7,0],"Perplexity-User":["partial","*",7,0],"Google-Extended":["partial","*",7,0],"Applebot-Extended":["partial","*",7,0],"CCBot":["partial","*",7,0],"Bytespider":["partial","*",7,0],"Amazonbot":["partial","*",7,0],"meta-externalagent":["partial","*",7,0],"cohere-ai":["partial","*",7,0],"Diffbot":["partial","*",7,0],"ContextFetchBot":["partial","*",7,0]},"llms":{"status":"missing","http":404,"bytes":4033},"home":{"status":"ok","http":200,"url":"https://rentry.co/","html_tokens":31434,"markdown_tokens":1090,"pruned_tokens":53}},
{"rank":1961,"domain":"sp.gov.br","origin":"https://sp.gov.br","robots":{"status":"ok","http":200,"url":"https://www.sp.gov.br/robots.txt","bytes":63,"groups":1},"bots":{"GPTBot":["allowed","*",0,1],"ChatGPT-User":["allowed","*",0,1],"OAI-SearchBot":["allowed","*",0,1],"ClaudeBot":["allowed","*",0,1],"Claude-User":["allowed","*",0,1],"Claude-SearchBot":["allowed","*",0,1],"anthropic-ai":["allowed","*",0,1],"PerplexityBot":["allowed","*",0,1],"Perplexity-User":["allowed","*",0,1],"Google-Extended":["allowed","*",0,1],"Applebot-Extended":["allowed","*",0,1],"CCBot":["allowed","*",0,1],"Bytespider":["allowed","*",0,1],"Amazonbot":["allowed","*",0,1],"meta-externalagent":["allowed","*",0,1],"cohere-ai":["allowed","*",0,1],"Diffbot":["allowed","*",0,1],"ContextFetchBot":["allowed","*",0,1]},"llms":{"status":"html","http":200,"bytes":1159},"home":{"status":"ok","http":200,"url":"https://sp.gov.br/","html_tokens":291,"markdown_tokens":21,"pruned_tokens":21}},
{"rank":1962,"domain":"kff.org","origin":"https://kff.org","robots":{"status":"ok","http":200,"url":"https://www.kff.org/robots.txt","bytes":319,"groups":1},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"missing","http":404,"bytes":94536},"home":{"status":"ok","http":200,"url":"https://www.kff.org/","html_tokens":34653,"markdown_tokens":392,"pruned_tokens":114}},
{"rank":1963,"domain":"china.com","origin":"https://www.china.com","robots":{"status":"missing","http":404,"url":"https://www.china.com/robots.txt","bytes":6671,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":6671},"home":{"status":"ok","http":200,"url":"https://www.china.com/","html_tokens":19076,"markdown_tokens":1009,"pruned_tokens":174}},
{"rank":1964,"domain":"androidauthority.com","origin":"https://androidauthority.com","robots":{"status":"ok","http":200,"url":"https://www.androidauthority.com/robots.txt","bytes":560,"groups":6},"bots":{"GPTBot":["partial","*",4,0],"ChatGPT-User":["partial","*",4,0],"OAI-SearchBot":["partial","*",4,0],"ClaudeBot":["partial","*",4,0],"Claude-User":["partial","*",4,0],"Claude-SearchBot":["partial","*",4,0],"anthropic-ai":["partial","*",4,0],"PerplexityBot":["partial","*",4,0],"Perplexity-User":["partial","*",4,0],"Google-Extended":["partial","*",4,0],"Applebot-Extended":["partial","*",4,0],"CCBot":["partial","*",4,0],"Bytespider":["partial","*",4,0],"Amazonbot":["partial","*",4,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["partial","*",4,0],"Diffbot":["partial","*",4,0],"ContextFetchBot":["partial","*",4,0]},"llms":{"status":"missing","http":404,"bytes":119553},"home":{"status":"http_error","http":403,"url":"https://androidauthority.com/"}},
{"rank":1965,"domain":"sfchronicle.com","origin":"https://sfchronicle.com","robots":{"status":"ok","http":200,"url":"https://www.sfchronicle.com/robots.txt","bytes":1144,"groups":3},"bots":{"GPTBot":["partial","*",10,0],"ChatGPT-User":["partial","*",10,0],"OAI-SearchBot":["partial","*",10,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["blocked","named",1,0],"Claude-SearchBot":["partial","*",10,0],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["blocked","named",1,0],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["partial","*",10,0],"meta-externalagent":["partial","*",10,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",10,0]},"llms":{"status":"missing","http":404,"bytes":63603},"home":{"status":"ok","http":200,"url":"https://www.sfchronicle.com/","html_tokens":306860,"markdown_tokens":29,"pruned_tokens":29}},
{"rank":1966,"domain":"boredpanda.com","origin":"https://boredpanda.com","robots":{"status":"ok","http":200,"url":"http://www.boredpanda.com/robots.txt","bytes":1321,"groups":8},"bots":{"GPTBot":["partial","*",5,22],"ChatGPT-User":["partial","*",5,22],"OAI-SearchBot":["partial","*",5,22],"ClaudeBot":["partial","*",5,22],"Claude-User":["partial","*",5,22],"Claude-SearchBot":["partial","*",5,22],"anthropic-ai":["partial","*",5,22],"PerplexityBot":["partial","*",5,22],"Perplexity-User":["partial","*",5,22],"Google-Extended":["partial","*",5,22],"Applebot-Extended":["partial","*",5,22],"CCBot":["partial","*",5,22],"Bytespider":["blocked","named",1,0],"Amazonbot":["partial","*",5,22],"meta-externalagent":["partial","*",5,22],"cohere-ai":["partial","*",5,22],"Diffbot":["partial","*",5,22],"ContextFetchBot":["partial","*",5,22]},"llms":{"status":"present","http":200,"bytes":1320},"home":{"status":"ok","http":200,"url":"https://www.boredpanda.com/","html_tokens":253677,"markdown_tokens":716,"pruned_tokens":168}},
{"rank":1967,"domain":"pp.ua","origin":"https://pp.ua","robots":{"status":"ok","http":200,"url":"https://pp.ua/robots.txt","bytes":119,"groups":1},"bots":{"GPTBot":["partial","*",2,1],"ChatGPT-User":["partial","*",2,1],"OAI-SearchBot":["partial","*",2,1],"ClaudeBot":["partial","*",2,1],"Claude-User":["partial","*",2,1],"Claude-SearchBot":["partial","*",2,1],"anthropic-ai":["partial","*",2,1],"PerplexityBot":["partial","*",2,1],"Perplexity-User":["partial","*",2,1],"Google-Extended":["partial","*",2,1],"Applebot-Extended":["partial","*",2,1],"CCBot":["partial","*",2,1],"Bytespider":["partial","*",2,1],"Amazonbot":["partial","*",2,1],"meta-externalagent":["partial","*",2,1],"cohere-ai":["partial","*",2,1],"Diffbot":["partial","*",2,1],"ContextFetchBot":["partial","*",2,1]},"llms":{"status":"missing","http":404,"bytes":44},"home":{"status":"ok","http":200,"url":"https://pp.ua/","html_tokens":2728,"markdown_tokens":209,"pruned_tokens":116}},
{"rank":1968,"domain":"theepochtimes.com","origin":"https://theepochtimes.com","robots":{"status":"ok","http":200,"url":"https://www.theepochtimes.com/robots.txt","bytes":6352,"groups":9},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["allowed","named",0,1],"OAI-SearchBot":["partial","named",1,11],"ClaudeBot":["blocked","named",1,0],"Claude-User":["allowed","named",0,1],"Claude-SearchBot":["partial","named",1,11],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["partial","named",1,11],"Perplexity-User":["allowed","named",0,1],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["allowed","named",0,1],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",101,0]},"llms":{"status":"missing","http":404,"bytes":58065},"home":{"status":"ok","http":200,"url":"https://www.theepochtimes.com/","html_tokens":317431,"markdown_tokens":659,"pruned_tokens":110}},
{"rank":1969,"domain":"mobirise.com","origin":"https://mobirise.com","robots":{"status":"missing","http":404,"url":"https://mobirise.com/robots.txt","bytes":0,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":0},"home":{"status":"ok","http":200,"url":"https://mobirise.com/","html_tokens":21834,"markdown_tokens":1300,"pruned_tokens":738}},
{"rank":1970,"domain":"unity3d.com","origin":"https://unity3d.com","robots":{"status":"ok","http":200,"url":"https://unity.com/robots.txt","bytes":264,"groups":5},"bots":{"GPTBot":["partial","*",3,0],"ChatGPT-User":["partial","*",3,0],"OAI-SearchBot":["partial","*",3,0],"ClaudeBot":["partial","*",3,0],"Claude-User":["partial","*",3,0],"Claude-SearchBot":["partial","*",3,0],"anthropic-ai":["partial","*",3,0],"PerplexityBot":["partial","*",3,0],"Perplexity-User":["partial","*",3,0],"Google-Extended":["partial","*",3,0],"Applebot-Extended":["partial","*",3,0],"CCBot":["partial","*",3,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["partial","*",3,0],"meta-externalagent":["partial","*",3,0],"cohere-ai":["partial","*",3,0],"Diffbot":["partial","*",3,0],"ContextFetchBot":["partial","*",3,0]},"llms":{"status":"present","http":200,"bytes":937665},"home":{"status":"ok","http":200,"url":"https://unity.com/","html_tokens":213310,"markdown_tokens":562,"pruned_tokens":103}},
{"rank":1971,"domain":"anydesk.com","origin":"https://anydesk.com","robots":{"status":"refused","http":403,"url":"https://anydesk.com/robots.txt","bytes":5506,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":1972,"domain":"taylorfrancis.com","origin":"https://taylorfrancis.com","robots":{"status":"ok","http":200,"url":"https://www.taylorfrancis.com/robots.txt","bytes":15679,"groups":4},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["partial","*",72,102],"OAI-SearchBot":["partial","*",72,102],"ClaudeBot":["partial","*",72,102],"Claude-User":["partial","*",72,102],"Claude-SearchBot":["partial","*",72,102],"anthropic-ai":["partial","*",72,102],"PerplexityBot":["partial","*",72,102],"Perplexity-User":["partial","*",72,102],"Google-Extended":["partial","*",72,102],"Applebot-Extended":["partial","*",72,102],"CCBot":["partial","*",72,102],"Bytespider":["partial","*",72,102],"Amazonbot":["partial","*",72,102],"meta-externalagent":["partial","*",72,102],"cohere-ai":["partial","*",72,102],"Diffbot":["partial","*",72,102],"ContextFetchBot":["partial","*",72,102]},"llms":{"status":"missing","http":404,"bytes":39974},"home":{"status":"http_error","http":403,"url":"https://www.taylorfrancis.com/"}},
{"rank":1973,"domain":"alz.org","origin":"https://alz.org","robots":{"status":"ok","http":200,"url":"https://www.alz.org/robots.txt","bytes":43,"groups":1},"bots":{"GPTBot":["allowed","*",0,1],"ChatGPT-User":["allowed","*",0,1],"OAI-SearchBot":["allowed","*",0,1],"ClaudeBot":["allowed","*",0,1],"Claude-User":["allowed","*",0,1],"Claude-SearchBot":["allowed","*",0,1],"anthropic-ai":["allowed","*",0,1],"PerplexityBot":["allowed","*",0,1],"Perplexity-User":["allowed","*",0,1],"Google-Extended":["allowed","*",0,1],"Applebot-Extended":["allowed","*",0,1],"CCBot":["allowed","*",0,1],"Bytespider":["allowed","*",0,1],"Amazonbot":["allowed","*",0,1],"meta-externalagent":["allowed","*",0,1],"cohere-ai":["allowed","*",0,1],"Diffbot":["allowed","*",0,1],"ContextFetchBot":["allowed","*",0,1]},"llms":{"status":"missing","http":404,"bytes":59159},"home":{"status":"ok","http":200,"url":"https://www.alz.org/","html_tokens":64988,"markdown_tokens":401,"pruned_tokens":42}},
{"rank":1974,"domain":"stern.de","origin":"https://stern.de","robots":{"status":"ok","http":200,"url":"https://www.stern.de/robots.txt","bytes":1055,"groups":13},"bots":{"GPTBot":["partial","named",1,2],"ChatGPT-User":["partial","named",1,2],"OAI-SearchBot":["partial","*",11,0],"ClaudeBot":["partial","*",11,0],"Claude-User":["partial","*",11,0],"Claude-SearchBot":["partial","*",11,0],"anthropic-ai":["partial","*",11,0],"PerplexityBot":["partial","*",11,0],"Perplexity-User":["partial","*",11,0],"Google-Extended":["partial","named",1,2],"Applebot-Extended":["partial","named",1,2],"CCBot":["blocked","named",1,0],"Bytespider":["partial","*",11,0],"Amazonbot":["partial","*",11,0],"meta-externalagent":["partial","*",11,0],"cohere-ai":["partial","*",11,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",11,0]},"llms":{"status":"missing","http":404,"bytes":103320},"home":{"status":"ok","http":200,"url":"https://www.stern.de/","html_tokens":116175,"markdown_tokens":68,"pruned_tokens":68}},
{"rank":1975,"domain":"gsu.edu","origin":"https://gsu.edu","robots":{"status":"ok","http":200,"url":"https://www.gsu.edu/robots.txt","bytes":437,"groups":1},"bots":{"GPTBot":["partial","*",13,0],"ChatGPT-User":["partial","*",13,0],"OAI-SearchBot":["partial","*",13,0],"ClaudeBot":["partial","*",13,0],"Claude-User":["partial","*",13,0],"Claude-SearchBot":["partial","*",13,0],"anthropic-ai":["partial","*",13,0],"PerplexityBot":["partial","*",13,0],"Perplexity-User":["partial","*",13,0],"Google-Extended":["partial","*",13,0],"Applebot-Extended":["partial","*",13,0],"CCBot":["partial","*",13,0],"Bytespider":["partial","*",13,0],"Amazonbot":["partial","*",13,0],"meta-externalagent":["partial","*",13,0],"cohere-ai":["partial","*",13,0],"Diffbot":["partial","*",13,0],"ContextFetchBot":["partial","*",13,0]},"llms":{"status":"html","http":200,"bytes":345764},"home":{"status":"ok","http":200,"url":"https://www.gsu.edu/","html_tokens":103223,"markdown_tokens":353,"pruned_tokens":146}},
{"rank":1976,"domain":"artnet.com","origin":"https://artnet.com","robots":{"status":"refused","http":403,"url":"https://artnet.com/robots.txt","bytes":5505,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":1977,"domain":"hbo.com","origin":"https://hbo.com","robots":{"status":"ok","http":200,"url":"https://www.hbo.com/robots.txt","bytes":87,"groups":1},"bots":{"GPTBot":["partial","*",1,0],"ChatGPT-User":["partial","*",1,0],"OAI-SearchBot":["partial","*",1,0],"ClaudeBot":["partial","*",1,0],"Claude-User":["partial","*",1,0],"Claude-SearchBot":["partial","*",1,0],"anthropic-ai":["partial","*",1,0],"PerplexityBot":["partial","*",1,0],"Perplexity-User":["partial","*",1,0],"Google-Extended":["partial","*",1,0],"Applebot-Extended":["partial","*",1,0],"CCBot":["partial","*",1,0],"Bytespider":["partial","*",1,0],"Amazonbot":["partial","*",1,0],"meta-externalagent":["partial","*",1,0],"cohere-ai":["partial","*",1,0],"Diffbot":["partial","*",1,0],"ContextFetchBot":["partial","*",1,0]},"llms":{"status":"http_error","http":403,"bytes":118},"home":{"status":"ok","http":200,"url":"https://www.hbo.com/","html_tokens":25264,"markdown_tokens":42,"pruned_tokens":42}},
{"rank":1978,"domain":"geocities.jp","origin":"https://www.geocities.jp","robots":{"status":"missing","http":404,"url":"https://www.geocities.jp/robots.txt","bytes":6885,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":6885},"home":{"status":"http_error","http":404,"url":"https://www.geocities.jp/"}},
{"rank":1979,"domain":"medicalxpress.com","origin":"https://medicalxpress.com","robots":{"status":"ok","http":200,"url":"https://medicalxpress.com/robots.txt","bytes":1199,"groups":16},"bots":{"GPTBot":["partial","named",4,0],"ChatGPT-User":["partial","*",5,1],"OAI-SearchBot":["partial","*",5,1],"ClaudeBot":["partial","named",4,0],"Claude-User":["partial","*",5,1],"Claude-SearchBot":["partial","*",5,1],"anthropic-ai":["partial","*",5,1],"PerplexityBot":["partial","*",5,1],"Perplexity-User":["partial","*",5,1],"Google-Extended":["partial","named",2,0],"Applebot-Extended":["partial","named",2,0],"CCBot":["partial","named",4,0],"Bytespider":["partial","named",4,0],"Amazonbot":["partial","*",5,1],"meta-externalagent":["partial","*",5,1],"cohere-ai":["partial","named",4,0],"Diffbot":["partial","*",5,1],"ContextFetchBot":["partial","*",5,1]},"llms":{"status":"present","http":200,"bytes":3145},"home":{"status":"ok","http":200,"url":"https://medicalxpress.com/","html_tokens":34512,"markdown_tokens":1088,"pruned_tokens":62}},
{"rank":1980,"domain":"eonline.com","origin":"https://eonline.com","robots":{"status":"ok","http":200,"url":"https://www.eonline.com/robots.txt","bytes":1545,"groups":23},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["blocked","named",1,0],"OAI-SearchBot":["partial","*",16,1],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",16,1],"Claude-SearchBot":["partial","*",16,1],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["partial","*",16,1],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["partial","*",16,1],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["partial","*",16,1],"cohere-ai":["partial","*",16,1],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",16,1]},"llms":{"status":"missing","http":404,"bytes":37761},"home":{"status":"ok","http":200,"url":"https://www.eonline.com/","html_tokens":123485,"markdown_tokens":144,"pruned_tokens":144}},
{"rank":1981,"domain":"lenta.ru","origin":"https://lenta.ru","robots":{"status":"ok","http":200,"url":"https://lenta.ru/robots.txt","bytes":1118,"groups":3},"bots":{"GPTBot":["partial","*",12,0],"ChatGPT-User":["partial","*",12,0],"OAI-SearchBot":["partial","*",12,0],"ClaudeBot":["partial","*",12,0],"Claude-User":["partial","*",12,0],"Claude-SearchBot":["partial","*",12,0],"anthropic-ai":["partial","*",12,0],"PerplexityBot":["partial","*",12,0],"Perplexity-User":["partial","*",12,0],"Google-Extended":["partial","*",12,0],"Applebot-Extended":["partial","*",12,0],"CCBot":["partial","*",12,0],"Bytespider":["partial","*",12,0],"Amazonbot":["partial","*",12,0],"meta-externalagent":["partial","*",12,0],"cohere-ai":["partial","*",12,0],"Diffbot":["partial","*",12,0],"ContextFetchBot":["partial","*",12,0]},"llms":{"status":"missing","http":404,"bytes":0},"home":{"status":"ok","http":200,"url":"https://lenta.ru/","html_tokens":32072,"markdown_tokens":34,"pruned_tokens":34}},
{"rank":1982,"domain":"yadi.sk","origin":"https://yadi.sk","robots":{"status":"ok","http":200,"url":"https://yadi.sk/robots.txt","bytes":677,"groups":1},"bots":{"GPTBot":["partial","*",27,2],"ChatGPT-User":["partial","*",27,2],"OAI-SearchBot":["partial","*",27,2],"ClaudeBot":["partial","*",27,2],"Claude-User":["partial","*",27,2],"Claude-SearchBot":["partial","*",27,2],"anthropic-ai":["partial","*",27,2],"PerplexityBot":["partial","*",27,2],"Perplexity-User":["partial","*",27,2],"Google-Extended":["partial","*",27,2],"Applebot-Extended":["partial","*",27,2],"CCBot":["partial","*",27,2],"Bytespider":["partial","*",27,2],"Amazonbot":["partial","*",27,2],"meta-externalagent":["partial","*",27,2],"cohere-ai":["partial","*",27,2],"Diffbot":["partial","*",27,2],"ContextFetchBot":["partial","*",27,2]},"llms":{"status":"missing","http":404,"bytes":18594},"home":{"status":"ok","http":200,"url":"https://360.yandex.com/disk/","html_tokens":201315,"markdown_tokens":85,"pruned_tokens":85}},
{"rank":1983,"domain":"networksolutions.com","origin":"https://networksolutions.com","robots":{"status":"refused","http":403,"url":"https://www.networksolutions.com/robots.txt","bytes":12241,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":1984,"domain":"nicovideo.jp","origin":"https://nicovideo.jp","robots":{"status":"ok","http":200,"url":"https://www.nicovideo.jp/robots.txt","bytes":1089,"groups":3},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["partial","*",18,1],"OAI-SearchBot":["partial","*",18,1],"ClaudeBot":["partial","*",18,1],"Claude-User":["partial","*",18,1],"Claude-SearchBot":["partial","*",18,1],"anthropic-ai":["partial","*",18,1],"PerplexityBot":["partial","*",18,1],"Perplexity-User":["partial","*",18,1],"Google-Extended":["partial","*",18,1],"Applebot-Extended":["partial","*",18,1],"CCBot":["partial","*",18,1],"Bytespider":["partial","*",18,1],"Amazonbot":["partial","*",18,1],"meta-externalagent":["partial","*",18,1],"cohere-ai":["partial","*",18,1],"Diffbot":["partial","*",18,1],"ContextFetchBot":["partial","*",18,1]},"llms":{"status":"missing","http":404,"bytes":10092},"home":{"status":"ok","http":200,"url":"https://www.nicovideo.jp/","html_tokens":35579,"markdown_tokens":13,"pruned_tokens":13}},
{"rank":1985,"domain":"discovermagazine.com","origin":"https://discovermagazine.com","robots":{"status":"ok","http":200,"url":"https://www.discovermagazine.com/robots.txt","bytes":402,"groups":1},"bots":{"GPTBot":["partial","*",10,0],"ChatGPT-User":["partial","*",10,0],"OAI-SearchBot":["partial","*",10,0],"ClaudeBot":["partial","*",10,0],"Claude-User":["partial","*",10,0],"Claude-SearchBot":["partial","*",10,0],"anthropic-ai":["partial","*",10,0],"PerplexityBot":["partial","*",10,0],"Perplexity-User":["partial","*",10,0],"Google-Extended":["partial","*",10,0],"Applebot-Extended":["partial","*",10,0],"CCBot":["partial","*",10,0],"Bytespider":["partial","*",10,0],"Amazonbot":["partial","*",10,0],"meta-externalagent":["partial","*",10,0],"cohere-ai":["partial","*",10,0],"Diffbot":["partial","*",10,0],"ContextFetchBot":["partial","*",10,0]},"llms":{"status":"http_error","http":403,"bytes":919},"home":{"status":"http_error","http":403,"url":"https://www.discovermagazine.com/"}},
{"rank":1986,"domain":"olympic.org","origin":"https://olympic.org","robots":{"status":"missing","http":404,"url":"https://olympic.org/robots.txt","bytes":162,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":162},"home":{"status":"disallowed","error":"robots.txt of www.olympics.com not readable (unreachable)"}},
{"rank":1987,"domain":"bitdefender.com","origin":"https://bitdefender.com","robots":{"status":"ok","http":200,"url":"https://www.bitdefender.com/robots.txt","bytes":1361,"groups":1},"bots":{"GPTBot":["partial","*",32,6],"ChatGPT-User":["partial","*",32,6],"OAI-SearchBot":["partial","*",32,6],"ClaudeBot":["partial","*",32,6],"Claude-User":["partial","*",32,6],"Claude-SearchBot":["partial","*",32,6],"anthropic-ai":["partial","*",32,6],"PerplexityBot":["partial","*",32,6],"Perplexity-User":["partial","*",32,6],"Google-Extended":["partial","*",32,6],"Applebot-Extended":["partial","*",32,6],"CCBot":["partial","*",32,6],"Bytespider":["partial","*",32,6],"Amazonbot":["partial","*",32,6],"meta-externalagent":["partial","*",32,6],"cohere-ai":["partial","*",32,6],"Diffbot":["partial","*",32,6],"ContextFetchBot":["partial","*",32,6]},"llms":{"status":"present","http":200,"bytes":15471},"home":{"status":"ok","http":200,"url":"https://www.bitdefender.com/en-us/","html_tokens":65796,"markdown_tokens":368,"pruned_tokens":137}},
{"rank":1988,"domain":"dnspod.cn","origin":"https://dnspod.cn","robots":{"status":"html","http":200,"url":"https://www.dnspod.cn/","bytes":11973,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":1989,"domain":"wordstream.com","origin":"https://wordstream.com","robots":{"status":"ok","http":200,"url":"https://www.wordstream.com/robots.txt","bytes":531,"groups":10},"bots":{"GPTBot":["partial","*",3,0],"ChatGPT-User":["partial","*",3,0],"OAI-SearchBot":["allowed","named",0,1],"ClaudeBot":["allowed","named",0,1],"Claude-User":["allowed","named",0,1],"Claude-SearchBot":["allowed","named",0,1],"anthropic-ai":["partial","*",3,0],"PerplexityBot":["partial","*",3,0],"Perplexity-User":["partial","*",3,0],"Google-Extended":["partial","*",3,0],"Applebot-Extended":["partial","*",3,0],"CCBot":["partial","*",3,0],"Bytespider":["partial","*",3,0],"Amazonbot":["partial","*",3,0],"meta-externalagent":["partial","*",3,0],"cohere-ai":["partial","*",3,0],"Diffbot":["partial","*",3,0],"ContextFetchBot":["partial","*",3,0]},"llms":{"status":"present","http":200,"bytes":163690},"home":{"status":"http_error","http":403,"url":"https://www.wordstream.com/"}},
{"rank":1990,"domain":"cal.com","origin":"https://cal.com","robots":{"status":"ok","http":200,"url":"https://cal.com/robots.txt","bytes":737,"groups":1},"bots":{"GPTBot":["partial","*",4,2],"ChatGPT-User":["partial","*",4,2],"OAI-SearchBot":["partial","*",4,2],"ClaudeBot":["partial","*",4,2],"Claude-User":["partial","*",4,2],"Claude-SearchBot":["partial","*",4,2],"anthropic-ai":["partial","*",4,2],"PerplexityBot":["partial","*",4,2],"Perplexity-User":["partial","*",4,2],"Google-Extended":["partial","*",4,2],"Applebot-Extended":["partial","*",4,2],"CCBot":["partial","*",4,2],"Bytespider":["partial","*",4,2],"Amazonbot":["partial","*",4,2],"meta-externalagent":["partial","*",4,2],"cohere-ai":["partial","*",4,2],"Diffbot":["partial","*",4,2],"ContextFetchBot":["partial","*",4,2]},"llms":{"status":"other_text","http":200,"bytes":14318},"home":{"status":"ok","http":200,"url":"https://cal.com/","html_tokens":579100,"markdown_tokens":1282,"pruned_tokens":789}},
{"rank":1991,"domain":"ehow.com","origin":"https://ehow.com","robots":{"status":"ok","http":200,"url":"https://www.ehow.com/robots.txt","bytes":456,"groups":2},"bots":{"GPTBot":["partial","*",10,0],"ChatGPT-User":["partial","*",10,0],"OAI-SearchBot":["partial","*",10,0],"ClaudeBot":["partial","*",10,0],"Claude-User":["partial","*",10,0],"Claude-SearchBot":["partial","*",10,0],"anthropic-ai":["partial","*",10,0],"PerplexityBot":["partial","*",10,0],"Perplexity-User":["partial","*",10,0],"Google-Extended":["partial","*",10,0],"Applebot-Extended":["partial","*",10,0],"CCBot":["partial","*",10,0],"Bytespider":["partial","*",10,0],"Amazonbot":["partial","*",10,0],"meta-externalagent":["partial","*",10,0],"cohere-ai":["partial","*",10,0],"Diffbot":["partial","*",10,0],"ContextFetchBot":["partial","*",10,0]},"llms":{"status":"missing","http":404,"bytes":0},"home":{"status":"http_error","http":403,"url":"https://www.ehow.com/"}},
{"rank":1992,"domain":"wwd.com","origin":"https://wwd.com","robots":{"status":"refused","http":403,"url":"https://wwd.com/robots.txt","bytes":0,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":1993,"domain":"sphinx-doc.org","origin":"https://sphinx-doc.org","robots":{"status":"ok","http":200,"url":"https://www.sphinx-doc.org/robots.txt","bytes":668,"groups":1},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"missing","http":404,"bytes":2353},"home":{"status":"ok","http":200,"url":"https://www.sphinx-doc.org/en/master/","html_tokens":7404,"markdown_tokens":557,"pruned_tokens":339}},
{"rank":1994,"domain":"sina.cn","origin":"https://sina.cn","robots":{"status":"ok","http":200,"url":"https://sina.cn/robots.txt","bytes":341,"groups":1},"bots":{"GPTBot":["allowed","*",0,1],"ChatGPT-User":["allowed","*",0,1],"OAI-SearchBot":["allowed","*",0,1],"ClaudeBot":["allowed","*",0,1],"Claude-User":["allowed","*",0,1],"Claude-SearchBot":["allowed","*",0,1],"anthropic-ai":["allowed","*",0,1],"PerplexityBot":["allowed","*",0,1],"Perplexity-User":["allowed","*",0,1],"Google-Extended":["allowed","*",0,1],"Applebot-Extended":["allowed","*",0,1],"CCBot":["allowed","*",0,1],"Bytespider":["allowed","*",0,1],"Amazonbot":["allowed","*",0,1],"meta-externalagent":["allowed","*",0,1],"cohere-ai":["allowed","*",0,1],"Diffbot":["allowed","*",0,1],"ContextFetchBot":["allowed","*",0,1]},"llms":{"status":"empty","http":200,"bytes":0},"home":{"status":"ok","http":200,"url":"https://sina.cn/","html_tokens":31940,"markdown_tokens":27,"pruned_tokens":27}},
{"rank":1995,"domain":"ku.edu","origin":"https://ku.edu","robots":{"status":"ok","http":200,"url":"https://ku.edu/robots.txt","bytes":2027,"groups":1},"bots":{"GPTBot":["partial","*",34,18],"ChatGPT-User":["partial","*",34,18],"OAI-SearchBot":["partial","*",34,18],"ClaudeBot":["partial","*",34,18],"Claude-User":["partial","*",34,18],"Claude-SearchBot":["partial","*",34,18],"anthropic-ai":["partial","*",34,18],"PerplexityBot":["partial","*",34,18],"Perplexity-User":["partial","*",34,18],"Google-Extended":["partial","*",34,18],"Applebot-Extended":["partial","*",34,18],"CCBot":["partial","*",34,18],"Bytespider":["partial","*",34,18],"Amazonbot":["partial","*",34,18],"meta-externalagent":["partial","*",34,18],"cohere-ai":["partial","*",34,18],"Diffbot":["partial","*",34,18],"ContextFetchBot":["partial","*",34,18]},"llms":{"status":"http_error","http":403,"bytes":146},"home":{"status":"ok","http":200,"url":"https://ku.edu/","html_tokens":18836,"markdown_tokens":200,"pruned_tokens":65}},
{"rank":1996,"domain":"ethereum.org","origin":"https://ethereum.org","robots":{"status":"ok","http":200,"url":"https://ethereum.org/robots.txt","bytes":264,"groups":1},"bots":{"GPTBot":["allowed","*",0,1],"ChatGPT-User":["allowed","*",0,1],"OAI-SearchBot":["allowed","*",0,1],"ClaudeBot":["allowed","*",0,1],"Claude-User":["allowed","*",0,1],"Claude-SearchBot":["allowed","*",0,1],"anthropic-ai":["allowed","*",0,1],"PerplexityBot":["allowed","*",0,1],"Perplexity-User":["allowed","*",0,1],"Google-Extended":["allowed","*",0,1],"Applebot-Extended":["allowed","*",0,1],"CCBot":["allowed","*",0,1],"Bytespider":["allowed","*",0,1],"Amazonbot":["allowed","*",0,1],"meta-externalagent":["allowed","*",0,1],"cohere-ai":["allowed","*",0,1],"Diffbot":["allowed","*",0,1],"ContextFetchBot":["allowed","*",0,1]},"llms":{"status":"present","http":200,"bytes":13239},"home":{"status":"ok","http":200,"url":"https://ethereum.org/","html_tokens":105357,"markdown_tokens":614,"pruned_tokens":121}},
{"rank":1997,"domain":"yamaha.com","origin":"https://yamaha.com","robots":{"status":"refused","http":403,"url":"http://www.yamaha.com/robots.txt","bytes":382,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":1998,"domain":"kp.ru","origin":"https://www.kp.ru","robots":{"status":"ok","http":200,"url":"https://www.kp.ru/robots.txt","bytes":1021,"groups":3},"bots":{"GPTBot":["partial","*",20,0],"ChatGPT-User":["partial","*",20,0],"OAI-SearchBot":["partial","*",20,0],"ClaudeBot":["partial","*",20,0],"Claude-User":["partial","*",20,0],"Claude-SearchBot":["partial","*",20,0],"anthropic-ai":["partial","*",20,0],"PerplexityBot":["partial","*",20,0],"Perplexity-User":["partial","*",20,0],"Google-Extended":["partial","*",20,0],"Applebot-Extended":["partial","*",20,0],"CCBot":["partial","*",20,0],"Bytespider":["partial","*",20,0],"Amazonbot":["partial","*",20,0],"meta-externalagent":["partial","*",20,0],"cohere-ai":["partial","*",20,0],"Diffbot":["partial","*",20,0],"ContextFetchBot":["partial","*",20,0]},"llms":{"status":"timeout","error":"timed out after 10 s"},"home":{"status":"ok","http":200,"url":"https://www.kp.ru/","html_tokens":153558,"markdown_tokens":38,"pruned_tokens":38}},
{"rank":1999,"domain":"primevideo.com","origin":"https://primevideo.com","robots":{"status":"ok","http":200,"url":"https://www.primevideo.com/robots.txt","bytes":4348,"groups":94},"bots":{"GPTBot":["partial","*",33,0],"ChatGPT-User":["blocked","named",1,0],"OAI-SearchBot":["partial","*",33,0],"ClaudeBot":["partial","*",33,0],"Claude-User":["blocked","named",1,0],"Claude-SearchBot":["partial","*",33,0],"anthropic-ai":["partial","*",33,0],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["blocked","named",1,0],"Google-Extended":["partial","*",33,0],"Applebot-Extended":["partial","*",33,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["partial","*",33,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",33,0]},"llms":{"status":"missing","http":404,"bytes":6626},"home":{"status":"ok","http":200,"url":"https://www.primevideo.com/","html_tokens":1150618,"markdown_tokens":429,"pruned_tokens":169}},
{"rank":2000,"domain":"roblox.com","origin":"https://roblox.com","robots":{"status":"ok","http":200,"url":"https://www.roblox.com/robots.txt","bytes":2181,"groups":1},"bots":{"GPTBot":["partial","*",28,0],"ChatGPT-User":["partial","*",28,0],"OAI-SearchBot":["partial","*",28,0],"ClaudeBot":["partial","*",28,0],"Claude-User":["partial","*",28,0],"Claude-SearchBot":["partial","*",28,0],"anthropic-ai":["partial","*",28,0],"PerplexityBot":["partial","*",28,0],"Perplexity-User":["partial","*",28,0],"Google-Extended":["partial","*",28,0],"Applebot-Extended":["partial","*",28,0],"CCBot":["partial","*",28,0],"Bytespider":["partial","*",28,0],"Amazonbot":["partial","*",28,0],"meta-externalagent":["partial","*",28,0],"cohere-ai":["partial","*",28,0],"Diffbot":["partial","*",28,0],"ContextFetchBot":["partial","*",28,0]},"llms":{"status":"missing","http":404,"bytes":67417},"home":{"status":"ok","http":200,"url":"https://www.roblox.com/","html_tokens":15969,"markdown_tokens":0,"pruned_tokens":0}},
{"rank":2001,"domain":"kriesi.at","origin":"https://kriesi.at","robots":{"status":"ok","http":200,"url":"https://kriesi.at/robots.txt","bytes":151,"groups":1},"bots":{"GPTBot":["partial","*",1,1],"ChatGPT-User":["partial","*",1,1],"OAI-SearchBot":["partial","*",1,1],"ClaudeBot":["partial","*",1,1],"Claude-User":["partial","*",1,1],"Claude-SearchBot":["partial","*",1,1],"anthropic-ai":["partial","*",1,1],"PerplexityBot":["partial","*",1,1],"Perplexity-User":["partial","*",1,1],"Google-Extended":["partial","*",1,1],"Applebot-Extended":["partial","*",1,1],"CCBot":["partial","*",1,1],"Bytespider":["partial","*",1,1],"Amazonbot":["partial","*",1,1],"meta-externalagent":["partial","*",1,1],"cohere-ai":["partial","*",1,1],"Diffbot":["partial","*",1,1],"ContextFetchBot":["partial","*",1,1]},"llms":{"status":"missing","http":404,"bytes":46868},"home":{"status":"ok","http":200,"url":"https://kriesi.at/","html_tokens":21529,"markdown_tokens":354,"pruned_tokens":138}},
{"rank":2002,"domain":"kyoto-u.ac.jp","origin":"https://kyoto-u.ac.jp","robots":{"status":"ok","http":200,"url":"https://www.kyoto-u.ac.jp/robots.txt","bytes":1950,"groups":2},"bots":{"GPTBot":["partial","*",28,18],"ChatGPT-User":["partial","*",28,18],"OAI-SearchBot":["partial","*",28,18],"ClaudeBot":["partial","*",28,18],"Claude-User":["partial","*",28,18],"Claude-SearchBot":["partial","*",28,18],"anthropic-ai":["partial","*",28,18],"PerplexityBot":["partial","*",28,18],"Perplexity-User":["partial","*",28,18],"Google-Extended":["partial","*",28,18],"Applebot-Extended":["partial","*",28,18],"CCBot":["partial","*",28,18],"Bytespider":["partial","*",28,18],"Amazonbot":["partial","*",28,18],"meta-externalagent":["partial","*",28,18],"cohere-ai":["partial","*",28,18],"Diffbot":["partial","*",28,18],"ContextFetchBot":["partial","*",28,18]},"llms":{"status":"missing","http":404,"bytes":191},"home":{"status":"ok","http":200,"url":"https://www.kyoto-u.ac.jp/ja","html_tokens":51316,"markdown_tokens":10,"pruned_tokens":10}},
{"rank":2003,"domain":"google.com.tw","origin":"https://google.com.tw","robots":{"status":"ok","http":200,"url":"https://www.google.com.tw/robots.txt","bytes":6624,"groups":4},"bots":{"GPTBot":["partial","*",169,64],"ChatGPT-User":["partial","*",169,64],"OAI-SearchBot":["partial","*",169,64],"ClaudeBot":["partial","*",169,64],"Claude-User":["partial","*",169,64],"Claude-SearchBot":["partial","*",169,64],"anthropic-ai":["partial","*",169,64],"PerplexityBot":["partial","*",169,64],"Perplexity-User":["partial","*",169,64],"Google-Extended":["partial","*",169,64],"Applebot-Extended":["partial","*",169,64],"CCBot":["partial","*",169,64],"Bytespider":["partial","*",169,64],"Amazonbot":["partial","*",169,64],"meta-externalagent":["partial","*",169,64],"cohere-ai":["partial","*",169,64],"Diffbot":["partial","*",169,64],"ContextFetchBot":["partial","*",169,64]},"llms":{"status":"missing","http":404,"bytes":1569},"home":{"status":"ok","http":200,"url":"https://www.google.com/","html_tokens":21296,"markdown_tokens":30,"pruned_tokens":30}},
{"rank":2004,"domain":"vc.ru","origin":"https://vc.ru","robots":{"status":"ok","http":200,"url":"https://vc.ru/robots.txt","bytes":556,"groups":1},"bots":{"GPTBot":["partial","*",25,0],"ChatGPT-User":["partial","*",25,0],"OAI-SearchBot":["partial","*",25,0],"ClaudeBot":["partial","*",25,0],"Claude-User":["partial","*",25,0],"Claude-SearchBot":["partial","*",25,0],"anthropic-ai":["partial","*",25,0],"PerplexityBot":["partial","*",25,0],"Perplexity-User":["partial","*",25,0],"Google-Extended":["partial","*",25,0],"Applebot-Extended":["partial","*",25,0],"CCBot":["partial","*",25,0],"Bytespider":["partial","*",25,0],"Amazonbot":["partial","*",25,0],"meta-externalagent":["partial","*",25,0],"cohere-ai":["partial","*",25,0],"Diffbot":["partial","*",25,0],"ContextFetchBot":["partial","*",25,0]},"llms":{"status":"missing","http":404,"bytes":564662},"home":{"status":"ok","http":200,"url":"https://vc.ru/","html_tokens":130669,"markdown_tokens":88,"pruned_tokens":88}},
{"rank":2005,"domain":"cbssports.com","origin":"https://cbssports.com","robots":{"status":"ok","http":200,"url":"https://www.cbssports.com/robots.txt","bytes":875,"groups":2},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["partial","*",17,0],"OAI-SearchBot":["partial","*",17,0],"ClaudeBot":["partial","*",17,0],"Claude-User":["partial","*",17,0],"Claude-SearchBot":["partial","*",17,0],"anthropic-ai":["partial","*",17,0],"PerplexityBot":["partial","*",17,0],"Perplexity-User":["partial","*",17,0],"Google-Extended":["partial","*",17,0],"Applebot-Extended":["partial","*",17,0],"CCBot":["partial","*",17,0],"Bytespider":["partial","*",17,0],"Amazonbot":["partial","*",17,0],"meta-externalagent":["partial","*",17,0],"cohere-ai":["partial","*",17,0],"Diffbot":["partial","*",17,0],"ContextFetchBot":["partial","*",17,0]},"llms":{"status":"missing","http":404,"bytes":424965},"home":{"status":"ok","http":200,"url":"https://www.cbssports.com/","html_tokens":276759,"markdown_tokens":4797,"pruned_tokens":80}},
{"rank":2006,"domain":"webnode.com","origin":"https://webnode.com","robots":{"status":"ok","http":200,"url":"https://www.webnode.com/robots.txt","bytes":677,"groups":2},"bots":{"GPTBot":["partial","*",25,0],"ChatGPT-User":["partial","*",25,0],"OAI-SearchBot":["partial","*",25,0],"ClaudeBot":["partial","*",25,0],"Claude-User":["partial","*",25,0],"Claude-SearchBot":["partial","*",25,0],"anthropic-ai":["partial","*",25,0],"PerplexityBot":["partial","*",25,0],"Perplexity-User":["partial","*",25,0],"Google-Extended":["partial","*",25,0],"Applebot-Extended":["partial","*",25,0],"CCBot":["partial","*",25,0],"Bytespider":["partial","*",25,0],"Amazonbot":["partial","*",25,0],"meta-externalagent":["partial","*",25,0],"cohere-ai":["partial","*",25,0],"Diffbot":["partial","*",25,0],"ContextFetchBot":["partial","*",25,0]},"llms":{"status":"present","http":200,"bytes":3499},"home":{"status":"ok","http":200,"url":"https://www.webnode.com/","html_tokens":61774,"markdown_tokens":81,"pruned_tokens":81}},
{"rank":2007,"domain":"esteri.it","origin":"https://esteri.it","robots":{"status":"ok","http":200,"url":"https://www.esteri.it/robots.txt","bytes":117,"groups":1},"bots":{"GPTBot":["partial","*",1,1],"ChatGPT-User":["partial","*",1,1],"OAI-SearchBot":["partial","*",1,1],"ClaudeBot":["partial","*",1,1],"Claude-User":["partial","*",1,1],"Claude-SearchBot":["partial","*",1,1],"anthropic-ai":["partial","*",1,1],"PerplexityBot":["partial","*",1,1],"Perplexity-User":["partial","*",1,1],"Google-Extended":["partial","*",1,1],"Applebot-Extended":["partial","*",1,1],"CCBot":["partial","*",1,1],"Bytespider":["partial","*",1,1],"Amazonbot":["partial","*",1,1],"meta-externalagent":["partial","*",1,1],"cohere-ai":["partial","*",1,1],"Diffbot":["partial","*",1,1],"ContextFetchBot":["partial","*",1,1]},"llms":{"status":"html","http":200,"bytes":2447},"home":{"status":"ok","http":200,"url":"https://www.esteri.it/it/","html_tokens":46991,"markdown_tokens":286,"pruned_tokens":71}},
{"rank":2008,"domain":"loom.com","origin":"https://loom.com","robots":{"status":"ok","http":200,"url":"https://www.loom.com/robots.txt","bytes":377,"groups":6},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["allowed","*",0,1],"OAI-SearchBot":["allowed","*",0,1],"ClaudeBot":["allowed","*",0,1],"Claude-User":["allowed","*",0,1],"Claude-SearchBot":["allowed","*",0,1],"anthropic-ai":["allowed","*",0,1],"PerplexityBot":["allowed","*",0,1],"Perplexity-User":["allowed","*",0,1],"Google-Extended":["allowed","*",0,1],"Applebot-Extended":["allowed","*",0,1],"CCBot":["allowed","*",0,1],"Bytespider":["allowed","*",0,1],"Amazonbot":["partial","named",1,0],"meta-externalagent":["allowed","*",0,1],"cohere-ai":["allowed","*",0,1],"Diffbot":["allowed","*",0,1],"ContextFetchBot":["allowed","*",0,1]},"llms":{"status":"present","http":200,"bytes":908},"home":{"status":"ok","http":200,"url":"https://www.loom.com/","html_tokens":51794,"markdown_tokens":231,"pruned_tokens":50}},
{"rank":2009,"domain":"fourseasons.com","origin":"https://fourseasons.com","robots":{"status":"refused","http":403,"url":"https://www.fourseasons.com/robots.txt","bytes":387,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":2010,"domain":"examiner.com","origin":"https://examiner.com","robots":{"status":"ok","http":200,"url":"https://examiner.com/robots.txt","bytes":228,"groups":1},"bots":{"GPTBot":["partial","*",10,0],"ChatGPT-User":["partial","*",10,0],"OAI-SearchBot":["partial","*",10,0],"ClaudeBot":["partial","*",10,0],"Claude-User":["partial","*",10,0],"Claude-SearchBot":["partial","*",10,0],"anthropic-ai":["partial","*",10,0],"PerplexityBot":["partial","*",10,0],"Perplexity-User":["partial","*",10,0],"Google-Extended":["partial","*",10,0],"Applebot-Extended":["partial","*",10,0],"CCBot":["partial","*",10,0],"Bytespider":["partial","*",10,0],"Amazonbot":["partial","*",10,0],"meta-externalagent":["partial","*",10,0],"cohere-ai":["partial","*",10,0],"Diffbot":["partial","*",10,0],"ContextFetchBot":["partial","*",10,0]},"llms":{"status":"html","http":200,"bytes":163490},"home":{"status":"ok","http":200,"url":"https://examiner.com/","html_tokens":40868,"markdown_tokens":160,"pruned_tokens":142}},
{"rank":2011,"domain":"lviv.ua","origin":"https://lviv.ua","robots":{"status":"unreachable","http":null,"url":"https://lviv.ua/robots.txt","bytes":0,"groups":0,"error":"cannot resolve lviv.ua"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":2012,"domain":"ryanair.com","origin":"https://ryanair.com","robots":{"status":"html","http":200,"url":"https://www.ryanair.com/us/en","bytes":4743,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":2013,"domain":"unm.edu","origin":"https://unm.edu","robots":{"status":"ok","http":200,"url":"https://www.unm.edu/robots.txt","bytes":99,"groups":1},"bots":{"GPTBot":["partial","*",3,0],"ChatGPT-User":["partial","*",3,0],"OAI-SearchBot":["partial","*",3,0],"ClaudeBot":["partial","*",3,0],"Claude-User":["partial","*",3,0],"Claude-SearchBot":["partial","*",3,0],"anthropic-ai":["partial","*",3,0],"PerplexityBot":["partial","*",3,0],"Perplexity-User":["partial","*",3,0],"Google-Extended":["partial","*",3,0],"Applebot-Extended":["partial","*",3,0],"CCBot":["partial","*",3,0],"Bytespider":["partial","*",3,0],"Amazonbot":["partial","*",3,0],"meta-externalagent":["partial","*",3,0],"cohere-ai":["partial","*",3,0],"Diffbot":["partial","*",3,0],"ContextFetchBot":["partial","*",3,0]},"llms":{"status":"missing","http":404,"bytes":30367},"home":{"status":"ok","http":200,"url":"https://www.unm.edu/","html_tokens":9570,"markdown_tokens":233,"pruned_tokens":117}},
{"rank":2014,"domain":"goal.com","origin":"https://goal.com","robots":{"status":"ok","http":200,"url":"https://www.goal.com/robots.txt","bytes":1558,"groups":1},"bots":{"GPTBot":["allowed","*",0,1],"ChatGPT-User":["allowed","*",0,1],"OAI-SearchBot":["allowed","*",0,1],"ClaudeBot":["allowed","*",0,1],"Claude-User":["allowed","*",0,1],"Claude-SearchBot":["allowed","*",0,1],"anthropic-ai":["allowed","*",0,1],"PerplexityBot":["allowed","*",0,1],"Perplexity-User":["allowed","*",0,1],"Google-Extended":["allowed","*",0,1],"Applebot-Extended":["allowed","*",0,1],"CCBot":["allowed","*",0,1],"Bytespider":["allowed","*",0,1],"Amazonbot":["allowed","*",0,1],"meta-externalagent":["allowed","*",0,1],"cohere-ai":["allowed","*",0,1],"Diffbot":["allowed","*",0,1],"ContextFetchBot":["allowed","*",0,1]},"llms":{"status":"missing","http":404,"bytes":131224},"home":{"status":"ok","http":200,"url":"https://www.goal.com/en-us","html_tokens":461881,"markdown_tokens":301,"pruned_tokens":158}},
{"rank":2015,"domain":"googlepages.com","origin":"http://googlepages.com","robots":{"status":"ok","http":200,"url":"https://sites.google.com/robots.txt","bytes":84,"groups":1},"bots":{"GPTBot":["partial","*",2,2],"ChatGPT-User":["partial","*",2,2],"OAI-SearchBot":["partial","*",2,2],"ClaudeBot":["partial","*",2,2],"Claude-User":["partial","*",2,2],"Claude-SearchBot":["partial","*",2,2],"anthropic-ai":["partial","*",2,2],"PerplexityBot":["partial","*",2,2],"Perplexity-User":["partial","*",2,2],"Google-Extended":["partial","*",2,2],"Applebot-Extended":["partial","*",2,2],"CCBot":["partial","*",2,2],"Bytespider":["partial","*",2,2],"Amazonbot":["partial","*",2,2],"meta-externalagent":["partial","*",2,2],"cohere-ai":["partial","*",2,2],"Diffbot":["partial","*",2,2],"ContextFetchBot":["partial","*",2,2]},"llms":{"status":"missing","http":404,"bytes":1652},"home":{"status":"ok","http":200,"url":"https://accounts.google.com/v3/signin/identifier?continue=https://sites.google.com/&followup=https://sites.google.com/&osid=1&passive=1209600&service=wise&flowName=WebLiteSignIn&flowEntry=ServiceLogin&dsh=S-2042940730:1792374309676145","html_tokens":236887,"markdown_tokens":7,"pruned_tokens":7}},
{"rank":2016,"domain":"nami.org","origin":"https://nami.org","robots":{"status":"ok","http":200,"url":"https://www.nami.org/robots.txt","bytes":171,"groups":1},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"other_text","http":200,"bytes":15631},"home":{"status":"ok","http":200,"url":"https://www.nami.org/","html_tokens":103952,"markdown_tokens":414,"pruned_tokens":99}},
{"rank":2017,"domain":"daum.net","origin":"https://daum.net","robots":{"status":"ok","http":200,"url":"https://daum.net/robots.txt","bytes":167,"groups":2},"bots":{"GPTBot":["partial","*",1,3],"ChatGPT-User":["partial","*",1,3],"OAI-SearchBot":["partial","*",1,3],"ClaudeBot":["partial","*",1,3],"Claude-User":["partial","*",1,3],"Claude-SearchBot":["partial","*",1,3],"anthropic-ai":["partial","*",1,3],"PerplexityBot":["partial","*",1,3],"Perplexity-User":["partial","*",1,3],"Google-Extended":["partial","*",1,3],"Applebot-Extended":["partial","*",1,3],"CCBot":["partial","*",1,3],"Bytespider":["partial","*",1,3],"Amazonbot":["partial","*",1,3],"meta-externalagent":["partial","*",1,3],"cohere-ai":["partial","*",1,3],"Diffbot":["partial","*",1,3],"ContextFetchBot":["partial","*",1,3]},"llms":{"status":"disallowed"},"home":{"status":"ok","http":200,"url":"https://www.daum.net/","html_tokens":142319,"markdown_tokens":21,"pruned_tokens":21}},
{"rank":2018,"domain":"lexisnexis.com","origin":"https://lexisnexis.com","robots":{"status":"ok","http":200,"url":"https://lexisnexis.com/robots.txt","bytes":242,"groups":1},"bots":{"GPTBot":["blocked","*",1,0],"ChatGPT-User":["blocked","*",1,0],"OAI-SearchBot":["blocked","*",1,0],"ClaudeBot":["blocked","*",1,0],"Claude-User":["blocked","*",1,0],"Claude-SearchBot":["blocked","*",1,0],"anthropic-ai":["blocked","*",1,0],"PerplexityBot":["blocked","*",1,0],"Perplexity-User":["blocked","*",1,0],"Google-Extended":["blocked","*",1,0],"Applebot-Extended":["blocked","*",1,0],"CCBot":["blocked","*",1,0],"Bytespider":["blocked","*",1,0],"Amazonbot":["blocked","*",1,0],"meta-externalagent":["blocked","*",1,0],"cohere-ai":["blocked","*",1,0],"Diffbot":["blocked","*",1,0],"ContextFetchBot":["blocked","*",1,0]},"llms":{"status":"disallowed"},"home":{"status":"disallowed"}},
{"rank":2019,"domain":"wolterskluwer.com","origin":"https://wolterskluwer.com","robots":{"status":"refused","http":403,"url":"https://www.wolterskluwer.com/robots.txt","bytes":5538,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":2020,"domain":"icrc.org","origin":"https://icrc.org","robots":{"status":"ok","http":200,"url":"https://www.icrc.org/robots.txt","bytes":2216,"groups":1},"bots":{"GPTBot":["partial","*",36,19],"ChatGPT-User":["partial","*",36,19],"OAI-SearchBot":["partial","*",36,19],"ClaudeBot":["partial","*",36,19],"Claude-User":["partial","*",36,19],"Claude-SearchBot":["partial","*",36,19],"anthropic-ai":["partial","*",36,19],"PerplexityBot":["partial","*",36,19],"Perplexity-User":["partial","*",36,19],"Google-Extended":["partial","*",36,19],"Applebot-Extended":["partial","*",36,19],"CCBot":["partial","*",36,19],"Bytespider":["partial","*",36,19],"Amazonbot":["partial","*",36,19],"meta-externalagent":["partial","*",36,19],"cohere-ai":["partial","*",36,19],"Diffbot":["partial","*",36,19],"ContextFetchBot":["partial","*",36,19]},"llms":{"status":"http_error","http":403,"bytes":146},"home":{"status":"ok","http":200,"url":"https://www.icrc.org/en","html_tokens":24753,"markdown_tokens":90,"pruned_tokens":90}},
{"rank":2021,"domain":"nationalgeographic.org","origin":"https://nationalgeographic.org","robots":{"status":"ok","http":200,"url":"https://www.nationalgeographic.org/robots.txt","bytes":2940,"groups":75},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["allowed","named",0,1],"OAI-SearchBot":["allowed","named",0,1],"ClaudeBot":["blocked","named",1,0],"Claude-User":["allowed","named",0,1],"Claude-SearchBot":["blocked","named",1,0],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["allowed","named",0,1],"Perplexity-User":["allowed","named",0,1],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["allowed","named",0,1],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":30065},"home":{"status":"ok","http":200,"url":"https://www.nationalgeographic.org/","html_tokens":78469,"markdown_tokens":244,"pruned_tokens":102}},
{"rank":2022,"domain":"lycos.com","origin":"https://lycos.com","robots":{"status":"missing","http":404,"url":"https://www.lycos.com/robots.txt","bytes":27477,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":27475},"home":{"status":"ok","http":200,"url":"https://www.lycos.com/","html_tokens":29257,"markdown_tokens":3787,"pruned_tokens":154}},
{"rank":2023,"domain":"obsproject.com","origin":"https://obsproject.com","robots":{"status":"ok","http":200,"url":"https://obsproject.com/robots.txt","bytes":712,"groups":2},"bots":{"GPTBot":["partial","*",31,0],"ChatGPT-User":["partial","*",31,0],"OAI-SearchBot":["partial","*",31,0],"ClaudeBot":["partial","*",31,0],"Claude-User":["partial","*",31,0],"Claude-SearchBot":["partial","*",31,0],"anthropic-ai":["partial","*",31,0],"PerplexityBot":["partial","*",31,0],"Perplexity-User":["partial","*",31,0],"Google-Extended":["partial","*",31,0],"Applebot-Extended":["partial","*",31,0],"CCBot":["partial","*",31,0],"Bytespider":["partial","*",31,0],"Amazonbot":["partial","*",31,0],"meta-externalagent":["partial","*",31,0],"cohere-ai":["partial","*",31,0],"Diffbot":["partial","*",31,0],"ContextFetchBot":["partial","*",31,0]},"llms":{"status":"missing","http":404,"bytes":10840},"home":{"status":"ok","http":200,"url":"https://obsproject.com/","html_tokens":5702,"markdown_tokens":545,"pruned_tokens":147}},
{"rank":2024,"domain":"vulture.com","origin":"https://vulture.com","robots":{"status":"ok","http":200,"url":"https://vulture.com/robots.txt","bytes":7536,"groups":108},"bots":{"GPTBot":["allowed","named",0,1],"ChatGPT-User":["partial","named",1,2],"OAI-SearchBot":["partial","*",6,0],"ClaudeBot":["partial","named",1,2],"Claude-User":["partial","named",1,2],"Claude-SearchBot":["partial","named",1,2],"anthropic-ai":["partial","named",1,2],"PerplexityBot":["partial","named",1,2],"Perplexity-User":["partial","named",1,2],"Google-Extended":["partial","named",1,2],"Applebot-Extended":["partial","named",1,2],"CCBot":["partial","named",1,2],"Bytespider":["partial","named",1,2],"Amazonbot":["allowed","named",0,1],"meta-externalagent":["partial","named",1,2],"cohere-ai":["partial","named",1,2],"Diffbot":["partial","named",1,2],"ContextFetchBot":["partial","*",6,0]},"llms":{"status":"missing","http":404,"bytes":47002},"home":{"status":"ok","http":200,"url":"https://www.vulture.com/","html_tokens":198338,"markdown_tokens":260,"pruned_tokens":166}},
{"rank":2025,"domain":"news24.com","origin":"https://news24.com","robots":{"status":"ok","http":200,"url":"https://www.news24.com/robots.txt","bytes":2701,"groups":63},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["blocked","named",1,0],"OAI-SearchBot":["blocked","named",1,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["blocked","named",1,0],"Claude-SearchBot":["blocked","named",1,0],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["blocked","named",1,0],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",2,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",9,3]},"llms":{"status":"missing","http":404,"bytes":0},"home":{"status":"http_error","http":403,"url":"https://news24.com/"}},
{"rank":2026,"domain":"react.dev","origin":"https://react.dev","robots":{"status":"missing","http":404,"url":"https://react.dev/robots.txt","bytes":9,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"present","http":200,"bytes":14476},"home":{"status":"http_error","http":404,"url":"https://react.dev/"}},
{"rank":2027,"domain":"messefrankfurt.com","origin":"https://messefrankfurt.com","robots":{"status":"ok","http":200,"url":"https://www.messefrankfurt.com/robots.txt","bytes":174,"groups":1},"bots":{"GPTBot":["partial","*",4,0],"ChatGPT-User":["partial","*",4,0],"OAI-SearchBot":["partial","*",4,0],"ClaudeBot":["partial","*",4,0],"Claude-User":["partial","*",4,0],"Claude-SearchBot":["partial","*",4,0],"anthropic-ai":["partial","*",4,0],"PerplexityBot":["partial","*",4,0],"Perplexity-User":["partial","*",4,0],"Google-Extended":["partial","*",4,0],"Applebot-Extended":["partial","*",4,0],"CCBot":["partial","*",4,0],"Bytespider":["partial","*",4,0],"Amazonbot":["partial","*",4,0],"meta-externalagent":["partial","*",4,0],"cohere-ai":["partial","*",4,0],"Diffbot":["partial","*",4,0],"ContextFetchBot":["partial","*",4,0]},"llms":{"status":"missing","http":404,"bytes":122345},"home":{"status":"ok","http":200,"url":"https://www.messefrankfurt.com/frankfurt/en.html","html_tokens":42030,"markdown_tokens":1010,"pruned_tokens":96}},
{"rank":2028,"domain":"freeprivacypolicy.com","origin":"https://freeprivacypolicy.com","robots":{"status":"ok","http":200,"url":"https://www.freeprivacypolicy.com/robots.txt","bytes":2134,"groups":1},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"http_error","http":403,"bytes":5560},"home":{"status":"http_error","http":403,"url":"https://freeprivacypolicy.com/"}},
{"rank":2029,"domain":"tawk.to","origin":"https://tawk.to","robots":{"status":"missing","http":404,"url":"https://tawk.to/robots.txt","bytes":1197,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":1197},"home":{"status":"ok","http":200,"url":"https://www.tawk.to/","html_tokens":152357,"markdown_tokens":776,"pruned_tokens":31}},
{"rank":2030,"domain":"sandiegouniontribune.com","origin":"https://sandiegouniontribune.com","robots":{"status":"ok","http":200,"url":"https://www.sandiegouniontribune.com/robots.txt","bytes":21147,"groups":229},"bots":{"GPTBot":["blocked","named",2,0],"ChatGPT-User":["blocked","named",2,0],"OAI-SearchBot":["blocked","named",2,0],"ClaudeBot":["blocked","named",2,0],"Claude-User":["blocked","named",2,0],"Claude-SearchBot":["blocked","named",2,0],"anthropic-ai":["blocked","named",2,0],"PerplexityBot":["blocked","named",2,0],"Perplexity-User":["blocked","named",2,0],"Google-Extended":["blocked","named",2,0],"Applebot-Extended":["blocked","named",2,0],"CCBot":["blocked","named",2,0],"Bytespider":["blocked","named",2,0],"Amazonbot":["blocked","named",2,0],"meta-externalagent":["blocked","named",2,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",2,0],"ContextFetchBot":["partial","*",8,1]},"llms":{"status":"missing","http":404,"bytes":96173},"home":{"status":"ok","http":200,"url":"https://www.sandiegouniontribune.com/","html_tokens":68037,"markdown_tokens":327,"pruned_tokens":167}},
{"rank":2031,"domain":"oclc.org","origin":"https://oclc.org","robots":{"status":"ok","http":200,"url":"https://www.oclc.org/robots.txt","bytes":5317,"groups":1},"bots":{"GPTBot":["partial","*",10,0],"ChatGPT-User":["partial","*",10,0],"OAI-SearchBot":["partial","*",10,0],"ClaudeBot":["partial","*",10,0],"Claude-User":["partial","*",10,0],"Claude-SearchBot":["partial","*",10,0],"anthropic-ai":["partial","*",10,0],"PerplexityBot":["partial","*",10,0],"Perplexity-User":["partial","*",10,0],"Google-Extended":["partial","*",10,0],"Applebot-Extended":["partial","*",10,0],"CCBot":["partial","*",10,0],"Bytespider":["partial","*",10,0],"Amazonbot":["partial","*",10,0],"meta-externalagent":["partial","*",10,0],"cohere-ai":["partial","*",10,0],"Diffbot":["partial","*",10,0],"ContextFetchBot":["partial","*",10,0]},"llms":{"status":"missing","http":404,"bytes":20555},"home":{"status":"ok","http":200,"url":"https://www.oclc.org/","html_tokens":101,"markdown_tokens":0,"pruned_tokens":0}},
{"rank":2032,"domain":"andersnoren.se","origin":"https://andersnoren.se","robots":{"status":"ok","http":200,"url":"https://andersnoren.se/robots.txt","bytes":172,"groups":1},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"missing","http":404,"bytes":70591},"home":{"status":"ok","http":200,"url":"https://andersnoren.se/","html_tokens":32294,"markdown_tokens":249,"pruned_tokens":134}},
{"rank":2033,"domain":"csic.es","origin":"https://csic.es","robots":{"status":"ok","http":200,"url":"https://www.csic.es/robots.txt","bytes":2051,"groups":1},"bots":{"GPTBot":["partial","*",32,18],"ChatGPT-User":["partial","*",32,18],"OAI-SearchBot":["partial","*",32,18],"ClaudeBot":["partial","*",32,18],"Claude-User":["partial","*",32,18],"Claude-SearchBot":["partial","*",32,18],"anthropic-ai":["partial","*",32,18],"PerplexityBot":["partial","*",32,18],"Perplexity-User":["partial","*",32,18],"Google-Extended":["partial","*",32,18],"Applebot-Extended":["partial","*",32,18],"CCBot":["partial","*",32,18],"Bytespider":["partial","*",32,18],"Amazonbot":["partial","*",32,18],"meta-externalagent":["partial","*",32,18],"cohere-ai":["partial","*",32,18],"Diffbot":["partial","*",32,18],"ContextFetchBot":["partial","*",32,18]},"llms":{"status":"missing","http":404,"bytes":184},"home":{"status":"ok","http":200,"url":"https://www.csic.es/es","html_tokens":48166,"markdown_tokens":365,"pruned_tokens":59}},
{"rank":2034,"domain":"esri.com","origin":"https://esri.com","robots":{"status":"ok","http":200,"url":"https://www.esri.com/robots.txt","bytes":3350,"groups":1},"bots":{"GPTBot":["partial","*",63,9],"ChatGPT-User":["partial","*",63,9],"OAI-SearchBot":["partial","*",63,9],"ClaudeBot":["partial","*",63,9],"Claude-User":["partial","*",63,9],"Claude-SearchBot":["partial","*",63,9],"anthropic-ai":["partial","*",63,9],"PerplexityBot":["partial","*",63,9],"Perplexity-User":["partial","*",63,9],"Google-Extended":["partial","*",63,9],"Applebot-Extended":["partial","*",63,9],"CCBot":["partial","*",63,9],"Bytespider":["partial","*",63,9],"Amazonbot":["partial","*",63,9],"meta-externalagent":["partial","*",63,9],"cohere-ai":["partial","*",63,9],"Diffbot":["partial","*",63,9],"ContextFetchBot":["partial","*",63,9]},"llms":{"status":"present","http":200,"bytes":10583},"home":{"status":"ok","http":200,"url":"https://www.esri.com/en-us/home","html_tokens":16853,"markdown_tokens":310,"pruned_tokens":68}},
{"rank":2035,"domain":"seagate.com","origin":"https://seagate.com","robots":{"status":"ok","http":200,"url":"https://www.seagate.com/robots.txt","bytes":4953,"groups":5},"bots":{"GPTBot":["partial","*",76,28],"ChatGPT-User":["partial","*",76,28],"OAI-SearchBot":["partial","*",76,28],"ClaudeBot":["partial","*",76,28],"Claude-User":["partial","*",76,28],"Claude-SearchBot":["partial","*",76,28],"anthropic-ai":["partial","*",76,28],"PerplexityBot":["partial","*",76,28],"Perplexity-User":["partial","*",76,28],"Google-Extended":["partial","*",76,28],"Applebot-Extended":["partial","*",76,28],"CCBot":["partial","*",76,28],"Bytespider":["partial","*",76,28],"Amazonbot":["partial","*",76,28],"meta-externalagent":["partial","*",76,28],"cohere-ai":["partial","*",76,28],"Diffbot":["partial","*",76,28],"ContextFetchBot":["partial","*",76,28]},"llms":{"status":"missing","http":404,"bytes":481525},"home":{"status":"ok","http":200,"url":"https://www.seagate.com/","html_tokens":132950,"markdown_tokens":460,"pruned_tokens":219}},
{"rank":2036,"domain":"rakuten.com","origin":"https://rakuten.com","robots":{"status":"ok","http":200,"url":"https://www.rakuten.com/robots.txt","bytes":2621,"groups":4},"bots":{"GPTBot":["partial","*",95,0],"ChatGPT-User":["partial","*",95,0],"OAI-SearchBot":["partial","*",95,0],"ClaudeBot":["partial","*",95,0],"Claude-User":["partial","*",95,0],"Claude-SearchBot":["partial","*",95,0],"anthropic-ai":["partial","*",95,0],"PerplexityBot":["partial","*",95,0],"Perplexity-User":["partial","*",95,0],"Google-Extended":["partial","*",95,0],"Applebot-Extended":["partial","*",95,0],"CCBot":["partial","*",95,0],"Bytespider":["partial","*",95,0],"Amazonbot":["partial","*",95,0],"meta-externalagent":["partial","*",95,0],"cohere-ai":["partial","*",95,0],"Diffbot":["partial","*",95,0],"ContextFetchBot":["partial","*",95,0]},"llms":{"status":"missing","http":404,"bytes":0},"home":{"status":"ok","http":200,"url":"https://www.rakuten.com/","html_tokens":489490,"markdown_tokens":336,"pruned_tokens":104}},
{"rank":2037,"domain":"mimecast.com","origin":"https://mimecast.com","robots":{"status":"ok","http":200,"url":"https://www.mimecast.com/robots.txt","bytes":70,"groups":1},"bots":{"GPTBot":["allowed","*",0,1],"ChatGPT-User":["allowed","*",0,1],"OAI-SearchBot":["allowed","*",0,1],"ClaudeBot":["allowed","*",0,1],"Claude-User":["allowed","*",0,1],"Claude-SearchBot":["allowed","*",0,1],"anthropic-ai":["allowed","*",0,1],"PerplexityBot":["allowed","*",0,1],"Perplexity-User":["allowed","*",0,1],"Google-Extended":["allowed","*",0,1],"Applebot-Extended":["allowed","*",0,1],"CCBot":["allowed","*",0,1],"Bytespider":["allowed","*",0,1],"Amazonbot":["allowed","*",0,1],"meta-externalagent":["allowed","*",0,1],"cohere-ai":["allowed","*",0,1],"Diffbot":["allowed","*",0,1],"ContextFetchBot":["allowed","*",0,1]},"llms":{"status":"missing","http":404,"bytes":71810},"home":{"status":"ok","http":200,"url":"https://www.mimecast.com/","html_tokens":35971,"markdown_tokens":417,"pruned_tokens":58}},
{"rank":2038,"domain":"statnews.com","origin":"https://statnews.com","robots":{"status":"ok","http":200,"url":"https://www.statnews.com/robots.txt","bytes":1338,"groups":26},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["partial","*",5,2],"OAI-SearchBot":["partial","*",5,2],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",5,2],"Claude-SearchBot":["partial","*",5,2],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["partial","*",5,2],"Perplexity-User":["partial","*",5,2],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",5,2]},"llms":{"status":"missing","http":404,"bytes":64765},"home":{"status":"ok","http":200,"url":"https://www.statnews.com/","html_tokens":87295,"markdown_tokens":61,"pruned_tokens":61}},
{"rank":2039,"domain":"speakerdeck.com","origin":"https://speakerdeck.com","robots":{"status":"ok","http":200,"url":"https://speakerdeck.com/robots.txt","bytes":665,"groups":1},"bots":{"GPTBot":["partial","*",7,20],"ChatGPT-User":["partial","*",7,20],"OAI-SearchBot":["partial","*",7,20],"ClaudeBot":["partial","*",7,20],"Claude-User":["partial","*",7,20],"Claude-SearchBot":["partial","*",7,20],"anthropic-ai":["partial","*",7,20],"PerplexityBot":["partial","*",7,20],"Perplexity-User":["partial","*",7,20],"Google-Extended":["partial","*",7,20],"Applebot-Extended":["partial","*",7,20],"CCBot":["partial","*",7,20],"Bytespider":["partial","*",7,20],"Amazonbot":["partial","*",7,20],"meta-externalagent":["partial","*",7,20],"cohere-ai":["partial","*",7,20],"Diffbot":["partial","*",7,20],"ContextFetchBot":["partial","*",7,20]},"llms":{"status":"missing","http":404,"bytes":0},"home":{"status":"ok","http":200,"url":"https://speakerdeck.com/","html_tokens":13673,"markdown_tokens":229,"pruned_tokens":44}},
{"rank":2040,"domain":"wbs-law.de","origin":"https://wbs-law.de","robots":{"status":"ok","http":200,"url":"https://www.wbs.legal/robots.txt","bytes":247,"groups":1},"bots":{"GPTBot":["partial","*",10,0],"ChatGPT-User":["partial","*",10,0],"OAI-SearchBot":["partial","*",10,0],"ClaudeBot":["partial","*",10,0],"Claude-User":["partial","*",10,0],"Claude-SearchBot":["partial","*",10,0],"anthropic-ai":["partial","*",10,0],"PerplexityBot":["partial","*",10,0],"Perplexity-User":["partial","*",10,0],"Google-Extended":["partial","*",10,0],"Applebot-Extended":["partial","*",10,0],"CCBot":["partial","*",10,0],"Bytespider":["partial","*",10,0],"Amazonbot":["partial","*",10,0],"meta-externalagent":["partial","*",10,0],"cohere-ai":["partial","*",10,0],"Diffbot":["partial","*",10,0],"ContextFetchBot":["partial","*",10,0]},"llms":{"status":"missing","http":404,"bytes":90508},"home":{"status":"ok","http":200,"url":"https://www.wbs.legal/","html_tokens":68658,"markdown_tokens":642,"pruned_tokens":101}},
{"rank":2041,"domain":"ab.ca","origin":"https://ab.ca","robots":{"status":"unreachable","http":null,"url":"https://ab.ca/robots.txt","bytes":0,"groups":0,"error":"cannot resolve ab.ca"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":2042,"domain":"energystar.gov","origin":"https://energystar.gov","robots":{"status":"ok","http":200,"url":"https://www.energystar.gov/robots.txt","bytes":2433,"groups":3},"bots":{"GPTBot":["partial","*",37,18],"ChatGPT-User":["partial","*",37,18],"OAI-SearchBot":["partial","*",37,18],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",37,18],"Claude-SearchBot":["partial","*",37,18],"anthropic-ai":["partial","*",37,18],"PerplexityBot":["partial","*",37,18],"Perplexity-User":["partial","*",37,18],"Google-Extended":["partial","*",37,18],"Applebot-Extended":["partial","*",37,18],"CCBot":["partial","*",37,18],"Bytespider":["partial","*",37,18],"Amazonbot":["partial","*",37,18],"meta-externalagent":["partial","*",37,18],"cohere-ai":["partial","*",37,18],"Diffbot":["partial","*",37,18],"ContextFetchBot":["partial","*",37,18]},"llms":{"status":"http_error","http":403,"bytes":146},"home":{"status":"ok","http":200,"url":"https://www.energystar.gov/","html_tokens":11585,"markdown_tokens":126,"pruned_tokens":115}},
{"rank":2043,"domain":"vkvideo.ru","origin":"https://vkvideo.ru","robots":{"status":"ok","http":200,"url":"https://vkvideo.ru/robots.txt","bytes":776,"groups":2},"bots":{"GPTBot":["partial","*",13,3],"ChatGPT-User":["partial","*",13,3],"OAI-SearchBot":["partial","*",13,3],"ClaudeBot":["partial","*",13,3],"Claude-User":["partial","*",13,3],"Claude-SearchBot":["partial","*",13,3],"anthropic-ai":["partial","*",13,3],"PerplexityBot":["partial","*",13,3],"Perplexity-User":["partial","*",13,3],"Google-Extended":["partial","*",13,3],"Applebot-Extended":["partial","*",13,3],"CCBot":["partial","*",13,3],"Bytespider":["partial","*",13,3],"Amazonbot":["partial","*",13,3],"meta-externalagent":["partial","*",13,3],"cohere-ai":["partial","*",13,3],"Diffbot":["partial","*",13,3],"ContextFetchBot":["partial","*",13,3]},"llms":{"status":"disallowed","error":"robots.txt of login.vk.ru disallows /?act=autologin&redirect_uri=https://vkvideo.ru&state=eyJzdGF0ZSI6IjRiOTVkYWFlLWI0OGYtNDllYS05NWJkLT"},"home":{"status":"disallowed","error":"robots.txt of login.vk.ru disallows /?act=autologin&redirect_uri=https://vkvideo.ru&state=eyJzdGF0ZSI6IjEzMGUzYjA0LWM4OTQtNDg4MC05YzJkLT"}},
{"rank":2044,"domain":"globaltimes.cn","origin":"https://globaltimes.cn","robots":{"status":"ok","http":200,"url":"https://www.globaltimes.cn/robots.txt","bytes":62,"groups":1},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"missing","http":404,"bytes":206},"home":{"status":"ok","http":200,"url":"https://www.globaltimes.cn/","html_tokens":5267,"markdown_tokens":190,"pruned_tokens":162}},
{"rank":2045,"domain":"kuleuven.be","origin":"https://kuleuven.be","robots":{"status":"ok","http":200,"url":"https://www.kuleuven.be/robots.txt","bytes":2014,"groups":1},"bots":{"GPTBot":["partial","*",39,0],"ChatGPT-User":["partial","*",39,0],"OAI-SearchBot":["partial","*",39,0],"ClaudeBot":["partial","*",39,0],"Claude-User":["partial","*",39,0],"Claude-SearchBot":["partial","*",39,0],"anthropic-ai":["partial","*",39,0],"PerplexityBot":["partial","*",39,0],"Perplexity-User":["partial","*",39,0],"Google-Extended":["partial","*",39,0],"Applebot-Extended":["partial","*",39,0],"CCBot":["partial","*",39,0],"Bytespider":["partial","*",39,0],"Amazonbot":["partial","*",39,0],"meta-externalagent":["partial","*",39,0],"cohere-ai":["partial","*",39,0],"Diffbot":["partial","*",39,0],"ContextFetchBot":["partial","*",39,0]},"llms":{"status":"missing","http":404,"bytes":26},"home":{"status":"ok","http":200,"url":"https://www.kuleuven.be/kuleuven/","html_tokens":9085,"markdown_tokens":308,"pruned_tokens":114}},
{"rank":2046,"domain":"japanpost.jp","origin":"https://www.japanpost.jp","robots":{"status":"ok","http":200,"url":"https://www.japanpost.jp/robots.txt","bytes":515,"groups":1},"bots":{"GPTBot":["partial","*",8,1],"ChatGPT-User":["partial","*",8,1],"OAI-SearchBot":["partial","*",8,1],"ClaudeBot":["partial","*",8,1],"Claude-User":["partial","*",8,1],"Claude-SearchBot":["partial","*",8,1],"anthropic-ai":["partial","*",8,1],"PerplexityBot":["partial","*",8,1],"Perplexity-User":["partial","*",8,1],"Google-Extended":["partial","*",8,1],"Applebot-Extended":["partial","*",8,1],"CCBot":["partial","*",8,1],"Bytespider":["partial","*",8,1],"Amazonbot":["partial","*",8,1],"meta-externalagent":["partial","*",8,1],"cohere-ai":["partial","*",8,1],"Diffbot":["partial","*",8,1],"ContextFetchBot":["partial","*",8,1]},"llms":{"status":"missing","http":404,"bytes":229911},"home":{"status":"ok","http":200,"url":"https://www.japanpost.jp/","html_tokens":54665,"markdown_tokens":440,"pruned_tokens":175}},
{"rank":2047,"domain":"custhelp.com","origin":"https://custhelp.com","robots":{"status":"timeout","http":null,"url":"https://custhelp.com/robots.txt","bytes":0,"groups":0,"error":"ConnectTimeout"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":2048,"domain":"ifixit.com","origin":"https://ifixit.com","robots":{"status":"ok","http":200,"url":"https://www.ifixit.com/robots.txt","bytes":17319,"groups":29},"bots":{"GPTBot":["partial","named",83,6],"ChatGPT-User":["partial","*",83,6],"OAI-SearchBot":["partial","*",83,6],"ClaudeBot":["partial","named",83,6],"Claude-User":["partial","*",83,6],"Claude-SearchBot":["partial","*",83,6],"anthropic-ai":["partial","named",83,6],"PerplexityBot":["partial","*",83,6],"Perplexity-User":["partial","*",83,6],"Google-Extended":["partial","named",83,6],"Applebot-Extended":["blocked","named",1,0],"CCBot":["partial","named",83,6],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",83,6]},"llms":{"status":"present","http":200,"bytes":5885},"home":{"status":"ok","http":200,"url":"https://www.ifixit.com/","html_tokens":41981,"markdown_tokens":264,"pruned_tokens":182}},
{"rank":2049,"domain":"gob.ec","origin":"https://www.gob.ec","robots":{"status":"ok","http":200,"url":"https://www.gob.ec/robots.txt","bytes":2005,"groups":1},"bots":{"GPTBot":["partial","*",33,18],"ChatGPT-User":["partial","*",33,18],"OAI-SearchBot":["partial","*",33,18],"ClaudeBot":["partial","*",33,18],"Claude-User":["partial","*",33,18],"Claude-SearchBot":["partial","*",33,18],"anthropic-ai":["partial","*",33,18],"PerplexityBot":["partial","*",33,18],"Perplexity-User":["partial","*",33,18],"Google-Extended":["partial","*",33,18],"Applebot-Extended":["partial","*",33,18],"CCBot":["partial","*",33,18],"Bytespider":["partial","*",33,18],"Amazonbot":["partial","*",33,18],"meta-externalagent":["partial","*",33,18],"cohere-ai":["partial","*",33,18],"Diffbot":["partial","*",33,18],"ContextFetchBot":["partial","*",33,18]},"llms":{"status":"missing","http":404,"bytes":153},"home":{"status":"ok","http":200,"url":"https://www.gob.ec/","html_tokens":7276,"markdown_tokens":12,"pruned_tokens":12}},
{"rank":2050,"domain":"sa.com","origin":"https://sa.com","robots":{"status":"unreachable","http":null,"url":"https://sa.com/robots.txt","bytes":0,"groups":0,"error":"cannot resolve sa.com"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":2051,"domain":"ubs.com","origin":"https://ubs.com","robots":{"status":"ok","http":200,"url":"https://www.ubs.com/robots.txt","bytes":1293,"groups":1},"bots":{"GPTBot":["partial","*",47,3],"ChatGPT-User":["partial","*",47,3],"OAI-SearchBot":["partial","*",47,3],"ClaudeBot":["partial","*",47,3],"Claude-User":["partial","*",47,3],"Claude-SearchBot":["partial","*",47,3],"anthropic-ai":["partial","*",47,3],"PerplexityBot":["partial","*",47,3],"Perplexity-User":["partial","*",47,3],"Google-Extended":["partial","*",47,3],"Applebot-Extended":["partial","*",47,3],"CCBot":["partial","*",47,3],"Bytespider":["partial","*",47,3],"Amazonbot":["partial","*",47,3],"meta-externalagent":["partial","*",47,3],"cohere-ai":["partial","*",47,3],"Diffbot":["partial","*",47,3],"ContextFetchBot":["partial","*",47,3]},"llms":{"status":"present","http":200,"bytes":99053},"home":{"status":"ok","http":200,"url":"https://www.ubs.com/us/en.html","html_tokens":35383,"markdown_tokens":294,"pruned_tokens":168}},
{"rank":2052,"domain":"bigcommerce.com","origin":"https://bigcommerce.com","robots":{"status":"ok","http":200,"url":"https://www.bigcommerce.com/robots.txt","bytes":2257,"groups":2},"bots":{"GPTBot":["partial","*",49,0],"ChatGPT-User":["partial","*",49,0],"OAI-SearchBot":["partial","*",49,0],"ClaudeBot":["partial","*",49,0],"Claude-User":["partial","*",49,0],"Claude-SearchBot":["partial","*",49,0],"anthropic-ai":["partial","*",49,0],"PerplexityBot":["partial","*",49,0],"Perplexity-User":["partial","*",49,0],"Google-Extended":["partial","*",49,0],"Applebot-Extended":["partial","*",49,0],"CCBot":["partial","*",49,0],"Bytespider":["partial","*",49,0],"Amazonbot":["partial","*",49,0],"meta-externalagent":["partial","*",49,0],"cohere-ai":["partial","*",49,0],"Diffbot":["partial","*",49,0],"ContextFetchBot":["partial","*",49,0]},"llms":{"status":"present","http":200,"bytes":3715},"home":{"status":"ok","http":200,"url":"https://www.bigcommerce.com/","html_tokens":332327,"markdown_tokens":1079,"pruned_tokens":95}},
{"rank":2053,"domain":"archive.ph","origin":"https://archive.ph","robots":{"status":"ok","http":200,"url":"https://archive.ph/robots.txt","bytes":439,"groups":1},"bots":{"GPTBot":["partial","*",14,2],"ChatGPT-User":["partial","*",14,2],"OAI-SearchBot":["partial","*",14,2],"ClaudeBot":["partial","*",14,2],"Claude-User":["partial","*",14,2],"Claude-SearchBot":["partial","*",14,2],"anthropic-ai":["partial","*",14,2],"PerplexityBot":["partial","*",14,2],"Perplexity-User":["partial","*",14,2],"Google-Extended":["partial","*",14,2],"Applebot-Extended":["partial","*",14,2],"CCBot":["partial","*",14,2],"Bytespider":["partial","*",14,2],"Amazonbot":["partial","*",14,2],"meta-externalagent":["partial","*",14,2],"cohere-ai":["partial","*",14,2],"Diffbot":["partial","*",14,2],"ContextFetchBot":["partial","*",14,2]},"llms":{"status":"missing","http":404,"bytes":13},"home":{"status":"ok","http":200,"url":"https://archive.ph/","html_tokens":3546,"markdown_tokens":154,"pruned_tokens":38}},
{"rank":2054,"domain":"lufthansa.com","origin":"https://lufthansa.com","robots":{"status":"ok","http":200,"url":"https://www.lufthansa.com/robots.txt","bytes":242,"groups":1},"bots":{"GPTBot":["partial","*",2,1],"ChatGPT-User":["partial","*",2,1],"OAI-SearchBot":["partial","*",2,1],"ClaudeBot":["partial","*",2,1],"Claude-User":["partial","*",2,1],"Claude-SearchBot":["partial","*",2,1],"anthropic-ai":["partial","*",2,1],"PerplexityBot":["partial","*",2,1],"Perplexity-User":["partial","*",2,1],"Google-Extended":["partial","*",2,1],"Applebot-Extended":["partial","*",2,1],"CCBot":["partial","*",2,1],"Bytespider":["partial","*",2,1],"Amazonbot":["partial","*",2,1],"meta-externalagent":["partial","*",2,1],"cohere-ai":["partial","*",2,1],"Diffbot":["partial","*",2,1],"ContextFetchBot":["partial","*",2,1]},"llms":{"status":"present","http":200,"bytes":3915},"home":{"status":"http_error","http":403,"url":"https://www.lufthansa.com/"}},
{"rank":2055,"domain":"go.ke","origin":"https://go.ke","robots":{"status":"unreachable","http":null,"url":"https://go.ke/robots.txt","bytes":0,"groups":0,"error":"cannot resolve go.ke"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":2056,"domain":"appleinsider.com","origin":"https://appleinsider.com","robots":{"status":"ok","http":200,"url":"https://appleinsider.com/robots.txt","bytes":154,"groups":1},"bots":{"GPTBot":["partial","*",4,0],"ChatGPT-User":["partial","*",4,0],"OAI-SearchBot":["partial","*",4,0],"ClaudeBot":["partial","*",4,0],"Claude-User":["partial","*",4,0],"Claude-SearchBot":["partial","*",4,0],"anthropic-ai":["partial","*",4,0],"PerplexityBot":["partial","*",4,0],"Perplexity-User":["partial","*",4,0],"Google-Extended":["partial","*",4,0],"Applebot-Extended":["partial","*",4,0],"CCBot":["partial","*",4,0],"Bytespider":["partial","*",4,0],"Amazonbot":["partial","*",4,0],"meta-externalagent":["partial","*",4,0],"cohere-ai":["partial","*",4,0],"Diffbot":["partial","*",4,0],"ContextFetchBot":["partial","*",4,0]},"llms":{"status":"missing","http":404,"bytes":81717},"home":{"status":"http_error","http":403,"url":"https://appleinsider.com/"}},
{"rank":2057,"domain":"dg-datenschutz.de","origin":"https://dg-datenschutz.de","robots":{"status":"ok","http":200,"url":"https://dg-datenschutz.de/robots.txt","bytes":2055,"groups":22},"bots":{"GPTBot":["partial","named",2,1],"ChatGPT-User":["allowed","named",0,1],"OAI-SearchBot":["allowed","named",0,1],"ClaudeBot":["partial","named",2,1],"Claude-User":["partial","*",9,1],"Claude-SearchBot":["partial","*",9,1],"anthropic-ai":["allowed","named",0,1],"PerplexityBot":["partial","named",2,1],"Perplexity-User":["allowed","named",0,1],"Google-Extended":["allowed","named",0,1],"Applebot-Extended":["allowed","named",0,1],"CCBot":["partial","named",2,1],"Bytespider":["partial","named",2,1],"Amazonbot":["allowed","named",0,1],"meta-externalagent":["allowed","named",0,1],"cohere-ai":["allowed","named",0,1],"Diffbot":["allowed","named",0,1],"ContextFetchBot":["partial","*",9,1]},"llms":{"status":"present","http":200,"bytes":2930},"home":{"status":"ok","http":200,"url":"https://dg-datenschutz.de/","html_tokens":25943,"markdown_tokens":1769,"pruned_tokens":711}},
{"rank":2058,"domain":"uni-hamburg.de","origin":"https://uni-hamburg.de","robots":{"status":"ok","http":200,"url":"https://www.uni-hamburg.de/robots.txt","bytes":228,"groups":1},"bots":{"GPTBot":["partial","*",6,0],"ChatGPT-User":["partial","*",6,0],"OAI-SearchBot":["partial","*",6,0],"ClaudeBot":["partial","*",6,0],"Claude-User":["partial","*",6,0],"Claude-SearchBot":["partial","*",6,0],"anthropic-ai":["partial","*",6,0],"PerplexityBot":["partial","*",6,0],"Perplexity-User":["partial","*",6,0],"Google-Extended":["partial","*",6,0],"Applebot-Extended":["partial","*",6,0],"CCBot":["partial","*",6,0],"Bytespider":["partial","*",6,0],"Amazonbot":["partial","*",6,0],"meta-externalagent":["partial","*",6,0],"cohere-ai":["partial","*",6,0],"Diffbot":["partial","*",6,0],"ContextFetchBot":["partial","*",6,0]},"llms":{"status":"missing","http":404,"bytes":34988},"home":{"status":"ok","http":200,"url":"https://www.uni-hamburg.de/","html_tokens":14214,"markdown_tokens":85,"pruned_tokens":85}},
{"rank":2059,"domain":"cnr.it","origin":"https://cnr.it","robots":{"status":"unreachable","http":null,"url":"https://cnr.it/robots.txt","bytes":0,"groups":0,"error":"[SSL: DH_KEY_TOO_SMALL] dh key too small (_ssl.c:1028)"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":2060,"domain":"exblog.jp","origin":"https://exblog.jp","robots":{"status":"missing","http":404,"url":"https://exblog.jp/robots.txt","bytes":0,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":0},"home":{"status":"not_html","error":"unsupported content type text/plain"}},
{"rank":2061,"domain":"worldpopulationreview.com","origin":"https://worldpopulationreview.com","robots":{"status":"ok","http":200,"url":"https://worldpopulationreview.com/robots.txt","bytes":86,"groups":1},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"missing","http":404,"bytes":72225},"home":{"status":"ok","http":200,"url":"https://worldpopulationreview.com/","html_tokens":315971,"markdown_tokens":5689,"pruned_tokens":157}},
{"rank":2062,"domain":"termly.io","origin":"https://termly.io","robots":{"status":"ok","http":200,"url":"https://termly.io/robots.txt","bytes":1067,"groups":19},"bots":{"GPTBot":["allowed","named",0,1],"ChatGPT-User":["allowed","named",0,1],"OAI-SearchBot":["allowed","named",0,1],"ClaudeBot":["allowed","named",0,1],"Claude-User":["allowed","named",0,1],"Claude-SearchBot":["partial","*",3,1],"anthropic-ai":["partial","*",3,1],"PerplexityBot":["allowed","named",0,1],"Perplexity-User":["allowed","named",0,1],"Google-Extended":["allowed","named",0,1],"Applebot-Extended":["allowed","named",0,1],"CCBot":["allowed","named",0,1],"Bytespider":["partial","*",3,1],"Amazonbot":["partial","*",3,1],"meta-externalagent":["partial","*",3,1],"cohere-ai":["partial","*",3,1],"Diffbot":["allowed","named",0,1],"ContextFetchBot":["partial","*",3,1]},"llms":{"status":"other_text","http":200,"bytes":13383},"home":{"status":"ok","http":200,"url":"https://termly.io/","html_tokens":79971,"markdown_tokens":962,"pruned_tokens":589}},
{"rank":2063,"domain":"unige.ch","origin":"https://unige.ch","robots":{"status":"ok","http":200,"url":"https://www.unige.ch/robots.txt","bytes":718,"groups":1},"bots":{"GPTBot":["partial","*",31,0],"ChatGPT-User":["partial","*",31,0],"OAI-SearchBot":["partial","*",31,0],"ClaudeBot":["partial","*",31,0],"Claude-User":["partial","*",31,0],"Claude-SearchBot":["partial","*",31,0],"anthropic-ai":["partial","*",31,0],"PerplexityBot":["partial","*",31,0],"Perplexity-User":["partial","*",31,0],"Google-Extended":["partial","*",31,0],"Applebot-Extended":["partial","*",31,0],"CCBot":["partial","*",31,0],"Bytespider":["partial","*",31,0],"Amazonbot":["partial","*",31,0],"meta-externalagent":["partial","*",31,0],"cohere-ai":["partial","*",31,0],"Diffbot":["partial","*",31,0],"ContextFetchBot":["partial","*",31,0]},"llms":{"status":"unreachable","error":"Server disconnected without sending a response."},"home":{"status":"ok","http":200,"url":"https://www.unige.ch/","html_tokens":18533,"markdown_tokens":187,"pruned_tokens":126}},
{"rank":2064,"domain":"wixstatic.com","origin":"https://wixstatic.com","robots":{"status":"unreachable","http":null,"url":"https://wixstatic.com/robots.txt","bytes":0,"groups":0,"error":"cannot resolve wixstatic.com"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":2065,"domain":"tokyo.jp","origin":"https://tokyo.jp","robots":{"status":"unreachable","http":null,"url":"https://tokyo.jp/robots.txt","bytes":0,"groups":0,"error":"cannot resolve tokyo.jp"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":2066,"domain":"northeastern.edu","origin":"https://northeastern.edu","robots":{"status":"ok","http":200,"url":"https://www.northeastern.edu/robots.txt","bytes":366,"groups":2},"bots":{"GPTBot":["partial","*",1,0],"ChatGPT-User":["partial","*",1,0],"OAI-SearchBot":["partial","*",1,0],"ClaudeBot":["partial","*",1,0],"Claude-User":["partial","*",1,0],"Claude-SearchBot":["partial","*",1,0],"anthropic-ai":["partial","*",1,0],"PerplexityBot":["partial","*",1,0],"Perplexity-User":["partial","*",1,0],"Google-Extended":["partial","*",1,0],"Applebot-Extended":["partial","*",1,0],"CCBot":["partial","*",1,0],"Bytespider":["partial","*",1,0],"Amazonbot":["partial","*",1,0],"meta-externalagent":["partial","*",1,0],"cohere-ai":["partial","*",1,0],"Diffbot":["partial","*",1,0],"ContextFetchBot":["partial","*",1,0]},"llms":{"status":"missing","http":404,"bytes":60},"home":{"status":"ok","http":200,"url":"https://www.northeastern.edu/","html_tokens":63174,"markdown_tokens":52,"pruned_tokens":52}},
{"rank":2067,"domain":"zotero.org","origin":"https://zotero.org","robots":{"status":"ok","http":200,"url":"https://www.zotero.org/robots.txt","bytes":120,"groups":1},"bots":{"GPTBot":["partial","*",5,0],"ChatGPT-User":["partial","*",5,0],"OAI-SearchBot":["partial","*",5,0],"ClaudeBot":["partial","*",5,0],"Claude-User":["partial","*",5,0],"Claude-SearchBot":["partial","*",5,0],"anthropic-ai":["partial","*",5,0],"PerplexityBot":["partial","*",5,0],"Perplexity-User":["partial","*",5,0],"Google-Extended":["partial","*",5,0],"Applebot-Extended":["partial","*",5,0],"CCBot":["partial","*",5,0],"Bytespider":["partial","*",5,0],"Amazonbot":["partial","*",5,0],"meta-externalagent":["partial","*",5,0],"cohere-ai":["partial","*",5,0],"Diffbot":["partial","*",5,0],"ContextFetchBot":["partial","*",5,0]},"llms":{"status":"missing","http":404,"bytes":8507},"home":{"status":"ok","http":200,"url":"https://www.zotero.org/","html_tokens":11257,"markdown_tokens":372,"pruned_tokens":175}},
{"rank":2068,"domain":"unibo.it","origin":"https://unibo.it","robots":{"status":"ok","http":200,"url":"https://www.unibo.it/robots.txt","bytes":613,"groups":1},"bots":{"GPTBot":["partial","*",14,0],"ChatGPT-User":["partial","*",14,0],"OAI-SearchBot":["partial","*",14,0],"ClaudeBot":["partial","*",14,0],"Claude-User":["partial","*",14,0],"Claude-SearchBot":["partial","*",14,0],"anthropic-ai":["partial","*",14,0],"PerplexityBot":["partial","*",14,0],"Perplexity-User":["partial","*",14,0],"Google-Extended":["partial","*",14,0],"Applebot-Extended":["partial","*",14,0],"CCBot":["partial","*",14,0],"Bytespider":["partial","*",14,0],"Amazonbot":["partial","*",14,0],"meta-externalagent":["partial","*",14,0],"cohere-ai":["partial","*",14,0],"Diffbot":["partial","*",14,0],"ContextFetchBot":["partial","*",14,0]},"llms":{"status":"missing","http":404,"bytes":27},"home":{"status":"ok","http":200,"url":"https://www.unibo.it/it","html_tokens":123720,"markdown_tokens":417,"pruned_tokens":149}},
{"rank":2069,"domain":"sydney.edu.au","origin":"https://sydney.edu.au","robots":{"status":"ok","http":200,"url":"https://www.sydney.edu.au/robots.txt","bytes":2137,"groups":2},"bots":{"GPTBot":["partial","*",34,4],"ChatGPT-User":["partial","*",34,4],"OAI-SearchBot":["partial","*",34,4],"ClaudeBot":["partial","*",34,4],"Claude-User":["partial","*",34,4],"Claude-SearchBot":["partial","*",34,4],"anthropic-ai":["partial","*",34,4],"PerplexityBot":["partial","*",34,4],"Perplexity-User":["partial","*",34,4],"Google-Extended":["partial","*",34,4],"Applebot-Extended":["partial","*",34,4],"CCBot":["partial","*",34,4],"Bytespider":["partial","*",34,4],"Amazonbot":["partial","*",34,4],"meta-externalagent":["partial","*",34,4],"cohere-ai":["partial","*",34,4],"Diffbot":["partial","*",34,4],"ContextFetchBot":["partial","*",34,4]},"llms":{"status":"html","http":200,"bytes":18537},"home":{"status":"ok","http":200,"url":"https://www.sydney.edu.au/","html_tokens":16972,"markdown_tokens":155,"pruned_tokens":80}},
{"rank":2070,"domain":"en-gb.facebook.com","origin":"https://en-gb.facebook.com","robots":{"status":"ok","http":200,"url":"https://en-gb.facebook.com/robots.txt","bytes":26872,"groups":36},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["blocked","*",1,0],"OAI-SearchBot":["blocked","*",1,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["blocked","*",1,0],"Claude-SearchBot":["blocked","*",1,0],"anthropic-ai":["blocked","*",1,0],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["blocked","*",1,0],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","*",1,0],"Bytespider":["blocked","*",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","*",1,0],"cohere-ai":["blocked","*",1,0],"Diffbot":["blocked","*",1,0],"ContextFetchBot":["blocked","*",1,0]},"llms":{"status":"disallowed"},"home":{"status":"disallowed"}},
{"rank":2071,"domain":"thehackernews.com","origin":"https://thehackernews.com","robots":{"status":"ok","http":200,"url":"https://thehackernews.com/robots.txt","bytes":356,"groups":1},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"present","http":200,"bytes":2721},"home":{"status":"ok","http":200,"url":"https://thehackernews.com/","html_tokens":48954,"markdown_tokens":38,"pruned_tokens":38}},
{"rank":2072,"domain":"biography.com","origin":"https://biography.com","robots":{"status":"ok","http":200,"url":"https://biography.com/robots.txt","bytes":963,"groups":3},"bots":{"GPTBot":["partial","named",32,0],"ChatGPT-User":["partial","named",32,0],"OAI-SearchBot":["partial","named",32,0],"ClaudeBot":["partial","*",19,0],"Claude-User":["partial","*",19,0],"Claude-SearchBot":["partial","*",19,0],"anthropic-ai":["partial","*",19,0],"PerplexityBot":["partial","*",19,0],"Perplexity-User":["partial","*",19,0],"Google-Extended":["partial","*",19,0],"Applebot-Extended":["partial","*",19,0],"CCBot":["partial","*",19,0],"Bytespider":["partial","*",19,0],"Amazonbot":["partial","*",19,0],"meta-externalagent":["partial","*",19,0],"cohere-ai":["partial","*",19,0],"Diffbot":["partial","*",19,0],"ContextFetchBot":["partial","*",19,0]},"llms":{"status":"missing","http":404,"bytes":84},"home":{"status":"ok","http":200,"url":"https://www.biography.com/","html_tokens":153902,"markdown_tokens":185,"pruned_tokens":145}},
{"rank":2073,"domain":"architecturaldigest.com","origin":"https://architecturaldigest.com","robots":{"status":"ok","http":200,"url":"https://www.architecturaldigest.com/robots.txt","bytes":2371,"groups":3},"bots":{"GPTBot":["partial","*",15,17],"ChatGPT-User":["partial","*",15,17],"OAI-SearchBot":["partial","*",15,17],"ClaudeBot":["blocked","named",1,0],"Claude-User":["blocked","named",1,0],"Claude-SearchBot":["blocked","named",1,0],"anthropic-ai":["partial","*",15,17],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["blocked","named",1,0],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",15,17]},"llms":{"status":"missing","http":404,"bytes":817278},"home":{"status":"ok","http":200,"url":"https://www.architecturaldigest.com/","html_tokens":629498,"markdown_tokens":78,"pruned_tokens":78}},
{"rank":2074,"domain":"screenrant.com","origin":"https://screenrant.com","robots":{"status":"ok","http":200,"url":"https://screenrant.com/robots.txt","bytes":2033,"groups":2},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["partial","*",22,0],"OAI-SearchBot":["partial","*",22,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",22,0],"Claude-SearchBot":["partial","*",22,0],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["partial","*",22,0],"Perplexity-User":["partial","*",22,0],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",22,0]},"llms":{"status":"missing","http":404,"bytes":108372},"home":{"status":"ok","http":200,"url":"https://screenrant.com/","html_tokens":231118,"markdown_tokens":580,"pruned_tokens":144}},
{"rank":2075,"domain":"dreamstime.com","origin":"https://dreamstime.com","robots":{"status":"ok","http":200,"url":"https://www.dreamstime.com/robots.txt","bytes":12584,"groups":18},"bots":{"GPTBot":["partial","*",276,0],"ChatGPT-User":["partial","*",276,0],"OAI-SearchBot":["partial","*",276,0],"ClaudeBot":["partial","*",276,0],"Claude-User":["partial","*",276,0],"Claude-SearchBot":["partial","*",276,0],"anthropic-ai":["partial","*",276,0],"PerplexityBot":["partial","*",276,0],"Perplexity-User":["partial","*",276,0],"Google-Extended":["partial","*",276,0],"Applebot-Extended":["partial","*",276,0],"CCBot":["partial","*",276,0],"Bytespider":["partial","*",276,0],"Amazonbot":["partial","*",276,0],"meta-externalagent":["partial","named",6,6],"cohere-ai":["partial","*",276,0],"Diffbot":["partial","*",276,0],"ContextFetchBot":["partial","*",276,0]},"llms":{"status":"missing","http":404,"bytes":63163},"home":{"status":"ok","http":200,"url":"https://www.dreamstime.com/","html_tokens":104054,"markdown_tokens":1255,"pruned_tokens":310}},
{"rank":2076,"domain":"ajc.com","origin":"https://ajc.com","robots":{"status":"ok","http":200,"url":"https://www.ajc.com/robots.txt","bytes":1598,"groups":16},"bots":{"GPTBot":["partial","*",5,0],"ChatGPT-User":["partial","*",5,0],"OAI-SearchBot":["partial","*",5,0],"ClaudeBot":["partial","*",5,0],"Claude-User":["partial","*",5,0],"Claude-SearchBot":["partial","*",5,0],"anthropic-ai":["partial","*",5,0],"PerplexityBot":["partial","*",5,0],"Perplexity-User":["partial","*",5,0],"Google-Extended":["partial","*",5,0],"Applebot-Extended":["partial","*",5,0],"CCBot":["partial","*",5,0],"Bytespider":["partial","*",5,0],"Amazonbot":["partial","*",5,0],"meta-externalagent":["partial","*",5,0],"cohere-ai":["partial","*",5,0],"Diffbot":["partial","*",5,0],"ContextFetchBot":["partial","*",5,0]},"llms":{"status":"missing","http":404,"bytes":124594},"home":{"status":"ok","http":200,"url":"https://www.ajc.com/","html_tokens":295417,"markdown_tokens":630,"pruned_tokens":101}},
{"rank":2077,"domain":"aad.org","origin":"https://aad.org","robots":{"status":"ok","http":200,"url":"https://www.aad.org/robots.txt","bytes":798,"groups":1},"bots":{"GPTBot":["partial","*",18,0],"ChatGPT-User":["partial","*",18,0],"OAI-SearchBot":["partial","*",18,0],"ClaudeBot":["partial","*",18,0],"Claude-User":["partial","*",18,0],"Claude-SearchBot":["partial","*",18,0],"anthropic-ai":["partial","*",18,0],"PerplexityBot":["partial","*",18,0],"Perplexity-User":["partial","*",18,0],"Google-Extended":["partial","*",18,0],"Applebot-Extended":["partial","*",18,0],"CCBot":["partial","*",18,0],"Bytespider":["partial","*",18,0],"Amazonbot":["partial","*",18,0],"meta-externalagent":["partial","*",18,0],"cohere-ai":["partial","*",18,0],"Diffbot":["partial","*",18,0],"ContextFetchBot":["partial","*",18,0]},"llms":{"status":"missing","http":404,"bytes":2485},"home":{"status":"ok","http":200,"url":"https://www.aad.org/","html_tokens":4191,"markdown_tokens":156,"pruned_tokens":100}},
{"rank":2078,"domain":"creativethemes.com","origin":"https://creativethemes.com","robots":{"status":"refused","http":403,"url":"https://creativethemes.com/robots.txt","bytes":5556,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":2079,"domain":"hashicorp.com","origin":"https://hashicorp.com","robots":{"status":"refused","http":429,"url":"https://hashicorp.com/robots.txt","bytes":34222,"groups":0,"error":"HTTP 429"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":2080,"domain":"bootstrapmade.com","origin":"https://bootstrapmade.com","robots":{"status":"ok","http":200,"url":"https://bootstrapmade.com/robots.txt","bytes":48,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":523},"home":{"status":"ok","http":200,"url":"https://bootstrapmade.com/","html_tokens":35668,"markdown_tokens":3158,"pruned_tokens":150}},
{"rank":2081,"domain":"opensea.io","origin":"https://opensea.io","robots":{"status":"ok","http":200,"url":"https://opensea.io/robots.txt","bytes":76,"groups":1},"bots":{"GPTBot":["partial","*",1,0],"ChatGPT-User":["partial","*",1,0],"OAI-SearchBot":["partial","*",1,0],"ClaudeBot":["partial","*",1,0],"Claude-User":["partial","*",1,0],"Claude-SearchBot":["partial","*",1,0],"anthropic-ai":["partial","*",1,0],"PerplexityBot":["partial","*",1,0],"Perplexity-User":["partial","*",1,0],"Google-Extended":["partial","*",1,0],"Applebot-Extended":["partial","*",1,0],"CCBot":["partial","*",1,0],"Bytespider":["partial","*",1,0],"Amazonbot":["partial","*",1,0],"meta-externalagent":["partial","*",1,0],"cohere-ai":["partial","*",1,0],"Diffbot":["partial","*",1,0],"ContextFetchBot":["partial","*",1,0]},"llms":{"status":"present","http":200,"bytes":2672},"home":{"status":"not_html","error":"unsupported content type text/markdown"}},
{"rank":2082,"domain":"kidshealth.org","origin":"https://kidshealth.org","robots":{"status":"ok","http":200,"url":"https://kidshealth.org/robots.txt","bytes":960,"groups":3},"bots":{"GPTBot":["partial","*",8,0],"ChatGPT-User":["partial","*",8,0],"OAI-SearchBot":["partial","*",8,0],"ClaudeBot":["partial","*",8,0],"Claude-User":["partial","*",8,0],"Claude-SearchBot":["partial","*",8,0],"anthropic-ai":["partial","*",8,0],"PerplexityBot":["partial","*",8,0],"Perplexity-User":["partial","*",8,0],"Google-Extended":["partial","*",8,0],"Applebot-Extended":["partial","*",8,0],"CCBot":["partial","*",8,0],"Bytespider":["partial","*",8,0],"Amazonbot":["partial","*",8,0],"meta-externalagent":["partial","*",8,0],"cohere-ai":["partial","*",8,0],"Diffbot":["partial","*",8,0],"ContextFetchBot":["partial","*",8,0]},"llms":{"status":"missing","http":404,"bytes":35348},"home":{"status":"ok","http":200,"url":"https://kidshealth.org/","html_tokens":11589,"markdown_tokens":70,"pruned_tokens":70}},
{"rank":2083,"domain":"thepaper.cn","origin":"https://thepaper.cn","robots":{"status":"ok","http":200,"url":"https://www.thepaper.cn/robots.txt","bytes":56916,"groups":1},"bots":{"GPTBot":["partial","*",1496,0],"ChatGPT-User":["partial","*",1496,0],"OAI-SearchBot":["partial","*",1496,0],"ClaudeBot":["partial","*",1496,0],"Claude-User":["partial","*",1496,0],"Claude-SearchBot":["partial","*",1496,0],"anthropic-ai":["partial","*",1496,0],"PerplexityBot":["partial","*",1496,0],"Perplexity-User":["partial","*",1496,0],"Google-Extended":["partial","*",1496,0],"Applebot-Extended":["partial","*",1496,0],"CCBot":["partial","*",1496,0],"Bytespider":["partial","*",1496,0],"Amazonbot":["partial","*",1496,0],"meta-externalagent":["partial","*",1496,0],"cohere-ai":["partial","*",1496,0],"Diffbot":["partial","*",1496,0],"ContextFetchBot":["partial","*",1496,0]},"llms":{"status":"missing","http":404,"bytes":3218},"home":{"status":"http_error","http":403,"url":"https://www.thepaper.cn/"}},
{"rank":2084,"domain":"rferl.org","origin":"https://rferl.org","robots":{"status":"ok","http":200,"url":"https://www.rferl.org/robots.txt","bytes":471,"groups":2},"bots":{"GPTBot":["partial","*",16,0],"ChatGPT-User":["partial","*",16,0],"OAI-SearchBot":["partial","*",16,0],"ClaudeBot":["partial","*",16,0],"Claude-User":["partial","*",16,0],"Claude-SearchBot":["partial","*",16,0],"anthropic-ai":["partial","*",16,0],"PerplexityBot":["partial","*",16,0],"Perplexity-User":["partial","*",16,0],"Google-Extended":["partial","*",16,0],"Applebot-Extended":["partial","*",16,0],"CCBot":["partial","*",16,0],"Bytespider":["partial","*",16,0],"Amazonbot":["partial","*",16,0],"meta-externalagent":["partial","*",16,0],"cohere-ai":["partial","*",16,0],"Diffbot":["partial","*",16,0],"ContextFetchBot":["partial","*",16,0]},"llms":{"status":"missing","http":404,"bytes":44219},"home":{"status":"ok","http":200,"url":"https://www.rferl.org/","html_tokens":41209,"markdown_tokens":7,"pruned_tokens":7}},
{"rank":2085,"domain":"ny.us","origin":"https://ny.us","robots":{"status":"unreachable","http":null,"url":"https://ny.us/robots.txt","bytes":0,"groups":0,"error":"cannot resolve ny.us"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":2086,"domain":"rijksoverheid.nl","origin":"https://rijksoverheid.nl","robots":{"status":"ok","http":200,"url":"https://www.rijksoverheid.nl/robots.txt","bytes":65,"groups":1},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"missing","http":404,"bytes":153947},"home":{"status":"ok","http":200,"url":"https://www.rijksoverheid.nl/","html_tokens":80492,"markdown_tokens":467,"pruned_tokens":172}},
{"rank":2087,"domain":"amazon.com.au","origin":"https://amazon.com.au","robots":{"status":"ok","http":200,"url":"https://www.amazon.com.au/robots.txt","bytes":6904,"groups":101},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["blocked","named",1,0],"OAI-SearchBot":["blocked","named",1,0],"ClaudeBot":["blocked","named",2,0],"Claude-User":["blocked","named",1,0],"Claude-SearchBot":["blocked","named",1,0],"anthropic-ai":["partial","*",85,6],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["blocked","named",1,0],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["partial","*",85,6],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["partial","*",85,6],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",85,6]},"llms":{"status":"missing","http":404,"bytes":2166},"home":{"status":"ok","http":200,"url":"https://www.amazon.com.au/","html_tokens":158699,"markdown_tokens":866,"pruned_tokens":174}},
{"rank":2088,"domain":"planalto.gov.br","origin":"https://planalto.gov.br","robots":{"status":"unreachable","http":null,"url":"https://planalto.gov.br/robots.txt","bytes":0,"groups":0,"error":"ReadError"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":2089,"domain":"huanqiu.com","origin":"https://huanqiu.com","robots":{"status":"ok","http":200,"url":"https://huanqiu.com/robots.txt","bytes":24,"groups":1},"bots":{"GPTBot":["allowed","*",0,1],"ChatGPT-User":["allowed","*",0,1],"OAI-SearchBot":["allowed","*",0,1],"ClaudeBot":["allowed","*",0,1],"Claude-User":["allowed","*",0,1],"Claude-SearchBot":["allowed","*",0,1],"anthropic-ai":["allowed","*",0,1],"PerplexityBot":["allowed","*",0,1],"Perplexity-User":["allowed","*",0,1],"Google-Extended":["allowed","*",0,1],"Applebot-Extended":["allowed","*",0,1],"CCBot":["allowed","*",0,1],"Bytespider":["allowed","*",0,1],"Amazonbot":["allowed","*",0,1],"meta-externalagent":["allowed","*",0,1],"cohere-ai":["allowed","*",0,1],"Diffbot":["allowed","*",0,1],"ContextFetchBot":["allowed","*",0,1]},"llms":{"status":"present","http":200,"bytes":11415},"home":{"status":"ok","http":200,"url":"https://huanqiu.com/","html_tokens":25938,"markdown_tokens":6,"pruned_tokens":6}},
{"rank":2090,"domain":"technorati.com","origin":"https://technorati.com","robots":{"status":"ok","http":200,"url":"https://technorati.com/robots.txt","bytes":1150,"groups":2},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["blocked","named",1,0],"OAI-SearchBot":["partial","*",8,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",8,0],"Claude-SearchBot":["partial","*",8,0],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["partial","*",8,0],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["partial","*",8,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["partial","*",8,0],"meta-externalagent":["partial","*",8,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",8,0]},"llms":{"status":"missing","http":404,"bytes":5446},"home":{"status":"ok","http":200,"url":"https://technorati.com/","html_tokens":2012,"markdown_tokens":0,"pruned_tokens":0}},
{"rank":2091,"domain":"scotsman.com","origin":"https://scotsman.com","robots":{"status":"ok","http":200,"url":"https://www.scotsman.com/robots.txt","bytes":843,"groups":2},"bots":{"GPTBot":["partial","*",3,0],"ChatGPT-User":["partial","*",3,0],"OAI-SearchBot":["partial","*",3,0],"ClaudeBot":["partial","*",3,0],"Claude-User":["partial","*",3,0],"Claude-SearchBot":["partial","*",3,0],"anthropic-ai":["partial","*",3,0],"PerplexityBot":["partial","*",3,0],"Perplexity-User":["partial","*",3,0],"Google-Extended":["partial","*",3,0],"Applebot-Extended":["partial","*",3,0],"CCBot":["partial","*",3,0],"Bytespider":["partial","*",3,0],"Amazonbot":["partial","*",3,0],"meta-externalagent":["partial","*",3,0],"cohere-ai":["partial","*",3,0],"Diffbot":["partial","*",3,0],"ContextFetchBot":["partial","*",3,0]},"llms":{"status":"http_error","http":403,"bytes":5527},"home":{"status":"http_error","http":403,"url":"https://www.scotsman.com/"}},
{"rank":2092,"domain":"topuniversities.com","origin":"https://topuniversities.com","robots":{"status":"ok","http":200,"url":"https://www.topuniversities.com/robots.txt","bytes":2223,"groups":1},"bots":{"GPTBot":["partial","*",39,18],"ChatGPT-User":["partial","*",39,18],"OAI-SearchBot":["partial","*",39,18],"ClaudeBot":["partial","*",39,18],"Claude-User":["partial","*",39,18],"Claude-SearchBot":["partial","*",39,18],"anthropic-ai":["partial","*",39,18],"PerplexityBot":["partial","*",39,18],"Perplexity-User":["partial","*",39,18],"Google-Extended":["partial","*",39,18],"Applebot-Extended":["partial","*",39,18],"CCBot":["partial","*",39,18],"Bytespider":["partial","*",39,18],"Amazonbot":["partial","*",39,18],"meta-externalagent":["partial","*",39,18],"cohere-ai":["partial","*",39,18],"Diffbot":["partial","*",39,18],"ContextFetchBot":["partial","*",39,18]},"llms":{"status":"missing","http":404,"bytes":558},"home":{"status":"http_error","http":403,"url":"https://topuniversities.com/"}},
{"rank":2093,"domain":"lowes.com","origin":"https://lowes.com","robots":{"status":"refused","http":403,"url":"https://www.lowes.com/robots.txt","bytes":381,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":2094,"domain":"opentable.com","origin":"https://opentable.com","robots":{"status":"unreachable","http":null,"url":"https://opentable.com/robots.txt","bytes":0,"groups":0,"error":"<StreamReset stream_id:1, error_code:2, remote_reset:True>"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":2095,"domain":"lu.se","origin":"https://lu.se","robots":{"status":"ok","http":200,"url":"https://www.lu.se/robots.txt","bytes":13942,"groups":1},"bots":{"GPTBot":["partial","*",249,126],"ChatGPT-User":["partial","*",249,126],"OAI-SearchBot":["partial","*",249,126],"ClaudeBot":["partial","*",249,126],"Claude-User":["partial","*",249,126],"Claude-SearchBot":["partial","*",249,126],"anthropic-ai":["partial","*",249,126],"PerplexityBot":["partial","*",249,126],"Perplexity-User":["partial","*",249,126],"Google-Extended":["partial","*",249,126],"Applebot-Extended":["partial","*",249,126],"CCBot":["partial","*",249,126],"Bytespider":["partial","*",249,126],"Amazonbot":["partial","*",249,126],"meta-externalagent":["partial","*",249,126],"cohere-ai":["partial","*",249,126],"Diffbot":["partial","*",249,126],"ContextFetchBot":["partial","*",249,126]},"llms":{"status":"missing","http":404,"bytes":183},"home":{"status":"ok","http":200,"url":"https://www.lu.se/","html_tokens":39192,"markdown_tokens":27,"pruned_tokens":27}},
{"rank":2096,"domain":"skysports.com","origin":"https://skysports.com","robots":{"status":"ok","http":200,"url":"https://www.skysports.com/robots.txt","bytes":2175,"groups":6},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["partial","*",58,0],"OAI-SearchBot":["partial","*",58,0],"ClaudeBot":["partial","*",58,0],"Claude-User":["partial","*",58,0],"Claude-SearchBot":["partial","*",58,0],"anthropic-ai":["partial","*",58,0],"PerplexityBot":["partial","*",58,0],"Perplexity-User":["partial","*",58,0],"Google-Extended":["partial","*",58,0],"Applebot-Extended":["partial","*",58,0],"CCBot":["blocked","named",1,0],"Bytespider":["partial","*",58,0],"Amazonbot":["partial","*",58,0],"meta-externalagent":["partial","*",58,0],"cohere-ai":["partial","*",58,0],"Diffbot":["partial","*",58,0],"ContextFetchBot":["partial","*",58,0]},"llms":{"status":"missing","http":404,"bytes":224201},"home":{"status":"ok","http":200,"url":"https://www.skysports.com/","html_tokens":147815,"markdown_tokens":32,"pruned_tokens":32}},
{"rank":2097,"domain":"illinois.gov","origin":"https://illinois.gov","robots":{"status":"ok","http":200,"url":"https://www.illinois.gov/robots.txt","bytes":2026,"groups":31},"bots":{"GPTBot":["partial","*",1,1],"ChatGPT-User":["partial","*",1,1],"OAI-SearchBot":["partial","*",1,1],"ClaudeBot":["partial","*",1,1],"Claude-User":["partial","*",1,1],"Claude-SearchBot":["partial","*",1,1],"anthropic-ai":["partial","*",1,1],"PerplexityBot":["partial","*",1,1],"Perplexity-User":["partial","*",1,1],"Google-Extended":["partial","*",1,1],"Applebot-Extended":["partial","*",1,1],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",2,0],"meta-externalagent":["partial","*",1,1],"cohere-ai":["partial","*",1,1],"Diffbot":["partial","*",1,1],"ContextFetchBot":["partial","*",1,1]},"llms":{"status":"missing","http":404,"bytes":196},"home":{"status":"ok","http":200,"url":"https://www.illinois.gov/","html_tokens":16907,"markdown_tokens":37,"pruned_tokens":37}},
{"rank":2098,"domain":"miami.edu","origin":"https://miami.edu","robots":{"status":"ok","http":200,"url":"https://welcome.miami.edu/robots.txt","bytes":130,"groups":1},"bots":{"GPTBot":["partial","*",2,0],"ChatGPT-User":["partial","*",2,0],"OAI-SearchBot":["partial","*",2,0],"ClaudeBot":["partial","*",2,0],"Claude-User":["partial","*",2,0],"Claude-SearchBot":["partial","*",2,0],"anthropic-ai":["partial","*",2,0],"PerplexityBot":["partial","*",2,0],"Perplexity-User":["partial","*",2,0],"Google-Extended":["partial","*",2,0],"Applebot-Extended":["partial","*",2,0],"CCBot":["partial","*",2,0],"Bytespider":["partial","*",2,0],"Amazonbot":["partial","*",2,0],"meta-externalagent":["partial","*",2,0],"cohere-ai":["partial","*",2,0],"Diffbot":["partial","*",2,0],"ContextFetchBot":["partial","*",2,0]},"llms":{"status":"missing","http":404,"bytes":59279},"home":{"status":"ok","http":200,"url":"https://welcome.miami.edu/","html_tokens":27614,"markdown_tokens":108,"pruned_tokens":15}},
{"rank":2099,"domain":"breitbart.com","origin":"https://breitbart.com","robots":{"status":"ok","http":200,"url":"https://www.breitbart.com/robots.txt","bytes":711,"groups":6},"bots":{"GPTBot":["partial","*",8,0],"ChatGPT-User":["partial","*",8,0],"OAI-SearchBot":["partial","*",8,0],"ClaudeBot":["allowed","named",0,0],"Claude-User":["partial","*",8,0],"Claude-SearchBot":["partial","*",8,0],"anthropic-ai":["partial","*",8,0],"PerplexityBot":["partial","*",8,0],"Perplexity-User":["partial","*",8,0],"Google-Extended":["partial","*",8,0],"Applebot-Extended":["partial","*",8,0],"CCBot":["partial","*",8,0],"Bytespider":["partial","*",8,0],"Amazonbot":["partial","*",8,0],"meta-externalagent":["partial","*",8,0],"cohere-ai":["partial","*",8,0],"Diffbot":["partial","*",8,0],"ContextFetchBot":["partial","*",8,0]},"llms":{"status":"missing","http":404,"bytes":1357},"home":{"status":"ok","http":200,"url":"https://www.breitbart.com/","html_tokens":42466,"markdown_tokens":63,"pruned_tokens":63}},
{"rank":2100,"domain":"visualcapitalist.com","origin":"https://visualcapitalist.com","robots":{"status":"refused","http":403,"url":"https://www.visualcapitalist.com/robots.txt","bytes":5583,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":2101,"domain":"annualreviews.org","origin":"https://annualreviews.org","robots":{"status":"ok","http":200,"url":"https://www.annualreviews.org/robots.txt","bytes":7970,"groups":6},"bots":{"GPTBot":["partial","*",53,0],"ChatGPT-User":["partial","*",53,0],"OAI-SearchBot":["partial","*",53,0],"ClaudeBot":["partial","*",53,0],"Claude-User":["partial","*",53,0],"Claude-SearchBot":["partial","*",53,0],"anthropic-ai":["partial","*",53,0],"PerplexityBot":["partial","*",53,0],"Perplexity-User":["partial","*",53,0],"Google-Extended":["partial","*",53,0],"Applebot-Extended":["partial","*",53,0],"CCBot":["partial","*",53,0],"Bytespider":["partial","*",53,0],"Amazonbot":["partial","*",53,0],"meta-externalagent":["partial","*",53,0],"cohere-ai":["partial","*",53,0],"Diffbot":["partial","*",53,0],"ContextFetchBot":["partial","*",53,0]},"llms":{"status":"missing","http":404,"bytes":50603},"home":{"status":"http_error","http":403,"url":"https://annualreviews.org/"}},
{"rank":2102,"domain":"newegg.com","origin":"https://newegg.com","robots":{"status":"ok","http":200,"url":"https://www.newegg.com/robots.txt","bytes":4596,"groups":12},"bots":{"GPTBot":["partial","*",70,2],"ChatGPT-User":["partial","*",70,2],"OAI-SearchBot":["partial","*",70,2],"ClaudeBot":["partial","*",70,2],"Claude-User":["partial","*",70,2],"Claude-SearchBot":["partial","*",70,2],"anthropic-ai":["partial","*",70,2],"PerplexityBot":["partial","*",70,2],"Perplexity-User":["partial","*",70,2],"Google-Extended":["partial","*",70,2],"Applebot-Extended":["partial","*",70,2],"CCBot":["partial","*",70,2],"Bytespider":["partial","*",70,2],"Amazonbot":["partial","*",70,2],"meta-externalagent":["partial","*",70,2],"cohere-ai":["partial","*",70,2],"Diffbot":["partial","*",70,2],"ContextFetchBot":["partial","*",70,2]},"llms":{"status":"present","http":200,"bytes":11960},"home":{"status":"http_error","http":403,"url":"https://www.newegg.com/"}},
{"rank":2103,"domain":"mailerlite.com","origin":"https://mailerlite.com","robots":{"status":"ok","http":200,"url":"https://www.mailerlite.com/robots.txt","bytes":337,"groups":1},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"present","http":200,"bytes":22461},"home":{"status":"ok","http":200,"url":"https://www.mailerlite.com/","html_tokens":78617,"markdown_tokens":238,"pruned_tokens":120}},
{"rank":2104,"domain":"bustle.com","origin":"https://bustle.com","robots":{"status":"ok","http":200,"url":"https://www.bustle.com/robots.txt","bytes":286,"groups":2},"bots":{"GPTBot":["partial","*",1,0],"ChatGPT-User":["partial","*",1,0],"OAI-SearchBot":["partial","*",1,0],"ClaudeBot":["partial","*",1,0],"Claude-User":["partial","*",1,0],"Claude-SearchBot":["partial","*",1,0],"anthropic-ai":["partial","*",1,0],"PerplexityBot":["partial","*",1,0],"Perplexity-User":["partial","*",1,0],"Google-Extended":["partial","*",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["partial","*",1,0],"Bytespider":["partial","*",1,0],"Amazonbot":["partial","*",1,0],"meta-externalagent":["partial","*",1,0],"cohere-ai":["partial","*",1,0],"Diffbot":["partial","*",1,0],"ContextFetchBot":["partial","*",1,0]},"llms":{"status":"missing","http":404,"bytes":35525},"home":{"status":"ok","http":200,"url":"https://www.bustle.com","html_tokens":98442,"markdown_tokens":1672,"pruned_tokens":149}},
{"rank":2105,"domain":"stitcher.com","origin":"https://www.stitcher.com","robots":{"status":"html","http":200,"url":"https://www.stitcher.com/roadblock/index.html","bytes":2850,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":2106,"domain":"menshealth.com","origin":"https://menshealth.com","robots":{"status":"ok","http":200,"url":"https://menshealth.com/robots.txt","bytes":950,"groups":3},"bots":{"GPTBot":["partial","named",31,0],"ChatGPT-User":["partial","named",31,0],"OAI-SearchBot":["partial","named",31,0],"ClaudeBot":["partial","*",19,0],"Claude-User":["partial","*",19,0],"Claude-SearchBot":["partial","*",19,0],"anthropic-ai":["partial","*",19,0],"PerplexityBot":["partial","*",19,0],"Perplexity-User":["partial","*",19,0],"Google-Extended":["partial","*",19,0],"Applebot-Extended":["partial","*",19,0],"CCBot":["partial","*",19,0],"Bytespider":["partial","*",19,0],"Amazonbot":["partial","*",19,0],"meta-externalagent":["partial","*",19,0],"cohere-ai":["partial","*",19,0],"Diffbot":["partial","*",19,0],"ContextFetchBot":["partial","*",19,0]},"llms":{"status":"missing","http":404,"bytes":84},"home":{"status":"ok","http":200,"url":"https://www.menshealth.com/","html_tokens":253246,"markdown_tokens":94,"pruned_tokens":95}},
{"rank":2107,"domain":"delicious.com","origin":"https://delicious.com","robots":{"status":"ok","http":200,"url":"https://delicious.com/robots.txt","bytes":12041,"groups":16},"bots":{"GPTBot":["partial","named",22,1],"ChatGPT-User":["partial","named",22,1],"OAI-SearchBot":["partial","named",22,1],"ClaudeBot":["partial","named",22,1],"Claude-User":["partial","named",22,1],"Claude-SearchBot":["partial","named",22,1],"anthropic-ai":["partial","named",22,1],"PerplexityBot":["partial","named",22,1],"Perplexity-User":["partial","named",22,1],"Google-Extended":["partial","named",22,1],"Applebot-Extended":["partial","named",22,1],"CCBot":["partial","*",22,1],"Bytespider":["partial","*",22,1],"Amazonbot":["partial","named",22,1],"meta-externalagent":["partial","*",22,1],"cohere-ai":["partial","named",22,1],"Diffbot":["partial","*",22,1],"ContextFetchBot":["partial","*",22,1]},"llms":{"status":"present","http":200,"bytes":2778},"home":{"status":"http_error","http":403,"url":"https://delicious.com/"}},
{"rank":2108,"domain":"cbs.com","origin":"https://cbs.com","robots":{"status":"ok","http":200,"url":"https://www.cbs.com/robots.txt","bytes":2049,"groups":1},"bots":{"GPTBot":["partial","*",52,4],"ChatGPT-User":["partial","*",52,4],"OAI-SearchBot":["partial","*",52,4],"ClaudeBot":["partial","*",52,4],"Claude-User":["partial","*",52,4],"Claude-SearchBot":["partial","*",52,4],"anthropic-ai":["partial","*",52,4],"PerplexityBot":["partial","*",52,4],"Perplexity-User":["partial","*",52,4],"Google-Extended":["partial","*",52,4],"Applebot-Extended":["partial","*",52,4],"CCBot":["partial","*",52,4],"Bytespider":["partial","*",52,4],"Amazonbot":["partial","*",52,4],"meta-externalagent":["partial","*",52,4],"cohere-ai":["partial","*",52,4],"Diffbot":["partial","*",52,4],"ContextFetchBot":["partial","*",52,4]},"llms":{"status":"missing","http":404,"bytes":3},"home":{"status":"ok","http":200,"url":"https://www.cbs.com/","html_tokens":47436,"markdown_tokens":200,"pruned_tokens":81}},
{"rank":2109,"domain":"macworld.com","origin":"https://macworld.com","robots":{"status":"ok","http":200,"url":"https://www.macworld.com/robots.txt","bytes":2611,"groups":41},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["partial","*",15,0],"OAI-SearchBot":["partial","*",15,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",15,0],"Claude-SearchBot":["partial","*",15,0],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["partial","*",15,0],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["partial","*",15,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",15,0]},"llms":{"status":"missing","http":404,"bytes":203338},"home":{"status":"ok","http":200,"url":"https://www.macworld.com","html_tokens":64082,"markdown_tokens":232,"pruned_tokens":72}},
{"rank":2110,"domain":"nap.edu","origin":"https://nap.edu","robots":{"status":"ok","http":200,"url":"https://nap.nationalacademies.org/robots.txt","bytes":1706,"groups":16},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["blocked","named",1,0],"OAI-SearchBot":["partial","*",10,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",10,0],"Claude-SearchBot":["partial","*",10,0],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["partial","*",10,0],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["partial","*",10,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["partial","*",10,0],"meta-externalagent":["partial","*",10,0],"cohere-ai":["partial","*",10,0],"Diffbot":["partial","*",10,0],"ContextFetchBot":["partial","*",10,0]},"llms":{"status":"missing","http":404,"bytes":9},"home":{"status":"ok","http":200,"url":"https://www.nationalacademies.org/publications/all","html_tokens":365596,"markdown_tokens":321,"pruned_tokens":124}},
{"rank":2111,"domain":"gencat.cat","origin":"https://gencat.cat","robots":{"status":"ok","http":200,"url":"https://gencat.cat/robots.txt","bytes":211,"groups":1},"bots":{"GPTBot":["partial","*",8,0],"ChatGPT-User":["partial","*",8,0],"OAI-SearchBot":["partial","*",8,0],"ClaudeBot":["partial","*",8,0],"Claude-User":["partial","*",8,0],"Claude-SearchBot":["partial","*",8,0],"anthropic-ai":["partial","*",8,0],"PerplexityBot":["partial","*",8,0],"Perplexity-User":["partial","*",8,0],"Google-Extended":["partial","*",8,0],"Applebot-Extended":["partial","*",8,0],"CCBot":["partial","*",8,0],"Bytespider":["partial","*",8,0],"Amazonbot":["partial","*",8,0],"meta-externalagent":["partial","*",8,0],"cohere-ai":["partial","*",8,0],"Diffbot":["partial","*",8,0],"ContextFetchBot":["partial","*",8,0]},"llms":{"status":"missing","http":404,"bytes":196},"home":{"status":"ok","http":200,"url":"https://gencat.cat/","html_tokens":45,"markdown_tokens":0,"pruned_tokens":0}},
{"rank":2112,"domain":"wikiwand.com","origin":"https://wikiwand.com","robots":{"status":"ok","http":200,"url":"https://www.wikiwand.com/robots.txt","bytes":182,"groups":1},"bots":{"GPTBot":["partial","*",6,1],"ChatGPT-User":["partial","*",6,1],"OAI-SearchBot":["partial","*",6,1],"ClaudeBot":["partial","*",6,1],"Claude-User":["partial","*",6,1],"Claude-SearchBot":["partial","*",6,1],"anthropic-ai":["partial","*",6,1],"PerplexityBot":["partial","*",6,1],"Perplexity-User":["partial","*",6,1],"Google-Extended":["partial","*",6,1],"Applebot-Extended":["partial","*",6,1],"CCBot":["partial","*",6,1],"Bytespider":["partial","*",6,1],"Amazonbot":["partial","*",6,1],"meta-externalagent":["partial","*",6,1],"cohere-ai":["partial","*",6,1],"Diffbot":["partial","*",6,1],"ContextFetchBot":["partial","*",6,1]},"llms":{"status":"missing","http":404,"bytes":68880},"home":{"status":"http_error","http":403,"url":"https://wikiwand.com/"}},
{"rank":2113,"domain":"linuxfoundation.org","origin":"https://linuxfoundation.org","robots":{"status":"ok","http":200,"url":"https://www.linuxfoundation.org/robots.txt","bytes":245,"groups":1},"bots":{"GPTBot":["partial","*",7,0],"ChatGPT-User":["partial","*",7,0],"OAI-SearchBot":["partial","*",7,0],"ClaudeBot":["partial","*",7,0],"Claude-User":["partial","*",7,0],"Claude-SearchBot":["partial","*",7,0],"anthropic-ai":["partial","*",7,0],"PerplexityBot":["partial","*",7,0],"Perplexity-User":["partial","*",7,0],"Google-Extended":["partial","*",7,0],"Applebot-Extended":["partial","*",7,0],"CCBot":["partial","*",7,0],"Bytespider":["partial","*",7,0],"Amazonbot":["partial","*",7,0],"meta-externalagent":["partial","*",7,0],"cohere-ai":["partial","*",7,0],"Diffbot":["partial","*",7,0],"ContextFetchBot":["partial","*",7,0]},"llms":{"status":"missing","http":404,"bytes":71340},"home":{"status":"ok","http":200,"url":"https://www.linuxfoundation.org","html_tokens":35474,"markdown_tokens":371,"pruned_tokens":158}},
{"rank":2114,"domain":"pravo.gov.ru","origin":"https://pravo.gov.ru","robots":{"status":"timeout","http":null,"url":"https://pravo.gov.ru/robots.txt","bytes":0,"groups":0,"error":"ConnectTimeout"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":2115,"domain":"theonion.com","origin":"https://theonion.com","robots":{"status":"ok","http":200,"url":"https://theonion.com/robots.txt","bytes":1548,"groups":35},"bots":{"GPTBot":["blocked","named",2,0],"ChatGPT-User":["blocked","named",1,0],"OAI-SearchBot":["blocked","named",1,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",2,1],"Claude-SearchBot":["partial","*",2,1],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["partial","*",2,1],"Google-Extended":["blocked","named",2,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",2,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",2,1]},"llms":{"status":"missing","http":404,"bytes":244338},"home":{"status":"ok","http":200,"url":"https://theonion.com/","html_tokens":129203,"markdown_tokens":412,"pruned_tokens":95}},
{"rank":2116,"domain":"kit.edu","origin":"https://kit.edu","robots":{"status":"ok","http":200,"url":"https://www.kit.edu/robots.txt","bytes":300,"groups":2},"bots":{"GPTBot":["partial","*",6,0],"ChatGPT-User":["partial","*",6,0],"OAI-SearchBot":["partial","*",6,0],"ClaudeBot":["partial","*",6,0],"Claude-User":["partial","*",6,0],"Claude-SearchBot":["partial","*",6,0],"anthropic-ai":["partial","*",6,0],"PerplexityBot":["partial","*",6,0],"Perplexity-User":["partial","*",6,0],"Google-Extended":["partial","*",6,0],"Applebot-Extended":["partial","*",6,0],"CCBot":["partial","*",6,0],"Bytespider":["partial","*",6,0],"Amazonbot":["partial","*",6,0],"meta-externalagent":["partial","*",6,0],"cohere-ai":["partial","*",6,0],"Diffbot":["partial","*",6,0],"ContextFetchBot":["partial","*",6,0]},"llms":{"status":"missing","http":404,"bytes":98946},"home":{"status":"ok","http":200,"url":"https://www.kit.edu/","html_tokens":18711,"markdown_tokens":209,"pruned_tokens":73}},
{"rank":2117,"domain":"baltimoresun.com","origin":"https://baltimoresun.com","robots":{"status":"ok","http":200,"url":"https://www.baltimoresun.com/robots.txt","bytes":4487,"groups":68},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["blocked","named",1,0],"OAI-SearchBot":["blocked","named",1,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["blocked","named",1,0],"Claude-SearchBot":["blocked","named",1,0],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["blocked","named",1,0],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["partial","*",8,1],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",8,1]},"llms":{"status":"missing","http":404,"bytes":83114},"home":{"status":"ok","http":200,"url":"https://www.baltimoresun.com/","html_tokens":65723,"markdown_tokens":378,"pruned_tokens":103}},
{"rank":2118,"domain":"ncl.ac.uk","origin":"https://ncl.ac.uk","robots":{"status":"ok","http":200,"url":"https://www.ncl.ac.uk/robots.txt","bytes":10649,"groups":10},"bots":{"GPTBot":["partial","*",111,0],"ChatGPT-User":["partial","*",111,0],"OAI-SearchBot":["partial","*",111,0],"ClaudeBot":["partial","*",111,0],"Claude-User":["partial","*",111,0],"Claude-SearchBot":["partial","*",111,0],"anthropic-ai":["partial","*",111,0],"PerplexityBot":["partial","*",111,0],"Perplexity-User":["partial","*",111,0],"Google-Extended":["partial","*",111,0],"Applebot-Extended":["partial","*",111,0],"CCBot":["partial","*",111,0],"Bytespider":["partial","*",111,0],"Amazonbot":["partial","*",111,0],"meta-externalagent":["partial","*",111,0],"cohere-ai":["partial","*",111,0],"Diffbot":["partial","*",111,0],"ContextFetchBot":["partial","*",111,0]},"llms":{"status":"missing","http":404,"bytes":31613},"home":{"status":"ok","http":200,"url":"https://www.ncl.ac.uk/","html_tokens":29325,"markdown_tokens":187,"pruned_tokens":122}},
{"rank":2119,"domain":"unece.org","origin":"https://unece.org","robots":{"status":"refused","http":403,"url":"https://unece.org/robots.txt","bytes":5397,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":2120,"domain":"synology.me","origin":"https://synology.me","robots":{"status":"unreachable","http":null,"url":"https://synology.me/robots.txt","bytes":0,"groups":0,"error":"cannot resolve synology.me"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":2121,"domain":"legacy.com","origin":"https://legacy.com","robots":{"status":"ok","http":200,"url":"https://www.legacy.com/robots.txt","bytes":8325,"groups":4},"bots":{"GPTBot":["partial","*",157,0],"ChatGPT-User":["partial","*",157,0],"OAI-SearchBot":["partial","*",157,0],"ClaudeBot":["partial","*",157,0],"Claude-User":["partial","*",157,0],"Claude-SearchBot":["partial","*",157,0],"anthropic-ai":["partial","*",157,0],"PerplexityBot":["partial","*",157,0],"Perplexity-User":["partial","*",157,0],"Google-Extended":["partial","*",157,0],"Applebot-Extended":["partial","*",157,0],"CCBot":["partial","*",157,0],"Bytespider":["partial","*",157,0],"Amazonbot":["partial","*",157,0],"meta-externalagent":["partial","*",157,0],"cohere-ai":["partial","*",157,0],"Diffbot":["partial","*",157,0],"ContextFetchBot":["partial","*",157,0]},"llms":{"status":"missing","http":404,"bytes":0},"home":{"status":"ok","http":200,"url":"https://www.legacy.com/","html_tokens":214110,"markdown_tokens":368,"pruned_tokens":21}},
{"rank":2122,"domain":"autoriteitpersoonsgegevens.nl","origin":"https://autoriteitpersoonsgegevens.nl","robots":{"status":"refused","http":403,"url":"https://autoriteitpersoonsgegevens.nl/robots.txt","bytes":245,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":2123,"domain":"govdelivery.com","origin":"https://govdelivery.com","robots":{"status":"refused","http":403,"url":"https://govdelivery.com/robots.txt","bytes":560,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":2124,"domain":"envoyer.io","origin":"https://envoyer.io","robots":{"status":"ok","http":200,"url":"https://envoyer.io/robots.txt","bytes":24,"groups":1},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"missing","http":404,"bytes":6603},"home":{"status":"ok","http":200,"url":"https://envoyer.io/","html_tokens":5746,"markdown_tokens":67,"pruned_tokens":53}},
{"rank":2125,"domain":"ollama.com","origin":"https://ollama.com","robots":{"status":"ok","http":200,"url":"https://ollama.com/robots.txt","bytes":451,"groups":1},"bots":{"GPTBot":["partial","*",17,1],"ChatGPT-User":["partial","*",17,1],"OAI-SearchBot":["partial","*",17,1],"ClaudeBot":["partial","*",17,1],"Claude-User":["partial","*",17,1],"Claude-SearchBot":["partial","*",17,1],"anthropic-ai":["partial","*",17,1],"PerplexityBot":["partial","*",17,1],"Perplexity-User":["partial","*",17,1],"Google-Extended":["partial","*",17,1],"Applebot-Extended":["partial","*",17,1],"CCBot":["partial","*",17,1],"Bytespider":["partial","*",17,1],"Amazonbot":["partial","*",17,1],"meta-externalagent":["partial","*",17,1],"cohere-ai":["partial","*",17,1],"Diffbot":["partial","*",17,1],"ContextFetchBot":["partial","*",17,1]},"llms":{"status":"present","http":200,"bytes":1786},"home":{"status":"ok","http":200,"url":"https://ollama.com/","html_tokens":4061,"markdown_tokens":78,"pruned_tokens":78}},
{"rank":2126,"domain":"gamblingcommission.gov.uk","origin":"https://gamblingcommission.gov.uk","robots":{"status":"ok","http":200,"url":"https://www.gamblingcommission.gov.uk/robots.txt","bytes":219,"groups":1},"bots":{"GPTBot":["partial","*",2,1],"ChatGPT-User":["partial","*",2,1],"OAI-SearchBot":["partial","*",2,1],"ClaudeBot":["partial","*",2,1],"Claude-User":["partial","*",2,1],"Claude-SearchBot":["partial","*",2,1],"anthropic-ai":["partial","*",2,1],"PerplexityBot":["partial","*",2,1],"Perplexity-User":["partial","*",2,1],"Google-Extended":["partial","*",2,1],"Applebot-Extended":["partial","*",2,1],"CCBot":["partial","*",2,1],"Bytespider":["partial","*",2,1],"Amazonbot":["partial","*",2,1],"meta-externalagent":["partial","*",2,1],"cohere-ai":["partial","*",2,1],"Diffbot":["partial","*",2,1],"ContextFetchBot":["partial","*",2,1]},"llms":{"status":"missing","http":404,"bytes":29159},"home":{"status":"ok","http":200,"url":"https://www.gamblingcommission.gov.uk/","html_tokens":13792,"markdown_tokens":473,"pruned_tokens":46}},
{"rank":2127,"domain":"nj.us","origin":"https://nj.us","robots":{"status":"unreachable","http":null,"url":"https://nj.us/robots.txt","bytes":0,"groups":0,"error":"cannot resolve nj.us"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":2128,"domain":"akc.org","origin":"https://akc.org","robots":{"status":"refused","http":403,"url":"https://akc.org/robots.txt","bytes":560,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":2129,"domain":"pingdom.com","origin":"https://pingdom.com","robots":{"status":"ok","http":200,"url":"https://www.pingdom.com/robots.txt","bytes":233,"groups":1},"bots":{"GPTBot":["partial","*",7,0],"ChatGPT-User":["partial","*",7,0],"OAI-SearchBot":["partial","*",7,0],"ClaudeBot":["partial","*",7,0],"Claude-User":["partial","*",7,0],"Claude-SearchBot":["partial","*",7,0],"anthropic-ai":["partial","*",7,0],"PerplexityBot":["partial","*",7,0],"Perplexity-User":["partial","*",7,0],"Google-Extended":["partial","*",7,0],"Applebot-Extended":["partial","*",7,0],"CCBot":["partial","*",7,0],"Bytespider":["partial","*",7,0],"Amazonbot":["partial","*",7,0],"meta-externalagent":["partial","*",7,0],"cohere-ai":["partial","*",7,0],"Diffbot":["partial","*",7,0],"ContextFetchBot":["partial","*",7,0]},"llms":{"status":"missing","http":404,"bytes":60},"home":{"status":"ok","http":200,"url":"https://www.pingdom.com/","html_tokens":41963,"markdown_tokens":1229,"pruned_tokens":473}},
{"rank":2130,"domain":"mainichi.jp","origin":"https://mainichi.jp","robots":{"status":"ok","http":200,"url":"https://mainichi.jp/robots.txt","bytes":1407,"groups":33},"bots":{"GPTBot":["partial","named",1,1],"ChatGPT-User":["partial","named",1,1],"OAI-SearchBot":["blocked","named",1,0],"ClaudeBot":["partial","named",1,1],"Claude-User":["partial","named",1,1],"Claude-SearchBot":["partial","named",1,1],"anthropic-ai":["partial","named",1,1],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["partial","named",1,2],"Applebot-Extended":["partial","named",1,1],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["partial","named",1,1],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":19369},"home":{"status":"ok","http":200,"url":"https://mainichi.jp/","html_tokens":50762,"markdown_tokens":51,"pruned_tokens":51}},
{"rank":2131,"domain":"tidal.com","origin":"https://tidal.com","robots":{"status":"ok","http":200,"url":"https://tidal.com/robots.txt","bytes":705,"groups":2},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["allowed","*",0,1],"OAI-SearchBot":["allowed","*",0,1],"ClaudeBot":["blocked","named",1,0],"Claude-User":["allowed","*",0,1],"Claude-SearchBot":["allowed","*",0,1],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["allowed","*",0,1],"Perplexity-User":["allowed","*",0,1],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["allowed","*",0,1],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["allowed","*",0,1],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["allowed","*",0,1]},"llms":{"status":"http_error","http":403,"bytes":263},"home":{"status":"http_error","http":403,"url":"https://tidal.com/"}},
{"rank":2132,"domain":"ku.dk","origin":"https://ku.dk","robots":{"status":"ok","http":200,"url":"https://www.ku.dk/robots.txt","bytes":612,"groups":2},"bots":{"GPTBot":["partial","*",8,4],"ChatGPT-User":["partial","*",8,4],"OAI-SearchBot":["partial","*",8,4],"ClaudeBot":["partial","*",8,4],"Claude-User":["partial","*",8,4],"Claude-SearchBot":["partial","*",8,4],"anthropic-ai":["partial","*",8,4],"PerplexityBot":["partial","*",8,4],"Perplexity-User":["partial","*",8,4],"Google-Extended":["partial","*",8,4],"Applebot-Extended":["partial","*",8,4],"CCBot":["partial","*",8,4],"Bytespider":["partial","*",8,4],"Amazonbot":["partial","*",8,4],"meta-externalagent":["partial","*",8,4],"cohere-ai":["partial","*",8,4],"Diffbot":["partial","*",8,4],"ContextFetchBot":["partial","*",8,4]},"llms":{"status":"missing","http":404,"bytes":81315},"home":{"status":"ok","http":200,"url":"https://www.ku.dk/","html_tokens":31005,"markdown_tokens":33,"pruned_tokens":33}},
{"rank":2133,"domain":"rb.gy","origin":"https://rb.gy","robots":{"status":"ok","http":200,"url":"https://rb.gy/robots.txt?rb.routing.mode=proxy&rb.routing.signature=634796","bytes":52,"groups":1},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"html","http":200,"bytes":267487},"home":{"status":"http_error","http":403,"url":"https://rb.gy/"}},
{"rank":2134,"domain":"yle.fi","origin":"https://yle.fi","robots":{"status":"ok","http":200,"url":"https://yle.fi/robots.txt","bytes":1713,"groups":39},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["blocked","named",1,0],"OAI-SearchBot":["partial","*",10,2],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",10,2],"Claude-SearchBot":["partial","*",10,2],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["partial","*",10,2],"Perplexity-User":["partial","*",10,2],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["partial","*",10,2],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",10,2]},"llms":{"status":"missing","http":404,"bytes":10859},"home":{"status":"ok","http":200,"url":"https://yle.fi/","html_tokens":109929,"markdown_tokens":199,"pruned_tokens":161}},
{"rank":2135,"domain":"acer.com","origin":"https://acer.com","robots":{"status":"unreachable","http":null,"url":"https://acer.com/robots.txt","bytes":0,"groups":0,"error":"<StreamReset stream_id:1, error_code:2, remote_reset:True>"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":2136,"domain":"zazzle.com","origin":"https://zazzle.com","robots":{"status":"refused","http":403,"url":"https://www.zazzle.com/robots.txt","bytes":4332,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":2137,"domain":"capgemini.com","origin":"https://capgemini.com","robots":{"status":"ok","http":200,"url":"https://www.capgemini.com/robots.txt","bytes":2769,"groups":2},"bots":{"GPTBot":["partial","*",8,2],"ChatGPT-User":["partial","*",8,2],"OAI-SearchBot":["partial","*",8,2],"ClaudeBot":["partial","*",8,2],"Claude-User":["partial","*",8,2],"Claude-SearchBot":["partial","*",8,2],"anthropic-ai":["partial","*",8,2],"PerplexityBot":["partial","*",8,2],"Perplexity-User":["partial","*",8,2],"Google-Extended":["partial","*",8,2],"Applebot-Extended":["partial","*",8,2],"CCBot":["partial","*",8,2],"Bytespider":["partial","*",8,2],"Amazonbot":["partial","*",8,2],"meta-externalagent":["partial","*",8,2],"cohere-ai":["partial","*",8,2],"Diffbot":["partial","*",8,2],"ContextFetchBot":["partial","*",8,2]},"llms":{"status":"html","http":200,"bytes":11744},"home":{"status":"ok","http":200,"url":"https://www.capgemini.com/","html_tokens":67361,"markdown_tokens":128,"pruned_tokens":101}},
{"rank":2138,"domain":"instapaper.com","origin":"https://instapaper.com","robots":{"status":"ok","http":200,"url":"https://instapaper.com/robots.txt","bytes":179,"groups":1},"bots":{"GPTBot":["partial","*",10,1],"ChatGPT-User":["partial","*",10,1],"OAI-SearchBot":["partial","*",10,1],"ClaudeBot":["partial","*",10,1],"Claude-User":["partial","*",10,1],"Claude-SearchBot":["partial","*",10,1],"anthropic-ai":["partial","*",10,1],"PerplexityBot":["partial","*",10,1],"Perplexity-User":["partial","*",10,1],"Google-Extended":["partial","*",10,1],"Applebot-Extended":["partial","*",10,1],"CCBot":["partial","*",10,1],"Bytespider":["partial","*",10,1],"Amazonbot":["partial","*",10,1],"meta-externalagent":["partial","*",10,1],"cohere-ai":["partial","*",10,1],"Diffbot":["partial","*",10,1],"ContextFetchBot":["partial","*",10,1]},"llms":{"status":"html","http":200,"bytes":5905},"home":{"status":"ok","http":200,"url":"https://instapaper.com/","html_tokens":1775,"markdown_tokens":0,"pruned_tokens":0}},
{"rank":2139,"domain":"zalo.me","origin":"https://zalo.me","robots":{"status":"ok","http":200,"url":"https://zalo.me/robots.txt","bytes":66,"groups":1},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"disallowed","error":"robots.txt of id.zalo.me not readable (html)"},"home":{"status":"http_error","http":502,"url":"https://zalo.me/"}},
{"rank":2140,"domain":"scoop.it","origin":"https://scoop.it","robots":{"status":"html","http":200,"url":"https://www.scoop.it/en/","bytes":40262,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":2141,"domain":"iucn.org","origin":"https://iucn.org","robots":{"status":"ok","http":200,"url":"https://iucn.org/robots.txt","bytes":3884,"groups":1},"bots":{"GPTBot":["partial","*",113,0],"ChatGPT-User":["partial","*",113,0],"OAI-SearchBot":["partial","*",113,0],"ClaudeBot":["partial","*",113,0],"Claude-User":["partial","*",113,0],"Claude-SearchBot":["partial","*",113,0],"anthropic-ai":["partial","*",113,0],"PerplexityBot":["partial","*",113,0],"Perplexity-User":["partial","*",113,0],"Google-Extended":["partial","*",113,0],"Applebot-Extended":["partial","*",113,0],"CCBot":["partial","*",113,0],"Bytespider":["partial","*",113,0],"Amazonbot":["partial","*",113,0],"meta-externalagent":["partial","*",113,0],"cohere-ai":["partial","*",113,0],"Diffbot":["partial","*",113,0],"ContextFetchBot":["partial","*",113,0]},"llms":{"status":"http_error","http":403,"bytes":513},"home":{"status":"ok","http":200,"url":"https://iucn.org/","html_tokens":33137,"markdown_tokens":760,"pruned_tokens":464}},
{"rank":2142,"domain":"people.cn","origin":"https://people.cn","robots":{"status":"ok","http":200,"url":"https://www.people.cn/robots.txt","bytes":90,"groups":1},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"crawl_delay","error":"Crawl-delay 120 s"},"home":{"status":"crawl_delay","error":"Crawl-delay 120 s"}},
{"rank":2143,"domain":"azureedge.net","origin":"https://www.azureedge.net","robots":{"status":"missing","http":404,"url":"https://www.azureedge.net/robots.txt","bytes":266389,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"error","error":"Invalid input ConnectionInputs.RECV_HEADERS in state ConnectionState.CLOSED"},"home":{"status":"http_error","http":404,"url":"https://www.azureedge.net/"}},
{"rank":2144,"domain":"daringfireball.net","origin":"https://daringfireball.net","robots":{"status":"ok","http":200,"url":"https://daringfireball.net/robots.txt","bytes":55,"groups":1},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":5721},"home":{"status":"ok","http":200,"url":"https://daringfireball.net/","html_tokens":31722,"markdown_tokens":19176,"pruned_tokens":148}},
{"rank":2145,"domain":"drugs.com","origin":"https://drugs.com","robots":{"status":"ok","http":200,"url":"https://www.drugs.com/robots.txt","bytes":1012,"groups":3},"bots":{"GPTBot":["partial","*",10,0],"ChatGPT-User":["partial","*",10,0],"OAI-SearchBot":["partial","*",10,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",10,0],"Claude-SearchBot":["partial","*",10,0],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["partial","*",10,0],"Google-Extended":["partial","*",10,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",10,0]},"llms":{"status":"http_error","http":403,"bytes":379},"home":{"status":"http_error","http":403,"url":"https://www.drugs.com/"}},
{"rank":2146,"domain":"agoda.com","origin":"https://agoda.com","robots":{"status":"ok","http":200,"url":"https://www.agoda.com/robots.txt","bytes":5086,"groups":9},"bots":{"GPTBot":["partial","*",36,4],"ChatGPT-User":["partial","*",36,4],"OAI-SearchBot":["partial","*",36,4],"ClaudeBot":["partial","*",36,4],"Claude-User":["partial","*",36,4],"Claude-SearchBot":["partial","*",36,4],"anthropic-ai":["partial","*",36,4],"PerplexityBot":["partial","*",36,4],"Perplexity-User":["partial","*",36,4],"Google-Extended":["partial","*",36,4],"Applebot-Extended":["partial","*",36,4],"CCBot":["partial","*",36,4],"Bytespider":["partial","*",36,4],"Amazonbot":["partial","*",36,4],"meta-externalagent":["partial","*",36,4],"cohere-ai":["partial","*",36,4],"Diffbot":["partial","*",36,4],"ContextFetchBot":["partial","*",36,4]},"llms":{"status":"missing","http":404,"bytes":85835},"home":{"status":"ok","http":200,"url":"https://www.agoda.com/","html_tokens":57173,"markdown_tokens":0,"pruned_tokens":0}},
{"rank":2147,"domain":"maine.gov","origin":"https://maine.gov","robots":{"status":"refused","http":403,"url":"https://www.maine.gov/robots.txt","bytes":919,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":2148,"domain":"uzh.ch","origin":"https://uzh.ch","robots":{"status":"ok","http":200,"url":"https://uzh.ch/robots.txt","bytes":467,"groups":2},"bots":{"GPTBot":["partial","*",14,1],"ChatGPT-User":["partial","*",14,1],"OAI-SearchBot":["partial","*",14,1],"ClaudeBot":["partial","*",14,1],"Claude-User":["partial","*",14,1],"Claude-SearchBot":["partial","*",14,1],"anthropic-ai":["partial","*",14,1],"PerplexityBot":["partial","*",14,1],"Perplexity-User":["partial","*",14,1],"Google-Extended":["partial","*",14,1],"Applebot-Extended":["partial","*",14,1],"CCBot":["partial","*",14,1],"Bytespider":["partial","*",14,1],"Amazonbot":["partial","*",14,1],"meta-externalagent":["partial","*",14,1],"cohere-ai":["partial","*",14,1],"Diffbot":["partial","*",14,1],"ContextFetchBot":["partial","*",14,1]},"llms":{"status":"missing","http":404,"bytes":133916},"home":{"status":"ok","http":200,"url":"https://www.uzh.ch/de.html","html_tokens":66904,"markdown_tokens":551,"pruned_tokens":89}},
{"rank":2149,"domain":"independent.ie","origin":"https://independent.ie","robots":{"status":"ok","http":200,"url":"https://www.independent.ie/robots.txt","bytes":2707,"groups":40},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["blocked","named",1,0],"OAI-SearchBot":["blocked","named",1,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["blocked","named",1,0],"Claude-SearchBot":["blocked","named",1,0],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["blocked","named",1,0],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["partial","*",1,2],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",1,2]},"llms":{"status":"http_error","http":403,"bytes":4547},"home":{"status":"http_error","http":403,"url":"https://www.independent.ie/"}},
{"rank":2150,"domain":"w3n7z2qt.top","origin":"https://w3n7z2qt.top","robots":{"status":"refused","http":403,"url":"https://w3n7z2qt.top/robots.txt","bytes":902,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":2151,"domain":"bp.com","origin":"https://bp.com","robots":{"status":"refused","http":403,"url":"https://www.bp.com/robots.txt","bytes":3939,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":2152,"domain":"okta.com","origin":"https://okta.com","robots":{"status":"ok","http":200,"url":"https://www.okta.com/robots.txt","bytes":2518,"groups":1},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"present","http":200,"bytes":225656},"home":{"status":"ok","http":200,"url":"https://www.okta.com/","html_tokens":106601,"markdown_tokens":77,"pruned_tokens":77}},
{"rank":2153,"domain":"livedoor.jp","origin":"http://livedoor.jp","robots":{"status":"missing","http":404,"url":"http://livedoor.jp/robots.txt","bytes":196,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":196},"home":{"status":"http_error","http":403,"url":"http://livedoor.jp/"}},
{"rank":2154,"domain":"i.pinimg.com","origin":"https://i.pinimg.com","robots":{"status":"ok","http":200,"url":"https://i.pinimg.com/robots.txt","bytes":8364,"groups":4},"bots":{"GPTBot":["blocked","*",1,0],"ChatGPT-User":["blocked","*",1,0],"OAI-SearchBot":["blocked","*",1,0],"ClaudeBot":["blocked","*",1,0],"Claude-User":["blocked","*",1,0],"Claude-SearchBot":["blocked","*",1,0],"anthropic-ai":["blocked","*",1,0],"PerplexityBot":["blocked","*",1,0],"Perplexity-User":["blocked","*",1,0],"Google-Extended":["blocked","*",1,0],"Applebot-Extended":["blocked","*",1,0],"CCBot":["blocked","*",1,0],"Bytespider":["blocked","*",1,0],"Amazonbot":["blocked","*",1,0],"meta-externalagent":["blocked","*",1,0],"cohere-ai":["blocked","*",1,0],"Diffbot":["blocked","*",1,0],"ContextFetchBot":["blocked","*",1,0]},"llms":{"status":"disallowed"},"home":{"status":"disallowed"}},
{"rank":2155,"domain":"windowsphone.com","origin":"https://www.windowsphone.com","robots":{"status":"html","http":200,"url":"https://support.microsoft.com/en-us/windows/deployment/updates-lifecycle/what-does-it-mean-if-windows-isn-t-supported","bytes":110493,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":2156,"domain":"hku.hk","origin":"https://hku.hk","robots":{"status":"ok","http":200,"url":"https://www.hku.hk/robots.txt","bytes":72,"groups":1},"bots":{"GPTBot":["allowed","*",0,1],"ChatGPT-User":["allowed","*",0,1],"OAI-SearchBot":["allowed","*",0,1],"ClaudeBot":["allowed","*",0,1],"Claude-User":["allowed","*",0,1],"Claude-SearchBot":["allowed","*",0,1],"anthropic-ai":["allowed","*",0,1],"PerplexityBot":["allowed","*",0,1],"Perplexity-User":["allowed","*",0,1],"Google-Extended":["allowed","*",0,1],"Applebot-Extended":["allowed","*",0,1],"CCBot":["allowed","*",0,1],"Bytespider":["allowed","*",0,1],"Amazonbot":["allowed","*",0,1],"meta-externalagent":["allowed","*",0,1],"cohere-ai":["allowed","*",0,1],"Diffbot":["allowed","*",0,1],"ContextFetchBot":["allowed","*",0,1]},"llms":{"status":"missing","http":404,"bytes":97971},"home":{"status":"ok","http":200,"url":"https://www.hku.hk/","html_tokens":24104,"markdown_tokens":529,"pruned_tokens":354}},
{"rank":2157,"domain":"clutch.co","origin":"https://clutch.co","robots":{"status":"refused","http":403,"url":"https://clutch.co/robots.txt","bytes":5525,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":2158,"domain":"linklist.bio","origin":"https://linklist.bio","robots":{"status":"ok","http":200,"url":"https://linklist.bio/robots.txt","bytes":196,"groups":1},"bots":{"GPTBot":["partial","*",2,0],"ChatGPT-User":["partial","*",2,0],"OAI-SearchBot":["partial","*",2,0],"ClaudeBot":["partial","*",2,0],"Claude-User":["partial","*",2,0],"Claude-SearchBot":["partial","*",2,0],"anthropic-ai":["partial","*",2,0],"PerplexityBot":["partial","*",2,0],"Perplexity-User":["partial","*",2,0],"Google-Extended":["partial","*",2,0],"Applebot-Extended":["partial","*",2,0],"CCBot":["partial","*",2,0],"Bytespider":["partial","*",2,0],"Amazonbot":["partial","*",2,0],"meta-externalagent":["partial","*",2,0],"cohere-ai":["partial","*",2,0],"Diffbot":["partial","*",2,0],"ContextFetchBot":["partial","*",2,0]},"llms":{"status":"http_error","http":403,"bytes":5844},"home":{"status":"http_error","http":403,"url":"https://linklist.bio/"}},
{"rank":2159,"domain":"nzz.ch","origin":"https://nzz.ch","robots":{"status":"ok","http":200,"url":"https://www.nzz.ch/robots.txt","bytes":4785,"groups":47},"bots":{"GPTBot":["partial","named",1,2],"ChatGPT-User":["partial","named",1,2],"OAI-SearchBot":["partial","named",1,2],"ClaudeBot":["partial","named",1,2],"Claude-User":["partial","*",24,0],"Claude-SearchBot":["partial","*",24,0],"anthropic-ai":["partial","named",1,2],"PerplexityBot":["partial","named",1,2],"Perplexity-User":["partial","named",1,2],"Google-Extended":["partial","named",1,2],"Applebot-Extended":["partial","named",1,2],"CCBot":["partial","named",1,2],"Bytespider":["partial","named",1,2],"Amazonbot":["partial","named",1,2],"meta-externalagent":["partial","named",2,4],"cohere-ai":["partial","named",1,2],"Diffbot":["partial","named",1,2],"ContextFetchBot":["partial","*",24,0]},"llms":{"status":"missing","http":404,"bytes":185453},"home":{"status":"ok","http":200,"url":"https://www.nzz.ch/","html_tokens":282934,"markdown_tokens":713,"pruned_tokens":143}},
{"rank":2160,"domain":"fortunebusinessinsights.com","origin":"https://fortunebusinessinsights.com","robots":{"status":"refused","http":403,"url":"https://fortunebusinessinsights.com/robots.txt","bytes":5458,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":2161,"domain":"hse.gov.uk","origin":"https://hse.gov.uk","robots":{"status":"ok","http":200,"url":"https://www.hse.gov.uk/robots.txt","bytes":975,"groups":1},"bots":{"GPTBot":["partial","*",19,0],"ChatGPT-User":["partial","*",19,0],"OAI-SearchBot":["partial","*",19,0],"ClaudeBot":["partial","*",19,0],"Claude-User":["partial","*",19,0],"Claude-SearchBot":["partial","*",19,0],"anthropic-ai":["partial","*",19,0],"PerplexityBot":["partial","*",19,0],"Perplexity-User":["partial","*",19,0],"Google-Extended":["partial","*",19,0],"Applebot-Extended":["partial","*",19,0],"CCBot":["partial","*",19,0],"Bytespider":["partial","*",19,0],"Amazonbot":["partial","*",19,0],"meta-externalagent":["partial","*",19,0],"cohere-ai":["partial","*",19,0],"Diffbot":["partial","*",19,0],"ContextFetchBot":["partial","*",19,0]},"llms":{"status":"missing","http":404,"bytes":35210},"home":{"status":"ok","http":200,"url":"https://www.hse.gov.uk/","html_tokens":11814,"markdown_tokens":111,"pruned_tokens":57}},
{"rank":2162,"domain":"ub.edu","origin":"https://ub.edu","robots":{"status":"ok","http":200,"url":"https://www.ub.edu/robots.txt","bytes":757,"groups":3},"bots":{"GPTBot":["partial","*",12,0],"ChatGPT-User":["partial","*",12,0],"OAI-SearchBot":["partial","*",12,0],"ClaudeBot":["partial","*",12,0],"Claude-User":["partial","*",12,0],"Claude-SearchBot":["partial","*",12,0],"anthropic-ai":["partial","*",12,0],"PerplexityBot":["partial","*",12,0],"Perplexity-User":["partial","*",12,0],"Google-Extended":["partial","*",12,0],"Applebot-Extended":["partial","*",12,0],"CCBot":["partial","*",12,0],"Bytespider":["partial","*",12,0],"Amazonbot":["partial","*",12,0],"meta-externalagent":["partial","*",12,0],"cohere-ai":["partial","*",12,0],"Diffbot":["partial","*",12,0],"ContextFetchBot":["partial","*",12,0]},"llms":{"status":"missing","http":404,"bytes":42114},"home":{"status":"http_error","http":403,"url":"https://web.ub.edu/web/ub/"}},
{"rank":2163,"domain":"msu.ru","origin":"https://msu.ru","robots":{"status":"ok","http":200,"url":"https://msu.ru/robots.txt","bytes":330,"groups":5},"bots":{"GPTBot":["partial","*",5,0],"ChatGPT-User":["partial","*",5,0],"OAI-SearchBot":["partial","*",5,0],"ClaudeBot":["partial","*",5,0],"Claude-User":["partial","*",5,0],"Claude-SearchBot":["partial","*",5,0],"anthropic-ai":["partial","*",5,0],"PerplexityBot":["partial","*",5,0],"Perplexity-User":["partial","*",5,0],"Google-Extended":["partial","*",5,0],"Applebot-Extended":["partial","*",5,0],"CCBot":["partial","*",5,0],"Bytespider":["partial","*",5,0],"Amazonbot":["partial","*",5,0],"meta-externalagent":["partial","*",5,0],"cohere-ai":["partial","*",5,0],"Diffbot":["partial","*",5,0],"ContextFetchBot":["partial","*",5,0]},"llms":{"status":"missing","http":404,"bytes":2209},"home":{"status":"ok","http":200,"url":"https://msu.ru/","html_tokens":973,"markdown_tokens":0,"pruned_tokens":0}},
{"rank":2164,"domain":"ontario.ca","origin":"https://ontario.ca","robots":{"status":"ok","http":200,"url":"https://www.ontario.ca/robots.txt","bytes":769,"groups":1},"bots":{"GPTBot":["partial","*",13,18],"ChatGPT-User":["partial","*",13,18],"OAI-SearchBot":["partial","*",13,18],"ClaudeBot":["partial","*",13,18],"Claude-User":["partial","*",13,18],"Claude-SearchBot":["partial","*",13,18],"anthropic-ai":["partial","*",13,18],"PerplexityBot":["partial","*",13,18],"Perplexity-User":["partial","*",13,18],"Google-Extended":["partial","*",13,18],"Applebot-Extended":["partial","*",13,18],"CCBot":["partial","*",13,18],"Bytespider":["partial","*",13,18],"Amazonbot":["partial","*",13,18],"meta-externalagent":["partial","*",13,18],"cohere-ai":["partial","*",13,18],"Diffbot":["partial","*",13,18],"ContextFetchBot":["partial","*",13,18]},"llms":{"status":"missing","http":404,"bytes":162},"home":{"status":"ok","http":200,"url":"https://www.ontario.ca/","html_tokens":22346,"markdown_tokens":24,"pruned_tokens":24}},
{"rank":2165,"domain":"health.com","origin":"https://health.com","robots":{"status":"ok","http":200,"url":"https://www.health.com/robots.txt","bytes":3465,"groups":6},"bots":{"GPTBot":["partial","named",1,0],"ChatGPT-User":["partial","named",1,0],"OAI-SearchBot":["partial","named",1,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["blocked","named",1,0],"Claude-SearchBot":["partial","named",1,2],"anthropic-ai":["partial","named",1,2],"PerplexityBot":["partial","named",1,2],"Perplexity-User":["blocked","named",1,0],"Google-Extended":["partial","named",1,2],"Applebot-Extended":["blocked","named",1,0],"CCBot":["partial","named",1,2],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["partial","named",1,2],"cohere-ai":["partial","named",1,2],"Diffbot":["partial","*",2,0],"ContextFetchBot":["partial","*",2,0]},"llms":{"status":"http_error","http":402,"bytes":612},"home":{"status":"http_error","http":402,"url":"https://www.health.com/"}},
{"rank":2166,"domain":"real.com","origin":"https://real.com","robots":{"status":"ok","http":200,"url":"https://www.real.com/robots.txt","bytes":782,"groups":1},"bots":{"GPTBot":["partial","*",24,1],"ChatGPT-User":["partial","*",24,1],"OAI-SearchBot":["partial","*",24,1],"ClaudeBot":["partial","*",24,1],"Claude-User":["partial","*",24,1],"Claude-SearchBot":["partial","*",24,1],"anthropic-ai":["partial","*",24,1],"PerplexityBot":["partial","*",24,1],"Perplexity-User":["partial","*",24,1],"Google-Extended":["partial","*",24,1],"Applebot-Extended":["partial","*",24,1],"CCBot":["partial","*",24,1],"Bytespider":["partial","*",24,1],"Amazonbot":["partial","*",24,1],"meta-externalagent":["partial","*",24,1],"cohere-ai":["partial","*",24,1],"Diffbot":["partial","*",24,1],"ContextFetchBot":["partial","*",24,1]},"llms":{"status":"present","http":200,"bytes":1747},"home":{"status":"ok","http":200,"url":"https://www.real.com/","html_tokens":26521,"markdown_tokens":168,"pruned_tokens":168}},
{"rank":2167,"domain":"nme.com","origin":"https://nme.com","robots":{"status":"ok","http":200,"url":"https://www.nme.com/robots.txt","bytes":2396,"groups":23},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["blocked","named",1,0],"OAI-SearchBot":["partial","*",27,1],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",27,1],"Claude-SearchBot":["partial","*",27,1],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["partial","*",27,1],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["partial","*",27,1],"cohere-ai":["blocked","named",1,0],"Diffbot":["partial","*",27,1],"ContextFetchBot":["partial","*",27,1]},"llms":{"status":"missing","http":404,"bytes":26606},"home":{"status":"ok","http":200,"url":"https://www.nme.com/","html_tokens":99561,"markdown_tokens":38,"pruned_tokens":38}},
{"rank":2168,"domain":"grafana.com","origin":"https://grafana.com","robots":{"status":"ok","http":200,"url":"https://grafana.com/robots.txt","bytes":84,"groups":1},"bots":{"GPTBot":["partial","*",1,1],"ChatGPT-User":["partial","*",1,1],"OAI-SearchBot":["partial","*",1,1],"ClaudeBot":["partial","*",1,1],"Claude-User":["partial","*",1,1],"Claude-SearchBot":["partial","*",1,1],"anthropic-ai":["partial","*",1,1],"PerplexityBot":["partial","*",1,1],"Perplexity-User":["partial","*",1,1],"Google-Extended":["partial","*",1,1],"Applebot-Extended":["partial","*",1,1],"CCBot":["partial","*",1,1],"Bytespider":["partial","*",1,1],"Amazonbot":["partial","*",1,1],"meta-externalagent":["partial","*",1,1],"cohere-ai":["partial","*",1,1],"Diffbot":["partial","*",1,1],"ContextFetchBot":["partial","*",1,1]},"llms":{"status":"missing","http":404,"bytes":19},"home":{"status":"ok","http":200,"url":"https://grafana.com/","html_tokens":191954,"markdown_tokens":118,"pruned_tokens":55}},
{"rank":2169,"domain":"51.la","origin":"https://51.la","robots":{"status":"html","http":200,"url":"https://51.la/robots.txt","bytes":1501,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":2170,"domain":"bleacherreport.com","origin":"https://bleacherreport.com","robots":{"status":"ok","http":200,"url":"https://bleacherreport.com/robots.txt","bytes":1175,"groups":6},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["partial","*",32,0],"OAI-SearchBot":["partial","*",32,0],"ClaudeBot":["partial","*",32,0],"Claude-User":["partial","*",32,0],"Claude-SearchBot":["partial","*",32,0],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["partial","*",32,0],"Perplexity-User":["partial","*",32,0],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["partial","*",32,0],"CCBot":["blocked","named",1,0],"Bytespider":["partial","*",32,0],"Amazonbot":["partial","*",32,0],"meta-externalagent":["partial","*",32,0],"cohere-ai":["partial","*",32,0],"Diffbot":["partial","*",32,0],"ContextFetchBot":["partial","*",32,0]},"llms":{"status":"missing","http":404,"bytes":736137},"home":{"status":"ok","http":200,"url":"https://bleacherreport.com/","html_tokens":704747,"markdown_tokens":7,"pruned_tokens":7}},
{"rank":2171,"domain":"yna.co.kr","origin":"https://yna.co.kr","robots":{"status":"ok","http":200,"url":"https://www.yna.co.kr/robots.txt","bytes":5332,"groups":34},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["partial","named",1,2],"OAI-SearchBot":["partial","named",1,2],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",55,1],"Claude-SearchBot":["partial","*",55,1],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["partial","*",55,1],"Google-Extended":["partial","named",1,2],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",55,1]},"llms":{"status":"missing","http":404,"bytes":78101},"home":{"status":"ok","http":200,"url":"https://www.yna.co.kr/","html_tokens":43374,"markdown_tokens":59,"pruned_tokens":59}},
{"rank":2172,"domain":"bayer.com","origin":"https://bayer.com","robots":{"status":"refused","http":403,"url":"https://www.bayer.com/robots.txt","bytes":9668,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":2173,"domain":"main.jp","origin":"http://main.jp","robots":{"status":"missing","http":404,"url":"http://err.lolipop.jp/404.html","bytes":19241,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":19241},"home":{"status":"http_error","http":404,"url":"http://err.lolipop.jp/404.html"}},
{"rank":2174,"domain":"ytimg.com","origin":"https://ytimg.com","robots":{"status":"unreachable","http":null,"url":"https://ytimg.com/robots.txt","bytes":0,"groups":0,"error":"cannot resolve ytimg.com"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":2175,"domain":"economictimes.com","origin":"https://economictimes.com","robots":{"status":"ok","http":200,"url":"https://economictimes.indiatimes.com/robots.txt","bytes":6944,"groups":7},"bots":{"GPTBot":["partial","*",101,1],"ChatGPT-User":["partial","*",101,1],"OAI-SearchBot":["partial","*",101,1],"ClaudeBot":["partial","*",101,1],"Claude-User":["partial","*",101,1],"Claude-SearchBot":["partial","*",101,1],"anthropic-ai":["partial","*",101,1],"PerplexityBot":["partial","*",101,1],"Perplexity-User":["partial","*",101,1],"Google-Extended":["partial","*",101,1],"Applebot-Extended":["partial","*",101,1],"CCBot":["partial","*",101,1],"Bytespider":["partial","*",101,1],"Amazonbot":["partial","*",101,1],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["partial","*",101,1],"Diffbot":["partial","*",101,1],"ContextFetchBot":["partial","*",101,1]},"llms":{"status":"missing","http":404,"bytes":152994},"home":{"status":"ok","http":200,"url":"https://economictimes.indiatimes.com/","html_tokens":330676,"markdown_tokens":894,"pruned_tokens":156}},
{"rank":2176,"domain":"weibo.cn","origin":"https://weibo.cn","robots":{"status":"ok","http":200,"url":"https://weibo.cn/robots.txt","bytes":95,"groups":1},"bots":{"GPTBot":["partial","*",3,1],"ChatGPT-User":["partial","*",3,1],"OAI-SearchBot":["partial","*",3,1],"ClaudeBot":["partial","*",3,1],"Claude-User":["partial","*",3,1],"Claude-SearchBot":["partial","*",3,1],"anthropic-ai":["partial","*",3,1],"PerplexityBot":["partial","*",3,1],"Perplexity-User":["partial","*",3,1],"Google-Extended":["partial","*",3,1],"Applebot-Extended":["partial","*",3,1],"CCBot":["partial","*",3,1],"Bytespider":["partial","*",3,1],"Amazonbot":["partial","*",3,1],"meta-externalagent":["partial","*",3,1],"cohere-ai":["partial","*",3,1],"Diffbot":["partial","*",3,1],"ContextFetchBot":["partial","*",3,1]},"llms":{"status":"disallowed"},"home":{"status":"disallowed"}},
{"rank":2177,"domain":"getty.edu","origin":"https://getty.edu","robots":{"status":"ok","http":200,"url":"https://www.getty.edu/robots.txt","bytes":251,"groups":6},"bots":{"GPTBot":["partial","*",3,0],"ChatGPT-User":["partial","*",3,0],"OAI-SearchBot":["partial","*",3,0],"ClaudeBot":["partial","*",3,0],"Claude-User":["partial","*",3,0],"Claude-SearchBot":["partial","*",3,0],"anthropic-ai":["partial","*",3,0],"PerplexityBot":["partial","*",3,0],"Perplexity-User":["partial","*",3,0],"Google-Extended":["partial","*",3,0],"Applebot-Extended":["partial","*",3,0],"CCBot":["partial","*",3,0],"Bytespider":["partial","*",3,0],"Amazonbot":["partial","*",3,0],"meta-externalagent":["partial","*",3,0],"cohere-ai":["partial","*",3,0],"Diffbot":["partial","*",3,0],"ContextFetchBot":["partial","*",3,0]},"llms":{"status":"missing","http":404,"bytes":6402},"home":{"status":"ok","http":200,"url":"https://www.getty.edu/","html_tokens":770936,"markdown_tokens":223,"pruned_tokens":69}},
{"rank":2178,"domain":"site123.me","origin":"https://site123.me","robots":{"status":"html","http":200,"url":"https://www.site123.com","bytes":104538,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":2179,"domain":"pa.us","origin":"https://pa.us","robots":{"status":"unreachable","http":null,"url":"https://pa.us/robots.txt","bytes":0,"groups":0,"error":"cannot resolve pa.us"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":2180,"domain":"hu-berlin.de","origin":"https://hu-berlin.de","robots":{"status":"ok","http":200,"url":"https://www.hu-berlin.de/robots.txt","bytes":78,"groups":1},"bots":{"GPTBot":["partial","*",1,0],"ChatGPT-User":["partial","*",1,0],"OAI-SearchBot":["partial","*",1,0],"ClaudeBot":["partial","*",1,0],"Claude-User":["partial","*",1,0],"Claude-SearchBot":["partial","*",1,0],"anthropic-ai":["partial","*",1,0],"PerplexityBot":["partial","*",1,0],"Perplexity-User":["partial","*",1,0],"Google-Extended":["partial","*",1,0],"Applebot-Extended":["partial","*",1,0],"CCBot":["partial","*",1,0],"Bytespider":["partial","*",1,0],"Amazonbot":["partial","*",1,0],"meta-externalagent":["partial","*",1,0],"cohere-ai":["partial","*",1,0],"Diffbot":["partial","*",1,0],"ContextFetchBot":["partial","*",1,0]},"llms":{"status":"html","http":200,"bytes":7771},"home":{"status":"ok","http":200,"url":"https://www.hu-berlin.de/","html_tokens":1940,"markdown_tokens":269,"pruned_tokens":153}},
{"rank":2181,"domain":"aruba.it","origin":"https://aruba.it","robots":{"status":"ok","http":200,"url":"https://www.aruba.it/robots.txt","bytes":616,"groups":1},"bots":{"GPTBot":["partial","*",16,0],"ChatGPT-User":["partial","*",16,0],"OAI-SearchBot":["partial","*",16,0],"ClaudeBot":["partial","*",16,0],"Claude-User":["partial","*",16,0],"Claude-SearchBot":["partial","*",16,0],"anthropic-ai":["partial","*",16,0],"PerplexityBot":["partial","*",16,0],"Perplexity-User":["partial","*",16,0],"Google-Extended":["partial","*",16,0],"Applebot-Extended":["partial","*",16,0],"CCBot":["partial","*",16,0],"Bytespider":["partial","*",16,0],"Amazonbot":["partial","*",16,0],"meta-externalagent":["partial","*",16,0],"cohere-ai":["partial","*",16,0],"Diffbot":["partial","*",16,0],"ContextFetchBot":["partial","*",16,0]},"llms":{"status":"missing","http":404,"bytes":114636},"home":{"status":"ok","http":200,"url":"https://www.aruba.it/en/home.aspx","html_tokens":47188,"markdown_tokens":1140,"pruned_tokens":240}},
{"rank":2182,"domain":"cpanel.com","origin":"https://cpanel.com","robots":{"status":"ok","http":200,"url":"https://www.cpanel.net/robots.txt","bytes":297,"groups":1},"bots":{"GPTBot":["partial","*",7,0],"ChatGPT-User":["partial","*",7,0],"OAI-SearchBot":["partial","*",7,0],"ClaudeBot":["partial","*",7,0],"Claude-User":["partial","*",7,0],"Claude-SearchBot":["partial","*",7,0],"anthropic-ai":["partial","*",7,0],"PerplexityBot":["partial","*",7,0],"Perplexity-User":["partial","*",7,0],"Google-Extended":["partial","*",7,0],"Applebot-Extended":["partial","*",7,0],"CCBot":["partial","*",7,0],"Bytespider":["partial","*",7,0],"Amazonbot":["partial","*",7,0],"meta-externalagent":["partial","*",7,0],"cohere-ai":["partial","*",7,0],"Diffbot":["partial","*",7,0],"ContextFetchBot":["partial","*",7,0]},"llms":{"status":"missing","http":404,"bytes":107830},"home":{"status":"ok","http":200,"url":"https://www.cpanel.net/","html_tokens":56509,"markdown_tokens":272,"pruned_tokens":128}},
{"rank":2183,"domain":"theaustralian.com.au","origin":"https://theaustralian.com.au","robots":{"status":"ok","http":200,"url":"https://www.theaustralian.com.au/robots.txt","bytes":3282,"groups":4},"bots":{"GPTBot":["allowed","named",0,0],"ChatGPT-User":["allowed","named",0,0],"OAI-SearchBot":["allowed","named",0,0],"ClaudeBot":["partial","*",1,2],"Claude-User":["partial","*",1,2],"Claude-SearchBot":["partial","*",1,2],"anthropic-ai":["partial","*",1,2],"PerplexityBot":["partial","*",1,2],"Perplexity-User":["partial","*",1,2],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["partial","*",1,2],"CCBot":["partial","*",1,2],"Bytespider":["partial","*",1,2],"Amazonbot":["partial","*",1,2],"meta-externalagent":["partial","*",1,2],"cohere-ai":["partial","*",1,2],"Diffbot":["partial","*",1,2],"ContextFetchBot":["partial","*",1,2]},"llms":{"status":"disallowed"},"home":{"status":"disallowed"}},
{"rank":2184,"domain":"nla.gov.au","origin":"https://nla.gov.au","robots":{"status":"ok","http":200,"url":"https://www.nla.gov.au/robots.txt","bytes":812,"groups":4},"bots":{"GPTBot":["partial","*",1,0],"ChatGPT-User":["partial","*",1,0],"OAI-SearchBot":["partial","*",1,0],"ClaudeBot":["partial","*",1,0],"Claude-User":["partial","*",1,0],"Claude-SearchBot":["partial","*",1,0],"anthropic-ai":["partial","*",1,0],"PerplexityBot":["partial","*",1,0],"Perplexity-User":["partial","*",1,0],"Google-Extended":["partial","*",1,0],"Applebot-Extended":["partial","*",1,0],"CCBot":["partial","*",1,0],"Bytespider":["partial","*",1,0],"Amazonbot":["partial","*",1,0],"meta-externalagent":["partial","*",1,0],"cohere-ai":["partial","*",1,0],"Diffbot":["partial","*",1,0],"ContextFetchBot":["partial","*",1,0]},"llms":{"status":"missing","http":404,"bytes":469583},"home":{"status":"ok","http":200,"url":"https://www.library.gov.au","html_tokens":85773,"markdown_tokens":316,"pruned_tokens":72}},
{"rank":2185,"domain":"global.fujitsu","origin":"https://global.fujitsu","robots":{"status":"refused","http":429,"url":"https://global.fujitsu/robots.txt","bytes":34227,"groups":0,"error":"HTTP 429"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":2186,"domain":"york.ac.uk","origin":"https://york.ac.uk","robots":{"status":"ok","http":200,"url":"https://www.york.ac.uk/robots.txt","bytes":13306,"groups":3},"bots":{"GPTBot":["partial","*",344,0],"ChatGPT-User":["partial","*",344,0],"OAI-SearchBot":["partial","*",344,0],"ClaudeBot":["partial","*",344,0],"Claude-User":["partial","*",344,0],"Claude-SearchBot":["partial","*",344,0],"anthropic-ai":["partial","*",344,0],"PerplexityBot":["partial","*",344,0],"Perplexity-User":["partial","*",344,0],"Google-Extended":["partial","*",344,0],"Applebot-Extended":["partial","*",344,0],"CCBot":["partial","*",344,0],"Bytespider":["partial","*",344,0],"Amazonbot":["partial","*",344,0],"meta-externalagent":["partial","*",344,0],"cohere-ai":["partial","*",344,0],"Diffbot":["partial","*",344,0],"ContextFetchBot":["partial","*",344,0]},"llms":{"status":"missing","http":404,"bytes":13074},"home":{"status":"ok","http":200,"url":"https://www.york.ac.uk/","html_tokens":16487,"markdown_tokens":150,"pruned_tokens":44}},
{"rank":2187,"domain":"research.google","origin":"https://research.google","robots":{"status":"ok","http":200,"url":"https://research.google/robots.txt","bytes":68,"groups":1},"bots":{"GPTBot":["allowed","*",0,1],"ChatGPT-User":["allowed","*",0,1],"OAI-SearchBot":["allowed","*",0,1],"ClaudeBot":["allowed","*",0,1],"Claude-User":["allowed","*",0,1],"Claude-SearchBot":["allowed","*",0,1],"anthropic-ai":["allowed","*",0,1],"PerplexityBot":["allowed","*",0,1],"Perplexity-User":["allowed","*",0,1],"Google-Extended":["allowed","*",0,1],"Applebot-Extended":["allowed","*",0,1],"CCBot":["allowed","*",0,1],"Bytespider":["allowed","*",0,1],"Amazonbot":["allowed","*",0,1],"meta-externalagent":["allowed","*",0,1],"cohere-ai":["allowed","*",0,1],"Diffbot":["allowed","*",0,1],"ContextFetchBot":["allowed","*",0,1]},"llms":{"status":"missing","http":404,"bytes":134819},"home":{"status":"ok","http":200,"url":"https://research.google/","html_tokens":64107,"markdown_tokens":268,"pruned_tokens":146}},
{"rank":2188,"domain":"cafepress.com","origin":"https://cafepress.com","robots":{"status":"ok","http":200,"url":"https://cafepress.com/robots.txt","bytes":563,"groups":8},"bots":{"GPTBot":["partial","*",7,0],"ChatGPT-User":["partial","*",7,0],"OAI-SearchBot":["partial","*",7,0],"ClaudeBot":["partial","*",7,0],"Claude-User":["partial","*",7,0],"Claude-SearchBot":["partial","*",7,0],"anthropic-ai":["partial","*",7,0],"PerplexityBot":["partial","*",7,0],"Perplexity-User":["partial","*",7,0],"Google-Extended":["partial","*",7,0],"Applebot-Extended":["partial","*",7,0],"CCBot":["partial","*",7,0],"Bytespider":["partial","*",7,0],"Amazonbot":["partial","*",7,0],"meta-externalagent":["partial","*",7,0],"cohere-ai":["partial","*",7,0],"Diffbot":["partial","*",7,0],"ContextFetchBot":["partial","*",7,0]},"llms":{"status":"missing","http":404,"bytes":13},"home":{"status":"ok","http":200,"url":"https://www.cafepress.com/","html_tokens":115653,"markdown_tokens":89,"pruned_tokens":89}},
{"rank":2189,"domain":"raspberrypi.com","origin":"https://raspberrypi.com","robots":{"status":"ok","http":200,"url":"https://www.raspberrypi.com/robots.txt","bytes":355,"groups":2},"bots":{"GPTBot":["partial","*",4,1],"ChatGPT-User":["partial","*",4,1],"OAI-SearchBot":["partial","*",4,1],"ClaudeBot":["partial","*",4,1],"Claude-User":["partial","*",4,1],"Claude-SearchBot":["partial","*",4,1],"anthropic-ai":["partial","*",4,1],"PerplexityBot":["partial","*",4,1],"Perplexity-User":["partial","*",4,1],"Google-Extended":["partial","*",4,1],"Applebot-Extended":["partial","*",4,1],"CCBot":["partial","*",4,1],"Bytespider":["partial","*",4,1],"Amazonbot":["partial","*",4,1],"meta-externalagent":["partial","*",4,1],"cohere-ai":["partial","*",4,1],"Diffbot":["partial","*",4,1],"ContextFetchBot":["partial","*",4,1]},"llms":{"status":"missing","http":404,"bytes":68920},"home":{"status":"http_error","http":403,"url":"https://raspberrypi.com/"}},
{"rank":2190,"domain":"experian.com","origin":"https://experian.com","robots":{"status":"ok","http":200,"url":"https://www.experian.com/robots.txt","bytes":1246,"groups":2},"bots":{"GPTBot":["partial","*",14,1],"ChatGPT-User":["partial","*",14,1],"OAI-SearchBot":["partial","*",14,1],"ClaudeBot":["partial","*",14,1],"Claude-User":["partial","*",14,1],"Claude-SearchBot":["partial","*",14,1],"anthropic-ai":["partial","*",14,1],"PerplexityBot":["partial","*",14,1],"Perplexity-User":["partial","*",14,1],"Google-Extended":["partial","*",14,1],"Applebot-Extended":["partial","*",14,1],"CCBot":["partial","*",14,1],"Bytespider":["partial","*",14,1],"Amazonbot":["partial","*",14,1],"meta-externalagent":["partial","*",14,1],"cohere-ai":["partial","*",14,1],"Diffbot":["partial","*",14,1],"ContextFetchBot":["partial","*",14,1]},"llms":{"status":"missing","http":404,"bytes":1819},"home":{"status":"ok","http":200,"url":"https://www.experian.com/","html_tokens":74789,"markdown_tokens":54,"pruned_tokens":54}},
{"rank":2191,"domain":"wakelet.com","origin":"https://wakelet.com","robots":{"status":"ok","http":200,"url":"https://wakelet.com/robots.txt","bytes":41,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"present","http":200,"bytes":10339},"home":{"status":"ok","http":200,"url":"https://wakelet.com/","html_tokens":13413,"markdown_tokens":119,"pruned_tokens":77}},
{"rank":2192,"domain":"rebrand.ly","origin":"https://rebrand.ly","robots":{"status":"ok","http":200,"url":"https://rebrand.ly/robots.txt?rb.routing.mode=proxy&rb.routing.signature=628996","bytes":52,"groups":1},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"disallowed","error":"robots.txt of app.rebrandly.com not readable (html)"},"home":{"status":"http_error","http":403,"url":"https://rebrand.ly/"}},
{"rank":2193,"domain":"postman.com","origin":"https://postman.com","robots":{"status":"ok","http":200,"url":"https://postman.com/robots.txt","bytes":3118,"groups":2},"bots":{"GPTBot":["partial","named",5,1],"ChatGPT-User":["partial","named",5,1],"OAI-SearchBot":["partial","named",5,1],"ClaudeBot":["partial","named",5,1],"Claude-User":["partial","named",5,1],"Claude-SearchBot":["partial","named",5,1],"anthropic-ai":["partial","named",5,1],"PerplexityBot":["partial","named",5,1],"Perplexity-User":["partial","named",5,1],"Google-Extended":["partial","named",5,1],"Applebot-Extended":["partial","named",5,1],"CCBot":["partial","named",5,1],"Bytespider":["blocked","named",1,0],"Amazonbot":["partial","named",5,1],"meta-externalagent":["partial","named",5,1],"cohere-ai":["partial","named",5,1],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",5,1]},"llms":{"status":"present","http":200,"bytes":21271},"home":{"status":"ok","http":200,"url":"https://www.postman.com/","html_tokens":125795,"markdown_tokens":337,"pruned_tokens":101}},
{"rank":2194,"domain":"jqueryui.com","origin":"https://jqueryui.com","robots":{"status":"ok","http":200,"url":"https://jqueryui.com/robots.txt","bytes":113,"groups":1},"bots":{"GPTBot":["partial","*",1,1],"ChatGPT-User":["partial","*",1,1],"OAI-SearchBot":["partial","*",1,1],"ClaudeBot":["partial","*",1,1],"Claude-User":["partial","*",1,1],"Claude-SearchBot":["partial","*",1,1],"anthropic-ai":["partial","*",1,1],"PerplexityBot":["partial","*",1,1],"Perplexity-User":["partial","*",1,1],"Google-Extended":["partial","*",1,1],"Applebot-Extended":["partial","*",1,1],"CCBot":["partial","*",1,1],"Bytespider":["partial","*",1,1],"Amazonbot":["partial","*",1,1],"meta-externalagent":["partial","*",1,1],"cohere-ai":["partial","*",1,1],"Diffbot":["partial","*",1,1],"ContextFetchBot":["partial","*",1,1]},"llms":{"status":"missing","http":404,"bytes":19153},"home":{"status":"ok","http":200,"url":"https://jqueryui.com/","html_tokens":6598,"markdown_tokens":357,"pruned_tokens":130}},
{"rank":2195,"domain":"washingtonexaminer.com","origin":"https://washingtonexaminer.com","robots":{"status":"ok","http":200,"url":"https://www.washingtonexaminer.com/robots.txt","bytes":645,"groups":13},"bots":{"GPTBot":["partial","*",5,1],"ChatGPT-User":["partial","*",5,1],"OAI-SearchBot":["partial","*",5,1],"ClaudeBot":["partial","*",5,1],"Claude-User":["partial","*",5,1],"Claude-SearchBot":["partial","*",5,1],"anthropic-ai":["partial","*",5,1],"PerplexityBot":["partial","*",5,1],"Perplexity-User":["partial","*",5,1],"Google-Extended":["partial","*",5,1],"Applebot-Extended":["partial","*",5,1],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["partial","*",5,1],"meta-externalagent":["partial","*",5,1],"cohere-ai":["partial","*",5,1],"Diffbot":["partial","*",5,1],"ContextFetchBot":["partial","*",5,1]},"llms":{"status":"missing","http":404,"bytes":88153},"home":{"status":"ok","http":200,"url":"https://www.washingtonexaminer.com/","html_tokens":69509,"markdown_tokens":27,"pruned_tokens":25}},
{"rank":2196,"domain":"gdpr-info.eu","origin":"https://gdpr-info.eu","robots":{"status":"ok","http":200,"url":"https://gdpr-info.eu/robots.txt","bytes":291,"groups":1},"bots":{"GPTBot":["partial","*",8,1],"ChatGPT-User":["partial","*",8,1],"OAI-SearchBot":["partial","*",8,1],"ClaudeBot":["partial","*",8,1],"Claude-User":["partial","*",8,1],"Claude-SearchBot":["partial","*",8,1],"anthropic-ai":["partial","*",8,1],"PerplexityBot":["partial","*",8,1],"Perplexity-User":["partial","*",8,1],"Google-Extended":["partial","*",8,1],"Applebot-Extended":["partial","*",8,1],"CCBot":["partial","*",8,1],"Bytespider":["partial","*",8,1],"Amazonbot":["partial","*",8,1],"meta-externalagent":["partial","*",8,1],"cohere-ai":["partial","*",8,1],"Diffbot":["partial","*",8,1],"ContextFetchBot":["partial","*",8,1]},"llms":{"status":"missing","http":404,"bytes":78740},"home":{"status":"ok","http":200,"url":"https://gdpr-info.eu/","html_tokens":30030,"markdown_tokens":277,"pruned_tokens":129}},
{"rank":2197,"domain":"elespanol.com","origin":"https://elespanol.com","robots":{"status":"ok","http":200,"url":"https://www.elespanol.com/robots.txt","bytes":4400,"groups":4},"bots":{"GPTBot":["partial","*",17,0],"ChatGPT-User":["partial","*",17,0],"OAI-SearchBot":["partial","*",17,0],"ClaudeBot":["partial","*",17,0],"Claude-User":["partial","*",17,0],"Claude-SearchBot":["partial","*",17,0],"anthropic-ai":["partial","*",17,0],"PerplexityBot":["partial","*",17,0],"Perplexity-User":["partial","*",17,0],"Google-Extended":["partial","*",17,0],"Applebot-Extended":["partial","*",17,0],"CCBot":["partial","*",17,0],"Bytespider":["partial","*",17,0],"Amazonbot":["partial","*",17,0],"meta-externalagent":["partial","*",17,0],"cohere-ai":["partial","*",17,0],"Diffbot":["partial","*",17,0],"ContextFetchBot":["partial","*",17,0]},"llms":{"status":"present","http":200,"bytes":6412},"home":{"status":"ok","http":200,"url":"https://www.elespanol.com/","html_tokens":167910,"markdown_tokens":90,"pruned_tokens":78}},
{"rank":2198,"domain":"modelcontextprotocol.io","origin":"https://modelcontextprotocol.io","robots":{"status":"ok","http":200,"url":"https://modelcontextprotocol.io/robots.txt","bytes":179,"groups":1},"bots":{"GPTBot":["partial","*",2,1],"ChatGPT-User":["partial","*",2,1],"OAI-SearchBot":["partial","*",2,1],"ClaudeBot":["partial","*",2,1],"Claude-User":["partial","*",2,1],"Claude-SearchBot":["partial","*",2,1],"anthropic-ai":["partial","*",2,1],"PerplexityBot":["partial","*",2,1],"Perplexity-User":["partial","*",2,1],"Google-Extended":["partial","*",2,1],"Applebot-Extended":["partial","*",2,1],"CCBot":["partial","*",2,1],"Bytespider":["partial","*",2,1],"Amazonbot":["partial","*",2,1],"meta-externalagent":["partial","*",2,1],"cohere-ai":["partial","*",2,1],"Diffbot":["partial","*",2,1],"ContextFetchBot":["partial","*",2,1]},"llms":{"status":"present","http":200,"bytes":50288},"home":{"status":"ok","http":200,"url":"https://modelcontextprotocol.io/docs/2026-07-28/getting-started/intro","html_tokens":78663,"markdown_tokens":570,"pruned_tokens":141}},
{"rank":2199,"domain":"military.com","origin":"https://military.com","robots":{"status":"ok","http":200,"url":"https://www.military.com/robots.txt","bytes":2782,"groups":5},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["blocked","named",1,0],"OAI-SearchBot":["partial","*",63,18],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",63,18],"Claude-SearchBot":["partial","*",63,18],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["partial","*",63,18],"Perplexity-User":["partial","*",63,18],"Google-Extended":["partial","*",63,18],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["partial","*",63,18],"meta-externalagent":["partial","*",63,18],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",63,18]},"llms":{"status":"missing","http":404,"bytes":121590},"home":{"status":"ok","http":200,"url":"https://www.military.com/","html_tokens":162044,"markdown_tokens":643,"pruned_tokens":74}},
{"rank":2200,"domain":"ssllabs.com","origin":"https://ssllabs.com","robots":{"status":"ok","http":200,"url":"https://www.ssllabs.com/robots.txt","bytes":115,"groups":1},"bots":{"GPTBot":["partial","*",4,0],"ChatGPT-User":["partial","*",4,0],"OAI-SearchBot":["partial","*",4,0],"ClaudeBot":["partial","*",4,0],"Claude-User":["partial","*",4,0],"Claude-SearchBot":["partial","*",4,0],"anthropic-ai":["partial","*",4,0],"PerplexityBot":["partial","*",4,0],"Perplexity-User":["partial","*",4,0],"Google-Extended":["partial","*",4,0],"Applebot-Extended":["partial","*",4,0],"CCBot":["partial","*",4,0],"Bytespider":["partial","*",4,0],"Amazonbot":["partial","*",4,0],"meta-externalagent":["partial","*",4,0],"cohere-ai":["partial","*",4,0],"Diffbot":["partial","*",4,0],"ContextFetchBot":["partial","*",4,0]},"llms":{"status":"missing","http":404,"bytes":515},"home":{"status":"ok","http":200,"url":"https://www.ssllabs.com/","html_tokens":1712,"markdown_tokens":276,"pruned_tokens":168}},
{"rank":2201,"domain":"x.ai","origin":"https://x.ai","robots":{"status":"ok","http":200,"url":"https://x.ai/robots.txt","bytes":225,"groups":1},"bots":{"GPTBot":["partial","*",1,1],"ChatGPT-User":["partial","*",1,1],"OAI-SearchBot":["partial","*",1,1],"ClaudeBot":["partial","*",1,1],"Claude-User":["partial","*",1,1],"Claude-SearchBot":["partial","*",1,1],"anthropic-ai":["partial","*",1,1],"PerplexityBot":["partial","*",1,1],"Perplexity-User":["partial","*",1,1],"Google-Extended":["partial","*",1,1],"Applebot-Extended":["partial","*",1,1],"CCBot":["partial","*",1,1],"Bytespider":["partial","*",1,1],"Amazonbot":["partial","*",1,1],"meta-externalagent":["partial","*",1,1],"cohere-ai":["partial","*",1,1],"Diffbot":["partial","*",1,1],"ContextFetchBot":["partial","*",1,1]},"llms":{"status":"missing","http":404,"bytes":79738},"home":{"status":"http_error","http":403,"url":"https://x.ai/"}},
{"rank":2202,"domain":"afp.com","origin":"https://www.afp.com","robots":{"status":"ok","http":200,"url":"https://www.afp.com/robots.txt","bytes":2027,"groups":1},"bots":{"GPTBot":["partial","*",34,18],"ChatGPT-User":["partial","*",34,18],"OAI-SearchBot":["partial","*",34,18],"ClaudeBot":["partial","*",34,18],"Claude-User":["partial","*",34,18],"Claude-SearchBot":["partial","*",34,18],"anthropic-ai":["partial","*",34,18],"PerplexityBot":["partial","*",34,18],"Perplexity-User":["partial","*",34,18],"Google-Extended":["partial","*",34,18],"Applebot-Extended":["partial","*",34,18],"CCBot":["partial","*",34,18],"Bytespider":["partial","*",34,18],"Amazonbot":["partial","*",34,18],"meta-externalagent":["partial","*",34,18],"cohere-ai":["partial","*",34,18],"Diffbot":["partial","*",34,18],"ContextFetchBot":["partial","*",34,18]},"llms":{"status":"present","http":200,"bytes":12},"home":{"status":"ok","http":200,"url":"https://www.afp.com/fr","html_tokens":35353,"markdown_tokens":61,"pruned_tokens":61}},
{"rank":2203,"domain":"helloasso.com","origin":"https://www.helloasso.com","robots":{"status":"ok","http":200,"url":"https://www.helloasso.com/robots.txt","bytes":4084,"groups":55},"bots":{"GPTBot":["partial","named",8,0],"ChatGPT-User":["partial","*",10,0],"OAI-SearchBot":["partial","*",10,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",10,0],"Claude-SearchBot":["partial","*",10,0],"anthropic-ai":["partial","*",10,0],"PerplexityBot":["partial","*",10,0],"Perplexity-User":["partial","*",10,0],"Google-Extended":["partial","*",10,0],"Applebot-Extended":["partial","*",10,0],"CCBot":["partial","*",10,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["partial","*",10,0],"meta-externalagent":["partial","*",10,0],"cohere-ai":["partial","*",10,0],"Diffbot":["partial","*",10,0],"ContextFetchBot":["partial","*",10,0]},"llms":{"status":"missing","http":404,"bytes":0},"home":{"status":"http_error","http":403,"url":"https://www.helloasso.com/"}},
{"rank":2204,"domain":"kobo.com","origin":"https://kobo.com","robots":{"status":"ok","http":200,"url":"https://www.kobo.com/robots.txt","bytes":3240,"groups":1},"bots":{"GPTBot":["partial","*",95,48],"ChatGPT-User":["partial","*",95,48],"OAI-SearchBot":["partial","*",95,48],"ClaudeBot":["partial","*",95,48],"Claude-User":["partial","*",95,48],"Claude-SearchBot":["partial","*",95,48],"anthropic-ai":["partial","*",95,48],"PerplexityBot":["partial","*",95,48],"Perplexity-User":["partial","*",95,48],"Google-Extended":["partial","*",95,48],"Applebot-Extended":["partial","*",95,48],"CCBot":["partial","*",95,48],"Bytespider":["partial","*",95,48],"Amazonbot":["partial","*",95,48],"meta-externalagent":["partial","*",95,48],"cohere-ai":["partial","*",95,48],"Diffbot":["partial","*",95,48],"ContextFetchBot":["partial","*",95,48]},"llms":{"status":"http_error","http":403,"bytes":84272},"home":{"status":"http_error","http":403,"url":"https://www.kobo.com/"}},
{"rank":2205,"domain":"ancestry.com","origin":"https://ancestry.com","robots":{"status":"ok","http":200,"url":"https://www.ancestry.com/robots.txt","bytes":3623,"groups":4},"bots":{"GPTBot":["partial","*",83,1],"ChatGPT-User":["partial","*",83,1],"OAI-SearchBot":["partial","*",83,1],"ClaudeBot":["partial","*",83,1],"Claude-User":["partial","*",83,1],"Claude-SearchBot":["partial","*",83,1],"anthropic-ai":["partial","*",83,1],"PerplexityBot":["partial","*",83,1],"Perplexity-User":["partial","*",83,1],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["partial","*",83,1],"CCBot":["partial","*",83,1],"Bytespider":["partial","*",83,1],"Amazonbot":["partial","*",83,1],"meta-externalagent":["partial","*",83,1],"cohere-ai":["partial","*",83,1],"Diffbot":["partial","*",83,1],"ContextFetchBot":["partial","*",83,1]},"llms":{"status":"missing","http":404,"bytes":53386},"home":{"status":"http_error","http":403,"url":"https://ancestry.com/"}},
{"rank":2206,"domain":"pearson.com","origin":"https://pearson.com","robots":{"status":"ok","http":200,"url":"https://www.pearson.com/robots.txt","bytes":5735,"groups":1},"bots":{"GPTBot":["partial","*",127,2],"ChatGPT-User":["partial","*",127,2],"OAI-SearchBot":["partial","*",127,2],"ClaudeBot":["partial","*",127,2],"Claude-User":["partial","*",127,2],"Claude-SearchBot":["partial","*",127,2],"anthropic-ai":["partial","*",127,2],"PerplexityBot":["partial","*",127,2],"Perplexity-User":["partial","*",127,2],"Google-Extended":["partial","*",127,2],"Applebot-Extended":["partial","*",127,2],"CCBot":["partial","*",127,2],"Bytespider":["partial","*",127,2],"Amazonbot":["partial","*",127,2],"meta-externalagent":["partial","*",127,2],"cohere-ai":["partial","*",127,2],"Diffbot":["partial","*",127,2],"ContextFetchBot":["partial","*",127,2]},"llms":{"status":"missing","http":404,"bytes":149575},"home":{"status":"ok","http":200,"url":"https://www.pearson.com/en-us","html_tokens":17144,"markdown_tokens":265,"pruned_tokens":172}},
{"rank":2207,"domain":"informer.com","origin":"https://informer.com","robots":{"status":"ok","http":200,"url":"https://www.informer.com/robots.txt","bytes":23,"groups":1},"bots":{"GPTBot":["allowed","*",0,1],"ChatGPT-User":["allowed","*",0,1],"OAI-SearchBot":["allowed","*",0,1],"ClaudeBot":["allowed","*",0,1],"Claude-User":["allowed","*",0,1],"Claude-SearchBot":["allowed","*",0,1],"anthropic-ai":["allowed","*",0,1],"PerplexityBot":["allowed","*",0,1],"Perplexity-User":["allowed","*",0,1],"Google-Extended":["allowed","*",0,1],"Applebot-Extended":["allowed","*",0,1],"CCBot":["allowed","*",0,1],"Bytespider":["allowed","*",0,1],"Amazonbot":["allowed","*",0,1],"meta-externalagent":["allowed","*",0,1],"cohere-ai":["allowed","*",0,1],"Diffbot":["allowed","*",0,1],"ContextFetchBot":["allowed","*",0,1]},"llms":{"status":"missing","http":404,"bytes":153},"home":{"status":"http_error","http":403,"url":"https://informer.com/"}},
{"rank":2208,"domain":"onamae.com","origin":"https://onamae.com","robots":{"status":"ok","http":200,"url":"https://www.onamae.com/robots.txt","bytes":1915,"groups":5},"bots":{"GPTBot":["allowed","named",0,1],"ChatGPT-User":["partial","*",42,0],"OAI-SearchBot":["allowed","named",0,1],"ClaudeBot":["partial","*",42,0],"Claude-User":["partial","*",42,0],"Claude-SearchBot":["partial","*",42,0],"anthropic-ai":["partial","*",42,0],"PerplexityBot":["allowed","named",0,1],"Perplexity-User":["partial","*",42,0],"Google-Extended":["partial","*",42,0],"Applebot-Extended":["partial","*",42,0],"CCBot":["partial","*",42,0],"Bytespider":["partial","*",42,0],"Amazonbot":["partial","*",42,0],"meta-externalagent":["partial","*",42,0],"cohere-ai":["partial","*",42,0],"Diffbot":["partial","*",42,0],"ContextFetchBot":["partial","*",42,0]},"llms":{"status":"present","http":200,"bytes":728345},"home":{"status":"http_error","http":403,"url":"https://onamae.com/"}},
{"rank":2209,"domain":"thunderbird.net","origin":"https://thunderbird.net","robots":{"status":"missing","http":404,"url":"https://www.thunderbird.net/robots.txt","bytes":157180,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":157180},"home":{"status":"http_error","http":403,"url":"https://thunderbird.net/"}},
{"rank":2210,"domain":"clickup.com","origin":"https://clickup.com","robots":{"status":"ok","http":200,"url":"https://clickup.com/robots.txt","bytes":1308,"groups":3},"bots":{"GPTBot":["partial","*",35,1],"ChatGPT-User":["partial","*",35,1],"OAI-SearchBot":["partial","*",35,1],"ClaudeBot":["partial","*",35,1],"Claude-User":["partial","*",35,1],"Claude-SearchBot":["partial","*",35,1],"anthropic-ai":["partial","*",35,1],"PerplexityBot":["partial","*",35,1],"Perplexity-User":["partial","*",35,1],"Google-Extended":["partial","*",35,1],"Applebot-Extended":["partial","*",35,1],"CCBot":["partial","*",35,1],"Bytespider":["partial","*",35,1],"Amazonbot":["partial","*",35,1],"meta-externalagent":["partial","*",35,1],"cohere-ai":["partial","*",35,1],"Diffbot":["partial","*",35,1],"ContextFetchBot":["partial","*",35,1]},"llms":{"status":"present","http":200,"bytes":8836},"home":{"status":"ok","http":200,"url":"https://clickup.com/","html_tokens":552853,"markdown_tokens":802,"pruned_tokens":233}},
{"rank":2211,"domain":"formula1.com","origin":"https://formula1.com","robots":{"status":"ok","http":200,"url":"https://www.formula1.com/robots.txt","bytes":97,"groups":1},"bots":{"GPTBot":["partial","*",1,1],"ChatGPT-User":["partial","*",1,1],"OAI-SearchBot":["partial","*",1,1],"ClaudeBot":["partial","*",1,1],"Claude-User":["partial","*",1,1],"Claude-SearchBot":["partial","*",1,1],"anthropic-ai":["partial","*",1,1],"PerplexityBot":["partial","*",1,1],"Perplexity-User":["partial","*",1,1],"Google-Extended":["partial","*",1,1],"Applebot-Extended":["partial","*",1,1],"CCBot":["partial","*",1,1],"Bytespider":["partial","*",1,1],"Amazonbot":["partial","*",1,1],"meta-externalagent":["partial","*",1,1],"cohere-ai":["partial","*",1,1],"Diffbot":["partial","*",1,1],"ContextFetchBot":["partial","*",1,1]},"llms":{"status":"missing","http":404,"bytes":16487},"home":{"status":"ok","http":200,"url":"https://www.formula1.com/","html_tokens":160645,"markdown_tokens":16,"pruned_tokens":16}},
{"rank":2212,"domain":"impress.co.jp","origin":"https://impress.co.jp","robots":{"status":"unreachable","http":null,"url":"https://impress.co.jp/robots.txt","bytes":0,"groups":0,"error":"[SSL: SSLV3_ALERT_HANDSHAKE_FAILURE] sslv3 alert handshake failure (_ssl.c:1028)"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":2213,"domain":"flightradar24.com","origin":"https://flightradar24.com","robots":{"status":"ok","http":200,"url":"https://www.flightradar24.com/robots.txt","bytes":617,"groups":5},"bots":{"GPTBot":["partial","named",1,2],"ChatGPT-User":["allowed","named",0,3],"OAI-SearchBot":["partial","*",11,0],"ClaudeBot":["partial","*",11,0],"Claude-User":["partial","*",11,0],"Claude-SearchBot":["partial","*",11,0],"anthropic-ai":["partial","*",11,0],"PerplexityBot":["partial","*",11,0],"Perplexity-User":["partial","*",11,0],"Google-Extended":["partial","*",11,0],"Applebot-Extended":["partial","*",11,0],"CCBot":["partial","named",1,2],"Bytespider":["partial","*",11,0],"Amazonbot":["partial","*",11,0],"meta-externalagent":["partial","*",11,0],"cohere-ai":["partial","*",11,0],"Diffbot":["partial","*",11,0],"ContextFetchBot":["partial","*",11,0]},"llms":{"status":"present","http":200,"bytes":2078},"home":{"status":"http_error","http":403,"url":"https://www.flightradar24.com/"}},
{"rank":2214,"domain":"thinkific.com","origin":"https://thinkific.com","robots":{"status":"refused","http":403,"url":"https://www.thinkific.com/robots.txt","bytes":5555,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":2215,"domain":"letterboxd.com","origin":"https://letterboxd.com","robots":{"status":"ok","http":200,"url":"https://letterboxd.com/robots.txt","bytes":3402,"groups":25},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["partial","*",12,0],"OAI-SearchBot":["partial","*",12,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",12,0],"Claude-SearchBot":["partial","*",12,0],"anthropic-ai":["partial","*",12,0],"PerplexityBot":["partial","*",12,0],"Perplexity-User":["partial","*",12,0],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["partial","*",12,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["partial","*",12,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",12,0]},"llms":{"status":"missing","http":404,"bytes":2140},"home":{"status":"http_error","http":403,"url":"https://letterboxd.com/"}},
{"rank":2216,"domain":"emojipedia.org","origin":"https://emojipedia.org","robots":{"status":"ok","http":200,"url":"https://emojipedia.org/robots.txt","bytes":588,"groups":11},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["allowed","*",0,1],"OAI-SearchBot":["allowed","*",0,1],"ClaudeBot":["blocked","named",1,0],"Claude-User":["allowed","*",0,1],"Claude-SearchBot":["allowed","*",0,1],"anthropic-ai":["allowed","*",0,1],"PerplexityBot":["allowed","*",0,1],"Perplexity-User":["allowed","*",0,1],"Google-Extended":["allowed","named",0,1],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["allowed","*",0,1],"Diffbot":["allowed","*",0,1],"ContextFetchBot":["allowed","*",0,1]},"llms":{"status":"missing","http":404,"bytes":11021},"home":{"status":"ok","http":200,"url":"https://emojipedia.org/","html_tokens":57565,"markdown_tokens":231,"pruned_tokens":117}},
{"rank":2217,"domain":"abebooks.com","origin":"https://abebooks.com","robots":{"status":"ok","http":200,"url":"https://www.abebooks.com/robots.txt","bytes":3415,"groups":9},"bots":{"GPTBot":["partial","*",11,0],"ChatGPT-User":["partial","*",11,0],"OAI-SearchBot":["partial","*",11,0],"ClaudeBot":["partial","*",11,0],"Claude-User":["partial","*",11,0],"Claude-SearchBot":["partial","*",11,0],"anthropic-ai":["partial","*",11,0],"PerplexityBot":["partial","*",11,0],"Perplexity-User":["partial","*",11,0],"Google-Extended":["partial","*",11,0],"Applebot-Extended":["partial","*",11,0],"CCBot":["partial","*",11,0],"Bytespider":["partial","*",11,0],"Amazonbot":["partial","*",11,0],"meta-externalagent":["partial","*",11,0],"cohere-ai":["partial","*",11,0],"Diffbot":["partial","*",11,0],"ContextFetchBot":["partial","*",11,0]},"llms":{"status":"missing","http":404,"bytes":22549},"home":{"status":"ok","http":200,"url":"https://www.abebooks.com/","html_tokens":77199,"markdown_tokens":480,"pruned_tokens":142}},
{"rank":2218,"domain":"sciencenews.org","origin":"https://sciencenews.org","robots":{"status":"ok","http":200,"url":"https://www.sciencenews.org/robots.txt","bytes":1253,"groups":28},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["partial","*",1,1],"OAI-SearchBot":["partial","*",1,1],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",1,1],"Claude-SearchBot":["partial","*",1,1],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["partial","*",1,1],"Perplexity-User":["partial","*",1,1],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",2,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",1,1]},"llms":{"status":"missing","http":404,"bytes":220659},"home":{"status":"ok","http":200,"url":"https://www.sciencenews.org/","html_tokens":213389,"markdown_tokens":67,"pruned_tokens":34}},
{"rank":2219,"domain":"psychcentral.com","origin":"https://psychcentral.com","robots":{"status":"ok","http":200,"url":"https://psychcentral.com/robots.txt","bytes":1680,"groups":28},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["partial","*",12,0],"OAI-SearchBot":["partial","*",12,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",12,0],"Claude-SearchBot":["partial","*",12,0],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["partial","*",12,0],"Perplexity-User":["partial","*",12,0],"Google-Extended":["partial","*",12,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["partial","*",12,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",12,0]},"llms":{"status":"missing","http":404,"bytes":87611},"home":{"status":"ok","http":200,"url":"https://psychcentral.com/","html_tokens":57481,"markdown_tokens":74,"pruned_tokens":74}},
{"rank":2220,"domain":"utwente.nl","origin":"https://utwente.nl","robots":{"status":"ok","http":200,"url":"https://www.utwente.nl/robots.txt","bytes":402,"groups":1},"bots":{"GPTBot":["partial","*",12,1],"ChatGPT-User":["partial","*",12,1],"OAI-SearchBot":["partial","*",12,1],"ClaudeBot":["partial","*",12,1],"Claude-User":["partial","*",12,1],"Claude-SearchBot":["partial","*",12,1],"anthropic-ai":["partial","*",12,1],"PerplexityBot":["partial","*",12,1],"Perplexity-User":["partial","*",12,1],"Google-Extended":["partial","*",12,1],"Applebot-Extended":["partial","*",12,1],"CCBot":["partial","*",12,1],"Bytespider":["partial","*",12,1],"Amazonbot":["partial","*",12,1],"meta-externalagent":["partial","*",12,1],"cohere-ai":["partial","*",12,1],"Diffbot":["partial","*",12,1],"ContextFetchBot":["partial","*",12,1]},"llms":{"status":"other_text","http":200,"bytes":3049},"home":{"status":"ok","http":200,"url":"https://www.utwente.nl/","html_tokens":19849,"markdown_tokens":92,"pruned_tokens":92}},
{"rank":2221,"domain":"dpreview.com","origin":"https://dpreview.com","robots":{"status":"refused","http":403,"url":"https://www.dpreview.com/robots.txt/","bytes":5536,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":2222,"domain":"getyourguide.com","origin":"https://getyourguide.com","robots":{"status":"refused","http":403,"url":"https://www.getyourguide.com/robots.txt","bytes":98598,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":2223,"domain":"tickettailor.com","origin":"https://tickettailor.com","robots":{"status":"ok","http":200,"url":"https://www.tickettailor.com/robots.txt","bytes":427,"groups":3},"bots":{"GPTBot":["partial","*",10,1],"ChatGPT-User":["partial","*",10,1],"OAI-SearchBot":["partial","named",1,1],"ClaudeBot":["partial","*",10,1],"Claude-User":["partial","*",10,1],"Claude-SearchBot":["partial","*",10,1],"anthropic-ai":["partial","*",10,1],"PerplexityBot":["partial","*",10,1],"Perplexity-User":["partial","*",10,1],"Google-Extended":["partial","*",10,1],"Applebot-Extended":["partial","*",10,1],"CCBot":["partial","*",10,1],"Bytespider":["partial","*",10,1],"Amazonbot":["partial","*",10,1],"meta-externalagent":["partial","*",10,1],"cohere-ai":["partial","*",10,1],"Diffbot":["partial","*",10,1],"ContextFetchBot":["partial","*",10,1]},"llms":{"status":"missing","http":404,"bytes":168474},"home":{"status":"ok","http":200,"url":"https://www.tickettailor.com/","html_tokens":439034,"markdown_tokens":430,"pruned_tokens":152}},
{"rank":2224,"domain":"marthastewart.com","origin":"https://marthastewart.com","robots":{"status":"ok","http":200,"url":"https://www.marthastewart.com/robots.txt","bytes":3479,"groups":6},"bots":{"GPTBot":["partial","named",1,0],"ChatGPT-User":["partial","named",1,0],"OAI-SearchBot":["partial","named",1,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["blocked","named",1,0],"Claude-SearchBot":["partial","named",1,2],"anthropic-ai":["partial","named",1,2],"PerplexityBot":["partial","named",1,2],"Perplexity-User":["blocked","named",1,0],"Google-Extended":["partial","named",1,2],"Applebot-Extended":["blocked","named",1,0],"CCBot":["partial","named",1,2],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["partial","named",1,2],"cohere-ai":["partial","named",1,2],"Diffbot":["partial","*",2,0],"ContextFetchBot":["partial","*",2,0]},"llms":{"status":"http_error","http":402,"bytes":612},"home":{"status":"http_error","http":402,"url":"https://www.marthastewart.com/"}},
{"rank":2225,"domain":"sputniknews.com","origin":"https://sputniknews.com","robots":{"status":"timeout","http":null,"url":"https://sputniknews.com/robots.txt","bytes":0,"groups":0,"error":"timed out after 10 s"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":2226,"domain":"syr.edu","origin":"https://syr.edu","robots":{"status":"ok","http":200,"url":"https://www.syracuse.edu/robots.txt","bytes":159,"groups":1},"bots":{"GPTBot":["partial","*",5,0],"ChatGPT-User":["partial","*",5,0],"OAI-SearchBot":["partial","*",5,0],"ClaudeBot":["partial","*",5,0],"Claude-User":["partial","*",5,0],"Claude-SearchBot":["partial","*",5,0],"anthropic-ai":["partial","*",5,0],"PerplexityBot":["partial","*",5,0],"Perplexity-User":["partial","*",5,0],"Google-Extended":["partial","*",5,0],"Applebot-Extended":["partial","*",5,0],"CCBot":["partial","*",5,0],"Bytespider":["partial","*",5,0],"Amazonbot":["partial","*",5,0],"meta-externalagent":["partial","*",5,0],"cohere-ai":["partial","*",5,0],"Diffbot":["partial","*",5,0],"ContextFetchBot":["partial","*",5,0]},"llms":{"status":"present","http":200,"bytes":6308},"home":{"status":"ok","http":200,"url":"https://www.syracuse.edu/","html_tokens":55939,"markdown_tokens":77,"pruned_tokens":77}},
{"rank":2227,"domain":"livechatinc.com","origin":"https://livechatinc.com","robots":{"status":"ok","http":200,"url":"https://www.livechat.com/robots.txt","bytes":1164,"groups":2},"bots":{"GPTBot":["partial","*",32,1],"ChatGPT-User":["partial","*",32,1],"OAI-SearchBot":["partial","*",32,1],"ClaudeBot":["partial","*",32,1],"Claude-User":["partial","*",32,1],"Claude-SearchBot":["partial","*",32,1],"anthropic-ai":["partial","*",32,1],"PerplexityBot":["partial","*",32,1],"Perplexity-User":["partial","*",32,1],"Google-Extended":["partial","*",32,1],"Applebot-Extended":["partial","*",32,1],"CCBot":["partial","*",32,1],"Bytespider":["partial","*",32,1],"Amazonbot":["partial","*",32,1],"meta-externalagent":["partial","*",32,1],"cohere-ai":["partial","*",32,1],"Diffbot":["partial","*",32,1],"ContextFetchBot":["partial","*",32,1]},"llms":{"status":"present","http":200,"bytes":238040},"home":{"status":"ok","http":200,"url":"https://www.livechat.com/","html_tokens":64538,"markdown_tokens":403,"pruned_tokens":82}},
{"rank":2228,"domain":"diabetes.org","origin":"https://diabetes.org","robots":{"status":"ok","http":200,"url":"https://diabetes.org/robots.txt","bytes":2604,"groups":1},"bots":{"GPTBot":["partial","*",53,18],"ChatGPT-User":["partial","*",53,18],"OAI-SearchBot":["partial","*",53,18],"ClaudeBot":["partial","*",53,18],"Claude-User":["partial","*",53,18],"Claude-SearchBot":["partial","*",53,18],"anthropic-ai":["partial","*",53,18],"PerplexityBot":["partial","*",53,18],"Perplexity-User":["partial","*",53,18],"Google-Extended":["partial","*",53,18],"Applebot-Extended":["partial","*",53,18],"CCBot":["partial","*",53,18],"Bytespider":["partial","*",53,18],"Amazonbot":["partial","*",53,18],"meta-externalagent":["partial","*",53,18],"cohere-ai":["partial","*",53,18],"Diffbot":["partial","*",53,18],"ContextFetchBot":["partial","*",53,18]},"llms":{"status":"missing","http":404,"bytes":652},"home":{"status":"ok","http":200,"url":"https://diabetes.org/","html_tokens":55563,"markdown_tokens":439,"pruned_tokens":168}},
{"rank":2229,"domain":"caranddriver.com","origin":"https://caranddriver.com","robots":{"status":"ok","http":200,"url":"https://caranddriver.com/robots.txt","bytes":1583,"groups":3},"bots":{"GPTBot":["partial","named",42,5],"ChatGPT-User":["partial","named",42,5],"OAI-SearchBot":["partial","named",42,5],"ClaudeBot":["partial","*",29,5],"Claude-User":["partial","*",29,5],"Claude-SearchBot":["partial","*",29,5],"anthropic-ai":["partial","*",29,5],"PerplexityBot":["partial","*",29,5],"Perplexity-User":["partial","*",29,5],"Google-Extended":["partial","*",29,5],"Applebot-Extended":["partial","*",29,5],"CCBot":["partial","*",29,5],"Bytespider":["partial","*",29,5],"Amazonbot":["partial","*",29,5],"meta-externalagent":["partial","*",29,5],"cohere-ai":["partial","*",29,5],"Diffbot":["partial","*",29,5],"ContextFetchBot":["partial","*",29,5]},"llms":{"status":"missing","http":404,"bytes":84},"home":{"status":"ok","http":200,"url":"https://www.caranddriver.com/","html_tokens":204291,"markdown_tokens":185,"pruned_tokens":102}},
{"rank":2230,"domain":"iom.int","origin":"https://iom.int","robots":{"status":"ok","http":200,"url":"https://www.iom.int/robots.txt","bytes":2027,"groups":1},"bots":{"GPTBot":["partial","*",34,18],"ChatGPT-User":["partial","*",34,18],"OAI-SearchBot":["partial","*",34,18],"ClaudeBot":["partial","*",34,18],"Claude-User":["partial","*",34,18],"Claude-SearchBot":["partial","*",34,18],"anthropic-ai":["partial","*",34,18],"PerplexityBot":["partial","*",34,18],"Perplexity-User":["partial","*",34,18],"Google-Extended":["partial","*",34,18],"Applebot-Extended":["partial","*",34,18],"CCBot":["partial","*",34,18],"Bytespider":["partial","*",34,18],"Amazonbot":["partial","*",34,18],"meta-externalagent":["partial","*",34,18],"cohere-ai":["partial","*",34,18],"Diffbot":["partial","*",34,18],"ContextFetchBot":["partial","*",34,18]},"llms":{"status":"missing","http":404,"bytes":291},"home":{"status":"ok","http":200,"url":"https://www.iom.int/","html_tokens":96532,"markdown_tokens":341,"pruned_tokens":166}},
{"rank":2231,"domain":"st-andrews.ac.uk","origin":"https://st-andrews.ac.uk","robots":{"status":"ok","http":200,"url":"https://www.st-andrews.ac.uk/robots.txt","bytes":602,"groups":4},"bots":{"GPTBot":["partial","*",20,0],"ChatGPT-User":["partial","*",20,0],"OAI-SearchBot":["partial","*",20,0],"ClaudeBot":["partial","*",20,0],"Claude-User":["partial","*",20,0],"Claude-SearchBot":["partial","*",20,0],"anthropic-ai":["partial","*",20,0],"PerplexityBot":["partial","*",20,0],"Perplexity-User":["partial","*",20,0],"Google-Extended":["partial","*",20,0],"Applebot-Extended":["partial","*",20,0],"CCBot":["partial","*",20,0],"Bytespider":["partial","*",20,0],"Amazonbot":["partial","*",20,0],"meta-externalagent":["partial","*",20,0],"cohere-ai":["partial","*",20,0],"Diffbot":["partial","*",20,0],"ContextFetchBot":["partial","*",20,0]},"llms":{"status":"html","http":200,"bytes":685374},"home":{"status":"ok","http":200,"url":"https://www.st-andrews.ac.uk/","html_tokens":7514,"markdown_tokens":286,"pruned_tokens":149}},
{"rank":2232,"domain":"ic3.gov","origin":"https://www.ic3.gov","robots":{"status":"ok","http":200,"url":"https://www.ic3.gov/robots.txt","bytes":165,"groups":1},"bots":{"GPTBot":["partial","*",6,0],"ChatGPT-User":["partial","*",6,0],"OAI-SearchBot":["partial","*",6,0],"ClaudeBot":["partial","*",6,0],"Claude-User":["partial","*",6,0],"Claude-SearchBot":["partial","*",6,0],"anthropic-ai":["partial","*",6,0],"PerplexityBot":["partial","*",6,0],"Perplexity-User":["partial","*",6,0],"Google-Extended":["partial","*",6,0],"Applebot-Extended":["partial","*",6,0],"CCBot":["partial","*",6,0],"Bytespider":["partial","*",6,0],"Amazonbot":["partial","*",6,0],"meta-externalagent":["partial","*",6,0],"cohere-ai":["partial","*",6,0],"Diffbot":["partial","*",6,0],"ContextFetchBot":["partial","*",6,0]},"llms":{"status":"missing","http":404,"bytes":33024},"home":{"status":"ok","http":200,"url":"https://www.ic3.gov/","html_tokens":13289,"markdown_tokens":885,"pruned_tokens":60}},
{"rank":2233,"domain":"taz.de","origin":"https://taz.de","robots":{"status":"ok","http":200,"url":"https://taz.de/robots.txt","bytes":834,"groups":3},"bots":{"GPTBot":["blocked","named",2,0],"ChatGPT-User":["partial","*",2,0],"OAI-SearchBot":["partial","*",2,0],"ClaudeBot":["partial","*",2,0],"Claude-User":["partial","*",2,0],"Claude-SearchBot":["partial","*",2,0],"anthropic-ai":["partial","*",2,0],"PerplexityBot":["partial","*",2,0],"Perplexity-User":["partial","*",2,0],"Google-Extended":["partial","*",2,0],"Applebot-Extended":["partial","*",2,0],"CCBot":["partial","*",2,0],"Bytespider":["partial","*",2,0],"Amazonbot":["partial","*",2,0],"meta-externalagent":["partial","*",2,0],"cohere-ai":["partial","*",2,0],"Diffbot":["partial","*",2,0],"ContextFetchBot":["partial","*",2,0]},"llms":{"status":"http_error","http":403,"bytes":210},"home":{"status":"ok","http":200,"url":"https://taz.de/","html_tokens":104388,"markdown_tokens":108,"pruned_tokens":108}},
{"rank":2234,"domain":"shareasale.com","origin":"https://shareasale.com","robots":{"status":"html","http":200,"url":"https://www.awin.com/us/getting-started-sas?utm_medium=redirect&utm_source=shareasale.com","bytes":147723,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":2235,"domain":"pinterest.de","origin":"https://pinterest.de","robots":{"status":"ok","http":200,"url":"https://de.pinterest.com/robots.txt","bytes":35607,"groups":8},"bots":{"GPTBot":["blocked","*",1,0],"ChatGPT-User":["blocked","*",1,0],"OAI-SearchBot":["blocked","*",1,0],"ClaudeBot":["blocked","*",1,0],"Claude-User":["blocked","*",1,0],"Claude-SearchBot":["blocked","*",1,0],"anthropic-ai":["blocked","*",1,0],"PerplexityBot":["blocked","*",1,0],"Perplexity-User":["blocked","*",1,0],"Google-Extended":["blocked","*",1,0],"Applebot-Extended":["blocked","*",1,0],"CCBot":["blocked","*",1,0],"Bytespider":["blocked","*",1,0],"Amazonbot":["blocked","*",1,0],"meta-externalagent":["blocked","*",1,0],"cohere-ai":["blocked","*",1,0],"Diffbot":["blocked","*",1,0],"ContextFetchBot":["blocked","*",1,0]},"llms":{"status":"disallowed"},"home":{"status":"disallowed"}},
{"rank":2236,"domain":"afthemes.com","origin":"https://afthemes.com","robots":{"status":"ok","http":200,"url":"https://afthemes.com/robots.txt","bytes":545,"groups":6},"bots":{"GPTBot":["partial","named",1,0],"ChatGPT-User":["partial","*",1,1],"OAI-SearchBot":["partial","*",1,1],"ClaudeBot":["partial","named",1,0],"Claude-User":["partial","*",1,1],"Claude-SearchBot":["partial","*",1,1],"anthropic-ai":["partial","*",1,1],"PerplexityBot":["allowed","named",0,0],"Perplexity-User":["partial","*",1,1],"Google-Extended":["partial","*",1,1],"Applebot-Extended":["partial","*",1,1],"CCBot":["partial","named",1,0],"Bytespider":["partial","named",2,0],"Amazonbot":["partial","*",1,1],"meta-externalagent":["partial","*",1,1],"cohere-ai":["partial","*",1,1],"Diffbot":["partial","*",1,1],"ContextFetchBot":["partial","*",1,1]},"llms":{"status":"present","http":200,"bytes":8177},"home":{"status":"ok","http":200,"url":"https://afthemes.com/","html_tokens":128108,"markdown_tokens":160,"pruned_tokens":160}},
{"rank":2237,"domain":"gsmarena.com","origin":"https://gsmarena.com","robots":{"status":"ok","http":200,"url":"https://www.gsmarena.com/robots.txt","bytes":2053,"groups":23},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["blocked","named",1,0],"OAI-SearchBot":["partial","named",9,2],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",26,0],"Claude-SearchBot":["blocked","named",1,0],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["partial","*",26,0],"Google-Extended":["partial","*",26,0],"Applebot-Extended":["partial","*",26,0],"CCBot":["blocked","named",1,0],"Bytespider":["partial","*",26,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["partial","*",26,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",26,0]},"llms":{"status":"missing","http":404,"bytes":19207},"home":{"status":"ok","http":200,"url":"https://www.gsmarena.com/","html_tokens":13535,"markdown_tokens":64,"pruned_tokens":64}},
{"rank":2238,"domain":"dnb.com","origin":"https://dnb.com","robots":{"status":"ok","http":200,"url":"https://www.dnb.com/robots.txt","bytes":1717,"groups":5},"bots":{"GPTBot":["partial","named",2,0],"ChatGPT-User":["partial","*",48,0],"OAI-SearchBot":["partial","*",48,0],"ClaudeBot":["partial","*",48,0],"Claude-User":["partial","*",48,0],"Claude-SearchBot":["partial","*",48,0],"anthropic-ai":["partial","*",48,0],"PerplexityBot":["partial","*",48,0],"Perplexity-User":["partial","*",48,0],"Google-Extended":["partial","*",48,0],"Applebot-Extended":["partial","*",48,0],"CCBot":["partial","*",48,0],"Bytespider":["partial","*",48,0],"Amazonbot":["partial","*",48,0],"meta-externalagent":["partial","*",48,0],"cohere-ai":["partial","*",48,0],"Diffbot":["partial","*",48,0],"ContextFetchBot":["partial","*",48,0]},"llms":{"status":"present","http":200,"bytes":5339},"home":{"status":"ok","http":200,"url":"https://www.dnb.com/en-us/","html_tokens":27354,"markdown_tokens":423,"pruned_tokens":110}},
{"rank":2239,"domain":"scirp.org","origin":"https://scirp.org","robots":{"status":"ok","http":200,"url":"https://scirp.org/robots.txt","bytes":75,"groups":1},"bots":{"GPTBot":["allowed","*",0,1],"ChatGPT-User":["allowed","*",0,1],"OAI-SearchBot":["allowed","*",0,1],"ClaudeBot":["allowed","*",0,1],"Claude-User":["allowed","*",0,1],"Claude-SearchBot":["allowed","*",0,1],"anthropic-ai":["allowed","*",0,1],"PerplexityBot":["allowed","*",0,1],"Perplexity-User":["allowed","*",0,1],"Google-Extended":["allowed","*",0,1],"Applebot-Extended":["allowed","*",0,1],"CCBot":["allowed","*",0,1],"Bytespider":["allowed","*",0,1],"Amazonbot":["allowed","*",0,1],"meta-externalagent":["allowed","*",0,1],"cohere-ai":["allowed","*",0,1],"Diffbot":["allowed","*",0,1],"ContextFetchBot":["allowed","*",0,1]},"llms":{"status":"html","http":200,"bytes":72688},"home":{"status":"ok","http":200,"url":"https://scirp.org/index","html_tokens":18122,"markdown_tokens":8,"pruned_tokens":8}},
{"rank":2240,"domain":"brew.sh","origin":"https://brew.sh","robots":{"status":"ok","http":200,"url":"https://brew.sh/robots.txt","bytes":114,"groups":1},"bots":{"GPTBot":["allowed","*",0,1],"ChatGPT-User":["allowed","*",0,1],"OAI-SearchBot":["allowed","*",0,1],"ClaudeBot":["allowed","*",0,1],"Claude-User":["allowed","*",0,1],"Claude-SearchBot":["allowed","*",0,1],"anthropic-ai":["allowed","*",0,1],"PerplexityBot":["allowed","*",0,1],"Perplexity-User":["allowed","*",0,1],"Google-Extended":["allowed","*",0,1],"Applebot-Extended":["allowed","*",0,1],"CCBot":["allowed","*",0,1],"Bytespider":["allowed","*",0,1],"Amazonbot":["allowed","*",0,1],"meta-externalagent":["allowed","*",0,1],"cohere-ai":["allowed","*",0,1],"Diffbot":["allowed","*",0,1],"ContextFetchBot":["allowed","*",0,1]},"llms":{"status":"missing","http":404,"bytes":9379},"home":{"status":"ok","http":200,"url":"https://brew.sh/","html_tokens":6580,"markdown_tokens":711,"pruned_tokens":139}},
{"rank":2241,"domain":"foreignaffairs.com","origin":"https://foreignaffairs.com","robots":{"status":"ok","http":200,"url":"https://www.foreignaffairs.com/robots.txt","bytes":5745,"groups":1},"bots":{"GPTBot":["partial","*",86,22],"ChatGPT-User":["partial","*",86,22],"OAI-SearchBot":["partial","*",86,22],"ClaudeBot":["partial","*",86,22],"Claude-User":["partial","*",86,22],"Claude-SearchBot":["partial","*",86,22],"anthropic-ai":["partial","*",86,22],"PerplexityBot":["partial","*",86,22],"Perplexity-User":["partial","*",86,22],"Google-Extended":["partial","*",86,22],"Applebot-Extended":["partial","*",86,22],"CCBot":["partial","*",86,22],"Bytespider":["partial","*",86,22],"Amazonbot":["partial","*",86,22],"meta-externalagent":["partial","*",86,22],"cohere-ai":["partial","*",86,22],"Diffbot":["partial","*",86,22],"ContextFetchBot":["partial","*",86,22]},"llms":{"status":"missing","http":404,"bytes":557},"home":{"status":"ok","http":200,"url":"https://www.foreignaffairs.com/","html_tokens":156467,"markdown_tokens":88,"pruned_tokens":72}},
{"rank":2242,"domain":"rust-lang.org","origin":"https://rust-lang.org","robots":{"status":"missing","http":404,"url":"https://rust-lang.org/robots.txt","bytes":9297,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":9297},"home":{"status":"ok","http":200,"url":"https://rust-lang.org/","html_tokens":4613,"markdown_tokens":375,"pruned_tokens":99}},
{"rank":2243,"domain":"superuser.com","origin":"https://superuser.com","robots":{"status":"refused","http":403,"url":"https://superuser.com/robots.txt","bytes":5530,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":2244,"domain":"bis.org","origin":"https://bis.org","robots":{"status":"ok","http":200,"url":"https://www.bis.org/robots.txt","bytes":2486,"groups":1},"bots":{"GPTBot":["partial","*",39,18],"ChatGPT-User":["partial","*",39,18],"OAI-SearchBot":["partial","*",39,18],"ClaudeBot":["partial","*",39,18],"Claude-User":["partial","*",39,18],"Claude-SearchBot":["partial","*",39,18],"anthropic-ai":["partial","*",39,18],"PerplexityBot":["partial","*",39,18],"Perplexity-User":["partial","*",39,18],"Google-Extended":["partial","*",39,18],"Applebot-Extended":["partial","*",39,18],"CCBot":["partial","*",39,18],"Bytespider":["partial","*",39,18],"Amazonbot":["partial","*",39,18],"meta-externalagent":["partial","*",39,18],"cohere-ai":["partial","*",39,18],"Diffbot":["partial","*",39,18],"ContextFetchBot":["partial","*",39,18]},"llms":{"status":"http_error","http":403,"bytes":239},"home":{"status":"ok","http":200,"url":"https://www.bis.org/","html_tokens":58252,"markdown_tokens":115,"pruned_tokens":115}},
{"rank":2245,"domain":"panda.org","origin":"https://panda.org","robots":{"status":"refused","http":403,"url":"https://wwf.panda.org/robots.txt","bytes":118,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":2246,"domain":"adidas.com","origin":"https://adidas.com","robots":{"status":"refused","http":403,"url":"https://www.adidas.com/robots.txt","bytes":3369,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":2247,"domain":"bc.edu","origin":"https://bc.edu","robots":{"status":"ok","http":200,"url":"https://www.bc.edu/robots.txt","bytes":38,"groups":1},"bots":{"GPTBot":["partial","*",1,0],"ChatGPT-User":["partial","*",1,0],"OAI-SearchBot":["partial","*",1,0],"ClaudeBot":["partial","*",1,0],"Claude-User":["partial","*",1,0],"Claude-SearchBot":["partial","*",1,0],"anthropic-ai":["partial","*",1,0],"PerplexityBot":["partial","*",1,0],"Perplexity-User":["partial","*",1,0],"Google-Extended":["partial","*",1,0],"Applebot-Extended":["partial","*",1,0],"CCBot":["partial","*",1,0],"Bytespider":["partial","*",1,0],"Amazonbot":["partial","*",1,0],"meta-externalagent":["partial","*",1,0],"cohere-ai":["partial","*",1,0],"Diffbot":["partial","*",1,0],"ContextFetchBot":["partial","*",1,0]},"llms":{"status":"missing","http":404,"bytes":52144},"home":{"status":"ok","http":200,"url":"https://www.bc.edu","html_tokens":29610,"markdown_tokens":193,"pruned_tokens":103}},
{"rank":2248,"domain":"uu.se","origin":"https://uu.se","robots":{"status":"ok","http":200,"url":"https://www.uu.se/robots.txt","bytes":2858,"groups":3},"bots":{"GPTBot":["partial","*",33,0],"ChatGPT-User":["partial","*",33,0],"OAI-SearchBot":["partial","*",33,0],"ClaudeBot":["partial","*",33,0],"Claude-User":["partial","*",33,0],"Claude-SearchBot":["partial","*",33,0],"anthropic-ai":["partial","*",33,0],"PerplexityBot":["partial","*",33,0],"Perplexity-User":["partial","*",33,0],"Google-Extended":["partial","*",33,0],"Applebot-Extended":["partial","*",33,0],"CCBot":["partial","*",33,0],"Bytespider":["partial","*",33,0],"Amazonbot":["partial","*",33,0],"meta-externalagent":["partial","*",33,0],"cohere-ai":["partial","*",33,0],"Diffbot":["partial","*",33,0],"ContextFetchBot":["partial","*",33,0]},"llms":{"status":"present","http":200,"bytes":12768},"home":{"status":"ok","http":200,"url":"https://www.uu.se/","html_tokens":42181,"markdown_tokens":526,"pruned_tokens":171}},
{"rank":2249,"domain":"nationalreview.com","origin":"https://nationalreview.com","robots":{"status":"ok","http":200,"url":"https://www.nationalreview.com/robots.txt","bytes":3600,"groups":2},"bots":{"GPTBot":["partial","*",2,1],"ChatGPT-User":["partial","*",2,1],"OAI-SearchBot":["partial","*",2,1],"ClaudeBot":["partial","*",2,1],"Claude-User":["partial","*",2,1],"Claude-SearchBot":["partial","*",2,1],"anthropic-ai":["partial","*",2,1],"PerplexityBot":["partial","*",2,1],"Perplexity-User":["partial","*",2,1],"Google-Extended":["partial","*",2,1],"Applebot-Extended":["partial","*",2,1],"CCBot":["partial","*",2,1],"Bytespider":["partial","*",2,1],"Amazonbot":["partial","*",2,1],"meta-externalagent":["partial","*",2,1],"cohere-ai":["partial","*",2,1],"Diffbot":["partial","*",2,1],"ContextFetchBot":["partial","*",2,1]},"llms":{"status":"http_error","http":403,"bytes":5536},"home":{"status":"http_error","http":403,"url":"https://www.nationalreview.com/"}},
{"rank":2250,"domain":"t-mobile.com","origin":"https://t-mobile.com","robots":{"status":"refused","http":403,"url":"https://t-mobile.com/robots.txt","bytes":378,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":2251,"domain":"c-span.org","origin":"https://c-span.org","robots":{"status":"ok","http":200,"url":"https://www.c-span.org/robots.txt","bytes":1281,"groups":3},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["blocked","named",1,0],"OAI-SearchBot":["partial","*",16,2],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",16,2],"Claude-SearchBot":["partial","*",16,2],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["partial","*",16,2],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["partial","*",16,2],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["partial","*",16,2],"cohere-ai":["partial","*",16,2],"Diffbot":["partial","*",16,2],"ContextFetchBot":["partial","*",16,2]},"llms":{"status":"missing","http":404,"bytes":46819},"home":{"status":"ok","http":202,"url":"https://www.c-span.org//","html_tokens":618,"markdown_tokens":40,"pruned_tokens":40}},
{"rank":2252,"domain":"anrdoezrs.net","origin":"https://www.anrdoezrs.net","robots":{"status":"ok","http":200,"url":"https://www.anrdoezrs.net/robots.txt","bytes":35,"groups":1},"bots":{"GPTBot":["blocked","*",1,0],"ChatGPT-User":["blocked","*",1,0],"OAI-SearchBot":["blocked","*",1,0],"ClaudeBot":["blocked","*",1,0],"Claude-User":["blocked","*",1,0],"Claude-SearchBot":["blocked","*",1,0],"anthropic-ai":["blocked","*",1,0],"PerplexityBot":["blocked","*",1,0],"Perplexity-User":["blocked","*",1,0],"Google-Extended":["blocked","*",1,0],"Applebot-Extended":["blocked","*",1,0],"CCBot":["blocked","*",1,0],"Bytespider":["blocked","*",1,0],"Amazonbot":["blocked","*",1,0],"meta-externalagent":["blocked","*",1,0],"cohere-ai":["blocked","*",1,0],"Diffbot":["blocked","*",1,0],"ContextFetchBot":["blocked","*",1,0]},"llms":{"status":"disallowed"},"home":{"status":"disallowed"}},
{"rank":2253,"domain":"usaid.gov","origin":"https://usaid.gov","robots":{"status":"html","http":200,"url":"https://usaid.gov/robots.txt","bytes":360,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":2254,"domain":"wri.org","origin":"https://wri.org","robots":{"status":"ok","http":200,"url":"https://www.wri.org/robots.txt","bytes":1841,"groups":1},"bots":{"GPTBot":["partial","*",40,7],"ChatGPT-User":["partial","*",40,7],"OAI-SearchBot":["partial","*",40,7],"ClaudeBot":["partial","*",40,7],"Claude-User":["partial","*",40,7],"Claude-SearchBot":["partial","*",40,7],"anthropic-ai":["partial","*",40,7],"PerplexityBot":["partial","*",40,7],"Perplexity-User":["partial","*",40,7],"Google-Extended":["partial","*",40,7],"Applebot-Extended":["partial","*",40,7],"CCBot":["partial","*",40,7],"Bytespider":["partial","*",40,7],"Amazonbot":["partial","*",40,7],"meta-externalagent":["partial","*",40,7],"cohere-ai":["partial","*",40,7],"Diffbot":["partial","*",40,7],"ContextFetchBot":["partial","*",40,7]},"llms":{"status":"present","http":200,"bytes":10336},"home":{"status":"ok","http":200,"url":"https://www.wri.org/","html_tokens":24820,"markdown_tokens":222,"pruned_tokens":158}},
{"rank":2255,"domain":"scholastic.com","origin":"https://scholastic.com","robots":{"status":"ok","http":200,"url":"https://www.scholastic.com/robots.txt","bytes":2492,"groups":1},"bots":{"GPTBot":["partial","*",74,0],"ChatGPT-User":["partial","*",74,0],"OAI-SearchBot":["partial","*",74,0],"ClaudeBot":["partial","*",74,0],"Claude-User":["partial","*",74,0],"Claude-SearchBot":["partial","*",74,0],"anthropic-ai":["partial","*",74,0],"PerplexityBot":["partial","*",74,0],"Perplexity-User":["partial","*",74,0],"Google-Extended":["partial","*",74,0],"Applebot-Extended":["partial","*",74,0],"CCBot":["partial","*",74,0],"Bytespider":["partial","*",74,0],"Amazonbot":["partial","*",74,0],"meta-externalagent":["partial","*",74,0],"cohere-ai":["partial","*",74,0],"Diffbot":["partial","*",74,0],"ContextFetchBot":["partial","*",74,0]},"llms":{"status":"html","http":200,"bytes":2569},"home":{"status":"ok","http":200,"url":"https://www.scholastic.com/home","html_tokens":24478,"markdown_tokens":19,"pruned_tokens":19}},
{"rank":2256,"domain":"adafruit.com","origin":"https://adafruit.com","robots":{"status":"ok","http":200,"url":"https://www.adafruit.com/robots.txt","bytes":310,"groups":2},"bots":{"GPTBot":["partial","*",9,2],"ChatGPT-User":["partial","*",9,2],"OAI-SearchBot":["partial","*",9,2],"ClaudeBot":["partial","*",9,2],"Claude-User":["partial","*",9,2],"Claude-SearchBot":["partial","*",9,2],"anthropic-ai":["partial","*",9,2],"PerplexityBot":["partial","*",9,2],"Perplexity-User":["partial","*",9,2],"Google-Extended":["partial","*",9,2],"Applebot-Extended":["partial","*",9,2],"CCBot":["partial","*",9,2],"Bytespider":["partial","*",9,2],"Amazonbot":["partial","*",9,2],"meta-externalagent":["partial","*",9,2],"cohere-ai":["partial","*",9,2],"Diffbot":["partial","*",9,2],"ContextFetchBot":["partial","*",9,2]},"llms":{"status":"missing","http":404,"bytes":32783},"home":{"status":"ok","http":200,"url":"https://www.adafruit.com/","html_tokens":10940,"markdown_tokens":18,"pruned_tokens":18}},
{"rank":2257,"domain":"springerlink.com","origin":"http://springerlink.com","robots":{"status":"missing","http":404,"url":"http://springerlink.com/robots.txt","bytes":711,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":709},"home":{"status":"ok","http":200,"url":"https://link.springer.com/","html_tokens":51401,"markdown_tokens":369,"pruned_tokens":42}},
{"rank":2258,"domain":"vedomosti.ru","origin":"https://vedomosti.ru","robots":{"status":"ok","http":200,"url":"https://www.vedomosti.ru/robots.txt","bytes":3636,"groups":25},"bots":{"GPTBot":["allowed","named",0,1],"ChatGPT-User":["allowed","named",0,1],"OAI-SearchBot":["allowed","named",0,1],"ClaudeBot":["allowed","named",0,1],"Claude-User":["partial","*",28,8],"Claude-SearchBot":["partial","*",28,8],"anthropic-ai":["allowed","named",0,1],"PerplexityBot":["allowed","named",0,1],"Perplexity-User":["allowed","named",0,1],"Google-Extended":["allowed","named",0,1],"Applebot-Extended":["allowed","named",0,1],"CCBot":["allowed","named",0,1],"Bytespider":["allowed","named",0,1],"Amazonbot":["allowed","named",0,1],"meta-externalagent":["allowed","named",0,1],"cohere-ai":["allowed","named",0,1],"Diffbot":["allowed","named",0,1],"ContextFetchBot":["partial","*",28,8]},"llms":{"status":"present","http":200,"bytes":20357},"home":{"status":"ok","http":200,"url":"https://www.vedomosti.ru/","html_tokens":781320,"markdown_tokens":19,"pruned_tokens":19}},
{"rank":2259,"domain":"eaton.com","origin":"https://eaton.com","robots":{"status":"unreachable","http":null,"url":"https://eaton.com/robots.txt","bytes":0,"groups":0,"error":"<StreamReset stream_id:1, error_code:2, remote_reset:True>"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":2260,"domain":"starwars.com","origin":"https://starwars.com","robots":{"status":"refused","http":403,"url":"https://starwars.com/robots.txt","bytes":376,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":2261,"domain":"towardsdatascience.com","origin":"https://towardsdatascience.com","robots":{"status":"ok","http":200,"url":"https://towardsdatascience.com/robots.txt","bytes":713,"groups":18},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["blocked","named",1,0],"OAI-SearchBot":["blocked","named",1,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",1,0],"Claude-SearchBot":["partial","*",1,0],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["partial","*",1,0],"Google-Extended":["partial","*",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["partial","*",1,0],"ContextFetchBot":["partial","*",1,0]},"llms":{"status":"missing","http":404,"bytes":63487},"home":{"status":"http_error","http":403,"url":"https://towardsdatascience.com/"}},
{"rank":2262,"domain":"case.edu","origin":"https://case.edu","robots":{"status":"refused","http":403,"url":"https://case.edu/robots.txt","bytes":560,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":2263,"domain":"nameshift.com","origin":"https://nameshift.com","robots":{"status":"ok","http":200,"url":"https://nameshift.com/robots.txt","bytes":73,"groups":1},"bots":{"GPTBot":["allowed","*",0,1],"ChatGPT-User":["allowed","*",0,1],"OAI-SearchBot":["allowed","*",0,1],"ClaudeBot":["allowed","*",0,1],"Claude-User":["allowed","*",0,1],"Claude-SearchBot":["allowed","*",0,1],"anthropic-ai":["allowed","*",0,1],"PerplexityBot":["allowed","*",0,1],"Perplexity-User":["allowed","*",0,1],"Google-Extended":["allowed","*",0,1],"Applebot-Extended":["allowed","*",0,1],"CCBot":["allowed","*",0,1],"Bytespider":["allowed","*",0,1],"Amazonbot":["allowed","*",0,1],"meta-externalagent":["allowed","*",0,1],"cohere-ai":["allowed","*",0,1],"Diffbot":["allowed","*",0,1],"ContextFetchBot":["allowed","*",0,1]},"llms":{"status":"missing","http":404,"bytes":21376},"home":{"status":"ok","http":200,"url":"https://nameshift.com/en-gb","html_tokens":29804,"markdown_tokens":1397,"pruned_tokens":197}},
{"rank":2264,"domain":"nrdc.org","origin":"https://nrdc.org","robots":{"status":"ok","http":200,"url":"https://www.nrdc.org/robots.txt","bytes":3614,"groups":24},"bots":{"GPTBot":["partial","*",42,18],"ChatGPT-User":["partial","*",42,18],"OAI-SearchBot":["partial","*",42,18],"ClaudeBot":["partial","*",42,18],"Claude-User":["partial","*",42,18],"Claude-SearchBot":["partial","*",42,18],"anthropic-ai":["partial","*",42,18],"PerplexityBot":["partial","*",42,18],"Perplexity-User":["partial","*",42,18],"Google-Extended":["partial","*",42,18],"Applebot-Extended":["partial","*",42,18],"CCBot":["partial","*",42,18],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["partial","*",42,18],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",42,18]},"llms":{"status":"missing","http":404,"bytes":1042},"home":{"status":"http_error","http":403,"url":"https://www.nrdc.org/"}},
{"rank":2265,"domain":"collinsdictionary.com","origin":"https://collinsdictionary.com","robots":{"status":"ok","http":200,"url":"https://www.collinsdictionary.com/robots.txt","bytes":6671,"groups":6},"bots":{"GPTBot":["partial","*",50,0],"ChatGPT-User":["partial","*",50,0],"OAI-SearchBot":["partial","*",50,0],"ClaudeBot":["partial","*",50,0],"Claude-User":["partial","*",50,0],"Claude-SearchBot":["partial","*",50,0],"anthropic-ai":["partial","*",50,0],"PerplexityBot":["partial","*",50,0],"Perplexity-User":["partial","*",50,0],"Google-Extended":["partial","*",50,0],"Applebot-Extended":["partial","*",50,0],"CCBot":["partial","*",50,0],"Bytespider":["partial","*",50,0],"Amazonbot":["partial","*",50,0],"meta-externalagent":["partial","*",50,0],"cohere-ai":["partial","*",50,0],"Diffbot":["partial","*",50,0],"ContextFetchBot":["partial","*",50,0]},"llms":{"status":"missing","http":404,"bytes":0},"home":{"status":"http_error","http":403,"url":"https://collinsdictionary.com/"}},
{"rank":2266,"domain":"ericsson.com","origin":"https://ericsson.com","robots":{"status":"missing","http":404,"url":"https://ericsson.com/robots.txt","bytes":150,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":150},"home":{"status":"disallowed","error":"robots.txt of www.ericsson.com not readable (refused)"}},
{"rank":2267,"domain":"wikimapia.org","origin":"https://wikimapia.org","robots":{"status":"ok","http":200,"url":"https://wikimapia.org/robots.txt","bytes":402,"groups":4},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["partial","*",14,0],"OAI-SearchBot":["partial","*",14,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",14,0],"Claude-SearchBot":["blocked","named",1,0],"anthropic-ai":["partial","*",14,0],"PerplexityBot":["partial","*",14,0],"Perplexity-User":["partial","*",14,0],"Google-Extended":["partial","*",14,0],"Applebot-Extended":["partial","*",14,0],"CCBot":["partial","*",14,0],"Bytespider":["partial","*",14,0],"Amazonbot":["partial","*",14,0],"meta-externalagent":["partial","*",14,0],"cohere-ai":["partial","*",14,0],"Diffbot":["partial","*",14,0],"ContextFetchBot":["partial","*",14,0]},"llms":{"status":"missing","http":404,"bytes":162},"home":{"status":"ok","http":200,"url":"https://wikimapia.org/","html_tokens":6622,"markdown_tokens":163,"pruned_tokens":163}},
{"rank":2268,"domain":"ets.org","origin":"https://ets.org","robots":{"status":"ok","http":200,"url":"https://www.ets.org/robots.txt","bytes":856,"groups":1},"bots":{"GPTBot":["partial","*",18,0],"ChatGPT-User":["partial","*",18,0],"OAI-SearchBot":["partial","*",18,0],"ClaudeBot":["partial","*",18,0],"Claude-User":["partial","*",18,0],"Claude-SearchBot":["partial","*",18,0],"anthropic-ai":["partial","*",18,0],"PerplexityBot":["partial","*",18,0],"Perplexity-User":["partial","*",18,0],"Google-Extended":["partial","*",18,0],"Applebot-Extended":["partial","*",18,0],"CCBot":["partial","*",18,0],"Bytespider":["partial","*",18,0],"Amazonbot":["partial","*",18,0],"meta-externalagent":["partial","*",18,0],"cohere-ai":["partial","*",18,0],"Diffbot":["partial","*",18,0],"ContextFetchBot":["partial","*",18,0]},"llms":{"status":"missing","http":404,"bytes":54138},"home":{"status":"ok","http":200,"url":"https://www.ets.org/","html_tokens":43022,"markdown_tokens":134,"pruned_tokens":134}},
{"rank":2269,"domain":"capterra.com","origin":"https://capterra.com","robots":{"status":"refused","http":403,"url":"https://www.capterra.com/robots.txt","bytes":4906,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":2270,"domain":"wattpad.com","origin":"https://wattpad.com","robots":{"status":"ok","http":200,"url":"https://www.wattpad.com/robots.txt","bytes":1580,"groups":3},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["partial","*",41,1],"OAI-SearchBot":["partial","*",41,1],"ClaudeBot":["partial","*",41,1],"Claude-User":["partial","*",41,1],"Claude-SearchBot":["partial","*",41,1],"anthropic-ai":["partial","*",41,1],"PerplexityBot":["partial","*",41,1],"Perplexity-User":["partial","*",41,1],"Google-Extended":["partial","*",41,1],"Applebot-Extended":["partial","*",41,1],"CCBot":["partial","*",41,1],"Bytespider":["partial","*",41,1],"Amazonbot":["partial","*",41,1],"meta-externalagent":["partial","*",41,1],"cohere-ai":["partial","*",41,1],"Diffbot":["partial","*",41,1],"ContextFetchBot":["partial","*",41,1]},"llms":{"status":"missing","http":404,"bytes":63410},"home":{"status":"ok","http":200,"url":"https://www.wattpad.com/","html_tokens":29711,"markdown_tokens":39,"pruned_tokens":21}},
{"rank":2271,"domain":"bhphotovideo.com","origin":"https://bhphotovideo.com","robots":{"status":"ok","http":200,"url":"https://www.bhphotovideo.com/robots.txt","bytes":4747,"groups":2},"bots":{"GPTBot":["partial","*",154,3],"ChatGPT-User":["partial","*",154,3],"OAI-SearchBot":["partial","*",154,3],"ClaudeBot":["partial","*",154,3],"Claude-User":["partial","*",154,3],"Claude-SearchBot":["partial","*",154,3],"anthropic-ai":["partial","*",154,3],"PerplexityBot":["partial","*",154,3],"Perplexity-User":["partial","*",154,3],"Google-Extended":["partial","*",154,3],"Applebot-Extended":["partial","*",154,3],"CCBot":["partial","*",154,3],"Bytespider":["partial","*",154,3],"Amazonbot":["partial","*",154,3],"meta-externalagent":["partial","*",154,3],"cohere-ai":["partial","*",154,3],"Diffbot":["partial","*",154,3],"ContextFetchBot":["partial","*",154,3]},"llms":{"status":"http_error","http":403,"bytes":5548},"home":{"status":"http_error","http":403,"url":"https://bhphotovideo.com/"}},
{"rank":2272,"domain":"allmusic.com","origin":"https://allmusic.com","robots":{"status":"ok","http":200,"url":"https://www.allmusic.com/robots.txt","bytes":5099,"groups":44},"bots":{"GPTBot":["allowed","named",0,1],"ChatGPT-User":["allowed","named",0,1],"OAI-SearchBot":["allowed","named",0,1],"ClaudeBot":["allowed","named",0,1],"Claude-User":["partial","*",42,0],"Claude-SearchBot":["partial","*",42,0],"anthropic-ai":["allowed","named",0,1],"PerplexityBot":["allowed","named",0,1],"Perplexity-User":["partial","*",42,0],"Google-Extended":["allowed","named",0,1],"Applebot-Extended":["allowed","named",0,1],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["allowed","named",0,1],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",42,0]},"llms":{"status":"missing","http":404,"bytes":37404},"home":{"status":"http_error","http":403,"url":"https://allmusic.com/"}},
{"rank":2273,"domain":"cuhk.edu.hk","origin":"https://cuhk.edu.hk","robots":{"status":"ok","http":200,"url":"https://cuhk.edu.hk/robots.txt","bytes":78,"groups":1},"bots":{"GPTBot":["partial","*",1,0],"ChatGPT-User":["partial","*",1,0],"OAI-SearchBot":["partial","*",1,0],"ClaudeBot":["partial","*",1,0],"Claude-User":["partial","*",1,0],"Claude-SearchBot":["partial","*",1,0],"anthropic-ai":["partial","*",1,0],"PerplexityBot":["partial","*",1,0],"Perplexity-User":["partial","*",1,0],"Google-Extended":["partial","*",1,0],"Applebot-Extended":["partial","*",1,0],"CCBot":["partial","*",1,0],"Bytespider":["partial","*",1,0],"Amazonbot":["partial","*",1,0],"meta-externalagent":["partial","*",1,0],"cohere-ai":["partial","*",1,0],"Diffbot":["partial","*",1,0],"ContextFetchBot":["partial","*",1,0]},"llms":{"status":"missing","http":404,"bytes":4533},"home":{"status":"ok","http":200,"url":"https://cuhk.edu.hk/","html_tokens":104,"markdown_tokens":28,"pruned_tokens":28}},
{"rank":2274,"domain":"wbur.org","origin":"https://wbur.org","robots":{"status":"ok","http":200,"url":"https://www.wbur.org/robots.txt","bytes":2834,"groups":5},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["allowed","named",0,1],"OAI-SearchBot":["allowed","named",0,1],"ClaudeBot":["blocked","named",1,0],"Claude-User":["allowed","named",0,1],"Claude-SearchBot":["allowed","named",0,1],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["allowed","named",0,1],"Perplexity-User":["partial","*",9,1],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",9,1]},"llms":{"status":"missing","http":404,"bytes":357405},"home":{"status":"ok","http":200,"url":"https://www.wbur.org/","html_tokens":334652,"markdown_tokens":45,"pruned_tokens":45}},
{"rank":2275,"domain":"cleveland.com","origin":"https://cleveland.com","robots":{"status":"ok","http":200,"url":"https://www.cleveland.com/robots.txt","bytes":2869,"groups":4},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["partial","named",1,13],"OAI-SearchBot":["partial","named",1,13],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","named",1,13],"Claude-SearchBot":["partial","named",1,13],"anthropic-ai":["partial","*",30,3],"PerplexityBot":["partial","named",1,13],"Perplexity-User":["partial","named",1,13],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["partial","*",30,3],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",30,3]},"llms":{"status":"missing","http":404,"bytes":403040},"home":{"status":"http_error","http":403,"url":"https://www.cleveland.com/"}},
{"rank":2276,"domain":"tensorflow.org","origin":"https://tensorflow.org","robots":{"status":"ok","http":200,"url":"https://www.tensorflow.org/robots.txt","bytes":62,"groups":1},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"missing","http":404,"bytes":78870},"home":{"status":"unreachable","error":"cannot resolve www.tensorflow.org"}},
{"rank":2277,"domain":"gamcare.org.uk","origin":"https://gamcare.org.uk","robots":{"status":"ok","http":200,"url":"https://www.gamcare.org.uk/robots.txt","bytes":250,"groups":2},"bots":{"GPTBot":["partial","*",1,1],"ChatGPT-User":["partial","*",1,1],"OAI-SearchBot":["partial","*",1,1],"ClaudeBot":["partial","*",1,1],"Claude-User":["partial","*",1,1],"Claude-SearchBot":["partial","*",1,1],"anthropic-ai":["partial","*",1,1],"PerplexityBot":["partial","*",1,1],"Perplexity-User":["partial","*",1,1],"Google-Extended":["partial","*",1,1],"Applebot-Extended":["partial","*",1,1],"CCBot":["partial","*",1,1],"Bytespider":["partial","*",1,1],"Amazonbot":["partial","*",1,1],"meta-externalagent":["partial","*",1,1],"cohere-ai":["partial","*",1,1],"Diffbot":["partial","*",1,1],"ContextFetchBot":["partial","*",1,1]},"llms":{"status":"missing","http":404,"bytes":145506},"home":{"status":"http_error","http":403,"url":"https://gamcare.org.uk/"}},
{"rank":2278,"domain":"uvic.ca","origin":"https://uvic.ca","robots":{"status":"ok","http":200,"url":"https://www.uvic.ca/robots.txt","bytes":337,"groups":1},"bots":{"GPTBot":["partial","*",3,0],"ChatGPT-User":["partial","*",3,0],"OAI-SearchBot":["partial","*",3,0],"ClaudeBot":["partial","*",3,0],"Claude-User":["partial","*",3,0],"Claude-SearchBot":["partial","*",3,0],"anthropic-ai":["partial","*",3,0],"PerplexityBot":["partial","*",3,0],"Perplexity-User":["partial","*",3,0],"Google-Extended":["partial","*",3,0],"Applebot-Extended":["partial","*",3,0],"CCBot":["partial","*",3,0],"Bytespider":["partial","*",3,0],"Amazonbot":["partial","*",3,0],"meta-externalagent":["partial","*",3,0],"cohere-ai":["partial","*",3,0],"Diffbot":["partial","*",3,0],"ContextFetchBot":["partial","*",3,0]},"llms":{"status":"missing","http":404,"bytes":17876},"home":{"status":"ok","http":200,"url":"https://www.uvic.ca/","html_tokens":27646,"markdown_tokens":278,"pruned_tokens":137}},
{"rank":2279,"domain":"krakow.pl","origin":"https://krakow.pl","robots":{"status":"ok","http":200,"url":"https://krakow.pl/robots.txt","bytes":36,"groups":1},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":8656},"home":{"status":"ok","http":200,"url":"https://krakow.pl/","html_tokens":26924,"markdown_tokens":391,"pruned_tokens":38}},
{"rank":2280,"domain":"nypl.org","origin":"https://nypl.org","robots":{"status":"html","http":200,"url":"https://www.nypl.org/robots.txt","bytes":212,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":2281,"domain":"vam.ac.uk","origin":"https://vam.ac.uk","robots":{"status":"ok","http":200,"url":"https://www.vam.ac.uk/robots.txt","bytes":1207,"groups":1},"bots":{"GPTBot":["partial","*",36,1],"ChatGPT-User":["partial","*",36,1],"OAI-SearchBot":["partial","*",36,1],"ClaudeBot":["partial","*",36,1],"Claude-User":["partial","*",36,1],"Claude-SearchBot":["partial","*",36,1],"anthropic-ai":["partial","*",36,1],"PerplexityBot":["partial","*",36,1],"Perplexity-User":["partial","*",36,1],"Google-Extended":["partial","*",36,1],"Applebot-Extended":["partial","*",36,1],"CCBot":["partial","*",36,1],"Bytespider":["partial","*",36,1],"Amazonbot":["partial","*",36,1],"meta-externalagent":["partial","*",36,1],"cohere-ai":["partial","*",36,1],"Diffbot":["partial","*",36,1],"ContextFetchBot":["partial","*",36,1]},"llms":{"status":"missing","http":404,"bytes":9991},"home":{"status":"ok","http":200,"url":"https://www.vam.ac.uk/","html_tokens":35380,"markdown_tokens":68,"pruned_tokens":68}},
{"rank":2282,"domain":"doodle.com","origin":"https://doodle.com","robots":{"status":"ok","http":200,"url":"https://doodle.com/robots.txt","bytes":804,"groups":2},"bots":{"GPTBot":["partial","*",24,1],"ChatGPT-User":["partial","*",24,1],"OAI-SearchBot":["partial","*",24,1],"ClaudeBot":["partial","*",24,1],"Claude-User":["partial","*",24,1],"Claude-SearchBot":["partial","*",24,1],"anthropic-ai":["partial","*",24,1],"PerplexityBot":["partial","*",24,1],"Perplexity-User":["partial","*",24,1],"Google-Extended":["partial","*",24,1],"Applebot-Extended":["partial","*",24,1],"CCBot":["partial","*",24,1],"Bytespider":["partial","*",24,1],"Amazonbot":["partial","*",24,1],"meta-externalagent":["partial","*",24,1],"cohere-ai":["partial","*",24,1],"Diffbot":["partial","*",24,1],"ContextFetchBot":["partial","*",24,1]},"llms":{"status":"missing","http":404,"bytes":1879},"home":{"status":"ok","http":200,"url":"https://doodle.com/en/","html_tokens":79249,"markdown_tokens":958,"pruned_tokens":407}},
{"rank":2283,"domain":"de.pinterest.com","origin":"https://de.pinterest.com","robots":{"status":"ok","http":200,"url":"https://de.pinterest.com/robots.txt","bytes":35607,"groups":8},"bots":{"GPTBot":["blocked","*",1,0],"ChatGPT-User":["blocked","*",1,0],"OAI-SearchBot":["blocked","*",1,0],"ClaudeBot":["blocked","*",1,0],"Claude-User":["blocked","*",1,0],"Claude-SearchBot":["blocked","*",1,0],"anthropic-ai":["blocked","*",1,0],"PerplexityBot":["blocked","*",1,0],"Perplexity-User":["blocked","*",1,0],"Google-Extended":["blocked","*",1,0],"Applebot-Extended":["blocked","*",1,0],"CCBot":["blocked","*",1,0],"Bytespider":["blocked","*",1,0],"Amazonbot":["blocked","*",1,0],"meta-externalagent":["blocked","*",1,0],"cohere-ai":["blocked","*",1,0],"Diffbot":["blocked","*",1,0],"ContextFetchBot":["blocked","*",1,0]},"llms":{"status":"disallowed"},"home":{"status":"disallowed"}},
{"rank":2284,"domain":"dji.com","origin":"https://dji.com","robots":{"status":"ok","http":200,"url":"https://www.dji.com/robots.txt","bytes":926,"groups":1},"bots":{"GPTBot":["partial","*",42,0],"ChatGPT-User":["partial","*",42,0],"OAI-SearchBot":["partial","*",42,0],"ClaudeBot":["partial","*",42,0],"Claude-User":["partial","*",42,0],"Claude-SearchBot":["partial","*",42,0],"anthropic-ai":["partial","*",42,0],"PerplexityBot":["partial","*",42,0],"Perplexity-User":["partial","*",42,0],"Google-Extended":["partial","*",42,0],"Applebot-Extended":["partial","*",42,0],"CCBot":["partial","*",42,0],"Bytespider":["partial","*",42,0],"Amazonbot":["partial","*",42,0],"meta-externalagent":["partial","*",42,0],"cohere-ai":["partial","*",42,0],"Diffbot":["partial","*",42,0],"ContextFetchBot":["partial","*",42,0]},"llms":{"status":"missing","http":404,"bytes":418347},"home":{"status":"ok","http":200,"url":"https://www.dji.com/","html_tokens":38280,"markdown_tokens":17,"pruned_tokens":17}},
{"rank":2285,"domain":"tfl.gov.uk","origin":"https://tfl.gov.uk","robots":{"status":"refused","http":403,"url":"https://tfl.gov.uk/robots.txt","bytes":4988,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":2286,"domain":"cas.cn","origin":"https://cas.cn","robots":{"status":"missing","http":404,"url":"https://www.cas.cn/robots.txt","bytes":1293,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":1291},"home":{"status":"ok","http":200,"url":"https://www.cas.cn/","html_tokens":26802,"markdown_tokens":23,"pruned_tokens":5}},
{"rank":2287,"domain":"usu.edu","origin":"https://usu.edu","robots":{"status":"ok","http":200,"url":"https://www.usu.edu/robots.txt","bytes":165,"groups":1},"bots":{"GPTBot":["partial","*",5,0],"ChatGPT-User":["partial","*",5,0],"OAI-SearchBot":["partial","*",5,0],"ClaudeBot":["partial","*",5,0],"Claude-User":["partial","*",5,0],"Claude-SearchBot":["partial","*",5,0],"anthropic-ai":["partial","*",5,0],"PerplexityBot":["partial","*",5,0],"Perplexity-User":["partial","*",5,0],"Google-Extended":["partial","*",5,0],"Applebot-Extended":["partial","*",5,0],"CCBot":["partial","*",5,0],"Bytespider":["partial","*",5,0],"Amazonbot":["partial","*",5,0],"meta-externalagent":["partial","*",5,0],"cohere-ai":["partial","*",5,0],"Diffbot":["partial","*",5,0],"ContextFetchBot":["partial","*",5,0]},"llms":{"status":"missing","http":404,"bytes":28995},"home":{"status":"ok","http":200,"url":"https://www.usu.edu/","html_tokens":15156,"markdown_tokens":280,"pruned_tokens":13}},
{"rank":2288,"domain":"zol.com.cn","origin":"https://zol.com.cn","robots":{"status":"ok","http":200,"url":"https://www.zol.com.cn/robots.txt","bytes":122,"groups":1},"bots":{"GPTBot":["partial","*",5,0],"ChatGPT-User":["partial","*",5,0],"OAI-SearchBot":["partial","*",5,0],"ClaudeBot":["partial","*",5,0],"Claude-User":["partial","*",5,0],"Claude-SearchBot":["partial","*",5,0],"anthropic-ai":["partial","*",5,0],"PerplexityBot":["partial","*",5,0],"Perplexity-User":["partial","*",5,0],"Google-Extended":["partial","*",5,0],"Applebot-Extended":["partial","*",5,0],"CCBot":["partial","*",5,0],"Bytespider":["partial","*",5,0],"Amazonbot":["partial","*",5,0],"meta-externalagent":["partial","*",5,0],"cohere-ai":["partial","*",5,0],"Diffbot":["partial","*",5,0],"ContextFetchBot":["partial","*",5,0]},"llms":{"status":"missing","http":404,"bytes":43309},"home":{"status":"ok","http":200,"url":"https://www.zol.com.cn/","html_tokens":425131,"markdown_tokens":109,"pruned_tokens":109}},
{"rank":2289,"domain":"thestar.com.my","origin":"https://thestar.com.my","robots":{"status":"ok","http":200,"url":"https://www.thestar.com.my/robots.txt","bytes":2774,"groups":36},"bots":{"GPTBot":["partial","*",8,0],"ChatGPT-User":["partial","*",8,0],"OAI-SearchBot":["partial","*",8,0],"ClaudeBot":["partial","*",8,0],"Claude-User":["partial","*",8,0],"Claude-SearchBot":["partial","*",8,0],"anthropic-ai":["partial","*",8,0],"PerplexityBot":["partial","*",8,0],"Perplexity-User":["partial","*",8,0],"Google-Extended":["partial","*",8,0],"Applebot-Extended":["partial","*",8,0],"CCBot":["partial","*",8,0],"Bytespider":["partial","*",8,0],"Amazonbot":["partial","*",8,0],"meta-externalagent":["partial","*",8,0],"cohere-ai":["partial","*",8,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",8,0]},"llms":{"status":"missing","http":404,"bytes":6336},"home":{"status":"ok","http":200,"url":"https://www.thestar.com.my","html_tokens":74420,"markdown_tokens":59,"pruned_tokens":59}},
{"rank":2290,"domain":"derstandard.at","origin":"https://derstandard.at","robots":{"status":"ok","http":200,"url":"https://www.derstandard.at/robots.txt","bytes":1241,"groups":27},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["blocked","named",1,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["blocked","named",1,0],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",2,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":16431},"home":{"status":"ok","http":200,"url":"https://www.derstandard.at/","html_tokens":126570,"markdown_tokens":9017,"pruned_tokens":148}},
{"rank":2291,"domain":"gosuslugi.ru","origin":"https://gosuslugi.ru","robots":{"status":"ok","http":200,"url":"https://www.gosuslugi.ru/robots.txt","bytes":8746,"groups":3},"bots":{"GPTBot":["partial","*",31,125],"ChatGPT-User":["partial","*",31,125],"OAI-SearchBot":["partial","*",31,125],"ClaudeBot":["partial","*",31,125],"Claude-User":["partial","*",31,125],"Claude-SearchBot":["partial","*",31,125],"anthropic-ai":["partial","*",31,125],"PerplexityBot":["partial","*",31,125],"Perplexity-User":["partial","*",31,125],"Google-Extended":["partial","*",31,125],"Applebot-Extended":["partial","*",31,125],"CCBot":["partial","*",31,125],"Bytespider":["partial","*",31,125],"Amazonbot":["partial","*",31,125],"meta-externalagent":["partial","*",31,125],"cohere-ai":["partial","*",31,125],"Diffbot":["partial","*",31,125],"ContextFetchBot":["partial","*",31,125]},"llms":{"status":"disallowed"},"home":{"status":"ok","http":200,"url":"https://gosuslugi.ru/","html_tokens":2309,"markdown_tokens":28,"pruned_tokens":22}},
{"rank":2292,"domain":"trimble.com","origin":"https://trimble.com","robots":{"status":"ok","http":200,"url":"https://www.trimble.com/robots.txt","bytes":92,"groups":1},"bots":{"GPTBot":["partial","*",1,1],"ChatGPT-User":["partial","*",1,1],"OAI-SearchBot":["partial","*",1,1],"ClaudeBot":["partial","*",1,1],"Claude-User":["partial","*",1,1],"Claude-SearchBot":["partial","*",1,1],"anthropic-ai":["partial","*",1,1],"PerplexityBot":["partial","*",1,1],"Perplexity-User":["partial","*",1,1],"Google-Extended":["partial","*",1,1],"Applebot-Extended":["partial","*",1,1],"CCBot":["partial","*",1,1],"Bytespider":["partial","*",1,1],"Amazonbot":["partial","*",1,1],"meta-externalagent":["partial","*",1,1],"cohere-ai":["partial","*",1,1],"Diffbot":["partial","*",1,1],"ContextFetchBot":["partial","*",1,1]},"llms":{"status":"present","http":200,"bytes":343},"home":{"status":"ok","http":200,"url":"https://www.trimble.com/en","html_tokens":382154,"markdown_tokens":556,"pruned_tokens":406}},
{"rank":2293,"domain":"ibtimes.co.uk","origin":"https://ibtimes.co.uk","robots":{"status":"ok","http":200,"url":"https://www.ibtimes.co.uk/robots.txt","bytes":431,"groups":1},"bots":{"GPTBot":["partial","*",12,0],"ChatGPT-User":["partial","*",12,0],"OAI-SearchBot":["partial","*",12,0],"ClaudeBot":["partial","*",12,0],"Claude-User":["partial","*",12,0],"Claude-SearchBot":["partial","*",12,0],"anthropic-ai":["partial","*",12,0],"PerplexityBot":["partial","*",12,0],"Perplexity-User":["partial","*",12,0],"Google-Extended":["partial","*",12,0],"Applebot-Extended":["partial","*",12,0],"CCBot":["partial","*",12,0],"Bytespider":["partial","*",12,0],"Amazonbot":["partial","*",12,0],"meta-externalagent":["partial","*",12,0],"cohere-ai":["partial","*",12,0],"Diffbot":["partial","*",12,0],"ContextFetchBot":["partial","*",12,0]},"llms":{"status":"missing","http":404,"bytes":134848},"home":{"status":"ok","http":200,"url":"https://www.ibtimes.co.uk/","html_tokens":53935,"markdown_tokens":41,"pruned_tokens":41}},
{"rank":2294,"domain":"ingentaconnect.com","origin":"https://ingentaconnect.com","robots":{"status":"refused","http":403,"url":"https://ingentaconnect.com/robots.txt","bytes":5428,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":2295,"domain":"microchip.com","origin":"https://microchip.com","robots":{"status":"refused","http":403,"url":"https://www.microchip.com/robots.txt","bytes":385,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":2296,"domain":"jianshu.com","origin":"https://jianshu.com","robots":{"status":"refused","http":403,"url":"https://www.jianshu.com/robots.txt","bytes":311,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":2297,"domain":"gob.cl","origin":"https://www.gob.cl","robots":{"status":"refused","http":403,"url":"https://www.gob.cl/robots.txt","bytes":5509,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":2298,"domain":"bloomberglaw.com","origin":"https://bloomberglaw.com","robots":{"status":"ok","http":200,"url":"https://www.bloomberglaw.com/robots.txt","bytes":166,"groups":5},"bots":{"GPTBot":["blocked","*",1,0],"ChatGPT-User":["blocked","*",1,0],"OAI-SearchBot":["blocked","*",1,0],"ClaudeBot":["blocked","*",1,0],"Claude-User":["blocked","*",1,0],"Claude-SearchBot":["blocked","*",1,0],"anthropic-ai":["blocked","*",1,0],"PerplexityBot":["blocked","*",1,0],"Perplexity-User":["blocked","*",1,0],"Google-Extended":["blocked","*",1,0],"Applebot-Extended":["blocked","*",1,0],"CCBot":["blocked","*",1,0],"Bytespider":["blocked","*",1,0],"Amazonbot":["blocked","*",1,0],"meta-externalagent":["blocked","*",1,0],"cohere-ai":["blocked","*",1,0],"Diffbot":["blocked","*",1,0],"ContextFetchBot":["blocked","*",1,0]},"llms":{"status":"disallowed"},"home":{"status":"disallowed"}},
{"rank":2299,"domain":"insidehighered.com","origin":"https://insidehighered.com","robots":{"status":"ok","http":200,"url":"https://www.insidehighered.com/robots.txt","bytes":2027,"groups":1},"bots":{"GPTBot":["partial","*",34,18],"ChatGPT-User":["partial","*",34,18],"OAI-SearchBot":["partial","*",34,18],"ClaudeBot":["partial","*",34,18],"Claude-User":["partial","*",34,18],"Claude-SearchBot":["partial","*",34,18],"anthropic-ai":["partial","*",34,18],"PerplexityBot":["partial","*",34,18],"Perplexity-User":["partial","*",34,18],"Google-Extended":["partial","*",34,18],"Applebot-Extended":["partial","*",34,18],"CCBot":["partial","*",34,18],"Bytespider":["partial","*",34,18],"Amazonbot":["partial","*",34,18],"meta-externalagent":["partial","*",34,18],"cohere-ai":["partial","*",34,18],"Diffbot":["partial","*",34,18],"ContextFetchBot":["partial","*",34,18]},"llms":{"status":"missing","http":404,"bytes":196},"home":{"status":"ok","http":200,"url":"https://www.insidehighered.com/","html_tokens":43912,"markdown_tokens":555,"pruned_tokens":79}},
{"rank":2300,"domain":"gamestop.com","origin":"https://gamestop.com","robots":{"status":"refused","http":403,"url":"https://www.gamestop.com//robots.txt","bytes":4545,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":2301,"domain":"myanimelist.net","origin":"https://myanimelist.net","robots":{"status":"ok","http":200,"url":"https://myanimelist.net/robots.txt","bytes":2126,"groups":5},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["partial","*",12,0],"OAI-SearchBot":["partial","*",12,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",12,0],"Claude-SearchBot":["partial","*",12,0],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["partial","*",12,0],"Perplexity-User":["partial","*",12,0],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["partial","*",12,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["partial","*",12,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",12,0]},"llms":{"status":"missing","http":404,"bytes":38086},"home":{"status":"ok","http":200,"url":"https://myanimelist.net/","html_tokens":45481,"markdown_tokens":695,"pruned_tokens":132}},
{"rank":2302,"domain":"rit.edu","origin":"https://rit.edu","robots":{"status":"ok","http":200,"url":"https://www.rit.edu/robots.txt","bytes":3136,"groups":14},"bots":{"GPTBot":["partial","*",52,0],"ChatGPT-User":["partial","*",52,0],"OAI-SearchBot":["partial","*",52,0],"ClaudeBot":["partial","*",52,0],"Claude-User":["partial","*",52,0],"Claude-SearchBot":["partial","*",52,0],"anthropic-ai":["partial","*",52,0],"PerplexityBot":["partial","*",52,0],"Perplexity-User":["partial","*",52,0],"Google-Extended":["partial","*",52,0],"Applebot-Extended":["partial","*",52,0],"CCBot":["partial","*",52,0],"Bytespider":["partial","*",52,0],"Amazonbot":["partial","*",52,0],"meta-externalagent":["partial","*",52,0],"cohere-ai":["partial","*",52,0],"Diffbot":["partial","*",52,0],"ContextFetchBot":["partial","*",52,0]},"llms":{"status":"missing","http":404,"bytes":185},"home":{"status":"ok","http":200,"url":"https://www.rit.edu/","html_tokens":53465,"markdown_tokens":3655,"pruned_tokens":54}},
{"rank":2303,"domain":"lifewire.com","origin":"https://lifewire.com","robots":{"status":"ok","http":200,"url":"https://www.lifewire.com/robots.txt","bytes":3469,"groups":6},"bots":{"GPTBot":["partial","named",1,0],"ChatGPT-User":["partial","named",1,0],"OAI-SearchBot":["partial","named",1,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["blocked","named",1,0],"Claude-SearchBot":["partial","named",1,2],"anthropic-ai":["partial","named",1,2],"PerplexityBot":["partial","named",1,2],"Perplexity-User":["blocked","named",1,0],"Google-Extended":["partial","named",1,2],"Applebot-Extended":["blocked","named",1,0],"CCBot":["partial","named",1,2],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["partial","named",1,2],"cohere-ai":["partial","named",1,2],"Diffbot":["partial","*",2,0],"ContextFetchBot":["partial","*",2,0]},"llms":{"status":"http_error","http":402,"bytes":612},"home":{"status":"http_error","http":402,"url":"https://www.lifewire.com/"}},
{"rank":2304,"domain":"aafp.org","origin":"https://aafp.org","robots":{"status":"ok","http":200,"url":"https://www.aafp.org/robots.txt","bytes":1046,"groups":23},"bots":{"GPTBot":["partial","*",22,0],"ChatGPT-User":["partial","*",22,0],"OAI-SearchBot":["partial","*",22,0],"ClaudeBot":["partial","*",22,0],"Claude-User":["partial","*",22,0],"Claude-SearchBot":["partial","*",22,0],"anthropic-ai":["partial","*",22,0],"PerplexityBot":["partial","*",22,0],"Perplexity-User":["partial","*",22,0],"Google-Extended":["partial","*",22,0],"Applebot-Extended":["partial","*",22,0],"CCBot":["partial","*",22,0],"Bytespider":["partial","*",22,0],"Amazonbot":["partial","*",22,0],"meta-externalagent":["partial","*",22,0],"cohere-ai":["partial","*",22,0],"Diffbot":["partial","*",22,0],"ContextFetchBot":["partial","*",22,0]},"llms":{"status":"missing","http":404,"bytes":271390},"home":{"status":"ok","http":200,"url":"https://www.aafp.org/","html_tokens":129496,"markdown_tokens":60,"pruned_tokens":31}},
{"rank":2305,"domain":"theknot.com","origin":"https://theknot.com","robots":{"status":"refused","http":403,"url":"https://www.theknot.com/robots.txt","bytes":379,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":2306,"domain":"rd.com","origin":"https://rd.com","robots":{"status":"ok","http":200,"url":"https://www.rd.com/robots.txt","bytes":689,"groups":1},"bots":{"GPTBot":["partial","*",14,1],"ChatGPT-User":["partial","*",14,1],"OAI-SearchBot":["partial","*",14,1],"ClaudeBot":["partial","*",14,1],"Claude-User":["partial","*",14,1],"Claude-SearchBot":["partial","*",14,1],"anthropic-ai":["partial","*",14,1],"PerplexityBot":["partial","*",14,1],"Perplexity-User":["partial","*",14,1],"Google-Extended":["partial","*",14,1],"Applebot-Extended":["partial","*",14,1],"CCBot":["partial","*",14,1],"Bytespider":["partial","*",14,1],"Amazonbot":["partial","*",14,1],"meta-externalagent":["partial","*",14,1],"cohere-ai":["partial","*",14,1],"Diffbot":["partial","*",14,1],"ContextFetchBot":["partial","*",14,1]},"llms":{"status":"missing","http":404,"bytes":160748},"home":{"status":"ok","http":200,"url":"https://www.rd.com/","html_tokens":63730,"markdown_tokens":40,"pruned_tokens":40}},
{"rank":2307,"domain":"viber.com","origin":"https://viber.com","robots":{"status":"ok","http":200,"url":"https://www.viber.com/robots.txt","bytes":181,"groups":1},"bots":{"GPTBot":["partial","*",3,1],"ChatGPT-User":["partial","*",3,1],"OAI-SearchBot":["partial","*",3,1],"ClaudeBot":["partial","*",3,1],"Claude-User":["partial","*",3,1],"Claude-SearchBot":["partial","*",3,1],"anthropic-ai":["partial","*",3,1],"PerplexityBot":["partial","*",3,1],"Perplexity-User":["partial","*",3,1],"Google-Extended":["partial","*",3,1],"Applebot-Extended":["partial","*",3,1],"CCBot":["partial","*",3,1],"Bytespider":["partial","*",3,1],"Amazonbot":["partial","*",3,1],"meta-externalagent":["partial","*",3,1],"cohere-ai":["partial","*",3,1],"Diffbot":["partial","*",3,1],"ContextFetchBot":["partial","*",3,1]},"llms":{"status":"html","http":200,"bytes":67149},"home":{"status":"ok","http":200,"url":"https://www.viber.com/en/","html_tokens":14951,"markdown_tokens":139,"pruned_tokens":139}},
{"rank":2308,"domain":"queensu.ca","origin":"https://queensu.ca","robots":{"status":"ok","http":200,"url":"https://www.queensu.ca/robots.txt","bytes":2746,"groups":1},"bots":{"GPTBot":["partial","*",80,0],"ChatGPT-User":["partial","*",80,0],"OAI-SearchBot":["partial","*",80,0],"ClaudeBot":["partial","*",80,0],"Claude-User":["partial","*",80,0],"Claude-SearchBot":["partial","*",80,0],"anthropic-ai":["partial","*",80,0],"PerplexityBot":["partial","*",80,0],"Perplexity-User":["partial","*",80,0],"Google-Extended":["partial","*",80,0],"Applebot-Extended":["partial","*",80,0],"CCBot":["partial","*",80,0],"Bytespider":["partial","*",80,0],"Amazonbot":["partial","*",80,0],"meta-externalagent":["partial","*",80,0],"cohere-ai":["partial","*",80,0],"Diffbot":["partial","*",80,0],"ContextFetchBot":["partial","*",80,0]},"llms":{"status":"present","http":200,"bytes":9524},"home":{"status":"ok","http":200,"url":"https://www.queensu.ca/","html_tokens":18591,"markdown_tokens":254,"pruned_tokens":116}},
{"rank":2309,"domain":"s-gift.app","origin":"https://www.s-gift.app","robots":{"status":"missing","http":404,"url":"https://www.s-gift.app/robots.txt","bytes":162,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":162},"home":{"status":"ok","http":200,"url":"https://www.s-gift.app/","html_tokens":2764,"markdown_tokens":21,"pruned_tokens":21}},
{"rank":2310,"domain":"thermofisher.com","origin":"http://thermofisher.com","robots":{"status":"ok","http":200,"url":"https://www.thermofisher.com/robots.txt","bytes":1370,"groups":2},"bots":{"GPTBot":["partial","*",17,0],"ChatGPT-User":["partial","*",17,0],"OAI-SearchBot":["partial","*",17,0],"ClaudeBot":["partial","*",17,0],"Claude-User":["partial","*",17,0],"Claude-SearchBot":["partial","*",17,0],"anthropic-ai":["partial","*",17,0],"PerplexityBot":["partial","*",17,0],"Perplexity-User":["partial","*",17,0],"Google-Extended":["partial","*",17,0],"Applebot-Extended":["partial","*",17,0],"CCBot":["partial","*",17,0],"Bytespider":["partial","*",17,0],"Amazonbot":["partial","*",17,0],"meta-externalagent":["partial","*",17,0],"cohere-ai":["partial","*",17,0],"Diffbot":["partial","*",17,0],"ContextFetchBot":["partial","*",17,0]},"llms":{"status":"disallowed","error":"robots.txt of www.thermofisher.com not readable (timeout)"},"home":{"status":"disallowed","error":"robots.txt of www.thermofisher.com not readable (timeout)"}},
{"rank":2311,"domain":"healthychildren.org","origin":"https://healthychildren.org","robots":{"status":"ok","http":200,"url":"https://www.healthychildren.org/robots.txt","bytes":624,"groups":8},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["partial","*",14,0],"OAI-SearchBot":["partial","*",14,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",14,0],"Claude-SearchBot":["partial","*",14,0],"anthropic-ai":["partial","*",14,0],"PerplexityBot":["partial","*",14,0],"Perplexity-User":["partial","*",14,0],"Google-Extended":["allowed","named",0,1],"Applebot-Extended":["partial","*",14,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["partial","*",14,0],"meta-externalagent":["partial","*",14,0],"cohere-ai":["partial","*",14,0],"Diffbot":["partial","*",14,0],"ContextFetchBot":["partial","*",14,0]},"llms":{"status":"present","http":200,"bytes":5137},"home":{"status":"ok","http":200,"url":"https://www.healthychildren.org/English/Pages/default.aspx","html_tokens":45105,"markdown_tokens":7,"pruned_tokens":7}},
{"rank":2312,"domain":"tokyo.lg.jp","origin":"https://tokyo.lg.jp","robots":{"status":"unreachable","http":null,"url":"https://tokyo.lg.jp/robots.txt","bytes":0,"groups":0,"error":"cannot resolve tokyo.lg.jp"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":2313,"domain":"dokuwiki.org","origin":"https://dokuwiki.org","robots":{"status":"ok","http":200,"url":"https://www.dokuwiki.org/robots.txt","bytes":138,"groups":4},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"http_error","http":402,"bytes":2205},"home":{"status":"http_error","http":402,"url":"https://www.dokuwiki.org/"}},
{"rank":2314,"domain":"truthsocial.com","origin":"https://truthsocial.com","robots":{"status":"ok","http":200,"url":"https://truthsocial.com/robots.txt","bytes":0,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"html","http":200,"bytes":2162},"home":{"status":"http_error","http":403,"url":"https://truthsocial.com/"}},
{"rank":2315,"domain":"ffmpeg.org","origin":"https://ffmpeg.org","robots":{"status":"ok","http":200,"url":"https://ffmpeg.org/robots.txt","bytes":378,"groups":1},"bots":{"GPTBot":["partial","*",11,0],"ChatGPT-User":["partial","*",11,0],"OAI-SearchBot":["partial","*",11,0],"ClaudeBot":["partial","*",11,0],"Claude-User":["partial","*",11,0],"Claude-SearchBot":["partial","*",11,0],"anthropic-ai":["partial","*",11,0],"PerplexityBot":["partial","*",11,0],"Perplexity-User":["partial","*",11,0],"Google-Extended":["partial","*",11,0],"Applebot-Extended":["partial","*",11,0],"CCBot":["partial","*",11,0],"Bytespider":["partial","*",11,0],"Amazonbot":["partial","*",11,0],"meta-externalagent":["partial","*",11,0],"cohere-ai":["partial","*",11,0],"Diffbot":["partial","*",11,0],"ContextFetchBot":["partial","*",11,0]},"llms":{"status":"missing","http":404,"bytes":196},"home":{"status":"ok","http":200,"url":"https://ffmpeg.org/","html_tokens":23339,"markdown_tokens":17395,"pruned_tokens":334}},
{"rank":2316,"domain":"3ds.com","origin":"https://3ds.com","robots":{"status":"ok","http":200,"url":"https://www.3ds.com/robots.txt","bytes":1649,"groups":2},"bots":{"GPTBot":["allowed","named",0,2],"ChatGPT-User":["allowed","named",0,2],"OAI-SearchBot":["partial","*",44,1],"ClaudeBot":["allowed","named",0,2],"Claude-User":["partial","*",44,1],"Claude-SearchBot":["partial","*",44,1],"anthropic-ai":["partial","*",44,1],"PerplexityBot":["allowed","named",0,2],"Perplexity-User":["partial","*",44,1],"Google-Extended":["allowed","named",0,2],"Applebot-Extended":["allowed","named",0,2],"CCBot":["allowed","named",0,2],"Bytespider":["allowed","named",0,2],"Amazonbot":["partial","*",44,1],"meta-externalagent":["partial","*",44,1],"cohere-ai":["allowed","named",0,2],"Diffbot":["partial","*",44,1],"ContextFetchBot":["partial","*",44,1]},"llms":{"status":"present","http":200,"bytes":12160},"home":{"status":"ok","http":200,"url":"https://www.3ds.com/","html_tokens":149737,"markdown_tokens":771,"pruned_tokens":102}},
{"rank":2317,"domain":"streamable.com","origin":"https://streamable.com","robots":{"status":"ok","http":200,"url":"https://streamable.com/robots.txt","bytes":509,"groups":5},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["blocked","named",1,0],"OAI-SearchBot":["partial","*",8,0],"ClaudeBot":["partial","*",8,0],"Claude-User":["partial","*",8,0],"Claude-SearchBot":["partial","*",8,0],"anthropic-ai":["partial","*",8,0],"PerplexityBot":["partial","*",8,0],"Perplexity-User":["partial","*",8,0],"Google-Extended":["partial","*",8,0],"Applebot-Extended":["partial","*",8,0],"CCBot":["blocked","named",1,0],"Bytespider":["partial","*",8,0],"Amazonbot":["partial","*",8,0],"meta-externalagent":["partial","*",8,0],"cohere-ai":["partial","*",8,0],"Diffbot":["partial","*",8,0],"ContextFetchBot":["partial","*",8,0]},"llms":{"status":"missing","http":404,"bytes":42715},"home":{"status":"ok","http":200,"url":"https://streamable.com/","html_tokens":1903,"markdown_tokens":0,"pruned_tokens":0}},
{"rank":2318,"domain":"benzinga.com","origin":"https://benzinga.com","robots":{"status":"ok","http":200,"url":"https://www.benzinga.com/robots.txt","bytes":1472,"groups":2},"bots":{"GPTBot":["partial","*",38,2],"ChatGPT-User":["partial","*",38,2],"OAI-SearchBot":["partial","*",38,2],"ClaudeBot":["partial","*",38,2],"Claude-User":["partial","*",38,2],"Claude-SearchBot":["partial","*",38,2],"anthropic-ai":["partial","*",38,2],"PerplexityBot":["partial","*",38,2],"Perplexity-User":["partial","*",38,2],"Google-Extended":["partial","*",38,2],"Applebot-Extended":["partial","*",38,2],"CCBot":["partial","*",38,2],"Bytespider":["partial","*",38,2],"Amazonbot":["partial","*",38,2],"meta-externalagent":["partial","*",38,2],"cohere-ai":["partial","*",38,2],"Diffbot":["partial","*",38,2],"ContextFetchBot":["partial","*",38,2]},"llms":{"status":"http_error","http":403,"bytes":4906},"home":{"status":"http_error","http":403,"url":"https://www.benzinga.com/"}},
{"rank":2319,"domain":"swr.de","origin":"https://swr.de","robots":{"status":"ok","http":200,"url":"https://www.swr.de/robots.txt","bytes":2699,"groups":21},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["partial","*",7,0],"OAI-SearchBot":["partial","*",7,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",7,0],"Claude-SearchBot":["partial","*",7,0],"anthropic-ai":["partial","*",7,0],"PerplexityBot":["partial","*",7,0],"Perplexity-User":["partial","*",7,0],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["partial","*",7,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["partial","*",7,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",7,0]},"llms":{"status":"missing","http":404,"bytes":236},"home":{"status":"ok","http":200,"url":"https://www.swr.de/","html_tokens":77870,"markdown_tokens":1385,"pruned_tokens":130}},
{"rank":2320,"domain":"hawaii.gov","origin":"https://hawaii.gov","robots":{"status":"ok","http":200,"url":"https://hawaii.gov/robots.txt","bytes":424,"groups":2},"bots":{"GPTBot":["partial","*",15,0],"ChatGPT-User":["partial","*",15,0],"OAI-SearchBot":["partial","*",15,0],"ClaudeBot":["partial","*",15,0],"Claude-User":["partial","*",15,0],"Claude-SearchBot":["partial","*",15,0],"anthropic-ai":["partial","*",15,0],"PerplexityBot":["partial","*",15,0],"Perplexity-User":["partial","*",15,0],"Google-Extended":["partial","*",15,0],"Applebot-Extended":["partial","*",15,0],"CCBot":["partial","*",15,0],"Bytespider":["partial","*",15,0],"Amazonbot":["partial","*",15,0],"meta-externalagent":["partial","*",15,0],"cohere-ai":["partial","*",15,0],"Diffbot":["partial","*",15,0],"ContextFetchBot":["partial","*",15,0]},"llms":{"status":"missing","http":404,"bytes":196},"home":{"status":"ok","http":200,"url":"https://hawaii.gov/","html_tokens":152,"markdown_tokens":0,"pruned_tokens":0}},
{"rank":2321,"domain":"formstack.com","origin":"https://formstack.com","robots":{"status":"ok","http":200,"url":"https://formstack.com/robots.txt","bytes":9472,"groups":2},"bots":{"GPTBot":["partial","*",30,0],"ChatGPT-User":["partial","*",30,0],"OAI-SearchBot":["partial","*",30,0],"ClaudeBot":["partial","*",30,0],"Claude-User":["partial","*",30,0],"Claude-SearchBot":["partial","*",30,0],"anthropic-ai":["partial","*",30,0],"PerplexityBot":["partial","*",30,0],"Perplexity-User":["partial","*",30,0],"Google-Extended":["partial","*",30,0],"Applebot-Extended":["partial","*",30,0],"CCBot":["partial","*",30,0],"Bytespider":["partial","*",30,0],"Amazonbot":["partial","*",30,0],"meta-externalagent":["partial","*",30,0],"cohere-ai":["partial","*",30,0],"Diffbot":["partial","*",30,0],"ContextFetchBot":["partial","*",30,0]},"llms":{"status":"present","http":200,"bytes":8555},"home":{"status":"ok","http":200,"url":"https://formstack.com/","html_tokens":41424,"markdown_tokens":473,"pruned_tokens":255}},
{"rank":2322,"domain":"cjb.net","origin":"https://cjb.net","robots":{"status":"unreachable","http":null,"url":"https://cjb.net/robots.txt","bytes":0,"groups":0,"error":"cannot resolve cjb.net"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":2323,"domain":"refinery29.com","origin":"https://refinery29.com","robots":{"status":"ok","http":200,"url":"https://www.refinery29.com/robots.txt","bytes":6302,"groups":134},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["blocked","named",1,0],"OAI-SearchBot":["blocked","named",1,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["blocked","named",1,0],"Claude-SearchBot":["blocked","named",1,0],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["blocked","named",1,0],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",2,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",15,0]},"llms":{"status":"missing","http":404,"bytes":42105},"home":{"status":"ok","http":200,"url":"https://www.refinery29.com/en-us","html_tokens":73872,"markdown_tokens":68,"pruned_tokens":68}},
{"rank":2324,"domain":"nottingham.ac.uk","origin":"https://nottingham.ac.uk","robots":{"status":"ok","http":200,"url":"https://www.nottingham.ac.uk/robots.txt","bytes":9536,"groups":50},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["blocked","named",1,0],"OAI-SearchBot":["blocked","named",1,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",68,0],"Claude-SearchBot":["partial","*",68,0],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["blocked","named",1,0],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",68,0]},"llms":{"status":"missing","http":404,"bytes":14},"home":{"status":"ok","http":200,"url":"https://www.nottingham.ac.uk/","html_tokens":48253,"markdown_tokens":341,"pruned_tokens":80}},
{"rank":2325,"domain":"google.com.ua","origin":"https://google.com.ua","robots":{"status":"ok","http":200,"url":"https://www.google.com.ua/robots.txt","bytes":6624,"groups":4},"bots":{"GPTBot":["partial","*",169,64],"ChatGPT-User":["partial","*",169,64],"OAI-SearchBot":["partial","*",169,64],"ClaudeBot":["partial","*",169,64],"Claude-User":["partial","*",169,64],"Claude-SearchBot":["partial","*",169,64],"anthropic-ai":["partial","*",169,64],"PerplexityBot":["partial","*",169,64],"Perplexity-User":["partial","*",169,64],"Google-Extended":["partial","*",169,64],"Applebot-Extended":["partial","*",169,64],"CCBot":["partial","*",169,64],"Bytespider":["partial","*",169,64],"Amazonbot":["partial","*",169,64],"meta-externalagent":["partial","*",169,64],"cohere-ai":["partial","*",169,64],"Diffbot":["partial","*",169,64],"ContextFetchBot":["partial","*",169,64]},"llms":{"status":"missing","http":404,"bytes":1569},"home":{"status":"ok","http":200,"url":"https://www.google.com/","html_tokens":21332,"markdown_tokens":30,"pruned_tokens":30}},
{"rank":2326,"domain":"webnode.page","origin":"https://webnode.page","robots":{"status":"missing","http":404,"url":"https://webnode.page/robots.txt","bytes":0,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"present","http":200,"bytes":3499},"home":{"status":"ok","http":200,"url":"https://www.webnode.com/","html_tokens":61774,"markdown_tokens":81,"pruned_tokens":81}},
{"rank":2327,"domain":"www.bl.uk","origin":"https://www.bl.uk","robots":{"status":"ok","http":200,"url":"https://www.bl.uk/robots.txt","bytes":63,"groups":1},"bots":{"GPTBot":["allowed","*",0,1],"ChatGPT-User":["allowed","*",0,1],"OAI-SearchBot":["allowed","*",0,1],"ClaudeBot":["allowed","*",0,1],"Claude-User":["allowed","*",0,1],"Claude-SearchBot":["allowed","*",0,1],"anthropic-ai":["allowed","*",0,1],"PerplexityBot":["allowed","*",0,1],"Perplexity-User":["allowed","*",0,1],"Google-Extended":["allowed","*",0,1],"Applebot-Extended":["allowed","*",0,1],"CCBot":["allowed","*",0,1],"Bytespider":["allowed","*",0,1],"Amazonbot":["allowed","*",0,1],"meta-externalagent":["allowed","*",0,1],"cohere-ai":["allowed","*",0,1],"Diffbot":["allowed","*",0,1],"ContextFetchBot":["allowed","*",0,1]},"llms":{"status":"missing","http":404,"bytes":41596},"home":{"status":"ok","http":200,"url":"https://www.bl.uk/","html_tokens":49262,"markdown_tokens":62,"pruned_tokens":62}},
{"rank":2328,"domain":"tu-dresden.de","origin":"https://tu-dresden.de","robots":{"status":"ok","http":200,"url":"https://tu-dresden.de/robots.txt","bytes":545,"groups":1},"bots":{"GPTBot":["partial","*",25,0],"ChatGPT-User":["partial","*",25,0],"OAI-SearchBot":["partial","*",25,0],"ClaudeBot":["partial","*",25,0],"Claude-User":["partial","*",25,0],"Claude-SearchBot":["partial","*",25,0],"anthropic-ai":["partial","*",25,0],"PerplexityBot":["partial","*",25,0],"Perplexity-User":["partial","*",25,0],"Google-Extended":["partial","*",25,0],"Applebot-Extended":["partial","*",25,0],"CCBot":["partial","*",25,0],"Bytespider":["partial","*",25,0],"Amazonbot":["partial","*",25,0],"meta-externalagent":["partial","*",25,0],"cohere-ai":["partial","*",25,0],"Diffbot":["partial","*",25,0],"ContextFetchBot":["partial","*",25,0]},"llms":{"status":"missing","http":404,"bytes":27762},"home":{"status":"ok","http":200,"url":"https://tu-dresden.de/","html_tokens":19811,"markdown_tokens":83,"pruned_tokens":84}},
{"rank":2329,"domain":"wikispaces.com","origin":"https://wikispaces.com","robots":{"status":"unreachable","http":null,"url":"https://wikispaces.com/robots.txt","bytes":0,"groups":0,"error":"cannot resolve wikispaces.com"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":2330,"domain":"cargocollective.com","origin":"https://cargocollective.com","robots":{"status":"ok","http":200,"url":"https://cargocollective.com/robots.txt","bytes":559,"groups":3},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["partial","named",5,0],"OAI-SearchBot":["blocked","named",1,0],"ClaudeBot":["blocked","*",1,0],"Claude-User":["blocked","*",1,0],"Claude-SearchBot":["blocked","*",1,0],"anthropic-ai":["blocked","*",1,0],"PerplexityBot":["blocked","*",1,0],"Perplexity-User":["blocked","*",1,0],"Google-Extended":["blocked","*",1,0],"Applebot-Extended":["blocked","*",1,0],"CCBot":["blocked","*",1,0],"Bytespider":["blocked","*",1,0],"Amazonbot":["blocked","*",1,0],"meta-externalagent":["blocked","*",1,0],"cohere-ai":["blocked","*",1,0],"Diffbot":["blocked","*",1,0],"ContextFetchBot":["blocked","*",1,0]},"llms":{"status":"disallowed"},"home":{"status":"disallowed"}},
{"rank":2331,"domain":"pxf.io","origin":"https://pxf.io","robots":{"status":"ok","http":200,"url":"https://pxf.io/robots.txt","bytes":289,"groups":8},"bots":{"GPTBot":["blocked","*",1,0],"ChatGPT-User":["blocked","*",1,0],"OAI-SearchBot":["blocked","*",1,0],"ClaudeBot":["blocked","*",1,0],"Claude-User":["blocked","*",1,0],"Claude-SearchBot":["blocked","*",1,0],"anthropic-ai":["blocked","*",1,0],"PerplexityBot":["blocked","*",1,0],"Perplexity-User":["blocked","*",1,0],"Google-Extended":["blocked","*",1,0],"Applebot-Extended":["blocked","*",1,0],"CCBot":["blocked","*",1,0],"Bytespider":["blocked","*",1,0],"Amazonbot":["blocked","*",1,0],"meta-externalagent":["blocked","*",1,0],"cohere-ai":["blocked","*",1,0],"Diffbot":["blocked","*",1,0],"ContextFetchBot":["blocked","*",1,0]},"llms":{"status":"disallowed"},"home":{"status":"disallowed"}},
{"rank":2332,"domain":"airbus.com","origin":"https://airbus.com","robots":{"status":"ok","http":200,"url":"https://www.airbus.com/robots.txt","bytes":6906,"groups":2},"bots":{"GPTBot":["partial","*",54,18],"ChatGPT-User":["partial","*",54,18],"OAI-SearchBot":["partial","*",54,18],"ClaudeBot":["partial","*",54,18],"Claude-User":["partial","*",54,18],"Claude-SearchBot":["partial","*",54,18],"anthropic-ai":["partial","*",54,18],"PerplexityBot":["partial","*",54,18],"Perplexity-User":["partial","*",54,18],"Google-Extended":["partial","*",54,18],"Applebot-Extended":["partial","*",54,18],"CCBot":["partial","*",54,18],"Bytespider":["blocked","named",1,0],"Amazonbot":["partial","*",54,18],"meta-externalagent":["partial","*",54,18],"cohere-ai":["partial","*",54,18],"Diffbot":["partial","*",54,18],"ContextFetchBot":["partial","*",54,18]},"llms":{"status":"missing","http":404,"bytes":196430},"home":{"status":"ok","http":200,"url":"https://www.airbus.com/en","html_tokens":62068,"markdown_tokens":72,"pruned_tokens":72}},
{"rank":2333,"domain":"docomo.ne.jp","origin":"https://docomo.ne.jp","robots":{"status":"missing","http":404,"url":"https://docomo.ne.jp/robots.txt","bytes":9,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":9},"home":{"status":"not_html","error":"unsupported content type text/plain"}},
{"rank":2334,"domain":"popsugar.com","origin":"https://popsugar.com","robots":{"status":"ok","http":200,"url":"https://www.popsugar.com/robots.txt","bytes":6088,"groups":107},"bots":{"GPTBot":["allowed","named",0,1],"ChatGPT-User":["partial","named",1,1],"OAI-SearchBot":["partial","*",32,3],"ClaudeBot":["partial","named",1,1],"Claude-User":["partial","named",1,1],"Claude-SearchBot":["partial","named",1,1],"anthropic-ai":["partial","named",1,1],"PerplexityBot":["partial","named",1,1],"Perplexity-User":["partial","named",1,1],"Google-Extended":["partial","named",1,1],"Applebot-Extended":["partial","named",2,1],"CCBot":["partial","named",1,1],"Bytespider":["partial","named",1,1],"Amazonbot":["allowed","named",0,1],"meta-externalagent":["partial","named",1,1],"cohere-ai":["partial","named",1,1],"Diffbot":["partial","named",1,1],"ContextFetchBot":["partial","*",32,3]},"llms":{"status":"missing","http":404,"bytes":162908},"home":{"status":"ok","http":200,"url":"https://www.popsugar.com/","html_tokens":337890,"markdown_tokens":14,"pruned_tokens":14}},
{"rank":2335,"domain":"monash.edu","origin":"https://monash.edu","robots":{"status":"refused","http":403,"url":"https://monash.edu/robots.txt","bytes":5505,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":2336,"domain":"simonandschuster.com","origin":"https://simonandschuster.com","robots":{"status":"ok","http":200,"url":"https://www.simonandschuster.com/robots.txt","bytes":728,"groups":1},"bots":{"GPTBot":["partial","*",2,7],"ChatGPT-User":["partial","*",2,7],"OAI-SearchBot":["partial","*",2,7],"ClaudeBot":["partial","*",2,7],"Claude-User":["partial","*",2,7],"Claude-SearchBot":["partial","*",2,7],"anthropic-ai":["partial","*",2,7],"PerplexityBot":["partial","*",2,7],"Perplexity-User":["partial","*",2,7],"Google-Extended":["partial","*",2,7],"Applebot-Extended":["partial","*",2,7],"CCBot":["partial","*",2,7],"Bytespider":["partial","*",2,7],"Amazonbot":["partial","*",2,7],"meta-externalagent":["partial","*",2,7],"cohere-ai":["partial","*",2,7],"Diffbot":["partial","*",2,7],"ContextFetchBot":["partial","*",2,7]},"llms":{"status":"missing","http":404,"bytes":8},"home":{"status":"http_error","http":403,"url":"https://simonandschuster.com/"}},
{"rank":2337,"domain":"sketchup.com","origin":"https://sketchup.com","robots":{"status":"ok","http":200,"url":"https://sketchup.trimble.com/robots.txt","bytes":97,"groups":1},"bots":{"GPTBot":["partial","*",1,1],"ChatGPT-User":["partial","*",1,1],"OAI-SearchBot":["partial","*",1,1],"ClaudeBot":["partial","*",1,1],"Claude-User":["partial","*",1,1],"Claude-SearchBot":["partial","*",1,1],"anthropic-ai":["partial","*",1,1],"PerplexityBot":["partial","*",1,1],"Perplexity-User":["partial","*",1,1],"Google-Extended":["partial","*",1,1],"Applebot-Extended":["partial","*",1,1],"CCBot":["partial","*",1,1],"Bytespider":["partial","*",1,1],"Amazonbot":["partial","*",1,1],"meta-externalagent":["partial","*",1,1],"cohere-ai":["partial","*",1,1],"Diffbot":["partial","*",1,1],"ContextFetchBot":["partial","*",1,1]},"llms":{"status":"present","http":200,"bytes":362},"home":{"status":"ok","http":200,"url":"https://sketchup.trimble.com/en","html_tokens":344651,"markdown_tokens":1086,"pruned_tokens":436}},
{"rank":2338,"domain":"bl.uk","origin":"https://bl.uk","robots":{"status":"ok","http":200,"url":"https://www.bl.uk/robots.txt","bytes":63,"groups":1},"bots":{"GPTBot":["allowed","*",0,1],"ChatGPT-User":["allowed","*",0,1],"OAI-SearchBot":["allowed","*",0,1],"ClaudeBot":["allowed","*",0,1],"Claude-User":["allowed","*",0,1],"Claude-SearchBot":["allowed","*",0,1],"anthropic-ai":["allowed","*",0,1],"PerplexityBot":["allowed","*",0,1],"Perplexity-User":["allowed","*",0,1],"Google-Extended":["allowed","*",0,1],"Applebot-Extended":["allowed","*",0,1],"CCBot":["allowed","*",0,1],"Bytespider":["allowed","*",0,1],"Amazonbot":["allowed","*",0,1],"meta-externalagent":["allowed","*",0,1],"cohere-ai":["allowed","*",0,1],"Diffbot":["allowed","*",0,1],"ContextFetchBot":["allowed","*",0,1]},"llms":{"status":"missing","http":404,"bytes":41682},"home":{"status":"ok","http":200,"url":"https://www.bl.uk/","html_tokens":49602,"markdown_tokens":62,"pruned_tokens":62}},
{"rank":2339,"domain":"ugent.be","origin":"https://ugent.be","robots":{"status":"ok","http":200,"url":"https://www.ugent.be/robots.txt","bytes":882,"groups":2},"bots":{"GPTBot":["partial","*",1,0],"ChatGPT-User":["partial","*",1,0],"OAI-SearchBot":["partial","*",1,0],"ClaudeBot":["partial","*",1,0],"Claude-User":["partial","*",1,0],"Claude-SearchBot":["partial","*",1,0],"anthropic-ai":["partial","*",1,0],"PerplexityBot":["partial","*",1,0],"Perplexity-User":["partial","*",1,0],"Google-Extended":["partial","*",1,0],"Applebot-Extended":["partial","*",1,0],"CCBot":["partial","*",1,0],"Bytespider":["partial","*",1,0],"Amazonbot":["partial","*",1,0],"meta-externalagent":["partial","*",1,0],"cohere-ai":["partial","*",1,0],"Diffbot":["partial","*",1,0],"ContextFetchBot":["partial","*",1,0]},"llms":{"status":"missing","http":404,"bytes":9994},"home":{"status":"ok","http":200,"url":"https://www.ugent.be/","html_tokens":8056,"markdown_tokens":24,"pruned_tokens":24}},
{"rank":2340,"domain":"blog.livedoor.jp","origin":"http://blog.livedoor.jp","robots":{"status":"ok","http":200,"url":"http://blog.livedoor.jp/robots.txt","bytes":59,"groups":1},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":2178},"home":{"status":"ok","http":200,"url":"http://blog.livedoor.jp/","html_tokens":2649,"markdown_tokens":340,"pruned_tokens":67}},
{"rank":2341,"domain":"digitaljournal.com","origin":"https://digitaljournal.com","robots":{"status":"refused","http":403,"url":"https://www.digitaljournal.com/robots.txt","bytes":7355,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":2342,"domain":"kompas.com","origin":"https://kompas.com","robots":{"status":"ok","http":200,"url":"https://kompas.com/robots.txt","bytes":6886,"groups":50},"bots":{"GPTBot":["partial","named",2,1],"ChatGPT-User":["partial","named",2,1],"OAI-SearchBot":["partial","named",3,1],"ClaudeBot":["blocked","named",2,0],"Claude-User":["blocked","named",1,0],"Claude-SearchBot":["blocked","named",1,0],"anthropic-ai":["blocked","named",2,0],"PerplexityBot":["blocked","named",2,0],"Perplexity-User":["blocked","named",1,0],"Google-Extended":["partial","*",6,0],"Applebot-Extended":["blocked","named",2,0],"CCBot":["blocked","named",2,0],"Bytespider":["blocked","named",2,0],"Amazonbot":["blocked","named",2,0],"meta-externalagent":["blocked","named",2,0],"cohere-ai":["blocked","named",2,0],"Diffbot":["blocked","named",2,0],"ContextFetchBot":["partial","*",6,0]},"llms":{"status":"missing","http":404,"bytes":162900},"home":{"status":"ok","http":200,"url":"https://kompas.com/","html_tokens":120379,"markdown_tokens":21,"pruned_tokens":21}},
{"rank":2343,"domain":"gsxt.gov.cn","origin":"https://www.gsxt.gov.cn","robots":{"status":"server_error","http":521,"url":"https://www.gsxt.gov.cn/robots.txt","bytes":826,"groups":0,"error":"HTTP 521"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":2344,"domain":"notepad-plus-plus.org","origin":"https://notepad-plus-plus.org","robots":{"status":"missing","http":404,"url":"https://notepad-plus-plus.org/robots.txt","bytes":236,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":236},"home":{"status":"ok","http":200,"url":"https://notepad-plus-plus.org/","html_tokens":4395,"markdown_tokens":191,"pruned_tokens":147}},
{"rank":2345,"domain":"gazeta.ru","origin":"https://gazeta.ru","robots":{"status":"ok","http":200,"url":"https://www.gazeta.ru/robots.txt","bytes":18660,"groups":2},"bots":{"GPTBot":["partial","*",250,6],"ChatGPT-User":["partial","*",250,6],"OAI-SearchBot":["partial","*",250,6],"ClaudeBot":["partial","*",250,6],"Claude-User":["partial","*",250,6],"Claude-SearchBot":["partial","*",250,6],"anthropic-ai":["partial","*",250,6],"PerplexityBot":["partial","*",250,6],"Perplexity-User":["partial","*",250,6],"Google-Extended":["partial","*",250,6],"Applebot-Extended":["partial","*",250,6],"CCBot":["partial","*",250,6],"Bytespider":["partial","*",250,6],"Amazonbot":["partial","*",250,6],"meta-externalagent":["partial","*",250,6],"cohere-ai":["partial","*",250,6],"Diffbot":["partial","*",250,6],"ContextFetchBot":["partial","*",250,6]},"llms":{"status":"missing","http":404,"bytes":76140},"home":{"status":"ok","http":200,"url":"https://www.gazeta.ru/auth/sso.shtml?origin=/","html_tokens":995,"markdown_tokens":0,"pruned_tokens":0}},
{"rank":2346,"domain":"medicare.gov","origin":"https://medicare.gov","robots":{"status":"ok","http":200,"url":"https://www.medicare.gov/robots.txt","bytes":1893,"groups":1},"bots":{"GPTBot":["partial","*",30,18],"ChatGPT-User":["partial","*",30,18],"OAI-SearchBot":["partial","*",30,18],"ClaudeBot":["partial","*",30,18],"Claude-User":["partial","*",30,18],"Claude-SearchBot":["partial","*",30,18],"anthropic-ai":["partial","*",30,18],"PerplexityBot":["partial","*",30,18],"Perplexity-User":["partial","*",30,18],"Google-Extended":["partial","*",30,18],"Applebot-Extended":["partial","*",30,18],"CCBot":["partial","*",30,18],"Bytespider":["partial","*",30,18],"Amazonbot":["partial","*",30,18],"meta-externalagent":["partial","*",30,18],"cohere-ai":["partial","*",30,18],"Diffbot":["partial","*",30,18],"ContextFetchBot":["partial","*",30,18]},"llms":{"status":"missing","http":404,"bytes":26321},"home":{"status":"ok","http":200,"url":"https://www.medicare.gov/","html_tokens":28213,"markdown_tokens":162,"pruned_tokens":162}},
{"rank":2347,"domain":"n-tv.de","origin":"https://n-tv.de","robots":{"status":"ok","http":200,"url":"https://www.n-tv.de/robots.txt","bytes":2415,"groups":4},"bots":{"GPTBot":["partial","named",1,12],"ChatGPT-User":["partial","*",6,0],"OAI-SearchBot":["partial","*",6,0],"ClaudeBot":["partial","*",6,0],"Claude-User":["partial","*",6,0],"Claude-SearchBot":["partial","*",6,0],"anthropic-ai":["partial","*",6,0],"PerplexityBot":["partial","*",6,0],"Perplexity-User":["partial","*",6,0],"Google-Extended":["partial","named",1,12],"Applebot-Extended":["partial","*",6,0],"CCBot":["partial","named",1,12],"Bytespider":["partial","*",6,0],"Amazonbot":["partial","*",6,0],"meta-externalagent":["partial","*",6,0],"cohere-ai":["partial","*",6,0],"Diffbot":["partial","*",6,0],"ContextFetchBot":["partial","*",6,0]},"llms":{"status":"missing","http":404,"bytes":580330},"home":{"status":"ok","http":200,"url":"https://www.n-tv.de/","html_tokens":334789,"markdown_tokens":8,"pruned_tokens":8}},
{"rank":2348,"domain":"ada.org","origin":"https://ada.org","robots":{"status":"ok","http":200,"url":"https://www.ada.org/robots.txt","bytes":68,"groups":1},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"missing","http":404,"bytes":103},"home":{"status":"ok","http":200,"url":"https://www.ada.org/","html_tokens":11193,"markdown_tokens":40,"pruned_tokens":40}},
{"rank":2349,"domain":"clarivate.com","origin":"https://clarivate.com","robots":{"status":"refused","http":403,"url":"https://clarivate.com/robots.txt","bytes":560,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":2350,"domain":"arbeitsagentur.de","origin":"https://arbeitsagentur.de","robots":{"status":"ok","http":200,"url":"https://www.arbeitsagentur.de/robots.txt","bytes":208,"groups":1},"bots":{"GPTBot":["allowed","*",0,1],"ChatGPT-User":["allowed","*",0,1],"OAI-SearchBot":["allowed","*",0,1],"ClaudeBot":["allowed","*",0,1],"Claude-User":["allowed","*",0,1],"Claude-SearchBot":["allowed","*",0,1],"anthropic-ai":["allowed","*",0,1],"PerplexityBot":["allowed","*",0,1],"Perplexity-User":["allowed","*",0,1],"Google-Extended":["allowed","*",0,1],"Applebot-Extended":["allowed","*",0,1],"CCBot":["allowed","*",0,1],"Bytespider":["allowed","*",0,1],"Amazonbot":["allowed","*",0,1],"meta-externalagent":["allowed","*",0,1],"cohere-ai":["allowed","*",0,1],"Diffbot":["allowed","*",0,1],"ContextFetchBot":["allowed","*",0,1]},"llms":{"status":"missing","http":404,"bytes":23288},"home":{"status":"ok","http":200,"url":"https://www.arbeitsagentur.de/","html_tokens":41206,"markdown_tokens":149,"pruned_tokens":43}},
{"rank":2351,"domain":"belgium.be","origin":"https://belgium.be","robots":{"status":"html","http":200,"url":"https://belgium.be/robots.txt","bytes":48727,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":2352,"domain":"beijing.gov.cn","origin":"https://www.beijing.gov.cn","robots":{"status":"ok","http":200,"url":"https://www.beijing.gov.cn/robots.txt","bytes":592,"groups":1},"bots":{"GPTBot":["partial","named",1,0],"ChatGPT-User":["partial","*",1,0],"OAI-SearchBot":["partial","*",1,0],"ClaudeBot":["partial","named",1,0],"Claude-User":["partial","*",1,0],"Claude-SearchBot":["partial","*",1,0],"anthropic-ai":["partial","named",1,0],"PerplexityBot":["partial","named",1,0],"Perplexity-User":["partial","*",1,0],"Google-Extended":["partial","named",1,0],"Applebot-Extended":["partial","*",1,0],"CCBot":["partial","named",1,0],"Bytespider":["partial","named",1,0],"Amazonbot":["partial","named",1,0],"meta-externalagent":["partial","*",1,0],"cohere-ai":["partial","named",1,0],"Diffbot":["partial","*",1,0],"ContextFetchBot":["partial","*",1,0]},"llms":{"status":"present","http":200,"bytes":10635},"home":{"status":"ok","http":200,"url":"https://www.beijing.gov.cn/","html_tokens":46262,"markdown_tokens":142,"pruned_tokens":142}},
{"rank":2353,"domain":"userway.org","origin":"https://userway.org","robots":{"status":"ok","http":200,"url":"https://userway.org/robots.txt","bytes":219,"groups":1},"bots":{"GPTBot":["partial","*",5,1],"ChatGPT-User":["partial","*",5,1],"OAI-SearchBot":["partial","*",5,1],"ClaudeBot":["partial","*",5,1],"Claude-User":["partial","*",5,1],"Claude-SearchBot":["partial","*",5,1],"anthropic-ai":["partial","*",5,1],"PerplexityBot":["partial","*",5,1],"Perplexity-User":["partial","*",5,1],"Google-Extended":["partial","*",5,1],"Applebot-Extended":["partial","*",5,1],"CCBot":["partial","*",5,1],"Bytespider":["partial","*",5,1],"Amazonbot":["partial","*",5,1],"meta-externalagent":["partial","*",5,1],"cohere-ai":["partial","*",5,1],"Diffbot":["partial","*",5,1],"ContextFetchBot":["partial","*",5,1]},"llms":{"status":"missing","http":404,"bytes":242475},"home":{"status":"ok","http":200,"url":"https://userway.org/","html_tokens":130420,"markdown_tokens":473,"pruned_tokens":174}},
{"rank":2354,"domain":"post-gazette.com","origin":"https://post-gazette.com","robots":{"status":"ok","http":200,"url":"https://post-gazette.com/robots.txt","bytes":206,"groups":1},"bots":{"GPTBot":["partial","*",3,0],"ChatGPT-User":["partial","*",3,0],"OAI-SearchBot":["partial","*",3,0],"ClaudeBot":["partial","*",3,0],"Claude-User":["partial","*",3,0],"Claude-SearchBot":["partial","*",3,0],"anthropic-ai":["partial","*",3,0],"PerplexityBot":["partial","*",3,0],"Perplexity-User":["partial","*",3,0],"Google-Extended":["partial","*",3,0],"Applebot-Extended":["partial","*",3,0],"CCBot":["partial","*",3,0],"Bytespider":["partial","*",3,0],"Amazonbot":["partial","*",3,0],"meta-externalagent":["partial","*",3,0],"cohere-ai":["partial","*",3,0],"Diffbot":["partial","*",3,0],"ContextFetchBot":["partial","*",3,0]},"llms":{"status":"html","http":200,"bytes":56268},"home":{"status":"ok","http":200,"url":"https://post-gazette.com/","html_tokens":71826,"markdown_tokens":398,"pruned_tokens":175}},
{"rank":2355,"domain":"boeing.com","origin":"https://boeing.com","robots":{"status":"ok","http":200,"url":"https://www.boeing.com/robots.txt","bytes":80,"groups":1},"bots":{"GPTBot":["partial","*",1,0],"ChatGPT-User":["partial","*",1,0],"OAI-SearchBot":["partial","*",1,0],"ClaudeBot":["partial","*",1,0],"Claude-User":["partial","*",1,0],"Claude-SearchBot":["partial","*",1,0],"anthropic-ai":["partial","*",1,0],"PerplexityBot":["partial","*",1,0],"Perplexity-User":["partial","*",1,0],"Google-Extended":["partial","*",1,0],"Applebot-Extended":["partial","*",1,0],"CCBot":["partial","*",1,0],"Bytespider":["partial","*",1,0],"Amazonbot":["partial","*",1,0],"meta-externalagent":["partial","*",1,0],"cohere-ai":["partial","*",1,0],"Diffbot":["partial","*",1,0],"ContextFetchBot":["partial","*",1,0]},"llms":{"status":"missing","http":404,"bytes":250960},"home":{"status":"ok","http":200,"url":"https://www.boeing.com/","html_tokens":97057,"markdown_tokens":202,"pruned_tokens":109}},
{"rank":2356,"domain":"chronicle.com","origin":"https://chronicle.com","robots":{"status":"ok","http":200,"url":"https://www.chronicle.com/robots.txt","bytes":2723,"groups":33},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["partial","*",1,1],"OAI-SearchBot":["partial","*",1,1],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",1,1],"Claude-SearchBot":["partial","*",1,1],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["partial","*",1,1],"Perplexity-User":["partial","*",1,1],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",1,1]},"llms":{"status":"missing","http":404,"bytes":271754},"home":{"status":"http_error","http":403,"url":"https://www.chronicle.com/"}},
{"rank":2357,"domain":"medicinenet.com","origin":"https://medicinenet.com","robots":{"status":"ok","http":200,"url":"https://www.medicinenet.com/robots.txt","bytes":75932,"groups":8},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["partial","*",2,0],"OAI-SearchBot":["partial","*",2,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",2,0],"Claude-SearchBot":["partial","*",2,0],"anthropic-ai":["partial","*",2,0],"PerplexityBot":["partial","*",2,0],"Perplexity-User":["partial","*",2,0],"Google-Extended":["partial","*",2,0],"Applebot-Extended":["partial","*",2,0],"CCBot":["blocked","named",1,0],"Bytespider":["partial","*",2,0],"Amazonbot":["partial","*",2,0],"meta-externalagent":["partial","*",2,0],"cohere-ai":["partial","*",2,0],"Diffbot":["partial","*",2,0],"ContextFetchBot":["partial","*",2,0]},"llms":{"status":"missing","http":404,"bytes":184670},"home":{"status":"ok","http":200,"url":"https://www.medicinenet.com/","html_tokens":115747,"markdown_tokens":97,"pruned_tokens":77}},
{"rank":2358,"domain":"depositphotos.com","origin":"https://depositphotos.com","robots":{"status":"ok","http":200,"url":"https://depositphotos.com/robots.txt","bytes":3118,"groups":2},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["partial","*",32,0],"OAI-SearchBot":["partial","*",32,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",32,0],"Claude-SearchBot":["partial","*",32,0],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["partial","*",32,0],"Perplexity-User":["partial","*",32,0],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["partial","*",32,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["partial","*",32,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",32,0]},"llms":{"status":"present","http":200,"bytes":10643},"home":{"status":"ok","http":200,"url":"https://depositphotos.com/","html_tokens":82254,"markdown_tokens":90,"pruned_tokens":90}},
{"rank":2359,"domain":"chsi.com.cn","origin":"https://www.chsi.com.cn","robots":{"status":"ok","http":200,"url":"https://www.chsi.com.cn/robots.txt","bytes":673,"groups":1},"bots":{"GPTBot":["partial","*",31,0],"ChatGPT-User":["partial","*",31,0],"OAI-SearchBot":["partial","*",31,0],"ClaudeBot":["partial","*",31,0],"Claude-User":["partial","*",31,0],"Claude-SearchBot":["partial","*",31,0],"anthropic-ai":["partial","*",31,0],"PerplexityBot":["partial","*",31,0],"Perplexity-User":["partial","*",31,0],"Google-Extended":["partial","*",31,0],"Applebot-Extended":["partial","*",31,0],"CCBot":["partial","*",31,0],"Bytespider":["partial","*",31,0],"Amazonbot":["partial","*",31,0],"meta-externalagent":["partial","*",31,0],"cohere-ai":["partial","*",31,0],"Diffbot":["partial","*",31,0],"ContextFetchBot":["partial","*",31,0]},"llms":{"status":"missing","http":404,"bytes":3168},"home":{"status":"ok","http":200,"url":"https://www.chsi.com.cn/","html_tokens":11073,"markdown_tokens":9,"pruned_tokens":9}},
{"rank":2360,"domain":"iz.ru","origin":"https://iz.ru","robots":{"status":"refused","http":403,"url":"https://iz.ru/robots.txt","bytes":49141,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":2361,"domain":"archive-it.org","origin":"https://archive-it.org","robots":{"status":"ok","http":200,"url":"https://archive-it.org/robots.txt","bytes":128,"groups":2},"bots":{"GPTBot":["partial","*",2,1],"ChatGPT-User":["partial","*",2,1],"OAI-SearchBot":["partial","*",2,1],"ClaudeBot":["partial","*",2,1],"Claude-User":["partial","*",2,1],"Claude-SearchBot":["partial","*",2,1],"anthropic-ai":["partial","*",2,1],"PerplexityBot":["partial","*",2,1],"Perplexity-User":["partial","*",2,1],"Google-Extended":["partial","*",2,1],"Applebot-Extended":["partial","*",2,1],"CCBot":["partial","*",2,1],"Bytespider":["partial","*",2,1],"Amazonbot":["partial","*",2,1],"meta-externalagent":["partial","*",2,1],"cohere-ai":["partial","*",2,1],"Diffbot":["partial","*",2,1],"ContextFetchBot":["partial","*",2,1]},"llms":{"status":"missing","http":404,"bytes":7731},"home":{"status":"ok","http":200,"url":"https://archive-it.org/","html_tokens":10109,"markdown_tokens":213,"pruned_tokens":156}},
{"rank":2362,"domain":"tp-link.com","origin":"https://tp-link.com","robots":{"status":"ok","http":200,"url":"https://www.tp-link.com/robots.txt","bytes":544,"groups":1},"bots":{"GPTBot":["partial","*",9,0],"ChatGPT-User":["partial","*",9,0],"OAI-SearchBot":["partial","*",9,0],"ClaudeBot":["partial","*",9,0],"Claude-User":["partial","*",9,0],"Claude-SearchBot":["partial","*",9,0],"anthropic-ai":["partial","*",9,0],"PerplexityBot":["partial","*",9,0],"Perplexity-User":["partial","*",9,0],"Google-Extended":["partial","*",9,0],"Applebot-Extended":["partial","*",9,0],"CCBot":["partial","*",9,0],"Bytespider":["partial","*",9,0],"Amazonbot":["partial","*",9,0],"meta-externalagent":["partial","*",9,0],"cohere-ai":["partial","*",9,0],"Diffbot":["partial","*",9,0],"ContextFetchBot":["partial","*",9,0]},"llms":{"status":"missing","http":404,"bytes":1867},"home":{"status":"ok","http":200,"url":"https://www.tp-link.com/us/","html_tokens":71877,"markdown_tokens":599,"pruned_tokens":113}},
{"rank":2363,"domain":"usenix.org","origin":"https://usenix.org","robots":{"status":"ok","http":200,"url":"https://www.usenix.org/robots.txt","bytes":2208,"groups":1},"bots":{"GPTBot":["partial","*",26,32],"ChatGPT-User":["partial","*",26,32],"OAI-SearchBot":["partial","*",26,32],"ClaudeBot":["partial","*",26,32],"Claude-User":["partial","*",26,32],"Claude-SearchBot":["partial","*",26,32],"anthropic-ai":["partial","*",26,32],"PerplexityBot":["partial","*",26,32],"Perplexity-User":["partial","*",26,32],"Google-Extended":["partial","*",26,32],"Applebot-Extended":["partial","*",26,32],"CCBot":["partial","*",26,32],"Bytespider":["partial","*",26,32],"Amazonbot":["partial","*",26,32],"meta-externalagent":["partial","*",26,32],"cohere-ai":["partial","*",26,32],"Diffbot":["partial","*",26,32],"ContextFetchBot":["partial","*",26,32]},"llms":{"status":"missing","http":404,"bytes":17256},"home":{"status":"ok","http":200,"url":"https://www.usenix.org/","html_tokens":7295,"markdown_tokens":209,"pruned_tokens":209}},
{"rank":2364,"domain":"wfp.org","origin":"https://wfp.org","robots":{"status":"ok","http":200,"url":"https://www.wfp.org//robots.txt","bytes":2582,"groups":1},"bots":{"GPTBot":["partial","*",47,22],"ChatGPT-User":["partial","*",47,22],"OAI-SearchBot":["partial","*",47,22],"ClaudeBot":["partial","*",47,22],"Claude-User":["partial","*",47,22],"Claude-SearchBot":["partial","*",47,22],"anthropic-ai":["partial","*",47,22],"PerplexityBot":["partial","*",47,22],"Perplexity-User":["partial","*",47,22],"Google-Extended":["partial","*",47,22],"Applebot-Extended":["partial","*",47,22],"CCBot":["partial","*",47,22],"Bytespider":["partial","*",47,22],"Amazonbot":["partial","*",47,22],"meta-externalagent":["partial","*",47,22],"cohere-ai":["partial","*",47,22],"Diffbot":["partial","*",47,22],"ContextFetchBot":["partial","*",47,22]},"llms":{"status":"missing","http":404,"bytes":185},"home":{"status":"ok","http":200,"url":"https://www.wfp.org/","html_tokens":24508,"markdown_tokens":242,"pruned_tokens":242}},
{"rank":2365,"domain":"tn.gov","origin":"https://tn.gov","robots":{"status":"refused","http":403,"url":"https://tn.gov/robots.txt","bytes":118,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":2366,"domain":"teachable.com","origin":"https://teachable.com","robots":{"status":"ok","http":200,"url":"https://www.teachable.com/robots.txt","bytes":797,"groups":2},"bots":{"GPTBot":["allowed","named",0,4],"ChatGPT-User":["allowed","named",0,4],"OAI-SearchBot":["partial","*",10,12],"ClaudeBot":["allowed","named",0,4],"Claude-User":["partial","*",10,12],"Claude-SearchBot":["partial","*",10,12],"anthropic-ai":["partial","*",10,12],"PerplexityBot":["allowed","named",0,4],"Perplexity-User":["partial","*",10,12],"Google-Extended":["allowed","named",0,4],"Applebot-Extended":["partial","*",10,12],"CCBot":["partial","*",10,12],"Bytespider":["partial","*",10,12],"Amazonbot":["partial","*",10,12],"meta-externalagent":["partial","*",10,12],"cohere-ai":["partial","*",10,12],"Diffbot":["partial","*",10,12],"ContextFetchBot":["partial","*",10,12]},"llms":{"status":"other_text","http":200,"bytes":1780},"home":{"status":"ok","http":200,"url":"https://www.teachable.com/","html_tokens":54647,"markdown_tokens":1778,"pruned_tokens":516}},
{"rank":2367,"domain":"transportation.gov","origin":"https://transportation.gov","robots":{"status":"refused","http":403,"url":"https://www.transportation.gov","bytes":376,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":2368,"domain":"css-tricks.com","origin":"https://css-tricks.com","robots":{"status":"ok","http":200,"url":"https://css-tricks.com/robots.txt","bytes":1197,"groups":14},"bots":{"GPTBot":["allowed","named",0,1],"ChatGPT-User":["allowed","named",0,1],"OAI-SearchBot":["partial","*",10,2],"ClaudeBot":["allowed","named",0,1],"Claude-User":["partial","*",10,2],"Claude-SearchBot":["partial","*",10,2],"anthropic-ai":["allowed","named",0,1],"PerplexityBot":["allowed","named",0,1],"Perplexity-User":["partial","*",10,2],"Google-Extended":["allowed","named",0,1],"Applebot-Extended":["partial","*",10,2],"CCBot":["allowed","named",0,1],"Bytespider":["allowed","named",0,1],"Amazonbot":["partial","*",10,2],"meta-externalagent":["partial","*",10,2],"cohere-ai":["partial","*",10,2],"Diffbot":["partial","*",10,2],"ContextFetchBot":["partial","*",10,2]},"llms":{"status":"present","http":200,"bytes":2808},"home":{"status":"ok","http":200,"url":"https://css-tricks.com/","html_tokens":38556,"markdown_tokens":110,"pruned_tokens":35}},
{"rank":2369,"domain":"wise.com","origin":"https://wise.com","robots":{"status":"ok","http":200,"url":"https://wise.com/robots.txt","bytes":2486,"groups":2},"bots":{"GPTBot":["partial","*",12,2],"ChatGPT-User":["partial","*",12,2],"OAI-SearchBot":["partial","*",12,2],"ClaudeBot":["partial","*",12,2],"Claude-User":["partial","*",12,2],"Claude-SearchBot":["partial","*",12,2],"anthropic-ai":["partial","*",12,2],"PerplexityBot":["partial","*",12,2],"Perplexity-User":["partial","*",12,2],"Google-Extended":["partial","*",12,2],"Applebot-Extended":["partial","*",12,2],"CCBot":["partial","*",12,2],"Bytespider":["partial","*",12,2],"Amazonbot":["partial","*",12,2],"meta-externalagent":["partial","*",12,2],"cohere-ai":["partial","*",12,2],"Diffbot":["partial","*",12,2],"ContextFetchBot":["partial","*",12,2]},"llms":{"status":"missing","http":404,"bytes":55436},"home":{"status":"ok","http":200,"url":"https://wise.com/","html_tokens":224118,"markdown_tokens":701,"pruned_tokens":237}},
{"rank":2370,"domain":"lanacion.com.ar","origin":"https://lanacion.com.ar","robots":{"status":"ok","http":200,"url":"https://www.lanacion.com.ar/robots.txt","bytes":1212,"groups":3},"bots":{"GPTBot":["partial","*",30,2],"ChatGPT-User":["partial","*",30,2],"OAI-SearchBot":["partial","*",30,2],"ClaudeBot":["partial","*",30,2],"Claude-User":["partial","*",30,2],"Claude-SearchBot":["partial","*",30,2],"anthropic-ai":["partial","*",30,2],"PerplexityBot":["partial","*",30,2],"Perplexity-User":["partial","*",30,2],"Google-Extended":["partial","*",30,2],"Applebot-Extended":["partial","*",30,2],"CCBot":["partial","*",30,2],"Bytespider":["partial","*",30,2],"Amazonbot":["partial","*",30,2],"meta-externalagent":["partial","*",30,2],"cohere-ai":["partial","*",30,2],"Diffbot":["partial","*",30,2],"ContextFetchBot":["partial","*",30,2]},"llms":{"status":"missing","http":404,"bytes":255017},"home":{"status":"ok","http":200,"url":"https://www.lanacion.com.ar/","html_tokens":260036,"markdown_tokens":8,"pruned_tokens":8}},
{"rank":2371,"domain":"astm.org","origin":"https://astm.org","robots":{"status":"refused","http":403,"url":"https://www.astm.org/robots.txt","bytes":919,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":2372,"domain":"988lifeline.org","origin":"https://988lifeline.org","robots":{"status":"ok","http":200,"url":"https://988lifeline.org/robots.txt","bytes":161,"groups":1},"bots":{"GPTBot":["partial","*",1,1],"ChatGPT-User":["partial","*",1,1],"OAI-SearchBot":["partial","*",1,1],"ClaudeBot":["partial","*",1,1],"Claude-User":["partial","*",1,1],"Claude-SearchBot":["partial","*",1,1],"anthropic-ai":["partial","*",1,1],"PerplexityBot":["partial","*",1,1],"Perplexity-User":["partial","*",1,1],"Google-Extended":["partial","*",1,1],"Applebot-Extended":["partial","*",1,1],"CCBot":["partial","*",1,1],"Bytespider":["partial","*",1,1],"Amazonbot":["partial","*",1,1],"meta-externalagent":["partial","*",1,1],"cohere-ai":["partial","*",1,1],"Diffbot":["partial","*",1,1],"ContextFetchBot":["partial","*",1,1]},"llms":{"status":"http_error","http":403,"bytes":5568},"home":{"status":"http_error","http":403,"url":"https://988lifeline.org/"}},
{"rank":2373,"domain":"mlive.com","origin":"https://mlive.com","robots":{"status":"ok","http":200,"url":"https://www.mlive.com/robots.txt","bytes":2875,"groups":4},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["partial","named",1,14],"OAI-SearchBot":["partial","named",1,14],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","named",1,14],"Claude-SearchBot":["partial","named",1,14],"anthropic-ai":["partial","*",30,3],"PerplexityBot":["partial","named",1,14],"Perplexity-User":["partial","named",1,14],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["partial","*",30,3],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",30,3]},"llms":{"status":"missing","http":404,"bytes":274599},"home":{"status":"http_error","http":403,"url":"https://www.mlive.com/"}},
{"rank":2374,"domain":"spacex.com","origin":"https://spacex.com","robots":{"status":"missing","http":404,"url":"https://www.spacex.com/robots.txt","bytes":682,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":682},"home":{"status":"ok","http":200,"url":"https://www.spacex.com/","html_tokens":874,"markdown_tokens":0,"pruned_tokens":0}},
{"rank":2375,"domain":"ucf.edu","origin":"https://ucf.edu","robots":{"status":"refused","http":403,"url":"https://www.ucf.edu/robots.txt","bytes":560,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":2376,"domain":"nextdoor.com","origin":"https://nextdoor.com","robots":{"status":"ok","http":200,"url":"https://nextdoor.com/robots.txt","bytes":2990,"groups":13},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["partial","*",1,2],"OAI-SearchBot":["partial","*",1,2],"ClaudeBot":["partial","*",1,2],"Claude-User":["partial","*",1,2],"Claude-SearchBot":["partial","*",1,2],"anthropic-ai":["partial","*",1,2],"PerplexityBot":["partial","*",1,2],"Perplexity-User":["partial","*",1,2],"Google-Extended":["partial","*",1,2],"Applebot-Extended":["partial","*",1,2],"CCBot":["partial","*",1,2],"Bytespider":["partial","*",1,2],"Amazonbot":["partial","*",1,2],"meta-externalagent":["partial","*",1,2],"cohere-ai":["partial","*",1,2],"Diffbot":["partial","*",1,2],"ContextFetchBot":["partial","*",1,2]},"llms":{"status":"disallowed"},"home":{"status":"disallowed"}},
{"rank":2377,"domain":"hackaday.com","origin":"https://hackaday.com","robots":{"status":"ok","http":200,"url":"https://hackaday.com/robots.txt","bytes":232,"groups":1},"bots":{"GPTBot":["partial","*",1,1],"ChatGPT-User":["partial","*",1,1],"OAI-SearchBot":["partial","*",1,1],"ClaudeBot":["partial","*",1,1],"Claude-User":["partial","*",1,1],"Claude-SearchBot":["partial","*",1,1],"anthropic-ai":["partial","*",1,1],"PerplexityBot":["partial","*",1,1],"Perplexity-User":["partial","*",1,1],"Google-Extended":["partial","*",1,1],"Applebot-Extended":["partial","*",1,1],"CCBot":["partial","*",1,1],"Bytespider":["partial","*",1,1],"Amazonbot":["partial","*",1,1],"meta-externalagent":["partial","*",1,1],"cohere-ai":["partial","*",1,1],"Diffbot":["partial","*",1,1],"ContextFetchBot":["partial","*",1,1]},"llms":{"status":"missing","http":404,"bytes":101315},"home":{"status":"ok","http":200,"url":"https://hackaday.com/","html_tokens":24695,"markdown_tokens":57,"pruned_tokens":57}},
{"rank":2378,"domain":"theweek.com","origin":"https://theweek.com","robots":{"status":"ok","http":200,"url":"https://theweek.com/robots.txt","bytes":1963,"groups":5},"bots":{"GPTBot":["partial","*",23,0],"ChatGPT-User":["partial","*",23,0],"OAI-SearchBot":["partial","*",23,0],"ClaudeBot":["partial","*",23,0],"Claude-User":["partial","*",23,0],"Claude-SearchBot":["partial","*",23,0],"anthropic-ai":["partial","*",23,0],"PerplexityBot":["partial","*",23,0],"Perplexity-User":["partial","*",23,0],"Google-Extended":["partial","*",23,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["partial","*",23,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["partial","*",23,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",23,0]},"llms":{"status":"missing","http":404,"bytes":1296949},"home":{"status":"ok","http":200,"url":"https://theweek.com/","html_tokens":390091,"markdown_tokens":68,"pruned_tokens":68}},
{"rank":2379,"domain":"smartsheet.com","origin":"https://smartsheet.com","robots":{"status":"ok","http":200,"url":"https://www.smartsheet.com/robots.txt","bytes":1888,"groups":1},"bots":{"GPTBot":["partial","*",35,18],"ChatGPT-User":["partial","*",35,18],"OAI-SearchBot":["partial","*",35,18],"ClaudeBot":["partial","*",35,18],"Claude-User":["partial","*",35,18],"Claude-SearchBot":["partial","*",35,18],"anthropic-ai":["partial","*",35,18],"PerplexityBot":["partial","*",35,18],"Perplexity-User":["partial","*",35,18],"Google-Extended":["partial","*",35,18],"Applebot-Extended":["partial","*",35,18],"CCBot":["partial","*",35,18],"Bytespider":["partial","*",35,18],"Amazonbot":["partial","*",35,18],"meta-externalagent":["partial","*",35,18],"cohere-ai":["partial","*",35,18],"Diffbot":["partial","*",35,18],"ContextFetchBot":["partial","*",35,18]},"llms":{"status":"present","http":200,"bytes":15331},"home":{"status":"ok","http":200,"url":"https://www.smartsheet.com/","html_tokens":53733,"markdown_tokens":199,"pruned_tokens":86}},
{"rank":2380,"domain":"auswaertiges-amt.de","origin":"https://auswaertiges-amt.de","robots":{"status":"ok","http":200,"url":"https://www.auswaertiges-amt.de/robots.txt","bytes":290,"groups":1},"bots":{"GPTBot":["partial","*",5,0],"ChatGPT-User":["partial","*",5,0],"OAI-SearchBot":["partial","*",5,0],"ClaudeBot":["partial","*",5,0],"Claude-User":["partial","*",5,0],"Claude-SearchBot":["partial","*",5,0],"anthropic-ai":["partial","*",5,0],"PerplexityBot":["partial","*",5,0],"Perplexity-User":["partial","*",5,0],"Google-Extended":["partial","*",5,0],"Applebot-Extended":["partial","*",5,0],"CCBot":["partial","*",5,0],"Bytespider":["partial","*",5,0],"Amazonbot":["partial","*",5,0],"meta-externalagent":["partial","*",5,0],"cohere-ai":["partial","*",5,0],"Diffbot":["partial","*",5,0],"ContextFetchBot":["partial","*",5,0]},"llms":{"status":"missing","http":404,"bytes":73631},"home":{"status":"ok","http":200,"url":"https://www.auswaertiges-amt.de/","html_tokens":31386,"markdown_tokens":87,"pruned_tokens":87}},
{"rank":2381,"domain":"liu.se","origin":"https://liu.se","robots":{"status":"ok","http":200,"url":"https://liu.se/robots.txt","bytes":174,"groups":1},"bots":{"GPTBot":["partial","*",5,0],"ChatGPT-User":["partial","*",5,0],"OAI-SearchBot":["partial","*",5,0],"ClaudeBot":["partial","*",5,0],"Claude-User":["partial","*",5,0],"Claude-SearchBot":["partial","*",5,0],"anthropic-ai":["partial","*",5,0],"PerplexityBot":["partial","*",5,0],"Perplexity-User":["partial","*",5,0],"Google-Extended":["partial","*",5,0],"Applebot-Extended":["partial","*",5,0],"CCBot":["partial","*",5,0],"Bytespider":["partial","*",5,0],"Amazonbot":["partial","*",5,0],"meta-externalagent":["partial","*",5,0],"cohere-ai":["partial","*",5,0],"Diffbot":["partial","*",5,0],"ContextFetchBot":["partial","*",5,0]},"llms":{"status":"missing","http":404,"bytes":32499},"home":{"status":"ok","http":200,"url":"https://liu.se/","html_tokens":13473,"markdown_tokens":92,"pruned_tokens":92}},
{"rank":2382,"domain":"afr.com","origin":"https://afr.com","robots":{"status":"ok","http":200,"url":"https://www.afr.com/robots.txt","bytes":4944,"groups":60},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["blocked","named",1,0],"OAI-SearchBot":["blocked","named",1,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",24,1],"Claude-SearchBot":["partial","*",24,1],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["blocked","named",1,0],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",24,1]},"llms":{"status":"missing","http":404,"bytes":157901},"home":{"status":"ok","http":200,"url":"https://www.afr.com/","html_tokens":280383,"markdown_tokens":415,"pruned_tokens":151}},
{"rank":2383,"domain":"colibriwp.com","origin":"https://colibriwp.com","robots":{"status":"ok","http":200,"url":"https://colibriwp.com/robots.txt","bytes":114,"groups":1},"bots":{"GPTBot":["partial","*",1,1],"ChatGPT-User":["partial","*",1,1],"OAI-SearchBot":["partial","*",1,1],"ClaudeBot":["partial","*",1,1],"Claude-User":["partial","*",1,1],"Claude-SearchBot":["partial","*",1,1],"anthropic-ai":["partial","*",1,1],"PerplexityBot":["partial","*",1,1],"Perplexity-User":["partial","*",1,1],"Google-Extended":["partial","*",1,1],"Applebot-Extended":["partial","*",1,1],"CCBot":["partial","*",1,1],"Bytespider":["partial","*",1,1],"Amazonbot":["partial","*",1,1],"meta-externalagent":["partial","*",1,1],"cohere-ai":["partial","*",1,1],"Diffbot":["partial","*",1,1],"ContextFetchBot":["partial","*",1,1]},"llms":{"status":"missing","http":404,"bytes":173938},"home":{"status":"ok","http":200,"url":"https://colibriwp.com/","html_tokens":242239,"markdown_tokens":2193,"pruned_tokens":603}},
{"rank":2384,"domain":"sa.gov.au","origin":"https://sa.gov.au","robots":{"status":"refused","http":403,"url":"https://www.sa.gov.au/robots.txt","bytes":5764,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":2385,"domain":"datenschutz-generator.de","origin":"https://datenschutz-generator.de","robots":{"status":"ok","http":200,"url":"https://datenschutz-generator.de/robots.txt","bytes":452,"groups":2},"bots":{"GPTBot":["partial","*",6,1],"ChatGPT-User":["partial","*",6,1],"OAI-SearchBot":["partial","*",6,1],"ClaudeBot":["partial","*",6,1],"Claude-User":["partial","*",6,1],"Claude-SearchBot":["partial","*",6,1],"anthropic-ai":["partial","*",6,1],"PerplexityBot":["partial","*",6,1],"Perplexity-User":["partial","*",6,1],"Google-Extended":["partial","*",6,1],"Applebot-Extended":["partial","*",6,1],"CCBot":["partial","*",6,1],"Bytespider":["partial","*",6,1],"Amazonbot":["partial","*",6,1],"meta-externalagent":["partial","*",6,1],"cohere-ai":["partial","*",6,1],"Diffbot":["partial","*",6,1],"ContextFetchBot":["partial","*",6,1]},"llms":{"status":"missing","http":404,"bytes":130795},"home":{"status":"ok","http":200,"url":"https://datenschutz-generator.de/","html_tokens":62821,"markdown_tokens":276,"pruned_tokens":140}},
{"rank":2386,"domain":"elevenlabs.io","origin":"https://elevenlabs.io","robots":{"status":"ok","http":200,"url":"https://elevenlabs.io/robots.txt","bytes":3222,"groups":5},"bots":{"GPTBot":["partial","*",17,6],"ChatGPT-User":["partial","*",17,6],"OAI-SearchBot":["partial","*",17,6],"ClaudeBot":["partial","*",17,6],"Claude-User":["partial","*",17,6],"Claude-SearchBot":["partial","*",17,6],"anthropic-ai":["partial","*",17,6],"PerplexityBot":["partial","*",17,6],"Perplexity-User":["partial","*",17,6],"Google-Extended":["partial","*",17,6],"Applebot-Extended":["partial","*",17,6],"CCBot":["partial","*",17,6],"Bytespider":["partial","*",17,6],"Amazonbot":["partial","*",17,6],"meta-externalagent":["partial","*",17,6],"cohere-ai":["partial","*",17,6],"Diffbot":["partial","*",17,6],"ContextFetchBot":["partial","*",17,6]},"llms":{"status":"present","http":200,"bytes":11314},"home":{"status":"not_html","error":"unsupported content type text/markdown"}},
{"rank":2387,"domain":"paho.org","origin":"https://paho.org","robots":{"status":"refused","http":403,"url":"https://paho.org/robots.txt","bytes":560,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":2388,"domain":"fasthosts.co.uk","origin":"https://fasthosts.co.uk","robots":{"status":"ok","http":200,"url":"https://www.fasthosts.co.uk/robots.txt","bytes":2855,"groups":4},"bots":{"GPTBot":["partial","*",53,33],"ChatGPT-User":["partial","*",53,33],"OAI-SearchBot":["allowed","named",0,1],"ClaudeBot":["partial","*",53,33],"Claude-User":["partial","*",53,33],"Claude-SearchBot":["partial","*",53,33],"anthropic-ai":["partial","*",53,33],"PerplexityBot":["allowed","named",0,1],"Perplexity-User":["partial","*",53,33],"Google-Extended":["partial","*",53,33],"Applebot-Extended":["partial","*",53,33],"CCBot":["partial","*",53,33],"Bytespider":["partial","*",53,33],"Amazonbot":["partial","*",53,33],"meta-externalagent":["partial","*",53,33],"cohere-ai":["partial","*",53,33],"Diffbot":["partial","*",53,33],"ContextFetchBot":["partial","*",53,33]},"llms":{"status":"present","http":200,"bytes":11040},"home":{"status":"ok","http":200,"url":"https://www.fasthosts.co.uk/","html_tokens":40009,"markdown_tokens":488,"pruned_tokens":107}},
{"rank":2389,"domain":"nginx.net","origin":"http://nginx.net","robots":{"status":"missing","http":404,"url":"http://nginx.net/robots.txt","bytes":153,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"disallowed","error":"robots.txt of nginx.com not readable (html)"},"home":{"status":"ok","http":200,"url":"http://nginx.org/","html_tokens":1680,"markdown_tokens":517,"pruned_tokens":175}},
{"rank":2390,"domain":"pmi.org","origin":"https://pmi.org","robots":{"status":"ok","http":200,"url":"https://www.pmi.org/robots.txt","bytes":240,"groups":1},"bots":{"GPTBot":["partial","*",4,1],"ChatGPT-User":["partial","*",4,1],"OAI-SearchBot":["partial","*",4,1],"ClaudeBot":["partial","*",4,1],"Claude-User":["partial","*",4,1],"Claude-SearchBot":["partial","*",4,1],"anthropic-ai":["partial","*",4,1],"PerplexityBot":["partial","*",4,1],"Perplexity-User":["partial","*",4,1],"Google-Extended":["partial","*",4,1],"Applebot-Extended":["partial","*",4,1],"CCBot":["partial","*",4,1],"Bytespider":["partial","*",4,1],"Amazonbot":["partial","*",4,1],"meta-externalagent":["partial","*",4,1],"cohere-ai":["partial","*",4,1],"Diffbot":["partial","*",4,1],"ContextFetchBot":["partial","*",4,1]},"llms":{"status":"present","http":200,"bytes":6446},"home":{"status":"http_error","http":403,"url":"https://www.pmi.org/"}},
{"rank":2391,"domain":"cnr.cn","origin":"https://www.cnr.cn","robots":{"status":"ok","http":200,"url":"https://www.cnr.cn/robots.txt","bytes":404,"groups":1},"bots":{"GPTBot":["partial","*",5,1],"ChatGPT-User":["partial","*",5,1],"OAI-SearchBot":["partial","*",5,1],"ClaudeBot":["partial","*",5,1],"Claude-User":["partial","*",5,1],"Claude-SearchBot":["partial","*",5,1],"anthropic-ai":["partial","*",5,1],"PerplexityBot":["partial","*",5,1],"Perplexity-User":["partial","*",5,1],"Google-Extended":["partial","*",5,1],"Applebot-Extended":["partial","*",5,1],"CCBot":["partial","*",5,1],"Bytespider":["partial","*",5,1],"Amazonbot":["partial","*",5,1],"meta-externalagent":["partial","*",5,1],"cohere-ai":["partial","*",5,1],"Diffbot":["partial","*",5,1],"ContextFetchBot":["partial","*",5,1]},"llms":{"status":"html","http":200,"bytes":6753},"home":{"status":"ok","http":200,"url":"https://www.cnr.cn/","html_tokens":37152,"markdown_tokens":73,"pruned_tokens":73}},
{"rank":2392,"domain":"mistral.ai","origin":"https://mistral.ai","robots":{"status":"ok","http":200,"url":"https://mistral.ai/robots.txt","bytes":70,"groups":1},"bots":{"GPTBot":["allowed","*",0,1],"ChatGPT-User":["allowed","*",0,1],"OAI-SearchBot":["allowed","*",0,1],"ClaudeBot":["allowed","*",0,1],"Claude-User":["allowed","*",0,1],"Claude-SearchBot":["allowed","*",0,1],"anthropic-ai":["allowed","*",0,1],"PerplexityBot":["allowed","*",0,1],"Perplexity-User":["allowed","*",0,1],"Google-Extended":["allowed","*",0,1],"Applebot-Extended":["allowed","*",0,1],"CCBot":["allowed","*",0,1],"Bytespider":["allowed","*",0,1],"Amazonbot":["allowed","*",0,1],"meta-externalagent":["allowed","*",0,1],"cohere-ai":["allowed","*",0,1],"Diffbot":["allowed","*",0,1],"ContextFetchBot":["allowed","*",0,1]},"llms":{"status":"present","http":200,"bytes":5477},"home":{"status":"ok","http":200,"url":"https://mistral.ai/","html_tokens":81473,"markdown_tokens":812,"pruned_tokens":632}},
{"rank":2393,"domain":"interfax.ru","origin":"https://interfax.ru","robots":{"status":"ok","http":200,"url":"https://www.interfax.ru/robots.txt","bytes":472,"groups":4},"bots":{"GPTBot":["partial","*",4,0],"ChatGPT-User":["partial","*",4,0],"OAI-SearchBot":["partial","*",4,0],"ClaudeBot":["partial","*",4,0],"Claude-User":["partial","*",4,0],"Claude-SearchBot":["partial","*",4,0],"anthropic-ai":["partial","*",4,0],"PerplexityBot":["partial","*",4,0],"Perplexity-User":["partial","*",4,0],"Google-Extended":["partial","*",4,0],"Applebot-Extended":["partial","*",4,0],"CCBot":["partial","*",4,0],"Bytespider":["partial","*",4,0],"Amazonbot":["partial","*",4,0],"meta-externalagent":["partial","*",4,0],"cohere-ai":["partial","*",4,0],"Diffbot":["partial","*",4,0],"ContextFetchBot":["partial","*",4,0]},"llms":{"status":"missing","http":404,"bytes":2596},"home":{"status":"ok","http":200,"url":"https://www.interfax.ru/","html_tokens":19001,"markdown_tokens":425,"pruned_tokens":61}},
{"rank":2394,"domain":"bol.com","origin":"https://bol.com","robots":{"status":"ok","http":200,"url":"https://www.bol.com/robots.txt","bytes":6805,"groups":6},"bots":{"GPTBot":["partial","*",55,8],"ChatGPT-User":["partial","*",55,8],"OAI-SearchBot":["partial","*",55,8],"ClaudeBot":["partial","*",55,8],"Claude-User":["partial","*",55,8],"Claude-SearchBot":["partial","*",55,8],"anthropic-ai":["partial","*",55,8],"PerplexityBot":["partial","*",55,8],"Perplexity-User":["partial","*",55,8],"Google-Extended":["partial","*",55,8],"Applebot-Extended":["partial","*",55,8],"CCBot":["partial","*",55,8],"Bytespider":["partial","*",55,8],"Amazonbot":["partial","*",55,8],"meta-externalagent":["partial","*",55,8],"cohere-ai":["partial","*",55,8],"Diffbot":["partial","*",55,8],"ContextFetchBot":["partial","*",55,8]},"llms":{"status":"missing","http":404,"bytes":11528},"home":{"status":"http_error","http":403,"url":"https://www.bol.com/"}},
{"rank":2395,"domain":"c212.net","origin":"https://c212.net","robots":{"status":"missing","http":404,"url":"https://c212.net/robots.txt","bytes":456,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":454},"home":{"status":"ok","http":200,"url":"https://c212.net/","html_tokens":52,"markdown_tokens":0,"pruned_tokens":0}},
{"rank":2396,"domain":"thecut.com","origin":"https://thecut.com","robots":{"status":"ok","http":200,"url":"https://thecut.com/robots.txt","bytes":7569,"groups":108},"bots":{"GPTBot":["allowed","named",0,1],"ChatGPT-User":["partial","named",1,2],"OAI-SearchBot":["partial","*",7,0],"ClaudeBot":["partial","named",1,2],"Claude-User":["partial","named",1,2],"Claude-SearchBot":["partial","named",1,2],"anthropic-ai":["partial","named",1,2],"PerplexityBot":["partial","named",1,2],"Perplexity-User":["partial","named",1,2],"Google-Extended":["partial","named",1,2],"Applebot-Extended":["partial","named",1,2],"CCBot":["partial","named",1,2],"Bytespider":["partial","named",1,2],"Amazonbot":["allowed","named",0,1],"meta-externalagent":["partial","named",1,2],"cohere-ai":["partial","named",1,2],"Diffbot":["partial","named",1,2],"ContextFetchBot":["partial","*",7,0]},"llms":{"status":"missing","http":404,"bytes":47002},"home":{"status":"ok","http":200,"url":"https://www.thecut.com/","html_tokens":132068,"markdown_tokens":115,"pruned_tokens":98}},
{"rank":2397,"domain":"tabelog.com","origin":"https://tabelog.com","robots":{"status":"refused","http":403,"url":"https://tabelog.com/robots.txt","bytes":5825,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":2398,"domain":"southampton.ac.uk","origin":"https://southampton.ac.uk","robots":{"status":"ok","http":200,"url":"https://www.southampton.ac.uk/robots.txt","bytes":2736,"groups":1},"bots":{"GPTBot":["partial","*",69,18],"ChatGPT-User":["partial","*",69,18],"OAI-SearchBot":["partial","*",69,18],"ClaudeBot":["partial","*",69,18],"Claude-User":["partial","*",69,18],"Claude-SearchBot":["partial","*",69,18],"anthropic-ai":["partial","*",69,18],"PerplexityBot":["partial","*",69,18],"Perplexity-User":["partial","*",69,18],"Google-Extended":["partial","*",69,18],"Applebot-Extended":["partial","*",69,18],"CCBot":["partial","*",69,18],"Bytespider":["partial","*",69,18],"Amazonbot":["partial","*",69,18],"meta-externalagent":["partial","*",69,18],"cohere-ai":["partial","*",69,18],"Diffbot":["partial","*",69,18],"ContextFetchBot":["partial","*",69,18]},"llms":{"status":"missing","http":404,"bytes":14114},"home":{"status":"ok","http":200,"url":"https://www.southampton.ac.uk/","html_tokens":72852,"markdown_tokens":531,"pruned_tokens":124}},
{"rank":2399,"domain":"ucm.es","origin":"https://ucm.es","robots":{"status":"missing","http":404,"url":"https://ucm.es/robots.txt","bytes":36165,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":36165},"home":{"status":"ok","http":200,"url":"https://ucm.es/","html_tokens":17106,"markdown_tokens":185,"pruned_tokens":132}},
{"rank":2400,"domain":"reason.com","origin":"https://reason.com","robots":{"status":"ok","http":200,"url":"https://reason.com/robots.txt","bytes":248,"groups":1},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"present","http":200,"bytes":28102},"home":{"status":"ok","http":200,"url":"https://reason.com/","html_tokens":75727,"markdown_tokens":532,"pruned_tokens":171}},
{"rank":2401,"domain":"eeoc.gov","origin":"https://eeoc.gov","robots":{"status":"ok","http":200,"url":"https://www.eeoc.gov/robots.txt","bytes":4726,"groups":1},"bots":{"GPTBot":["partial","*",130,22],"ChatGPT-User":["partial","*",130,22],"OAI-SearchBot":["partial","*",130,22],"ClaudeBot":["partial","*",130,22],"Claude-User":["partial","*",130,22],"Claude-SearchBot":["partial","*",130,22],"anthropic-ai":["partial","*",130,22],"PerplexityBot":["partial","*",130,22],"Perplexity-User":["partial","*",130,22],"Google-Extended":["partial","*",130,22],"Applebot-Extended":["partial","*",130,22],"CCBot":["partial","*",130,22],"Bytespider":["partial","*",130,22],"Amazonbot":["partial","*",130,22],"meta-externalagent":["partial","*",130,22],"cohere-ai":["partial","*",130,22],"Diffbot":["partial","*",130,22],"ContextFetchBot":["partial","*",130,22]},"llms":{"status":"missing","http":404,"bytes":62132},"home":{"status":"ok","http":200,"url":"https://www.eeoc.gov/","html_tokens":20480,"markdown_tokens":155,"pruned_tokens":150}},
{"rank":2402,"domain":"bartleby.com","origin":"https://bartleby.com","robots":{"status":"ok","http":200,"url":"https://bartleby.com/robots.txt","bytes":711,"groups":1},"bots":{"GPTBot":["partial","*",26,0],"ChatGPT-User":["partial","*",26,0],"OAI-SearchBot":["partial","*",26,0],"ClaudeBot":["partial","*",26,0],"Claude-User":["partial","*",26,0],"Claude-SearchBot":["partial","*",26,0],"anthropic-ai":["partial","*",26,0],"PerplexityBot":["partial","*",26,0],"Perplexity-User":["partial","*",26,0],"Google-Extended":["partial","*",26,0],"Applebot-Extended":["partial","*",26,0],"CCBot":["partial","*",26,0],"Bytespider":["partial","*",26,0],"Amazonbot":["partial","*",26,0],"meta-externalagent":["partial","*",26,0],"cohere-ai":["partial","*",26,0],"Diffbot":["partial","*",26,0],"ContextFetchBot":["partial","*",26,0]},"llms":{"status":"http_error","http":403,"bytes":919},"home":{"status":"http_error","http":403,"url":"https://bartleby.com/"}},
{"rank":2403,"domain":"20minutos.es","origin":"https://20minutos.es","robots":{"status":"ok","http":200,"url":"https://www.20minutos.es/robots.txt","bytes":7659,"groups":120},"bots":{"GPTBot":["partial","named",1,3],"ChatGPT-User":["partial","*",28,0],"OAI-SearchBot":["partial","*",28,0],"ClaudeBot":["partial","named",1,3],"Claude-User":["partial","*",28,0],"Claude-SearchBot":["partial","*",28,0],"anthropic-ai":["partial","named",1,3],"PerplexityBot":["partial","*",28,0],"Perplexity-User":["partial","*",28,0],"Google-Extended":["partial","*",28,0],"Applebot-Extended":["partial","named",1,3],"CCBot":["partial","named",1,3],"Bytespider":["partial","named",1,3],"Amazonbot":["partial","named",1,3],"meta-externalagent":["partial","named",1,3],"cohere-ai":["partial","named",1,3],"Diffbot":["partial","named",1,3],"ContextFetchBot":["partial","*",28,0]},"llms":{"status":"missing","http":404,"bytes":73430},"home":{"status":"ok","http":200,"url":"https://www.20minutos.es/","html_tokens":92307,"markdown_tokens":8,"pruned_tokens":8}},
{"rank":2404,"domain":"clemson.edu","origin":"https://clemson.edu","robots":{"status":"ok","http":200,"url":"https://www.clemson.edu/robots.txt","bytes":681,"groups":2},"bots":{"GPTBot":["partial","*",1,0],"ChatGPT-User":["partial","*",1,0],"OAI-SearchBot":["partial","*",1,0],"ClaudeBot":["partial","*",1,0],"Claude-User":["partial","*",1,0],"Claude-SearchBot":["partial","*",1,0],"anthropic-ai":["partial","*",1,0],"PerplexityBot":["partial","*",1,0],"Perplexity-User":["partial","*",1,0],"Google-Extended":["partial","*",1,0],"Applebot-Extended":["partial","*",1,0],"CCBot":["partial","*",1,0],"Bytespider":["partial","*",1,0],"Amazonbot":["partial","*",1,0],"meta-externalagent":["partial","*",1,0],"cohere-ai":["partial","*",1,0],"Diffbot":["partial","*",1,0],"ContextFetchBot":["partial","*",1,0]},"llms":{"status":"missing","http":404,"bytes":143193},"home":{"status":"ok","http":200,"url":"https://www.clemson.edu/","html_tokens":101388,"markdown_tokens":334,"pruned_tokens":40}},
{"rank":2405,"domain":"maps.google.co.uk","origin":"https://maps.google.co.uk","robots":{"status":"ok","http":200,"url":"https://maps.google.co.uk/robots.txt","bytes":6624,"groups":4},"bots":{"GPTBot":["partial","*",169,64],"ChatGPT-User":["partial","*",169,64],"OAI-SearchBot":["partial","*",169,64],"ClaudeBot":["partial","*",169,64],"Claude-User":["partial","*",169,64],"Claude-SearchBot":["partial","*",169,64],"anthropic-ai":["partial","*",169,64],"PerplexityBot":["partial","*",169,64],"Perplexity-User":["partial","*",169,64],"Google-Extended":["partial","*",169,64],"Applebot-Extended":["partial","*",169,64],"CCBot":["partial","*",169,64],"Bytespider":["partial","*",169,64],"Amazonbot":["partial","*",169,64],"meta-externalagent":["partial","*",169,64],"cohere-ai":["partial","*",169,64],"Diffbot":["partial","*",169,64],"ContextFetchBot":["partial","*",169,64]},"llms":{"status":"missing","http":404,"bytes":1569},"home":{"status":"ok","http":200,"url":"https://www.google.co.uk/maps","html_tokens":54080,"markdown_tokens":30,"pruned_tokens":30}},
{"rank":2406,"domain":"credly.com","origin":"https://credly.com","robots":{"status":"ok","http":200,"url":"https://credly.com/robots.txt","bytes":72,"groups":2},"bots":{"GPTBot":["partial","*",1,0],"ChatGPT-User":["partial","*",1,0],"OAI-SearchBot":["partial","*",1,0],"ClaudeBot":["partial","*",1,0],"Claude-User":["partial","*",1,0],"Claude-SearchBot":["partial","*",1,0],"anthropic-ai":["partial","*",1,0],"PerplexityBot":["partial","*",1,0],"Perplexity-User":["partial","*",1,0],"Google-Extended":["partial","*",1,0],"Applebot-Extended":["partial","*",1,0],"CCBot":["partial","*",1,0],"Bytespider":["partial","*",1,0],"Amazonbot":["partial","*",1,0],"meta-externalagent":["partial","*",1,0],"cohere-ai":["partial","*",1,0],"Diffbot":["partial","*",1,0],"ContextFetchBot":["partial","*",1,0]},"llms":{"status":"missing","http":404,"bytes":43415},"home":{"status":"ok","http":200,"url":"https://info.credly.com/","html_tokens":25556,"markdown_tokens":217,"pruned_tokens":65}},
{"rank":2407,"domain":"finra.org","origin":"https://finra.org","robots":{"status":"ok","http":200,"url":"https://www.finra.org/robots.txt","bytes":9195,"groups":1},"bots":{"GPTBot":["partial","*",137,18],"ChatGPT-User":["partial","*",137,18],"OAI-SearchBot":["partial","*",137,18],"ClaudeBot":["partial","*",137,18],"Claude-User":["partial","*",137,18],"Claude-SearchBot":["partial","*",137,18],"anthropic-ai":["partial","*",137,18],"PerplexityBot":["partial","*",137,18],"Perplexity-User":["partial","*",137,18],"Google-Extended":["partial","*",137,18],"Applebot-Extended":["partial","*",137,18],"CCBot":["partial","*",137,18],"Bytespider":["partial","*",137,18],"Amazonbot":["partial","*",137,18],"meta-externalagent":["partial","*",137,18],"cohere-ai":["partial","*",137,18],"Diffbot":["partial","*",137,18],"ContextFetchBot":["partial","*",137,18]},"llms":{"status":"missing","http":404,"bytes":187},"home":{"status":"ok","http":200,"url":"https://www.finra.org/","html_tokens":23085,"markdown_tokens":244,"pruned_tokens":244}},
{"rank":2408,"domain":"pfizer.com","origin":"https://pfizer.com","robots":{"status":"refused","http":403,"url":"https://www.pfizer.com/robots.txt","bytes":5531,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":2409,"domain":"mercedes-benz.com","origin":"https://mercedes-benz.com","robots":{"status":"refused","http":403,"url":"https://mercedes-benz.com/robots.txt","bytes":385,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":2410,"domain":"adac.de","origin":"https://adac.de","robots":{"status":"ok","http":200,"url":"https://www.adac.de/robots.txt","bytes":819,"groups":1},"bots":{"GPTBot":["partial","*",10,0],"ChatGPT-User":["partial","*",10,0],"OAI-SearchBot":["partial","*",10,0],"ClaudeBot":["partial","*",10,0],"Claude-User":["partial","*",10,0],"Claude-SearchBot":["partial","*",10,0],"anthropic-ai":["partial","*",10,0],"PerplexityBot":["partial","*",10,0],"Perplexity-User":["partial","*",10,0],"Google-Extended":["partial","*",10,0],"Applebot-Extended":["partial","*",10,0],"CCBot":["partial","*",10,0],"Bytespider":["partial","*",10,0],"Amazonbot":["partial","*",10,0],"meta-externalagent":["partial","*",10,0],"cohere-ai":["partial","*",10,0],"Diffbot":["partial","*",10,0],"ContextFetchBot":["partial","*",10,0]},"llms":{"status":"missing","http":404,"bytes":11871},"home":{"status":"ok","http":200,"url":"https://www.adac.de/","html_tokens":220679,"markdown_tokens":23,"pruned_tokens":23}},
{"rank":2411,"domain":"biz.pl","origin":"https://biz.pl","robots":{"status":"unreachable","http":null,"url":"https://biz.pl/robots.txt","bytes":0,"groups":0,"error":"cannot resolve biz.pl"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":2412,"domain":"nikkeibp.co.jp","origin":"https://nikkeibp.co.jp","robots":{"status":"ok","http":200,"url":"https://www.nikkeibp.co.jp/robots.txt","bytes":1607,"groups":36},"bots":{"GPTBot":["partial","named",1,1],"ChatGPT-User":["partial","named",1,1],"OAI-SearchBot":["blocked","named",1,0],"ClaudeBot":["partial","named",1,1],"Claude-User":["partial","named",1,1],"Claude-SearchBot":["partial","named",1,1],"anthropic-ai":["partial","named",1,1],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["blocked","named",1,0],"Google-Extended":["partial","named",1,1],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",3,3]},"llms":{"status":"missing","http":404,"bytes":2346},"home":{"status":"ok","http":200,"url":"https://www.nikkeibp.co.jp/","html_tokens":3323,"markdown_tokens":15,"pruned_tokens":15}},
{"rank":2413,"domain":"rpi.edu","origin":"https://rpi.edu","robots":{"status":"refused","http":403,"url":"https://rpi.edu/robots.txt","bytes":560,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":2414,"domain":"techspot.com","origin":"https://techspot.com","robots":{"status":"ok","http":200,"url":"https://www.techspot.com/robots.txt","bytes":1014,"groups":3},"bots":{"GPTBot":["partial","*",25,1],"ChatGPT-User":["partial","*",25,1],"OAI-SearchBot":["partial","*",25,1],"ClaudeBot":["partial","*",25,1],"Claude-User":["partial","*",25,1],"Claude-SearchBot":["partial","*",25,1],"anthropic-ai":["partial","*",25,1],"PerplexityBot":["partial","*",25,1],"Perplexity-User":["partial","*",25,1],"Google-Extended":["partial","*",25,1],"Applebot-Extended":["partial","*",25,1],"CCBot":["partial","*",25,1],"Bytespider":["partial","*",25,1],"Amazonbot":["partial","*",25,1],"meta-externalagent":["partial","*",25,1],"cohere-ai":["partial","*",25,1],"Diffbot":["partial","*",25,1],"ContextFetchBot":["partial","*",25,1]},"llms":{"status":"missing","http":404,"bytes":12347},"home":{"status":"http_error","http":403,"url":"https://techspot.com/"}},
{"rank":2415,"domain":"ntu.edu.tw","origin":"https://www.ntu.edu.tw","robots":{"status":"missing","http":404,"url":"https://www.ntu.edu.tw/robots.txt","bytes":239,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":239},"home":{"status":"ok","http":200,"url":"https://www.ntu.edu.tw/","html_tokens":49373,"markdown_tokens":8,"pruned_tokens":8}},
{"rank":2416,"domain":"amebaownd.com","origin":"https://www.amebaownd.com","robots":{"status":"ok","http":200,"url":"https://www.amebaownd.com/robots.txt","bytes":71,"groups":1},"bots":{"GPTBot":["allowed","*",0,1],"ChatGPT-User":["allowed","*",0,1],"OAI-SearchBot":["allowed","*",0,1],"ClaudeBot":["allowed","*",0,1],"Claude-User":["allowed","*",0,1],"Claude-SearchBot":["allowed","*",0,1],"anthropic-ai":["allowed","*",0,1],"PerplexityBot":["allowed","*",0,1],"Perplexity-User":["allowed","*",0,1],"Google-Extended":["allowed","*",0,1],"Applebot-Extended":["allowed","*",0,1],"CCBot":["allowed","*",0,1],"Bytespider":["allowed","*",0,1],"Amazonbot":["allowed","*",0,1],"meta-externalagent":["allowed","*",0,1],"cohere-ai":["allowed","*",0,1],"Diffbot":["allowed","*",0,1],"ContextFetchBot":["allowed","*",0,1]},"llms":{"status":"missing","http":404,"bytes":832},"home":{"status":"ok","http":200,"url":"https://www.amebaownd.com/","html_tokens":1591,"markdown_tokens":0,"pruned_tokens":0}},
{"rank":2417,"domain":"askubuntu.com","origin":"https://askubuntu.com","robots":{"status":"refused","http":403,"url":"https://askubuntu.com/robots.txt","bytes":5530,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":2418,"domain":"nhm.ac.uk","origin":"https://nhm.ac.uk","robots":{"status":"ok","http":200,"url":"https://www.nhm.ac.uk/robots.txt","bytes":871,"groups":1},"bots":{"GPTBot":["partial","*",20,0],"ChatGPT-User":["partial","*",20,0],"OAI-SearchBot":["partial","*",20,0],"ClaudeBot":["partial","*",20,0],"Claude-User":["partial","*",20,0],"Claude-SearchBot":["partial","*",20,0],"anthropic-ai":["partial","*",20,0],"PerplexityBot":["partial","*",20,0],"Perplexity-User":["partial","*",20,0],"Google-Extended":["partial","*",20,0],"Applebot-Extended":["partial","*",20,0],"CCBot":["partial","*",20,0],"Bytespider":["partial","*",20,0],"Amazonbot":["partial","*",20,0],"meta-externalagent":["partial","*",20,0],"cohere-ai":["partial","*",20,0],"Diffbot":["partial","*",20,0],"ContextFetchBot":["partial","*",20,0]},"llms":{"status":"missing","http":404,"bytes":4168},"home":{"status":"ok","http":200,"url":"https://www.nhm.ac.uk/","html_tokens":112270,"markdown_tokens":428,"pruned_tokens":137}},
{"rank":2419,"domain":"env.go.jp","origin":"https://www.env.go.jp","robots":{"status":"ok","http":200,"url":"https://www.env.go.jp/robots.txt","bytes":33,"groups":1},"bots":{"GPTBot":["partial","*",1,0],"ChatGPT-User":["partial","*",1,0],"OAI-SearchBot":["partial","*",1,0],"ClaudeBot":["partial","*",1,0],"Claude-User":["partial","*",1,0],"Claude-SearchBot":["partial","*",1,0],"anthropic-ai":["partial","*",1,0],"PerplexityBot":["partial","*",1,0],"Perplexity-User":["partial","*",1,0],"Google-Extended":["partial","*",1,0],"Applebot-Extended":["partial","*",1,0],"CCBot":["partial","*",1,0],"Bytespider":["partial","*",1,0],"Amazonbot":["partial","*",1,0],"meta-externalagent":["partial","*",1,0],"cohere-ai":["partial","*",1,0],"Diffbot":["partial","*",1,0],"ContextFetchBot":["partial","*",1,0]},"llms":{"status":"missing","http":404,"bytes":45364},"home":{"status":"ok","http":200,"url":"https://www.env.go.jp/","html_tokens":14517,"markdown_tokens":38,"pruned_tokens":38}},
{"rank":2420,"domain":"marketsandmarkets.com","origin":"https://marketsandmarkets.com","robots":{"status":"ok","http":200,"url":"https://www.marketsandmarkets.com/robots.txt","bytes":5954,"groups":10},"bots":{"GPTBot":["allowed","named",0,1],"ChatGPT-User":["allowed","named",0,1],"OAI-SearchBot":["partial","*",98,1],"ClaudeBot":["allowed","named",0,1],"Claude-User":["partial","*",98,1],"Claude-SearchBot":["partial","*",98,1],"anthropic-ai":["partial","*",98,1],"PerplexityBot":["allowed","named",0,1],"Perplexity-User":["partial","*",98,1],"Google-Extended":["allowed","named",0,1],"Applebot-Extended":["partial","*",98,1],"CCBot":["partial","*",98,1],"Bytespider":["partial","*",98,1],"Amazonbot":["partial","*",98,1],"meta-externalagent":["partial","*",98,1],"cohere-ai":["allowed","named",0,1],"Diffbot":["partial","*",98,1],"ContextFetchBot":["partial","*",98,1]},"llms":{"status":"present","http":200,"bytes":26938},"home":{"status":"ok","http":200,"url":"https://www.marketsandmarkets.com/","html_tokens":38039,"markdown_tokens":566,"pruned_tokens":65}},
{"rank":2421,"domain":"tistory.com","origin":"https://tistory.com","robots":{"status":"missing","http":404,"url":"https://www.tistory.com/robots.txt","bytes":22730,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":22728},"home":{"status":"ok","http":200,"url":"https://www.tistory.com/","html_tokens":13187,"markdown_tokens":14,"pruned_tokens":14}},
{"rank":2422,"domain":"udn.com","origin":"https://udn.com","robots":{"status":"ok","http":200,"url":"https://udn.com/robots.txt","bytes":1192,"groups":7},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["partial","*",11,3],"OAI-SearchBot":["partial","*",11,3],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",11,3],"Claude-SearchBot":["partial","*",11,3],"anthropic-ai":["partial","*",11,3],"PerplexityBot":["partial","*",11,3],"Perplexity-User":["partial","*",11,3],"Google-Extended":["partial","*",11,3],"Applebot-Extended":["partial","*",11,3],"CCBot":["partial","*",11,3],"Bytespider":["partial","*",11,3],"Amazonbot":["partial","*",11,3],"meta-externalagent":["allowed","named",0,1],"cohere-ai":["partial","*",11,3],"Diffbot":["partial","*",11,3],"ContextFetchBot":["partial","*",11,3]},"llms":{"status":"missing","http":404,"bytes":61200},"home":{"status":"ok","http":200,"url":"https://udn.com/","html_tokens":90012,"markdown_tokens":138,"pruned_tokens":138}},
{"rank":2423,"domain":"basf.com","origin":"https://basf.com","robots":{"status":"ok","http":200,"url":"https://www.basf.com/robots.txt","bytes":198,"groups":4},"bots":{"GPTBot":["allowed","named",0,1],"ChatGPT-User":["allowed","*",0,1],"OAI-SearchBot":["allowed","*",0,1],"ClaudeBot":["allowed","*",0,1],"Claude-User":["allowed","*",0,1],"Claude-SearchBot":["allowed","*",0,1],"anthropic-ai":["allowed","*",0,1],"PerplexityBot":["allowed","named",0,1],"Perplexity-User":["allowed","*",0,1],"Google-Extended":["allowed","named",0,1],"Applebot-Extended":["allowed","*",0,1],"CCBot":["allowed","*",0,1],"Bytespider":["allowed","*",0,1],"Amazonbot":["allowed","*",0,1],"meta-externalagent":["allowed","*",0,1],"cohere-ai":["allowed","*",0,1],"Diffbot":["allowed","*",0,1],"ContextFetchBot":["allowed","*",0,1]},"llms":{"status":"html","http":200,"bytes":622228},"home":{"status":"ok","http":200,"url":"https://www.basf.com/us/en","html_tokens":155521,"markdown_tokens":75,"pruned_tokens":75}},
{"rank":2424,"domain":"ky.gov","origin":"http://ky.gov","robots":{"status":"missing","http":404,"url":"http://ky.gov/robots.txt","bytes":208,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":206},"home":{"status":"ok","http":200,"url":"http://ky.gov/","html_tokens":88,"markdown_tokens":0,"pruned_tokens":0}},
{"rank":2425,"domain":"google.com.mx","origin":"https://google.com.mx","robots":{"status":"ok","http":200,"url":"https://www.google.com.mx/robots.txt","bytes":6624,"groups":4},"bots":{"GPTBot":["partial","*",169,64],"ChatGPT-User":["partial","*",169,64],"OAI-SearchBot":["partial","*",169,64],"ClaudeBot":["partial","*",169,64],"Claude-User":["partial","*",169,64],"Claude-SearchBot":["partial","*",169,64],"anthropic-ai":["partial","*",169,64],"PerplexityBot":["partial","*",169,64],"Perplexity-User":["partial","*",169,64],"Google-Extended":["partial","*",169,64],"Applebot-Extended":["partial","*",169,64],"CCBot":["partial","*",169,64],"Bytespider":["partial","*",169,64],"Amazonbot":["partial","*",169,64],"meta-externalagent":["partial","*",169,64],"cohere-ai":["partial","*",169,64],"Diffbot":["partial","*",169,64],"ContextFetchBot":["partial","*",169,64]},"llms":{"status":"missing","http":404,"bytes":1569},"home":{"status":"ok","http":200,"url":"https://www.google.com/","html_tokens":21347,"markdown_tokens":30,"pruned_tokens":30}},
{"rank":2426,"domain":"nos.nl","origin":"https://nos.nl","robots":{"status":"ok","http":200,"url":"https://nos.nl/robots.txt","bytes":1287,"groups":26},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["blocked","named",1,0],"OAI-SearchBot":["partial","*",6,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",6,0],"Claude-SearchBot":["partial","*",6,0],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["partial","*",6,0],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["partial","*",6,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",6,0]},"llms":{"status":"missing","http":404,"bytes":78386},"home":{"status":"ok","http":200,"url":"https://nos.nl/","html_tokens":199933,"markdown_tokens":781,"pruned_tokens":168}},
{"rank":2427,"domain":"nta.go.jp","origin":"https://www.nta.go.jp","robots":{"status":"ok","http":200,"url":"https://www.nta.go.jp/robots.txt","bytes":853,"groups":2},"bots":{"GPTBot":["partial","*",13,0],"ChatGPT-User":["partial","*",13,0],"OAI-SearchBot":["partial","*",13,0],"ClaudeBot":["partial","*",13,0],"Claude-User":["partial","*",13,0],"Claude-SearchBot":["partial","*",13,0],"anthropic-ai":["partial","*",13,0],"PerplexityBot":["partial","*",13,0],"Perplexity-User":["partial","*",13,0],"Google-Extended":["partial","*",13,0],"Applebot-Extended":["partial","*",13,0],"CCBot":["partial","*",13,0],"Bytespider":["partial","*",13,0],"Amazonbot":["partial","*",13,0],"meta-externalagent":["partial","*",13,0],"cohere-ai":["partial","*",13,0],"Diffbot":["partial","*",13,0],"ContextFetchBot":["partial","*",13,0]},"llms":{"status":"html","http":200,"bytes":2007},"home":{"status":"ok","http":200,"url":"https://www.nta.go.jp/","html_tokens":14886,"markdown_tokens":371,"pruned_tokens":194}},
{"rank":2428,"domain":"ushmm.org","origin":"https://ushmm.org","robots":{"status":"ok","http":200,"url":"https://www.ushmm.org/robots.txt","bytes":404,"groups":1},"bots":{"GPTBot":["partial","*",6,1],"ChatGPT-User":["partial","*",6,1],"OAI-SearchBot":["partial","*",6,1],"ClaudeBot":["partial","*",6,1],"Claude-User":["partial","*",6,1],"Claude-SearchBot":["partial","*",6,1],"anthropic-ai":["partial","*",6,1],"PerplexityBot":["partial","*",6,1],"Perplexity-User":["partial","*",6,1],"Google-Extended":["partial","*",6,1],"Applebot-Extended":["partial","*",6,1],"CCBot":["partial","*",6,1],"Bytespider":["partial","*",6,1],"Amazonbot":["partial","*",6,1],"meta-externalagent":["partial","*",6,1],"cohere-ai":["partial","*",6,1],"Diffbot":["partial","*",6,1],"ContextFetchBot":["partial","*",6,1]},"llms":{"status":"missing","http":404,"bytes":221610},"home":{"status":"ok","http":200,"url":"https://www.ushmm.org/","html_tokens":116510,"markdown_tokens":199,"pruned_tokens":86}},
{"rank":2429,"domain":"abril.com.br","origin":"https://abril.com.br","robots":{"status":"refused","http":403,"url":"https://abril.com.br/robots.txt","bytes":0,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":2430,"domain":"aljazeera.net","origin":"https://aljazeera.net","robots":{"status":"ok","http":200,"url":"https://www.aljazeera.net/robots.txt","bytes":1834,"groups":10},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["blocked","named",1,0],"OAI-SearchBot":["partial","*",15,1],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",15,1],"Claude-SearchBot":["partial","*",15,1],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["partial","*",15,1],"Google-Extended":["partial","*",15,1],"Applebot-Extended":["partial","*",15,1],"CCBot":["partial","*",15,1],"Bytespider":["blocked","named",1,0],"Amazonbot":["partial","*",15,1],"meta-externalagent":["partial","*",15,1],"cohere-ai":["blocked","named",1,0],"Diffbot":["partial","*",15,1],"ContextFetchBot":["partial","*",15,1]},"llms":{"status":"missing","http":404,"bytes":41475},"home":{"status":"ok","http":200,"url":"https://www.aljazeera.net/","html_tokens":244879,"markdown_tokens":144,"pruned_tokens":144}},
{"rank":2431,"domain":"ce.cn","origin":"https://www.ce.cn","robots":{"status":"ok","http":200,"url":"http://www.ce.cn/robots.txt","bytes":42,"groups":1},"bots":{"GPTBot":["partial","*",1,1],"ChatGPT-User":["partial","*",1,1],"OAI-SearchBot":["partial","*",1,1],"ClaudeBot":["partial","*",1,1],"Claude-User":["partial","*",1,1],"Claude-SearchBot":["partial","*",1,1],"anthropic-ai":["partial","*",1,1],"PerplexityBot":["partial","*",1,1],"Perplexity-User":["partial","*",1,1],"Google-Extended":["partial","*",1,1],"Applebot-Extended":["partial","*",1,1],"CCBot":["partial","*",1,1],"Bytespider":["partial","*",1,1],"Amazonbot":["partial","*",1,1],"meta-externalagent":["partial","*",1,1],"cohere-ai":["partial","*",1,1],"Diffbot":["partial","*",1,1],"ContextFetchBot":["partial","*",1,1]},"llms":{"status":"missing","http":404,"bytes":2375},"home":{"status":"ok","http":200,"url":"http://www.ce.cn/","html_tokens":24219,"markdown_tokens":103,"pruned_tokens":103}},
{"rank":2432,"domain":"uni-koeln.de","origin":"https://uni-koeln.de","robots":{"status":"ok","http":200,"url":"https://uni-koeln.de/robots.txt","bytes":3014,"groups":4},"bots":{"GPTBot":["partial","named",5,0],"ChatGPT-User":["partial","*",48,0],"OAI-SearchBot":["partial","*",48,0],"ClaudeBot":["partial","*",48,0],"Claude-User":["partial","*",48,0],"Claude-SearchBot":["partial","*",48,0],"anthropic-ai":["partial","*",48,0],"PerplexityBot":["partial","*",48,0],"Perplexity-User":["partial","*",48,0],"Google-Extended":["partial","*",48,0],"Applebot-Extended":["partial","*",48,0],"CCBot":["partial","*",48,0],"Bytespider":["partial","*",48,0],"Amazonbot":["partial","*",48,0],"meta-externalagent":["partial","*",48,0],"cohere-ai":["partial","*",48,0],"Diffbot":["partial","*",48,0],"ContextFetchBot":["partial","*",48,0]},"llms":{"status":"missing","http":404,"bytes":26731},"home":{"status":"ok","http":200,"url":"https://uni-koeln.de/","html_tokens":84110,"markdown_tokens":92,"pruned_tokens":92}},
{"rank":2433,"domain":"apachehaus.com","origin":"https://apachehaus.com","robots":{"status":"ok","http":200,"url":"https://www.apachehaus.com/robots.txt","bytes":634,"groups":1},"bots":{"GPTBot":["partial","*",16,1],"ChatGPT-User":["partial","*",16,1],"OAI-SearchBot":["partial","*",16,1],"ClaudeBot":["partial","*",16,1],"Claude-User":["partial","*",16,1],"Claude-SearchBot":["partial","*",16,1],"anthropic-ai":["partial","*",16,1],"PerplexityBot":["partial","*",16,1],"Perplexity-User":["partial","*",16,1],"Google-Extended":["partial","*",16,1],"Applebot-Extended":["partial","*",16,1],"CCBot":["partial","*",16,1],"Bytespider":["partial","*",16,1],"Amazonbot":["partial","*",16,1],"meta-externalagent":["partial","*",16,1],"cohere-ai":["partial","*",16,1],"Diffbot":["partial","*",16,1],"ContextFetchBot":["partial","*",16,1]},"llms":{"status":"missing","http":404,"bytes":4477},"home":{"status":"ok","http":200,"url":"https://www.apachehaus.com/","html_tokens":2737,"markdown_tokens":627,"pruned_tokens":339}},
{"rank":2434,"domain":"cntv.cn","origin":"https://cntv.cn","robots":{"status":"html","http":200,"url":"https://www.cctv.com/","bytes":318881,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":2435,"domain":"kuaishou.com","origin":"https://www.kuaishou.com","robots":{"status":"ok","http":200,"url":"https://www.kuaishou.com/robots.txt","bytes":604,"groups":15},"bots":{"GPTBot":["allowed","named",0,1],"ChatGPT-User":["allowed","named",0,1],"OAI-SearchBot":["allowed","named",0,1],"ClaudeBot":["blocked","*",1,0],"Claude-User":["blocked","*",1,0],"Claude-SearchBot":["blocked","*",1,0],"anthropic-ai":["blocked","*",1,0],"PerplexityBot":["allowed","named",0,1],"Perplexity-User":["allowed","named",0,1],"Google-Extended":["blocked","*",1,0],"Applebot-Extended":["blocked","*",1,0],"CCBot":["blocked","*",1,0],"Bytespider":["blocked","*",1,0],"Amazonbot":["blocked","*",1,0],"meta-externalagent":["blocked","*",1,0],"cohere-ai":["blocked","*",1,0],"Diffbot":["blocked","*",1,0],"ContextFetchBot":["blocked","*",1,0]},"llms":{"status":"disallowed"},"home":{"status":"disallowed"}},
{"rank":2436,"domain":"bigthink.com","origin":"https://bigthink.com","robots":{"status":"ok","http":200,"url":"https://bigthink.com/robots.txt","bytes":170,"groups":1},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"missing","http":404,"bytes":252410},"home":{"status":"http_error","http":403,"url":"https://bigthink.com/"}},
{"rank":2437,"domain":"sjv.io","origin":"https://www.sjv.io","robots":{"status":"ok","http":200,"url":"https://www.sjv.io/robots.txt","bytes":289,"groups":8},"bots":{"GPTBot":["blocked","*",1,0],"ChatGPT-User":["blocked","*",1,0],"OAI-SearchBot":["blocked","*",1,0],"ClaudeBot":["blocked","*",1,0],"Claude-User":["blocked","*",1,0],"Claude-SearchBot":["blocked","*",1,0],"anthropic-ai":["blocked","*",1,0],"PerplexityBot":["blocked","*",1,0],"Perplexity-User":["blocked","*",1,0],"Google-Extended":["blocked","*",1,0],"Applebot-Extended":["blocked","*",1,0],"CCBot":["blocked","*",1,0],"Bytespider":["blocked","*",1,0],"Amazonbot":["blocked","*",1,0],"meta-externalagent":["blocked","*",1,0],"cohere-ai":["blocked","*",1,0],"Diffbot":["blocked","*",1,0],"ContextFetchBot":["blocked","*",1,0]},"llms":{"status":"disallowed"},"home":{"status":"disallowed"}},
{"rank":2438,"domain":"umontreal.ca","origin":"https://umontreal.ca","robots":{"status":"ok","http":200,"url":"https://umontreal.ca/robots.txt","bytes":64,"groups":1},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"missing","http":404,"bytes":37641},"home":{"status":"ok","http":200,"url":"https://www.umontreal.ca/","html_tokens":16823,"markdown_tokens":154,"pruned_tokens":154}},
{"rank":2439,"domain":"rei.com","origin":"https://rei.com","robots":{"status":"refused","http":403,"url":"https://rei.com/robots.txt","bytes":369,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":2440,"domain":"mn.gov","origin":"https://mn.gov","robots":{"status":"ok","http":200,"url":"https://mn.gov/robots.txt","bytes":524,"groups":3},"bots":{"GPTBot":["partial","*",5,0],"ChatGPT-User":["partial","*",5,0],"OAI-SearchBot":["partial","*",5,0],"ClaudeBot":["partial","*",5,0],"Claude-User":["partial","*",5,0],"Claude-SearchBot":["partial","*",5,0],"anthropic-ai":["partial","*",5,0],"PerplexityBot":["partial","*",5,0],"Perplexity-User":["partial","*",5,0],"Google-Extended":["partial","*",5,0],"Applebot-Extended":["partial","*",5,0],"CCBot":["partial","*",5,0],"Bytespider":["partial","*",5,0],"Amazonbot":["partial","*",5,0],"meta-externalagent":["partial","*",5,0],"cohere-ai":["partial","*",5,0],"Diffbot":["partial","*",5,0],"ContextFetchBot":["partial","*",5,0]},"llms":{"status":"missing","http":404,"bytes":5822},"home":{"status":"ok","http":200,"url":"https://mn.gov/portal/","html_tokens":14417,"markdown_tokens":21,"pruned_tokens":21}},
{"rank":2441,"domain":"fivethirtyeight.com","origin":"https://fivethirtyeight.com","robots":{"status":"html","http":200,"url":"https://abcnews.com/politics","bytes":326924,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":2442,"domain":"jiathis.com","origin":"https://jiathis.com","robots":{"status":"ok","http":200,"url":"https://jiathis.com/robots.txt","bytes":1248,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":513},"home":{"status":"ok","http":200,"url":"https://jiathis.com/","html_tokens":1088,"markdown_tokens":71,"pruned_tokens":71}},
{"rank":2443,"domain":"thewrap.com","origin":"https://thewrap.com","robots":{"status":"refused","http":403,"url":"https://www.thewrap.com/robots.txt","bytes":560,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":2444,"domain":"exeter.ac.uk","origin":"https://exeter.ac.uk","robots":{"status":"ok","http":200,"url":"https://www.exeter.ac.uk/robots.txt","bytes":215,"groups":1},"bots":{"GPTBot":["partial","*",4,0],"ChatGPT-User":["partial","*",4,0],"OAI-SearchBot":["partial","*",4,0],"ClaudeBot":["partial","*",4,0],"Claude-User":["partial","*",4,0],"Claude-SearchBot":["partial","*",4,0],"anthropic-ai":["partial","*",4,0],"PerplexityBot":["partial","*",4,0],"Perplexity-User":["partial","*",4,0],"Google-Extended":["partial","*",4,0],"Applebot-Extended":["partial","*",4,0],"CCBot":["partial","*",4,0],"Bytespider":["partial","*",4,0],"Amazonbot":["partial","*",4,0],"meta-externalagent":["partial","*",4,0],"cohere-ai":["partial","*",4,0],"Diffbot":["partial","*",4,0],"ContextFetchBot":["partial","*",4,0]},"llms":{"status":"present","http":200,"bytes":6612},"home":{"status":"ok","http":200,"url":"https://www.exeter.ac.uk/","html_tokens":18277,"markdown_tokens":95,"pruned_tokens":95}},
{"rank":2445,"domain":"indiewire.com","origin":"https://indiewire.com","robots":{"status":"ok","http":200,"url":"https://www.indiewire.com/robots.txt","bytes":1258,"groups":25},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["blocked","named",1,0],"OAI-SearchBot":["blocked","named",1,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",11,1],"Claude-SearchBot":["partial","*",11,1],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["partial","*",11,1],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["partial","*",11,1],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",11,1]},"llms":{"status":"missing","http":404,"bytes":335168},"home":{"status":"ok","http":200,"url":"https://www.indiewire.com/","html_tokens":142117,"markdown_tokens":241,"pruned_tokens":104}},
{"rank":2446,"domain":"informationweek.com","origin":"https://informationweek.com","robots":{"status":"ok","http":200,"url":"https://www.informationweek.com/robots.txt","bytes":1669,"groups":2},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["partial","*",6,0],"OAI-SearchBot":["partial","*",6,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",6,0],"Claude-SearchBot":["partial","*",6,0],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["partial","*",6,0],"Perplexity-User":["partial","*",6,0],"Google-Extended":["partial","*",6,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",6,0]},"llms":{"status":"missing","http":404,"bytes":361189},"home":{"status":"ok","http":200,"url":"https://www.informationweek.com/","html_tokens":127843,"markdown_tokens":213,"pruned_tokens":140}},
{"rank":2447,"domain":"signal.org","origin":"https://signal.org","robots":{"status":"missing","http":404,"url":"https://signal.org/robots.txt","bytes":7450,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":7450},"home":{"status":"ok","http":200,"url":"https://signal.org/","html_tokens":5471,"markdown_tokens":311,"pruned_tokens":257}},
{"rank":2448,"domain":"openjdk.org","origin":"https://openjdk.org","robots":{"status":"timeout","http":null,"url":"https://openjdk.org/robots.txt","bytes":0,"groups":0,"error":"timed out after 10 s"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":2449,"domain":"tcd.ie","origin":"https://tcd.ie","robots":{"status":"ok","http":200,"url":"https://www.tcd.ie/robots.txt","bytes":625,"groups":1},"bots":{"GPTBot":["partial","*",15,4],"ChatGPT-User":["partial","*",15,4],"OAI-SearchBot":["partial","*",15,4],"ClaudeBot":["partial","*",15,4],"Claude-User":["partial","*",15,4],"Claude-SearchBot":["partial","*",15,4],"anthropic-ai":["partial","*",15,4],"PerplexityBot":["partial","*",15,4],"Perplexity-User":["partial","*",15,4],"Google-Extended":["partial","*",15,4],"Applebot-Extended":["partial","*",15,4],"CCBot":["partial","*",15,4],"Bytespider":["partial","*",15,4],"Amazonbot":["partial","*",15,4],"meta-externalagent":["partial","*",15,4],"cohere-ai":["partial","*",15,4],"Diffbot":["partial","*",15,4],"ContextFetchBot":["partial","*",15,4]},"llms":{"status":"missing","http":404,"bytes":14212},"home":{"status":"ok","http":200,"url":"https://www.tcd.ie/","html_tokens":6460,"markdown_tokens":104,"pruned_tokens":104}},
{"rank":2450,"domain":"marvel.com","origin":"https://marvel.com","robots":{"status":"refused","http":403,"url":"https://marvel.com/robots.txt","bytes":919,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":2451,"domain":"tsa.gov","origin":"https://tsa.gov","robots":{"status":"refused","http":403,"url":"https://tsa.gov/robots.txt","bytes":371,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":2452,"domain":"pixiv.net","origin":"https://pixiv.net","robots":{"status":"ok","http":200,"url":"https://www.pixiv.net/robots.txt","bytes":1025,"groups":2},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["partial","*",22,2],"OAI-SearchBot":["partial","*",22,2],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",22,2],"Claude-SearchBot":["partial","*",22,2],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["partial","*",22,2],"Perplexity-User":["partial","*",22,2],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["partial","*",22,2],"Amazonbot":["partial","*",22,2],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",22,2]},"llms":{"status":"missing","http":404,"bytes":1714},"home":{"status":"http_error","http":403,"url":"https://www.pixiv.net/en/"}},
{"rank":2453,"domain":"copernicus.eu","origin":"https://copernicus.eu","robots":{"status":"ok","http":200,"url":"https://www.copernicus.eu/robots.txt","bytes":2566,"groups":1},"bots":{"GPTBot":["partial","*",62,18],"ChatGPT-User":["partial","*",62,18],"OAI-SearchBot":["partial","*",62,18],"ClaudeBot":["partial","*",62,18],"Claude-User":["partial","*",62,18],"Claude-SearchBot":["partial","*",62,18],"anthropic-ai":["partial","*",62,18],"PerplexityBot":["partial","*",62,18],"Perplexity-User":["partial","*",62,18],"Google-Extended":["partial","*",62,18],"Applebot-Extended":["partial","*",62,18],"CCBot":["partial","*",62,18],"Bytespider":["partial","*",62,18],"Amazonbot":["partial","*",62,18],"meta-externalagent":["partial","*",62,18],"cohere-ai":["partial","*",62,18],"Diffbot":["partial","*",62,18],"ContextFetchBot":["partial","*",62,18]},"llms":{"status":"missing","http":404,"bytes":154},"home":{"status":"ok","http":200,"url":"https://www.copernicus.eu/en","html_tokens":33298,"markdown_tokens":256,"pruned_tokens":176}},
{"rank":2454,"domain":"rockstargames.com","origin":"https://rockstargames.com","robots":{"status":"ok","http":200,"url":"https://www.rockstargames.com/robots.txt","bytes":311,"groups":8},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":2447},"home":{"status":"ok","http":200,"url":"https://www.rockstargames.com/","html_tokens":887,"markdown_tokens":0,"pruned_tokens":0}},
{"rank":2455,"domain":"motorola.com","origin":"https://motorola.com","robots":{"status":"ok","http":200,"url":"https://www.motorola.com/robots.txt","bytes":4068,"groups":19},"bots":{"GPTBot":["allowed","named",0,1],"ChatGPT-User":["allowed","named",0,1],"OAI-SearchBot":["allowed","named",0,2],"ClaudeBot":["allowed","named",0,1],"Claude-User":["partial","*",20,0],"Claude-SearchBot":["partial","*",20,0],"anthropic-ai":["partial","*",20,0],"PerplexityBot":["allowed","named",0,1],"Perplexity-User":["partial","*",20,0],"Google-Extended":["allowed","named",0,1],"Applebot-Extended":["partial","*",20,0],"CCBot":["allowed","named",0,1],"Bytespider":["partial","*",20,0],"Amazonbot":["allowed","named",0,1],"meta-externalagent":["partial","*",20,0],"cohere-ai":["partial","*",20,0],"Diffbot":["partial","*",20,0],"ContextFetchBot":["partial","*",20,0]},"llms":{"status":"present","http":200,"bytes":12646},"home":{"status":"ok","http":200,"url":"https://www.motorola.com/us/en/homepage","html_tokens":389210,"markdown_tokens":50,"pruned_tokens":50}},
{"rank":2456,"domain":"znaki.fm","origin":"https://znaki.fm","robots":{"status":"ok","http":200,"url":"https://znaki.fm/robots.txt","bytes":40,"groups":1},"bots":{"GPTBot":["partial","*",1,1],"ChatGPT-User":["partial","*",1,1],"OAI-SearchBot":["partial","*",1,1],"ClaudeBot":["partial","*",1,1],"Claude-User":["partial","*",1,1],"Claude-SearchBot":["partial","*",1,1],"anthropic-ai":["partial","*",1,1],"PerplexityBot":["partial","*",1,1],"Perplexity-User":["partial","*",1,1],"Google-Extended":["partial","*",1,1],"Applebot-Extended":["partial","*",1,1],"CCBot":["partial","*",1,1],"Bytespider":["partial","*",1,1],"Amazonbot":["partial","*",1,1],"meta-externalagent":["partial","*",1,1],"cohere-ai":["partial","*",1,1],"Diffbot":["partial","*",1,1],"ContextFetchBot":["partial","*",1,1]},"llms":{"status":"missing","http":404,"bytes":32783},"home":{"status":"ok","http":200,"url":"https://znaki.fm/","html_tokens":47706,"markdown_tokens":191,"pruned_tokens":174}},
{"rank":2457,"domain":"jnj.com","origin":"https://jnj.com","robots":{"status":"ok","http":200,"url":"https://www.jnj.com/robots.txt","bytes":2340,"groups":1},"bots":{"GPTBot":["partial","*",4,1],"ChatGPT-User":["partial","*",4,1],"OAI-SearchBot":["partial","*",4,1],"ClaudeBot":["partial","*",4,1],"Claude-User":["partial","*",4,1],"Claude-SearchBot":["partial","*",4,1],"anthropic-ai":["partial","*",4,1],"PerplexityBot":["partial","*",4,1],"Perplexity-User":["partial","*",4,1],"Google-Extended":["partial","*",4,1],"Applebot-Extended":["partial","*",4,1],"CCBot":["partial","*",4,1],"Bytespider":["partial","*",4,1],"Amazonbot":["partial","*",4,1],"meta-externalagent":["partial","*",4,1],"cohere-ai":["partial","*",4,1],"Diffbot":["partial","*",4,1],"ContextFetchBot":["partial","*",4,1]},"llms":{"status":"missing","http":404,"bytes":289762},"home":{"status":"ok","http":200,"url":"https://www.jnj.com/","html_tokens":95138,"markdown_tokens":816,"pruned_tokens":113}},
{"rank":2458,"domain":"ul.com","origin":"https://ul.com","robots":{"status":"refused","http":403,"url":"https://ul.com/robots.txt","bytes":560,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":2459,"domain":"nature.org","origin":"https://nature.org","robots":{"status":"ok","http":200,"url":"https://www.nature.org/robots.txt","bytes":299,"groups":3},"bots":{"GPTBot":["partial","*",4,0],"ChatGPT-User":["partial","*",4,0],"OAI-SearchBot":["partial","*",4,0],"ClaudeBot":["partial","*",4,0],"Claude-User":["partial","*",4,0],"Claude-SearchBot":["partial","*",4,0],"anthropic-ai":["partial","*",4,0],"PerplexityBot":["partial","*",4,0],"Perplexity-User":["partial","*",4,0],"Google-Extended":["partial","*",4,0],"Applebot-Extended":["partial","*",4,0],"CCBot":["partial","*",4,0],"Bytespider":["partial","*",4,0],"Amazonbot":["partial","*",4,0],"meta-externalagent":["partial","*",4,0],"cohere-ai":["partial","*",4,0],"Diffbot":["partial","*",4,0],"ContextFetchBot":["partial","*",4,0]},"llms":{"status":"missing","http":404,"bytes":29077},"home":{"status":"ok","http":200,"url":"https://www.nature.org/en-us/","html_tokens":50323,"markdown_tokens":209,"pruned_tokens":90}},
{"rank":2460,"domain":"mcmaster.ca","origin":"https://mcmaster.ca","robots":{"status":"ok","http":200,"url":"https://mcmaster.ca/robots.txt","bytes":933,"groups":1},"bots":{"GPTBot":["partial","*",36,0],"ChatGPT-User":["partial","*",36,0],"OAI-SearchBot":["partial","*",36,0],"ClaudeBot":["partial","*",36,0],"Claude-User":["partial","*",36,0],"Claude-SearchBot":["partial","*",36,0],"anthropic-ai":["partial","*",36,0],"PerplexityBot":["partial","*",36,0],"Perplexity-User":["partial","*",36,0],"Google-Extended":["partial","*",36,0],"Applebot-Extended":["partial","*",36,0],"CCBot":["partial","*",36,0],"Bytespider":["partial","*",36,0],"Amazonbot":["partial","*",36,0],"meta-externalagent":["partial","*",36,0],"cohere-ai":["partial","*",36,0],"Diffbot":["partial","*",36,0],"ContextFetchBot":["partial","*",36,0]},"llms":{"status":"missing","http":404,"bytes":196},"home":{"status":"ok","http":200,"url":"https://www.mcmaster.ca/","html_tokens":22420,"markdown_tokens":94,"pruned_tokens":94}},
{"rank":2461,"domain":"newsday.com","origin":"https://newsday.com","robots":{"status":"ok","http":200,"url":"https://www.newsday.com/robots.txt","bytes":1030,"groups":3},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["partial","*",11,0],"OAI-SearchBot":["partial","*",11,0],"ClaudeBot":["partial","*",11,0],"Claude-User":["partial","*",11,0],"Claude-SearchBot":["partial","*",11,0],"anthropic-ai":["partial","*",11,0],"PerplexityBot":["partial","*",11,0],"Perplexity-User":["partial","*",11,0],"Google-Extended":["partial","*",11,0],"Applebot-Extended":["partial","*",11,0],"CCBot":["blocked","named",1,0],"Bytespider":["partial","*",11,0],"Amazonbot":["partial","*",11,0],"meta-externalagent":["partial","*",11,0],"cohere-ai":["partial","*",11,0],"Diffbot":["partial","*",11,0],"ContextFetchBot":["partial","*",11,0]},"llms":{"status":"missing","http":404,"bytes":37017},"home":{"status":"ok","http":200,"url":"https://www.newsday.com/","html_tokens":573153,"markdown_tokens":230,"pruned_tokens":148}},
{"rank":2462,"domain":"gsa.gov","origin":"https://gsa.gov","robots":{"status":"ok","http":200,"url":"https://www.gsa.gov/robots.txt","bytes":2027,"groups":1},"bots":{"GPTBot":["partial","*",34,18],"ChatGPT-User":["partial","*",34,18],"OAI-SearchBot":["partial","*",34,18],"ClaudeBot":["partial","*",34,18],"Claude-User":["partial","*",34,18],"Claude-SearchBot":["partial","*",34,18],"anthropic-ai":["partial","*",34,18],"PerplexityBot":["partial","*",34,18],"Perplexity-User":["partial","*",34,18],"Google-Extended":["partial","*",34,18],"Applebot-Extended":["partial","*",34,18],"CCBot":["partial","*",34,18],"Bytespider":["partial","*",34,18],"Amazonbot":["partial","*",34,18],"meta-externalagent":["partial","*",34,18],"cohere-ai":["partial","*",34,18],"Diffbot":["partial","*",34,18],"ContextFetchBot":["partial","*",34,18]},"llms":{"status":"missing","http":404,"bytes":185},"home":{"status":"ok","http":200,"url":"https://www.gsa.gov/","html_tokens":44921,"markdown_tokens":149,"pruned_tokens":149}},
{"rank":2463,"domain":"asos.com","origin":"https://asos.com","robots":{"status":"unreachable","http":null,"url":"https://asos.com/robots.txt","bytes":0,"groups":0,"error":"<StreamReset stream_id:1, error_code:2, remote_reset:True>"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":2464,"domain":"presscustomizr.com","origin":"https://presscustomizr.com","robots":{"status":"ok","http":200,"url":"https://presscustomizr.com/robots.txt","bytes":165,"groups":1},"bots":{"GPTBot":["partial","*",1,1],"ChatGPT-User":["partial","*",1,1],"OAI-SearchBot":["partial","*",1,1],"ClaudeBot":["partial","*",1,1],"Claude-User":["partial","*",1,1],"Claude-SearchBot":["partial","*",1,1],"anthropic-ai":["partial","*",1,1],"PerplexityBot":["partial","*",1,1],"Perplexity-User":["partial","*",1,1],"Google-Extended":["partial","*",1,1],"Applebot-Extended":["partial","*",1,1],"CCBot":["partial","*",1,1],"Bytespider":["partial","*",1,1],"Amazonbot":["partial","*",1,1],"meta-externalagent":["partial","*",1,1],"cohere-ai":["partial","*",1,1],"Diffbot":["partial","*",1,1],"ContextFetchBot":["partial","*",1,1]},"llms":{"status":"missing","http":404,"bytes":70016},"home":{"status":"ok","http":200,"url":"https://presscustomizr.com/","html_tokens":72768,"markdown_tokens":999,"pruned_tokens":54}},
{"rank":2465,"domain":"sjtu.edu.cn","origin":"https://www.sjtu.edu.cn","robots":{"status":"missing","http":404,"url":"https://www.sjtu.edu.cn/robots.txt","bytes":146,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":146},"home":{"status":"ok","http":200,"url":"https://www.sjtu.edu.cn/","html_tokens":17976,"markdown_tokens":8,"pruned_tokens":8}},
{"rank":2466,"domain":"cuni.cz","origin":"https://cuni.cz","robots":{"status":"ok","http":200,"url":"https://cuni.cz/robots.txt","bytes":23,"groups":1},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"missing","http":404,"bytes":196},"home":{"status":"ok","http":200,"url":"https://cuni.cz/","html_tokens":17550,"markdown_tokens":991,"pruned_tokens":135}},
{"rank":2467,"domain":"qiita.com","origin":"https://qiita.com","robots":{"status":"ok","http":200,"url":"https://qiita.com/robots.txt","bytes":379,"groups":1},"bots":{"GPTBot":["partial","*",8,1],"ChatGPT-User":["partial","*",8,1],"OAI-SearchBot":["partial","*",8,1],"ClaudeBot":["partial","*",8,1],"Claude-User":["partial","*",8,1],"Claude-SearchBot":["partial","*",8,1],"anthropic-ai":["partial","*",8,1],"PerplexityBot":["partial","*",8,1],"Perplexity-User":["partial","*",8,1],"Google-Extended":["partial","*",8,1],"Applebot-Extended":["partial","*",8,1],"CCBot":["partial","*",8,1],"Bytespider":["partial","*",8,1],"Amazonbot":["partial","*",8,1],"meta-externalagent":["partial","*",8,1],"cohere-ai":["partial","*",8,1],"Diffbot":["partial","*",8,1],"ContextFetchBot":["partial","*",8,1]},"llms":{"status":"missing","http":404,"bytes":0},"home":{"status":"ok","http":200,"url":"https://qiita.com/","html_tokens":76188,"markdown_tokens":81,"pruned_tokens":42}},
{"rank":2468,"domain":"helpguide.org","origin":"https://helpguide.org","robots":{"status":"ok","http":200,"url":"https://helpguide.org/robots.txt","bytes":209,"groups":2},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["partial","*",6,0],"OAI-SearchBot":["partial","*",6,0],"ClaudeBot":["partial","*",6,0],"Claude-User":["partial","*",6,0],"Claude-SearchBot":["partial","*",6,0],"anthropic-ai":["partial","*",6,0],"PerplexityBot":["partial","*",6,0],"Perplexity-User":["partial","*",6,0],"Google-Extended":["partial","*",6,0],"Applebot-Extended":["partial","*",6,0],"CCBot":["partial","*",6,0],"Bytespider":["partial","*",6,0],"Amazonbot":["partial","*",6,0],"meta-externalagent":["partial","*",6,0],"cohere-ai":["partial","*",6,0],"Diffbot":["partial","*",6,0],"ContextFetchBot":["partial","*",6,0]},"llms":{"status":"missing","http":404,"bytes":395253},"home":{"status":"ok","http":200,"url":"https://www.helpguide.org/","html_tokens":134468,"markdown_tokens":355,"pruned_tokens":255}},
{"rank":2469,"domain":"webofscience.com","origin":"https://webofscience.com","robots":{"status":"html","http":200,"url":"https://www.webofscience.com/wos/","bytes":3063,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":2470,"domain":"cato.org","origin":"https://cato.org","robots":{"status":"refused","http":403,"url":"https://www.cato.org/robots.txt","bytes":880,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":2471,"domain":"mdr.de","origin":"https://mdr.de","robots":{"status":"ok","http":200,"url":"https://www.mdr.de/robots.txt","bytes":3381,"groups":29},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["partial","*",19,4],"OAI-SearchBot":["partial","*",19,4],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",19,4],"Claude-SearchBot":["partial","*",19,4],"anthropic-ai":["partial","*",19,4],"PerplexityBot":["partial","*",19,4],"Perplexity-User":["partial","*",19,4],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["partial","*",19,4],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["partial","*",19,4],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",19,4]},"llms":{"status":"missing","http":404,"bytes":46801},"home":{"status":"ok","http":200,"url":"https://www.mdr.de/","html_tokens":69241,"markdown_tokens":425,"pruned_tokens":134}},
{"rank":2472,"domain":"gamesradar.com","origin":"https://gamesradar.com","robots":{"status":"ok","http":200,"url":"https://www.gamesradar.com/robots.txt","bytes":1957,"groups":5},"bots":{"GPTBot":["partial","*",22,0],"ChatGPT-User":["partial","*",22,0],"OAI-SearchBot":["partial","*",22,0],"ClaudeBot":["partial","*",22,0],"Claude-User":["partial","*",22,0],"Claude-SearchBot":["partial","*",22,0],"anthropic-ai":["partial","*",22,0],"PerplexityBot":["partial","*",22,0],"Perplexity-User":["partial","*",22,0],"Google-Extended":["partial","*",22,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["partial","*",22,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["partial","*",22,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",22,0]},"llms":{"status":"missing","http":404,"bytes":1565007},"home":{"status":"ok","http":200,"url":"https://www.gamesradar.com/","html_tokens":622895,"markdown_tokens":169,"pruned_tokens":123}},
{"rank":2473,"domain":"google.se","origin":"https://google.se","robots":{"status":"ok","http":200,"url":"https://www.google.se/robots.txt","bytes":6624,"groups":4},"bots":{"GPTBot":["partial","*",169,64],"ChatGPT-User":["partial","*",169,64],"OAI-SearchBot":["partial","*",169,64],"ClaudeBot":["partial","*",169,64],"Claude-User":["partial","*",169,64],"Claude-SearchBot":["partial","*",169,64],"anthropic-ai":["partial","*",169,64],"PerplexityBot":["partial","*",169,64],"Perplexity-User":["partial","*",169,64],"Google-Extended":["partial","*",169,64],"Applebot-Extended":["partial","*",169,64],"CCBot":["partial","*",169,64],"Bytespider":["partial","*",169,64],"Amazonbot":["partial","*",169,64],"meta-externalagent":["partial","*",169,64],"cohere-ai":["partial","*",169,64],"Diffbot":["partial","*",169,64],"ContextFetchBot":["partial","*",169,64]},"llms":{"status":"missing","http":404,"bytes":1569},"home":{"status":"ok","http":200,"url":"https://www.google.com/","html_tokens":21363,"markdown_tokens":30,"pruned_tokens":30}},
{"rank":2474,"domain":"siteorigin.com","origin":"https://siteorigin.com","robots":{"status":"ok","http":200,"url":"https://siteorigin.com/robots.txt","bytes":155,"groups":3},"bots":{"GPTBot":["partial","*",3,0],"ChatGPT-User":["partial","*",3,0],"OAI-SearchBot":["partial","*",3,0],"ClaudeBot":["partial","*",3,0],"Claude-User":["partial","*",3,0],"Claude-SearchBot":["partial","*",3,0],"anthropic-ai":["partial","*",3,0],"PerplexityBot":["partial","*",3,0],"Perplexity-User":["partial","*",3,0],"Google-Extended":["partial","*",3,0],"Applebot-Extended":["partial","*",3,0],"CCBot":["partial","*",3,0],"Bytespider":["partial","*",3,0],"Amazonbot":["partial","*",3,0],"meta-externalagent":["partial","*",3,0],"cohere-ai":["partial","*",3,0],"Diffbot":["partial","*",3,0],"ContextFetchBot":["partial","*",3,0]},"llms":{"status":"other_text","http":200,"bytes":6945},"home":{"status":"ok","http":200,"url":"https://siteorigin.com/","html_tokens":28922,"markdown_tokens":830,"pruned_tokens":347}},
{"rank":2475,"domain":"va.us","origin":"https://va.us","robots":{"status":"unreachable","http":null,"url":"https://va.us/robots.txt","bytes":0,"groups":0,"error":"cannot resolve va.us"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":2476,"domain":"infoq.com","origin":"https://infoq.com","robots":{"status":"ok","http":200,"url":"https://www.infoq.com/robots.txt","bytes":2585,"groups":9},"bots":{"GPTBot":["partial","*",13,0],"ChatGPT-User":["partial","*",13,0],"OAI-SearchBot":["partial","*",13,0],"ClaudeBot":["partial","*",13,0],"Claude-User":["partial","*",13,0],"Claude-SearchBot":["partial","*",13,0],"anthropic-ai":["partial","*",13,0],"PerplexityBot":["partial","*",13,0],"Perplexity-User":["partial","*",13,0],"Google-Extended":["partial","*",13,0],"Applebot-Extended":["partial","*",13,0],"CCBot":["partial","*",13,0],"Bytespider":["partial","*",13,0],"Amazonbot":["partial","*",13,0],"meta-externalagent":["partial","*",13,0],"cohere-ai":["partial","*",13,0],"Diffbot":["partial","*",13,0],"ContextFetchBot":["partial","*",13,0]},"llms":{"status":"missing","http":404,"bytes":259},"home":{"status":"ok","http":200,"url":"https://www.infoq.com/","html_tokens":61536,"markdown_tokens":39,"pruned_tokens":39}},
{"rank":2477,"domain":"zeffy.com","origin":"https://zeffy.com","robots":{"status":"ok","http":200,"url":"https://www.zeffy.com/robots.txt","bytes":645,"groups":1},"bots":{"GPTBot":["partial","*",9,0],"ChatGPT-User":["partial","*",9,0],"OAI-SearchBot":["partial","*",9,0],"ClaudeBot":["partial","*",9,0],"Claude-User":["partial","*",9,0],"Claude-SearchBot":["partial","*",9,0],"anthropic-ai":["partial","*",9,0],"PerplexityBot":["partial","*",9,0],"Perplexity-User":["partial","*",9,0],"Google-Extended":["partial","*",9,0],"Applebot-Extended":["partial","*",9,0],"CCBot":["partial","*",9,0],"Bytespider":["partial","*",9,0],"Amazonbot":["partial","*",9,0],"meta-externalagent":["partial","*",9,0],"cohere-ai":["partial","*",9,0],"Diffbot":["partial","*",9,0],"ContextFetchBot":["partial","*",9,0]},"llms":{"status":"missing","http":404,"bytes":920862},"home":{"status":"ok","http":200,"url":"https://www.zeffy.com","html_tokens":106726,"markdown_tokens":501,"pruned_tokens":437}},
{"rank":2478,"domain":"financialexpress.com","origin":"https://financialexpress.com","robots":{"status":"ok","http":200,"url":"https://www.financialexpress.com/robots.txt","bytes":1789,"groups":20},"bots":{"GPTBot":["partial","*",25,0],"ChatGPT-User":["partial","*",25,0],"OAI-SearchBot":["partial","*",25,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",25,0],"Claude-SearchBot":["partial","*",25,0],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["partial","*",25,0],"Google-Extended":["partial","*",25,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["partial","*",25,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["partial","*",25,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",25,0]},"llms":{"status":"missing","http":404,"bytes":768738},"home":{"status":"ok","http":200,"url":"https://www.financialexpress.com/","html_tokens":293066,"markdown_tokens":79,"pruned_tokens":79}},
{"rank":2479,"domain":"thediplomat.com","origin":"https://thediplomat.com","robots":{"status":"ok","http":200,"url":"https://thediplomat.com/robots.txt","bytes":281,"groups":5},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["partial","*",2,1],"OAI-SearchBot":["partial","*",2,1],"ClaudeBot":["partial","*",2,1],"Claude-User":["partial","*",2,1],"Claude-SearchBot":["partial","*",2,1],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["partial","*",2,1],"Perplexity-User":["partial","*",2,1],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["partial","*",2,1],"CCBot":["blocked","named",1,0],"Bytespider":["partial","*",2,1],"Amazonbot":["partial","*",2,1],"meta-externalagent":["partial","*",2,1],"cohere-ai":["partial","*",2,1],"Diffbot":["partial","*",2,1],"ContextFetchBot":["partial","*",2,1]},"llms":{"status":"missing","http":404,"bytes":53303},"home":{"status":"ok","http":200,"url":"https://thediplomat.com/","html_tokens":29569,"markdown_tokens":1425,"pruned_tokens":110}},
{"rank":2480,"domain":"foxsports.com","origin":"https://foxsports.com","robots":{"status":"ok","http":200,"url":"https://www.foxsports.com/robots.txt","bytes":183,"groups":1},"bots":{"GPTBot":["partial","*",3,0],"ChatGPT-User":["partial","*",3,0],"OAI-SearchBot":["partial","*",3,0],"ClaudeBot":["partial","*",3,0],"Claude-User":["partial","*",3,0],"Claude-SearchBot":["partial","*",3,0],"anthropic-ai":["partial","*",3,0],"PerplexityBot":["partial","*",3,0],"Perplexity-User":["partial","*",3,0],"Google-Extended":["partial","*",3,0],"Applebot-Extended":["partial","*",3,0],"CCBot":["partial","*",3,0],"Bytespider":["partial","*",3,0],"Amazonbot":["partial","*",3,0],"meta-externalagent":["partial","*",3,0],"cohere-ai":["partial","*",3,0],"Diffbot":["partial","*",3,0],"ContextFetchBot":["partial","*",3,0]},"llms":{"status":"missing","http":404,"bytes":137202},"home":{"status":"ok","http":200,"url":"https://www.foxsports.com/","html_tokens":231142,"markdown_tokens":7,"pruned_tokens":7}},
{"rank":2481,"domain":"eldiario.es","origin":"https://eldiario.es","robots":{"status":"ok","http":200,"url":"https://www.eldiario.es/robots.txt","bytes":880,"groups":1},"bots":{"GPTBot":["partial","*",23,0],"ChatGPT-User":["partial","*",23,0],"OAI-SearchBot":["partial","*",23,0],"ClaudeBot":["partial","*",23,0],"Claude-User":["partial","*",23,0],"Claude-SearchBot":["partial","*",23,0],"anthropic-ai":["partial","*",23,0],"PerplexityBot":["partial","*",23,0],"Perplexity-User":["partial","*",23,0],"Google-Extended":["partial","*",23,0],"Applebot-Extended":["partial","*",23,0],"CCBot":["partial","*",23,0],"Bytespider":["partial","*",23,0],"Amazonbot":["partial","*",23,0],"meta-externalagent":["partial","*",23,0],"cohere-ai":["partial","*",23,0],"Diffbot":["partial","*",23,0],"ContextFetchBot":["partial","*",23,0]},"llms":{"status":"missing","http":404,"bytes":216888},"home":{"status":"ok","http":200,"url":"https://www.eldiario.es/","html_tokens":222127,"markdown_tokens":115,"pruned_tokens":115}},
{"rank":2482,"domain":"lsu.edu","origin":"https://lsu.edu","robots":{"status":"ok","http":200,"url":"https://lsu.edu/robots.txt","bytes":199,"groups":2},"bots":{"GPTBot":["partial","*",8,0],"ChatGPT-User":["partial","*",8,0],"OAI-SearchBot":["partial","*",8,0],"ClaudeBot":["partial","*",8,0],"Claude-User":["partial","*",8,0],"Claude-SearchBot":["partial","*",8,0],"anthropic-ai":["partial","*",8,0],"PerplexityBot":["partial","*",8,0],"Perplexity-User":["partial","*",8,0],"Google-Extended":["partial","*",8,0],"Applebot-Extended":["partial","*",8,0],"CCBot":["partial","*",8,0],"Bytespider":["partial","*",8,0],"Amazonbot":["partial","*",8,0],"meta-externalagent":["partial","*",8,0],"cohere-ai":["partial","*",8,0],"Diffbot":["partial","*",8,0],"ContextFetchBot":["partial","*",8,0]},"llms":{"status":"missing","http":404,"bytes":51356},"home":{"status":"ok","http":200,"url":"https://lsu.edu/","html_tokens":20699,"markdown_tokens":461,"pruned_tokens":300}},
{"rank":2483,"domain":"bhg.com","origin":"https://bhg.com","robots":{"status":"ok","http":200,"url":"https://www.bhg.com/robots.txt","bytes":3459,"groups":6},"bots":{"GPTBot":["partial","named",1,0],"ChatGPT-User":["partial","named",1,0],"OAI-SearchBot":["partial","named",1,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["blocked","named",1,0],"Claude-SearchBot":["partial","named",1,2],"anthropic-ai":["partial","named",1,2],"PerplexityBot":["partial","named",1,2],"Perplexity-User":["blocked","named",1,0],"Google-Extended":["partial","named",1,2],"Applebot-Extended":["blocked","named",1,0],"CCBot":["partial","named",1,2],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["partial","named",1,2],"cohere-ai":["partial","named",1,2],"Diffbot":["partial","*",2,0],"ContextFetchBot":["partial","*",2,0]},"llms":{"status":"http_error","http":402,"bytes":612},"home":{"status":"http_error","http":402,"url":"https://www.bhg.com/"}},
{"rank":2484,"domain":"discourse.org","origin":"https://discourse.org","robots":{"status":"ok","http":200,"url":"https://www.discourse.org/robots.txt","bytes":61,"groups":1},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"missing","http":404,"bytes":45548},"home":{"status":"ok","http":200,"url":"https://www.discourse.org/","html_tokens":28090,"markdown_tokens":602,"pruned_tokens":144}},
{"rank":2485,"domain":"themoscowtimes.com","origin":"https://themoscowtimes.com","robots":{"status":"ok","http":200,"url":"https://www.themoscowtimes.com/robots.txt","bytes":290,"groups":2},"bots":{"GPTBot":["partial","*",4,0],"ChatGPT-User":["partial","*",4,0],"OAI-SearchBot":["partial","*",4,0],"ClaudeBot":["partial","*",4,0],"Claude-User":["partial","*",4,0],"Claude-SearchBot":["partial","*",4,0],"anthropic-ai":["partial","*",4,0],"PerplexityBot":["partial","*",4,0],"Perplexity-User":["partial","*",4,0],"Google-Extended":["partial","*",4,0],"Applebot-Extended":["partial","*",4,0],"CCBot":["partial","*",4,0],"Bytespider":["partial","*",4,0],"Amazonbot":["partial","*",4,0],"meta-externalagent":["partial","*",4,0],"cohere-ai":["partial","*",4,0],"Diffbot":["partial","*",4,0],"ContextFetchBot":["partial","*",4,0]},"llms":{"status":"missing","http":404,"bytes":72899},"home":{"status":"ok","http":200,"url":"https://www.themoscowtimes.com/","html_tokens":30180,"markdown_tokens":134,"pruned_tokens":134}},
{"rank":2486,"domain":"activecampaign.com","origin":"https://activecampaign.com","robots":{"status":"ok","http":200,"url":"https://www.activecampaign.com/robots.txt","bytes":1502,"groups":3},"bots":{"GPTBot":["partial","*",27,3],"ChatGPT-User":["partial","*",27,3],"OAI-SearchBot":["partial","*",27,3],"ClaudeBot":["partial","*",27,3],"Claude-User":["partial","*",27,3],"Claude-SearchBot":["partial","*",27,3],"anthropic-ai":["partial","*",27,3],"PerplexityBot":["partial","*",27,3],"Perplexity-User":["partial","*",27,3],"Google-Extended":["partial","*",27,3],"Applebot-Extended":["partial","*",27,3],"CCBot":["partial","*",27,3],"Bytespider":["partial","*",27,3],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["partial","*",27,3],"cohere-ai":["partial","*",27,3],"Diffbot":["partial","*",27,3],"ContextFetchBot":["partial","*",27,3]},"llms":{"status":"missing","http":404,"bytes":176303},"home":{"status":"ok","http":200,"url":"https://www.activecampaign.com/","html_tokens":114729,"markdown_tokens":517,"pruned_tokens":274}},
{"rank":2487,"domain":"volvocars.com","origin":"https://volvocars.com","robots":{"status":"refused","http":403,"url":"https://www.volvocars.com/robots.txt/","bytes":388,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":2488,"domain":"tue.nl","origin":"https://tue.nl","robots":{"status":"server_error","http":500,"url":"https://tue.nl/robots.txt","bytes":39108,"groups":0,"error":"HTTP 500"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":2489,"domain":"onet.pl","origin":"https://onet.pl","robots":{"status":"ok","http":200,"url":"https://www.onet.pl/robots.txt","bytes":586,"groups":15},"bots":{"GPTBot":["partial","*",1,0],"ChatGPT-User":["partial","*",1,0],"OAI-SearchBot":["partial","*",1,0],"ClaudeBot":["partial","*",1,0],"Claude-User":["partial","*",1,0],"Claude-SearchBot":["partial","*",1,0],"anthropic-ai":["partial","*",1,0],"PerplexityBot":["partial","*",1,0],"Perplexity-User":["partial","*",1,0],"Google-Extended":["partial","*",1,0],"Applebot-Extended":["partial","*",1,0],"CCBot":["partial","*",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",1,0]},"llms":{"status":"missing","http":404,"bytes":111819},"home":{"status":"ok","http":200,"url":"https://www.onet.pl/","html_tokens":84669,"markdown_tokens":110,"pruned_tokens":110}},
{"rank":2490,"domain":"connect.ok.ru","origin":"https://connect.ok.ru","robots":{"status":"ok","http":200,"url":"https://connect.ok.ru/robots.txt","bytes":25,"groups":1},"bots":{"GPTBot":["blocked","*",1,0],"ChatGPT-User":["blocked","*",1,0],"OAI-SearchBot":["blocked","*",1,0],"ClaudeBot":["blocked","*",1,0],"Claude-User":["blocked","*",1,0],"Claude-SearchBot":["blocked","*",1,0],"anthropic-ai":["blocked","*",1,0],"PerplexityBot":["blocked","*",1,0],"Perplexity-User":["blocked","*",1,0],"Google-Extended":["blocked","*",1,0],"Applebot-Extended":["blocked","*",1,0],"CCBot":["blocked","*",1,0],"Bytespider":["blocked","*",1,0],"Amazonbot":["blocked","*",1,0],"meta-externalagent":["blocked","*",1,0],"cohere-ai":["blocked","*",1,0],"Diffbot":["blocked","*",1,0],"ContextFetchBot":["blocked","*",1,0]},"llms":{"status":"disallowed"},"home":{"status":"disallowed"}},
{"rank":2491,"domain":"msi.com","origin":"https://msi.com","robots":{"status":"refused","http":403,"url":"https://www.msi.com/robots.txt","bytes":379,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":2492,"domain":"sh.gov.cn","origin":"https://sh.gov.cn","robots":{"status":"missing","http":404,"url":"https://www.shanghai.gov.cn/robots.txt","bytes":1091,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":1091},"home":{"status":"ok","http":200,"url":"https://www.shanghai.gov.cn/","html_tokens":29797,"markdown_tokens":16,"pruned_tokens":16}},
{"rank":2493,"domain":"cochrane.org","origin":"https://cochrane.org","robots":{"status":"ok","http":200,"url":"https://www.cochrane.org/robots.txt","bytes":2247,"groups":1},"bots":{"GPTBot":["partial","*",36,22],"ChatGPT-User":["partial","*",36,22],"OAI-SearchBot":["partial","*",36,22],"ClaudeBot":["partial","*",36,22],"Claude-User":["partial","*",36,22],"Claude-SearchBot":["partial","*",36,22],"anthropic-ai":["partial","*",36,22],"PerplexityBot":["partial","*",36,22],"Perplexity-User":["partial","*",36,22],"Google-Extended":["partial","*",36,22],"Applebot-Extended":["partial","*",36,22],"CCBot":["partial","*",36,22],"Bytespider":["partial","*",36,22],"Amazonbot":["partial","*",36,22],"meta-externalagent":["partial","*",36,22],"cohere-ai":["partial","*",36,22],"Diffbot":["partial","*",36,22],"ContextFetchBot":["partial","*",36,22]},"llms":{"status":"missing","http":404,"bytes":291},"home":{"status":"ok","http":200,"url":"https://www.cochrane.org/","html_tokens":18254,"markdown_tokens":386,"pruned_tokens":112}},
{"rank":2494,"domain":"newstatesman.com","origin":"https://newstatesman.com","robots":{"status":"refused","http":403,"url":"https://newstatesman.com/robots.txt","bytes":560,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":2495,"domain":"health.gov","origin":"https://health.gov","robots":{"status":"ok","http":200,"url":"https://health.gov/robots.txt","bytes":2027,"groups":1},"bots":{"GPTBot":["partial","*",34,18],"ChatGPT-User":["partial","*",34,18],"OAI-SearchBot":["partial","*",34,18],"ClaudeBot":["partial","*",34,18],"Claude-User":["partial","*",34,18],"Claude-SearchBot":["partial","*",34,18],"anthropic-ai":["partial","*",34,18],"PerplexityBot":["partial","*",34,18],"Perplexity-User":["partial","*",34,18],"Google-Extended":["partial","*",34,18],"Applebot-Extended":["partial","*",34,18],"CCBot":["partial","*",34,18],"Bytespider":["partial","*",34,18],"Amazonbot":["partial","*",34,18],"meta-externalagent":["partial","*",34,18],"cohere-ai":["partial","*",34,18],"Diffbot":["partial","*",34,18],"ContextFetchBot":["partial","*",34,18]},"llms":{"status":"missing","http":404,"bytes":184},"home":{"status":"ok","http":200,"url":"https://health.gov/","html_tokens":21573,"markdown_tokens":706,"pruned_tokens":123}},
{"rank":2496,"domain":"provenexpert.com","origin":"https://provenexpert.com","robots":{"status":"ok","http":200,"url":"https://www.provenexpert.com/robots.txt","bytes":168,"groups":1},"bots":{"GPTBot":["partial","*",1,0],"ChatGPT-User":["partial","*",1,0],"OAI-SearchBot":["partial","*",1,0],"ClaudeBot":["partial","*",1,0],"Claude-User":["partial","*",1,0],"Claude-SearchBot":["partial","*",1,0],"anthropic-ai":["partial","*",1,0],"PerplexityBot":["partial","*",1,0],"Perplexity-User":["partial","*",1,0],"Google-Extended":["partial","*",1,0],"Applebot-Extended":["partial","*",1,0],"CCBot":["partial","*",1,0],"Bytespider":["partial","*",1,0],"Amazonbot":["partial","*",1,0],"meta-externalagent":["partial","*",1,0],"cohere-ai":["partial","*",1,0],"Diffbot":["partial","*",1,0],"ContextFetchBot":["partial","*",1,0]},"llms":{"status":"missing","http":410,"bytes":54959},"home":{"status":"ok","http":200,"url":"https://www.provenexpert.com/en-us/","html_tokens":195854,"markdown_tokens":886,"pruned_tokens":183}},
{"rank":2497,"domain":"uwo.ca","origin":"https://uwo.ca","robots":{"status":"ok","http":200,"url":"https://www.uwo.ca/robots.txt","bytes":6848,"groups":2},"bots":{"GPTBot":["partial","*",114,0],"ChatGPT-User":["partial","*",114,0],"OAI-SearchBot":["partial","*",114,0],"ClaudeBot":["partial","*",114,0],"Claude-User":["partial","*",114,0],"Claude-SearchBot":["partial","*",114,0],"anthropic-ai":["partial","*",114,0],"PerplexityBot":["partial","*",114,0],"Perplexity-User":["partial","*",114,0],"Google-Extended":["partial","*",114,0],"Applebot-Extended":["partial","*",114,0],"CCBot":["partial","*",114,0],"Bytespider":["partial","*",114,0],"Amazonbot":["partial","*",114,0],"meta-externalagent":["partial","*",114,0],"cohere-ai":["partial","*",114,0],"Diffbot":["partial","*",114,0],"ContextFetchBot":["partial","*",114,0]},"llms":{"status":"missing","http":404,"bytes":21405},"home":{"status":"ok","http":200,"url":"https://www.uwo.ca/","html_tokens":40875,"markdown_tokens":463,"pruned_tokens":121}},
{"rank":2498,"domain":"rsf.org","origin":"https://rsf.org","robots":{"status":"ok","http":200,"url":"https://rsf.org/robots.txt","bytes":2202,"groups":4},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["blocked","named",1,0],"OAI-SearchBot":["partial","*",45,18],"ClaudeBot":["partial","*",45,18],"Claude-User":["partial","*",45,18],"Claude-SearchBot":["partial","*",45,18],"anthropic-ai":["partial","*",45,18],"PerplexityBot":["partial","*",45,18],"Perplexity-User":["partial","*",45,18],"Google-Extended":["partial","*",45,18],"Applebot-Extended":["partial","*",45,18],"CCBot":["partial","*",45,18],"Bytespider":["blocked","named",1,0],"Amazonbot":["partial","*",45,18],"meta-externalagent":["partial","*",45,18],"cohere-ai":["partial","*",45,18],"Diffbot":["partial","*",45,18],"ContextFetchBot":["partial","*",45,18]},"llms":{"status":"missing","http":404,"bytes":181},"home":{"status":"ok","http":200,"url":"https://rsf.org/en","html_tokens":29541,"markdown_tokens":122,"pruned_tokens":122}},
{"rank":2499,"domain":"icq.com","origin":"https://icq.com","robots":{"status":"ok","http":200,"url":"https://icq.com/robots.txt","bytes":150,"groups":2},"bots":{"GPTBot":["partial","*",5,0],"ChatGPT-User":["partial","*",5,0],"OAI-SearchBot":["partial","*",5,0],"ClaudeBot":["partial","*",5,0],"Claude-User":["partial","*",5,0],"Claude-SearchBot":["partial","*",5,0],"anthropic-ai":["partial","*",5,0],"PerplexityBot":["partial","*",5,0],"Perplexity-User":["partial","*",5,0],"Google-Extended":["partial","*",5,0],"Applebot-Extended":["partial","*",5,0],"CCBot":["partial","*",5,0],"Bytespider":["partial","*",5,0],"Amazonbot":["partial","*",5,0],"meta-externalagent":["partial","*",5,0],"cohere-ai":["partial","*",5,0],"Diffbot":["partial","*",5,0],"ContextFetchBot":["partial","*",5,0]},"llms":{"status":"missing","http":404,"bytes":5475},"home":{"status":"ok","http":200,"url":"https://icq.com/desktop/en#windows","html_tokens":10461,"markdown_tokens":27,"pruned_tokens":27}},
{"rank":2500,"domain":"auth0.com","origin":"https://auth0.com","robots":{"status":"ok","http":200,"url":"https://auth0.com/robots.txt","bytes":320,"groups":1},"bots":{"GPTBot":["partial","*",10,1],"ChatGPT-User":["partial","*",10,1],"OAI-SearchBot":["partial","*",10,1],"ClaudeBot":["partial","*",10,1],"Claude-User":["partial","*",10,1],"Claude-SearchBot":["partial","*",10,1],"anthropic-ai":["partial","*",10,1],"PerplexityBot":["partial","*",10,1],"Perplexity-User":["partial","*",10,1],"Google-Extended":["partial","*",10,1],"Applebot-Extended":["partial","*",10,1],"CCBot":["partial","*",10,1],"Bytespider":["partial","*",10,1],"Amazonbot":["partial","*",10,1],"meta-externalagent":["partial","*",10,1],"cohere-ai":["partial","*",10,1],"Diffbot":["partial","*",10,1],"ContextFetchBot":["partial","*",10,1]},"llms":{"status":"present","http":200,"bytes":4232},"home":{"status":"ok","http":200,"url":"https://auth0.com/","html_tokens":112468,"markdown_tokens":1088,"pruned_tokens":111}},
{"rank":2501,"domain":"msdmanuals.com","origin":"https://msdmanuals.com","robots":{"status":"ok","http":200,"url":"https://www.msdmanuals.com/robots.txt","bytes":1736,"groups":4},"bots":{"GPTBot":["partial","*",24,0],"ChatGPT-User":["partial","*",24,0],"OAI-SearchBot":["partial","*",24,0],"ClaudeBot":["partial","*",24,0],"Claude-User":["partial","*",24,0],"Claude-SearchBot":["partial","*",24,0],"anthropic-ai":["partial","*",24,0],"PerplexityBot":["partial","*",24,0],"Perplexity-User":["partial","*",24,0],"Google-Extended":["partial","*",24,0],"Applebot-Extended":["partial","*",24,0],"CCBot":["partial","*",24,0],"Bytespider":["partial","*",24,0],"Amazonbot":["partial","*",24,0],"meta-externalagent":["partial","*",24,0],"cohere-ai":["partial","*",24,0],"Diffbot":["partial","*",24,0],"ContextFetchBot":["partial","*",24,0]},"llms":{"status":"missing","http":404,"bytes":115273},"home":{"status":"ok","http":200,"url":"https://www.msdmanuals.com/","html_tokens":32730,"markdown_tokens":610,"pruned_tokens":122}},
{"rank":2502,"domain":"boardgamegeek.com","origin":"https://boardgamegeek.com","robots":{"status":"ok","http":200,"url":"https://boardgamegeek.com/robots.txt","bytes":4205,"groups":3},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["partial","*",114,1],"OAI-SearchBot":["partial","*",114,1],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",114,1],"Claude-SearchBot":["partial","*",114,1],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["blocked","named",1,0],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",114,1]},"llms":{"status":"missing","http":404,"bytes":6212},"home":{"status":"http_error","http":403,"url":"https://boardgamegeek.com/"}},
{"rank":2503,"domain":"metoffice.gov.uk","origin":"https://metoffice.gov.uk","robots":{"status":"ok","http":200,"url":"https://www.metoffice.gov.uk/robots.txt","bytes":408,"groups":2},"bots":{"GPTBot":["partial","*",13,0],"ChatGPT-User":["partial","*",13,0],"OAI-SearchBot":["partial","*",13,0],"ClaudeBot":["partial","*",13,0],"Claude-User":["partial","*",13,0],"Claude-SearchBot":["partial","*",13,0],"anthropic-ai":["partial","*",13,0],"PerplexityBot":["partial","*",13,0],"Perplexity-User":["partial","*",13,0],"Google-Extended":["partial","*",13,0],"Applebot-Extended":["partial","*",13,0],"CCBot":["partial","*",13,0],"Bytespider":["partial","*",13,0],"Amazonbot":["partial","*",13,0],"meta-externalagent":["partial","*",13,0],"cohere-ai":["partial","*",13,0],"Diffbot":["partial","*",13,0],"ContextFetchBot":["partial","*",13,0]},"llms":{"status":"missing","http":404,"bytes":1473},"home":{"status":"ok","http":200,"url":"https://www.metoffice.gov.uk/","html_tokens":42143,"markdown_tokens":69,"pruned_tokens":69}},
{"rank":2504,"domain":"ionos.com","origin":"https://ionos.com","robots":{"status":"ok","http":200,"url":"https://www.ionos.com/robots.txt","bytes":450,"groups":2},"bots":{"GPTBot":["partial","*",8,1],"ChatGPT-User":["partial","*",8,1],"OAI-SearchBot":["partial","*",8,1],"ClaudeBot":["partial","*",8,1],"Claude-User":["partial","*",8,1],"Claude-SearchBot":["partial","*",8,1],"anthropic-ai":["partial","*",8,1],"PerplexityBot":["partial","*",8,1],"Perplexity-User":["partial","*",8,1],"Google-Extended":["partial","*",8,1],"Applebot-Extended":["partial","*",8,1],"CCBot":["partial","*",8,1],"Bytespider":["partial","*",8,1],"Amazonbot":["partial","*",8,1],"meta-externalagent":["partial","*",8,1],"cohere-ai":["partial","*",8,1],"Diffbot":["partial","*",8,1],"ContextFetchBot":["partial","*",8,1]},"llms":{"status":"missing","http":404,"bytes":596356},"home":{"status":"ok","http":200,"url":"https://www.ionos.com/","html_tokens":186229,"markdown_tokens":56,"pruned_tokens":56}},
{"rank":2505,"domain":"outsideonline.com","origin":"https://outsideonline.com","robots":{"status":"ok","http":200,"url":"https://www.outsideonline.com/robots.txt","bytes":1079,"groups":2},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["partial","*",4,1],"OAI-SearchBot":["partial","*",4,1],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",4,1],"Claude-SearchBot":["partial","*",4,1],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["partial","*",4,1],"Perplexity-User":["partial","*",4,1],"Google-Extended":["partial","*",4,1],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",4,1]},"llms":{"status":"missing","http":404,"bytes":246826},"home":{"status":"ok","http":200,"url":"https://www.outsideonline.com/featured","html_tokens":102926,"markdown_tokens":1780,"pruned_tokens":147}},
{"rank":2506,"domain":"techopedia.com","origin":"https://techopedia.com","robots":{"status":"ok","http":200,"url":"https://www.techopedia.com/robots.txt","bytes":1023,"groups":4},"bots":{"GPTBot":["partial","*",18,3],"ChatGPT-User":["partial","*",18,3],"OAI-SearchBot":["partial","*",18,3],"ClaudeBot":["partial","*",18,3],"Claude-User":["partial","*",18,3],"Claude-SearchBot":["partial","*",18,3],"anthropic-ai":["partial","*",18,3],"PerplexityBot":["partial","*",18,3],"Perplexity-User":["partial","*",18,3],"Google-Extended":["partial","*",18,3],"Applebot-Extended":["partial","*",18,3],"CCBot":["partial","*",18,3],"Bytespider":["partial","*",18,3],"Amazonbot":["partial","*",18,3],"meta-externalagent":["partial","*",18,3],"cohere-ai":["partial","*",18,3],"Diffbot":["partial","*",18,3],"ContextFetchBot":["partial","*",18,3]},"llms":{"status":"missing","http":404,"bytes":93196},"home":{"status":"ok","http":200,"url":"https://www.techopedia.com/","html_tokens":32636,"markdown_tokens":103,"pruned_tokens":103}},
{"rank":2507,"domain":"researchandmarkets.com","origin":"https://researchandmarkets.com","robots":{"status":"ok","http":200,"url":"https://www.researchandmarkets.com/robots.txt","bytes":1426,"groups":12},"bots":{"GPTBot":["partial","*",23,0],"ChatGPT-User":["partial","*",23,0],"OAI-SearchBot":["partial","*",23,0],"ClaudeBot":["partial","*",23,0],"Claude-User":["partial","*",23,0],"Claude-SearchBot":["partial","*",23,0],"anthropic-ai":["partial","*",23,0],"PerplexityBot":["partial","*",23,0],"Perplexity-User":["partial","*",23,0],"Google-Extended":["partial","*",23,0],"Applebot-Extended":["partial","*",23,0],"CCBot":["partial","*",23,0],"Bytespider":["partial","*",23,0],"Amazonbot":["partial","*",23,0],"meta-externalagent":["partial","*",23,0],"cohere-ai":["partial","*",23,0],"Diffbot":["partial","*",23,0],"ContextFetchBot":["partial","*",23,0]},"llms":{"status":"missing","http":404,"bytes":50817},"home":{"status":"ok","http":200,"url":"https://www.researchandmarkets.com/","html_tokens":25586,"markdown_tokens":95,"pruned_tokens":85}},
{"rank":2508,"domain":"sas.com","origin":"https://sas.com","robots":{"status":"ok","http":200,"url":"https://www.sas.com/robots.txt","bytes":4010,"groups":2},"bots":{"GPTBot":["partial","named",84,1],"ChatGPT-User":["partial","named",84,1],"OAI-SearchBot":["partial","named",84,1],"ClaudeBot":["partial","named",84,1],"Claude-User":["partial","named",84,1],"Claude-SearchBot":["partial","named",84,1],"anthropic-ai":["blocked","*",1,0],"PerplexityBot":["partial","named",84,1],"Perplexity-User":["partial","named",84,1],"Google-Extended":["partial","named",84,1],"Applebot-Extended":["blocked","*",1,0],"CCBot":["partial","named",84,1],"Bytespider":["partial","named",84,1],"Amazonbot":["partial","named",84,1],"meta-externalagent":["blocked","*",1,0],"cohere-ai":["blocked","*",1,0],"Diffbot":["blocked","*",1,0],"ContextFetchBot":["blocked","*",1,0]},"llms":{"status":"disallowed"},"home":{"status":"disallowed"}},
{"rank":2509,"domain":"temple.edu","origin":"https://temple.edu","robots":{"status":"html","http":200,"url":"https://temple.edu/robots.txt","bytes":269,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":2510,"domain":"yubico.com","origin":"https://yubico.com","robots":{"status":"ok","http":200,"url":"https://www.yubico.com/robots.txt","bytes":213,"groups":1},"bots":{"GPTBot":["partial","*",5,0],"ChatGPT-User":["partial","*",5,0],"OAI-SearchBot":["partial","*",5,0],"ClaudeBot":["partial","*",5,0],"Claude-User":["partial","*",5,0],"Claude-SearchBot":["partial","*",5,0],"anthropic-ai":["partial","*",5,0],"PerplexityBot":["partial","*",5,0],"Perplexity-User":["partial","*",5,0],"Google-Extended":["partial","*",5,0],"Applebot-Extended":["partial","*",5,0],"CCBot":["partial","*",5,0],"Bytespider":["partial","*",5,0],"Amazonbot":["partial","*",5,0],"meta-externalagent":["partial","*",5,0],"cohere-ai":["partial","*",5,0],"Diffbot":["partial","*",5,0],"ContextFetchBot":["partial","*",5,0]},"llms":{"status":"missing","http":404,"bytes":416467},"home":{"status":"ok","http":200,"url":"https://www.yubico.com/","html_tokens":130474,"markdown_tokens":296,"pruned_tokens":166}},
{"rank":2511,"domain":"soumu.go.jp","origin":"https://www.soumu.go.jp","robots":{"status":"ok","http":200,"url":"https://www.soumu.go.jp/robots.txt","bytes":35,"groups":1},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"http_error","http":502,"bytes":996},"home":{"status":"ok","http":200,"url":"https://www.soumu.go.jp/","html_tokens":56299,"markdown_tokens":1281,"pruned_tokens":171}},
{"rank":2512,"domain":"intercom.com","origin":"https://intercom.com","robots":{"status":"ok","http":200,"url":"https://www.intercom.com/robots.txt","bytes":261,"groups":1},"bots":{"GPTBot":["partial","*",7,0],"ChatGPT-User":["partial","*",7,0],"OAI-SearchBot":["partial","*",7,0],"ClaudeBot":["partial","*",7,0],"Claude-User":["partial","*",7,0],"Claude-SearchBot":["partial","*",7,0],"anthropic-ai":["partial","*",7,0],"PerplexityBot":["partial","*",7,0],"Perplexity-User":["partial","*",7,0],"Google-Extended":["partial","*",7,0],"Applebot-Extended":["partial","*",7,0],"CCBot":["partial","*",7,0],"Bytespider":["partial","*",7,0],"Amazonbot":["partial","*",7,0],"meta-externalagent":["partial","*",7,0],"cohere-ai":["partial","*",7,0],"Diffbot":["partial","*",7,0],"ContextFetchBot":["partial","*",7,0]},"llms":{"status":"present","http":200,"bytes":9094},"home":{"status":"not_html","error":"unsupported content type text/markdown"}},
{"rank":2513,"domain":"sc.edu","origin":"https://sc.edu","robots":{"status":"ok","http":200,"url":"https://sc.edu/robots.txt","bytes":1063,"groups":6},"bots":{"GPTBot":["partial","*",28,0],"ChatGPT-User":["partial","*",28,0],"OAI-SearchBot":["partial","*",28,0],"ClaudeBot":["partial","*",28,0],"Claude-User":["partial","*",28,0],"Claude-SearchBot":["partial","*",28,0],"anthropic-ai":["partial","*",28,0],"PerplexityBot":["partial","*",28,0],"Perplexity-User":["partial","*",28,0],"Google-Extended":["partial","*",28,0],"Applebot-Extended":["partial","*",28,0],"CCBot":["partial","*",28,0],"Bytespider":["partial","*",28,0],"Amazonbot":["partial","*",28,0],"meta-externalagent":["partial","*",28,0],"cohere-ai":["partial","*",28,0],"Diffbot":["partial","*",28,0],"ContextFetchBot":["partial","*",28,0]},"llms":{"status":"html","http":200,"bytes":46138},"home":{"status":"ok","http":200,"url":"https://sc.edu/","html_tokens":15150,"markdown_tokens":128,"pruned_tokens":129}},
{"rank":2514,"domain":"pitchfork.com","origin":"https://pitchfork.com","robots":{"status":"ok","http":200,"url":"https://pitchfork.com/robots.txt","bytes":2149,"groups":3},"bots":{"GPTBot":["partial","*",13,14],"ChatGPT-User":["partial","*",13,14],"OAI-SearchBot":["partial","*",13,14],"ClaudeBot":["blocked","named",1,0],"Claude-User":["blocked","named",1,0],"Claude-SearchBot":["blocked","named",1,0],"anthropic-ai":["partial","*",13,14],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["blocked","named",1,0],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",13,14]},"llms":{"status":"missing","http":404,"bytes":579443},"home":{"status":"ok","http":200,"url":"https://pitchfork.com/","html_tokens":338574,"markdown_tokens":799,"pruned_tokens":400}},
{"rank":2515,"domain":"ewg.org","origin":"https://ewg.org","robots":{"status":"ok","http":200,"url":"https://www.ewg.org/robots.txt","bytes":2027,"groups":1},"bots":{"GPTBot":["partial","*",34,18],"ChatGPT-User":["partial","*",34,18],"OAI-SearchBot":["partial","*",34,18],"ClaudeBot":["partial","*",34,18],"Claude-User":["partial","*",34,18],"Claude-SearchBot":["partial","*",34,18],"anthropic-ai":["partial","*",34,18],"PerplexityBot":["partial","*",34,18],"Perplexity-User":["partial","*",34,18],"Google-Extended":["partial","*",34,18],"Applebot-Extended":["partial","*",34,18],"CCBot":["partial","*",34,18],"Bytespider":["partial","*",34,18],"Amazonbot":["partial","*",34,18],"meta-externalagent":["partial","*",34,18],"cohere-ai":["partial","*",34,18],"Diffbot":["partial","*",34,18],"ContextFetchBot":["partial","*",34,18]},"llms":{"status":"present","http":200,"bytes":12773},"home":{"status":"http_error","http":403,"url":"https://www.ewg.org/"}},
{"rank":2516,"domain":"twitpic.com","origin":"https://twitpic.com","robots":{"status":"html","http":200,"url":"https://twitpic.com","bytes":10123,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":2517,"domain":"bankofengland.co.uk","origin":"https://www.bankofengland.co.uk","robots":{"status":"ok","http":200,"url":"https://www.bankofengland.co.uk/robots.txt","bytes":357,"groups":1},"bots":{"GPTBot":["partial","*",9,0],"ChatGPT-User":["partial","*",9,0],"OAI-SearchBot":["partial","*",9,0],"ClaudeBot":["partial","*",9,0],"Claude-User":["partial","*",9,0],"Claude-SearchBot":["partial","*",9,0],"anthropic-ai":["partial","*",9,0],"PerplexityBot":["partial","*",9,0],"Perplexity-User":["partial","*",9,0],"Google-Extended":["partial","*",9,0],"Applebot-Extended":["partial","*",9,0],"CCBot":["partial","*",9,0],"Bytespider":["partial","*",9,0],"Amazonbot":["partial","*",9,0],"meta-externalagent":["partial","*",9,0],"cohere-ai":["partial","*",9,0],"Diffbot":["partial","*",9,0],"ContextFetchBot":["partial","*",9,0]},"llms":{"status":"missing","http":404,"bytes":55911},"home":{"status":"ok","http":200,"url":"https://www.bankofengland.co.uk/","html_tokens":21812,"markdown_tokens":63,"pruned_tokens":63}},
{"rank":2518,"domain":"arm.com","origin":"https://arm.com","robots":{"status":"ok","http":200,"url":"https://www.arm.com/robots.txt","bytes":2109,"groups":24},"bots":{"GPTBot":["allowed","named",0,1],"ChatGPT-User":["allowed","named",0,1],"OAI-SearchBot":["partial","*",15,0],"ClaudeBot":["allowed","named",0,1],"Claude-User":["partial","*",15,0],"Claude-SearchBot":["partial","*",15,0],"anthropic-ai":["allowed","named",0,1],"PerplexityBot":["allowed","named",0,1],"Perplexity-User":["partial","*",15,0],"Google-Extended":["allowed","named",0,1],"Applebot-Extended":["partial","*",15,0],"CCBot":["allowed","named",0,1],"Bytespider":["allowed","named",0,1],"Amazonbot":["allowed","named",0,1],"meta-externalagent":["partial","*",15,0],"cohere-ai":["partial","*",15,0],"Diffbot":["partial","*",15,0],"ContextFetchBot":["partial","*",15,0]},"llms":{"status":"present","http":200,"bytes":28445},"home":{"status":"ok","http":200,"url":"https://www.arm.com/","html_tokens":135506,"markdown_tokens":395,"pruned_tokens":134}},
{"rank":2519,"domain":"geekwire.com","origin":"https://geekwire.com","robots":{"status":"ok","http":200,"url":"https://www.geekwire.com/robots.txt","bytes":282,"groups":1},"bots":{"GPTBot":["partial","*",7,1],"ChatGPT-User":["partial","*",7,1],"OAI-SearchBot":["partial","*",7,1],"ClaudeBot":["partial","*",7,1],"Claude-User":["partial","*",7,1],"Claude-SearchBot":["partial","*",7,1],"anthropic-ai":["partial","*",7,1],"PerplexityBot":["partial","*",7,1],"Perplexity-User":["partial","*",7,1],"Google-Extended":["partial","*",7,1],"Applebot-Extended":["partial","*",7,1],"CCBot":["partial","*",7,1],"Bytespider":["partial","*",7,1],"Amazonbot":["partial","*",7,1],"meta-externalagent":["partial","*",7,1],"cohere-ai":["partial","*",7,1],"Diffbot":["partial","*",7,1],"ContextFetchBot":["partial","*",7,1]},"llms":{"status":"missing","http":404,"bytes":94232},"home":{"status":"http_error","http":403,"url":"https://www.geekwire.com/"}},
{"rank":2520,"domain":"avito.ru","origin":"https://avito.ru","robots":{"status":"ok","http":200,"url":"https://www.avito.ru/robots.txt","bytes":14700,"groups":5},"bots":{"GPTBot":["partial","*",126,13],"ChatGPT-User":["partial","*",126,13],"OAI-SearchBot":["partial","*",126,13],"ClaudeBot":["partial","*",126,13],"Claude-User":["partial","*",126,13],"Claude-SearchBot":["partial","*",126,13],"anthropic-ai":["partial","*",126,13],"PerplexityBot":["partial","*",126,13],"Perplexity-User":["partial","*",126,13],"Google-Extended":["partial","*",126,13],"Applebot-Extended":["partial","*",126,13],"CCBot":["partial","*",126,13],"Bytespider":["partial","*",126,13],"Amazonbot":["partial","*",126,13],"meta-externalagent":["partial","*",126,13],"cohere-ai":["partial","*",126,13],"Diffbot":["partial","*",126,13],"ContextFetchBot":["partial","*",126,13]},"llms":{"status":"missing","http":404,"bytes":29907},"home":{"status":"http_error","http":429,"url":"https://www.avito.ru/"}},
{"rank":2521,"domain":"au.dk","origin":"https://au.dk","robots":{"status":"missing","http":404,"url":"https://www.au.dk/robots.txt","bytes":8436,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":8436},"home":{"status":"ok","http":200,"url":"https://www.au.dk/","html_tokens":20131,"markdown_tokens":150,"pruned_tokens":150}},
{"rank":2522,"domain":"vecteezy.com","origin":"https://vecteezy.com","robots":{"status":"ok","http":200,"url":"https://www.vecteezy.com/robots.txt","bytes":2676,"groups":40},"bots":{"GPTBot":["partial","*",33,0],"ChatGPT-User":["partial","*",33,0],"OAI-SearchBot":["partial","*",33,0],"ClaudeBot":["partial","*",33,0],"Claude-User":["partial","*",33,0],"Claude-SearchBot":["partial","*",33,0],"anthropic-ai":["partial","*",33,0],"PerplexityBot":["partial","*",33,0],"Perplexity-User":["partial","*",33,0],"Google-Extended":["partial","*",33,0],"Applebot-Extended":["partial","*",33,0],"CCBot":["partial","*",33,0],"Bytespider":["partial","*",33,0],"Amazonbot":["partial","*",33,0],"meta-externalagent":["partial","*",33,0],"cohere-ai":["partial","*",33,0],"Diffbot":["partial","*",33,0],"ContextFetchBot":["partial","*",33,0]},"llms":{"status":"http_error","http":403,"bytes":5505},"home":{"status":"http_error","http":403,"url":"https://vecteezy.com/"}},
{"rank":2523,"domain":"nii.ac.jp","origin":"https://www.nii.ac.jp","robots":{"status":"ok","http":200,"url":"https://www.nii.ac.jp/robots.txt","bytes":64,"groups":1},"bots":{"GPTBot":["partial","*",2,0],"ChatGPT-User":["partial","*",2,0],"OAI-SearchBot":["partial","*",2,0],"ClaudeBot":["partial","*",2,0],"Claude-User":["partial","*",2,0],"Claude-SearchBot":["partial","*",2,0],"anthropic-ai":["partial","*",2,0],"PerplexityBot":["partial","*",2,0],"Perplexity-User":["partial","*",2,0],"Google-Extended":["partial","*",2,0],"Applebot-Extended":["partial","*",2,0],"CCBot":["partial","*",2,0],"Bytespider":["partial","*",2,0],"Amazonbot":["partial","*",2,0],"meta-externalagent":["partial","*",2,0],"cohere-ai":["partial","*",2,0],"Diffbot":["partial","*",2,0],"ContextFetchBot":["partial","*",2,0]},"llms":{"status":"missing","http":404,"bytes":32597},"home":{"status":"ok","http":200,"url":"https://www.nii.ac.jp/","html_tokens":11255,"markdown_tokens":29,"pruned_tokens":29}},
{"rank":2524,"domain":"sacbee.com","origin":"https://sacbee.com","robots":{"status":"unreachable","http":null,"url":"https://sacbee.com/robots.txt","bytes":0,"groups":0,"error":"<StreamReset stream_id:1, error_code:2, remote_reset:True>"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":2525,"domain":"deutschlandfunk.de","origin":"https://deutschlandfunk.de","robots":{"status":"ok","http":200,"url":"https://www.deutschlandfunk.de/robots.txt","bytes":994,"groups":1},"bots":{"GPTBot":["partial","*",2,0],"ChatGPT-User":["partial","*",2,0],"OAI-SearchBot":["partial","*",2,0],"ClaudeBot":["partial","*",2,0],"Claude-User":["partial","*",2,0],"Claude-SearchBot":["partial","*",2,0],"anthropic-ai":["partial","*",2,0],"PerplexityBot":["partial","*",2,0],"Perplexity-User":["partial","*",2,0],"Google-Extended":["partial","*",2,0],"Applebot-Extended":["partial","*",2,0],"CCBot":["partial","*",2,0],"Bytespider":["partial","*",2,0],"Amazonbot":["partial","*",2,0],"meta-externalagent":["partial","*",2,0],"cohere-ai":["partial","*",2,0],"Diffbot":["partial","*",2,0],"ContextFetchBot":["partial","*",2,0]},"llms":{"status":"missing","http":404,"bytes":337804},"home":{"status":"ok","http":200,"url":"https://www.deutschlandfunk.de/","html_tokens":359988,"markdown_tokens":124,"pruned_tokens":121}},
{"rank":2526,"domain":"numbeo.com","origin":"https://numbeo.com","robots":{"status":"ok","http":200,"url":"https://www.numbeo.com/robots.txt","bytes":68,"groups":1},"bots":{"GPTBot":["partial","*",2,0],"ChatGPT-User":["partial","*",2,0],"OAI-SearchBot":["partial","*",2,0],"ClaudeBot":["partial","*",2,0],"Claude-User":["partial","*",2,0],"Claude-SearchBot":["partial","*",2,0],"anthropic-ai":["partial","*",2,0],"PerplexityBot":["partial","*",2,0],"Perplexity-User":["partial","*",2,0],"Google-Extended":["partial","*",2,0],"Applebot-Extended":["partial","*",2,0],"CCBot":["partial","*",2,0],"Bytespider":["partial","*",2,0],"Amazonbot":["partial","*",2,0],"meta-externalagent":["partial","*",2,0],"cohere-ai":["partial","*",2,0],"Diffbot":["partial","*",2,0],"ContextFetchBot":["partial","*",2,0]},"llms":{"status":"missing","http":404,"bytes":4525},"home":{"status":"ok","http":200,"url":"https://www.numbeo.com/cost-of-living/","html_tokens":14564,"markdown_tokens":187,"pruned_tokens":150}},
{"rank":2527,"domain":"canalblog.com","origin":"https://canalblog.com","robots":{"status":"ok","http":200,"url":"https://canalblog.com/robots.txt","bytes":23,"groups":1},"bots":{"GPTBot":["allowed","*",0,1],"ChatGPT-User":["allowed","*",0,1],"OAI-SearchBot":["allowed","*",0,1],"ClaudeBot":["allowed","*",0,1],"Claude-User":["allowed","*",0,1],"Claude-SearchBot":["allowed","*",0,1],"anthropic-ai":["allowed","*",0,1],"PerplexityBot":["allowed","*",0,1],"Perplexity-User":["allowed","*",0,1],"Google-Extended":["allowed","*",0,1],"Applebot-Extended":["allowed","*",0,1],"CCBot":["allowed","*",0,1],"Bytespider":["allowed","*",0,1],"Amazonbot":["allowed","*",0,1],"meta-externalagent":["allowed","*",0,1],"cohere-ai":["allowed","*",0,1],"Diffbot":["allowed","*",0,1],"ContextFetchBot":["allowed","*",0,1]},"llms":{"status":"missing","http":404,"bytes":15622},"home":{"status":"ok","http":200,"url":"https://www.canalblog.com/","html_tokens":47373,"markdown_tokens":200,"pruned_tokens":57}},
{"rank":2528,"domain":"karger.com","origin":"https://karger.com","robots":{"status":"ok","http":200,"url":"https://karger.com/robots.txt","bytes":1179,"groups":2},"bots":{"GPTBot":["partial","*",36,3],"ChatGPT-User":["partial","*",36,3],"OAI-SearchBot":["partial","*",36,3],"ClaudeBot":["partial","*",36,3],"Claude-User":["partial","*",36,3],"Claude-SearchBot":["partial","*",36,3],"anthropic-ai":["partial","*",36,3],"PerplexityBot":["partial","*",36,3],"Perplexity-User":["partial","*",36,3],"Google-Extended":["partial","*",36,3],"Applebot-Extended":["partial","*",36,3],"CCBot":["partial","*",36,3],"Bytespider":["partial","*",36,3],"Amazonbot":["partial","*",36,3],"meta-externalagent":["partial","*",36,3],"cohere-ai":["partial","*",36,3],"Diffbot":["partial","*",36,3],"ContextFetchBot":["partial","*",36,3]},"llms":{"status":"http_error","http":403,"bytes":5477},"home":{"status":"http_error","http":403,"url":"https://karger.com/"}},
{"rank":2529,"domain":"tradedoubler.com","origin":"https://tradedoubler.com","robots":{"status":"ok","http":200,"url":"https://tradedoubler.com/robots.txt","bytes":886,"groups":4},"bots":{"GPTBot":["partial","*",1,1],"ChatGPT-User":["partial","*",1,1],"OAI-SearchBot":["partial","*",1,1],"ClaudeBot":["partial","*",1,1],"Claude-User":["partial","*",1,1],"Claude-SearchBot":["partial","*",1,1],"anthropic-ai":["partial","*",1,1],"PerplexityBot":["partial","*",1,1],"Perplexity-User":["partial","*",1,1],"Google-Extended":["partial","*",1,1],"Applebot-Extended":["partial","*",1,1],"CCBot":["partial","*",1,1],"Bytespider":["partial","*",1,1],"Amazonbot":["partial","*",1,1],"meta-externalagent":["partial","*",1,1],"cohere-ai":["partial","*",1,1],"Diffbot":["partial","*",1,1],"ContextFetchBot":["partial","*",1,1]},"llms":{"status":"present","http":200,"bytes":3201},"home":{"status":"ok","http":200,"url":"https://www.tradedoubler.com/","html_tokens":443352,"markdown_tokens":630,"pruned_tokens":113}},
{"rank":2530,"domain":"alaska.gov","origin":"https://alaska.gov","robots":{"status":"html","http":200,"url":"https://alaska.gov/robots.txt","bytes":245,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":2531,"domain":"insee.fr","origin":"https://insee.fr","robots":{"status":"ok","http":200,"url":"https://www.insee.fr/robots.txt","bytes":163,"groups":1},"bots":{"GPTBot":["partial","*",6,0],"ChatGPT-User":["partial","*",6,0],"OAI-SearchBot":["partial","*",6,0],"ClaudeBot":["partial","*",6,0],"Claude-User":["partial","*",6,0],"Claude-SearchBot":["partial","*",6,0],"anthropic-ai":["partial","*",6,0],"PerplexityBot":["partial","*",6,0],"Perplexity-User":["partial","*",6,0],"Google-Extended":["partial","*",6,0],"Applebot-Extended":["partial","*",6,0],"CCBot":["partial","*",6,0],"Bytespider":["partial","*",6,0],"Amazonbot":["partial","*",6,0],"meta-externalagent":["partial","*",6,0],"cohere-ai":["partial","*",6,0],"Diffbot":["partial","*",6,0],"ContextFetchBot":["partial","*",6,0]},"llms":{"status":"http_error","http":500,"bytes":44145},"home":{"status":"ok","http":200,"url":"https://www.insee.fr/fr/accueil","html_tokens":16329,"markdown_tokens":40,"pruned_tokens":40}},
{"rank":2532,"domain":"lwn.net","origin":"https://lwn.net","robots":{"status":"ok","http":200,"url":"https://lwn.net/robots.txt","bytes":589,"groups":2},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["partial","*",2,0],"OAI-SearchBot":["partial","*",2,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",2,0],"Claude-SearchBot":["partial","*",2,0],"anthropic-ai":["partial","*",2,0],"PerplexityBot":["partial","*",2,0],"Perplexity-User":["partial","*",2,0],"Google-Extended":["partial","*",2,0],"Applebot-Extended":["partial","*",2,0],"CCBot":["partial","*",2,0],"Bytespider":["partial","*",2,0],"Amazonbot":["partial","*",2,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["partial","*",2,0],"Diffbot":["partial","*",2,0],"ContextFetchBot":["partial","*",2,0]},"llms":{"status":"missing","http":404,"bytes":3898},"home":{"status":"ok","http":200,"url":"https://lwn.net/","html_tokens":7573,"markdown_tokens":3950,"pruned_tokens":210}},
{"rank":2533,"domain":"redfin.com","origin":"https://redfin.com","robots":{"status":"ok","http":200,"url":"https://www.redfin.com/robots.txt","bytes":10605,"groups":12},"bots":{"GPTBot":["partial","*",68,6],"ChatGPT-User":["partial","*",68,6],"OAI-SearchBot":["partial","*",68,6],"ClaudeBot":["partial","*",68,6],"Claude-User":["partial","*",68,6],"Claude-SearchBot":["partial","*",68,6],"anthropic-ai":["partial","*",68,6],"PerplexityBot":["partial","*",68,6],"Perplexity-User":["partial","*",68,6],"Google-Extended":["partial","*",68,6],"Applebot-Extended":["partial","*",68,6],"CCBot":["partial","*",68,6],"Bytespider":["partial","*",68,6],"Amazonbot":["partial","*",68,6],"meta-externalagent":["partial","*",68,6],"cohere-ai":["partial","*",68,6],"Diffbot":["partial","*",68,6],"ContextFetchBot":["partial","*",68,6]},"llms":{"status":"missing","http":404,"bytes":24758},"home":{"status":"ok","http":200,"url":"https://www.redfin.com/","html_tokens":71101,"markdown_tokens":702,"pruned_tokens":257}},
{"rank":2534,"domain":"blurb.com","origin":"https://blurb.com","robots":{"status":"ok","http":200,"url":"https://www.blurb.com/robots.txt","bytes":5395,"groups":36},"bots":{"GPTBot":["partial","*",28,0],"ChatGPT-User":["partial","*",28,0],"OAI-SearchBot":["partial","*",28,0],"ClaudeBot":["partial","*",28,0],"Claude-User":["partial","*",28,0],"Claude-SearchBot":["partial","*",28,0],"anthropic-ai":["partial","*",28,0],"PerplexityBot":["partial","*",28,0],"Perplexity-User":["partial","*",28,0],"Google-Extended":["partial","*",28,0],"Applebot-Extended":["partial","*",28,0],"CCBot":["partial","*",28,0],"Bytespider":["partial","*",28,0],"Amazonbot":["partial","*",28,0],"meta-externalagent":["partial","*",28,0],"cohere-ai":["partial","*",28,0],"Diffbot":["partial","*",28,0],"ContextFetchBot":["partial","*",28,0]},"llms":{"status":"missing","http":404,"bytes":44450},"home":{"status":"ok","http":200,"url":"https://www.blurb.com/","html_tokens":51676,"markdown_tokens":2166,"pruned_tokens":469}},
{"rank":2535,"domain":"ge.com","origin":"https://www.ge.com","robots":{"status":"ok","http":200,"url":"https://www.ge.com/robots.txt","bytes":2027,"groups":1},"bots":{"GPTBot":["partial","*",34,18],"ChatGPT-User":["partial","*",34,18],"OAI-SearchBot":["partial","*",34,18],"ClaudeBot":["partial","*",34,18],"Claude-User":["partial","*",34,18],"Claude-SearchBot":["partial","*",34,18],"anthropic-ai":["partial","*",34,18],"PerplexityBot":["partial","*",34,18],"Perplexity-User":["partial","*",34,18],"Google-Extended":["partial","*",34,18],"Applebot-Extended":["partial","*",34,18],"CCBot":["partial","*",34,18],"Bytespider":["partial","*",34,18],"Amazonbot":["partial","*",34,18],"meta-externalagent":["partial","*",34,18],"cohere-ai":["partial","*",34,18],"Diffbot":["partial","*",34,18],"ContextFetchBot":["partial","*",34,18]},"llms":{"status":"missing","http":404,"bytes":545},"home":{"status":"ok","http":200,"url":"https://www.ge.com/","html_tokens":17098,"markdown_tokens":147,"pruned_tokens":16}},
{"rank":2536,"domain":"quizlet.com","origin":"https://quizlet.com","robots":{"status":"refused","http":403,"url":"https://quizlet.com/robots.txt","bytes":160451,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":2537,"domain":"nordvpn.com","origin":"https://nordvpn.com","robots":{"status":"ok","http":200,"url":"https://nordvpn.com/robots.txt","bytes":15951,"groups":6},"bots":{"GPTBot":["partial","*",4,0],"ChatGPT-User":["partial","*",4,0],"OAI-SearchBot":["allowed","named",0,1],"ClaudeBot":["partial","*",4,0],"Claude-User":["partial","*",4,0],"Claude-SearchBot":["partial","*",4,0],"anthropic-ai":["partial","*",4,0],"PerplexityBot":["partial","*",4,0],"Perplexity-User":["partial","*",4,0],"Google-Extended":["partial","*",4,0],"Applebot-Extended":["partial","*",4,0],"CCBot":["partial","*",4,0],"Bytespider":["partial","*",4,0],"Amazonbot":["partial","*",4,0],"meta-externalagent":["partial","*",4,0],"cohere-ai":["partial","*",4,0],"Diffbot":["partial","*",4,0],"ContextFetchBot":["partial","*",4,0]},"llms":{"status":"present","http":200,"bytes":2415},"home":{"status":"http_error","http":403,"url":"https://nordvpn.com/"}},
{"rank":2538,"domain":"rapidshare.com","origin":"https://rapidshare.com","robots":{"status":"unreachable","http":null,"url":"https://rapidshare.com/robots.txt","bytes":0,"groups":0,"error":"rapidshare.com is not a public address"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":2539,"domain":"zara.com","origin":"https://zara.com","robots":{"status":"refused","http":403,"url":"https://zara.com/robots.txt","bytes":372,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":2540,"domain":"copernicus.org","origin":"https://copernicus.org","robots":{"status":"ok","http":200,"url":"https://copernicus.org/robots.txt","bytes":44,"groups":1},"bots":{"GPTBot":["partial","*",1,0],"ChatGPT-User":["partial","*",1,0],"OAI-SearchBot":["partial","*",1,0],"ClaudeBot":["partial","*",1,0],"Claude-User":["partial","*",1,0],"Claude-SearchBot":["partial","*",1,0],"anthropic-ai":["partial","*",1,0],"PerplexityBot":["partial","*",1,0],"Perplexity-User":["partial","*",1,0],"Google-Extended":["partial","*",1,0],"Applebot-Extended":["partial","*",1,0],"CCBot":["partial","*",1,0],"Bytespider":["partial","*",1,0],"Amazonbot":["partial","*",1,0],"meta-externalagent":["partial","*",1,0],"cohere-ai":["partial","*",1,0],"Diffbot":["partial","*",1,0],"ContextFetchBot":["partial","*",1,0]},"llms":{"status":"missing","http":404,"bytes":30988},"home":{"status":"ok","http":200,"url":"https://copernicus.org/","html_tokens":10344,"markdown_tokens":415,"pruned_tokens":107}},
{"rank":2541,"domain":"martinfowler.com","origin":"https://martinfowler.com","robots":{"status":"ok","http":200,"url":"https://martinfowler.com/robots.txt","bytes":23,"groups":1},"bots":{"GPTBot":["allowed","*",0,1],"ChatGPT-User":["allowed","*",0,1],"OAI-SearchBot":["allowed","*",0,1],"ClaudeBot":["allowed","*",0,1],"Claude-User":["allowed","*",0,1],"Claude-SearchBot":["allowed","*",0,1],"anthropic-ai":["allowed","*",0,1],"PerplexityBot":["allowed","*",0,1],"Perplexity-User":["allowed","*",0,1],"Google-Extended":["allowed","*",0,1],"Applebot-Extended":["allowed","*",0,1],"CCBot":["allowed","*",0,1],"Bytespider":["allowed","*",0,1],"Amazonbot":["allowed","*",0,1],"meta-externalagent":["allowed","*",0,1],"cohere-ai":["allowed","*",0,1],"Diffbot":["allowed","*",0,1],"ContextFetchBot":["allowed","*",0,1]},"llms":{"status":"missing","http":404,"bytes":10978},"home":{"status":"ok","http":200,"url":"https://martinfowler.com/","html_tokens":9064,"markdown_tokens":1448,"pruned_tokens":167}},
{"rank":2542,"domain":"courtlistener.com","origin":"https://courtlistener.com","robots":{"status":"refused","http":403,"url":"https://courtlistener.com/robots.txt","bytes":919,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":2543,"domain":"heritage.org","origin":"https://heritage.org","robots":{"status":"ok","http":200,"url":"https://www.heritage.org/robots.txt","bytes":2027,"groups":1},"bots":{"GPTBot":["partial","*",34,18],"ChatGPT-User":["partial","*",34,18],"OAI-SearchBot":["partial","*",34,18],"ClaudeBot":["partial","*",34,18],"Claude-User":["partial","*",34,18],"Claude-SearchBot":["partial","*",34,18],"anthropic-ai":["partial","*",34,18],"PerplexityBot":["partial","*",34,18],"Perplexity-User":["partial","*",34,18],"Google-Extended":["partial","*",34,18],"Applebot-Extended":["partial","*",34,18],"CCBot":["partial","*",34,18],"Bytespider":["partial","*",34,18],"Amazonbot":["partial","*",34,18],"meta-externalagent":["partial","*",34,18],"cohere-ai":["partial","*",34,18],"Diffbot":["partial","*",34,18],"ContextFetchBot":["partial","*",34,18]},"llms":{"status":"missing","http":404,"bytes":7301},"home":{"status":"http_error","http":403,"url":"https://heritage.org/"}},
{"rank":2544,"domain":"womenshealthmag.com","origin":"https://womenshealthmag.com","robots":{"status":"ok","http":200,"url":"https://womenshealthmag.com/robots.txt","bytes":960,"groups":3},"bots":{"GPTBot":["partial","named",31,0],"ChatGPT-User":["partial","named",31,0],"OAI-SearchBot":["partial","named",31,0],"ClaudeBot":["partial","*",19,0],"Claude-User":["partial","*",19,0],"Claude-SearchBot":["partial","*",19,0],"anthropic-ai":["partial","*",19,0],"PerplexityBot":["partial","*",19,0],"Perplexity-User":["partial","*",19,0],"Google-Extended":["partial","*",19,0],"Applebot-Extended":["partial","*",19,0],"CCBot":["partial","*",19,0],"Bytespider":["partial","*",19,0],"Amazonbot":["partial","*",19,0],"meta-externalagent":["partial","*",19,0],"cohere-ai":["partial","*",19,0],"Diffbot":["partial","*",19,0],"ContextFetchBot":["partial","*",19,0]},"llms":{"status":"missing","http":404,"bytes":84},"home":{"status":"ok","http":200,"url":"https://www.womenshealthmag.com/","html_tokens":240872,"markdown_tokens":230,"pruned_tokens":65}},
{"rank":2545,"domain":"cntraveler.com","origin":"https://cntraveler.com","robots":{"status":"ok","http":200,"url":"https://www.cntraveler.com/robots.txt","bytes":2285,"groups":3},"bots":{"GPTBot":["partial","*",15,15],"ChatGPT-User":["partial","*",15,15],"OAI-SearchBot":["partial","*",15,15],"ClaudeBot":["blocked","named",1,0],"Claude-User":["blocked","named",1,0],"Claude-SearchBot":["blocked","named",1,0],"anthropic-ai":["partial","*",15,15],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["blocked","named",1,0],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",15,15]},"llms":{"status":"missing","http":404,"bytes":911777},"home":{"status":"ok","http":200,"url":"https://www.cntraveler.com/","html_tokens":488946,"markdown_tokens":164,"pruned_tokens":164}},
{"rank":2546,"domain":"thebalance.com","origin":"https://thebalance.com","robots":{"status":"ok","http":200,"url":"https://www.thebalancemoney.com/robots.txt","bytes":3483,"groups":6},"bots":{"GPTBot":["partial","named",1,0],"ChatGPT-User":["partial","named",1,0],"OAI-SearchBot":["partial","named",1,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["blocked","named",1,0],"Claude-SearchBot":["partial","named",1,2],"anthropic-ai":["partial","named",1,2],"PerplexityBot":["partial","named",1,2],"Perplexity-User":["blocked","named",1,0],"Google-Extended":["partial","named",1,2],"Applebot-Extended":["blocked","named",1,0],"CCBot":["partial","named",1,2],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["partial","named",1,2],"cohere-ai":["partial","named",1,2],"Diffbot":["partial","*",2,0],"ContextFetchBot":["partial","*",2,0]},"llms":{"status":"http_error","http":402,"bytes":612},"home":{"status":"http_error","http":402,"url":"https://www.thebalancemoney.com/"}},
{"rank":2547,"domain":"gog.com","origin":"https://gog.com","robots":{"status":"ok","http":200,"url":"https://www.gog.com/robots.txt","bytes":152,"groups":1},"bots":{"GPTBot":["partial","*",6,0],"ChatGPT-User":["partial","*",6,0],"OAI-SearchBot":["partial","*",6,0],"ClaudeBot":["partial","*",6,0],"Claude-User":["partial","*",6,0],"Claude-SearchBot":["partial","*",6,0],"anthropic-ai":["partial","*",6,0],"PerplexityBot":["partial","*",6,0],"Perplexity-User":["partial","*",6,0],"Google-Extended":["partial","*",6,0],"Applebot-Extended":["partial","*",6,0],"CCBot":["partial","*",6,0],"Bytespider":["partial","*",6,0],"Amazonbot":["partial","*",6,0],"meta-externalagent":["partial","*",6,0],"cohere-ai":["partial","*",6,0],"Diffbot":["partial","*",6,0],"ContextFetchBot":["partial","*",6,0]},"llms":{"status":"missing","http":404,"bytes":153},"home":{"status":"ok","http":200,"url":"https://www.gog.com/en/","html_tokens":383669,"markdown_tokens":26,"pruned_tokens":9}},
{"rank":2548,"domain":"jooble.org","origin":"https://jooble.org","robots":{"status":"ok","http":200,"url":"https://jooble.org/robots.txt","bytes":1648,"groups":2},"bots":{"GPTBot":["partial","*",53,0],"ChatGPT-User":["partial","*",53,0],"OAI-SearchBot":["partial","*",53,0],"ClaudeBot":["partial","*",53,0],"Claude-User":["partial","*",53,0],"Claude-SearchBot":["partial","*",53,0],"anthropic-ai":["partial","*",53,0],"PerplexityBot":["partial","*",53,0],"Perplexity-User":["partial","*",53,0],"Google-Extended":["partial","*",53,0],"Applebot-Extended":["partial","*",53,0],"CCBot":["partial","*",53,0],"Bytespider":["partial","*",53,0],"Amazonbot":["partial","*",53,0],"meta-externalagent":["partial","*",53,0],"cohere-ai":["partial","*",53,0],"Diffbot":["partial","*",53,0],"ContextFetchBot":["partial","*",53,0]},"llms":{"status":"present","http":200,"bytes":68400},"home":{"status":"http_error","http":403,"url":"https://jooble.org/"}},
{"rank":2549,"domain":"europapress.es","origin":"https://europapress.es","robots":{"status":"ok","http":200,"url":"https://www.europapress.es/robots.txt","bytes":1014,"groups":11},"bots":{"GPTBot":["partial","*",10,0],"ChatGPT-User":["partial","*",10,0],"OAI-SearchBot":["partial","*",10,0],"ClaudeBot":["partial","*",10,0],"Claude-User":["partial","*",10,0],"Claude-SearchBot":["partial","*",10,0],"anthropic-ai":["partial","*",10,0],"PerplexityBot":["partial","*",10,0],"Perplexity-User":["partial","*",10,0],"Google-Extended":["partial","*",10,0],"Applebot-Extended":["partial","*",10,0],"CCBot":["partial","*",10,0],"Bytespider":["partial","*",10,0],"Amazonbot":["partial","*",10,0],"meta-externalagent":["partial","*",10,0],"cohere-ai":["partial","*",10,0],"Diffbot":["partial","*",10,0],"ContextFetchBot":["partial","*",10,0]},"llms":{"status":"missing","http":404,"bytes":63897},"home":{"status":"ok","http":200,"url":"https://www.europapress.es/","html_tokens":157436,"markdown_tokens":1027,"pruned_tokens":175}},
{"rank":2550,"domain":"al.com","origin":"https://al.com","robots":{"status":"ok","http":200,"url":"https://www.al.com/robots.txt","bytes":2857,"groups":4},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["partial","named",1,14],"OAI-SearchBot":["partial","named",1,14],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","named",1,14],"Claude-SearchBot":["partial","named",1,14],"anthropic-ai":["partial","*",30,3],"PerplexityBot":["partial","named",1,14],"Perplexity-User":["partial","named",1,14],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["partial","*",30,3],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",30,3]},"llms":{"status":"missing","http":404,"bytes":258871},"home":{"status":"http_error","http":403,"url":"https://www.al.com/"}},
{"rank":2551,"domain":"toasttab.com","origin":"https://toasttab.com","robots":{"status":"ok","http":200,"url":"https://www.toasttab.com/robots.txt","bytes":770,"groups":3},"bots":{"GPTBot":["partial","*",26,0],"ChatGPT-User":["partial","*",26,0],"OAI-SearchBot":["partial","*",26,0],"ClaudeBot":["partial","*",26,0],"Claude-User":["partial","*",26,0],"Claude-SearchBot":["partial","*",26,0],"anthropic-ai":["partial","*",26,0],"PerplexityBot":["partial","*",26,0],"Perplexity-User":["partial","*",26,0],"Google-Extended":["partial","*",26,0],"Applebot-Extended":["partial","*",26,0],"CCBot":["partial","*",26,0],"Bytespider":["partial","*",26,0],"Amazonbot":["partial","*",26,0],"meta-externalagent":["partial","*",26,0],"cohere-ai":["partial","*",26,0],"Diffbot":["partial","*",26,0],"ContextFetchBot":["partial","*",26,0]},"llms":{"status":"disallowed","error":"robots.txt of pos.toasttab.com not readable (refused)"},"home":{"status":"disallowed","error":"robots.txt of pos.toasttab.com not readable (refused)"}},
{"rank":2552,"domain":"shutterfly.com","origin":"https://shutterfly.com","robots":{"status":"ok","http":200,"url":"https://www.shutterfly.com/robots.txt","bytes":1914,"groups":6},"bots":{"GPTBot":["partial","*",28,0],"ChatGPT-User":["partial","*",28,0],"OAI-SearchBot":["partial","*",28,0],"ClaudeBot":["partial","*",28,0],"Claude-User":["partial","*",28,0],"Claude-SearchBot":["partial","*",28,0],"anthropic-ai":["partial","*",28,0],"PerplexityBot":["partial","*",28,0],"Perplexity-User":["partial","*",28,0],"Google-Extended":["partial","*",28,0],"Applebot-Extended":["partial","*",28,0],"CCBot":["partial","*",28,0],"Bytespider":["partial","*",28,0],"Amazonbot":["partial","*",28,0],"meta-externalagent":["partial","*",28,0],"cohere-ai":["partial","*",28,0],"Diffbot":["partial","*",28,0],"ContextFetchBot":["partial","*",28,0]},"llms":{"status":"timeout","error":"timed out after 10 s"},"home":{"status":"ok","http":200,"url":"https://www.shutterfly.com/","html_tokens":327486,"markdown_tokens":67,"pruned_tokens":67}},
{"rank":2553,"domain":"mongabay.com","origin":"https://mongabay.com","robots":{"status":"ok","http":200,"url":"https://mongabay.com/robots.txt","bytes":229,"groups":1},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"http_error","http":403,"bytes":6360},"home":{"status":"http_error","http":403,"url":"https://mongabay.com/"}},
{"rank":2554,"domain":"iyfubh.com","origin":"http://iyfubh.com","robots":{"status":"ok","http":200,"url":"http://iyfubh.com/robots.txt","bytes":292,"groups":6},"bots":{"GPTBot":["blocked","*",6,0],"ChatGPT-User":["blocked","*",6,0],"OAI-SearchBot":["blocked","*",6,0],"ClaudeBot":["blocked","*",6,0],"Claude-User":["blocked","*",6,0],"Claude-SearchBot":["blocked","*",6,0],"anthropic-ai":["blocked","*",6,0],"PerplexityBot":["blocked","*",6,0],"Perplexity-User":["blocked","*",6,0],"Google-Extended":["blocked","*",6,0],"Applebot-Extended":["blocked","*",6,0],"CCBot":["blocked","*",6,0],"Bytespider":["blocked","*",6,0],"Amazonbot":["blocked","*",6,0],"meta-externalagent":["blocked","*",6,0],"cohere-ai":["blocked","*",6,0],"Diffbot":["blocked","*",6,0],"ContextFetchBot":["blocked","*",6,0]},"llms":{"status":"disallowed"},"home":{"status":"disallowed"}},
{"rank":2555,"domain":"wp.pl","origin":"https://wp.pl","robots":{"status":"ok","http":200,"url":"https://www.wp.pl/robots.txt","bytes":640,"groups":7},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["allowed","named",0,1],"OAI-SearchBot":["allowed","named",0,1],"ClaudeBot":["partial","*",15,0],"Claude-User":["partial","*",15,0],"Claude-SearchBot":["partial","*",15,0],"anthropic-ai":["partial","*",15,0],"PerplexityBot":["allowed","named",0,1],"Perplexity-User":["allowed","named",0,1],"Google-Extended":["partial","*",15,0],"Applebot-Extended":["partial","*",15,0],"CCBot":["blocked","named",1,0],"Bytespider":["partial","*",15,0],"Amazonbot":["partial","*",15,0],"meta-externalagent":["partial","*",15,0],"cohere-ai":["partial","*",15,0],"Diffbot":["partial","*",15,0],"ContextFetchBot":["partial","*",15,0]},"llms":{"status":"html","http":200,"bytes":427539},"home":{"status":"ok","http":200,"url":"https://www.wp.pl/","html_tokens":106338,"markdown_tokens":277,"pruned_tokens":174}},
{"rank":2556,"domain":"ihk.de","origin":"https://ihk.de","robots":{"status":"html","http":200,"url":"https://www.ihk.de/robots.txt","bytes":269,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":2557,"domain":"marxists.org","origin":"https://marxists.org","robots":{"status":"ok","http":200,"url":"https://www.marxists.org/robots.txt","bytes":777,"groups":2},"bots":{"GPTBot":["partial","*",14,0],"ChatGPT-User":["partial","*",14,0],"OAI-SearchBot":["partial","*",14,0],"ClaudeBot":["partial","*",14,0],"Claude-User":["partial","*",14,0],"Claude-SearchBot":["partial","*",14,0],"anthropic-ai":["partial","*",14,0],"PerplexityBot":["partial","*",14,0],"Perplexity-User":["partial","*",14,0],"Google-Extended":["partial","*",14,0],"Applebot-Extended":["partial","*",14,0],"CCBot":["partial","*",14,0],"Bytespider":["partial","*",14,0],"Amazonbot":["partial","*",14,0],"meta-externalagent":["partial","*",14,0],"cohere-ai":["partial","*",14,0],"Diffbot":["partial","*",14,0],"ContextFetchBot":["partial","*",14,0]},"llms":{"status":"missing","http":404,"bytes":1989},"home":{"status":"ok","http":200,"url":"https://www.marxists.org/","html_tokens":1338,"markdown_tokens":0,"pruned_tokens":0}},
{"rank":2558,"domain":"red-bean.com","origin":"https://red-bean.com","robots":{"status":"ok","http":200,"url":"https://red-bean.com/robots.txt","bytes":191,"groups":2},"bots":{"GPTBot":["partial","*",4,0],"ChatGPT-User":["partial","*",4,0],"OAI-SearchBot":["partial","*",4,0],"ClaudeBot":["partial","*",4,0],"Claude-User":["partial","*",4,0],"Claude-SearchBot":["partial","*",4,0],"anthropic-ai":["partial","*",4,0],"PerplexityBot":["partial","*",4,0],"Perplexity-User":["partial","*",4,0],"Google-Extended":["partial","*",4,0],"Applebot-Extended":["partial","*",4,0],"CCBot":["partial","*",4,0],"Bytespider":["partial","*",4,0],"Amazonbot":["partial","*",4,0],"meta-externalagent":["partial","*",4,0],"cohere-ai":["partial","*",4,0],"Diffbot":["partial","*",4,0],"ContextFetchBot":["partial","*",4,0]},"llms":{"status":"missing","http":404,"bytes":315},"home":{"status":"ok","http":200,"url":"https://red-bean.com/","html_tokens":837,"markdown_tokens":352,"pruned_tokens":59}},
{"rank":2559,"domain":"pearltrees.com","origin":"https://pearltrees.com","robots":{"status":"ok","http":200,"url":"https://pearltrees.com/robots.txt","bytes":76,"groups":1},"bots":{"GPTBot":["partial","*",1,0],"ChatGPT-User":["partial","*",1,0],"OAI-SearchBot":["partial","*",1,0],"ClaudeBot":["partial","*",1,0],"Claude-User":["partial","*",1,0],"Claude-SearchBot":["partial","*",1,0],"anthropic-ai":["partial","*",1,0],"PerplexityBot":["partial","*",1,0],"Perplexity-User":["partial","*",1,0],"Google-Extended":["partial","*",1,0],"Applebot-Extended":["partial","*",1,0],"CCBot":["partial","*",1,0],"Bytespider":["partial","*",1,0],"Amazonbot":["partial","*",1,0],"meta-externalagent":["partial","*",1,0],"cohere-ai":["partial","*",1,0],"Diffbot":["partial","*",1,0],"ContextFetchBot":["partial","*",1,0]},"llms":{"status":"html","http":200,"bytes":1174},"home":{"status":"ok","http":200,"url":"https://www.pearltrees.com/","html_tokens":12658,"markdown_tokens":5,"pruned_tokens":5}},
{"rank":2560,"domain":"rs-online.com","origin":"https://rs-online.com","robots":{"status":"html","http":200,"url":"https://www.rs-online.com/","bytes":124583,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":2561,"domain":"uschamber.com","origin":"https://uschamber.com","robots":{"status":"ok","http":200,"url":"https://www.uschamber.com/robots.txt","bytes":580,"groups":1},"bots":{"GPTBot":["partial","*",11,0],"ChatGPT-User":["partial","*",11,0],"OAI-SearchBot":["partial","*",11,0],"ClaudeBot":["partial","*",11,0],"Claude-User":["partial","*",11,0],"Claude-SearchBot":["partial","*",11,0],"anthropic-ai":["partial","*",11,0],"PerplexityBot":["partial","*",11,0],"Perplexity-User":["partial","*",11,0],"Google-Extended":["partial","*",11,0],"Applebot-Extended":["partial","*",11,0],"CCBot":["partial","*",11,0],"Bytespider":["partial","*",11,0],"Amazonbot":["partial","*",11,0],"meta-externalagent":["partial","*",11,0],"cohere-ai":["partial","*",11,0],"Diffbot":["partial","*",11,0],"ContextFetchBot":["partial","*",11,0]},"llms":{"status":"missing","http":404,"bytes":79451},"home":{"status":"ok","http":200,"url":"https://www.uschamber.com/","html_tokens":145183,"markdown_tokens":976,"pruned_tokens":127}},
{"rank":2562,"domain":"dc.gov","origin":"https://dc.gov","robots":{"status":"ok","http":200,"url":"https://dc.gov/robots.txt","bytes":2027,"groups":1},"bots":{"GPTBot":["partial","*",34,18],"ChatGPT-User":["partial","*",34,18],"OAI-SearchBot":["partial","*",34,18],"ClaudeBot":["partial","*",34,18],"Claude-User":["partial","*",34,18],"Claude-SearchBot":["partial","*",34,18],"anthropic-ai":["partial","*",34,18],"PerplexityBot":["partial","*",34,18],"Perplexity-User":["partial","*",34,18],"Google-Extended":["partial","*",34,18],"Applebot-Extended":["partial","*",34,18],"CCBot":["partial","*",34,18],"Bytespider":["partial","*",34,18],"Amazonbot":["partial","*",34,18],"meta-externalagent":["partial","*",34,18],"cohere-ai":["partial","*",34,18],"Diffbot":["partial","*",34,18],"ContextFetchBot":["partial","*",34,18]},"llms":{"status":"missing","http":404,"bytes":1158},"home":{"status":"ok","http":200,"url":"https://dc.gov/","html_tokens":23668,"markdown_tokens":101,"pruned_tokens":71}},
{"rank":2563,"domain":"wondershare.com","origin":"https://wondershare.com","robots":{"status":"ok","http":200,"url":"https://www.wondershare.com/robots.txt","bytes":97,"groups":1},"bots":{"GPTBot":["partial","*",4,0],"ChatGPT-User":["partial","*",4,0],"OAI-SearchBot":["partial","*",4,0],"ClaudeBot":["partial","*",4,0],"Claude-User":["partial","*",4,0],"Claude-SearchBot":["partial","*",4,0],"anthropic-ai":["partial","*",4,0],"PerplexityBot":["partial","*",4,0],"Perplexity-User":["partial","*",4,0],"Google-Extended":["partial","*",4,0],"Applebot-Extended":["partial","*",4,0],"CCBot":["partial","*",4,0],"Bytespider":["partial","*",4,0],"Amazonbot":["partial","*",4,0],"meta-externalagent":["partial","*",4,0],"cohere-ai":["partial","*",4,0],"Diffbot":["partial","*",4,0],"ContextFetchBot":["partial","*",4,0]},"llms":{"status":"missing","http":404,"bytes":138086},"home":{"status":"ok","http":200,"url":"https://www.wondershare.com/","html_tokens":62672,"markdown_tokens":219,"pruned_tokens":25}},
{"rank":2564,"domain":"vu.nl","origin":"https://vu.nl","robots":{"status":"ok","http":200,"url":"https://vu.nl/robots.txt","bytes":110,"groups":1},"bots":{"GPTBot":["partial","*",2,0],"ChatGPT-User":["partial","*",2,0],"OAI-SearchBot":["partial","*",2,0],"ClaudeBot":["partial","*",2,0],"Claude-User":["partial","*",2,0],"Claude-SearchBot":["partial","*",2,0],"anthropic-ai":["partial","*",2,0],"PerplexityBot":["partial","*",2,0],"Perplexity-User":["partial","*",2,0],"Google-Extended":["partial","*",2,0],"Applebot-Extended":["partial","*",2,0],"CCBot":["partial","*",2,0],"Bytespider":["partial","*",2,0],"Amazonbot":["partial","*",2,0],"meta-externalagent":["partial","*",2,0],"cohere-ai":["partial","*",2,0],"Diffbot":["partial","*",2,0],"ContextFetchBot":["partial","*",2,0]},"llms":{"status":"missing","http":404,"bytes":37928},"home":{"status":"ok","http":200,"url":"https://vu.nl/nl","html_tokens":12541,"markdown_tokens":63,"pruned_tokens":63}},
{"rank":2565,"domain":"or.us","origin":"https://or.us","robots":{"status":"unreachable","http":null,"url":"https://or.us/robots.txt","bytes":0,"groups":0,"error":"cannot resolve or.us"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":2566,"domain":"emb-japan.go.jp","origin":"https://emb-japan.go.jp","robots":{"status":"unreachable","http":null,"url":"https://emb-japan.go.jp/robots.txt","bytes":0,"groups":0,"error":"cannot resolve emb-japan.go.jp"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":2567,"domain":"detroitnews.com","origin":"https://detroitnews.com","robots":{"status":"ok","http":200,"url":"https://www.detroitnews.com/robots.txt","bytes":12671,"groups":288},"bots":{"GPTBot":["partial","named",1,4],"ChatGPT-User":["partial","named",1,4],"OAI-SearchBot":["partial","named",1,4],"ClaudeBot":["blocked","named",1,0],"Claude-User":["blocked","named",1,0],"Claude-SearchBot":["blocked","named",1,0],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["partial","named",1,4],"Perplexity-User":["blocked","named",1,0],"Google-Extended":["partial","named",1,4],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",2,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",32,0]},"llms":{"status":"http_error","http":402,"bytes":254},"home":{"status":"http_error","http":402,"url":"https://www.detroitnews.com/"}},
{"rank":2568,"domain":"morganstanley.com","origin":"https://morganstanley.com","robots":{"status":"unreachable","http":null,"url":"https://morganstanley.com/robots.txt","bytes":0,"groups":0,"error":"<StreamReset stream_id:1, error_code:2, remote_reset:True>"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":2569,"domain":"zblogcn.com","origin":"https://www.zblogcn.com","robots":{"status":"ok","http":200,"url":"https://www.zblogcn.com/robots.txt","bytes":42,"groups":1},"bots":{"GPTBot":["partial","*",1,1],"ChatGPT-User":["partial","*",1,1],"OAI-SearchBot":["partial","*",1,1],"ClaudeBot":["partial","*",1,1],"Claude-User":["partial","*",1,1],"Claude-SearchBot":["partial","*",1,1],"anthropic-ai":["partial","*",1,1],"PerplexityBot":["partial","*",1,1],"Perplexity-User":["partial","*",1,1],"Google-Extended":["partial","*",1,1],"Applebot-Extended":["partial","*",1,1],"CCBot":["partial","*",1,1],"Bytespider":["partial","*",1,1],"Amazonbot":["partial","*",1,1],"meta-externalagent":["partial","*",1,1],"cohere-ai":["partial","*",1,1],"Diffbot":["partial","*",1,1],"ContextFetchBot":["partial","*",1,1]},"llms":{"status":"missing","http":404,"bytes":4939},"home":{"status":"ok","http":200,"url":"https://www.zblogcn.com/","html_tokens":6652,"markdown_tokens":16,"pruned_tokens":16}},
{"rank":2570,"domain":"cgtn.com","origin":"https://cgtn.com","robots":{"status":"ok","http":200,"url":"https://www.cgtn.com/robots.txt","bytes":1328,"groups":1},"bots":{"GPTBot":["partial","*",3,0],"ChatGPT-User":["partial","*",3,0],"OAI-SearchBot":["partial","*",3,0],"ClaudeBot":["partial","*",3,0],"Claude-User":["partial","*",3,0],"Claude-SearchBot":["partial","*",3,0],"anthropic-ai":["partial","*",3,0],"PerplexityBot":["partial","*",3,0],"Perplexity-User":["partial","*",3,0],"Google-Extended":["partial","*",3,0],"Applebot-Extended":["partial","*",3,0],"CCBot":["partial","*",3,0],"Bytespider":["partial","*",3,0],"Amazonbot":["partial","*",3,0],"meta-externalagent":["partial","*",3,0],"cohere-ai":["partial","*",3,0],"Diffbot":["partial","*",3,0],"ContextFetchBot":["partial","*",3,0]},"llms":{"status":"missing","http":404,"bytes":116626},"home":{"status":"ok","http":200,"url":"https://www.cgtn.com/","html_tokens":100581,"markdown_tokens":41,"pruned_tokens":41}},
{"rank":2571,"domain":"auckland.ac.nz","origin":"https://auckland.ac.nz","robots":{"status":"ok","http":200,"url":"https://www.auckland.ac.nz/robots.txt","bytes":76,"groups":1},"bots":{"GPTBot":["allowed","*",0,1],"ChatGPT-User":["allowed","*",0,1],"OAI-SearchBot":["allowed","*",0,1],"ClaudeBot":["allowed","*",0,1],"Claude-User":["allowed","*",0,1],"Claude-SearchBot":["allowed","*",0,1],"anthropic-ai":["allowed","*",0,1],"PerplexityBot":["allowed","*",0,1],"Perplexity-User":["allowed","*",0,1],"Google-Extended":["allowed","*",0,1],"Applebot-Extended":["allowed","*",0,1],"CCBot":["allowed","*",0,1],"Bytespider":["allowed","*",0,1],"Amazonbot":["allowed","*",0,1],"meta-externalagent":["allowed","*",0,1],"cohere-ai":["allowed","*",0,1],"Diffbot":["allowed","*",0,1],"ContextFetchBot":["allowed","*",0,1]},"llms":{"status":"missing","http":404,"bytes":177444},"home":{"status":"ok","http":200,"url":"https://www.auckland.ac.nz/en.html","html_tokens":51247,"markdown_tokens":52,"pruned_tokens":52}},
{"rank":2572,"domain":"dzone.com","origin":"https://dzone.com","robots":{"status":"ok","http":200,"url":"https://dzone.com/robots.txt","bytes":280,"groups":2},"bots":{"GPTBot":["partial","*",10,0],"ChatGPT-User":["partial","*",10,0],"OAI-SearchBot":["partial","*",10,0],"ClaudeBot":["partial","*",10,0],"Claude-User":["partial","*",10,0],"Claude-SearchBot":["partial","*",10,0],"anthropic-ai":["partial","*",10,0],"PerplexityBot":["partial","*",10,0],"Perplexity-User":["partial","*",10,0],"Google-Extended":["partial","*",10,0],"Applebot-Extended":["partial","*",10,0],"CCBot":["partial","*",10,0],"Bytespider":["partial","*",10,0],"Amazonbot":["partial","*",10,0],"meta-externalagent":["partial","*",10,0],"cohere-ai":["partial","*",10,0],"Diffbot":["partial","*",10,0],"ContextFetchBot":["partial","*",10,0]},"llms":{"status":"missing","http":404,"bytes":78572},"home":{"status":"http_error","http":403,"url":"https://dzone.com/"}},
{"rank":2573,"domain":"realsimple.com","origin":"https://realsimple.com","robots":{"status":"ok","http":200,"url":"https://www.realsimple.com/robots.txt","bytes":3473,"groups":6},"bots":{"GPTBot":["partial","named",1,0],"ChatGPT-User":["partial","named",1,0],"OAI-SearchBot":["partial","named",1,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["blocked","named",1,0],"Claude-SearchBot":["partial","named",1,2],"anthropic-ai":["partial","named",1,2],"PerplexityBot":["partial","named",1,2],"Perplexity-User":["blocked","named",1,0],"Google-Extended":["partial","named",1,2],"Applebot-Extended":["blocked","named",1,0],"CCBot":["partial","named",1,2],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["partial","named",1,2],"cohere-ai":["partial","named",1,2],"Diffbot":["partial","*",2,0],"ContextFetchBot":["partial","*",2,0]},"llms":{"status":"http_error","http":402,"bytes":612},"home":{"status":"http_error","http":402,"url":"https://www.realsimple.com/"}},
{"rank":2574,"domain":"computerweekly.com","origin":"https://computerweekly.com","robots":{"status":"ok","http":200,"url":"https://www.computerweekly.com/robots.txt","bytes":2067,"groups":7},"bots":{"GPTBot":["partial","*",10,0],"ChatGPT-User":["partial","*",10,0],"OAI-SearchBot":["partial","*",10,0],"ClaudeBot":["partial","*",10,0],"Claude-User":["partial","*",10,0],"Claude-SearchBot":["partial","*",10,0],"anthropic-ai":["partial","*",10,0],"PerplexityBot":["partial","*",10,0],"Perplexity-User":["partial","*",10,0],"Google-Extended":["partial","*",10,0],"Applebot-Extended":["partial","*",10,0],"CCBot":["partial","*",10,0],"Bytespider":["partial","*",10,0],"Amazonbot":["partial","*",10,0],"meta-externalagent":["partial","*",10,0],"cohere-ai":["partial","*",10,0],"Diffbot":["partial","*",10,0],"ContextFetchBot":["partial","*",10,0]},"llms":{"status":"missing","http":404,"bytes":59831},"home":{"status":"http_error","http":403,"url":"https://www.computerweekly.com/"}},
{"rank":2575,"domain":"oculus.com","origin":"https://oculus.com","robots":{"status":"ok","http":200,"url":"https://www.oculus.com/robots.txt/","bytes":439,"groups":2},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"html","http":200,"bytes":241322},"home":{"status":"ok","http":200,"url":"https://www.meta.com/quest/","html_tokens":123046,"markdown_tokens":50,"pruned_tokens":50}},
{"rank":2576,"domain":"mext.go.jp","origin":"https://www.mext.go.jp","robots":{"status":"missing","http":404,"url":"https://www.mext.go.jp/robots.txt","bytes":5794,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":5794},"home":{"status":"ok","http":200,"url":"https://www.mext.go.jp/","html_tokens":3621,"markdown_tokens":28,"pruned_tokens":28}},
{"rank":2577,"domain":"umweltbundesamt.de","origin":"https://umweltbundesamt.de","robots":{"status":"ok","http":200,"url":"https://www.umweltbundesamt.de/robots.txt","bytes":2118,"groups":1},"bots":{"GPTBot":["partial","*",37,18],"ChatGPT-User":["partial","*",37,18],"OAI-SearchBot":["partial","*",37,18],"ClaudeBot":["partial","*",37,18],"Claude-User":["partial","*",37,18],"Claude-SearchBot":["partial","*",37,18],"anthropic-ai":["partial","*",37,18],"PerplexityBot":["partial","*",37,18],"Perplexity-User":["partial","*",37,18],"Google-Extended":["partial","*",37,18],"Applebot-Extended":["partial","*",37,18],"CCBot":["partial","*",37,18],"Bytespider":["partial","*",37,18],"Amazonbot":["partial","*",37,18],"meta-externalagent":["partial","*",37,18],"cohere-ai":["partial","*",37,18],"Diffbot":["partial","*",37,18],"ContextFetchBot":["partial","*",37,18]},"llms":{"status":"missing","http":404,"bytes":206},"home":{"status":"ok","http":200,"url":"https://www.umweltbundesamt.de/","html_tokens":99549,"markdown_tokens":665,"pruned_tokens":135}},
{"rank":2578,"domain":"foodandwine.com","origin":"https://foodandwine.com","robots":{"status":"ok","http":200,"url":"https://www.foodandwine.com/robots.txt","bytes":3475,"groups":6},"bots":{"GPTBot":["partial","named",1,0],"ChatGPT-User":["partial","named",1,0],"OAI-SearchBot":["partial","named",1,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["blocked","named",1,0],"Claude-SearchBot":["partial","named",1,2],"anthropic-ai":["partial","named",1,2],"PerplexityBot":["partial","named",1,2],"Perplexity-User":["blocked","named",1,0],"Google-Extended":["partial","named",1,2],"Applebot-Extended":["blocked","named",1,0],"CCBot":["partial","named",1,2],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["partial","named",1,2],"cohere-ai":["partial","named",1,2],"Diffbot":["partial","*",2,0],"ContextFetchBot":["partial","*",2,0]},"llms":{"status":"http_error","http":402,"bytes":612},"home":{"status":"http_error","http":402,"url":"https://www.foodandwine.com/"}},
{"rank":2579,"domain":"canon.com","origin":"https://www.canon.com","robots":{"status":"missing","http":404,"url":"https://www.canon.com/robots.txt","bytes":315,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":315},"home":{"status":"ok","http":200,"url":"https://global.canon/en/index.html","html_tokens":25436,"markdown_tokens":1393,"pruned_tokens":355}},
{"rank":2580,"domain":"readwrite.com","origin":"https://readwrite.com","robots":{"status":"ok","http":200,"url":"https://readwrite.com/robots.txt","bytes":332,"groups":1},"bots":{"GPTBot":["partial","*",5,1],"ChatGPT-User":["partial","*",5,1],"OAI-SearchBot":["partial","*",5,1],"ClaudeBot":["partial","*",5,1],"Claude-User":["partial","*",5,1],"Claude-SearchBot":["partial","*",5,1],"anthropic-ai":["partial","*",5,1],"PerplexityBot":["partial","*",5,1],"Perplexity-User":["partial","*",5,1],"Google-Extended":["partial","*",5,1],"Applebot-Extended":["partial","*",5,1],"CCBot":["partial","*",5,1],"Bytespider":["partial","*",5,1],"Amazonbot":["partial","*",5,1],"meta-externalagent":["partial","*",5,1],"cohere-ai":["partial","*",5,1],"Diffbot":["partial","*",5,1],"ContextFetchBot":["partial","*",5,1]},"llms":{"status":"missing","http":404,"bytes":125658},"home":{"status":"ok","http":200,"url":"https://readwrite.com/","html_tokens":87176,"markdown_tokens":165,"pruned_tokens":113}},
{"rank":2581,"domain":"financialpost.com","origin":"https://financialpost.com","robots":{"status":"ok","http":200,"url":"https://financialpost.com/robots.txt","bytes":1233,"groups":15},"bots":{"GPTBot":["partial","*",3,6],"ChatGPT-User":["partial","*",3,6],"OAI-SearchBot":["partial","*",3,6],"ClaudeBot":["partial","*",3,6],"Claude-User":["partial","*",3,6],"Claude-SearchBot":["partial","*",3,6],"anthropic-ai":["partial","*",3,6],"PerplexityBot":["partial","*",3,6],"Perplexity-User":["partial","*",3,6],"Google-Extended":["partial","*",3,6],"Applebot-Extended":["partial","*",3,6],"CCBot":["partial","*",3,6],"Bytespider":["partial","*",3,6],"Amazonbot":["partial","*",3,6],"meta-externalagent":["partial","*",3,6],"cohere-ai":["partial","*",3,6],"Diffbot":["partial","*",3,6],"ContextFetchBot":["partial","*",3,6]},"llms":{"status":"missing","http":404,"bytes":258537},"home":{"status":"ok","http":200,"url":"https://financialpost.com/","html_tokens":194658,"markdown_tokens":287,"pruned_tokens":173}},
{"rank":2582,"domain":"study.com","origin":"https://study.com","robots":{"status":"ok","http":200,"url":"https://study.com/robots.txt","bytes":25061,"groups":5},"bots":{"GPTBot":["partial","*",4,1],"ChatGPT-User":["partial","*",4,1],"OAI-SearchBot":["partial","*",4,1],"ClaudeBot":["partial","*",4,1],"Claude-User":["partial","*",4,1],"Claude-SearchBot":["partial","*",4,1],"anthropic-ai":["partial","*",4,1],"PerplexityBot":["partial","*",4,1],"Perplexity-User":["partial","*",4,1],"Google-Extended":["partial","*",4,1],"Applebot-Extended":["partial","*",4,1],"CCBot":["partial","named",2,467],"Bytespider":["partial","*",4,1],"Amazonbot":["partial","*",4,1],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["partial","*",4,1],"Diffbot":["partial","*",4,1],"ContextFetchBot":["partial","*",4,1]},"llms":{"status":"missing","http":404,"bytes":2143},"home":{"status":"http_error","http":403,"url":"https://study.com/"}},
{"rank":2583,"domain":"mb.ca","origin":"https://mb.ca","robots":{"status":"unreachable","http":null,"url":"https://mb.ca/robots.txt","bytes":0,"groups":0,"error":"cannot resolve mb.ca"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":2584,"domain":"alberta.ca","origin":"https://alberta.ca","robots":{"status":"ok","http":200,"url":"https://www.alberta.ca/robots.txt","bytes":2357,"groups":1},"bots":{"GPTBot":["partial","*",45,18],"ChatGPT-User":["partial","*",45,18],"OAI-SearchBot":["partial","*",45,18],"ClaudeBot":["partial","*",45,18],"Claude-User":["partial","*",45,18],"Claude-SearchBot":["partial","*",45,18],"anthropic-ai":["partial","*",45,18],"PerplexityBot":["partial","*",45,18],"Perplexity-User":["partial","*",45,18],"Google-Extended":["partial","*",45,18],"Applebot-Extended":["partial","*",45,18],"CCBot":["partial","*",45,18],"Bytespider":["partial","*",45,18],"Amazonbot":["partial","*",45,18],"meta-externalagent":["partial","*",45,18],"cohere-ai":["partial","*",45,18],"Diffbot":["partial","*",45,18],"ContextFetchBot":["partial","*",45,18]},"llms":{"status":"unreachable","error":"[SSL: CERTIFICATE_VERIFY_FAILED] certificate verify failed: unable to get local issuer certificate (_ssl.c:1028)"},"home":{"status":"ok","http":200,"url":"https://www.alberta.ca/","html_tokens":11194,"markdown_tokens":221,"pruned_tokens":46}},
{"rank":2585,"domain":"forbes.ru","origin":"https://forbes.ru","robots":{"status":"html","http":200,"url":"https://www.forbes.ru/robots.txt","bytes":13602,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":2586,"domain":"audacityteam.org","origin":"https://audacityteam.org","robots":{"status":"missing","http":404,"url":"https://www.audacityteam.org/robots.txt","bytes":91684,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":91684},"home":{"status":"ok","http":200,"url":"https://www.audacityteam.org/","html_tokens":31220,"markdown_tokens":468,"pruned_tokens":313}},
{"rank":2587,"domain":"xero.com","origin":"https://xero.com","robots":{"status":"ok","http":200,"url":"https://www.xero.com/robots.txt","bytes":260,"groups":1},"bots":{"GPTBot":["partial","*",8,0],"ChatGPT-User":["partial","*",8,0],"OAI-SearchBot":["partial","*",8,0],"ClaudeBot":["partial","*",8,0],"Claude-User":["partial","*",8,0],"Claude-SearchBot":["partial","*",8,0],"anthropic-ai":["partial","*",8,0],"PerplexityBot":["partial","*",8,0],"Perplexity-User":["partial","*",8,0],"Google-Extended":["partial","*",8,0],"Applebot-Extended":["partial","*",8,0],"CCBot":["partial","*",8,0],"Bytespider":["partial","*",8,0],"Amazonbot":["partial","*",8,0],"meta-externalagent":["partial","*",8,0],"cohere-ai":["partial","*",8,0],"Diffbot":["partial","*",8,0],"ContextFetchBot":["partial","*",8,0]},"llms":{"status":"present","http":200,"bytes":13353},"home":{"status":"ok","http":200,"url":"https://www.xero.com/us/","html_tokens":114475,"markdown_tokens":326,"pruned_tokens":222}},
{"rank":2588,"domain":"adelaide.edu.au","origin":"https://adelaide.edu.au","robots":{"status":"ok","http":200,"url":"https://adelaide.edu.au/robots.txt","bytes":123,"groups":1},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"missing","http":404,"bytes":236},"home":{"status":"ok","http":200,"url":"https://adelaide.edu.au/","html_tokens":56347,"markdown_tokens":159,"pruned_tokens":131}},
{"rank":2589,"domain":"replit.com","origin":"https://replit.com","robots":{"status":"ok","http":200,"url":"https://replit.com/robots.txt","bytes":432,"groups":2},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"present","http":200,"bytes":20191},"home":{"status":"ok","http":200,"url":"https://replit.com/","html_tokens":170064,"markdown_tokens":304,"pruned_tokens":100}},
{"rank":2590,"domain":"arabnews.com","origin":"https://arabnews.com","robots":{"status":"ok","http":200,"url":"https://www.arabnews.com/robots.txt","bytes":453,"groups":1},"bots":{"GPTBot":["partial","*",8,2],"ChatGPT-User":["partial","*",8,2],"OAI-SearchBot":["partial","*",8,2],"ClaudeBot":["partial","*",8,2],"Claude-User":["partial","*",8,2],"Claude-SearchBot":["partial","*",8,2],"anthropic-ai":["partial","*",8,2],"PerplexityBot":["partial","*",8,2],"Perplexity-User":["partial","*",8,2],"Google-Extended":["partial","*",8,2],"Applebot-Extended":["partial","*",8,2],"CCBot":["partial","*",8,2],"Bytespider":["partial","*",8,2],"Amazonbot":["partial","*",8,2],"meta-externalagent":["partial","*",8,2],"cohere-ai":["partial","*",8,2],"Diffbot":["partial","*",8,2],"ContextFetchBot":["partial","*",8,2]},"llms":{"status":"present","http":200,"bytes":1863},"home":{"status":"ok","http":200,"url":"https://www.arabnews.com/","html_tokens":71379,"markdown_tokens":56,"pruned_tokens":56}},
{"rank":2591,"domain":"fsc.org","origin":"https://fsc.org","robots":{"status":"ok","http":200,"url":"https://fsc.org/robots.txt","bytes":2178,"groups":1},"bots":{"GPTBot":["partial","*",37,22],"ChatGPT-User":["partial","*",37,22],"OAI-SearchBot":["partial","*",37,22],"ClaudeBot":["partial","*",37,22],"Claude-User":["partial","*",37,22],"Claude-SearchBot":["partial","*",37,22],"anthropic-ai":["partial","*",37,22],"PerplexityBot":["partial","*",37,22],"Perplexity-User":["partial","*",37,22],"Google-Extended":["partial","*",37,22],"Applebot-Extended":["partial","*",37,22],"CCBot":["partial","*",37,22],"Bytespider":["partial","*",37,22],"Amazonbot":["partial","*",37,22],"meta-externalagent":["partial","*",37,22],"cohere-ai":["partial","*",37,22],"Diffbot":["partial","*",37,22],"ContextFetchBot":["partial","*",37,22]},"llms":{"status":"missing","http":404,"bytes":181},"home":{"status":"ok","http":200,"url":"https://fsc.org/en","html_tokens":112173,"markdown_tokens":257,"pruned_tokens":234}},
{"rank":2592,"domain":"clarin.com","origin":"https://clarin.com","robots":{"status":"ok","http":200,"url":"https://www.clarin.com/robots.txt","bytes":457,"groups":1},"bots":{"GPTBot":["partial","*",12,0],"ChatGPT-User":["partial","*",12,0],"OAI-SearchBot":["partial","*",12,0],"ClaudeBot":["partial","*",12,0],"Claude-User":["partial","*",12,0],"Claude-SearchBot":["partial","*",12,0],"anthropic-ai":["partial","*",12,0],"PerplexityBot":["partial","*",12,0],"Perplexity-User":["partial","*",12,0],"Google-Extended":["partial","*",12,0],"Applebot-Extended":["partial","*",12,0],"CCBot":["partial","*",12,0],"Bytespider":["partial","*",12,0],"Amazonbot":["partial","*",12,0],"meta-externalagent":["partial","*",12,0],"cohere-ai":["partial","*",12,0],"Diffbot":["partial","*",12,0],"ContextFetchBot":["partial","*",12,0]},"llms":{"status":"missing","http":404,"bytes":248160},"home":{"status":"ok","http":200,"url":"https://www.clarin.com/","html_tokens":1023376,"markdown_tokens":1512,"pruned_tokens":173}},
{"rank":2593,"domain":"idnes.cz","origin":"https://idnes.cz","robots":{"status":"ok","http":200,"url":"https://www.idnes.cz/robots.txt","bytes":1105,"groups":18},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["partial","*",16,1],"OAI-SearchBot":["partial","*",16,1],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",16,1],"Claude-SearchBot":["partial","*",16,1],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["partial","*",16,1],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["partial","*",16,1],"CCBot":["partial","*",16,1],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["partial","*",16,1],"cohere-ai":["partial","*",16,1],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",16,1]},"llms":{"status":"missing","http":404,"bytes":3407},"home":{"status":"ok","http":200,"url":"https://www.idnes.cz/","html_tokens":52037,"markdown_tokens":1928,"pruned_tokens":163}},
{"rank":2594,"domain":"aa.com","origin":"https://aa.com","robots":{"status":"ok","http":200,"url":"https://www.aa.com/robots.txt","bytes":587,"groups":1},"bots":{"GPTBot":["partial","*",18,0],"ChatGPT-User":["partial","*",18,0],"OAI-SearchBot":["partial","*",18,0],"ClaudeBot":["partial","*",18,0],"Claude-User":["partial","*",18,0],"Claude-SearchBot":["partial","*",18,0],"anthropic-ai":["partial","*",18,0],"PerplexityBot":["partial","*",18,0],"Perplexity-User":["partial","*",18,0],"Google-Extended":["partial","*",18,0],"Applebot-Extended":["partial","*",18,0],"CCBot":["partial","*",18,0],"Bytespider":["partial","*",18,0],"Amazonbot":["partial","*",18,0],"meta-externalagent":["partial","*",18,0],"cohere-ai":["partial","*",18,0],"Diffbot":["partial","*",18,0],"ContextFetchBot":["partial","*",18,0]},"llms":{"status":"missing","http":404,"bytes":291},"home":{"status":"http_error","http":403,"url":"https://aa.com/"}},
{"rank":2595,"domain":"vanguardngr.com","origin":"https://vanguardngr.com","robots":{"status":"ok","http":200,"url":"https://www.vanguardngr.com/robots.txt","bytes":383,"groups":3},"bots":{"GPTBot":["partial","*",6,2],"ChatGPT-User":["partial","*",6,2],"OAI-SearchBot":["partial","*",6,2],"ClaudeBot":["partial","*",6,2],"Claude-User":["partial","*",6,2],"Claude-SearchBot":["partial","*",6,2],"anthropic-ai":["partial","*",6,2],"PerplexityBot":["partial","*",6,2],"Perplexity-User":["partial","*",6,2],"Google-Extended":["partial","*",6,2],"Applebot-Extended":["partial","*",6,2],"CCBot":["partial","*",6,2],"Bytespider":["partial","*",6,2],"Amazonbot":["partial","*",6,2],"meta-externalagent":["partial","*",6,2],"cohere-ai":["partial","*",6,2],"Diffbot":["partial","*",6,2],"ContextFetchBot":["partial","*",6,2]},"llms":{"status":"missing","http":404,"bytes":147136},"home":{"status":"http_error","http":403,"url":"https://vanguardngr.com/"}},
{"rank":2596,"domain":"fordham.edu","origin":"https://fordham.edu","robots":{"status":"ok","http":200,"url":"https://www.fordham.edu/robots.txt","bytes":1000,"groups":11},"bots":{"GPTBot":["allowed","named",0,1],"ChatGPT-User":["allowed","named",0,1],"OAI-SearchBot":["partial","*",19,0],"ClaudeBot":["allowed","named",0,1],"Claude-User":["allowed","named",0,1],"Claude-SearchBot":["allowed","named",0,1],"anthropic-ai":["partial","*",19,0],"PerplexityBot":["allowed","named",0,1],"Perplexity-User":["allowed","named",0,1],"Google-Extended":["allowed","named",0,1],"Applebot-Extended":["partial","*",19,0],"CCBot":["partial","*",19,0],"Bytespider":["partial","*",19,0],"Amazonbot":["partial","*",19,0],"meta-externalagent":["partial","*",19,0],"cohere-ai":["partial","*",19,0],"Diffbot":["partial","*",19,0],"ContextFetchBot":["partial","*",19,0]},"llms":{"status":"present","http":200,"bytes":3080},"home":{"status":"ok","http":200,"url":"https://www.fordham.edu/","html_tokens":34300,"markdown_tokens":176,"pruned_tokens":64}},
{"rank":2597,"domain":"designboom.com","origin":"https://designboom.com","robots":{"status":"ok","http":200,"url":"https://www.designboom.com/robots.txt","bytes":1223,"groups":1},"bots":{"GPTBot":["partial","*",21,21],"ChatGPT-User":["partial","*",21,21],"OAI-SearchBot":["partial","*",21,21],"ClaudeBot":["partial","*",21,21],"Claude-User":["partial","*",21,21],"Claude-SearchBot":["partial","*",21,21],"anthropic-ai":["partial","*",21,21],"PerplexityBot":["partial","*",21,21],"Perplexity-User":["partial","*",21,21],"Google-Extended":["partial","*",21,21],"Applebot-Extended":["partial","*",21,21],"CCBot":["partial","*",21,21],"Bytespider":["partial","*",21,21],"Amazonbot":["partial","*",21,21],"meta-externalagent":["partial","*",21,21],"cohere-ai":["partial","*",21,21],"Diffbot":["partial","*",21,21],"ContextFetchBot":["partial","*",21,21]},"llms":{"status":"missing","http":404,"bytes":146},"home":{"status":"ok","http":200,"url":"https://www.designboom.com/","html_tokens":74041,"markdown_tokens":558,"pruned_tokens":43}},
{"rank":2598,"domain":"eurogamer.net","origin":"https://eurogamer.net","robots":{"status":"ok","http":200,"url":"https://www.eurogamer.net/robots.txt","bytes":5657,"groups":2},"bots":{"GPTBot":["partial","named",1,1],"ChatGPT-User":["partial","named",1,1],"OAI-SearchBot":["partial","named",1,1],"ClaudeBot":["partial","named",1,1],"Claude-User":["partial","named",1,1],"Claude-SearchBot":["partial","named",1,1],"anthropic-ai":["partial","named",1,1],"PerplexityBot":["partial","named",1,1],"Perplexity-User":["partial","named",1,1],"Google-Extended":["partial","*",9,0],"Applebot-Extended":["partial","named",1,1],"CCBot":["partial","named",1,1],"Bytespider":["partial","named",1,1],"Amazonbot":["partial","named",1,1],"meta-externalagent":["partial","named",1,1],"cohere-ai":["partial","named",1,1],"Diffbot":["partial","named",1,1],"ContextFetchBot":["partial","*",9,0]},"llms":{"status":"missing","http":404,"bytes":2283},"home":{"status":"ok","http":200,"url":"https://www.eurogamer.net/","html_tokens":70889,"markdown_tokens":106,"pruned_tokens":48}},
{"rank":2599,"domain":"britishairways.com","origin":"http://britishairways.com","robots":{"status":"html","http":200,"url":"http://britishairways.com/robots.txt","bytes":8995,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":2600,"domain":"matrix.org","origin":"https://matrix.org","robots":{"status":"ok","http":200,"url":"https://matrix.org/robots.txt","bytes":73,"groups":1},"bots":{"GPTBot":["allowed","*",0,1],"ChatGPT-User":["allowed","*",0,1],"OAI-SearchBot":["allowed","*",0,1],"ClaudeBot":["allowed","*",0,1],"Claude-User":["allowed","*",0,1],"Claude-SearchBot":["allowed","*",0,1],"anthropic-ai":["allowed","*",0,1],"PerplexityBot":["allowed","*",0,1],"Perplexity-User":["allowed","*",0,1],"Google-Extended":["allowed","*",0,1],"Applebot-Extended":["allowed","*",0,1],"CCBot":["allowed","*",0,1],"Bytespider":["allowed","*",0,1],"Amazonbot":["allowed","*",0,1],"meta-externalagent":["allowed","*",0,1],"cohere-ai":["allowed","*",0,1],"Diffbot":["allowed","*",0,1],"ContextFetchBot":["allowed","*",0,1]},"llms":{"status":"missing","http":404,"bytes":16294},"home":{"status":"ok","http":200,"url":"https://matrix.org/","html_tokens":12681,"markdown_tokens":127,"pruned_tokens":92}},
{"rank":2601,"domain":"emirates.com","origin":"https://emirates.com","robots":{"status":"unreachable","http":null,"url":"https://emirates.com/robots.txt","bytes":0,"groups":0,"error":"<StreamReset stream_id:1, error_code:2, remote_reset:True>"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":2602,"domain":"gitbook.com","origin":"https://gitbook.com","robots":{"status":"ok","http":200,"url":"https://www.gitbook.com/robots.txt","bytes":432,"groups":1},"bots":{"GPTBot":["allowed","*",0,1],"ChatGPT-User":["allowed","*",0,1],"OAI-SearchBot":["allowed","*",0,1],"ClaudeBot":["allowed","*",0,1],"Claude-User":["allowed","*",0,1],"Claude-SearchBot":["allowed","*",0,1],"anthropic-ai":["allowed","*",0,1],"PerplexityBot":["allowed","*",0,1],"Perplexity-User":["allowed","*",0,1],"Google-Extended":["allowed","*",0,1],"Applebot-Extended":["allowed","*",0,1],"CCBot":["allowed","*",0,1],"Bytespider":["allowed","*",0,1],"Amazonbot":["allowed","*",0,1],"meta-externalagent":["allowed","*",0,1],"cohere-ai":["allowed","*",0,1],"Diffbot":["allowed","*",0,1],"ContextFetchBot":["allowed","*",0,1]},"llms":{"status":"present","http":200,"bytes":11068},"home":{"status":"ok","http":200,"url":"https://www.gitbook.com/","html_tokens":375972,"markdown_tokens":305,"pruned_tokens":123}},
{"rank":2603,"domain":"shell.com","origin":"https://shell.com","robots":{"status":"ok","http":200,"url":"https://www.shell.com/robots.txt","bytes":18776,"groups":3},"bots":{"GPTBot":["partial","*",208,0],"ChatGPT-User":["partial","*",208,0],"OAI-SearchBot":["partial","*",208,0],"ClaudeBot":["partial","*",208,0],"Claude-User":["partial","*",208,0],"Claude-SearchBot":["partial","*",208,0],"anthropic-ai":["partial","*",208,0],"PerplexityBot":["partial","*",208,0],"Perplexity-User":["partial","*",208,0],"Google-Extended":["partial","*",208,0],"Applebot-Extended":["partial","*",208,0],"CCBot":["partial","*",208,0],"Bytespider":["partial","*",208,0],"Amazonbot":["partial","*",208,0],"meta-externalagent":["partial","*",208,0],"cohere-ai":["partial","*",208,0],"Diffbot":["partial","*",208,0],"ContextFetchBot":["partial","*",208,0]},"llms":{"status":"missing","http":404,"bytes":196},"home":{"status":"ok","http":200,"url":"https://www.shell.com/","html_tokens":2291,"markdown_tokens":0,"pruned_tokens":0}},
{"rank":2604,"domain":"samr.gov.cn","origin":"https://www.samr.gov.cn","robots":{"status":"missing","http":404,"url":"https://www.samr.gov.cn/robots.txt","bytes":146,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":146},"home":{"status":"ok","http":200,"url":"https://www.samr.gov.cn/","html_tokens":21270,"markdown_tokens":25,"pruned_tokens":25}},
{"rank":2605,"domain":"cdbaby.com","origin":"https://cdbaby.com","robots":{"status":"ok","http":200,"url":"https://cdbaby.com/robots.txt","bytes":636,"groups":13},"bots":{"GPTBot":["allowed","named",0,1],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","named",0,1],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","named",0,1],"PerplexityBot":["allowed","named",0,1],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","named",0,1],"Applebot-Extended":["allowed","named",0,1],"CCBot":["allowed","named",0,1],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","named",0,1],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","named",0,1],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"http_error","http":403,"bytes":5541},"home":{"status":"http_error","http":403,"url":"https://cdbaby.com/"}},
{"rank":2606,"domain":"ala.org","origin":"https://ala.org","robots":{"status":"refused","http":403,"url":"https://ala.org/robots.txt","bytes":560,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":2607,"domain":"ojp.gov","origin":"https://ojp.gov","robots":{"status":"ok","http":200,"url":"https://www.ojp.gov/robots.txt","bytes":1978,"groups":1},"bots":{"GPTBot":["partial","*",31,18],"ChatGPT-User":["partial","*",31,18],"OAI-SearchBot":["partial","*",31,18],"ClaudeBot":["partial","*",31,18],"Claude-User":["partial","*",31,18],"Claude-SearchBot":["partial","*",31,18],"anthropic-ai":["partial","*",31,18],"PerplexityBot":["partial","*",31,18],"Perplexity-User":["partial","*",31,18],"Google-Extended":["partial","*",31,18],"Applebot-Extended":["partial","*",31,18],"CCBot":["partial","*",31,18],"Bytespider":["partial","*",31,18],"Amazonbot":["partial","*",31,18],"meta-externalagent":["partial","*",31,18],"cohere-ai":["partial","*",31,18],"Diffbot":["partial","*",31,18],"ContextFetchBot":["partial","*",31,18]},"llms":{"status":"missing","http":404,"bytes":185},"home":{"status":"ok","http":200,"url":"https://www.ojp.gov/","html_tokens":13508,"markdown_tokens":186,"pruned_tokens":136}},
{"rank":2608,"domain":"lepoint.fr","origin":"https://lepoint.fr","robots":{"status":"refused","http":403,"url":"https://www.lepoint.fr/robots.txt","bytes":770,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":2609,"domain":"nbcsports.com","origin":"https://nbcsports.com","robots":{"status":"ok","http":200,"url":"https://www.nbcsports.com/robots.txt","bytes":1914,"groups":2},"bots":{"GPTBot":["partial","*",9,1],"ChatGPT-User":["partial","*",9,1],"OAI-SearchBot":["partial","*",9,1],"ClaudeBot":["partial","*",9,1],"Claude-User":["partial","*",9,1],"Claude-SearchBot":["partial","*",9,1],"anthropic-ai":["partial","*",9,1],"PerplexityBot":["partial","*",9,1],"Perplexity-User":["partial","*",9,1],"Google-Extended":["partial","*",9,1],"Applebot-Extended":["partial","*",9,1],"CCBot":["partial","*",9,1],"Bytespider":["partial","*",9,1],"Amazonbot":["partial","*",9,1],"meta-externalagent":["partial","*",9,1],"cohere-ai":["partial","*",9,1],"Diffbot":["partial","*",9,1],"ContextFetchBot":["partial","*",9,1]},"llms":{"status":"missing","http":404,"bytes":478350},"home":{"status":"ok","http":200,"url":"https://www.nbcsports.com/","html_tokens":290022,"markdown_tokens":1535,"pruned_tokens":152}},
{"rank":2610,"domain":"cancerresearchuk.org","origin":"https://cancerresearchuk.org","robots":{"status":"ok","http":200,"url":"https://www.cancerresearchuk.org/robots.txt","bytes":2440,"groups":2},"bots":{"GPTBot":["partial","*",75,3],"ChatGPT-User":["partial","*",75,3],"OAI-SearchBot":["partial","*",75,3],"ClaudeBot":["partial","*",75,3],"Claude-User":["partial","*",75,3],"Claude-SearchBot":["partial","*",75,3],"anthropic-ai":["partial","*",75,3],"PerplexityBot":["partial","*",75,3],"Perplexity-User":["partial","*",75,3],"Google-Extended":["partial","*",75,3],"Applebot-Extended":["partial","*",75,3],"CCBot":["partial","*",75,3],"Bytespider":["partial","*",75,3],"Amazonbot":["partial","*",75,3],"meta-externalagent":["partial","*",75,3],"cohere-ai":["partial","*",75,3],"Diffbot":["partial","*",75,3],"ContextFetchBot":["partial","*",75,3]},"llms":{"status":"missing","http":404,"bytes":1861985},"home":{"status":"ok","http":200,"url":"https://www.cancerresearchuk.org/","html_tokens":530086,"markdown_tokens":360,"pruned_tokens":89}},
{"rank":2611,"domain":"westerndigital.com","origin":"https://westerndigital.com","robots":{"status":"ok","http":200,"url":"https://www.westerndigital.com/robots.txt","bytes":697,"groups":3},"bots":{"GPTBot":["partial","*",5,4],"ChatGPT-User":["partial","*",5,4],"OAI-SearchBot":["partial","named",3,4],"ClaudeBot":["partial","*",5,4],"Claude-User":["partial","*",5,4],"Claude-SearchBot":["partial","*",5,4],"anthropic-ai":["partial","*",5,4],"PerplexityBot":["partial","*",5,4],"Perplexity-User":["partial","*",5,4],"Google-Extended":["partial","*",5,4],"Applebot-Extended":["partial","*",5,4],"CCBot":["partial","*",5,4],"Bytespider":["partial","*",5,4],"Amazonbot":["partial","*",5,4],"meta-externalagent":["partial","*",5,4],"cohere-ai":["partial","*",5,4],"Diffbot":["partial","*",5,4],"ContextFetchBot":["partial","*",5,4]},"llms":{"status":"present","http":200,"bytes":23109},"home":{"status":"ok","http":200,"url":"https://www.westerndigital.com/","html_tokens":298905,"markdown_tokens":174,"pruned_tokens":174}},
{"rank":2612,"domain":"active.macromedia.com","origin":"https://active.macromedia.com","robots":{"status":"unreachable","http":null,"url":"https://active.macromedia.com/robots.txt","bytes":0,"groups":0,"error":"cannot resolve active.macromedia.com"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":2613,"domain":"hal.science","origin":"https://hal.science","robots":{"status":"ok","http":200,"url":"https://hal.science/robots","bytes":2993,"groups":50},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["blocked","named",1,0],"OAI-SearchBot":["blocked","named",1,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",34,0],"Claude-SearchBot":["partial","*",34,0],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["partial","*",34,0],"Google-Extended":["blocked","named",2,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",34,0]},"llms":{"status":"missing","http":404,"bytes":48597},"home":{"status":"ok","http":200,"url":"https://hal.science/","html_tokens":8112,"markdown_tokens":196,"pruned_tokens":104}},
{"rank":2614,"domain":"businessnewsdaily.com","origin":"https://businessnewsdaily.com","robots":{"status":"ok","http":200,"url":"https://www.businessnewsdaily.com/robots.txt","bytes":552,"groups":2},"bots":{"GPTBot":["allowed","named",0,1],"ChatGPT-User":["allowed","named",0,1],"OAI-SearchBot":["allowed","named",0,1],"ClaudeBot":["allowed","named",0,1],"Claude-User":["partial","*",1,0],"Claude-SearchBot":["partial","*",1,0],"anthropic-ai":["allowed","named",0,1],"PerplexityBot":["allowed","named",0,1],"Perplexity-User":["allowed","named",0,1],"Google-Extended":["allowed","named",0,1],"Applebot-Extended":["allowed","named",0,1],"CCBot":["partial","*",1,0],"Bytespider":["partial","*",1,0],"Amazonbot":["allowed","named",0,1],"meta-externalagent":["allowed","named",0,1],"cohere-ai":["partial","*",1,0],"Diffbot":["partial","*",1,0],"ContextFetchBot":["partial","*",1,0]},"llms":{"status":"missing","http":404,"bytes":97849},"home":{"status":"ok","http":200,"url":"https://www.businessnewsdaily.com/","html_tokens":42091,"markdown_tokens":69,"pruned_tokens":66}},
{"rank":2615,"domain":"cairn.info","origin":"https://cairn.info","robots":{"status":"refused","http":403,"url":"https://shs.cairn.info/","bytes":770,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":2616,"domain":"intechopen.com","origin":"https://intechopen.com","robots":{"status":"ok","http":200,"url":"https://www.intechopen.com/robots.txt","bytes":3084,"groups":24},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["partial","*",1,1],"OAI-SearchBot":["allowed","named",0,1],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",1,1],"Claude-SearchBot":["partial","*",1,1],"anthropic-ai":["partial","*",1,1],"PerplexityBot":["allowed","named",0,1],"Perplexity-User":["partial","*",1,1],"Google-Extended":["allowed","named",0,1],"Applebot-Extended":["allowed","named",0,1],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["allowed","named",0,1],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",1,1]},"llms":{"status":"present","http":200,"bytes":5265},"home":{"status":"ok","http":200,"url":"https://www.intechopen.com/","html_tokens":346438,"markdown_tokens":177,"pruned_tokens":69}},
{"rank":2617,"domain":"ask.com","origin":"https://ask.com","robots":{"status":"missing","http":404,"url":"https://www.ask.com/robots.txt","bytes":2353,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":2353},"home":{"status":"ok","http":200,"url":"https://www.ask.com/","html_tokens":895,"markdown_tokens":159,"pruned_tokens":148}},
{"rank":2618,"domain":"doordash.com","origin":"https://doordash.com","robots":{"status":"refused","http":403,"url":"https://www.doordash.com/robots.txt","bytes":5575,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":2619,"domain":"inquirer.net","origin":"https://inquirer.net","robots":{"status":"ok","http":200,"url":"https://inquirer.net/robots.txt","bytes":441,"groups":4},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["partial","*",12,0],"OAI-SearchBot":["partial","*",12,0],"ClaudeBot":["partial","*",12,0],"Claude-User":["partial","*",12,0],"Claude-SearchBot":["partial","*",12,0],"anthropic-ai":["partial","*",12,0],"PerplexityBot":["partial","*",12,0],"Perplexity-User":["partial","*",12,0],"Google-Extended":["partial","*",12,0],"Applebot-Extended":["partial","*",12,0],"CCBot":["partial","*",12,0],"Bytespider":["partial","*",12,0],"Amazonbot":["partial","*",12,0],"meta-externalagent":["partial","*",12,0],"cohere-ai":["partial","*",12,0],"Diffbot":["partial","*",12,0],"ContextFetchBot":["partial","*",12,0]},"llms":{"status":"http_error","http":403,"bytes":5543},"home":{"status":"http_error","http":403,"url":"https://inquirer.net/"}},
{"rank":2620,"domain":"analog.com","origin":"https://analog.com","robots":{"status":"unreachable","http":null,"url":"https://analog.com/robots.txt","bytes":0,"groups":0,"error":"<StreamReset stream_id:1, error_code:2, remote_reset:True>"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":2621,"domain":"gofund.me","origin":"https://gofund.me","robots":{"status":"ok","http":200,"url":"https://gofund.me/robots.txt","bytes":24,"groups":1},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"html","http":200,"bytes":582810},"home":{"status":"ok","http":200,"url":"https://www.gofundme.com","html_tokens":145270,"markdown_tokens":316,"pruned_tokens":40}},
{"rank":2622,"domain":"firstpost.com","origin":"https://firstpost.com","robots":{"status":"refused","http":403,"url":"https://www.firstpost.com/robots.txt","bytes":383,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":2623,"domain":"f-droid.org","origin":"https://f-droid.org","robots":{"status":"ok","http":200,"url":"https://f-droid.org/robots.txt","bytes":774,"groups":21},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["blocked","named",1,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"missing","http":404,"bytes":12799},"home":{"status":"ok","http":200,"url":"https://f-droid.org/","html_tokens":3279,"markdown_tokens":68,"pruned_tokens":68}},
{"rank":2624,"domain":"foodnetwork.co.uk","origin":"https://foodnetwork.co.uk","robots":{"status":"ok","http":200,"url":"https://foodnetwork.co.uk/robots.txt","bytes":259,"groups":4},"bots":{"GPTBot":["partial","*",1,0],"ChatGPT-User":["partial","*",1,0],"OAI-SearchBot":["partial","*",1,0],"ClaudeBot":["partial","*",1,0],"Claude-User":["partial","*",1,0],"Claude-SearchBot":["partial","*",1,0],"anthropic-ai":["partial","*",1,0],"PerplexityBot":["partial","*",1,0],"Perplexity-User":["partial","*",1,0],"Google-Extended":["partial","*",1,0],"Applebot-Extended":["partial","*",1,0],"CCBot":["partial","*",1,0],"Bytespider":["partial","*",1,0],"Amazonbot":["partial","*",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["partial","*",1,0],"Diffbot":["partial","*",1,0],"ContextFetchBot":["partial","*",1,0]},"llms":{"status":"missing","http":404,"bytes":233999},"home":{"status":"ok","http":200,"url":"https://foodnetwork.co.uk/","html_tokens":213619,"markdown_tokens":65,"pruned_tokens":65}},
{"rank":2625,"domain":"metrika.yandex.ru","origin":"https://metrika.yandex.ru","robots":{"status":"ok","http":200,"url":"https://metrika.yandex.ru/robots.txt","bytes":78,"groups":1},"bots":{"GPTBot":["partial","*",2,3],"ChatGPT-User":["partial","*",2,3],"OAI-SearchBot":["partial","*",2,3],"ClaudeBot":["partial","*",2,3],"Claude-User":["partial","*",2,3],"Claude-SearchBot":["partial","*",2,3],"anthropic-ai":["partial","*",2,3],"PerplexityBot":["partial","*",2,3],"Perplexity-User":["partial","*",2,3],"Google-Extended":["partial","*",2,3],"Applebot-Extended":["partial","*",2,3],"CCBot":["partial","*",2,3],"Bytespider":["partial","*",2,3],"Amazonbot":["partial","*",2,3],"meta-externalagent":["partial","*",2,3],"cohere-ai":["partial","*",2,3],"Diffbot":["partial","*",2,3],"ContextFetchBot":["partial","*",2,3]},"llms":{"status":"html","http":200,"bytes":618798},"home":{"status":"ok","http":200,"url":"https://b2b.yandex.ru/adv/metrika","html_tokens":145628,"markdown_tokens":99,"pruned_tokens":99}},
{"rank":2626,"domain":"kubiobuilder.com","origin":"https://kubiobuilder.com","robots":{"status":"missing","http":404,"url":"https://kubiobuilder.com/robots.txt","bytes":925,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":160342},"home":{"status":"ok","http":200,"url":"https://kubiobuilder.com/","html_tokens":119142,"markdown_tokens":898,"pruned_tokens":634}},
{"rank":2627,"domain":"opentext.com","origin":"https://opentext.com","robots":{"status":"ok","http":200,"url":"https://www.opentext.com/robots.txt","bytes":4043,"groups":13},"bots":{"GPTBot":["partial","*",67,1],"ChatGPT-User":["partial","*",67,1],"OAI-SearchBot":["partial","*",67,1],"ClaudeBot":["partial","*",67,1],"Claude-User":["partial","*",67,1],"Claude-SearchBot":["partial","*",67,1],"anthropic-ai":["partial","*",67,1],"PerplexityBot":["partial","*",67,1],"Perplexity-User":["partial","*",67,1],"Google-Extended":["partial","*",67,1],"Applebot-Extended":["partial","*",67,1],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["partial","*",67,1],"meta-externalagent":["partial","*",67,1],"cohere-ai":["partial","*",67,1],"Diffbot":["partial","*",67,1],"ContextFetchBot":["partial","*",67,1]},"llms":{"status":"present","http":200,"bytes":16507},"home":{"status":"ok","http":200,"url":"https://www.opentext.com/","html_tokens":97816,"markdown_tokens":500,"pruned_tokens":263}},
{"rank":2628,"domain":"lifehack.org","origin":"https://lifehack.org","robots":{"status":"ok","http":200,"url":"https://www.lifehack.org/robots.txt","bytes":5451,"groups":135},"bots":{"GPTBot":["partial","*",7,1],"ChatGPT-User":["partial","*",7,1],"OAI-SearchBot":["partial","*",7,1],"ClaudeBot":["partial","*",7,1],"Claude-User":["partial","*",7,1],"Claude-SearchBot":["partial","*",7,1],"anthropic-ai":["partial","*",7,1],"PerplexityBot":["partial","*",7,1],"Perplexity-User":["partial","*",7,1],"Google-Extended":["partial","*",7,1],"Applebot-Extended":["partial","*",7,1],"CCBot":["partial","*",7,1],"Bytespider":["partial","*",7,1],"Amazonbot":["partial","*",7,1],"meta-externalagent":["partial","*",7,1],"cohere-ai":["partial","*",7,1],"Diffbot":["partial","*",7,1],"ContextFetchBot":["partial","*",7,1]},"llms":{"status":"missing","http":404,"bytes":46426},"home":{"status":"ok","http":200,"url":"https://www.lifehack.org/","html_tokens":16774,"markdown_tokens":231,"pruned_tokens":111}},
{"rank":2629,"domain":"elconfidencial.com","origin":"https://elconfidencial.com","robots":{"status":"ok","http":200,"url":"https://www.elconfidencial.com/robots.txt","bytes":10124,"groups":61},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["blocked","named",1,0],"OAI-SearchBot":["blocked","named",1,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",127,2],"Claude-SearchBot":["partial","*",127,2],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["partial","*",127,2],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",127,2]},"llms":{"status":"missing","http":404,"bytes":161147},"home":{"status":"ok","http":200,"url":"https://www.elconfidencial.com/","html_tokens":148394,"markdown_tokens":385,"pruned_tokens":146}},
{"rank":2630,"domain":"tbs.co.jp","origin":"https://tbs.co.jp","robots":{"status":"missing","http":404,"url":"https://tbs.co.jp/robots.txt","bytes":872,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":872},"home":{"status":"http_error","http":404,"url":"https://tbs.co.jp/"}},
{"rank":2631,"domain":"moneycontrol.com","origin":"https://moneycontrol.com","robots":{"status":"refused","http":403,"url":"https://www.moneycontrol.com/robots.txt","bytes":386,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":2632,"domain":"zerohedge.com","origin":"https://zerohedge.com","robots":{"status":"ok","http":200,"url":"https://www.zerohedge.com/robots.txt","bytes":50,"groups":1},"bots":{"GPTBot":["partial","*",1,1],"ChatGPT-User":["partial","*",1,1],"OAI-SearchBot":["partial","*",1,1],"ClaudeBot":["partial","*",1,1],"Claude-User":["partial","*",1,1],"Claude-SearchBot":["partial","*",1,1],"anthropic-ai":["partial","*",1,1],"PerplexityBot":["partial","*",1,1],"Perplexity-User":["partial","*",1,1],"Google-Extended":["partial","*",1,1],"Applebot-Extended":["partial","*",1,1],"CCBot":["partial","*",1,1],"Bytespider":["partial","*",1,1],"Amazonbot":["partial","*",1,1],"meta-externalagent":["partial","*",1,1],"cohere-ai":["partial","*",1,1],"Diffbot":["partial","*",1,1],"ContextFetchBot":["partial","*",1,1]},"llms":{"status":"missing","http":404,"bytes":8964},"home":{"status":"ok","http":200,"url":"https://www.zerohedge.com/","html_tokens":38216,"markdown_tokens":677,"pruned_tokens":167}},
{"rank":2633,"domain":"netdna-ssl.com","origin":"https://netdna-ssl.com","robots":{"status":"unreachable","http":null,"url":"https://netdna-ssl.com/robots.txt","bytes":0,"groups":0,"error":"cannot resolve netdna-ssl.com"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":2634,"domain":"bloomsbury.com","origin":"https://bloomsbury.com","robots":{"status":"ok","http":200,"url":"https://www.bloomsbury.com/robots.txt","bytes":811,"groups":13},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["partial","*",3,0],"OAI-SearchBot":["allowed","named",0,1],"ClaudeBot":["partial","*",3,0],"Claude-User":["partial","*",3,0],"Claude-SearchBot":["partial","*",3,0],"anthropic-ai":["partial","*",3,0],"PerplexityBot":["partial","*",3,0],"Perplexity-User":["partial","*",3,0],"Google-Extended":["partial","*",3,0],"Applebot-Extended":["partial","*",3,0],"CCBot":["partial","*",3,0],"Bytespider":["partial","*",3,0],"Amazonbot":["partial","*",3,0],"meta-externalagent":["partial","*",3,0],"cohere-ai":["partial","*",3,0],"Diffbot":["partial","*",3,0],"ContextFetchBot":["partial","*",3,0]},"llms":{"status":"missing","http":404,"bytes":0},"home":{"status":"http_error","http":403,"url":"https://www.bloomsbury.com/"}},
{"rank":2635,"domain":"ready.gov","origin":"https://ready.gov","robots":{"status":"refused","http":403,"url":"https://ready.gov/robots.txt","bytes":373,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":2636,"domain":"dafont.com","origin":"https://dafont.com","robots":{"status":"ok","http":200,"url":"https://www.dafont.com/robots.txt","bytes":1754,"groups":2},"bots":{"GPTBot":["partial","*",63,0],"ChatGPT-User":["partial","*",63,0],"OAI-SearchBot":["partial","*",63,0],"ClaudeBot":["partial","*",63,0],"Claude-User":["partial","*",63,0],"Claude-SearchBot":["partial","*",63,0],"anthropic-ai":["partial","*",63,0],"PerplexityBot":["partial","*",63,0],"Perplexity-User":["partial","*",63,0],"Google-Extended":["partial","*",63,0],"Applebot-Extended":["partial","*",63,0],"CCBot":["partial","*",63,0],"Bytespider":["partial","*",63,0],"Amazonbot":["partial","*",63,0],"meta-externalagent":["partial","*",63,0],"cohere-ai":["partial","*",63,0],"Diffbot":["partial","*",63,0],"ContextFetchBot":["partial","*",63,0]},"llms":{"status":"missing","http":404,"bytes":1417},"home":{"status":"ok","http":200,"url":"https://www.dafont.com/","html_tokens":4193,"markdown_tokens":328,"pruned_tokens":109}},
{"rank":2637,"domain":"united.com","origin":"https://united.com","robots":{"status":"unreachable","http":null,"url":"https://united.com/robots.txt","bytes":0,"groups":0,"error":"<StreamReset stream_id:1, error_code:2, remote_reset:True>"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":2638,"domain":"fiu.edu","origin":"https://fiu.edu","robots":{"status":"ok","http":200,"url":"https://www.fiu.edu/robots.txt","bytes":50,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":66766},"home":{"status":"ok","http":200,"url":"https://www.fiu.edu/","html_tokens":14863,"markdown_tokens":119,"pruned_tokens":119}},
{"rank":2639,"domain":"geocaching.com","origin":"https://geocaching.com","robots":{"status":"ok","http":200,"url":"https://www.geocaching.com/robots.txt","bytes":547,"groups":4},"bots":{"GPTBot":["partial","*",12,0],"ChatGPT-User":["partial","*",12,0],"OAI-SearchBot":["partial","*",12,0],"ClaudeBot":["partial","*",12,0],"Claude-User":["partial","*",12,0],"Claude-SearchBot":["partial","*",12,0],"anthropic-ai":["partial","*",12,0],"PerplexityBot":["partial","*",12,0],"Perplexity-User":["partial","*",12,0],"Google-Extended":["partial","*",12,0],"Applebot-Extended":["partial","*",12,0],"CCBot":["partial","*",12,0],"Bytespider":["partial","*",12,0],"Amazonbot":["partial","*",12,0],"meta-externalagent":["partial","*",12,0],"cohere-ai":["partial","*",12,0],"Diffbot":["partial","*",12,0],"ContextFetchBot":["partial","*",12,0]},"llms":{"status":"missing","http":404,"bytes":2288},"home":{"status":"ok","http":200,"url":"https://www.geocaching.com/play","html_tokens":7109,"markdown_tokens":115,"pruned_tokens":115}},
{"rank":2640,"domain":"fnac.com","origin":"https://fnac.com","robots":{"status":"refused","http":403,"url":"https://www.fnac.com/robots.txt","bytes":65479,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":2641,"domain":"commondreams.org","origin":"https://commondreams.org","robots":{"status":"ok","http":200,"url":"https://www.commondreams.org/robots.txt","bytes":338,"groups":1},"bots":{"GPTBot":["partial","*",3,0],"ChatGPT-User":["partial","*",3,0],"OAI-SearchBot":["partial","*",3,0],"ClaudeBot":["partial","*",3,0],"Claude-User":["partial","*",3,0],"Claude-SearchBot":["partial","*",3,0],"anthropic-ai":["partial","*",3,0],"PerplexityBot":["partial","*",3,0],"Perplexity-User":["partial","*",3,0],"Google-Extended":["partial","*",3,0],"Applebot-Extended":["partial","*",3,0],"CCBot":["partial","*",3,0],"Bytespider":["partial","*",3,0],"Amazonbot":["partial","*",3,0],"meta-externalagent":["partial","*",3,0],"cohere-ai":["partial","*",3,0],"Diffbot":["partial","*",3,0],"ContextFetchBot":["partial","*",3,0]},"llms":{"status":"present","http":200,"bytes":254188},"home":{"status":"ok","http":200,"url":"https://www.commondreams.org/","html_tokens":227673,"markdown_tokens":20580,"pruned_tokens":154}},
{"rank":2642,"domain":"keybase.io","origin":"https://keybase.io","robots":{"status":"ok","http":200,"url":"https://keybase.io/robots.txt","bytes":15,"groups":1},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"missing","http":404,"bytes":23},"home":{"status":"ok","http":200,"url":"https://keybase.io/","html_tokens":4758,"markdown_tokens":62,"pruned_tokens":29}},
{"rank":2643,"domain":"websitedemos.net","origin":"https://websitedemos.net","robots":{"status":"ok","http":200,"url":"https://websitedemos.net/robots.txt","bytes":6663,"groups":1},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"missing","http":404,"bytes":150362},"home":{"status":"ok","http":200,"url":"https://websitedemos.net/","html_tokens":111037,"markdown_tokens":231,"pruned_tokens":71}},
{"rank":2644,"domain":"blogspot.it","origin":"https://blogspot.it","robots":{"status":"ok","http":200,"url":"https://blogspot.it/robots.txt","bytes":6624,"groups":4},"bots":{"GPTBot":["partial","*",169,64],"ChatGPT-User":["partial","*",169,64],"OAI-SearchBot":["partial","*",169,64],"ClaudeBot":["partial","*",169,64],"Claude-User":["partial","*",169,64],"Claude-SearchBot":["partial","*",169,64],"anthropic-ai":["partial","*",169,64],"PerplexityBot":["partial","*",169,64],"Perplexity-User":["partial","*",169,64],"Google-Extended":["partial","*",169,64],"Applebot-Extended":["partial","*",169,64],"CCBot":["partial","*",169,64],"Bytespider":["partial","*",169,64],"Amazonbot":["partial","*",169,64],"meta-externalagent":["partial","*",169,64],"cohere-ai":["partial","*",169,64],"Diffbot":["partial","*",169,64],"ContextFetchBot":["partial","*",169,64]},"llms":{"status":"missing","http":404,"bytes":1569},"home":{"status":"ok","http":200,"url":"https://www.google.com/","html_tokens":21364,"markdown_tokens":30,"pruned_tokens":30}},
{"rank":2645,"domain":"interestingengineering.com","origin":"https://interestingengineering.com","robots":{"status":"ok","http":200,"url":"https://interestingengineering.com/robots.txt","bytes":2553,"groups":33},"bots":{"GPTBot":["partial","*",19,18],"ChatGPT-User":["partial","*",19,18],"OAI-SearchBot":["partial","*",19,18],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",19,18],"Claude-SearchBot":["partial","*",19,18],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["partial","*",19,18],"Perplexity-User":["partial","*",19,18],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",19,18]},"llms":{"status":"missing","http":404,"bytes":124048},"home":{"status":"ok","http":200,"url":"https://interestingengineering.com/","html_tokens":194827,"markdown_tokens":187,"pruned_tokens":58}},
{"rank":2646,"domain":"signupgenius.com","origin":"https://signupgenius.com","robots":{"status":"ok","http":200,"url":"https://www.signupgenius.com/robots.txt","bytes":1232,"groups":18},"bots":{"GPTBot":["partial","*",13,1],"ChatGPT-User":["partial","*",13,1],"OAI-SearchBot":["partial","*",13,1],"ClaudeBot":["partial","*",13,1],"Claude-User":["partial","*",13,1],"Claude-SearchBot":["partial","*",13,1],"anthropic-ai":["partial","*",13,1],"PerplexityBot":["partial","*",13,1],"Perplexity-User":["partial","*",13,1],"Google-Extended":["partial","*",13,1],"Applebot-Extended":["partial","*",13,1],"CCBot":["partial","*",13,1],"Bytespider":["partial","*",13,1],"Amazonbot":["partial","*",13,1],"meta-externalagent":["partial","*",13,1],"cohere-ai":["partial","*",13,1],"Diffbot":["partial","*",13,1],"ContextFetchBot":["partial","*",13,1]},"llms":{"status":"missing","http":404,"bytes":429},"home":{"status":"ok","http":200,"url":"https://www.signupgenius.com/","html_tokens":42623,"markdown_tokens":425,"pruned_tokens":287}},
{"rank":2647,"domain":"livestrong.com","origin":"https://livestrong.com","robots":{"status":"ok","http":200,"url":"https://www.everydayhealth.com/robots.txt","bytes":183249,"groups":4},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["blocked","named",1,0],"OAI-SearchBot":["blocked","named",1,0],"ClaudeBot":["partial","named",1,2],"Claude-User":["partial","named",1,2],"Claude-SearchBot":["partial","named",1,2],"anthropic-ai":["partial","named",1,2],"PerplexityBot":["partial","named",1,2],"Perplexity-User":["partial","named",1,2],"Google-Extended":["partial","named",2764,0],"Applebot-Extended":["partial","named",1,2],"CCBot":["partial","named",1,2],"Bytespider":["partial","named",1,2],"Amazonbot":["partial","named",1,2],"meta-externalagent":["partial","named",1,2],"cohere-ai":["partial","named",1,2],"Diffbot":["partial","named",1,2],"ContextFetchBot":["partial","*",14,0]},"llms":{"status":"html","http":200,"bytes":298132},"home":{"status":"disallowed","error":"robots.txt of livestrong.org not readable (refused)"}},
{"rank":2648,"domain":"genially.com","origin":"https://genially.com","robots":{"status":"ok","http":200,"url":"https://genially.com/robots.txt","bytes":68,"groups":1},"bots":{"GPTBot":["allowed","*",0,1],"ChatGPT-User":["allowed","*",0,1],"OAI-SearchBot":["allowed","*",0,1],"ClaudeBot":["allowed","*",0,1],"Claude-User":["allowed","*",0,1],"Claude-SearchBot":["allowed","*",0,1],"anthropic-ai":["allowed","*",0,1],"PerplexityBot":["allowed","*",0,1],"Perplexity-User":["allowed","*",0,1],"Google-Extended":["allowed","*",0,1],"Applebot-Extended":["allowed","*",0,1],"CCBot":["allowed","*",0,1],"Bytespider":["allowed","*",0,1],"Amazonbot":["allowed","*",0,1],"meta-externalagent":["allowed","*",0,1],"cohere-ai":["allowed","*",0,1],"Diffbot":["allowed","*",0,1],"ContextFetchBot":["allowed","*",0,1]},"llms":{"status":"present","http":200,"bytes":7501},"home":{"status":"ok","http":200,"url":"https://genially.com/","html_tokens":44736,"markdown_tokens":129,"pruned_tokens":39}},
{"rank":2649,"domain":"easyjet.com","origin":"https://easyjet.com","robots":{"status":"refused","http":403,"url":"https://easyjet.com/robots.txt","bytes":375,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":2650,"domain":"fdic.gov","origin":"https://fdic.gov","robots":{"status":"ok","http":200,"url":"https://www.fdic.gov/robots.txt","bytes":2173,"groups":1},"bots":{"GPTBot":["partial","*",36,18],"ChatGPT-User":["partial","*",36,18],"OAI-SearchBot":["partial","*",36,18],"ClaudeBot":["partial","*",36,18],"Claude-User":["partial","*",36,18],"Claude-SearchBot":["partial","*",36,18],"anthropic-ai":["partial","*",36,18],"PerplexityBot":["partial","*",36,18],"Perplexity-User":["partial","*",36,18],"Google-Extended":["partial","*",36,18],"Applebot-Extended":["partial","*",36,18],"CCBot":["partial","*",36,18],"Bytespider":["partial","*",36,18],"Amazonbot":["partial","*",36,18],"meta-externalagent":["partial","*",36,18],"cohere-ai":["partial","*",36,18],"Diffbot":["partial","*",36,18],"ContextFetchBot":["partial","*",36,18]},"llms":{"status":"missing","http":404,"bytes":186},"home":{"status":"ok","http":200,"url":"https://www.fdic.gov/","html_tokens":26284,"markdown_tokens":139,"pruned_tokens":126}},
{"rank":2651,"domain":"ctrip.com","origin":"https://ctrip.com","robots":{"status":"ok","http":200,"url":"https://www.ctrip.com/robots.txt","bytes":433,"groups":1},"bots":{"GPTBot":["partial","*",18,1],"ChatGPT-User":["partial","*",18,1],"OAI-SearchBot":["partial","*",18,1],"ClaudeBot":["partial","*",18,1],"Claude-User":["partial","*",18,1],"Claude-SearchBot":["partial","*",18,1],"anthropic-ai":["partial","*",18,1],"PerplexityBot":["partial","*",18,1],"Perplexity-User":["partial","*",18,1],"Google-Extended":["partial","*",18,1],"Applebot-Extended":["partial","*",18,1],"CCBot":["partial","*",18,1],"Bytespider":["partial","*",18,1],"Amazonbot":["partial","*",18,1],"meta-externalagent":["partial","*",18,1],"cohere-ai":["partial","*",18,1],"Diffbot":["partial","*",18,1],"ContextFetchBot":["partial","*",18,1]},"llms":{"status":"present","http":200,"bytes":8556},"home":{"status":"ok","http":200,"url":"https://www.ctrip.com/","html_tokens":71383,"markdown_tokens":9,"pruned_tokens":9}},
{"rank":2652,"domain":"mattel.com","origin":"https://mattel.com","robots":{"status":"ok","http":200,"url":"https://mattel.com/robots.txt","bytes":24,"groups":1},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"present","http":200,"bytes":14093},"home":{"status":"ok","http":200,"url":"https://about.mattel.com/","html_tokens":496968,"markdown_tokens":0,"pruned_tokens":0}},
{"rank":2653,"domain":"windowscentral.com","origin":"https://windowscentral.com","robots":{"status":"ok","http":200,"url":"https://www.windowscentral.com/robots.txt","bytes":1674,"groups":5},"bots":{"GPTBot":["partial","*",19,0],"ChatGPT-User":["partial","*",19,0],"OAI-SearchBot":["partial","*",19,0],"ClaudeBot":["partial","*",19,0],"Claude-User":["partial","*",19,0],"Claude-SearchBot":["partial","*",19,0],"anthropic-ai":["partial","*",19,0],"PerplexityBot":["partial","*",19,0],"Perplexity-User":["partial","*",19,0],"Google-Extended":["partial","*",19,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["partial","*",19,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["partial","*",19,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",19,0]},"llms":{"status":"missing","http":404,"bytes":1243865},"home":{"status":"ok","http":200,"url":"https://www.windowscentral.com/","html_tokens":493831,"markdown_tokens":78,"pruned_tokens":78}},
{"rank":2654,"domain":"psychiatry.org","origin":"https://psychiatry.org","robots":{"status":"ok","http":200,"url":"https://www.psychiatry.org/robots.txt","bytes":637,"groups":1},"bots":{"GPTBot":["partial","*",8,1],"ChatGPT-User":["partial","*",8,1],"OAI-SearchBot":["partial","*",8,1],"ClaudeBot":["partial","*",8,1],"Claude-User":["partial","*",8,1],"Claude-SearchBot":["partial","*",8,1],"anthropic-ai":["partial","*",8,1],"PerplexityBot":["partial","*",8,1],"Perplexity-User":["partial","*",8,1],"Google-Extended":["partial","*",8,1],"Applebot-Extended":["partial","*",8,1],"CCBot":["partial","*",8,1],"Bytespider":["partial","*",8,1],"Amazonbot":["partial","*",8,1],"meta-externalagent":["partial","*",8,1],"cohere-ai":["partial","*",8,1],"Diffbot":["partial","*",8,1],"ContextFetchBot":["partial","*",8,1]},"llms":{"status":"missing","http":404,"bytes":186339},"home":{"status":"ok","http":200,"url":"https://www.psychiatry.org/","html_tokens":51616,"markdown_tokens":307,"pruned_tokens":85}},
{"rank":2655,"domain":"tampabay.com","origin":"https://tampabay.com","robots":{"status":"ok","http":200,"url":"https://www.tampabay.com/robots.txt","bytes":362,"groups":10},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["allowed","*",0,1],"OAI-SearchBot":["allowed","*",0,1],"ClaudeBot":["blocked","named",1,0],"Claude-User":["allowed","*",0,1],"Claude-SearchBot":["allowed","*",0,1],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["allowed","*",0,1],"Perplexity-User":["allowed","*",0,1],"Google-Extended":["allowed","named",0,1],"Applebot-Extended":["allowed","*",0,1],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["allowed","*",0,1],"meta-externalagent":["allowed","*",0,1],"cohere-ai":["allowed","*",0,1],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["allowed","*",0,1]},"llms":{"status":"missing","http":404,"bytes":50699},"home":{"status":"ok","http":200,"url":"https://www.tampabay.com/","html_tokens":197603,"markdown_tokens":95,"pruned_tokens":57}},
{"rank":2656,"domain":"unh.edu","origin":"https://unh.edu","robots":{"status":"ok","http":200,"url":"https://www.unh.edu/robots.txt","bytes":2138,"groups":2},"bots":{"GPTBot":["partial","*",35,18],"ChatGPT-User":["partial","*",35,18],"OAI-SearchBot":["partial","*",35,18],"ClaudeBot":["partial","*",35,18],"Claude-User":["partial","*",35,18],"Claude-SearchBot":["partial","*",35,18],"anthropic-ai":["partial","*",35,18],"PerplexityBot":["partial","*",35,18],"Perplexity-User":["partial","*",35,18],"Google-Extended":["partial","*",35,18],"Applebot-Extended":["partial","*",35,18],"CCBot":["partial","*",35,18],"Bytespider":["partial","*",35,18],"Amazonbot":["partial","*",35,18],"meta-externalagent":["partial","*",35,18],"cohere-ai":["partial","*",35,18],"Diffbot":["partial","*",35,18],"ContextFetchBot":["partial","*",35,18]},"llms":{"status":"missing","http":404,"bytes":23759},"home":{"status":"ok","http":200,"url":"https://www.unh.edu/","html_tokens":60776,"markdown_tokens":112,"pruned_tokens":64}},
{"rank":2657,"domain":"il.us","origin":"https://il.us","robots":{"status":"unreachable","http":null,"url":"https://il.us/robots.txt","bytes":0,"groups":0,"error":"cannot resolve il.us"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":2658,"domain":"nationaltrust.org.uk","origin":"https://nationaltrust.org.uk","robots":{"status":"ok","http":200,"url":"https://www.nationaltrust.org.uk/robots.txt","bytes":731,"groups":1},"bots":{"GPTBot":["partial","*",13,0],"ChatGPT-User":["partial","*",13,0],"OAI-SearchBot":["partial","*",13,0],"ClaudeBot":["partial","*",13,0],"Claude-User":["partial","*",13,0],"Claude-SearchBot":["partial","*",13,0],"anthropic-ai":["partial","*",13,0],"PerplexityBot":["partial","*",13,0],"Perplexity-User":["partial","*",13,0],"Google-Extended":["partial","*",13,0],"Applebot-Extended":["partial","*",13,0],"CCBot":["partial","*",13,0],"Bytespider":["partial","*",13,0],"Amazonbot":["partial","*",13,0],"meta-externalagent":["partial","*",13,0],"cohere-ai":["partial","*",13,0],"Diffbot":["partial","*",13,0],"ContextFetchBot":["partial","*",13,0]},"llms":{"status":"disallowed","error":"robots.txt of static.nationaltrust.org.uk not readable (refused)"},"home":{"status":"ok","http":200,"url":"https://www.nationaltrust.org.uk/","html_tokens":158413,"markdown_tokens":213,"pruned_tokens":76}},
{"rank":2659,"domain":"xrea.com","origin":"https://xrea.com","robots":{"status":"missing","http":404,"url":"https://www.xrea.com/robots.txt","bytes":19914,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":19910},"home":{"status":"ok","http":200,"url":"https://www.xrea.com/","html_tokens":8395,"markdown_tokens":293,"pruned_tokens":43}},
{"rank":2660,"domain":"complex.com","origin":"https://complex.com","robots":{"status":"ok","http":200,"url":"https://www.complex.com/robots.txt","bytes":3766,"groups":22},"bots":{"GPTBot":["partial","*",53,3],"ChatGPT-User":["allowed","named",0,1],"OAI-SearchBot":["allowed","named",0,1],"ClaudeBot":["partial","*",53,3],"Claude-User":["partial","*",53,3],"Claude-SearchBot":["partial","*",53,3],"anthropic-ai":["partial","*",53,3],"PerplexityBot":["partial","*",53,3],"Perplexity-User":["partial","*",53,3],"Google-Extended":["partial","*",53,3],"Applebot-Extended":["partial","*",53,3],"CCBot":["partial","*",53,3],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["partial","*",53,3],"cohere-ai":["partial","*",53,3],"Diffbot":["partial","*",53,3],"ContextFetchBot":["partial","*",53,3]},"llms":{"status":"present","http":200,"bytes":483185},"home":{"status":"ok","http":200,"url":"https://www.complex.com/","html_tokens":166516,"markdown_tokens":107,"pruned_tokens":22}},
{"rank":2661,"domain":"thenounproject.com","origin":"https://thenounproject.com","robots":{"status":"ok","http":200,"url":"https://thenounproject.com/robots.txt","bytes":301,"groups":1},"bots":{"GPTBot":["partial","*",3,0],"ChatGPT-User":["partial","*",3,0],"OAI-SearchBot":["partial","*",3,0],"ClaudeBot":["partial","*",3,0],"Claude-User":["partial","*",3,0],"Claude-SearchBot":["partial","*",3,0],"anthropic-ai":["partial","*",3,0],"PerplexityBot":["partial","*",3,0],"Perplexity-User":["partial","*",3,0],"Google-Extended":["partial","*",3,0],"Applebot-Extended":["partial","*",3,0],"CCBot":["partial","*",3,0],"Bytespider":["partial","*",3,0],"Amazonbot":["partial","*",3,0],"meta-externalagent":["partial","*",3,0],"cohere-ai":["partial","*",3,0],"Diffbot":["partial","*",3,0],"ContextFetchBot":["partial","*",3,0]},"llms":{"status":"present","http":200,"bytes":33270},"home":{"status":"ok","http":200,"url":"https://thenounproject.com/","html_tokens":61859,"markdown_tokens":459,"pruned_tokens":154}},
{"rank":2662,"domain":"seesaa.net","origin":"https://seesaa.net","robots":{"status":"ok","http":200,"url":"https://seesaa.net/robots.txt","bytes":462,"groups":5},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["partial","*",8,0],"OAI-SearchBot":["partial","*",8,0],"ClaudeBot":["partial","*",8,0],"Claude-User":["partial","*",8,0],"Claude-SearchBot":["partial","*",8,0],"anthropic-ai":["partial","*",8,0],"PerplexityBot":["partial","*",8,0],"Perplexity-User":["partial","*",8,0],"Google-Extended":["partial","*",8,0],"Applebot-Extended":["partial","*",8,0],"CCBot":["partial","*",8,0],"Bytespider":["partial","*",8,0],"Amazonbot":["partial","*",8,0],"meta-externalagent":["partial","*",8,0],"cohere-ai":["partial","*",8,0],"Diffbot":["partial","*",8,0],"ContextFetchBot":["partial","*",8,0]},"llms":{"status":"html","http":200,"bytes":32367},"home":{"status":"ok","http":200,"url":"https://blog.seesaa.jp","html_tokens":7183,"markdown_tokens":116,"pruned_tokens":116}},
{"rank":2663,"domain":"datareportal.com","origin":"https://datareportal.com","robots":{"status":"ok","http":200,"url":"https://datareportal.com/robots.txt","bytes":1507,"groups":1},"bots":{"GPTBot":["partial","named",27,1],"ChatGPT-User":["partial","*",27,1],"OAI-SearchBot":["partial","*",27,1],"ClaudeBot":["partial","named",27,1],"Claude-User":["partial","*",27,1],"Claude-SearchBot":["partial","*",27,1],"anthropic-ai":["partial","named",27,1],"PerplexityBot":["partial","*",27,1],"Perplexity-User":["partial","*",27,1],"Google-Extended":["partial","named",27,1],"Applebot-Extended":["partial","named",27,1],"CCBot":["partial","named",27,1],"Bytespider":["partial","named",27,1],"Amazonbot":["partial","named",27,1],"meta-externalagent":["partial","named",27,1],"cohere-ai":["partial","named",27,1],"Diffbot":["partial","*",27,1],"ContextFetchBot":["partial","*",27,1]},"llms":{"status":"missing","http":404,"bytes":0},"home":{"status":"ok","http":200,"url":"https://datareportal.com/","html_tokens":46914,"markdown_tokens":842,"pruned_tokens":162}},
{"rank":2664,"domain":"physiology.org","origin":"https://physiology.org","robots":{"status":"ok","http":200,"url":"https://www.physiology.org/robots.txt","bytes":391,"groups":1},"bots":{"GPTBot":["partial","*",4,0],"ChatGPT-User":["partial","*",4,0],"OAI-SearchBot":["partial","*",4,0],"ClaudeBot":["partial","*",4,0],"Claude-User":["partial","*",4,0],"Claude-SearchBot":["partial","*",4,0],"anthropic-ai":["partial","*",4,0],"PerplexityBot":["partial","*",4,0],"Perplexity-User":["partial","*",4,0],"Google-Extended":["partial","*",4,0],"Applebot-Extended":["partial","*",4,0],"CCBot":["partial","*",4,0],"Bytespider":["partial","*",4,0],"Amazonbot":["partial","*",4,0],"meta-externalagent":["partial","*",4,0],"cohere-ai":["partial","*",4,0],"Diffbot":["partial","*",4,0],"ContextFetchBot":["partial","*",4,0]},"llms":{"status":"crawl_delay","error":"Crawl-delay 120 s"},"home":{"status":"crawl_delay","error":"Crawl-delay 120 s"}},
{"rank":2665,"domain":"unfpa.org","origin":"https://unfpa.org","robots":{"status":"refused","http":403,"url":"https://unfpa.org/robots.txt","bytes":560,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":2666,"domain":"war.gov","origin":"https://war.gov","robots":{"status":"refused","http":403,"url":"https://war.gov/robots.txt","bytes":369,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":2667,"domain":"ndrc.gov.cn","origin":"https://www.ndrc.gov.cn","robots":{"status":"refused","http":403,"url":"https://www.ndrc.gov.cn/robots.txt","bytes":144,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":2668,"domain":"bangkokpost.com","origin":"https://bangkokpost.com","robots":{"status":"ok","http":200,"url":"https://www.bangkokpost.com/robots.txt","bytes":539,"groups":3},"bots":{"GPTBot":["partial","*",16,0],"ChatGPT-User":["partial","*",16,0],"OAI-SearchBot":["partial","*",16,0],"ClaudeBot":["partial","*",16,0],"Claude-User":["partial","*",16,0],"Claude-SearchBot":["partial","*",16,0],"anthropic-ai":["partial","*",16,0],"PerplexityBot":["partial","*",16,0],"Perplexity-User":["partial","*",16,0],"Google-Extended":["partial","*",16,0],"Applebot-Extended":["partial","*",16,0],"CCBot":["partial","*",16,0],"Bytespider":["partial","*",16,0],"Amazonbot":["partial","*",16,0],"meta-externalagent":["partial","*",16,0],"cohere-ai":["partial","*",16,0],"Diffbot":["partial","*",16,0],"ContextFetchBot":["partial","*",16,0]},"llms":{"status":"missing","http":404,"bytes":58394},"home":{"status":"ok","http":200,"url":"https://www.bangkokpost.com/","html_tokens":50834,"markdown_tokens":16,"pruned_tokens":16}},
{"rank":2669,"domain":"icondrawer.com","origin":"https://icondrawer.com","robots":{"status":"unreachable","http":null,"url":"https://icondrawer.com/robots.txt","bytes":0,"groups":0,"error":"cannot resolve icondrawer.com"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":2670,"domain":"ada.gov","origin":"https://ada.gov","robots":{"status":"html","http":200,"url":"https://www.ada.gov/","bytes":63831,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":2671,"domain":"kqed.org","origin":"https://kqed.org","robots":{"status":"ok","http":200,"url":"https://www.kqed.org/robots.txt","bytes":131,"groups":2},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","named",0,1],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":3852},"home":{"status":"ok","http":200,"url":"https://www.kqed.org/","html_tokens":166928,"markdown_tokens":14,"pruned_tokens":14}},
{"rank":2672,"domain":"ucar.edu","origin":"https://ucar.edu","robots":{"status":"refused","http":403,"url":"https://ucar.edu/robots.txt","bytes":560,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":2673,"domain":"delaware.gov","origin":"https://delaware.gov","robots":{"status":"html","http":200,"url":"https://delaware.gov/robots.txt","bytes":244,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":2674,"domain":"kth.se","origin":"https://kth.se","robots":{"status":"ok","http":200,"url":"https://www.kth.se/robots.txt","bytes":980,"groups":1},"bots":{"GPTBot":["partial","*",38,0],"ChatGPT-User":["partial","*",38,0],"OAI-SearchBot":["partial","*",38,0],"ClaudeBot":["partial","*",38,0],"Claude-User":["partial","*",38,0],"Claude-SearchBot":["partial","*",38,0],"anthropic-ai":["partial","*",38,0],"PerplexityBot":["partial","*",38,0],"Perplexity-User":["partial","*",38,0],"Google-Extended":["partial","*",38,0],"Applebot-Extended":["partial","*",38,0],"CCBot":["partial","*",38,0],"Bytespider":["partial","*",38,0],"Amazonbot":["partial","*",38,0],"meta-externalagent":["partial","*",38,0],"cohere-ai":["partial","*",38,0],"Diffbot":["partial","*",38,0],"ContextFetchBot":["partial","*",38,0]},"llms":{"status":"missing","http":404,"bytes":13070},"home":{"status":"ok","http":200,"url":"https://www.kth.se/","html_tokens":16898,"markdown_tokens":201,"pruned_tokens":68}},
{"rank":2675,"domain":"pinterest.fr","origin":"https://pinterest.fr","robots":{"status":"ok","http":200,"url":"https://fr.pinterest.com/robots.txt","bytes":34476,"groups":8},"bots":{"GPTBot":["blocked","*",1,0],"ChatGPT-User":["blocked","*",1,0],"OAI-SearchBot":["blocked","*",1,0],"ClaudeBot":["blocked","*",1,0],"Claude-User":["blocked","*",1,0],"Claude-SearchBot":["blocked","*",1,0],"anthropic-ai":["blocked","*",1,0],"PerplexityBot":["blocked","*",1,0],"Perplexity-User":["blocked","*",1,0],"Google-Extended":["blocked","*",1,0],"Applebot-Extended":["blocked","*",1,0],"CCBot":["blocked","*",1,0],"Bytespider":["blocked","*",1,0],"Amazonbot":["blocked","*",1,0],"meta-externalagent":["blocked","*",1,0],"cohere-ai":["blocked","*",1,0],"Diffbot":["blocked","*",1,0],"ContextFetchBot":["blocked","*",1,0]},"llms":{"status":"disallowed"},"home":{"status":"disallowed"}},
{"rank":2676,"domain":"umeng.com","origin":"https://www.umeng.com","robots":{"status":"ok","http":200,"url":"https://www.umeng.com/robots.txt","bytes":1625,"groups":26},"bots":{"GPTBot":["allowed","named",0,1],"ChatGPT-User":["allowed","named",0,1],"OAI-SearchBot":["allowed","named",0,1],"ClaudeBot":["allowed","named",0,1],"Claude-User":["allowed","*",0,1],"Claude-SearchBot":["allowed","*",0,1],"anthropic-ai":["allowed","named",0,1],"PerplexityBot":["allowed","named",0,1],"Perplexity-User":["allowed","*",0,1],"Google-Extended":["allowed","named",0,1],"Applebot-Extended":["allowed","*",0,1],"CCBot":["allowed","*",0,1],"Bytespider":["allowed","named",0,1],"Amazonbot":["allowed","*",0,1],"meta-externalagent":["allowed","*",0,1],"cohere-ai":["allowed","*",0,1],"Diffbot":["allowed","*",0,1],"ContextFetchBot":["allowed","*",0,1]},"llms":{"status":"present","http":200,"bytes":5158},"home":{"status":"ok","http":200,"url":"https://www.umeng.com/","html_tokens":159777,"markdown_tokens":589,"pruned_tokens":115}},
{"rank":2677,"domain":"paramountplus.com","origin":"https://paramountplus.com","robots":{"status":"ok","http":200,"url":"https://www.paramountplus.com/robots.txt","bytes":2120,"groups":1},"bots":{"GPTBot":["partial","*",54,5],"ChatGPT-User":["partial","*",54,5],"OAI-SearchBot":["partial","*",54,5],"ClaudeBot":["partial","*",54,5],"Claude-User":["partial","*",54,5],"Claude-SearchBot":["partial","*",54,5],"anthropic-ai":["partial","*",54,5],"PerplexityBot":["partial","*",54,5],"Perplexity-User":["partial","*",54,5],"Google-Extended":["partial","*",54,5],"Applebot-Extended":["partial","*",54,5],"CCBot":["partial","*",54,5],"Bytespider":["partial","*",54,5],"Amazonbot":["partial","*",54,5],"meta-externalagent":["partial","*",54,5],"cohere-ai":["partial","*",54,5],"Diffbot":["partial","*",54,5],"ContextFetchBot":["partial","*",54,5]},"llms":{"status":"missing","http":404,"bytes":3},"home":{"status":"ok","http":200,"url":"https://www.paramountplus.com","html_tokens":122711,"markdown_tokens":113,"pruned_tokens":113}},
{"rank":2678,"domain":"bjx.com.cn","origin":"https://bjx.com.cn","robots":{"status":"ok","http":200,"url":"https://www.bjx.com.cn/robots.txt","bytes":30,"groups":1},"bots":{"GPTBot":["partial","*",1,0],"ChatGPT-User":["partial","*",1,0],"OAI-SearchBot":["partial","*",1,0],"ClaudeBot":["partial","*",1,0],"Claude-User":["partial","*",1,0],"Claude-SearchBot":["partial","*",1,0],"anthropic-ai":["partial","*",1,0],"PerplexityBot":["partial","*",1,0],"Perplexity-User":["partial","*",1,0],"Google-Extended":["partial","*",1,0],"Applebot-Extended":["partial","*",1,0],"CCBot":["partial","*",1,0],"Bytespider":["partial","*",1,0],"Amazonbot":["partial","*",1,0],"meta-externalagent":["partial","*",1,0],"cohere-ai":["partial","*",1,0],"Diffbot":["partial","*",1,0],"ContextFetchBot":["partial","*",1,0]},"llms":{"status":"missing","http":404,"bytes":21833},"home":{"status":"ok","http":200,"url":"https://www.bjx.com.cn/","html_tokens":39007,"markdown_tokens":42,"pruned_tokens":42}},
{"rank":2679,"domain":"jsonline.com","origin":"https://jsonline.com","robots":{"status":"ok","http":200,"url":"https://www.jsonline.com/robots.txt","bytes":12835,"groups":288},"bots":{"GPTBot":["partial","named",1,4],"ChatGPT-User":["partial","named",1,4],"OAI-SearchBot":["partial","named",1,4],"ClaudeBot":["blocked","named",1,0],"Claude-User":["blocked","named",1,0],"Claude-SearchBot":["blocked","named",1,0],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["partial","named",1,4],"Perplexity-User":["blocked","named",1,0],"Google-Extended":["partial","named",1,4],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",2,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",34,0]},"llms":{"status":"http_error","http":402,"bytes":254},"home":{"status":"http_error","http":402,"url":"https://www.jsonline.com/"}},
{"rank":2680,"domain":"orange.fr","origin":"https://orange.fr","robots":{"status":"ok","http":200,"url":"https://www.orange.fr/robots.txt","bytes":66,"groups":1},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"html","http":200,"bytes":1193},"home":{"status":"ok","http":200,"url":"https://www.orange.fr/portail","html_tokens":299,"markdown_tokens":0,"pruned_tokens":0}},
{"rank":2681,"domain":"cbr.ru","origin":"https://cbr.ru","robots":{"status":"ok","http":200,"url":"https://cbr.ru/robots.txt","bytes":300,"groups":1},"bots":{"GPTBot":["partial","*",5,0],"ChatGPT-User":["partial","*",5,0],"OAI-SearchBot":["partial","*",5,0],"ClaudeBot":["partial","*",5,0],"Claude-User":["partial","*",5,0],"Claude-SearchBot":["partial","*",5,0],"anthropic-ai":["partial","*",5,0],"PerplexityBot":["partial","*",5,0],"Perplexity-User":["partial","*",5,0],"Google-Extended":["partial","*",5,0],"Applebot-Extended":["partial","*",5,0],"CCBot":["partial","*",5,0],"Bytespider":["partial","*",5,0],"Amazonbot":["partial","*",5,0],"meta-externalagent":["partial","*",5,0],"cohere-ai":["partial","*",5,0],"Diffbot":["partial","*",5,0],"ContextFetchBot":["partial","*",5,0]},"llms":{"status":"missing","http":404,"bytes":11101},"home":{"status":"ok","http":200,"url":"https://cbr.ru/","html_tokens":17236,"markdown_tokens":57,"pruned_tokens":57}},
{"rank":2682,"domain":"google.com.tr","origin":"https://google.com.tr","robots":{"status":"ok","http":200,"url":"https://www.google.com.tr/robots.txt","bytes":6624,"groups":4},"bots":{"GPTBot":["partial","*",169,64],"ChatGPT-User":["partial","*",169,64],"OAI-SearchBot":["partial","*",169,64],"ClaudeBot":["partial","*",169,64],"Claude-User":["partial","*",169,64],"Claude-SearchBot":["partial","*",169,64],"anthropic-ai":["partial","*",169,64],"PerplexityBot":["partial","*",169,64],"Perplexity-User":["partial","*",169,64],"Google-Extended":["partial","*",169,64],"Applebot-Extended":["partial","*",169,64],"CCBot":["partial","*",169,64],"Bytespider":["partial","*",169,64],"Amazonbot":["partial","*",169,64],"meta-externalagent":["partial","*",169,64],"cohere-ai":["partial","*",169,64],"Diffbot":["partial","*",169,64],"ContextFetchBot":["partial","*",169,64]},"llms":{"status":"missing","http":404,"bytes":1569},"home":{"status":"ok","http":200,"url":"https://www.google.com/","html_tokens":21323,"markdown_tokens":30,"pruned_tokens":30}},
{"rank":2683,"domain":"example.org","origin":"https://example.org","robots":{"status":"missing","http":404,"url":"https://example.org/robots.txt","bytes":577,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":577},"home":{"status":"ok","http":200,"url":"https://example.org/","html_tokens":145,"markdown_tokens":39,"pruned_tokens":39}},
{"rank":2684,"domain":"chinatax.gov.cn","origin":"https://www.chinatax.gov.cn","robots":{"status":"refused","http":403,"url":"https://www.chinatax.gov.cn/robots.txt","bytes":6688,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":2685,"domain":"wikileaks.org","origin":"https://wikileaks.org","robots":{"status":"ok","http":200,"url":"https://wikileaks.org/robots.txt","bytes":228,"groups":1},"bots":{"GPTBot":["partial","*",6,0],"ChatGPT-User":["partial","*",6,0],"OAI-SearchBot":["partial","*",6,0],"ClaudeBot":["partial","*",6,0],"Claude-User":["partial","*",6,0],"Claude-SearchBot":["partial","*",6,0],"anthropic-ai":["partial","*",6,0],"PerplexityBot":["partial","*",6,0],"Perplexity-User":["partial","*",6,0],"Google-Extended":["partial","*",6,0],"Applebot-Extended":["partial","*",6,0],"CCBot":["partial","*",6,0],"Bytespider":["partial","*",6,0],"Amazonbot":["partial","*",6,0],"meta-externalagent":["partial","*",6,0],"cohere-ai":["partial","*",6,0],"Diffbot":["partial","*",6,0],"ContextFetchBot":["partial","*",6,0]},"llms":{"status":"missing","http":404,"bytes":655},"home":{"status":"ok","http":200,"url":"https://wikileaks.org/","html_tokens":7087,"markdown_tokens":2566,"pruned_tokens":349}},
{"rank":2686,"domain":"hotmail.com","origin":"https://hotmail.com","robots":{"status":"html","http":200,"url":"https://outlook.live.com/mail/","bytes":14277,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":2687,"domain":"headspace.com","origin":"https://headspace.com","robots":{"status":"ok","http":200,"url":"https://www.headspace.com/robots.txt","bytes":675,"groups":2},"bots":{"GPTBot":["partial","*",27,0],"ChatGPT-User":["partial","*",27,0],"OAI-SearchBot":["partial","*",27,0],"ClaudeBot":["partial","*",27,0],"Claude-User":["partial","*",27,0],"Claude-SearchBot":["partial","*",27,0],"anthropic-ai":["partial","*",27,0],"PerplexityBot":["partial","*",27,0],"Perplexity-User":["partial","*",27,0],"Google-Extended":["partial","*",27,0],"Applebot-Extended":["partial","*",27,0],"CCBot":["partial","*",27,0],"Bytespider":["partial","*",27,0],"Amazonbot":["partial","*",27,0],"meta-externalagent":["partial","*",27,0],"cohere-ai":["partial","*",27,0],"Diffbot":["partial","*",27,0],"ContextFetchBot":["partial","*",27,0]},"llms":{"status":"present","http":200,"bytes":9716},"home":{"status":"ok","http":200,"url":"https://www.headspace.com/","html_tokens":137605,"markdown_tokens":478,"pruned_tokens":274}},
{"rank":2688,"domain":"wyndhamhotels.com","origin":"https://wyndhamhotels.com","robots":{"status":"ok","http":200,"url":"https://www.wyndhamhotels.com/robots.txt","bytes":1680,"groups":5},"bots":{"GPTBot":["partial","*",30,1],"ChatGPT-User":["partial","*",30,1],"OAI-SearchBot":["partial","*",30,1],"ClaudeBot":["partial","*",30,1],"Claude-User":["partial","*",30,1],"Claude-SearchBot":["partial","*",30,1],"anthropic-ai":["partial","*",30,1],"PerplexityBot":["partial","*",30,1],"Perplexity-User":["partial","*",30,1],"Google-Extended":["partial","*",30,1],"Applebot-Extended":["partial","*",30,1],"CCBot":["partial","*",30,1],"Bytespider":["partial","*",30,1],"Amazonbot":["partial","*",30,1],"meta-externalagent":["partial","*",30,1],"cohere-ai":["partial","*",30,1],"Diffbot":["partial","*",30,1],"ContextFetchBot":["partial","*",30,1]},"llms":{"status":"present","http":200,"bytes":15933},"home":{"status":"ok","http":200,"url":"https://www.wyndhamhotels.com/","html_tokens":98156,"markdown_tokens":1352,"pruned_tokens":89}},
{"rank":2689,"domain":"linux.org","origin":"https://linux.org","robots":{"status":"ok","http":200,"url":"https://www.linux.org/robots.txt","bytes":338,"groups":1},"bots":{"GPTBot":["partial","*",13,1],"ChatGPT-User":["partial","*",13,1],"OAI-SearchBot":["partial","*",13,1],"ClaudeBot":["partial","*",13,1],"Claude-User":["partial","*",13,1],"Claude-SearchBot":["partial","*",13,1],"anthropic-ai":["partial","*",13,1],"PerplexityBot":["partial","*",13,1],"Perplexity-User":["partial","*",13,1],"Google-Extended":["partial","*",13,1],"Applebot-Extended":["partial","*",13,1],"CCBot":["partial","*",13,1],"Bytespider":["partial","*",13,1],"Amazonbot":["partial","*",13,1],"meta-externalagent":["partial","*",13,1],"cohere-ai":["partial","*",13,1],"Diffbot":["partial","*",13,1],"ContextFetchBot":["partial","*",13,1]},"llms":{"status":"missing","http":404,"bytes":31832},"home":{"status":"http_error","http":403,"url":"https://linux.org/"}},
{"rank":2690,"domain":"softbank.jp","origin":"https://softbank.jp","robots":{"status":"html","http":200,"url":"https://www.softbank.jp/","bytes":54297,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":2691,"domain":"jpn.com","origin":"https://jpn.com","robots":{"status":"ok","http":200,"url":"https://jpn.com/robots.txt","bytes":0,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":8025},"home":{"status":"ok","http":200,"url":"https://jpn.com/","html_tokens":10255,"markdown_tokens":880,"pruned_tokens":864}},
{"rank":2692,"domain":"uni-stuttgart.de","origin":"https://uni-stuttgart.de","robots":{"status":"ok","http":200,"url":"https://www.uni-stuttgart.de/robots.txt","bytes":2116,"groups":7},"bots":{"GPTBot":["partial","*",42,0],"ChatGPT-User":["partial","*",42,0],"OAI-SearchBot":["partial","*",42,0],"ClaudeBot":["partial","*",42,0],"Claude-User":["partial","*",42,0],"Claude-SearchBot":["partial","*",42,0],"anthropic-ai":["partial","*",42,0],"PerplexityBot":["partial","*",42,0],"Perplexity-User":["partial","*",42,0],"Google-Extended":["partial","*",42,0],"Applebot-Extended":["partial","*",42,0],"CCBot":["partial","*",42,0],"Bytespider":["partial","*",42,0],"Amazonbot":["partial","*",42,0],"meta-externalagent":["partial","*",42,0],"cohere-ai":["partial","*",42,0],"Diffbot":["partial","*",42,0],"ContextFetchBot":["partial","*",42,0]},"llms":{"status":"missing","http":404,"bytes":61002},"home":{"status":"ok","http":200,"url":"https://www.uni-stuttgart.de","html_tokens":39870,"markdown_tokens":23,"pruned_tokens":23}},
{"rank":2693,"domain":"20minutes.fr","origin":"https://20minutes.fr","robots":{"status":"ok","http":200,"url":"https://www.20minutes.fr/robots.txt","bytes":2945,"groups":4},"bots":{"GPTBot":["partial","named",1,10],"ChatGPT-User":["partial","named",1,9],"OAI-SearchBot":["partial","named",1,9],"ClaudeBot":["partial","named",1,10],"Claude-User":["partial","named",1,9],"Claude-SearchBot":["partial","named",1,9],"anthropic-ai":["partial","named",1,10],"PerplexityBot":["partial","named",1,10],"Perplexity-User":["partial","named",1,9],"Google-Extended":["partial","named",2,19],"Applebot-Extended":["partial","named",1,10],"CCBot":["partial","named",1,10],"Bytespider":["partial","*",5,0],"Amazonbot":["partial","*",5,0],"meta-externalagent":["partial","*",5,0],"cohere-ai":["partial","named",1,10],"Diffbot":["partial","*",5,0],"ContextFetchBot":["partial","*",5,0]},"llms":{"status":"missing","http":404,"bytes":430666},"home":{"status":"ok","http":200,"url":"https://www.20minutes.fr/","html_tokens":197764,"markdown_tokens":200,"pruned_tokens":154}},
{"rank":2694,"domain":"lastampa.it","origin":"https://lastampa.it","robots":{"status":"ok","http":200,"url":"https://lastampa.it/robots.txt","bytes":4052,"groups":101},"bots":{"GPTBot":["partial","*",1,0],"ChatGPT-User":["partial","*",1,0],"OAI-SearchBot":["partial","*",1,0],"ClaudeBot":["partial","*",1,0],"Claude-User":["partial","*",1,0],"Claude-SearchBot":["partial","*",1,0],"anthropic-ai":["partial","*",1,0],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["partial","*",1,0],"Google-Extended":["partial","*",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["partial","*",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["allowed","named",0,1],"meta-externalagent":["partial","*",1,0],"cohere-ai":["partial","*",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",1,0]},"llms":{"status":"missing","http":404,"bytes":64148},"home":{"status":"ok","http":200,"url":"https://www.lastampa.it/","html_tokens":133074,"markdown_tokens":28,"pruned_tokens":28}},
{"rank":2695,"domain":"computer.org","origin":"https://computer.org","robots":{"status":"ok","http":200,"url":"https://www.computer.org/robots.txt","bytes":823,"groups":1},"bots":{"GPTBot":["partial","*",29,2],"ChatGPT-User":["partial","*",29,2],"OAI-SearchBot":["partial","*",29,2],"ClaudeBot":["partial","*",29,2],"Claude-User":["partial","*",29,2],"Claude-SearchBot":["partial","*",29,2],"anthropic-ai":["partial","*",29,2],"PerplexityBot":["partial","*",29,2],"Perplexity-User":["partial","*",29,2],"Google-Extended":["partial","*",29,2],"Applebot-Extended":["partial","*",29,2],"CCBot":["partial","*",29,2],"Bytespider":["partial","*",29,2],"Amazonbot":["partial","*",29,2],"meta-externalagent":["partial","*",29,2],"cohere-ai":["partial","*",29,2],"Diffbot":["partial","*",29,2],"ContextFetchBot":["partial","*",29,2]},"llms":{"status":"present","http":200,"bytes":12724},"home":{"status":"ok","http":200,"url":"https://www.computer.org/","html_tokens":143820,"markdown_tokens":589,"pruned_tokens":175}},
{"rank":2696,"domain":"metacritic.com","origin":"https://metacritic.com","robots":{"status":"ok","http":200,"url":"https://www.metacritic.com/robots.txt","bytes":1105,"groups":15},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["partial","*",7,0],"OAI-SearchBot":["blocked","named",1,0],"ClaudeBot":["partial","*",7,0],"Claude-User":["partial","*",7,0],"Claude-SearchBot":["partial","*",7,0],"anthropic-ai":["partial","*",7,0],"PerplexityBot":["partial","*",7,0],"Perplexity-User":["partial","*",7,0],"Google-Extended":["partial","*",7,0],"Applebot-Extended":["partial","*",7,0],"CCBot":["blocked","named",1,0],"Bytespider":["partial","*",7,0],"Amazonbot":["partial","*",7,0],"meta-externalagent":["partial","*",7,0],"cohere-ai":["partial","*",7,0],"Diffbot":["partial","*",7,0],"ContextFetchBot":["partial","*",7,0]},"llms":{"status":"missing","http":404,"bytes":135804},"home":{"status":"ok","http":200,"url":"https://www.metacritic.com/","html_tokens":364321,"markdown_tokens":77,"pruned_tokens":77}},
{"rank":2697,"domain":"uvm.edu","origin":"https://uvm.edu","robots":{"status":"ok","http":200,"url":"https://www.uvm.edu/robots.txt","bytes":2108,"groups":1},"bots":{"GPTBot":["partial","*",36,19],"ChatGPT-User":["partial","*",36,19],"OAI-SearchBot":["partial","*",36,19],"ClaudeBot":["partial","*",36,19],"Claude-User":["partial","*",36,19],"Claude-SearchBot":["partial","*",36,19],"anthropic-ai":["partial","*",36,19],"PerplexityBot":["partial","*",36,19],"Perplexity-User":["partial","*",36,19],"Google-Extended":["partial","*",36,19],"Applebot-Extended":["partial","*",36,19],"CCBot":["partial","*",36,19],"Bytespider":["partial","*",36,19],"Amazonbot":["partial","*",36,19],"meta-externalagent":["partial","*",36,19],"cohere-ai":["partial","*",36,19],"Diffbot":["partial","*",36,19],"ContextFetchBot":["partial","*",36,19]},"llms":{"status":"disallowed","error":"robots.txt of www.uvm.edu disallows /search/google?keys=/llms.txt"},"home":{"status":"ok","http":200,"url":"https://www.uvm.edu/","html_tokens":38118,"markdown_tokens":90,"pruned_tokens":90}},
{"rank":2698,"domain":"getresponse.com","origin":"https://getresponse.com","robots":{"status":"ok","http":200,"url":"https://www.getresponse.com/robots.txt","bytes":6152,"groups":1},"bots":{"GPTBot":["partial","*",145,7],"ChatGPT-User":["partial","*",145,7],"OAI-SearchBot":["partial","*",145,7],"ClaudeBot":["partial","*",145,7],"Claude-User":["partial","*",145,7],"Claude-SearchBot":["partial","*",145,7],"anthropic-ai":["partial","*",145,7],"PerplexityBot":["partial","*",145,7],"Perplexity-User":["partial","*",145,7],"Google-Extended":["partial","*",145,7],"Applebot-Extended":["partial","*",145,7],"CCBot":["partial","*",145,7],"Bytespider":["partial","*",145,7],"Amazonbot":["partial","*",145,7],"meta-externalagent":["partial","*",145,7],"cohere-ai":["partial","*",145,7],"Diffbot":["partial","*",145,7],"ContextFetchBot":["partial","*",145,7]},"llms":{"status":"present","http":200,"bytes":15105},"home":{"status":"not_html","error":"unsupported content type text/markdown"}},
{"rank":2699,"domain":"mendeley.com","origin":"https://mendeley.com","robots":{"status":"refused","http":403,"url":"https://www.mendeley.com/robots.txt","bytes":4906,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":2700,"domain":"christies.com","origin":"https://christies.com","robots":{"status":"timeout","http":null,"url":"https://christies.com/robots.txt","bytes":0,"groups":0,"error":"timed out after 10 s"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":2701,"domain":"tvtropes.org","origin":"https://tvtropes.org","robots":{"status":"refused","http":403,"url":"https://tvtropes.org/robots.txt","bytes":5485,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":2702,"domain":"youdao.com","origin":"https://youdao.com","robots":{"status":"html","http":200,"url":"https://www.youdao.com/","bytes":294296,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":2703,"domain":"brainyquote.com","origin":"https://brainyquote.com","robots":{"status":"ok","http":200,"url":"https://www.brainyquote.com/robots.txt","bytes":5535,"groups":12},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["partial","named",3,1],"OAI-SearchBot":["partial","named",3,1],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","named",3,1],"Claude-SearchBot":["partial","named",3,1],"anthropic-ai":["partial","*",3,1],"PerplexityBot":["partial","named",3,1],"Perplexity-User":["partial","named",3,1],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["partial","*",3,1],"Diffbot":["partial","*",3,1],"ContextFetchBot":["partial","*",3,1]},"llms":{"status":"missing","http":404,"bytes":9},"home":{"status":"http_error","http":403,"url":"https://brainyquote.com/"}},
{"rank":2704,"domain":"bt.com","origin":"https://bt.com","robots":{"status":"ok","http":200,"url":"https://www.bt.com/robots.txt","bytes":574,"groups":1},"bots":{"GPTBot":["partial","*",22,1],"ChatGPT-User":["partial","*",22,1],"OAI-SearchBot":["partial","*",22,1],"ClaudeBot":["partial","*",22,1],"Claude-User":["partial","*",22,1],"Claude-SearchBot":["partial","*",22,1],"anthropic-ai":["partial","*",22,1],"PerplexityBot":["partial","*",22,1],"Perplexity-User":["partial","*",22,1],"Google-Extended":["partial","*",22,1],"Applebot-Extended":["partial","*",22,1],"CCBot":["partial","*",22,1],"Bytespider":["partial","*",22,1],"Amazonbot":["partial","*",22,1],"meta-externalagent":["partial","*",22,1],"cohere-ai":["partial","*",22,1],"Diffbot":["partial","*",22,1],"ContextFetchBot":["partial","*",22,1]},"llms":{"status":"missing","http":404,"bytes":1},"home":{"status":"ok","http":200,"url":"https://www.bt.com/","html_tokens":57127,"markdown_tokens":375,"pruned_tokens":106}},
{"rank":2705,"domain":"ibisworld.com","origin":"https://ibisworld.com","robots":{"status":"ok","http":200,"url":"https://www.ibisworld.com/robots.txt","bytes":1030,"groups":2},"bots":{"GPTBot":["partial","*",30,2],"ChatGPT-User":["partial","*",30,2],"OAI-SearchBot":["partial","*",30,2],"ClaudeBot":["partial","*",30,2],"Claude-User":["partial","*",30,2],"Claude-SearchBot":["partial","*",30,2],"anthropic-ai":["partial","*",30,2],"PerplexityBot":["partial","*",30,2],"Perplexity-User":["partial","*",30,2],"Google-Extended":["partial","*",30,2],"Applebot-Extended":["partial","*",30,2],"CCBot":["partial","*",30,2],"Bytespider":["partial","*",30,2],"Amazonbot":["partial","*",30,2],"meta-externalagent":["partial","*",30,2],"cohere-ai":["partial","*",30,2],"Diffbot":["partial","*",30,2],"ContextFetchBot":["partial","*",30,2]},"llms":{"status":"missing","http":404,"bytes":45054},"home":{"status":"ok","http":200,"url":"https://www.ibisworld.com/","html_tokens":179082,"markdown_tokens":458,"pruned_tokens":190}},
{"rank":2706,"domain":"hgtv.com","origin":"https://hgtv.com","robots":{"status":"ok","http":200,"url":"https://www.hgtv.com/robots.txt","bytes":442,"groups":3},"bots":{"GPTBot":["partial","*",6,5],"ChatGPT-User":["partial","*",6,5],"OAI-SearchBot":["partial","*",6,5],"ClaudeBot":["partial","*",6,5],"Claude-User":["partial","*",6,5],"Claude-SearchBot":["partial","*",6,5],"anthropic-ai":["partial","*",6,5],"PerplexityBot":["partial","*",6,5],"Perplexity-User":["partial","*",6,5],"Google-Extended":["partial","*",6,5],"Applebot-Extended":["partial","*",6,5],"CCBot":["partial","*",6,5],"Bytespider":["partial","*",6,5],"Amazonbot":["partial","*",6,5],"meta-externalagent":["partial","*",6,5],"cohere-ai":["partial","*",6,5],"Diffbot":["partial","*",6,5],"ContextFetchBot":["partial","*",6,5]},"llms":{"status":"missing","http":404,"bytes":277614},"home":{"status":"ok","http":200,"url":"https://www.hgtv.com/","html_tokens":117309,"markdown_tokens":306,"pruned_tokens":216}},
{"rank":2707,"domain":"web.de","origin":"https://web.de","robots":{"status":"ok","http":200,"url":"https://web.de/robots.txt","bytes":1003,"groups":4},"bots":{"GPTBot":["partial","named",3,3],"ChatGPT-User":["partial","named",2,2],"OAI-SearchBot":["partial","named",2,2],"ClaudeBot":["partial","named",3,3],"Claude-User":["partial","named",2,2],"Claude-SearchBot":["partial","named",2,2],"anthropic-ai":["partial","*",2,0],"PerplexityBot":["partial","named",2,2],"Perplexity-User":["partial","named",2,2],"Google-Extended":["partial","named",3,3],"Applebot-Extended":["partial","named",3,3],"CCBot":["partial","named",3,3],"Bytespider":["partial","*",2,0],"Amazonbot":["partial","named",3,3],"meta-externalagent":["partial","named",3,3],"cohere-ai":["partial","named",3,3],"Diffbot":["partial","named",3,3],"ContextFetchBot":["partial","*",2,0]},"llms":{"status":"missing","http":404,"bytes":406151},"home":{"status":"ok","http":200,"url":"https://web.de/","html_tokens":101557,"markdown_tokens":87,"pruned_tokens":88}},
{"rank":2708,"domain":"trip.com","origin":"https://trip.com","robots":{"status":"ok","http":200,"url":"https://www.trip.com/robots.txt","bytes":4587,"groups":4},"bots":{"GPTBot":["partial","*",88,40],"ChatGPT-User":["partial","*",88,40],"OAI-SearchBot":["partial","*",88,40],"ClaudeBot":["partial","*",88,40],"Claude-User":["partial","*",88,40],"Claude-SearchBot":["partial","*",88,40],"anthropic-ai":["partial","*",88,40],"PerplexityBot":["partial","*",88,40],"Perplexity-User":["partial","*",88,40],"Google-Extended":["partial","*",88,40],"Applebot-Extended":["partial","*",88,40],"CCBot":["partial","*",88,40],"Bytespider":["partial","*",88,40],"Amazonbot":["partial","*",88,40],"meta-externalagent":["partial","*",88,40],"cohere-ai":["partial","*",88,40],"Diffbot":["partial","*",88,40],"ContextFetchBot":["partial","*",88,40]},"llms":{"status":"present","http":200,"bytes":49117},"home":{"status":"ok","http":200,"url":"https://www.trip.com/","html_tokens":44736,"markdown_tokens":5,"pruned_tokens":5}},
{"rank":2709,"domain":"st.com","origin":"https://st.com","robots":{"status":"unreachable","http":null,"url":"https://st.com/robots.txt","bytes":0,"groups":0,"error":"<StreamReset stream_id:1, error_code:2, remote_reset:True>"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":2710,"domain":"networkworld.com","origin":"https://networkworld.com","robots":{"status":"ok","http":200,"url":"https://www.networkworld.com/robots.txt","bytes":1541,"groups":1},"bots":{"GPTBot":["partial","*",24,0],"ChatGPT-User":["partial","*",24,0],"OAI-SearchBot":["partial","*",24,0],"ClaudeBot":["partial","*",24,0],"Claude-User":["partial","*",24,0],"Claude-SearchBot":["partial","*",24,0],"anthropic-ai":["partial","*",24,0],"PerplexityBot":["partial","*",24,0],"Perplexity-User":["partial","*",24,0],"Google-Extended":["partial","*",24,0],"Applebot-Extended":["partial","*",24,0],"CCBot":["partial","*",24,0],"Bytespider":["partial","*",24,0],"Amazonbot":["partial","*",24,0],"meta-externalagent":["partial","*",24,0],"cohere-ai":["partial","*",24,0],"Diffbot":["partial","*",24,0],"ContextFetchBot":["partial","*",24,0]},"llms":{"status":"missing","http":404,"bytes":171072},"home":{"status":"ok","http":200,"url":"https://www.networkworld.com","html_tokens":74962,"markdown_tokens":259,"pruned_tokens":130}},
{"rank":2711,"domain":"tu-berlin.de","origin":"https://www.tu-berlin.de","robots":{"status":"ok","http":200,"url":"https://www.tu-berlin.de/robots.txt","bytes":4502,"groups":3},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["blocked","named",1,0],"OAI-SearchBot":["blocked","named",1,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["blocked","named",1,0],"Claude-SearchBot":["blocked","named",1,0],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["blocked","named",1,0],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["allowed","*",0,1]},"llms":{"status":"html","http":200,"bytes":107372},"home":{"status":"ok","http":200,"url":"https://www.tu.berlin","html_tokens":26795,"markdown_tokens":73,"pruned_tokens":73}},
{"rank":2712,"domain":"die.net","origin":"https://die.net","robots":{"status":"ok","http":200,"url":"https://www.die.net/robots.txt","bytes":380,"groups":2},"bots":{"GPTBot":["partial","*",5,0],"ChatGPT-User":["partial","*",5,0],"OAI-SearchBot":["partial","*",5,0],"ClaudeBot":["partial","*",5,0],"Claude-User":["partial","*",5,0],"Claude-SearchBot":["partial","*",5,0],"anthropic-ai":["partial","*",5,0],"PerplexityBot":["partial","*",5,0],"Perplexity-User":["partial","*",5,0],"Google-Extended":["partial","*",5,0],"Applebot-Extended":["partial","*",5,0],"CCBot":["partial","*",5,0],"Bytespider":["partial","*",5,0],"Amazonbot":["partial","*",5,0],"meta-externalagent":["partial","*",5,0],"cohere-ai":["partial","*",5,0],"Diffbot":["partial","*",5,0],"ContextFetchBot":["partial","*",5,0]},"llms":{"status":"missing","http":404,"bytes":44152},"home":{"status":"ok","http":200,"url":"https://www.die.net/","html_tokens":967,"markdown_tokens":320,"pruned_tokens":174}},
{"rank":2713,"domain":"criteo.com","origin":"https://criteo.com","robots":{"status":"refused","http":403,"url":"https://criteo.com/robots.txt","bytes":560,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":2714,"domain":"fidelity.com","origin":"https://fidelity.com","robots":{"status":"ok","http":200,"url":"https://www.fidelity.com/robots.txt","bytes":12403,"groups":3},"bots":{"GPTBot":["partial","*",227,0],"ChatGPT-User":["partial","*",227,0],"OAI-SearchBot":["partial","*",227,0],"ClaudeBot":["partial","*",227,0],"Claude-User":["partial","*",227,0],"Claude-SearchBot":["partial","*",227,0],"anthropic-ai":["partial","*",227,0],"PerplexityBot":["partial","*",227,0],"Perplexity-User":["partial","*",227,0],"Google-Extended":["partial","*",227,0],"Applebot-Extended":["partial","*",227,0],"CCBot":["partial","*",227,0],"Bytespider":["partial","*",227,0],"Amazonbot":["partial","*",227,0],"meta-externalagent":["partial","*",227,0],"cohere-ai":["partial","*",227,0],"Diffbot":["partial","*",227,0],"ContextFetchBot":["partial","*",227,0]},"llms":{"status":"http_error","http":403,"bytes":243},"home":{"status":"ok","http":200,"url":"https://www.fidelity.com/","html_tokens":118300,"markdown_tokens":103,"pruned_tokens":68}},
{"rank":2715,"domain":"sdsu.edu","origin":"https://sdsu.edu","robots":{"status":"ok","http":200,"url":"https://www.sdsu.edu/robots.txt","bytes":136,"groups":1},"bots":{"GPTBot":["partial","*",3,0],"ChatGPT-User":["partial","*",3,0],"OAI-SearchBot":["partial","*",3,0],"ClaudeBot":["partial","*",3,0],"Claude-User":["partial","*",3,0],"Claude-SearchBot":["partial","*",3,0],"anthropic-ai":["partial","*",3,0],"PerplexityBot":["partial","*",3,0],"Perplexity-User":["partial","*",3,0],"Google-Extended":["partial","*",3,0],"Applebot-Extended":["partial","*",3,0],"CCBot":["partial","*",3,0],"Bytespider":["partial","*",3,0],"Amazonbot":["partial","*",3,0],"meta-externalagent":["partial","*",3,0],"cohere-ai":["partial","*",3,0],"Diffbot":["partial","*",3,0],"ContextFetchBot":["partial","*",3,0]},"llms":{"status":"missing","http":404,"bytes":31715},"home":{"status":"ok","http":200,"url":"https://www.sdsu.edu/","html_tokens":9188,"markdown_tokens":184,"pruned_tokens":89}},
{"rank":2716,"domain":"tuwien.ac.at","origin":"https://tuwien.ac.at","robots":{"status":"ok","http":200,"url":"https://www.tuwien.at/robots.txt","bytes":285,"groups":2},"bots":{"GPTBot":["partial","*",4,1],"ChatGPT-User":["partial","*",4,1],"OAI-SearchBot":["partial","*",4,1],"ClaudeBot":["partial","*",4,1],"Claude-User":["partial","*",4,1],"Claude-SearchBot":["partial","*",4,1],"anthropic-ai":["partial","*",4,1],"PerplexityBot":["partial","*",4,1],"Perplexity-User":["partial","*",4,1],"Google-Extended":["partial","*",4,1],"Applebot-Extended":["partial","*",4,1],"CCBot":["partial","*",4,1],"Bytespider":["partial","*",4,1],"Amazonbot":["partial","*",4,1],"meta-externalagent":["partial","*",4,1],"cohere-ai":["partial","*",4,1],"Diffbot":["partial","*",4,1],"ContextFetchBot":["partial","*",4,1]},"llms":{"status":"missing","http":404,"bytes":256994},"home":{"status":"ok","http":200,"url":"https://www.tuwien.at/","html_tokens":98225,"markdown_tokens":237,"pruned_tokens":115}},
{"rank":2717,"domain":"google.dev","origin":"https://google.dev","robots":{"status":"missing","http":404,"url":"https://developers.google.com/profile/robots.txt","bytes":46806,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":46804},"home":{"status":"ok","http":200,"url":"https://developers.google.com/program","html_tokens":33213,"markdown_tokens":615,"pruned_tokens":110}},
{"rank":2718,"domain":"leg.br","origin":"https://leg.br","robots":{"status":"unreachable","http":null,"url":"https://leg.br/robots.txt","bytes":0,"groups":0,"error":"cannot resolve leg.br"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":2719,"domain":"news18.com","origin":"https://news18.com","robots":{"status":"refused","http":403,"url":"https://news18.com/robots.txt","bytes":374,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":2720,"domain":"battle.net","origin":"https://battle.net","robots":{"status":"refused","http":403,"url":"https://battle.net/robots.txt","bytes":118,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":2721,"domain":"imgix.net","origin":"https://www.imgix.net","robots":{"status":"ok","http":200,"url":"https://www.imgix.net/robots.txt","bytes":48,"groups":1},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"missing","http":404,"bytes":29},"home":{"status":"ok","http":200,"url":"https://www.imgix.net/","html_tokens":689,"markdown_tokens":5,"pruned_tokens":5}},
{"rank":2722,"domain":"atlanticcouncil.org","origin":"https://atlanticcouncil.org","robots":{"status":"refused","http":403,"url":"https://atlanticcouncil.org/robots.txt","bytes":560,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":2723,"domain":"sheffield.ac.uk","origin":"https://sheffield.ac.uk","robots":{"status":"ok","http":200,"url":"https://sheffield.ac.uk/robots.txt","bytes":1801,"groups":1},"bots":{"GPTBot":["partial","*",28,18],"ChatGPT-User":["partial","*",28,18],"OAI-SearchBot":["partial","*",28,18],"ClaudeBot":["partial","*",28,18],"Claude-User":["partial","*",28,18],"Claude-SearchBot":["partial","*",28,18],"anthropic-ai":["partial","*",28,18],"PerplexityBot":["partial","*",28,18],"Perplexity-User":["partial","*",28,18],"Google-Extended":["partial","*",28,18],"Applebot-Extended":["partial","*",28,18],"CCBot":["partial","*",28,18],"Bytespider":["partial","*",28,18],"Amazonbot":["partial","*",28,18],"meta-externalagent":["partial","*",28,18],"cohere-ai":["partial","*",28,18],"Diffbot":["partial","*",28,18],"ContextFetchBot":["partial","*",28,18]},"llms":{"status":"missing","http":404,"bytes":24217},"home":{"status":"ok","http":200,"url":"https://sheffield.ac.uk/","html_tokens":28979,"markdown_tokens":70,"pruned_tokens":70}},
{"rank":2724,"domain":"coingecko.com","origin":"https://coingecko.com","robots":{"status":"ok","http":200,"url":"https://www.coingecko.com/robots.txt","bytes":8686,"groups":2},"bots":{"GPTBot":["partial","*",132,1],"ChatGPT-User":["partial","*",132,1],"OAI-SearchBot":["partial","*",132,1],"ClaudeBot":["partial","*",132,1],"Claude-User":["partial","*",132,1],"Claude-SearchBot":["partial","*",132,1],"anthropic-ai":["partial","*",132,1],"PerplexityBot":["partial","*",132,1],"Perplexity-User":["partial","*",132,1],"Google-Extended":["partial","named",135,1],"Applebot-Extended":["partial","*",132,1],"CCBot":["partial","*",132,1],"Bytespider":["partial","*",132,1],"Amazonbot":["partial","*",132,1],"meta-externalagent":["partial","*",132,1],"cohere-ai":["partial","*",132,1],"Diffbot":["partial","*",132,1],"ContextFetchBot":["partial","*",132,1]},"llms":{"status":"missing","http":404,"bytes":14},"home":{"status":"http_error","http":403,"url":"https://www.coingecko.com/"}},
{"rank":2725,"domain":"drexel.edu","origin":"https://drexel.edu","robots":{"status":"ok","http":200,"url":"https://drexel.edu/robots.txt","bytes":588,"groups":2},"bots":{"GPTBot":["partial","*",24,0],"ChatGPT-User":["partial","*",24,0],"OAI-SearchBot":["partial","*",24,0],"ClaudeBot":["partial","*",24,0],"Claude-User":["partial","*",24,0],"Claude-SearchBot":["partial","*",24,0],"anthropic-ai":["partial","*",24,0],"PerplexityBot":["partial","*",24,0],"Perplexity-User":["partial","*",24,0],"Google-Extended":["partial","*",24,0],"Applebot-Extended":["partial","*",24,0],"CCBot":["partial","*",24,0],"Bytespider":["partial","*",24,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["partial","*",24,0],"cohere-ai":["partial","*",24,0],"Diffbot":["partial","*",24,0],"ContextFetchBot":["partial","*",24,0]},"llms":{"status":"missing","http":404,"bytes":69460},"home":{"status":"ok","http":200,"url":"https://drexel.edu/","html_tokens":33275,"markdown_tokens":872,"pruned_tokens":448}},
{"rank":2726,"domain":"constantcontactpages.com","origin":"http://constantcontactpages.com","robots":{"status":"refused","http":403,"url":"https://www.constantcontact.com/","bytes":5531,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":2727,"domain":"aspca.org","origin":"https://aspca.org","robots":{"status":"refused","http":403,"url":"https://aspca.org/robots.txt","bytes":560,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":2728,"domain":"bbcgoodfood.com","origin":"https://bbcgoodfood.com","robots":{"status":"ok","http":200,"url":"https://www.bbcgoodfood.com/robots.txt","bytes":1828,"groups":12},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["partial","*",36,2],"OAI-SearchBot":["partial","*",36,2],"ClaudeBot":["partial","*",36,2],"Claude-User":["partial","*",36,2],"Claude-SearchBot":["partial","*",36,2],"anthropic-ai":["partial","*",36,2],"PerplexityBot":["partial","*",36,2],"Perplexity-User":["partial","*",36,2],"Google-Extended":["partial","*",36,2],"Applebot-Extended":["partial","*",36,2],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["partial","*",36,2],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["partial","*",36,2],"Diffbot":["partial","*",36,2],"ContextFetchBot":["partial","*",36,2]},"llms":{"status":"missing","http":404,"bytes":9},"home":{"status":"ok","http":200,"url":"https://www.bbcgoodfood.com/","html_tokens":209109,"markdown_tokens":71,"pruned_tokens":71}},
{"rank":2729,"domain":"democracynow.org","origin":"https://democracynow.org","robots":{"status":"ok","http":200,"url":"https://www.democracynow.org/robots.txt","bytes":275,"groups":1},"bots":{"GPTBot":["partial","*",7,1],"ChatGPT-User":["partial","*",7,1],"OAI-SearchBot":["partial","*",7,1],"ClaudeBot":["partial","*",7,1],"Claude-User":["partial","*",7,1],"Claude-SearchBot":["partial","*",7,1],"anthropic-ai":["partial","*",7,1],"PerplexityBot":["partial","*",7,1],"Perplexity-User":["partial","*",7,1],"Google-Extended":["partial","*",7,1],"Applebot-Extended":["partial","*",7,1],"CCBot":["partial","*",7,1],"Bytespider":["partial","*",7,1],"Amazonbot":["partial","*",7,1],"meta-externalagent":["partial","*",7,1],"cohere-ai":["partial","*",7,1],"Diffbot":["partial","*",7,1],"ContextFetchBot":["partial","*",7,1]},"llms":{"status":"missing","http":404,"bytes":12896},"home":{"status":"ok","http":200,"url":"https://www.democracynow.org/","html_tokens":44569,"markdown_tokens":190,"pruned_tokens":140}},
{"rank":2730,"domain":"uqam.ca","origin":"https://uqam.ca","robots":{"status":"ok","http":200,"url":"https://uqam.ca/robots.txt","bytes":274,"groups":1},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"missing","http":404,"bytes":46849},"home":{"status":"ok","http":200,"url":"https://uqam.ca/","html_tokens":45543,"markdown_tokens":136,"pruned_tokens":69}},
{"rank":2731,"domain":"brill.com","origin":"https://brill.com","robots":{"status":"ok","http":200,"url":"https://brill.com/robots.txt","bytes":966,"groups":1},"bots":{"GPTBot":["partial","*",30,0],"ChatGPT-User":["partial","*",30,0],"OAI-SearchBot":["partial","*",30,0],"ClaudeBot":["partial","*",30,0],"Claude-User":["partial","*",30,0],"Claude-SearchBot":["partial","*",30,0],"anthropic-ai":["partial","*",30,0],"PerplexityBot":["partial","*",30,0],"Perplexity-User":["partial","*",30,0],"Google-Extended":["partial","*",30,0],"Applebot-Extended":["partial","*",30,0],"CCBot":["partial","*",30,0],"Bytespider":["partial","*",30,0],"Amazonbot":["partial","*",30,0],"meta-externalagent":["partial","*",30,0],"cohere-ai":["partial","*",30,0],"Diffbot":["partial","*",30,0],"ContextFetchBot":["partial","*",30,0]},"llms":{"status":"missing","http":404,"bytes":132201},"home":{"status":"ok","http":200,"url":"https://brill.com/","html_tokens":56020,"markdown_tokens":4,"pruned_tokens":4}},
{"rank":2732,"domain":"189.cn","origin":"https://189.cn","robots":{"status":"missing","http":404,"url":"https://189.cn/robots.txt","bytes":146,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":146},"home":{"status":"http_error","http":412,"url":"https://189.cn/"}},
{"rank":2733,"domain":"amnh.org","origin":"https://amnh.org","robots":{"status":"ok","http":200,"url":"https://www.amnh.org/robots.txt","bytes":453,"groups":1},"bots":{"GPTBot":["partial","*",14,1],"ChatGPT-User":["partial","*",14,1],"OAI-SearchBot":["partial","*",14,1],"ClaudeBot":["partial","*",14,1],"Claude-User":["partial","*",14,1],"Claude-SearchBot":["partial","*",14,1],"anthropic-ai":["partial","*",14,1],"PerplexityBot":["partial","*",14,1],"Perplexity-User":["partial","*",14,1],"Google-Extended":["partial","*",14,1],"Applebot-Extended":["partial","*",14,1],"CCBot":["partial","*",14,1],"Bytespider":["partial","*",14,1],"Amazonbot":["partial","*",14,1],"meta-externalagent":["partial","*",14,1],"cohere-ai":["partial","*",14,1],"Diffbot":["partial","*",14,1],"ContextFetchBot":["partial","*",14,1]},"llms":{"status":"missing","http":404,"bytes":241730},"home":{"status":"http_error","http":403,"url":"https://amnh.org/"}},
{"rank":2734,"domain":"avaaz.org","origin":"https://avaaz.org","robots":{"status":"ok","http":200,"url":"https://avaaz.org/robots.txt","bytes":642,"groups":2},"bots":{"GPTBot":["partial","*",10,0],"ChatGPT-User":["partial","*",10,0],"OAI-SearchBot":["partial","*",10,0],"ClaudeBot":["partial","*",10,0],"Claude-User":["partial","*",10,0],"Claude-SearchBot":["partial","*",10,0],"anthropic-ai":["partial","*",10,0],"PerplexityBot":["partial","*",10,0],"Perplexity-User":["partial","*",10,0],"Google-Extended":["partial","*",10,0],"Applebot-Extended":["partial","*",10,0],"CCBot":["partial","*",10,0],"Bytespider":["partial","*",10,0],"Amazonbot":["partial","*",10,0],"meta-externalagent":["partial","named",10,0],"cohere-ai":["partial","*",10,0],"Diffbot":["partial","*",10,0],"ContextFetchBot":["partial","*",10,0]},"llms":{"status":"timeout","error":"timed out after 10 s"},"home":{"status":"ok","http":200,"url":"https://avaaz.org/","html_tokens":75347,"markdown_tokens":5990,"pruned_tokens":119}},
{"rank":2735,"domain":"trade.gov","origin":"https://trade.gov","robots":{"status":"ok","http":200,"url":"https://trade.gov/robots.txt","bytes":2090,"groups":2},"bots":{"GPTBot":["partial","*",23,18],"ChatGPT-User":["partial","*",23,18],"OAI-SearchBot":["partial","*",23,18],"ClaudeBot":["partial","*",23,18],"Claude-User":["partial","*",23,18],"Claude-SearchBot":["partial","*",23,18],"anthropic-ai":["partial","*",23,18],"PerplexityBot":["partial","*",23,18],"Perplexity-User":["partial","*",23,18],"Google-Extended":["partial","*",23,18],"Applebot-Extended":["partial","*",23,18],"CCBot":["partial","*",23,18],"Bytespider":["partial","*",23,18],"Amazonbot":["partial","*",23,18],"meta-externalagent":["partial","*",23,18],"cohere-ai":["partial","*",23,18],"Diffbot":["partial","*",23,18],"ContextFetchBot":["partial","*",23,18]},"llms":{"status":"missing","http":404,"bytes":187},"home":{"status":"ok","http":200,"url":"https://www.trade.gov/","html_tokens":64592,"markdown_tokens":207,"pruned_tokens":59}},
{"rank":2736,"domain":"hc360.com","origin":"https://hc360.com","robots":{"status":"unreachable","http":null,"url":"https://hc360.com/robots.txt","bytes":0,"groups":0,"error":"cannot resolve hc360.com"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":2737,"domain":"rtl.de","origin":"https://rtl.de","robots":{"status":"ok","http":200,"url":"https://www.rtl.de/robots.txt","bytes":1230,"groups":4},"bots":{"GPTBot":["partial","named",1,4],"ChatGPT-User":["partial","*",9,0],"OAI-SearchBot":["partial","*",9,0],"ClaudeBot":["partial","*",9,0],"Claude-User":["partial","*",9,0],"Claude-SearchBot":["partial","*",9,0],"anthropic-ai":["partial","*",9,0],"PerplexityBot":["partial","*",9,0],"Perplexity-User":["partial","*",9,0],"Google-Extended":["partial","named",1,4],"Applebot-Extended":["partial","*",9,0],"CCBot":["partial","named",1,4],"Bytespider":["partial","*",9,0],"Amazonbot":["partial","*",9,0],"meta-externalagent":["partial","*",9,0],"cohere-ai":["partial","*",9,0],"Diffbot":["partial","*",9,0],"ContextFetchBot":["partial","*",9,0]},"llms":{"status":"missing","http":404,"bytes":261876},"home":{"status":"ok","http":200,"url":"https://www.rtl.de/","html_tokens":771520,"markdown_tokens":212,"pruned_tokens":127}},
{"rank":2738,"domain":"libero.it","origin":"https://libero.it","robots":{"status":"ok","http":200,"url":"https://www.libero.it/robots.txt","bytes":2712,"groups":42},"bots":{"GPTBot":["partial","*",3,0],"ChatGPT-User":["partial","*",3,0],"OAI-SearchBot":["allowed","named",0,2],"ClaudeBot":["partial","*",3,0],"Claude-User":["partial","*",3,0],"Claude-SearchBot":["partial","*",3,0],"anthropic-ai":["partial","*",3,0],"PerplexityBot":["partial","*",3,0],"Perplexity-User":["partial","*",3,0],"Google-Extended":["partial","*",3,0],"Applebot-Extended":["partial","*",3,0],"CCBot":["partial","*",3,0],"Bytespider":["partial","*",3,0],"Amazonbot":["partial","*",3,0],"meta-externalagent":["partial","*",3,0],"cohere-ai":["partial","*",3,0],"Diffbot":["partial","*",3,0],"ContextFetchBot":["partial","*",3,0]},"llms":{"status":"missing","http":404,"bytes":151703},"home":{"status":"ok","http":200,"url":"https://www.libero.it/","html_tokens":178604,"markdown_tokens":18,"pruned_tokens":18}},
{"rank":2739,"domain":"beehiiv.com","origin":"https://beehiiv.com","robots":{"status":"ok","http":200,"url":"https://www.beehiiv.com/robots.txt","bytes":5903,"groups":3},"bots":{"GPTBot":["partial","*",39,2],"ChatGPT-User":["partial","*",39,2],"OAI-SearchBot":["partial","named",39,2],"ClaudeBot":["partial","*",39,2],"Claude-User":["partial","*",39,2],"Claude-SearchBot":["partial","*",39,2],"anthropic-ai":["partial","*",39,2],"PerplexityBot":["partial","*",39,2],"Perplexity-User":["partial","*",39,2],"Google-Extended":["partial","*",39,2],"Applebot-Extended":["partial","*",39,2],"CCBot":["partial","*",39,2],"Bytespider":["partial","*",39,2],"Amazonbot":["partial","*",39,2],"meta-externalagent":["partial","*",39,2],"cohere-ai":["partial","*",39,2],"Diffbot":["partial","*",39,2],"ContextFetchBot":["partial","*",39,2]},"llms":{"status":"present","http":200,"bytes":10157},"home":{"status":"ok","http":200,"url":"https://www.beehiiv.com/","html_tokens":47533,"markdown_tokens":573,"pruned_tokens":61}},
{"rank":2740,"domain":"socialmediatoday.com","origin":"https://socialmediatoday.com","robots":{"status":"ok","http":200,"url":"https://www.socialmediatoday.com/robots.txt","bytes":4954,"groups":13},"bots":{"GPTBot":["partial","*",15,2],"ChatGPT-User":["partial","*",15,2],"OAI-SearchBot":["partial","*",15,2],"ClaudeBot":["partial","*",15,2],"Claude-User":["partial","*",15,2],"Claude-SearchBot":["partial","*",15,2],"anthropic-ai":["partial","*",15,2],"PerplexityBot":["partial","*",15,2],"Perplexity-User":["partial","*",15,2],"Google-Extended":["partial","*",15,2],"Applebot-Extended":["partial","*",15,2],"CCBot":["partial","*",15,2],"Bytespider":["partial","*",15,2],"Amazonbot":["partial","named",17,2],"meta-externalagent":["partial","*",15,2],"cohere-ai":["partial","*",15,2],"Diffbot":["partial","*",15,2],"ContextFetchBot":["partial","*",15,2]},"llms":{"status":"missing","http":404,"bytes":123187},"home":{"status":"ok","http":200,"url":"https://www.socialmediatoday.com/","html_tokens":63583,"markdown_tokens":129,"pruned_tokens":129}},
{"rank":2741,"domain":"ocn.ne.jp","origin":"https://ocn.ne.jp","robots":{"status":"refused","http":403,"url":"https://www.ocn.ne.jp/robots.txt","bytes":3721,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":2742,"domain":"rtbf.be","origin":"https://rtbf.be","robots":{"status":"ok","http":200,"url":"https://www.rtbf.be/robots.txt","bytes":916,"groups":24},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["blocked","named",1,0],"OAI-SearchBot":["blocked","named",1,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",4,1],"Claude-SearchBot":["partial","*",4,1],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["partial","*",4,1],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",4,1]},"llms":{"status":"missing","http":404,"bytes":190519},"home":{"status":"ok","http":200,"url":"https://www.rtbf.be/","html_tokens":455802,"markdown_tokens":115,"pruned_tokens":52}},
{"rank":2743,"domain":"healthcare.gov","origin":"https://healthcare.gov","robots":{"status":"ok","http":200,"url":"https://www.healthcare.gov/robots.txt","bytes":3118,"groups":3},"bots":{"GPTBot":["partial","*",98,0],"ChatGPT-User":["partial","*",98,0],"OAI-SearchBot":["partial","*",98,0],"ClaudeBot":["partial","*",98,0],"Claude-User":["partial","*",98,0],"Claude-SearchBot":["partial","*",98,0],"anthropic-ai":["partial","*",98,0],"PerplexityBot":["partial","*",98,0],"Perplexity-User":["partial","*",98,0],"Google-Extended":["partial","*",98,0],"Applebot-Extended":["partial","*",98,0],"CCBot":["partial","*",98,0],"Bytespider":["partial","*",98,0],"Amazonbot":["partial","*",98,0],"meta-externalagent":["partial","*",98,0],"cohere-ai":["partial","*",98,0],"Diffbot":["partial","*",98,0],"ContextFetchBot":["partial","*",98,0]},"llms":{"status":"missing","http":404,"bytes":380045},"home":{"status":"ok","http":200,"url":"https://www.healthcare.gov/","html_tokens":107084,"markdown_tokens":65,"pruned_tokens":47}},
{"rank":2744,"domain":"recaptcha.net","origin":"https://recaptcha.net","robots":{"status":"ok","http":200,"url":"https://recaptcha.net/robots.txt","bytes":153,"groups":1},"bots":{"GPTBot":["partial","*",5,0],"ChatGPT-User":["partial","*",5,0],"OAI-SearchBot":["partial","*",5,0],"ClaudeBot":["partial","*",5,0],"Claude-User":["partial","*",5,0],"Claude-SearchBot":["partial","*",5,0],"anthropic-ai":["partial","*",5,0],"PerplexityBot":["partial","*",5,0],"Perplexity-User":["partial","*",5,0],"Google-Extended":["partial","*",5,0],"Applebot-Extended":["partial","*",5,0],"CCBot":["partial","*",5,0],"Bytespider":["partial","*",5,0],"Amazonbot":["partial","*",5,0],"meta-externalagent":["partial","*",5,0],"cohere-ai":["partial","*",5,0],"Diffbot":["partial","*",5,0],"ContextFetchBot":["partial","*",5,0]},"llms":{"status":"missing","http":404,"bytes":1621},"home":{"status":"http_error","http":404,"url":"https://recaptcha.net/"}},
{"rank":2745,"domain":"beacons.ai","origin":"https://beacons.ai","robots":{"status":"ok","http":200,"url":"https://beacons.ai/robots.txt","bytes":13,"groups":1},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"present","http":200,"bytes":5225},"home":{"status":"http_error","http":403,"url":"https://beacons.ai/"}},
{"rank":2746,"domain":"hbomax.com","origin":"https://hbomax.com","robots":{"status":"ok","http":200,"url":"https://www.hbomax.com/robots.txt","bytes":41,"groups":1},"bots":{"GPTBot":["partial","*",1,1],"ChatGPT-User":["partial","*",1,1],"OAI-SearchBot":["partial","*",1,1],"ClaudeBot":["partial","*",1,1],"Claude-User":["partial","*",1,1],"Claude-SearchBot":["partial","*",1,1],"anthropic-ai":["partial","*",1,1],"PerplexityBot":["partial","*",1,1],"Perplexity-User":["partial","*",1,1],"Google-Extended":["partial","*",1,1],"Applebot-Extended":["partial","*",1,1],"CCBot":["partial","*",1,1],"Bytespider":["partial","*",1,1],"Amazonbot":["partial","*",1,1],"meta-externalagent":["partial","*",1,1],"cohere-ai":["partial","*",1,1],"Diffbot":["partial","*",1,1],"ContextFetchBot":["partial","*",1,1]},"llms":{"status":"missing","http":404,"bytes":30},"home":{"status":"ok","http":200,"url":"https://www.hbomax.com/","html_tokens":722489,"markdown_tokens":601,"pruned_tokens":118}},
{"rank":2747,"domain":"wordreference.com","origin":"https://wordreference.com","robots":{"status":"ok","http":200,"url":"https://wordreference.com/robots.txt","bytes":5280,"groups":4},"bots":{"GPTBot":["partial","*",1,2],"ChatGPT-User":["partial","*",1,2],"OAI-SearchBot":["partial","*",1,2],"ClaudeBot":["partial","*",1,2],"Claude-User":["partial","*",1,2],"Claude-SearchBot":["partial","*",1,2],"anthropic-ai":["partial","*",1,2],"PerplexityBot":["partial","*",1,2],"Perplexity-User":["partial","*",1,2],"Google-Extended":["partial","named",14,2],"Applebot-Extended":["partial","*",1,2],"CCBot":["partial","*",1,2],"Bytespider":["partial","*",1,2],"Amazonbot":["partial","*",1,2],"meta-externalagent":["partial","named",1,2],"cohere-ai":["partial","*",1,2],"Diffbot":["partial","*",1,2],"ContextFetchBot":["partial","*",1,2]},"llms":{"status":"disallowed"},"home":{"status":"disallowed"}},
{"rank":2748,"domain":"res.in","origin":"https://res.in","robots":{"status":"unreachable","http":null,"url":"https://res.in/robots.txt","bytes":0,"groups":0,"error":"cannot resolve res.in"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":2749,"domain":"law.com","origin":"https://law.com","robots":{"status":"ok","http":200,"url":"https://www.law.com/robots.txt","bytes":16361,"groups":69},"bots":{"GPTBot":["partial","named",1,1],"ChatGPT-User":["partial","named",1,1],"OAI-SearchBot":["partial","named",1,1],"ClaudeBot":["partial","named",1,1],"Claude-User":["partial","named",1,1],"Claude-SearchBot":["partial","named",1,1],"anthropic-ai":["partial","named",1,1],"PerplexityBot":["partial","named",1,1],"Perplexity-User":["partial","named",1,1],"Google-Extended":["partial","named",1,1],"Applebot-Extended":["partial","named",1,1],"CCBot":["partial","named",1,1],"Bytespider":["partial","named",1,1],"Amazonbot":["partial","named",1,1],"meta-externalagent":["partial","named",1,1],"cohere-ai":["partial","named",1,1],"Diffbot":["partial","named",1,1],"ContextFetchBot":["partial","*",241,1]},"llms":{"status":"missing","http":404,"bytes":148310},"home":{"status":"ok","http":200,"url":"https://www.law.com/","html_tokens":93035,"markdown_tokens":127,"pruned_tokens":124}},
{"rank":2750,"domain":"bio.site","origin":"https://bio.site","robots":{"status":"missing","http":404,"url":"https://bio.site/robots.txt","bytes":44,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":44},"home":{"status":"ok","http":200,"url":"https://biosites.com/","html_tokens":21101,"markdown_tokens":151,"pruned_tokens":50}},
{"rank":2751,"domain":"sitepoint.com","origin":"https://sitepoint.com","robots":{"status":"ok","http":200,"url":"https://www.sitepoint.com/robots.txt","bytes":2853,"groups":4},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["partial","*",35,3],"OAI-SearchBot":["partial","*",35,3],"ClaudeBot":["partial","*",35,3],"Claude-User":["partial","*",35,3],"Claude-SearchBot":["partial","*",35,3],"anthropic-ai":["partial","*",35,3],"PerplexityBot":["partial","*",35,3],"Perplexity-User":["partial","*",35,3],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["partial","*",35,3],"CCBot":["blocked","named",1,0],"Bytespider":["partial","*",35,3],"Amazonbot":["partial","*",35,3],"meta-externalagent":["partial","*",35,3],"cohere-ai":["partial","*",35,3],"Diffbot":["partial","*",35,3],"ContextFetchBot":["partial","*",35,3]},"llms":{"status":"missing","http":404,"bytes":8541},"home":{"status":"http_error","http":403,"url":"https://www.sitepoint.com/"}},
{"rank":2752,"domain":"techtimes.com","origin":"https://techtimes.com","robots":{"status":"ok","http":200,"url":"https://www.techtimes.com/robots.txt","bytes":1946,"groups":2},"bots":{"GPTBot":["partial","*",12,0],"ChatGPT-User":["partial","*",12,0],"OAI-SearchBot":["allowed","named",0,0],"ClaudeBot":["allowed","named",0,0],"Claude-User":["allowed","named",0,0],"Claude-SearchBot":["allowed","named",0,0],"anthropic-ai":["allowed","named",0,0],"PerplexityBot":["allowed","named",0,0],"Perplexity-User":["allowed","named",0,0],"Google-Extended":["partial","*",12,0],"Applebot-Extended":["allowed","named",0,0],"CCBot":["allowed","named",0,0],"Bytespider":["allowed","named",0,0],"Amazonbot":["allowed","named",0,0],"meta-externalagent":["allowed","named",0,0],"cohere-ai":["allowed","named",0,0],"Diffbot":["partial","*",12,0],"ContextFetchBot":["partial","*",12,0]},"llms":{"status":"missing","http":404,"bytes":104},"home":{"status":"ok","http":200,"url":"https://www.techtimes.com/","html_tokens":96042,"markdown_tokens":172,"pruned_tokens":172}},
{"rank":2753,"domain":"51job.com","origin":"https://51job.com","robots":{"status":"html","http":200,"url":"https://51job.com/in/missing.php","bytes":29593,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":2754,"domain":"ncsc.gov.uk","origin":"https://ncsc.gov.uk","robots":{"status":"missing","http":404,"url":"http://www.ncsc.gov.uk/robots.txt","bytes":874,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":874},"home":{"status":"ok","http":200,"url":"https://www.ncsc.gov.uk/","html_tokens":62879,"markdown_tokens":372,"pruned_tokens":55}},
{"rank":2755,"domain":"walgreens.com","origin":"https://walgreens.com","robots":{"status":"ok","http":200,"url":"https://www.walgreens.com/robots.txt","bytes":2593,"groups":1},"bots":{"GPTBot":["partial","*",60,2],"ChatGPT-User":["partial","*",60,2],"OAI-SearchBot":["partial","*",60,2],"ClaudeBot":["partial","*",60,2],"Claude-User":["partial","*",60,2],"Claude-SearchBot":["partial","*",60,2],"anthropic-ai":["partial","*",60,2],"PerplexityBot":["partial","*",60,2],"Perplexity-User":["partial","*",60,2],"Google-Extended":["partial","*",60,2],"Applebot-Extended":["partial","*",60,2],"CCBot":["partial","*",60,2],"Bytespider":["partial","*",60,2],"Amazonbot":["partial","*",60,2],"meta-externalagent":["partial","*",60,2],"cohere-ai":["partial","*",60,2],"Diffbot":["partial","*",60,2],"ContextFetchBot":["partial","*",60,2]},"llms":{"status":"html","http":200,"bytes":961387},"home":{"status":"ok","http":200,"url":"https://www.walgreens.com/","html_tokens":92324,"markdown_tokens":60,"pruned_tokens":60}},
{"rank":2756,"domain":"gtmetrix.com","origin":"https://gtmetrix.com","robots":{"status":"ok","http":200,"url":"https://gtmetrix.com/robots.txt","bytes":908,"groups":2},"bots":{"GPTBot":["partial","*",32,0],"ChatGPT-User":["partial","*",32,0],"OAI-SearchBot":["partial","*",32,0],"ClaudeBot":["partial","*",32,0],"Claude-User":["partial","*",32,0],"Claude-SearchBot":["partial","*",32,0],"anthropic-ai":["partial","*",32,0],"PerplexityBot":["partial","*",32,0],"Perplexity-User":["partial","*",32,0],"Google-Extended":["partial","*",32,0],"Applebot-Extended":["partial","*",32,0],"CCBot":["partial","*",32,0],"Bytespider":["partial","*",32,0],"Amazonbot":["partial","*",32,0],"meta-externalagent":["partial","*",32,0],"cohere-ai":["partial","*",32,0],"Diffbot":["partial","*",32,0],"ContextFetchBot":["partial","*",32,0]},"llms":{"status":"missing","http":404,"bytes":14216},"home":{"status":"http_error","http":403,"url":"https://gtmetrix.com/"}},
{"rank":2757,"domain":"cyberpanel.net","origin":"https://cyberpanel.net","robots":{"status":"ok","http":200,"url":"https://cyberpanel.net/robots.txt","bytes":118,"groups":1},"bots":{"GPTBot":["partial","*",1,1],"ChatGPT-User":["partial","*",1,1],"OAI-SearchBot":["partial","*",1,1],"ClaudeBot":["partial","*",1,1],"Claude-User":["partial","*",1,1],"Claude-SearchBot":["partial","*",1,1],"anthropic-ai":["partial","*",1,1],"PerplexityBot":["partial","*",1,1],"Perplexity-User":["partial","*",1,1],"Google-Extended":["partial","*",1,1],"Applebot-Extended":["partial","*",1,1],"CCBot":["partial","*",1,1],"Bytespider":["partial","*",1,1],"Amazonbot":["partial","*",1,1],"meta-externalagent":["partial","*",1,1],"cohere-ai":["partial","*",1,1],"Diffbot":["partial","*",1,1],"ContextFetchBot":["partial","*",1,1]},"llms":{"status":"missing","http":404,"bytes":157233},"home":{"status":"ok","http":200,"url":"https://cyberpanel.net/","html_tokens":57557,"markdown_tokens":1054,"pruned_tokens":135}},
{"rank":2758,"domain":"jdoqocy.com","origin":"https://www.jdoqocy.com","robots":{"status":"ok","http":200,"url":"https://www.jdoqocy.com/robots.txt","bytes":35,"groups":1},"bots":{"GPTBot":["blocked","*",1,0],"ChatGPT-User":["blocked","*",1,0],"OAI-SearchBot":["blocked","*",1,0],"ClaudeBot":["blocked","*",1,0],"Claude-User":["blocked","*",1,0],"Claude-SearchBot":["blocked","*",1,0],"anthropic-ai":["blocked","*",1,0],"PerplexityBot":["blocked","*",1,0],"Perplexity-User":["blocked","*",1,0],"Google-Extended":["blocked","*",1,0],"Applebot-Extended":["blocked","*",1,0],"CCBot":["blocked","*",1,0],"Bytespider":["blocked","*",1,0],"Amazonbot":["blocked","*",1,0],"meta-externalagent":["blocked","*",1,0],"cohere-ai":["blocked","*",1,0],"Diffbot":["blocked","*",1,0],"ContextFetchBot":["blocked","*",1,0]},"llms":{"status":"disallowed"},"home":{"status":"disallowed"}},
{"rank":2759,"domain":"koreatimes.co.kr","origin":"https://koreatimes.co.kr","robots":{"status":"ok","http":200,"url":"https://www.koreatimes.co.kr/robots.txt","bytes":447,"groups":10},"bots":{"GPTBot":["partial","*",1,0],"ChatGPT-User":["partial","*",1,0],"OAI-SearchBot":["partial","*",1,0],"ClaudeBot":["partial","*",1,0],"Claude-User":["partial","*",1,0],"Claude-SearchBot":["partial","*",1,0],"anthropic-ai":["partial","*",1,0],"PerplexityBot":["partial","*",1,0],"Perplexity-User":["partial","*",1,0],"Google-Extended":["partial","*",1,0],"Applebot-Extended":["partial","*",1,0],"CCBot":["partial","*",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["partial","*",1,0],"cohere-ai":["partial","*",1,0],"Diffbot":["partial","*",1,0],"ContextFetchBot":["partial","*",1,0]},"llms":{"status":"present","http":200,"bytes":2803},"home":{"status":"ok","http":200,"url":"https://www.koreatimes.co.kr/","html_tokens":105976,"markdown_tokens":280,"pruned_tokens":124}},
{"rank":2760,"domain":"americanprogress.org","origin":"https://americanprogress.org","robots":{"status":"ok","http":200,"url":"https://www.americanprogress.org/robots.txt","bytes":554,"groups":3},"bots":{"GPTBot":["partial","*",8,1],"ChatGPT-User":["partial","*",8,1],"OAI-SearchBot":["partial","*",8,1],"ClaudeBot":["partial","*",8,1],"Claude-User":["partial","*",8,1],"Claude-SearchBot":["partial","*",8,1],"anthropic-ai":["partial","*",8,1],"PerplexityBot":["partial","*",8,1],"Perplexity-User":["partial","*",8,1],"Google-Extended":["partial","*",8,1],"Applebot-Extended":["partial","*",8,1],"CCBot":["partial","*",8,1],"Bytespider":["partial","*",8,1],"Amazonbot":["partial","*",8,1],"meta-externalagent":["partial","*",8,1],"cohere-ai":["partial","*",8,1],"Diffbot":["partial","*",8,1],"ContextFetchBot":["partial","*",8,1]},"llms":{"status":"missing","http":404,"bytes":109422},"home":{"status":"http_error","http":403,"url":"https://www.americanprogress.org/"}},
{"rank":2761,"domain":"item.jd.com","origin":"https://item.jd.com","robots":{"status":"ok","http":200,"url":"https://item.jd.com/robots.txt","bytes":105,"groups":3},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"disallowed","error":"robots.txt of pc-frequent-pro.pf.jd.com not readable (html)"},"home":{"status":"disallowed","error":"robots.txt of pc-frequent-pro.pf.jd.com not readable (html)"}},
{"rank":2762,"domain":"viator.com","origin":"https://viator.com","robots":{"status":"refused","http":403,"url":"https://www.viator.com/robots.txt","bytes":778,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":2763,"domain":"jsfiddle.net","origin":"https://jsfiddle.net","robots":{"status":"ok","http":200,"url":"https://jsfiddle.net/robots.txt","bytes":98,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":2231},"home":{"status":"ok","http":200,"url":"https://jsfiddle.net/","html_tokens":13683,"markdown_tokens":104,"pruned_tokens":104}},
{"rank":2764,"domain":"zohopublic.com","origin":"https://zohopublic.com","robots":{"status":"missing","http":404,"url":"https://zohopublic.com/robots.txt","bytes":105,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":105},"home":{"status":"ok","http":200,"url":"https://zohopublic.com/","html_tokens":33,"markdown_tokens":8,"pruned_tokens":8}},
{"rank":2765,"domain":"escholarship.org","origin":"https://escholarship.org","robots":{"status":"refused","http":403,"url":"https://escholarship.org/robots.txt","bytes":919,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":2766,"domain":"unt.edu","origin":"https://unt.edu","robots":{"status":"ok","http":200,"url":"https://www.unt.edu/robots.txt","bytes":439,"groups":1},"bots":{"GPTBot":["partial","*",11,1],"ChatGPT-User":["partial","*",11,1],"OAI-SearchBot":["partial","*",11,1],"ClaudeBot":["partial","*",11,1],"Claude-User":["partial","*",11,1],"Claude-SearchBot":["partial","*",11,1],"anthropic-ai":["partial","*",11,1],"PerplexityBot":["partial","*",11,1],"Perplexity-User":["partial","*",11,1],"Google-Extended":["partial","*",11,1],"Applebot-Extended":["partial","*",11,1],"CCBot":["partial","*",11,1],"Bytespider":["partial","*",11,1],"Amazonbot":["partial","*",11,1],"meta-externalagent":["partial","*",11,1],"cohere-ai":["partial","*",11,1],"Diffbot":["partial","*",11,1],"ContextFetchBot":["partial","*",11,1]},"llms":{"status":"missing","http":404,"bytes":45519},"home":{"status":"ok","http":200,"url":"https://www.unt.edu/","html_tokens":17592,"markdown_tokens":103,"pruned_tokens":103}},
{"rank":2767,"domain":"diplo.de","origin":"https://diplo.de","robots":{"status":"ok","http":200,"url":"https://www.auswaertiges-amt.de/robots.txt","bytes":290,"groups":1},"bots":{"GPTBot":["partial","*",5,0],"ChatGPT-User":["partial","*",5,0],"OAI-SearchBot":["partial","*",5,0],"ClaudeBot":["partial","*",5,0],"Claude-User":["partial","*",5,0],"Claude-SearchBot":["partial","*",5,0],"anthropic-ai":["partial","*",5,0],"PerplexityBot":["partial","*",5,0],"Perplexity-User":["partial","*",5,0],"Google-Extended":["partial","*",5,0],"Applebot-Extended":["partial","*",5,0],"CCBot":["partial","*",5,0],"Bytespider":["partial","*",5,0],"Amazonbot":["partial","*",5,0],"meta-externalagent":["partial","*",5,0],"cohere-ai":["partial","*",5,0],"Diffbot":["partial","*",5,0],"ContextFetchBot":["partial","*",5,0]},"llms":{"status":"missing","http":404,"bytes":73631},"home":{"status":"ok","http":200,"url":"https://www.auswaertiges-amt.de/","html_tokens":31386,"markdown_tokens":87,"pruned_tokens":87}},
{"rank":2768,"domain":"acog.org","origin":"https://acog.org","robots":{"status":"ok","http":200,"url":"https://www.acog.org/robots.txt","bytes":113,"groups":1},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"missing","http":404,"bytes":49189},"home":{"status":"ok","http":200,"url":"https://www.acog.org/","html_tokens":22575,"markdown_tokens":160,"pruned_tokens":96}},
{"rank":2769,"domain":"ncsl.org","origin":"https://ncsl.org","robots":{"status":"ok","http":200,"url":"https://www.ncsl.org/robots.txt","bytes":1045,"groups":3},"bots":{"GPTBot":["allowed","named",0,0],"ChatGPT-User":["partial","*",13,0],"OAI-SearchBot":["partial","*",13,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",13,0],"Claude-SearchBot":["partial","*",13,0],"anthropic-ai":["partial","*",13,0],"PerplexityBot":["partial","*",13,0],"Perplexity-User":["partial","*",13,0],"Google-Extended":["partial","*",13,0],"Applebot-Extended":["partial","*",13,0],"CCBot":["allowed","named",0,0],"Bytespider":["partial","*",13,0],"Amazonbot":["allowed","named",0,0],"meta-externalagent":["partial","*",13,0],"cohere-ai":["partial","*",13,0],"Diffbot":["partial","*",13,0],"ContextFetchBot":["partial","*",13,0]},"llms":{"status":"missing","http":404,"bytes":1606},"home":{"status":"ok","http":200,"url":"https://www.ncsl.org/","html_tokens":19491,"markdown_tokens":102,"pruned_tokens":102}},
{"rank":2770,"domain":"ddns.net","origin":"http://ddns.net","robots":{"status":"html","http":200,"url":"http://freeddns.noip.com?d=ddns.net&u=ZGRucy5uZXQvcm9ib3RzLnR4dA==","bytes":60533,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":2771,"domain":"plala.or.jp","origin":"https://www.plala.or.jp","robots":{"status":"ok","http":200,"url":"https://www.plala.or.jp/robots.txt","bytes":541,"groups":1},"bots":{"GPTBot":["partial","*",18,1],"ChatGPT-User":["partial","*",18,1],"OAI-SearchBot":["partial","*",18,1],"ClaudeBot":["partial","*",18,1],"Claude-User":["partial","*",18,1],"Claude-SearchBot":["partial","*",18,1],"anthropic-ai":["partial","*",18,1],"PerplexityBot":["partial","*",18,1],"Perplexity-User":["partial","*",18,1],"Google-Extended":["partial","*",18,1],"Applebot-Extended":["partial","*",18,1],"CCBot":["partial","*",18,1],"Bytespider":["partial","*",18,1],"Amazonbot":["partial","*",18,1],"meta-externalagent":["partial","*",18,1],"cohere-ai":["partial","*",18,1],"Diffbot":["partial","*",18,1],"ContextFetchBot":["partial","*",18,1]},"llms":{"status":"missing","http":404,"bytes":4879},"home":{"status":"ok","http":200,"url":"https://www.plala.or.jp/","html_tokens":6097,"markdown_tokens":43,"pruned_tokens":43}},
{"rank":2772,"domain":"uni-tuebingen.de","origin":"https://uni-tuebingen.de","robots":{"status":"ok","http":200,"url":"https://uni-tuebingen.de/robots.txt","bytes":255,"groups":2},"bots":{"GPTBot":["partial","*",2,0],"ChatGPT-User":["partial","*",2,0],"OAI-SearchBot":["partial","*",2,0],"ClaudeBot":["partial","*",2,0],"Claude-User":["partial","*",2,0],"Claude-SearchBot":["partial","*",2,0],"anthropic-ai":["partial","*",2,0],"PerplexityBot":["partial","*",2,0],"Perplexity-User":["partial","*",2,0],"Google-Extended":["partial","*",2,0],"Applebot-Extended":["partial","*",2,0],"CCBot":["partial","*",2,0],"Bytespider":["partial","*",2,0],"Amazonbot":["partial","*",2,0],"meta-externalagent":["partial","*",2,0],"cohere-ai":["partial","*",2,0],"Diffbot":["partial","*",2,0],"ContextFetchBot":["partial","*",2,0]},"llms":{"status":"missing","http":404,"bytes":672245},"home":{"status":"ok","http":200,"url":"https://uni-tuebingen.de/","html_tokens":174191,"markdown_tokens":50,"pruned_tokens":50}},
{"rank":2773,"domain":"lit.link","origin":"https://lit.link","robots":{"status":"ok","http":200,"url":"https://lit.link/robots.txt","bytes":53,"groups":1},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"html","http":200,"bytes":175687},"home":{"status":"ok","http":200,"url":"https://lit.link/en","html_tokens":38476,"markdown_tokens":0,"pruned_tokens":0}},
{"rank":2774,"domain":"tilda.cc","origin":"https://tilda.cc","robots":{"status":"ok","http":200,"url":"https://tilda.cc/robots.txt","bytes":1637,"groups":2},"bots":{"GPTBot":["partial","*",15,3],"ChatGPT-User":["partial","*",15,3],"OAI-SearchBot":["partial","*",15,3],"ClaudeBot":["partial","*",15,3],"Claude-User":["partial","*",15,3],"Claude-SearchBot":["partial","*",15,3],"anthropic-ai":["partial","*",15,3],"PerplexityBot":["partial","*",15,3],"Perplexity-User":["partial","*",15,3],"Google-Extended":["partial","*",15,3],"Applebot-Extended":["partial","*",15,3],"CCBot":["partial","*",15,3],"Bytespider":["partial","*",15,3],"Amazonbot":["partial","*",15,3],"meta-externalagent":["partial","*",15,3],"cohere-ai":["partial","*",15,3],"Diffbot":["partial","*",15,3],"ContextFetchBot":["partial","*",15,3]},"llms":{"status":"missing","http":404,"bytes":369},"home":{"status":"ok","http":200,"url":"https://tilda.cc/","html_tokens":117491,"markdown_tokens":191,"pruned_tokens":27}},
{"rank":2775,"domain":"pantone.com","origin":"https://pantone.com","robots":{"status":"ok","http":200,"url":"https://www.pantone.com/robots.txt","bytes":2322,"groups":5},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["partial","named",10,1],"OAI-SearchBot":["partial","named",10,1],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",10,1],"Claude-SearchBot":["partial","*",10,1],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["partial","named",10,1],"Perplexity-User":["partial","*",10,1],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["partial","named",10,1],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",10,1]},"llms":{"status":"missing","http":404,"bytes":207544},"home":{"status":"ok","http":200,"url":"https://www.pantone.com/","html_tokens":122509,"markdown_tokens":284,"pruned_tokens":115}},
{"rank":2776,"domain":"open.ac.uk","origin":"https://www.open.ac.uk","robots":{"status":"refused","http":403,"url":"https://www.open.ac.uk/robots.txt","bytes":5552,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":2777,"domain":"mof.gov.cn","origin":"https://www.mof.gov.cn","robots":{"status":"missing","http":404,"url":"https://www.mof.gov.cn/robots.txt","bytes":7588,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":7588},"home":{"status":"ok","http":200,"url":"https://www.mof.gov.cn/","html_tokens":634,"markdown_tokens":0,"pruned_tokens":0}},
{"rank":2778,"domain":"onelink.to","origin":"https://onelink.to","robots":{"status":"ok","http":200,"url":"https://onelink.to/robots.txt","bytes":11,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"html","http":200,"bytes":17556},"home":{"status":"ok","http":200,"url":"https://www.onelink.to","html_tokens":98559,"markdown_tokens":198,"pruned_tokens":126}},
{"rank":2779,"domain":"ovh.net","origin":"https://ovh.net","robots":{"status":"refused","http":429,"url":"https://proof.ovh.net","bytes":162,"groups":0,"error":"HTTP 429"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":2780,"domain":"mobiri.se","origin":"https://mobiri.se","robots":{"status":"ok","http":200,"url":"https://mobiri.se/robots.txt","bytes":63,"groups":1},"bots":{"GPTBot":["allowed","*",0,1],"ChatGPT-User":["allowed","*",0,1],"OAI-SearchBot":["allowed","*",0,1],"ClaudeBot":["allowed","*",0,1],"Claude-User":["allowed","*",0,1],"Claude-SearchBot":["allowed","*",0,1],"anthropic-ai":["allowed","*",0,1],"PerplexityBot":["allowed","*",0,1],"Perplexity-User":["allowed","*",0,1],"Google-Extended":["allowed","*",0,1],"Applebot-Extended":["allowed","*",0,1],"CCBot":["allowed","*",0,1],"Bytespider":["allowed","*",0,1],"Amazonbot":["allowed","*",0,1],"meta-externalagent":["allowed","*",0,1],"cohere-ai":["allowed","*",0,1],"Diffbot":["allowed","*",0,1],"ContextFetchBot":["allowed","*",0,1]},"llms":{"status":"present","http":200,"bytes":886},"home":{"status":"ok","http":200,"url":"https://mobiri.se/","html_tokens":13657,"markdown_tokens":4246,"pruned_tokens":315}},
{"rank":2781,"domain":"poznan.pl","origin":"https://poznan.pl","robots":{"status":"ok","http":200,"url":"https://www.poznan.pl/robots.txt","bytes":1009,"groups":12},"bots":{"GPTBot":["partial","*",21,0],"ChatGPT-User":["partial","*",21,0],"OAI-SearchBot":["partial","*",21,0],"ClaudeBot":["partial","*",21,0],"Claude-User":["partial","*",21,0],"Claude-SearchBot":["partial","*",21,0],"anthropic-ai":["partial","*",21,0],"PerplexityBot":["partial","*",21,0],"Perplexity-User":["partial","*",21,0],"Google-Extended":["partial","*",21,0],"Applebot-Extended":["partial","*",21,0],"CCBot":["partial","*",21,0],"Bytespider":["partial","*",21,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["partial","*",21,0],"cohere-ai":["partial","*",21,0],"Diffbot":["partial","*",21,0],"ContextFetchBot":["partial","*",21,0]},"llms":{"status":"missing","http":404,"bytes":38860},"home":{"status":"ok","http":200,"url":"https://www.poznan.pl/","html_tokens":55587,"markdown_tokens":66,"pruned_tokens":66}},
{"rank":2782,"domain":"kyiv.ua","origin":"https://www.kyiv.ua","robots":{"status":"missing","http":404,"url":"https://tviy.kyiv.ua/robots.txt","bytes":146,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":146},"home":{"status":"ok","http":200,"url":"https://tviy.kyiv.ua/","html_tokens":514,"markdown_tokens":39,"pruned_tokens":39}},
{"rank":2783,"domain":"self.com","origin":"https://self.com","robots":{"status":"ok","http":200,"url":"https://www.self.com/robots.txt","bytes":2098,"groups":3},"bots":{"GPTBot":["partial","*",13,14],"ChatGPT-User":["partial","*",13,14],"OAI-SearchBot":["partial","*",13,14],"ClaudeBot":["blocked","named",1,0],"Claude-User":["blocked","named",1,0],"Claude-SearchBot":["blocked","named",1,0],"anthropic-ai":["partial","*",13,14],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["blocked","named",1,0],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",13,14]},"llms":{"status":"missing","http":404,"bytes":518606},"home":{"status":"ok","http":200,"url":"https://www.self.com/","html_tokens":424485,"markdown_tokens":404,"pruned_tokens":175}},
{"rank":2784,"domain":"hessen.de","origin":"https://hessen.de","robots":{"status":"ok","http":200,"url":"https://hessen.de/robots.txt","bytes":2111,"groups":4},"bots":{"GPTBot":["partial","*",33,19],"ChatGPT-User":["partial","*",33,19],"OAI-SearchBot":["partial","*",33,19],"ClaudeBot":["partial","*",33,19],"Claude-User":["partial","*",33,19],"Claude-SearchBot":["partial","*",33,19],"anthropic-ai":["partial","*",33,19],"PerplexityBot":["partial","*",33,19],"Perplexity-User":["partial","*",33,19],"Google-Extended":["partial","*",33,19],"Applebot-Extended":["partial","*",33,19],"CCBot":["partial","*",33,19],"Bytespider":["partial","*",33,19],"Amazonbot":["partial","*",33,19],"meta-externalagent":["partial","*",33,19],"cohere-ai":["partial","*",33,19],"Diffbot":["partial","*",33,19],"ContextFetchBot":["partial","*",33,19]},"llms":{"status":"missing","http":404,"bytes":183},"home":{"status":"ok","http":200,"url":"https://hessen.de/","html_tokens":245610,"markdown_tokens":1486,"pruned_tokens":296}},
{"rank":2785,"domain":"justpaste.it","origin":"https://justpaste.it","robots":{"status":"ok","http":200,"url":"https://justpaste.it/robots.txt","bytes":49,"groups":1},"bots":{"GPTBot":["partial","*",2,0],"ChatGPT-User":["partial","*",2,0],"OAI-SearchBot":["partial","*",2,0],"ClaudeBot":["partial","*",2,0],"Claude-User":["partial","*",2,0],"Claude-SearchBot":["partial","*",2,0],"anthropic-ai":["partial","*",2,0],"PerplexityBot":["partial","*",2,0],"Perplexity-User":["partial","*",2,0],"Google-Extended":["partial","*",2,0],"Applebot-Extended":["partial","*",2,0],"CCBot":["partial","*",2,0],"Bytespider":["partial","*",2,0],"Amazonbot":["partial","*",2,0],"meta-externalagent":["partial","*",2,0],"cohere-ai":["partial","*",2,0],"Diffbot":["partial","*",2,0],"ContextFetchBot":["partial","*",2,0]},"llms":{"status":"missing","http":404,"bytes":6132},"home":{"status":"ok","http":200,"url":"https://justpaste.it/","html_tokens":1879,"markdown_tokens":10,"pruned_tokens":10}},
{"rank":2786,"domain":"ohio-state.edu","origin":"https://ohio-state.edu","robots":{"status":"unreachable","http":null,"url":"https://ohio-state.edu/robots.txt","bytes":0,"groups":0,"error":"cannot resolve ohio-state.edu"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":2787,"domain":"europeana.eu","origin":"https://europeana.eu","robots":{"status":"refused","http":403,"url":"https://www.europeana.eu/robots.txt","bytes":5597,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":2788,"domain":"american.edu","origin":"https://american.edu","robots":{"status":"ok","http":200,"url":"https://american.edu/robots.txt","bytes":625,"groups":1},"bots":{"GPTBot":["partial","*",13,0],"ChatGPT-User":["partial","*",13,0],"OAI-SearchBot":["partial","*",13,0],"ClaudeBot":["partial","*",13,0],"Claude-User":["partial","*",13,0],"Claude-SearchBot":["partial","*",13,0],"anthropic-ai":["partial","*",13,0],"PerplexityBot":["partial","*",13,0],"Perplexity-User":["partial","*",13,0],"Google-Extended":["partial","*",13,0],"Applebot-Extended":["partial","*",13,0],"CCBot":["partial","*",13,0],"Bytespider":["partial","*",13,0],"Amazonbot":["partial","*",13,0],"meta-externalagent":["partial","*",13,0],"cohere-ai":["partial","*",13,0],"Diffbot":["partial","*",13,0],"ContextFetchBot":["partial","*",13,0]},"llms":{"status":"missing","http":404,"bytes":35794},"home":{"status":"http_error","http":403,"url":"https://american.edu/"}},
{"rank":2789,"domain":"radio-canada.ca","origin":"https://radio-canada.ca","robots":{"status":"ok","http":200,"url":"https://radio-canada.ca/robots.txt","bytes":24,"groups":1},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"missing","http":404,"bytes":328476},"home":{"status":"ok","http":200,"url":"https://ici.radio-canada.ca","html_tokens":256134,"markdown_tokens":880,"pruned_tokens":7}},
{"rank":2790,"domain":"gentoo.org","origin":"https://gentoo.org","robots":{"status":"ok","http":200,"url":"https://www.gentoo.org/robots.txt","bytes":2441,"groups":3},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["blocked","named",1,0],"OAI-SearchBot":["blocked","named",1,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["blocked","named",1,0],"Claude-SearchBot":["blocked","named",1,0],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["blocked","named",1,0],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["allowed","*",0,1]},"llms":{"status":"missing","http":404,"bytes":2362},"home":{"status":"ok","http":200,"url":"https://www.gentoo.org/","html_tokens":6104,"markdown_tokens":216,"pruned_tokens":216}},
{"rank":2791,"domain":"birmingham.ac.uk","origin":"https://birmingham.ac.uk","robots":{"status":"ok","http":200,"url":"https://www.birmingham.ac.uk/robots.txt","bytes":1476,"groups":2},"bots":{"GPTBot":["partial","*",33,1],"ChatGPT-User":["partial","*",33,1],"OAI-SearchBot":["partial","*",33,1],"ClaudeBot":["partial","*",33,1],"Claude-User":["partial","*",33,1],"Claude-SearchBot":["partial","*",33,1],"anthropic-ai":["partial","*",33,1],"PerplexityBot":["partial","*",33,1],"Perplexity-User":["partial","*",33,1],"Google-Extended":["partial","*",33,1],"Applebot-Extended":["partial","*",33,1],"CCBot":["partial","*",33,1],"Bytespider":["partial","*",33,1],"Amazonbot":["partial","*",33,1],"meta-externalagent":["partial","*",33,1],"cohere-ai":["partial","*",33,1],"Diffbot":["partial","*",33,1],"ContextFetchBot":["partial","*",33,1]},"llms":{"status":"missing","http":404,"bytes":14},"home":{"status":"ok","http":200,"url":"https://www.birmingham.ac.uk/","html_tokens":49687,"markdown_tokens":164,"pruned_tokens":95}},
{"rank":2792,"domain":"pymnts.com","origin":"https://pymnts.com","robots":{"status":"ok","http":200,"url":"https://www.pymnts.com/robots.txt","bytes":332,"groups":2},"bots":{"GPTBot":["partial","*",4,0],"ChatGPT-User":["partial","*",4,0],"OAI-SearchBot":["partial","*",4,0],"ClaudeBot":["partial","*",4,0],"Claude-User":["partial","*",4,0],"Claude-SearchBot":["partial","*",4,0],"anthropic-ai":["partial","*",4,0],"PerplexityBot":["partial","*",4,0],"Perplexity-User":["partial","*",4,0],"Google-Extended":["partial","*",4,0],"Applebot-Extended":["partial","*",4,0],"CCBot":["partial","*",4,0],"Bytespider":["partial","*",4,0],"Amazonbot":["partial","*",4,0],"meta-externalagent":["partial","*",4,0],"cohere-ai":["partial","*",4,0],"Diffbot":["partial","*",4,0],"ContextFetchBot":["partial","*",4,0]},"llms":{"status":"present","http":200,"bytes":209972},"home":{"status":"ok","http":200,"url":"https://www.pymnts.com/","html_tokens":53007,"markdown_tokens":2215,"pruned_tokens":171}},
{"rank":2793,"domain":"cognitoforms.com","origin":"https://cognitoforms.com","robots":{"status":"ok","http":200,"url":"https://www.cognitoforms.com/robots.txt","bytes":336,"groups":1},"bots":{"GPTBot":["partial","*",11,0],"ChatGPT-User":["partial","*",11,0],"OAI-SearchBot":["partial","*",11,0],"ClaudeBot":["partial","*",11,0],"Claude-User":["partial","*",11,0],"Claude-SearchBot":["partial","*",11,0],"anthropic-ai":["partial","*",11,0],"PerplexityBot":["partial","*",11,0],"Perplexity-User":["partial","*",11,0],"Google-Extended":["partial","*",11,0],"Applebot-Extended":["partial","*",11,0],"CCBot":["partial","*",11,0],"Bytespider":["partial","*",11,0],"Amazonbot":["partial","*",11,0],"meta-externalagent":["partial","*",11,0],"cohere-ai":["partial","*",11,0],"Diffbot":["partial","*",11,0],"ContextFetchBot":["partial","*",11,0]},"llms":{"status":"missing","http":404,"bytes":201677},"home":{"status":"ok","http":200,"url":"https://www.cognitoforms.com/","html_tokens":79491,"markdown_tokens":103,"pruned_tokens":48}},
{"rank":2794,"domain":"rki.de","origin":"https://rki.de","robots":{"status":"ok","http":200,"url":"https://www.rki.de/robots.txt","bytes":239,"groups":1},"bots":{"GPTBot":["partial","*",3,3],"ChatGPT-User":["partial","*",3,3],"OAI-SearchBot":["partial","*",3,3],"ClaudeBot":["partial","*",3,3],"Claude-User":["partial","*",3,3],"Claude-SearchBot":["partial","*",3,3],"anthropic-ai":["partial","*",3,3],"PerplexityBot":["partial","*",3,3],"Perplexity-User":["partial","*",3,3],"Google-Extended":["partial","*",3,3],"Applebot-Extended":["partial","*",3,3],"CCBot":["partial","*",3,3],"Bytespider":["partial","*",3,3],"Amazonbot":["partial","*",3,3],"meta-externalagent":["partial","*",3,3],"cohere-ai":["partial","*",3,3],"Diffbot":["partial","*",3,3],"ContextFetchBot":["partial","*",3,3]},"llms":{"status":"missing","http":404,"bytes":165921},"home":{"status":"timeout","error":"timed out after 15 s"}},
{"rank":2795,"domain":"cookiebot.com","origin":"https://cookiebot.com","robots":{"status":"ok","http":200,"url":"https://www.cookiebot.com/robots.txt","bytes":636,"groups":1},"bots":{"GPTBot":["allowed","*",0,1],"ChatGPT-User":["allowed","*",0,1],"OAI-SearchBot":["allowed","*",0,1],"ClaudeBot":["allowed","*",0,1],"Claude-User":["allowed","*",0,1],"Claude-SearchBot":["allowed","*",0,1],"anthropic-ai":["allowed","*",0,1],"PerplexityBot":["allowed","*",0,1],"Perplexity-User":["allowed","*",0,1],"Google-Extended":["allowed","*",0,1],"Applebot-Extended":["allowed","*",0,1],"CCBot":["allowed","*",0,1],"Bytespider":["allowed","*",0,1],"Amazonbot":["allowed","*",0,1],"meta-externalagent":["allowed","*",0,1],"cohere-ai":["allowed","*",0,1],"Diffbot":["allowed","*",0,1],"ContextFetchBot":["allowed","*",0,1]},"llms":{"status":"http_error","http":403,"bytes":5544},"home":{"status":"http_error","http":403,"url":"https://cookiebot.com/"}},
{"rank":2796,"domain":"idaho.gov","origin":"https://idaho.gov","robots":{"status":"ok","http":200,"url":"https://idaho.gov/robots.txt","bytes":107,"groups":1},"bots":{"GPTBot":["partial","*",1,1],"ChatGPT-User":["partial","*",1,1],"OAI-SearchBot":["partial","*",1,1],"ClaudeBot":["partial","*",1,1],"Claude-User":["partial","*",1,1],"Claude-SearchBot":["partial","*",1,1],"anthropic-ai":["partial","*",1,1],"PerplexityBot":["partial","*",1,1],"Perplexity-User":["partial","*",1,1],"Google-Extended":["partial","*",1,1],"Applebot-Extended":["partial","*",1,1],"CCBot":["partial","*",1,1],"Bytespider":["partial","*",1,1],"Amazonbot":["partial","*",1,1],"meta-externalagent":["partial","*",1,1],"cohere-ai":["partial","*",1,1],"Diffbot":["partial","*",1,1],"ContextFetchBot":["partial","*",1,1]},"llms":{"status":"missing","http":404,"bytes":159815},"home":{"status":"ok","http":200,"url":"https://idaho.gov/","html_tokens":130815,"markdown_tokens":688,"pruned_tokens":483}},
{"rank":2797,"domain":"wistia.net","origin":"https://wistia.net","robots":{"status":"unreachable","http":null,"url":"https://wistia.net/robots.txt","bytes":0,"groups":0,"error":"cannot resolve wistia.net"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":2798,"domain":"intensedebate.com","origin":"https://intensedebate.com","robots":{"status":"html","http":200,"url":"https://intensedebate.com/robots.txt","bytes":7507,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":2799,"domain":"hyundai.com","origin":"https://hyundai.com","robots":{"status":"ok","http":200,"url":"https://www.hyundai.com/robots.txt","bytes":5909,"groups":3},"bots":{"GPTBot":["partial","named",25,1],"ChatGPT-User":["partial","named",25,1],"OAI-SearchBot":["partial","named",25,1],"ClaudeBot":["partial","named",25,1],"Claude-User":["partial","named",25,1],"Claude-SearchBot":["partial","named",25,1],"anthropic-ai":["partial","*",27,1],"PerplexityBot":["partial","named",25,1],"Perplexity-User":["partial","*",27,1],"Google-Extended":["partial","named",25,1],"Applebot-Extended":["partial","*",27,1],"CCBot":["partial","*",27,1],"Bytespider":["partial","*",27,1],"Amazonbot":["partial","*",27,1],"meta-externalagent":["partial","*",27,1],"cohere-ai":["partial","*",27,1],"Diffbot":["partial","*",27,1],"ContextFetchBot":["partial","*",27,1]},"llms":{"status":"present","http":200,"bytes":285},"home":{"status":"too_large","error":"response larger than 5 MB"}},
{"rank":2800,"domain":"ornl.gov","origin":"https://ornl.gov","robots":{"status":"ok","http":200,"url":"https://www.ornl.gov/robots.txt","bytes":128,"groups":1},"bots":{"GPTBot":["partial","*",1,1],"ChatGPT-User":["partial","*",1,1],"OAI-SearchBot":["partial","*",1,1],"ClaudeBot":["partial","*",1,1],"Claude-User":["partial","*",1,1],"Claude-SearchBot":["partial","*",1,1],"anthropic-ai":["partial","*",1,1],"PerplexityBot":["partial","*",1,1],"Perplexity-User":["partial","*",1,1],"Google-Extended":["partial","*",1,1],"Applebot-Extended":["partial","*",1,1],"CCBot":["partial","*",1,1],"Bytespider":["partial","*",1,1],"Amazonbot":["partial","*",1,1],"meta-externalagent":["partial","*",1,1],"cohere-ai":["partial","*",1,1],"Diffbot":["partial","*",1,1],"ContextFetchBot":["partial","*",1,1]},"llms":{"status":"http_error","http":403,"bytes":5539},"home":{"status":"http_error","http":403,"url":"https://ornl.gov/"}},
{"rank":2801,"domain":"embedly.com","origin":"https://www.embedly.com","robots":{"status":"missing","http":404,"url":"https://embed.ly/robots.txt","bytes":925,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":430},"home":{"status":"ok","http":200,"url":"https://embed.ly/","html_tokens":4378,"markdown_tokens":130,"pruned_tokens":130}},
{"rank":2802,"domain":"futurelearn.com","origin":"https://futurelearn.com","robots":{"status":"ok","http":200,"url":"https://assets.futurelearn.com/robots.txt","bytes":343,"groups":1},"bots":{"GPTBot":["partial","*",7,2],"ChatGPT-User":["partial","*",7,2],"OAI-SearchBot":["partial","*",7,2],"ClaudeBot":["partial","*",7,2],"Claude-User":["partial","*",7,2],"Claude-SearchBot":["partial","*",7,2],"anthropic-ai":["partial","*",7,2],"PerplexityBot":["partial","*",7,2],"Perplexity-User":["partial","*",7,2],"Google-Extended":["partial","*",7,2],"Applebot-Extended":["partial","*",7,2],"CCBot":["partial","*",7,2],"Bytespider":["partial","*",7,2],"Amazonbot":["partial","*",7,2],"meta-externalagent":["partial","*",7,2],"cohere-ai":["partial","*",7,2],"Diffbot":["partial","*",7,2],"ContextFetchBot":["partial","*",7,2]},"llms":{"status":"missing","http":404,"bytes":173438},"home":{"status":"http_error","http":403,"url":"https://futurelearn.com/"}},
{"rank":2803,"domain":"garant.ru","origin":"https://garant.ru","robots":{"status":"ok","http":200,"url":"https://www.garant.ru/robots.txt","bytes":4294,"groups":3},"bots":{"GPTBot":["partial","*",24,1],"ChatGPT-User":["partial","*",24,1],"OAI-SearchBot":["partial","*",24,1],"ClaudeBot":["partial","*",24,1],"Claude-User":["partial","*",24,1],"Claude-SearchBot":["partial","*",24,1],"anthropic-ai":["partial","*",24,1],"PerplexityBot":["partial","*",24,1],"Perplexity-User":["partial","*",24,1],"Google-Extended":["partial","*",24,1],"Applebot-Extended":["partial","*",24,1],"CCBot":["partial","*",24,1],"Bytespider":["partial","*",24,1],"Amazonbot":["partial","*",24,1],"meta-externalagent":["partial","*",24,1],"cohere-ai":["partial","*",24,1],"Diffbot":["partial","*",24,1],"ContextFetchBot":["partial","*",24,1]},"llms":{"status":"present","http":200,"bytes":2675},"home":{"status":"ok","http":200,"url":"https://www.garant.ru/","html_tokens":66411,"markdown_tokens":83,"pruned_tokens":83}},
{"rank":2804,"domain":"bitchute.com","origin":"https://bitchute.com","robots":{"status":"ok","http":200,"url":"https://bitchute.com/robots.txt","bytes":2676,"groups":32},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["allowed","*",0,1],"OAI-SearchBot":["allowed","*",0,1],"ClaudeBot":["blocked","named",1,0],"Claude-User":["allowed","*",0,1],"Claude-SearchBot":["allowed","*",0,1],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["allowed","*",0,1],"Perplexity-User":["allowed","*",0,1],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["allowed","*",0,1]},"llms":{"status":"html","http":200,"bytes":6780},"home":{"status":"ok","http":200,"url":"https://www.bitchute.com/","html_tokens":1930,"markdown_tokens":0,"pruned_tokens":0}},
{"rank":2805,"domain":"uc.edu","origin":"https://uc.edu","robots":{"status":"ok","http":200,"url":"https://www.uc.edu/robots.txt","bytes":636,"groups":3},"bots":{"GPTBot":["partial","*",9,7],"ChatGPT-User":["partial","*",9,7],"OAI-SearchBot":["partial","*",9,7],"ClaudeBot":["partial","*",9,7],"Claude-User":["partial","*",9,7],"Claude-SearchBot":["partial","*",9,7],"anthropic-ai":["partial","*",9,7],"PerplexityBot":["partial","*",9,7],"Perplexity-User":["partial","*",9,7],"Google-Extended":["partial","*",9,7],"Applebot-Extended":["partial","*",9,7],"CCBot":["partial","*",9,7],"Bytespider":["blocked","named",1,0],"Amazonbot":["partial","*",9,7],"meta-externalagent":["partial","*",9,7],"cohere-ai":["partial","*",9,7],"Diffbot":["partial","*",9,7],"ContextFetchBot":["partial","*",9,7]},"llms":{"status":"present","http":200,"bytes":500},"home":{"status":"ok","http":200,"url":"https://www.uc.edu/","html_tokens":301762,"markdown_tokens":956,"pruned_tokens":218}},
{"rank":2806,"domain":"bcorporation.net","origin":"https://bcorporation.net","robots":{"status":"ok","http":200,"url":"https://www.bcorporation.net/robots.txt","bytes":111,"groups":1},"bots":{"GPTBot":["allowed","*",0,1],"ChatGPT-User":["allowed","*",0,1],"OAI-SearchBot":["allowed","*",0,1],"ClaudeBot":["allowed","*",0,1],"Claude-User":["allowed","*",0,1],"Claude-SearchBot":["allowed","*",0,1],"anthropic-ai":["allowed","*",0,1],"PerplexityBot":["allowed","*",0,1],"Perplexity-User":["allowed","*",0,1],"Google-Extended":["allowed","*",0,1],"Applebot-Extended":["allowed","*",0,1],"CCBot":["allowed","*",0,1],"Bytespider":["allowed","*",0,1],"Amazonbot":["allowed","*",0,1],"meta-externalagent":["allowed","*",0,1],"cohere-ai":["allowed","*",0,1],"Diffbot":["allowed","*",0,1],"ContextFetchBot":["allowed","*",0,1]},"llms":{"status":"missing","http":404,"bytes":26409},"home":{"status":"http_error","http":403,"url":"https://bcorporation.net/"}},
{"rank":2807,"domain":"media-amazon.com","origin":"https://media-amazon.com","robots":{"status":"unreachable","http":null,"url":"https://media-amazon.com/robots.txt","bytes":0,"groups":0,"error":"cannot resolve media-amazon.com"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":2808,"domain":"12377.cn","origin":"https://www.12377.cn","robots":{"status":"refused","http":403,"url":"https://www.12377.cn/robots.txt","bytes":7384,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":2809,"domain":"q4cdn.com","origin":"http://q4cdn.com","robots":{"status":"refused","http":403,"url":"http://q4cdn.com/robots.txt","bytes":4542,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":2810,"domain":"inaturalist.org","origin":"https://inaturalist.org","robots":{"status":"ok","http":200,"url":"https://www.inaturalist.org/robots.txt","bytes":6693,"groups":33},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["partial","*",147,4],"OAI-SearchBot":["partial","*",147,4],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",147,4],"Claude-SearchBot":["partial","*",147,4],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["partial","*",147,4],"Perplexity-User":["partial","*",147,4],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",147,4]},"llms":{"status":"http_error","http":403,"bytes":5551},"home":{"status":"ok","http":200,"url":"https://www.inaturalist.org/","html_tokens":17533,"markdown_tokens":900,"pruned_tokens":408}},
{"rank":2811,"domain":"mos.ru","origin":"https://mos.ru","robots":{"status":"ok","http":200,"url":"https://www.mos.ru/robots.txt","bytes":43178,"groups":740},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["blocked","named",1,0],"OAI-SearchBot":["partial","*",167,10],"ClaudeBot":["partial","*",167,10],"Claude-User":["partial","*",167,10],"Claude-SearchBot":["partial","*",167,10],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["partial","*",167,10],"Perplexity-User":["partial","*",167,10],"Google-Extended":["partial","*",167,10],"Applebot-Extended":["partial","*",167,10],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["partial","*",167,10],"meta-externalagent":["partial","*",167,10],"cohere-ai":["blocked","named",1,0],"Diffbot":["partial","*",167,10],"ContextFetchBot":["partial","*",167,10]},"llms":{"status":"missing","http":404,"bytes":39484},"home":{"status":"ok","http":200,"url":"https://www.mos.ru/","html_tokens":87577,"markdown_tokens":98,"pruned_tokens":98}},
{"rank":2812,"domain":"kayak.com","origin":"https://kayak.com","robots":{"status":"ok","http":200,"url":"https://www.kayak.com/robots.txt","bytes":42020,"groups":23},"bots":{"GPTBot":["partial","*",115,23],"ChatGPT-User":["partial","named",19,5],"OAI-SearchBot":["partial","*",115,23],"ClaudeBot":["partial","*",115,23],"Claude-User":["partial","*",115,23],"Claude-SearchBot":["partial","*",115,23],"anthropic-ai":["partial","*",115,23],"PerplexityBot":["partial","*",115,23],"Perplexity-User":["partial","*",115,23],"Google-Extended":["partial","*",115,23],"Applebot-Extended":["partial","*",115,23],"CCBot":["partial","*",115,23],"Bytespider":["partial","*",115,23],"Amazonbot":["partial","*",115,23],"meta-externalagent":["partial","*",115,23],"cohere-ai":["partial","*",115,23],"Diffbot":["partial","*",115,23],"ContextFetchBot":["partial","*",115,23]},"llms":{"status":"present","http":200,"bytes":7474},"home":{"status":"ok","http":200,"url":"https://www.kayak.com/","html_tokens":408759,"markdown_tokens":622,"pruned_tokens":494}},
{"rank":2813,"domain":"lexpress.fr","origin":"https://lexpress.fr","robots":{"status":"ok","http":200,"url":"https://www.lexpress.fr/robots.txt","bytes":6008,"groups":59},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["partial","named",1,4],"OAI-SearchBot":["partial","named",1,4],"ClaudeBot":["partial","named",1,4],"Claude-User":["partial","named",1,4],"Claude-SearchBot":["partial","named",1,4],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["partial","named",1,4],"Perplexity-User":["partial","*",7,0],"Google-Extended":["partial","named",1,4],"Applebot-Extended":["partial","named",1,4],"CCBot":["blocked","named",1,0],"Bytespider":["partial","*",7,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["partial","*",7,0],"ContextFetchBot":["partial","*",7,0]},"llms":{"status":"missing","http":404,"bytes":85648},"home":{"status":"ok","http":200,"url":"https://www.lexpress.fr/","html_tokens":382967,"markdown_tokens":202,"pruned_tokens":150}},
{"rank":2814,"domain":"prf.hn","origin":"https://prf.hn","robots":{"status":"ok","http":204,"url":"https://prf.hn/robots.txt","bytes":0,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"http_error","http":403,"bytes":146},"home":{"status":"not_html","error":"unsupported content type application/octet-stream"}},
{"rank":2815,"domain":"bristol.ac.uk","origin":"https://bristol.ac.uk","robots":{"status":"refused","http":403,"url":"https://www.bristol.ac.uk/robots.txt","bytes":3175,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":2816,"domain":"artsy.net","origin":"https://artsy.net","robots":{"status":"ok","http":200,"url":"https://www.artsy.net/robots.txt","bytes":1160,"groups":1},"bots":{"GPTBot":["partial","*",5,0],"ChatGPT-User":["partial","*",5,0],"OAI-SearchBot":["partial","*",5,0],"ClaudeBot":["partial","*",5,0],"Claude-User":["partial","*",5,0],"Claude-SearchBot":["partial","*",5,0],"anthropic-ai":["partial","*",5,0],"PerplexityBot":["partial","*",5,0],"Perplexity-User":["partial","*",5,0],"Google-Extended":["partial","*",5,0],"Applebot-Extended":["partial","*",5,0],"CCBot":["partial","*",5,0],"Bytespider":["partial","*",5,0],"Amazonbot":["partial","*",5,0],"meta-externalagent":["partial","*",5,0],"cohere-ai":["partial","*",5,0],"Diffbot":["partial","*",5,0],"ContextFetchBot":["partial","*",5,0]},"llms":{"status":"present","http":200,"bytes":4452},"home":{"status":"ok","http":200,"url":"https://www.artsy.net/","html_tokens":66757,"markdown_tokens":43,"pruned_tokens":43}},
{"rank":2817,"domain":"harpercollins.com","origin":"https://harpercollins.com","robots":{"status":"ok","http":200,"url":"https://www.harpercollins.com/robots.txt","bytes":3683,"groups":2},"bots":{"GPTBot":["partial","*",40,23],"ChatGPT-User":["partial","*",40,23],"OAI-SearchBot":["partial","*",40,23],"ClaudeBot":["partial","*",40,23],"Claude-User":["partial","*",40,23],"Claude-SearchBot":["partial","*",40,23],"anthropic-ai":["partial","*",40,23],"PerplexityBot":["partial","*",40,23],"Perplexity-User":["partial","*",40,23],"Google-Extended":["partial","*",40,23],"Applebot-Extended":["partial","*",40,23],"CCBot":["partial","*",40,23],"Bytespider":["partial","*",40,23],"Amazonbot":["partial","*",40,23],"meta-externalagent":["partial","*",40,23],"cohere-ai":["partial","*",40,23],"Diffbot":["partial","*",40,23],"ContextFetchBot":["partial","*",40,23]},"llms":{"status":"present","http":200,"bytes":5838},"home":{"status":"http_error","http":403,"url":"https://www.harpercollins.com/"}},
{"rank":2818,"domain":"electrek.co","origin":"https://electrek.co","robots":{"status":"ok","http":200,"url":"https://electrek.co/robots.txt","bytes":911,"groups":4},"bots":{"GPTBot":["partial","*",4,0],"ChatGPT-User":["partial","*",4,0],"OAI-SearchBot":["partial","*",4,0],"ClaudeBot":["partial","*",4,0],"Claude-User":["partial","*",4,0],"Claude-SearchBot":["partial","*",4,0],"anthropic-ai":["partial","*",4,0],"PerplexityBot":["partial","*",4,0],"Perplexity-User":["partial","*",4,0],"Google-Extended":["partial","*",4,0],"Applebot-Extended":["partial","*",4,0],"CCBot":["partial","*",4,0],"Bytespider":["partial","*",4,0],"Amazonbot":["partial","*",4,0],"meta-externalagent":["partial","*",4,0],"cohere-ai":["partial","*",4,0],"Diffbot":["partial","*",4,0],"ContextFetchBot":["partial","*",4,0]},"llms":{"status":"missing","http":404,"bytes":109040},"home":{"status":"ok","http":200,"url":"https://electrek.co/","html_tokens":73302,"markdown_tokens":2121,"pruned_tokens":113}},
{"rank":2819,"domain":"soton.ac.uk","origin":"https://www.soton.ac.uk","robots":{"status":"ok","http":200,"url":"https://www.southampton.ac.uk/robots.txt","bytes":2736,"groups":1},"bots":{"GPTBot":["partial","*",69,18],"ChatGPT-User":["partial","*",69,18],"OAI-SearchBot":["partial","*",69,18],"ClaudeBot":["partial","*",69,18],"Claude-User":["partial","*",69,18],"Claude-SearchBot":["partial","*",69,18],"anthropic-ai":["partial","*",69,18],"PerplexityBot":["partial","*",69,18],"Perplexity-User":["partial","*",69,18],"Google-Extended":["partial","*",69,18],"Applebot-Extended":["partial","*",69,18],"CCBot":["partial","*",69,18],"Bytespider":["partial","*",69,18],"Amazonbot":["partial","*",69,18],"meta-externalagent":["partial","*",69,18],"cohere-ai":["partial","*",69,18],"Diffbot":["partial","*",69,18],"ContextFetchBot":["partial","*",69,18]},"llms":{"status":"missing","http":404,"bytes":14114},"home":{"status":"ok","http":200,"url":"https://www.southampton.ac.uk/","html_tokens":72852,"markdown_tokens":531,"pruned_tokens":124}},
{"rank":2820,"domain":"gm.com","origin":"https://gm.com","robots":{"status":"refused","http":403,"url":"https://www.gm.com/robots.txt","bytes":378,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":2821,"domain":"chip.de","origin":"https://chip.de","robots":{"status":"refused","http":403,"url":"https://chip.de/robots.txt","bytes":371,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":2822,"domain":"postimg.org","origin":"https://postimg.org","robots":{"status":"unreachable","http":null,"url":"https://postimg.org/robots.txt","bytes":0,"groups":0,"error":"cannot resolve postimg.org"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":2823,"domain":"m.media-amazon.com","origin":"https://m.media-amazon.com","robots":{"status":"ok","http":200,"url":"https://m.media-amazon.com/robots.txt","bytes":263,"groups":4},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["blocked","named",1,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"http_error","http":400,"bytes":11},"home":{"status":"not_html","error":"unsupported content type text/plain"}},
{"rank":2824,"domain":"jekyllrb.com","origin":"https://jekyllrb.com","robots":{"status":"ok","http":200,"url":"https://jekyllrb.com/robots.txt","bytes":42,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":5445},"home":{"status":"ok","http":200,"url":"https://jekyllrb.com/","html_tokens":2593,"markdown_tokens":129,"pruned_tokens":92}},
{"rank":2825,"domain":"amzn.eu","origin":"https://amzn.eu","robots":{"status":"missing","http":404,"url":"https://amzn.eu/robots.txt","bytes":28,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":28},"home":{"status":"http_error","http":404,"url":"https://www.amazon.co.uk/gp/amazon-eu.html"}},
{"rank":2826,"domain":"ocregister.com","origin":"https://ocregister.com","robots":{"status":"ok","http":200,"url":"https://www.ocregister.com/robots.txt","bytes":4480,"groups":68},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["blocked","named",1,0],"OAI-SearchBot":["blocked","named",1,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["blocked","named",1,0],"Claude-SearchBot":["blocked","named",1,0],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["blocked","named",1,0],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["partial","*",8,1],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",8,1]},"llms":{"status":"missing","http":404,"bytes":118047},"home":{"status":"ok","http":200,"url":"https://www.ocregister.com/","html_tokens":61611,"markdown_tokens":226,"pruned_tokens":175}},
{"rank":2827,"domain":"findagrave.com","origin":"https://findagrave.com","robots":{"status":"ok","http":200,"url":"https://www.findagrave.com/robots.txt","bytes":207,"groups":1},"bots":{"GPTBot":["partial","*",5,0],"ChatGPT-User":["partial","*",5,0],"OAI-SearchBot":["partial","*",5,0],"ClaudeBot":["partial","*",5,0],"Claude-User":["partial","*",5,0],"Claude-SearchBot":["partial","*",5,0],"anthropic-ai":["partial","*",5,0],"PerplexityBot":["partial","*",5,0],"Perplexity-User":["partial","*",5,0],"Google-Extended":["partial","*",5,0],"Applebot-Extended":["partial","*",5,0],"CCBot":["partial","*",5,0],"Bytespider":["partial","*",5,0],"Amazonbot":["partial","*",5,0],"meta-externalagent":["partial","*",5,0],"cohere-ai":["partial","*",5,0],"Diffbot":["partial","*",5,0],"ContextFetchBot":["partial","*",5,0]},"llms":{"status":"http_error","http":403,"bytes":5525},"home":{"status":"http_error","http":403,"url":"https://findagrave.com/"}},
{"rank":2828,"domain":"dpbolvw.net","origin":"https://www.dpbolvw.net","robots":{"status":"ok","http":200,"url":"https://www.dpbolvw.net/robots.txt","bytes":35,"groups":1},"bots":{"GPTBot":["blocked","*",1,0],"ChatGPT-User":["blocked","*",1,0],"OAI-SearchBot":["blocked","*",1,0],"ClaudeBot":["blocked","*",1,0],"Claude-User":["blocked","*",1,0],"Claude-SearchBot":["blocked","*",1,0],"anthropic-ai":["blocked","*",1,0],"PerplexityBot":["blocked","*",1,0],"Perplexity-User":["blocked","*",1,0],"Google-Extended":["blocked","*",1,0],"Applebot-Extended":["blocked","*",1,0],"CCBot":["blocked","*",1,0],"Bytespider":["blocked","*",1,0],"Amazonbot":["blocked","*",1,0],"meta-externalagent":["blocked","*",1,0],"cohere-ai":["blocked","*",1,0],"Diffbot":["blocked","*",1,0],"ContextFetchBot":["blocked","*",1,0]},"llms":{"status":"disallowed"},"home":{"status":"disallowed"}},
{"rank":2829,"domain":"aappublications.org","origin":"https://aappublications.org","robots":{"status":"ok","http":200,"url":"https://publications.aap.org/robots.txt","bytes":3400,"groups":4},"bots":{"GPTBot":["partial","*",36,3],"ChatGPT-User":["partial","*",36,3],"OAI-SearchBot":["partial","*",36,3],"ClaudeBot":["partial","*",36,3],"Claude-User":["partial","*",36,3],"Claude-SearchBot":["partial","*",36,3],"anthropic-ai":["partial","*",36,3],"PerplexityBot":["partial","*",36,3],"Perplexity-User":["partial","*",36,3],"Google-Extended":["allowed","named",0,1],"Applebot-Extended":["partial","*",36,3],"CCBot":["partial","*",36,3],"Bytespider":["partial","*",36,3],"Amazonbot":["partial","*",36,3],"meta-externalagent":["partial","*",36,3],"cohere-ai":["partial","*",36,3],"Diffbot":["partial","*",36,3],"ContextFetchBot":["partial","*",36,3]},"llms":{"status":"http_error","http":403,"bytes":5531},"home":{"status":"http_error","http":403,"url":"https://publications.aap.org"}},
{"rank":2830,"domain":"gd.gov.cn","origin":"https://www.gd.gov.cn","robots":{"status":"missing","http":404,"url":"https://www.gd.gov.cn/robots.txt","bytes":146,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":146},"home":{"status":"ok","http":200,"url":"https://www.gd.gov.cn/","html_tokens":26901,"markdown_tokens":68,"pruned_tokens":68}},
{"rank":2831,"domain":"gamblingtherapy.org","origin":"https://gamblingtherapy.org","robots":{"status":"timeout","http":null,"url":"https://gamblingtherapy.org/robots.txt","bytes":0,"groups":0,"error":"timed out after 10 s"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":2832,"domain":"posterous.com","origin":"http://posterous.com","robots":{"status":"missing","http":404,"url":"http://posterous.com/robots.txt","bytes":548,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":548},"home":{"status":"http_error","http":404,"url":"http://posterous.com/"}},
{"rank":2833,"domain":"home.pl","origin":"https://home.pl","robots":{"status":"ok","http":200,"url":"https://home.pl/robots.txt","bytes":82,"groups":1},"bots":{"GPTBot":["allowed","*",0,1],"ChatGPT-User":["allowed","*",0,1],"OAI-SearchBot":["allowed","*",0,1],"ClaudeBot":["allowed","*",0,1],"Claude-User":["allowed","*",0,1],"Claude-SearchBot":["allowed","*",0,1],"anthropic-ai":["allowed","*",0,1],"PerplexityBot":["allowed","*",0,1],"Perplexity-User":["allowed","*",0,1],"Google-Extended":["allowed","*",0,1],"Applebot-Extended":["allowed","*",0,1],"CCBot":["allowed","*",0,1],"Bytespider":["allowed","*",0,1],"Amazonbot":["allowed","*",0,1],"meta-externalagent":["allowed","*",0,1],"cohere-ai":["allowed","*",0,1],"Diffbot":["allowed","*",0,1],"ContextFetchBot":["allowed","*",0,1]},"llms":{"status":"present","http":200,"bytes":6629},"home":{"status":"ok","http":200,"url":"https://home.pl/","html_tokens":140644,"markdown_tokens":208,"pruned_tokens":50}},
{"rank":2834,"domain":"familysearch.org","origin":"https://familysearch.org","robots":{"status":"ok","http":200,"url":"https://www.familysearch.org/robots.txt","bytes":2723,"groups":6},"bots":{"GPTBot":["partial","*",44,4],"ChatGPT-User":["partial","*",44,4],"OAI-SearchBot":["partial","*",44,4],"ClaudeBot":["partial","*",44,4],"Claude-User":["partial","*",44,4],"Claude-SearchBot":["partial","*",44,4],"anthropic-ai":["partial","*",44,4],"PerplexityBot":["partial","*",44,4],"Perplexity-User":["partial","*",44,4],"Google-Extended":["partial","*",44,4],"Applebot-Extended":["partial","*",44,4],"CCBot":["partial","*",44,4],"Bytespider":["partial","*",44,4],"Amazonbot":["partial","*",44,4],"meta-externalagent":["partial","*",44,4],"cohere-ai":["partial","*",44,4],"Diffbot":["partial","*",44,4],"ContextFetchBot":["partial","*",44,4]},"llms":{"status":"present","http":200,"bytes":2698},"home":{"status":"http_error","http":403,"url":"https://www.familysearch.org/"}},
{"rank":2835,"domain":"unrealengine.com","origin":"https://unrealengine.com","robots":{"status":"refused","http":403,"url":"https://www.unrealengine.com/robots.txt","bytes":25363,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":2836,"domain":"krebsonsecurity.com","origin":"https://krebsonsecurity.com","robots":{"status":"ok","http":200,"url":"https://krebsonsecurity.com/robots.txt","bytes":260,"groups":4},"bots":{"GPTBot":["partial","*",4,0],"ChatGPT-User":["partial","*",4,0],"OAI-SearchBot":["partial","*",4,0],"ClaudeBot":["partial","*",4,0],"Claude-User":["partial","*",4,0],"Claude-SearchBot":["partial","*",4,0],"anthropic-ai":["partial","*",4,0],"PerplexityBot":["partial","*",4,0],"Perplexity-User":["partial","*",4,0],"Google-Extended":["partial","*",4,0],"Applebot-Extended":["partial","*",4,0],"CCBot":["partial","*",4,0],"Bytespider":["partial","*",4,0],"Amazonbot":["partial","*",4,0],"meta-externalagent":["partial","*",4,0],"cohere-ai":["partial","*",4,0],"Diffbot":["partial","*",4,0],"ContextFetchBot":["partial","*",4,0]},"llms":{"status":"missing","http":404,"bytes":30641},"home":{"status":"ok","http":200,"url":"https://krebsonsecurity.com/","html_tokens":28435,"markdown_tokens":1967,"pruned_tokens":246}},
{"rank":2837,"domain":"transparency.org","origin":"https://transparency.org","robots":{"status":"ok","http":200,"url":"https://www.transparency.org/robots.txt","bytes":745,"groups":1},"bots":{"GPTBot":["partial","*",5,0],"ChatGPT-User":["partial","*",5,0],"OAI-SearchBot":["partial","*",5,0],"ClaudeBot":["partial","*",5,0],"Claude-User":["partial","*",5,0],"Claude-SearchBot":["partial","*",5,0],"anthropic-ai":["partial","*",5,0],"PerplexityBot":["partial","*",5,0],"Perplexity-User":["partial","*",5,0],"Google-Extended":["partial","*",5,0],"Applebot-Extended":["partial","*",5,0],"CCBot":["partial","*",5,0],"Bytespider":["partial","*",5,0],"Amazonbot":["partial","*",5,0],"meta-externalagent":["partial","*",5,0],"cohere-ai":["partial","*",5,0],"Diffbot":["partial","*",5,0],"ContextFetchBot":["partial","*",5,0]},"llms":{"status":"present","http":200,"bytes":84625},"home":{"status":"ok","http":200,"url":"https://www.transparency.org/en/","html_tokens":76256,"markdown_tokens":416,"pruned_tokens":42}},
{"rank":2838,"domain":"venmo.com","origin":"https://venmo.com","robots":{"status":"ok","http":200,"url":"https://venmo.com/robots.txt","bytes":239,"groups":1},"bots":{"GPTBot":["partial","*",8,1],"ChatGPT-User":["partial","*",8,1],"OAI-SearchBot":["partial","*",8,1],"ClaudeBot":["partial","*",8,1],"Claude-User":["partial","*",8,1],"Claude-SearchBot":["partial","*",8,1],"anthropic-ai":["partial","*",8,1],"PerplexityBot":["partial","*",8,1],"Perplexity-User":["partial","*",8,1],"Google-Extended":["partial","*",8,1],"Applebot-Extended":["partial","*",8,1],"CCBot":["partial","*",8,1],"Bytespider":["partial","*",8,1],"Amazonbot":["partial","*",8,1],"meta-externalagent":["partial","*",8,1],"cohere-ai":["partial","*",8,1],"Diffbot":["partial","*",8,1],"ContextFetchBot":["partial","*",8,1]},"llms":{"status":"missing","http":404,"bytes":0},"home":{"status":"ok","http":200,"url":"https://venmo.com/","html_tokens":65225,"markdown_tokens":121,"pruned_tokens":121}},
{"rank":2839,"domain":"petapixel.com","origin":"https://petapixel.com","robots":{"status":"ok","http":200,"url":"https://petapixel.com/robots.txt","bytes":970,"groups":9},"bots":{"GPTBot":["partial","*",23,2],"ChatGPT-User":["partial","*",23,2],"OAI-SearchBot":["partial","*",23,2],"ClaudeBot":["partial","*",23,2],"Claude-User":["partial","*",23,2],"Claude-SearchBot":["partial","*",23,2],"anthropic-ai":["partial","*",23,2],"PerplexityBot":["partial","*",23,2],"Perplexity-User":["partial","*",23,2],"Google-Extended":["partial","*",23,2],"Applebot-Extended":["partial","*",23,2],"CCBot":["partial","*",23,2],"Bytespider":["partial","*",23,2],"Amazonbot":["partial","*",23,2],"meta-externalagent":["partial","*",23,2],"cohere-ai":["partial","*",23,2],"Diffbot":["partial","*",23,2],"ContextFetchBot":["partial","*",23,2]},"llms":{"status":"missing","http":404,"bytes":60},"home":{"status":"http_error","http":403,"url":"https://petapixel.com/"}},
{"rank":2840,"domain":"itmedia.co.jp","origin":"https://itmedia.co.jp","robots":{"status":"html","http":200,"url":"https://www.itmedia.co.jp/","bytes":84140,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":2841,"domain":"devowl.io","origin":"https://devowl.io","robots":{"status":"ok","http":200,"url":"https://devowl.io/robots.txt","bytes":111,"groups":1},"bots":{"GPTBot":["partial","*",1,1],"ChatGPT-User":["partial","*",1,1],"OAI-SearchBot":["partial","*",1,1],"ClaudeBot":["partial","*",1,1],"Claude-User":["partial","*",1,1],"Claude-SearchBot":["partial","*",1,1],"anthropic-ai":["partial","*",1,1],"PerplexityBot":["partial","*",1,1],"Perplexity-User":["partial","*",1,1],"Google-Extended":["partial","*",1,1],"Applebot-Extended":["partial","*",1,1],"CCBot":["partial","*",1,1],"Bytespider":["partial","*",1,1],"Amazonbot":["partial","*",1,1],"meta-externalagent":["partial","*",1,1],"cohere-ai":["partial","*",1,1],"Diffbot":["partial","*",1,1],"ContextFetchBot":["partial","*",1,1]},"llms":{"status":"missing","http":404,"bytes":209296},"home":{"status":"ok","http":200,"url":"https://devowl.io/","html_tokens":62026,"markdown_tokens":603,"pruned_tokens":116}},
{"rank":2842,"domain":"vine.co","origin":"https://vine.co","robots":{"status":"ok","http":200,"url":"https://vine.co/robots.txt","bytes":88,"groups":2},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"missing","http":404,"bytes":5383},"home":{"status":"ok","http":200,"url":"https://vine.co/","html_tokens":1680,"markdown_tokens":39,"pruned_tokens":39}},
{"rank":2843,"domain":"icons8.com","origin":"https://icons8.com","robots":{"status":"ok","http":200,"url":"https://icons8.com/robots.txt","bytes":7231,"groups":6},"bots":{"GPTBot":["partial","named",2,0],"ChatGPT-User":["partial","*",268,8],"OAI-SearchBot":["partial","*",268,8],"ClaudeBot":["partial","named",2,0],"Claude-User":["partial","*",268,8],"Claude-SearchBot":["partial","*",268,8],"anthropic-ai":["partial","*",268,8],"PerplexityBot":["partial","*",268,8],"Perplexity-User":["partial","*",268,8],"Google-Extended":["partial","*",268,8],"Applebot-Extended":["partial","*",268,8],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["partial","*",268,8],"meta-externalagent":["partial","*",268,8],"cohere-ai":["partial","*",268,8],"Diffbot":["partial","*",268,8],"ContextFetchBot":["partial","*",268,8]},"llms":{"status":"missing","http":404,"bytes":226645},"home":{"status":"http_error","http":403,"url":"https://icons8.com/"}},
{"rank":2844,"domain":"metacafe.com","origin":"https://metacafe.com","robots":{"status":"unreachable","http":null,"url":"https://metacafe.com/robots.txt","bytes":0,"groups":0,"error":"cannot resolve metacafe.com"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":2845,"domain":"perfectdomain.com","origin":"https://perfectdomain.com","robots":{"status":"ok","http":200,"url":"https://perfectdomain.com/robots.txt","bytes":69,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":12709},"home":{"status":"http_error","http":403,"url":"https://perfectdomain.com/"}},
{"rank":2846,"domain":"costco.com","origin":"https://costco.com","robots":{"status":"ok","http":200,"url":"https://www.costco.com/robots.txt","bytes":3997,"groups":1},"bots":{"GPTBot":["partial","*",54,1],"ChatGPT-User":["partial","*",54,1],"OAI-SearchBot":["partial","*",54,1],"ClaudeBot":["partial","*",54,1],"Claude-User":["partial","*",54,1],"Claude-SearchBot":["partial","*",54,1],"anthropic-ai":["partial","*",54,1],"PerplexityBot":["partial","*",54,1],"Perplexity-User":["partial","*",54,1],"Google-Extended":["partial","*",54,1],"Applebot-Extended":["partial","*",54,1],"CCBot":["partial","*",54,1],"Bytespider":["partial","*",54,1],"Amazonbot":["partial","*",54,1],"meta-externalagent":["partial","*",54,1],"cohere-ai":["partial","*",54,1],"Diffbot":["partial","*",54,1],"ContextFetchBot":["partial","*",54,1]},"llms":{"status":"missing","http":404,"bytes":161212},"home":{"status":"ok","http":200,"url":"https://www.costco.com/","html_tokens":795070,"markdown_tokens":29,"pruned_tokens":29}},
{"rank":2847,"domain":"cision.com","origin":"https://cision.com","robots":{"status":"ok","http":200,"url":"https://www.cision.com/robots.txt","bytes":146,"groups":1},"bots":{"GPTBot":["partial","*",3,0],"ChatGPT-User":["partial","*",3,0],"OAI-SearchBot":["partial","*",3,0],"ClaudeBot":["partial","*",3,0],"Claude-User":["partial","*",3,0],"Claude-SearchBot":["partial","*",3,0],"anthropic-ai":["partial","*",3,0],"PerplexityBot":["partial","*",3,0],"Perplexity-User":["partial","*",3,0],"Google-Extended":["partial","*",3,0],"Applebot-Extended":["partial","*",3,0],"CCBot":["partial","*",3,0],"Bytespider":["partial","*",3,0],"Amazonbot":["partial","*",3,0],"meta-externalagent":["partial","*",3,0],"cohere-ai":["partial","*",3,0],"Diffbot":["partial","*",3,0],"ContextFetchBot":["partial","*",3,0]},"llms":{"status":"missing","http":404,"bytes":68409},"home":{"status":"ok","http":200,"url":"https://www.cision.com/","html_tokens":35859,"markdown_tokens":285,"pruned_tokens":285}},
{"rank":2848,"domain":"jmir.org","origin":"https://jmir.org","robots":{"status":"ok","http":200,"url":"https://www.jmir.org/robots.txt","bytes":1514,"groups":2},"bots":{"GPTBot":["partial","*",23,0],"ChatGPT-User":["partial","*",23,0],"OAI-SearchBot":["partial","*",23,0],"ClaudeBot":["partial","*",23,0],"Claude-User":["partial","*",23,0],"Claude-SearchBot":["partial","*",23,0],"anthropic-ai":["partial","*",23,0],"PerplexityBot":["partial","*",23,0],"Perplexity-User":["partial","*",23,0],"Google-Extended":["partial","*",23,0],"Applebot-Extended":["partial","*",23,0],"CCBot":["partial","*",23,0],"Bytespider":["partial","*",23,0],"Amazonbot":["partial","*",23,0],"meta-externalagent":["partial","*",23,0],"cohere-ai":["partial","*",23,0],"Diffbot":["partial","*",23,0],"ContextFetchBot":["partial","*",23,0]},"llms":{"status":"missing","http":404,"bytes":0},"home":{"status":"ok","http":202,"url":"https://jmir.org/","html_tokens":531,"markdown_tokens":40,"pruned_tokens":40}},
{"rank":2849,"domain":"wponetap.com","origin":"https://wponetap.com","robots":{"status":"ok","http":200,"url":"https://wponetap.com/robots.txt","bytes":378,"groups":7},"bots":{"GPTBot":["partial","*",1,2],"ChatGPT-User":["partial","*",1,2],"OAI-SearchBot":["partial","*",1,2],"ClaudeBot":["partial","*",1,2],"Claude-User":["partial","*",1,2],"Claude-SearchBot":["partial","*",1,2],"anthropic-ai":["partial","*",1,2],"PerplexityBot":["partial","*",1,2],"Perplexity-User":["partial","*",1,2],"Google-Extended":["partial","*",1,2],"Applebot-Extended":["partial","*",1,2],"CCBot":["partial","*",1,2],"Bytespider":["partial","*",1,2],"Amazonbot":["partial","*",1,2],"meta-externalagent":["partial","*",1,2],"cohere-ai":["partial","*",1,2],"Diffbot":["partial","*",1,2],"ContextFetchBot":["partial","*",1,2]},"llms":{"status":"other_text","http":200,"bytes":3566},"home":{"status":"ok","http":200,"url":"https://wponetap.com/","html_tokens":215997,"markdown_tokens":2022,"pruned_tokens":112}},
{"rank":2850,"domain":"control-webpanel.com","origin":"https://control-webpanel.com","robots":{"status":"ok","http":200,"url":"https://control-webpanel.com/robots.txt","bytes":121,"groups":1},"bots":{"GPTBot":["partial","*",1,1],"ChatGPT-User":["partial","*",1,1],"OAI-SearchBot":["partial","*",1,1],"ClaudeBot":["partial","*",1,1],"Claude-User":["partial","*",1,1],"Claude-SearchBot":["partial","*",1,1],"anthropic-ai":["partial","*",1,1],"PerplexityBot":["partial","*",1,1],"Perplexity-User":["partial","*",1,1],"Google-Extended":["partial","*",1,1],"Applebot-Extended":["partial","*",1,1],"CCBot":["partial","*",1,1],"Bytespider":["partial","*",1,1],"Amazonbot":["partial","*",1,1],"meta-externalagent":["partial","*",1,1],"cohere-ai":["partial","*",1,1],"Diffbot":["partial","*",1,1],"ContextFetchBot":["partial","*",1,1]},"llms":{"status":"missing","http":404,"bytes":69403},"home":{"status":"ok","http":200,"url":"https://control-webpanel.com/","html_tokens":34267,"markdown_tokens":222,"pruned_tokens":84}},
{"rank":2851,"domain":"patagonia.com","origin":"https://patagonia.com","robots":{"status":"html","http":200,"url":"https://www.patagonia.com/robots.txt","bytes":14373,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":2852,"domain":"gamma.app","origin":"https://gamma.app","robots":{"status":"refused","http":403,"url":"https://gamma.app/robots.txt","bytes":679138,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":2853,"domain":"googlecode.com","origin":"https://googlecode.com","robots":{"status":"missing","http":404,"url":"https://googlecode.com/robots.txt","bytes":1571,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":1569},"home":{"status":"http_error","http":404,"url":"https://googlecode.com/"}},
{"rank":2854,"domain":"pixnet.net","origin":"https://pixnet.net","robots":{"status":"missing","http":404,"url":"https://www.pixnet.net/robots.txt","bytes":17884,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":17884},"home":{"status":"ok","http":200,"url":"https://www.pixnet.net/","html_tokens":105724,"markdown_tokens":199,"pruned_tokens":161}},
{"rank":2855,"domain":"uib.no","origin":"https://uib.no","robots":{"status":"ok","http":200,"url":"https://www.uib.no/robots.txt","bytes":2189,"groups":1},"bots":{"GPTBot":["partial","*",36,32],"ChatGPT-User":["partial","*",36,32],"OAI-SearchBot":["partial","*",36,32],"ClaudeBot":["partial","*",36,32],"Claude-User":["partial","*",36,32],"Claude-SearchBot":["partial","*",36,32],"anthropic-ai":["partial","*",36,32],"PerplexityBot":["partial","*",36,32],"Perplexity-User":["partial","*",36,32],"Google-Extended":["partial","*",36,32],"Applebot-Extended":["partial","*",36,32],"CCBot":["partial","*",36,32],"Bytespider":["partial","*",36,32],"Amazonbot":["partial","*",36,32],"meta-externalagent":["partial","*",36,32],"cohere-ai":["partial","*",36,32],"Diffbot":["partial","*",36,32],"ContextFetchBot":["partial","*",36,32]},"llms":{"status":"missing","http":404,"bytes":294},"home":{"status":"ok","http":200,"url":"https://www4.uib.no","html_tokens":11478,"markdown_tokens":283,"pruned_tokens":82}},
{"rank":2856,"domain":"caesars.com","origin":"https://caesars.com","robots":{"status":"ok","http":200,"url":"https://www.caesars.com/robots.txt","bytes":431,"groups":1},"bots":{"GPTBot":["partial","*",3,0],"ChatGPT-User":["partial","*",3,0],"OAI-SearchBot":["partial","*",3,0],"ClaudeBot":["partial","*",3,0],"Claude-User":["partial","*",3,0],"Claude-SearchBot":["partial","*",3,0],"anthropic-ai":["partial","*",3,0],"PerplexityBot":["partial","*",3,0],"Perplexity-User":["partial","*",3,0],"Google-Extended":["partial","*",3,0],"Applebot-Extended":["partial","*",3,0],"CCBot":["partial","*",3,0],"Bytespider":["partial","*",3,0],"Amazonbot":["partial","*",3,0],"meta-externalagent":["partial","*",3,0],"cohere-ai":["partial","*",3,0],"Diffbot":["partial","*",3,0],"ContextFetchBot":["partial","*",3,0]},"llms":{"status":"missing","http":404,"bytes":3686},"home":{"status":"ok","http":200,"url":"https://caesars.com/","html_tokens":232,"markdown_tokens":21,"pruned_tokens":21}},
{"rank":2857,"domain":"edweek.org","origin":"https://edweek.org","robots":{"status":"ok","http":200,"url":"https://www.edweek.org/robots.txt","bytes":356,"groups":3},"bots":{"GPTBot":["partial","*",6,0],"ChatGPT-User":["partial","*",6,0],"OAI-SearchBot":["partial","*",6,0],"ClaudeBot":["partial","*",6,0],"Claude-User":["partial","*",6,0],"Claude-SearchBot":["partial","*",6,0],"anthropic-ai":["partial","*",6,0],"PerplexityBot":["partial","*",6,0],"Perplexity-User":["partial","*",6,0],"Google-Extended":["partial","*",6,0],"Applebot-Extended":["partial","*",6,0],"CCBot":["partial","*",6,0],"Bytespider":["partial","*",6,0],"Amazonbot":["partial","*",6,0],"meta-externalagent":["partial","*",6,0],"cohere-ai":["partial","*",6,0],"Diffbot":["partial","*",6,0],"ContextFetchBot":["partial","*",6,0]},"llms":{"status":"missing","http":404,"bytes":126270},"home":{"status":"ok","http":200,"url":"https://www.edweek.org/","html_tokens":87799,"markdown_tokens":1291,"pruned_tokens":121}},
{"rank":2858,"domain":"government.ru","origin":"http://government.ru","robots":{"status":"missing","http":404,"url":"http://government.ru/robots.txt","bytes":39807,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":39807},"home":{"status":"ok","http":200,"url":"http://government.ru/","html_tokens":14933,"markdown_tokens":66,"pruned_tokens":66}},
{"rank":2859,"domain":"tkqlhce.com","origin":"https://www.tkqlhce.com","robots":{"status":"ok","http":200,"url":"https://www.tkqlhce.com/robots.txt","bytes":35,"groups":1},"bots":{"GPTBot":["blocked","*",1,0],"ChatGPT-User":["blocked","*",1,0],"OAI-SearchBot":["blocked","*",1,0],"ClaudeBot":["blocked","*",1,0],"Claude-User":["blocked","*",1,0],"Claude-SearchBot":["blocked","*",1,0],"anthropic-ai":["blocked","*",1,0],"PerplexityBot":["blocked","*",1,0],"Perplexity-User":["blocked","*",1,0],"Google-Extended":["blocked","*",1,0],"Applebot-Extended":["blocked","*",1,0],"CCBot":["blocked","*",1,0],"Bytespider":["blocked","*",1,0],"Amazonbot":["blocked","*",1,0],"meta-externalagent":["blocked","*",1,0],"cohere-ai":["blocked","*",1,0],"Diffbot":["blocked","*",1,0],"ContextFetchBot":["blocked","*",1,0]},"llms":{"status":"disallowed"},"home":{"status":"disallowed"}},
{"rank":2860,"domain":"google.cz","origin":"https://google.cz","robots":{"status":"ok","http":200,"url":"https://www.google.cz/robots.txt","bytes":6624,"groups":4},"bots":{"GPTBot":["partial","*",169,64],"ChatGPT-User":["partial","*",169,64],"OAI-SearchBot":["partial","*",169,64],"ClaudeBot":["partial","*",169,64],"Claude-User":["partial","*",169,64],"Claude-SearchBot":["partial","*",169,64],"anthropic-ai":["partial","*",169,64],"PerplexityBot":["partial","*",169,64],"Perplexity-User":["partial","*",169,64],"Google-Extended":["partial","*",169,64],"Applebot-Extended":["partial","*",169,64],"CCBot":["partial","*",169,64],"Bytespider":["partial","*",169,64],"Amazonbot":["partial","*",169,64],"meta-externalagent":["partial","*",169,64],"cohere-ai":["partial","*",169,64],"Diffbot":["partial","*",169,64],"ContextFetchBot":["partial","*",169,64]},"llms":{"status":"missing","http":404,"bytes":1569},"home":{"status":"ok","http":200,"url":"https://www.google.com/","html_tokens":21353,"markdown_tokens":30,"pruned_tokens":30}},
{"rank":2861,"domain":"docs.rs","origin":"https://docs.rs","robots":{"status":"ok","http":200,"url":"https://docs.rs/-/static/robots.txt","bytes":375,"groups":1},"bots":{"GPTBot":["partial","*",3,0],"ChatGPT-User":["partial","*",3,0],"OAI-SearchBot":["partial","*",3,0],"ClaudeBot":["partial","*",3,0],"Claude-User":["partial","*",3,0],"Claude-SearchBot":["partial","*",3,0],"anthropic-ai":["partial","*",3,0],"PerplexityBot":["partial","*",3,0],"Perplexity-User":["partial","*",3,0],"Google-Extended":["partial","*",3,0],"Applebot-Extended":["partial","*",3,0],"CCBot":["partial","*",3,0],"Bytespider":["partial","*",3,0],"Amazonbot":["partial","*",3,0],"meta-externalagent":["partial","*",3,0],"cohere-ai":["partial","*",3,0],"Diffbot":["partial","*",3,0],"ContextFetchBot":["partial","*",3,0]},"llms":{"status":"http_error","http":400,"bytes":6662},"home":{"status":"ok","http":200,"url":"https://docs.rs/","html_tokens":4828,"markdown_tokens":27,"pruned_tokens":27}},
{"rank":2862,"domain":"blogspot.in","origin":"https://blogspot.in","robots":{"status":"ok","http":200,"url":"https://blogspot.in/robots.txt","bytes":6624,"groups":4},"bots":{"GPTBot":["partial","*",169,64],"ChatGPT-User":["partial","*",169,64],"OAI-SearchBot":["partial","*",169,64],"ClaudeBot":["partial","*",169,64],"Claude-User":["partial","*",169,64],"Claude-SearchBot":["partial","*",169,64],"anthropic-ai":["partial","*",169,64],"PerplexityBot":["partial","*",169,64],"Perplexity-User":["partial","*",169,64],"Google-Extended":["partial","*",169,64],"Applebot-Extended":["partial","*",169,64],"CCBot":["partial","*",169,64],"Bytespider":["partial","*",169,64],"Amazonbot":["partial","*",169,64],"meta-externalagent":["partial","*",169,64],"cohere-ai":["partial","*",169,64],"Diffbot":["partial","*",169,64],"ContextFetchBot":["partial","*",169,64]},"llms":{"status":"missing","http":404,"bytes":1569},"home":{"status":"ok","http":200,"url":"https://www.google.com/","html_tokens":21451,"markdown_tokens":30,"pruned_tokens":30}},
{"rank":2863,"domain":"storify.com","origin":"https://storify.com","robots":{"status":"unreachable","http":null,"url":"https://storify.com/robots.txt","bytes":0,"groups":0,"error":"cannot resolve storify.com"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":2864,"domain":"wko.at","origin":"https://wko.at","robots":{"status":"ok","http":200,"url":"https://www.wko.at/robots.txt","bytes":762,"groups":1},"bots":{"GPTBot":["partial","*",3,0],"ChatGPT-User":["partial","*",3,0],"OAI-SearchBot":["partial","*",3,0],"ClaudeBot":["partial","*",3,0],"Claude-User":["partial","*",3,0],"Claude-SearchBot":["partial","*",3,0],"anthropic-ai":["partial","*",3,0],"PerplexityBot":["partial","*",3,0],"Perplexity-User":["partial","*",3,0],"Google-Extended":["partial","*",3,0],"Applebot-Extended":["partial","*",3,0],"CCBot":["partial","*",3,0],"Bytespider":["partial","*",3,0],"Amazonbot":["partial","*",3,0],"meta-externalagent":["partial","*",3,0],"cohere-ai":["partial","*",3,0],"Diffbot":["partial","*",3,0],"ContextFetchBot":["partial","*",3,0]},"llms":{"status":"missing","http":404,"bytes":106955},"home":{"status":"ok","http":200,"url":"https://www.wko.at/","html_tokens":45609,"markdown_tokens":89,"pruned_tokens":89}},
{"rank":2865,"domain":"taplink.cc","origin":"https://taplink.cc","robots":{"status":"ok","http":200,"url":"https://taplink.cc/robots.txt","bytes":215,"groups":1},"bots":{"GPTBot":["partial","*",9,0],"ChatGPT-User":["partial","*",9,0],"OAI-SearchBot":["partial","*",9,0],"ClaudeBot":["partial","*",9,0],"Claude-User":["partial","*",9,0],"Claude-SearchBot":["partial","*",9,0],"anthropic-ai":["partial","*",9,0],"PerplexityBot":["partial","*",9,0],"Perplexity-User":["partial","*",9,0],"Google-Extended":["partial","*",9,0],"Applebot-Extended":["partial","*",9,0],"CCBot":["partial","*",9,0],"Bytespider":["partial","*",9,0],"Amazonbot":["partial","*",9,0],"meta-externalagent":["partial","*",9,0],"cohere-ai":["partial","*",9,0],"Diffbot":["partial","*",9,0],"ContextFetchBot":["partial","*",9,0]},"llms":{"status":"missing","http":404,"bytes":1899},"home":{"status":"ok","http":200,"url":"https://taplink.cc/","html_tokens":3267,"markdown_tokens":546,"pruned_tokens":49}},
{"rank":2866,"domain":"pornhub.com","origin":"https://pornhub.com","robots":{"status":"missing","http":410,"url":"https://pornhub.com/robots.txt","bytes":0,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":410,"bytes":0},"home":{"status":"http_error","http":410,"url":"https://pornhub.com/"}},
{"rank":2867,"domain":"nrel.gov","origin":"https://nrel.gov","robots":{"status":"unreachable","http":null,"url":"https://nrel.gov/robots.txt","bytes":0,"groups":0,"error":"cannot resolve nrel.gov"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":2868,"domain":"123rf.com","origin":"https://123rf.com","robots":{"status":"ok","http":200,"url":"https://www.123rf.com/robots.txt","bytes":639,"groups":7},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["partial","*",13,0],"OAI-SearchBot":["partial","*",13,0],"ClaudeBot":["partial","*",13,0],"Claude-User":["partial","*",13,0],"Claude-SearchBot":["partial","*",13,0],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["partial","*",13,0],"Perplexity-User":["partial","*",13,0],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["partial","*",13,0],"CCBot":["blocked","named",1,0],"Bytespider":["partial","*",13,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["partial","*",13,0],"cohere-ai":["partial","*",13,0],"Diffbot":["partial","*",13,0],"ContextFetchBot":["partial","*",13,0]},"llms":{"status":"missing","http":404,"bytes":146},"home":{"status":"ok","http":200,"url":"https://www.123rf.com/","html_tokens":69904,"markdown_tokens":70,"pruned_tokens":70}},
{"rank":2869,"domain":"fujifilm.com","origin":"https://fujifilm.com","robots":{"status":"ok","http":200,"url":"https://www.fujifilm.com/robots.txt","bytes":25627,"groups":1},"bots":{"GPTBot":["partial","*",720,18],"ChatGPT-User":["partial","*",720,18],"OAI-SearchBot":["partial","*",720,18],"ClaudeBot":["partial","*",720,18],"Claude-User":["partial","*",720,18],"Claude-SearchBot":["partial","*",720,18],"anthropic-ai":["partial","*",720,18],"PerplexityBot":["partial","*",720,18],"Perplexity-User":["partial","*",720,18],"Google-Extended":["partial","*",720,18],"Applebot-Extended":["partial","*",720,18],"CCBot":["partial","*",720,18],"Bytespider":["partial","*",720,18],"Amazonbot":["partial","*",720,18],"meta-externalagent":["partial","*",720,18],"cohere-ai":["partial","*",720,18],"Diffbot":["partial","*",720,18],"ContextFetchBot":["partial","*",720,18]},"llms":{"status":"missing","http":404,"bytes":804},"home":{"status":"ok","http":200,"url":"https://global.fujifilm.com/en","html_tokens":65711,"markdown_tokens":124,"pruned_tokens":124}},
{"rank":2870,"domain":"vcu.edu","origin":"https://vcu.edu","robots":{"status":"ok","http":200,"url":"https://www.vcu.edu/robots.txt","bytes":269,"groups":2},"bots":{"GPTBot":["partial","*",2,1],"ChatGPT-User":["partial","*",2,1],"OAI-SearchBot":["partial","*",2,1],"ClaudeBot":["partial","*",2,1],"Claude-User":["partial","*",2,1],"Claude-SearchBot":["partial","*",2,1],"anthropic-ai":["partial","*",2,1],"PerplexityBot":["partial","*",2,1],"Perplexity-User":["partial","*",2,1],"Google-Extended":["partial","*",2,1],"Applebot-Extended":["partial","*",2,1],"CCBot":["partial","*",2,1],"Bytespider":["partial","*",2,1],"Amazonbot":["partial","*",2,1],"meta-externalagent":["partial","*",2,1],"cohere-ai":["partial","*",2,1],"Diffbot":["partial","*",2,1],"ContextFetchBot":["partial","*",2,1]},"llms":{"status":"missing","http":404,"bytes":169534},"home":{"status":"ok","http":200,"url":"https://www.vcu.edu/","html_tokens":77481,"markdown_tokens":79,"pruned_tokens":30}},
{"rank":2871,"domain":"usdoj.gov","origin":"https://usdoj.gov","robots":{"status":"html","http":200,"url":"https://www.justice.gov/","bytes":102726,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":2872,"domain":"openbsd.org","origin":"https://openbsd.org","robots":{"status":"ok","http":200,"url":"https://www.openbsd.org/robots.txt","bytes":60,"groups":1},"bots":{"GPTBot":["partial","*",2,0],"ChatGPT-User":["partial","*",2,0],"OAI-SearchBot":["partial","*",2,0],"ClaudeBot":["partial","*",2,0],"Claude-User":["partial","*",2,0],"Claude-SearchBot":["partial","*",2,0],"anthropic-ai":["partial","*",2,0],"PerplexityBot":["partial","*",2,0],"Perplexity-User":["partial","*",2,0],"Google-Extended":["partial","*",2,0],"Applebot-Extended":["partial","*",2,0],"CCBot":["partial","*",2,0],"Bytespider":["partial","*",2,0],"Amazonbot":["partial","*",2,0],"meta-externalagent":["partial","*",2,0],"cohere-ai":["partial","*",2,0],"Diffbot":["partial","*",2,0],"ContextFetchBot":["partial","*",2,0]},"llms":{"status":"missing","http":404,"bytes":494},"home":{"status":"ok","http":200,"url":"https://www.openbsd.org/","html_tokens":906,"markdown_tokens":193,"pruned_tokens":175}},
{"rank":2873,"domain":"uni-freiburg.de","origin":"https://uni-freiburg.de","robots":{"status":"ok","http":200,"url":"https://uni-freiburg.de/robots.txt","bytes":116,"groups":1},"bots":{"GPTBot":["partial","*",1,1],"ChatGPT-User":["partial","*",1,1],"OAI-SearchBot":["partial","*",1,1],"ClaudeBot":["partial","*",1,1],"Claude-User":["partial","*",1,1],"Claude-SearchBot":["partial","*",1,1],"anthropic-ai":["partial","*",1,1],"PerplexityBot":["partial","*",1,1],"Perplexity-User":["partial","*",1,1],"Google-Extended":["partial","*",1,1],"Applebot-Extended":["partial","*",1,1],"CCBot":["partial","*",1,1],"Bytespider":["partial","*",1,1],"Amazonbot":["partial","*",1,1],"meta-externalagent":["partial","*",1,1],"cohere-ai":["partial","*",1,1],"Diffbot":["partial","*",1,1],"ContextFetchBot":["partial","*",1,1]},"llms":{"status":"missing","http":404,"bytes":826244},"home":{"status":"ok","http":200,"url":"https://uni-freiburg.de/","html_tokens":230530,"markdown_tokens":631,"pruned_tokens":268}},
{"rank":2874,"domain":"business2community.com","origin":"https://business2community.com","robots":{"status":"ok","http":200,"url":"https://www.business2community.com/robots.txt","bytes":485,"groups":2},"bots":{"GPTBot":["partial","*",10,0],"ChatGPT-User":["partial","*",10,0],"OAI-SearchBot":["partial","*",10,0],"ClaudeBot":["partial","*",10,0],"Claude-User":["partial","*",10,0],"Claude-SearchBot":["partial","*",10,0],"anthropic-ai":["partial","*",10,0],"PerplexityBot":["partial","*",10,0],"Perplexity-User":["partial","*",10,0],"Google-Extended":["partial","*",10,0],"Applebot-Extended":["partial","*",10,0],"CCBot":["partial","*",10,0],"Bytespider":["partial","*",10,0],"Amazonbot":["partial","*",10,0],"meta-externalagent":["partial","*",10,0],"cohere-ai":["partial","*",10,0],"Diffbot":["partial","*",10,0],"ContextFetchBot":["partial","*",10,0]},"llms":{"status":"missing","http":404,"bytes":71868},"home":{"status":"ok","http":200,"url":"https://www.business2community.com/","html_tokens":101842,"markdown_tokens":192,"pruned_tokens":102}},
{"rank":2875,"domain":"netcraft.com","origin":"https://netcraft.com","robots":{"status":"ok","http":200,"url":"https://www.netcraft.com/robots.txt","bytes":69,"groups":1},"bots":{"GPTBot":["allowed","*",0,1],"ChatGPT-User":["allowed","*",0,1],"OAI-SearchBot":["allowed","*",0,1],"ClaudeBot":["allowed","*",0,1],"Claude-User":["allowed","*",0,1],"Claude-SearchBot":["allowed","*",0,1],"anthropic-ai":["allowed","*",0,1],"PerplexityBot":["allowed","*",0,1],"Perplexity-User":["allowed","*",0,1],"Google-Extended":["allowed","*",0,1],"Applebot-Extended":["allowed","*",0,1],"CCBot":["allowed","*",0,1],"Bytespider":["allowed","*",0,1],"Amazonbot":["allowed","*",0,1],"meta-externalagent":["allowed","*",0,1],"cohere-ai":["allowed","*",0,1],"Diffbot":["allowed","*",0,1],"ContextFetchBot":["allowed","*",0,1]},"llms":{"status":"present","http":200,"bytes":4986},"home":{"status":"ok","http":200,"url":"https://www.netcraft.com/","html_tokens":321898,"markdown_tokens":706,"pruned_tokens":166}},
{"rank":2876,"domain":"inews.co.uk","origin":"https://inews.co.uk","robots":{"status":"ok","http":200,"url":"https://inews.co.uk/robots.txt","bytes":4533,"groups":4},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["blocked","named",1,0],"OAI-SearchBot":["blocked","named",1,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["blocked","named",1,0],"Claude-SearchBot":["blocked","named",1,0],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["blocked","named",1,0],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",18,1]},"llms":{"status":"missing","http":404,"bytes":184314},"home":{"status":"ok","http":200,"url":"https://inews.co.uk/","html_tokens":82097,"markdown_tokens":248,"pruned_tokens":147}},
{"rank":2877,"domain":"1password.com","origin":"https://1password.com","robots":{"status":"ok","http":200,"url":"https://1password.com/robots.txt","bytes":141,"groups":1},"bots":{"GPTBot":["partial","*",2,0],"ChatGPT-User":["partial","*",2,0],"OAI-SearchBot":["partial","*",2,0],"ClaudeBot":["partial","*",2,0],"Claude-User":["partial","*",2,0],"Claude-SearchBot":["partial","*",2,0],"anthropic-ai":["partial","*",2,0],"PerplexityBot":["partial","*",2,0],"Perplexity-User":["partial","*",2,0],"Google-Extended":["partial","*",2,0],"Applebot-Extended":["partial","*",2,0],"CCBot":["partial","*",2,0],"Bytespider":["partial","*",2,0],"Amazonbot":["partial","*",2,0],"meta-externalagent":["partial","*",2,0],"cohere-ai":["partial","*",2,0],"Diffbot":["partial","*",2,0],"ContextFetchBot":["partial","*",2,0]},"llms":{"status":"missing","http":404,"bytes":108324},"home":{"status":"ok","http":200,"url":"https://1password.com/","html_tokens":230677,"markdown_tokens":105,"pruned_tokens":85}},
{"rank":2878,"domain":"pbworks.com","origin":"https://pbworks.com","robots":{"status":"missing","http":404,"url":"https://www.pbworks.com/robots.txt","bytes":29374,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":29374},"home":{"status":"ok","http":200,"url":"https://www.pbworks.com/","html_tokens":20299,"markdown_tokens":447,"pruned_tokens":99}},
{"rank":2879,"domain":"seriouseats.com","origin":"https://seriouseats.com","robots":{"status":"ok","http":200,"url":"https://www.seriouseats.com/robots.txt","bytes":3475,"groups":6},"bots":{"GPTBot":["partial","named",1,0],"ChatGPT-User":["partial","named",1,0],"OAI-SearchBot":["partial","named",1,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["blocked","named",1,0],"Claude-SearchBot":["partial","named",1,2],"anthropic-ai":["partial","named",1,2],"PerplexityBot":["partial","named",1,2],"Perplexity-User":["blocked","named",1,0],"Google-Extended":["partial","named",1,2],"Applebot-Extended":["blocked","named",1,0],"CCBot":["partial","named",1,2],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["partial","named",1,2],"cohere-ai":["partial","named",1,2],"Diffbot":["partial","*",2,0],"ContextFetchBot":["partial","*",2,0]},"llms":{"status":"http_error","http":402,"bytes":612},"home":{"status":"http_error","http":402,"url":"https://www.seriouseats.com/"}},
{"rank":2880,"domain":"tutorialspoint.com","origin":"https://tutorialspoint.com","robots":{"status":"ok","http":200,"url":"https://www.tutorialspoint.com/robots.txt","bytes":850,"groups":4},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["blocked","named",1,0],"OAI-SearchBot":["partial","*",24,0],"ClaudeBot":["partial","*",24,0],"Claude-User":["partial","*",24,0],"Claude-SearchBot":["partial","*",24,0],"anthropic-ai":["partial","*",24,0],"PerplexityBot":["partial","*",24,0],"Perplexity-User":["partial","*",24,0],"Google-Extended":["partial","*",24,0],"Applebot-Extended":["partial","*",24,0],"CCBot":["blocked","named",1,0],"Bytespider":["partial","*",24,0],"Amazonbot":["partial","*",24,0],"meta-externalagent":["partial","*",24,0],"cohere-ai":["partial","*",24,0],"Diffbot":["partial","*",24,0],"ContextFetchBot":["partial","*",24,0]},"llms":{"status":"missing","http":404,"bytes":14893},"home":{"status":"ok","http":200,"url":"https://www.tutorialspoint.com/","html_tokens":41766,"markdown_tokens":331,"pruned_tokens":61}},
{"rank":2881,"domain":"gatesfoundation.org","origin":"https://gatesfoundation.org","robots":{"status":"ok","http":200,"url":"https://www.gatesfoundation.org/robots.txt","bytes":85,"groups":1},"bots":{"GPTBot":["allowed","*",0,1],"ChatGPT-User":["allowed","*",0,1],"OAI-SearchBot":["allowed","*",0,1],"ClaudeBot":["allowed","*",0,1],"Claude-User":["allowed","*",0,1],"Claude-SearchBot":["allowed","*",0,1],"anthropic-ai":["allowed","*",0,1],"PerplexityBot":["allowed","*",0,1],"Perplexity-User":["allowed","*",0,1],"Google-Extended":["allowed","*",0,1],"Applebot-Extended":["allowed","*",0,1],"CCBot":["allowed","*",0,1],"Bytespider":["allowed","*",0,1],"Amazonbot":["allowed","*",0,1],"meta-externalagent":["allowed","*",0,1],"cohere-ai":["allowed","*",0,1],"Diffbot":["allowed","*",0,1],"ContextFetchBot":["allowed","*",0,1]},"llms":{"status":"missing","http":404,"bytes":131769},"home":{"status":"ok","http":200,"url":"https://www.gatesfoundation.org/","html_tokens":54662,"markdown_tokens":293,"pruned_tokens":31}},
{"rank":2882,"domain":"wanadoo.fr","origin":"https://wanadoo.fr","robots":{"status":"ok","http":200,"url":"https://www.orange.fr/robots.txt","bytes":66,"groups":1},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"html","http":200,"bytes":1193},"home":{"status":"ok","http":200,"url":"https://www.orange.fr/portail","html_tokens":299,"markdown_tokens":0,"pruned_tokens":0}},
{"rank":2883,"domain":"suno.com","origin":"https://suno.com","robots":{"status":"ok","http":200,"url":"https://suno.com/robots.txt","bytes":427,"groups":1},"bots":{"GPTBot":["partial","*",4,1],"ChatGPT-User":["partial","*",4,1],"OAI-SearchBot":["partial","*",4,1],"ClaudeBot":["partial","*",4,1],"Claude-User":["partial","*",4,1],"Claude-SearchBot":["partial","*",4,1],"anthropic-ai":["partial","*",4,1],"PerplexityBot":["partial","*",4,1],"Perplexity-User":["partial","*",4,1],"Google-Extended":["partial","*",4,1],"Applebot-Extended":["partial","*",4,1],"CCBot":["partial","*",4,1],"Bytespider":["partial","*",4,1],"Amazonbot":["partial","*",4,1],"meta-externalagent":["partial","*",4,1],"cohere-ai":["partial","*",4,1],"Diffbot":["partial","*",4,1],"ContextFetchBot":["partial","*",4,1]},"llms":{"status":"present","http":200,"bytes":11816},"home":{"status":"ok","http":200,"url":"https://suno.com/","html_tokens":64221,"markdown_tokens":652,"pruned_tokens":288}},
{"rank":2884,"domain":"edublogs.org","origin":"https://edublogs.org","robots":{"status":"ok","http":200,"url":"https://edublogs.org/robots.txt","bytes":1253,"groups":25},"bots":{"GPTBot":["partial","*",1,1],"ChatGPT-User":["partial","*",1,1],"OAI-SearchBot":["partial","*",1,1],"ClaudeBot":["partial","*",1,1],"Claude-User":["partial","*",1,1],"Claude-SearchBot":["partial","*",1,1],"anthropic-ai":["partial","*",1,1],"PerplexityBot":["partial","*",1,1],"Perplexity-User":["partial","*",1,1],"Google-Extended":["partial","*",1,1],"Applebot-Extended":["partial","*",1,1],"CCBot":["partial","*",1,1],"Bytespider":["partial","*",1,1],"Amazonbot":["partial","*",1,1],"meta-externalagent":["partial","*",1,1],"cohere-ai":["partial","*",1,1],"Diffbot":["partial","*",1,1],"ContextFetchBot":["partial","*",1,1]},"llms":{"status":"missing","http":404,"bytes":25217},"home":{"status":"ok","http":200,"url":"https://edublogs.org/","html_tokens":30697,"markdown_tokens":660,"pruned_tokens":89}},
{"rank":2885,"domain":"acuityscheduling.com","origin":"https://acuityscheduling.com","robots":{"status":"ok","http":200,"url":"https://acuityscheduling.com/robots.txt","bytes":941,"groups":1},"bots":{"GPTBot":["partial","*",20,0],"ChatGPT-User":["partial","*",20,0],"OAI-SearchBot":["partial","*",20,0],"ClaudeBot":["partial","*",20,0],"Claude-User":["partial","*",20,0],"Claude-SearchBot":["partial","*",20,0],"anthropic-ai":["partial","*",20,0],"PerplexityBot":["partial","*",20,0],"Perplexity-User":["partial","*",20,0],"Google-Extended":["partial","*",20,0],"Applebot-Extended":["partial","*",20,0],"CCBot":["partial","*",20,0],"Bytespider":["partial","*",20,0],"Amazonbot":["partial","*",20,0],"meta-externalagent":["partial","*",20,0],"cohere-ai":["partial","*",20,0],"Diffbot":["partial","*",20,0],"ContextFetchBot":["partial","*",20,0]},"llms":{"status":"missing","http":404,"bytes":59815},"home":{"status":"ok","http":200,"url":"https://acuityscheduling.com/","html_tokens":23879,"markdown_tokens":95,"pruned_tokens":95}},
{"rank":2886,"domain":"nextjs.org","origin":"https://nextjs.org","robots":{"status":"ok","http":200,"url":"https://nextjs.org/robots.txt","bytes":40,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"present","http":200,"bytes":14542},"home":{"status":"ok","http":200,"url":"https://nextjs.org/","html_tokens":56066,"markdown_tokens":599,"pruned_tokens":170}},
{"rank":2887,"domain":"thalesgroup.com","origin":"https://thalesgroup.com","robots":{"status":"ok","http":200,"url":"https://www.thalesgroup.com/robots.txt","bytes":2344,"groups":2},"bots":{"GPTBot":["partial","*",80,0],"ChatGPT-User":["partial","*",80,0],"OAI-SearchBot":["partial","*",80,0],"ClaudeBot":["partial","*",80,0],"Claude-User":["partial","*",80,0],"Claude-SearchBot":["partial","*",80,0],"anthropic-ai":["partial","*",80,0],"PerplexityBot":["partial","*",80,0],"Perplexity-User":["partial","*",80,0],"Google-Extended":["partial","*",80,0],"Applebot-Extended":["partial","*",80,0],"CCBot":["partial","*",80,0],"Bytespider":["partial","*",80,0],"Amazonbot":["partial","*",80,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["partial","*",80,0],"Diffbot":["partial","*",80,0],"ContextFetchBot":["partial","*",80,0]},"llms":{"status":"missing","http":404,"bytes":1030},"home":{"status":"ok","http":200,"url":"https://www.thalesgroup.com/","html_tokens":53,"markdown_tokens":0,"pruned_tokens":0}},
{"rank":2888,"domain":"unibe.ch","origin":"https://unibe.ch","robots":{"status":"ok","http":200,"url":"https://www.unibe.ch/robots.txt","bytes":1190,"groups":7},"bots":{"GPTBot":["partial","*",34,0],"ChatGPT-User":["partial","*",34,0],"OAI-SearchBot":["partial","*",34,0],"ClaudeBot":["partial","*",34,0],"Claude-User":["partial","*",34,0],"Claude-SearchBot":["partial","*",34,0],"anthropic-ai":["partial","*",34,0],"PerplexityBot":["partial","*",34,0],"Perplexity-User":["partial","*",34,0],"Google-Extended":["partial","*",34,0],"Applebot-Extended":["partial","*",34,0],"CCBot":["partial","*",34,0],"Bytespider":["partial","*",34,0],"Amazonbot":["partial","*",34,0],"meta-externalagent":["partial","*",34,0],"cohere-ai":["partial","*",34,0],"Diffbot":["partial","*",34,0],"ContextFetchBot":["partial","*",34,0]},"llms":{"status":"missing","http":404,"bytes":7229},"home":{"status":"ok","http":200,"url":"https://www.unibe.ch/","html_tokens":18295,"markdown_tokens":75,"pruned_tokens":75}},
{"rank":2889,"domain":"unirioja.es","origin":"https://www.unirioja.es","robots":{"status":"ok","http":200,"url":"https://www.unirioja.es/robots.txt","bytes":312,"groups":1},"bots":{"GPTBot":["partial","*",10,1],"ChatGPT-User":["partial","*",10,1],"OAI-SearchBot":["partial","*",10,1],"ClaudeBot":["partial","*",10,1],"Claude-User":["partial","*",10,1],"Claude-SearchBot":["partial","*",10,1],"anthropic-ai":["partial","*",10,1],"PerplexityBot":["partial","*",10,1],"Perplexity-User":["partial","*",10,1],"Google-Extended":["partial","*",10,1],"Applebot-Extended":["partial","*",10,1],"CCBot":["partial","*",10,1],"Bytespider":["partial","*",10,1],"Amazonbot":["partial","*",10,1],"meta-externalagent":["partial","*",10,1],"cohere-ai":["partial","*",10,1],"Diffbot":["partial","*",10,1],"ContextFetchBot":["partial","*",10,1]},"llms":{"status":"html","http":200,"bytes":521008},"home":{"status":"ok","http":200,"url":"https://www.unirioja.es/","html_tokens":291645,"markdown_tokens":433,"pruned_tokens":116}},
{"rank":2890,"domain":"pagesperso-orange.fr","origin":"https://pagesperso-orange.fr","robots":{"status":"unreachable","http":null,"url":"https://pagesperso-orange.fr/robots.txt","bytes":0,"groups":0,"error":"cannot resolve pages.perso.orange.fr"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":2891,"domain":"geo.io","origin":"https://geo.io","robots":{"status":"ok","http":200,"url":"https://geo.io/robots.txt","bytes":2676,"groups":32},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["allowed","*",0,1],"OAI-SearchBot":["allowed","*",0,1],"ClaudeBot":["blocked","named",1,0],"Claude-User":["allowed","*",0,1],"Claude-SearchBot":["allowed","*",0,1],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["allowed","*",0,1],"Perplexity-User":["allowed","*",0,1],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["allowed","*",0,1]},"llms":{"status":"http_error","http":403,"bytes":5495},"home":{"status":"http_error","http":403,"url":"https://geo.io/"}},
{"rank":2892,"domain":"libguides.com","origin":"https://libguides.com","robots":{"status":"missing","http":404,"url":"https://libguides.com/robots.txt","bytes":1444,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":1217},"home":{"status":"http_error","http":404,"url":"https://libguides.com/"}},
{"rank":2893,"domain":"nec.com","origin":"https://nec.com","robots":{"status":"refused","http":403,"url":"https://www.nec.com/","bytes":365,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":2894,"domain":"tudou.com","origin":"https://tudou.com","robots":{"status":"html","http":200,"url":"https://tudou.com/robots.txt","bytes":247134,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":2895,"domain":"nar.realtor","origin":"https://nar.realtor","robots":{"status":"ok","http":200,"url":"https://www.nar.realtor/robots.txt","bytes":216,"groups":1},"bots":{"GPTBot":["partial","*",8,1],"ChatGPT-User":["partial","*",8,1],"OAI-SearchBot":["partial","*",8,1],"ClaudeBot":["partial","*",8,1],"Claude-User":["partial","*",8,1],"Claude-SearchBot":["partial","*",8,1],"anthropic-ai":["partial","*",8,1],"PerplexityBot":["partial","*",8,1],"Perplexity-User":["partial","*",8,1],"Google-Extended":["partial","*",8,1],"Applebot-Extended":["partial","*",8,1],"CCBot":["partial","*",8,1],"Bytespider":["partial","*",8,1],"Amazonbot":["partial","*",8,1],"meta-externalagent":["partial","*",8,1],"cohere-ai":["partial","*",8,1],"Diffbot":["partial","*",8,1],"ContextFetchBot":["partial","*",8,1]},"llms":{"status":"missing","http":404,"bytes":20976},"home":{"status":"ok","http":200,"url":"https://www.nar.realtor/","html_tokens":349376,"markdown_tokens":234,"pruned_tokens":50}},
{"rank":2896,"domain":"www.parliament.uk","origin":"https://www.parliament.uk","robots":{"status":"refused","http":403,"url":"https://www.parliament.uk/robots.txt","bytes":5534,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":2897,"domain":"strikingly.com","origin":"https://strikingly.com","robots":{"status":"ok","http":200,"url":"https://www.strikingly.com/robots.txt","bytes":698,"groups":5},"bots":{"GPTBot":["partial","*",9,0],"ChatGPT-User":["partial","*",9,0],"OAI-SearchBot":["partial","*",9,0],"ClaudeBot":["partial","*",9,0],"Claude-User":["partial","*",9,0],"Claude-SearchBot":["partial","*",9,0],"anthropic-ai":["partial","*",9,0],"PerplexityBot":["partial","*",9,0],"Perplexity-User":["partial","*",9,0],"Google-Extended":["partial","*",9,0],"Applebot-Extended":["partial","*",9,0],"CCBot":["partial","*",9,0],"Bytespider":["partial","*",9,0],"Amazonbot":["partial","*",9,0],"meta-externalagent":["partial","*",9,0],"cohere-ai":["partial","*",9,0],"Diffbot":["partial","*",9,0],"ContextFetchBot":["partial","*",9,0]},"llms":{"status":"missing","http":404,"bytes":2088},"home":{"status":"ok","http":200,"url":"https://www.strikingly.com/","html_tokens":30939,"markdown_tokens":573,"pruned_tokens":226}},
{"rank":2898,"domain":"peta.org","origin":"https://peta.org","robots":{"status":"ok","http":200,"url":"https://www.peta.org/robots.txt","bytes":190,"groups":1},"bots":{"GPTBot":["partial","*",1,1],"ChatGPT-User":["partial","*",1,1],"OAI-SearchBot":["partial","*",1,1],"ClaudeBot":["partial","*",1,1],"Claude-User":["partial","*",1,1],"Claude-SearchBot":["partial","*",1,1],"anthropic-ai":["partial","*",1,1],"PerplexityBot":["partial","*",1,1],"Perplexity-User":["partial","*",1,1],"Google-Extended":["partial","*",1,1],"Applebot-Extended":["partial","*",1,1],"CCBot":["partial","*",1,1],"Bytespider":["partial","*",1,1],"Amazonbot":["partial","*",1,1],"meta-externalagent":["partial","*",1,1],"cohere-ai":["partial","*",1,1],"Diffbot":["partial","*",1,1],"ContextFetchBot":["partial","*",1,1]},"llms":{"status":"missing","http":404,"bytes":188458},"home":{"status":"ok","http":200,"url":"https://www.peta.org/","html_tokens":108882,"markdown_tokens":447,"pruned_tokens":35}},
{"rank":2899,"domain":"nrk.no","origin":"https://nrk.no","robots":{"status":"ok","http":200,"url":"https://www.nrk.no/robots.txt","bytes":3480,"groups":55},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["blocked","named",1,0],"OAI-SearchBot":["blocked","named",1,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["blocked","named",1,0],"Claude-SearchBot":["blocked","named",1,0],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["partial","*",37,0],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",37,0]},"llms":{"status":"missing","http":404,"bytes":84931},"home":{"status":"ok","http":200,"url":"https://www.nrk.no/","html_tokens":102098,"markdown_tokens":111,"pruned_tokens":27}},
{"rank":2900,"domain":"netlify.com","origin":"https://netlify.com","robots":{"status":"ok","http":200,"url":"https://netlify.com/robots.txt","bytes":764,"groups":8},"bots":{"GPTBot":["allowed","named",0,1],"ChatGPT-User":["allowed","named",0,1],"OAI-SearchBot":["allowed","named",0,1],"ClaudeBot":["allowed","named",0,1],"Claude-User":["allowed","*",0,1],"Claude-SearchBot":["allowed","*",0,1],"anthropic-ai":["allowed","*",0,1],"PerplexityBot":["allowed","named",0,1],"Perplexity-User":["allowed","*",0,1],"Google-Extended":["allowed","named",0,1],"Applebot-Extended":["allowed","*",0,1],"CCBot":["allowed","*",0,1],"Bytespider":["allowed","*",0,1],"Amazonbot":["allowed","*",0,1],"meta-externalagent":["allowed","*",0,1],"cohere-ai":["allowed","*",0,1],"Diffbot":["allowed","*",0,1],"ContextFetchBot":["allowed","*",0,1]},"llms":{"status":"present","http":200,"bytes":2818},"home":{"status":"not_html","error":"unsupported content type text/markdown"}},
{"rank":2901,"domain":"deseret.com","origin":"https://deseret.com","robots":{"status":"ok","http":200,"url":"https://www.deseret.com/robots.txt","bytes":990,"groups":18},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["allowed","named",0,1],"OAI-SearchBot":["allowed","named",0,1],"ClaudeBot":["blocked","named",1,0],"Claude-User":["allowed","named",0,1],"Claude-SearchBot":["allowed","named",0,1],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","named",0,1],"Perplexity-User":["allowed","named",0,1],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"missing","http":404,"bytes":160463},"home":{"status":"ok","http":200,"url":"https://www.deseret.com/","html_tokens":86710,"markdown_tokens":1887,"pruned_tokens":175}},
{"rank":2902,"domain":"toyota.com","origin":"https://toyota.com","robots":{"status":"ok","http":200,"url":"https://www.toyota.com/robots.txt","bytes":2188,"groups":3},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["partial","named",16,0],"OAI-SearchBot":["partial","named",16,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","named",16,0],"Claude-SearchBot":["partial","named",16,0],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["partial","named",16,0],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",15,0]},"llms":{"status":"missing","http":404,"bytes":128666},"home":{"status":"ok","http":200,"url":"https://www.toyota.com/","html_tokens":168944,"markdown_tokens":38,"pruned_tokens":38}},
{"rank":2903,"domain":"spring.io","origin":"https://spring.io","robots":{"status":"ok","http":200,"url":"https://spring.io/robots.txt","bytes":92,"groups":1},"bots":{"GPTBot":["allowed","*",0,1],"ChatGPT-User":["allowed","*",0,1],"OAI-SearchBot":["allowed","*",0,1],"ClaudeBot":["allowed","*",0,1],"Claude-User":["allowed","*",0,1],"Claude-SearchBot":["allowed","*",0,1],"anthropic-ai":["allowed","*",0,1],"PerplexityBot":["allowed","*",0,1],"Perplexity-User":["allowed","*",0,1],"Google-Extended":["allowed","*",0,1],"Applebot-Extended":["allowed","*",0,1],"CCBot":["allowed","*",0,1],"Bytespider":["allowed","*",0,1],"Amazonbot":["allowed","*",0,1],"meta-externalagent":["allowed","*",0,1],"cohere-ai":["allowed","*",0,1],"Diffbot":["allowed","*",0,1],"ContextFetchBot":["allowed","*",0,1]},"llms":{"status":"missing","http":404,"bytes":349777},"home":{"status":"ok","http":200,"url":"https://spring.io/","html_tokens":101280,"markdown_tokens":71,"pruned_tokens":71}},
{"rank":2904,"domain":"wur.nl","origin":"https://wur.nl","robots":{"status":"ok","http":200,"url":"https://www.wur.nl/robots.txt","bytes":1119,"groups":1},"bots":{"GPTBot":["partial","*",40,0],"ChatGPT-User":["partial","*",40,0],"OAI-SearchBot":["partial","*",40,0],"ClaudeBot":["partial","*",40,0],"Claude-User":["partial","*",40,0],"Claude-SearchBot":["partial","*",40,0],"anthropic-ai":["partial","*",40,0],"PerplexityBot":["partial","*",40,0],"Perplexity-User":["partial","*",40,0],"Google-Extended":["partial","*",40,0],"Applebot-Extended":["partial","*",40,0],"CCBot":["partial","*",40,0],"Bytespider":["partial","*",40,0],"Amazonbot":["partial","*",40,0],"meta-externalagent":["partial","*",40,0],"cohere-ai":["partial","*",40,0],"Diffbot":["partial","*",40,0],"ContextFetchBot":["partial","*",40,0]},"llms":{"status":"missing","http":404,"bytes":23703},"home":{"status":"ok","http":200,"url":"https://www.wur.nl/","html_tokens":66362,"markdown_tokens":69,"pruned_tokens":51}},
{"rank":2905,"domain":"archives-ouvertes.fr","origin":"https://archives-ouvertes.fr","robots":{"status":"ok","http":200,"url":"https://hal.science/robots","bytes":2993,"groups":50},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["blocked","named",1,0],"OAI-SearchBot":["blocked","named",1,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",34,0],"Claude-SearchBot":["partial","*",34,0],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["partial","*",34,0],"Google-Extended":["blocked","named",2,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",34,0]},"llms":{"status":"missing","http":404,"bytes":48597},"home":{"status":"ok","http":200,"url":"https://hal.science/","html_tokens":8112,"markdown_tokens":196,"pruned_tokens":58}},
{"rank":2906,"domain":"wellsfargo.com","origin":"https://wellsfargo.com","robots":{"status":"ok","http":200,"url":"https://www.wellsfargo.com/robots.txt","bytes":621,"groups":1},"bots":{"GPTBot":["partial","*",18,1],"ChatGPT-User":["partial","*",18,1],"OAI-SearchBot":["partial","*",18,1],"ClaudeBot":["partial","*",18,1],"Claude-User":["partial","*",18,1],"Claude-SearchBot":["partial","*",18,1],"anthropic-ai":["partial","*",18,1],"PerplexityBot":["partial","*",18,1],"Perplexity-User":["partial","*",18,1],"Google-Extended":["partial","*",18,1],"Applebot-Extended":["partial","*",18,1],"CCBot":["partial","*",18,1],"Bytespider":["partial","*",18,1],"Amazonbot":["partial","*",18,1],"meta-externalagent":["partial","*",18,1],"cohere-ai":["partial","*",18,1],"Diffbot":["partial","*",18,1],"ContextFetchBot":["partial","*",18,1]},"llms":{"status":"missing","http":404,"bytes":7958},"home":{"status":"ok","http":200,"url":"https://www.wellsfargo.com/","html_tokens":31027,"markdown_tokens":119,"pruned_tokens":62}},
{"rank":2907,"domain":"infomaniak.com","origin":"https://infomaniak.com","robots":{"status":"ok","http":200,"url":"https://www.infomaniak.com/robots.txt","bytes":1716,"groups":2},"bots":{"GPTBot":["partial","named",26,0],"ChatGPT-User":["partial","named",26,0],"OAI-SearchBot":["partial","named",26,0],"ClaudeBot":["partial","named",26,0],"Claude-User":["partial","*",7,1],"Claude-SearchBot":["partial","named",26,0],"anthropic-ai":["partial","named",26,0],"PerplexityBot":["partial","named",26,0],"Perplexity-User":["partial","named",26,0],"Google-Extended":["partial","*",7,1],"Applebot-Extended":["partial","named",26,0],"CCBot":["partial","named",26,0],"Bytespider":["partial","named",26,0],"Amazonbot":["partial","named",26,0],"meta-externalagent":["partial","named",26,0],"cohere-ai":["partial","named",26,0],"Diffbot":["partial","named",26,0],"ContextFetchBot":["partial","*",7,1]},"llms":{"status":"missing","http":404,"bytes":106340},"home":{"status":"ok","http":200,"url":"https://www.infomaniak.com/en","html_tokens":678178,"markdown_tokens":396,"pruned_tokens":180}},
{"rank":2908,"domain":"dawn.com","origin":"https://dawn.com","robots":{"status":"ok","http":200,"url":"https://www.dawn.com/robots.txt","bytes":469,"groups":1},"bots":{"GPTBot":["partial","*",14,0],"ChatGPT-User":["partial","*",14,0],"OAI-SearchBot":["partial","*",14,0],"ClaudeBot":["partial","*",14,0],"Claude-User":["partial","*",14,0],"Claude-SearchBot":["partial","*",14,0],"anthropic-ai":["partial","*",14,0],"PerplexityBot":["partial","*",14,0],"Perplexity-User":["partial","*",14,0],"Google-Extended":["partial","*",14,0],"Applebot-Extended":["partial","*",14,0],"CCBot":["partial","*",14,0],"Bytespider":["partial","*",14,0],"Amazonbot":["partial","*",14,0],"meta-externalagent":["partial","*",14,0],"cohere-ai":["partial","*",14,0],"Diffbot":["partial","*",14,0],"ContextFetchBot":["partial","*",14,0]},"llms":{"status":"missing","http":404,"bytes":84278},"home":{"status":"http_error","http":403,"url":"https://dawn.com/"}},
{"rank":2909,"domain":"ctan.org","origin":"https://ctan.org","robots":{"status":"ok","http":200,"url":"https://ctan.org/robots.txt","bytes":152,"groups":1},"bots":{"GPTBot":["partial","*",7,0],"ChatGPT-User":["partial","*",7,0],"OAI-SearchBot":["partial","*",7,0],"ClaudeBot":["partial","*",7,0],"Claude-User":["partial","*",7,0],"Claude-SearchBot":["partial","*",7,0],"anthropic-ai":["partial","*",7,0],"PerplexityBot":["partial","*",7,0],"Perplexity-User":["partial","*",7,0],"Google-Extended":["partial","*",7,0],"Applebot-Extended":["partial","*",7,0],"CCBot":["partial","*",7,0],"Bytespider":["partial","*",7,0],"Amazonbot":["partial","*",7,0],"meta-externalagent":["partial","*",7,0],"cohere-ai":["partial","*",7,0],"Diffbot":["partial","*",7,0],"ContextFetchBot":["partial","*",7,0]},"llms":{"status":"present","http":200,"bytes":4367},"home":{"status":"ok","http":200,"url":"https://ctan.org/","html_tokens":4846,"markdown_tokens":662,"pruned_tokens":162}},
{"rank":2910,"domain":"okstate.edu","origin":"https://okstate.edu","robots":{"status":"refused","http":403,"url":"https://okstate.edu/robots.txt","bytes":5506,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":2911,"domain":"xerox.com","origin":"https://xerox.com","robots":{"status":"ok","http":200,"url":"https://www.xerox.com/robots.txt","bytes":2304,"groups":2},"bots":{"GPTBot":["partial","*",70,0],"ChatGPT-User":["partial","*",70,0],"OAI-SearchBot":["partial","*",70,0],"ClaudeBot":["partial","*",70,0],"Claude-User":["partial","*",70,0],"Claude-SearchBot":["partial","*",70,0],"anthropic-ai":["partial","*",70,0],"PerplexityBot":["partial","*",70,0],"Perplexity-User":["partial","*",70,0],"Google-Extended":["partial","*",70,0],"Applebot-Extended":["partial","*",70,0],"CCBot":["partial","*",70,0],"Bytespider":["partial","*",70,0],"Amazonbot":["partial","*",70,0],"meta-externalagent":["partial","*",70,0],"cohere-ai":["partial","*",70,0],"Diffbot":["partial","*",70,0],"ContextFetchBot":["partial","*",70,0]},"llms":{"status":"missing","http":404,"bytes":266611},"home":{"status":"ok","http":200,"url":"https://www.xerox.com/en-us","html_tokens":104872,"markdown_tokens":103,"pruned_tokens":40}},
{"rank":2912,"domain":"ad.doubleclick.net","origin":"https://ad.doubleclick.net","robots":{"status":"ok","http":200,"url":"https://ad.doubleclick.net/robots.txt","bytes":28,"groups":1},"bots":{"GPTBot":["blocked","*",1,0],"ChatGPT-User":["blocked","*",1,0],"OAI-SearchBot":["blocked","*",1,0],"ClaudeBot":["blocked","*",1,0],"Claude-User":["blocked","*",1,0],"Claude-SearchBot":["blocked","*",1,0],"anthropic-ai":["blocked","*",1,0],"PerplexityBot":["blocked","*",1,0],"Perplexity-User":["blocked","*",1,0],"Google-Extended":["blocked","*",1,0],"Applebot-Extended":["blocked","*",1,0],"CCBot":["blocked","*",1,0],"Bytespider":["blocked","*",1,0],"Amazonbot":["blocked","*",1,0],"meta-externalagent":["blocked","*",1,0],"cohere-ai":["blocked","*",1,0],"Diffbot":["blocked","*",1,0],"ContextFetchBot":["blocked","*",1,0]},"llms":{"status":"disallowed"},"home":{"status":"disallowed"}},
{"rank":2913,"domain":"clickbank.net","origin":"https://www.clickbank.net","robots":{"status":"ok","http":200,"url":"https://www.clickbank.com/robots.txt","bytes":71,"groups":1},"bots":{"GPTBot":["allowed","*",0,1],"ChatGPT-User":["allowed","*",0,1],"OAI-SearchBot":["allowed","*",0,1],"ClaudeBot":["allowed","*",0,1],"Claude-User":["allowed","*",0,1],"Claude-SearchBot":["allowed","*",0,1],"anthropic-ai":["allowed","*",0,1],"PerplexityBot":["allowed","*",0,1],"Perplexity-User":["allowed","*",0,1],"Google-Extended":["allowed","*",0,1],"Applebot-Extended":["allowed","*",0,1],"CCBot":["allowed","*",0,1],"Bytespider":["allowed","*",0,1],"Amazonbot":["allowed","*",0,1],"meta-externalagent":["allowed","*",0,1],"cohere-ai":["allowed","*",0,1],"Diffbot":["allowed","*",0,1],"ContextFetchBot":["allowed","*",0,1]},"llms":{"status":"http_error","http":403,"bytes":63087},"home":{"status":"ok","http":200,"url":"https://www.clickbank.com/","html_tokens":98139,"markdown_tokens":1314,"pruned_tokens":892}},
{"rank":2914,"domain":"cloudwaysapps.com","origin":"https://cloudwaysapps.com","robots":{"status":"unreachable","http":null,"url":"https://cloudwaysapps.com/robots.txt","bytes":0,"groups":0,"error":"cannot resolve cloudwaysapps.com"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":2915,"domain":"liveleak.com","origin":"https://liveleak.com","robots":{"status":"unreachable","http":null,"url":"https://liveleak.com/robots.txt","bytes":0,"groups":0,"error":"[SSL: CERTIFICATE_VERIFY_FAILED] certificate verify failed: unable to get local issuer certificate (_ssl.c:1028)"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":2916,"domain":"diva-portal.org","origin":"https://diva-portal.org","robots":{"status":"ok","http":200,"url":"https://diva-portal.org/robots.txt","bytes":622,"groups":11},"bots":{"GPTBot":["partial","*",7,0],"ChatGPT-User":["partial","*",7,0],"OAI-SearchBot":["partial","*",7,0],"ClaudeBot":["partial","*",7,0],"Claude-User":["partial","*",7,0],"Claude-SearchBot":["partial","*",7,0],"anthropic-ai":["partial","*",7,0],"PerplexityBot":["partial","*",7,0],"Perplexity-User":["partial","*",7,0],"Google-Extended":["partial","*",7,0],"Applebot-Extended":["partial","*",7,0],"CCBot":["partial","*",7,0],"Bytespider":["partial","*",7,0],"Amazonbot":["partial","*",7,0],"meta-externalagent":["partial","*",7,0],"cohere-ai":["partial","*",7,0],"Diffbot":["partial","*",7,0],"ContextFetchBot":["partial","*",7,0]},"llms":{"status":"missing","http":404,"bytes":278},"home":{"status":"ok","http":200,"url":"https://www.diva-portal.org/smash/","html_tokens":4663,"markdown_tokens":84,"pruned_tokens":40}},
{"rank":2917,"domain":"quark.cn","origin":"https://quark.cn","robots":{"status":"ok","http":200,"url":"https://quark.cn/robots.txt","bytes":650,"groups":1},"bots":{"GPTBot":["partial","*",29,1],"ChatGPT-User":["partial","*",29,1],"OAI-SearchBot":["partial","*",29,1],"ClaudeBot":["partial","*",29,1],"Claude-User":["partial","*",29,1],"Claude-SearchBot":["partial","*",29,1],"anthropic-ai":["partial","*",29,1],"PerplexityBot":["partial","*",29,1],"Perplexity-User":["partial","*",29,1],"Google-Extended":["partial","*",29,1],"Applebot-Extended":["partial","*",29,1],"CCBot":["partial","*",29,1],"Bytespider":["partial","*",29,1],"Amazonbot":["partial","*",29,1],"meta-externalagent":["partial","*",29,1],"cohere-ai":["partial","*",29,1],"Diffbot":["partial","*",29,1],"ContextFetchBot":["partial","*",29,1]},"llms":{"status":"missing","http":404,"bytes":2125},"home":{"status":"ok","http":200,"url":"https://www.quark.cn/","html_tokens":26155,"markdown_tokens":422,"pruned_tokens":87}},
{"rank":2918,"domain":"viglink.com","origin":"https://viglink.com","robots":{"status":"refused","http":403,"url":"https://sovrn.com/commerce/?utm_source=viglink","bytes":5637,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":2919,"domain":"lmu.de","origin":"https://lmu.de","robots":{"status":"ok","http":200,"url":"https://www.lmu.de/robots.txt","bytes":3293,"groups":1},"bots":{"GPTBot":["partial","*",44,0],"ChatGPT-User":["partial","*",44,0],"OAI-SearchBot":["partial","*",44,0],"ClaudeBot":["partial","*",44,0],"Claude-User":["partial","*",44,0],"Claude-SearchBot":["partial","*",44,0],"anthropic-ai":["partial","*",44,0],"PerplexityBot":["partial","*",44,0],"Perplexity-User":["partial","*",44,0],"Google-Extended":["partial","*",44,0],"Applebot-Extended":["partial","*",44,0],"CCBot":["partial","*",44,0],"Bytespider":["partial","*",44,0],"Amazonbot":["partial","*",44,0],"meta-externalagent":["partial","*",44,0],"cohere-ai":["partial","*",44,0],"Diffbot":["partial","*",44,0],"ContextFetchBot":["partial","*",44,0]},"llms":{"status":"missing","http":404,"bytes":18469},"home":{"status":"ok","http":200,"url":"https://www.lmu.de/de/index.html","html_tokens":10638,"markdown_tokens":414,"pruned_tokens":130}},
{"rank":2920,"domain":"surrey.ac.uk","origin":"https://surrey.ac.uk","robots":{"status":"ok","http":200,"url":"https://www.surrey.ac.uk/robots.txt","bytes":2982,"groups":2},"bots":{"GPTBot":["partial","*",31,18],"ChatGPT-User":["partial","*",31,18],"OAI-SearchBot":["partial","*",31,18],"ClaudeBot":["partial","*",31,18],"Claude-User":["partial","*",31,18],"Claude-SearchBot":["partial","*",31,18],"anthropic-ai":["partial","*",31,18],"PerplexityBot":["partial","*",31,18],"Perplexity-User":["partial","*",31,18],"Google-Extended":["partial","*",31,18],"Applebot-Extended":["partial","*",31,18],"CCBot":["partial","*",31,18],"Bytespider":["partial","*",31,18],"Amazonbot":["partial","*",31,18],"meta-externalagent":["partial","*",31,18],"cohere-ai":["partial","*",31,18],"Diffbot":["partial","*",31,18],"ContextFetchBot":["partial","*",31,18]},"llms":{"status":"missing","http":404,"bytes":190},"home":{"status":"ok","http":200,"url":"https://www.surrey.ac.uk/","html_tokens":51381,"markdown_tokens":111,"pruned_tokens":111}},
{"rank":2921,"domain":"kqzyfj.com","origin":"https://www.kqzyfj.com","robots":{"status":"ok","http":200,"url":"https://www.kqzyfj.com/robots.txt","bytes":35,"groups":1},"bots":{"GPTBot":["blocked","*",1,0],"ChatGPT-User":["blocked","*",1,0],"OAI-SearchBot":["blocked","*",1,0],"ClaudeBot":["blocked","*",1,0],"Claude-User":["blocked","*",1,0],"Claude-SearchBot":["blocked","*",1,0],"anthropic-ai":["blocked","*",1,0],"PerplexityBot":["blocked","*",1,0],"Perplexity-User":["blocked","*",1,0],"Google-Extended":["blocked","*",1,0],"Applebot-Extended":["blocked","*",1,0],"CCBot":["blocked","*",1,0],"Bytespider":["blocked","*",1,0],"Amazonbot":["blocked","*",1,0],"meta-externalagent":["blocked","*",1,0],"cohere-ai":["blocked","*",1,0],"Diffbot":["blocked","*",1,0],"ContextFetchBot":["blocked","*",1,0]},"llms":{"status":"disallowed"},"home":{"status":"disallowed"}},
{"rank":2922,"domain":"usgbc.org","origin":"https://usgbc.org","robots":{"status":"ok","http":200,"url":"https://www.usgbc.org/robots.txt","bytes":2357,"groups":3},"bots":{"GPTBot":["blocked","*",1,0],"ChatGPT-User":["blocked","*",1,0],"OAI-SearchBot":["blocked","*",1,0],"ClaudeBot":["blocked","*",1,0],"Claude-User":["blocked","*",1,0],"Claude-SearchBot":["blocked","*",1,0],"anthropic-ai":["blocked","*",1,0],"PerplexityBot":["blocked","*",1,0],"Perplexity-User":["blocked","*",1,0],"Google-Extended":["blocked","*",1,0],"Applebot-Extended":["blocked","*",1,0],"CCBot":["blocked","*",1,0],"Bytespider":["blocked","*",1,0],"Amazonbot":["blocked","*",1,0],"meta-externalagent":["blocked","*",1,0],"cohere-ai":["blocked","*",1,0],"Diffbot":["blocked","*",1,0],"ContextFetchBot":["blocked","*",1,0]},"llms":{"status":"disallowed"},"home":{"status":"disallowed"}},
{"rank":2923,"domain":"parliament.uk","origin":"https://parliament.uk","robots":{"status":"refused","http":403,"url":"https://www.parliament.uk/robots.txt","bytes":5534,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":2924,"domain":"uni-frankfurt.de","origin":"https://uni-frankfurt.de","robots":{"status":"ok","http":200,"url":"https://www.uni-frankfurt.de/robots.txt","bytes":2347,"groups":3},"bots":{"GPTBot":["partial","*",30,56],"ChatGPT-User":["partial","*",30,56],"OAI-SearchBot":["partial","*",30,56],"ClaudeBot":["partial","*",30,56],"Claude-User":["partial","*",30,56],"Claude-SearchBot":["partial","*",30,56],"anthropic-ai":["partial","*",30,56],"PerplexityBot":["partial","*",30,56],"Perplexity-User":["partial","*",30,56],"Google-Extended":["partial","*",30,56],"Applebot-Extended":["partial","*",30,56],"CCBot":["blocked","named",1,0],"Bytespider":["partial","*",30,56],"Amazonbot":["partial","*",30,56],"meta-externalagent":["partial","*",30,56],"cohere-ai":["partial","*",30,56],"Diffbot":["partial","*",30,56],"ContextFetchBot":["partial","*",30,56]},"llms":{"status":"missing","http":404,"bytes":16617},"home":{"status":"ok","http":200,"url":"https://www.uni-frankfurt.de/en","html_tokens":85498,"markdown_tokens":565,"pruned_tokens":44}},
{"rank":2925,"domain":"alarabiya.net","origin":"https://alarabiya.net","robots":{"status":"ok","http":200,"url":"https://www.alarabiya.net/robots.txt","bytes":406,"groups":1},"bots":{"GPTBot":["partial","*",5,0],"ChatGPT-User":["partial","*",5,0],"OAI-SearchBot":["partial","*",5,0],"ClaudeBot":["partial","*",5,0],"Claude-User":["partial","*",5,0],"Claude-SearchBot":["partial","*",5,0],"anthropic-ai":["partial","*",5,0],"PerplexityBot":["partial","*",5,0],"Perplexity-User":["partial","*",5,0],"Google-Extended":["partial","*",5,0],"Applebot-Extended":["partial","*",5,0],"CCBot":["partial","*",5,0],"Bytespider":["partial","*",5,0],"Amazonbot":["partial","*",5,0],"meta-externalagent":["partial","*",5,0],"cohere-ai":["partial","*",5,0],"Diffbot":["partial","*",5,0],"ContextFetchBot":["partial","*",5,0]},"llms":{"status":"missing","http":404,"bytes":75817},"home":{"status":"http_error","http":403,"url":"https://www.alarabiya.net/"}},
{"rank":2926,"domain":"wi.gov","origin":"https://wi.gov","robots":{"status":"html","http":200,"url":"https://www.wisconsin.gov/Pages/Home.aspx","bytes":113989,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":2927,"domain":"goto.com","origin":"https://goto.com","robots":{"status":"ok","http":200,"url":"https://www.goto.com/robots.txt","bytes":356,"groups":1},"bots":{"GPTBot":["partial","*",5,1],"ChatGPT-User":["partial","*",5,1],"OAI-SearchBot":["partial","*",5,1],"ClaudeBot":["partial","*",5,1],"Claude-User":["partial","*",5,1],"Claude-SearchBot":["partial","*",5,1],"anthropic-ai":["partial","*",5,1],"PerplexityBot":["partial","*",5,1],"Perplexity-User":["partial","*",5,1],"Google-Extended":["partial","*",5,1],"Applebot-Extended":["partial","*",5,1],"CCBot":["partial","*",5,1],"Bytespider":["partial","*",5,1],"Amazonbot":["partial","*",5,1],"meta-externalagent":["partial","*",5,1],"cohere-ai":["partial","*",5,1],"Diffbot":["partial","*",5,1],"ContextFetchBot":["partial","*",5,1]},"llms":{"status":"missing","http":404,"bytes":176554},"home":{"status":"ok","http":200,"url":"https://www.goto.com/","html_tokens":74838,"markdown_tokens":410,"pruned_tokens":130}},
{"rank":2928,"domain":"avada.com","origin":"https://avada.com","robots":{"status":"ok","http":200,"url":"https://avada.com/robots.txt","bytes":419,"groups":2},"bots":{"GPTBot":["partial","*",6,1],"ChatGPT-User":["partial","*",6,1],"OAI-SearchBot":["partial","*",6,1],"ClaudeBot":["partial","*",6,1],"Claude-User":["partial","*",6,1],"Claude-SearchBot":["partial","*",6,1],"anthropic-ai":["partial","*",6,1],"PerplexityBot":["partial","*",6,1],"Perplexity-User":["partial","*",6,1],"Google-Extended":["partial","*",6,1],"Applebot-Extended":["partial","*",6,1],"CCBot":["partial","*",6,1],"Bytespider":["partial","*",6,1],"Amazonbot":["partial","*",6,1],"meta-externalagent":["partial","*",6,1],"cohere-ai":["partial","*",6,1],"Diffbot":["partial","*",6,1],"ContextFetchBot":["partial","*",6,1]},"llms":{"status":"http_error","http":403,"bytes":5540},"home":{"status":"http_error","http":403,"url":"https://avada.com/"}},
{"rank":2929,"domain":"mgmresorts.com","origin":"https://mgmresorts.com","robots":{"status":"unreachable","http":null,"url":"https://mgmresorts.com/robots.txt","bytes":0,"groups":0,"error":"<StreamReset stream_id:1, error_code:2, remote_reset:True>"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":2930,"domain":"me.com","origin":"https://me.com","robots":{"status":"html","http":200,"url":"https://www.icloud.com/","bytes":31328,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":2931,"domain":"which.co.uk","origin":"https://which.co.uk","robots":{"status":"ok","http":200,"url":"https://www.which.co.uk/robots.txt","bytes":1739,"groups":3},"bots":{"GPTBot":["partial","named",2,0],"ChatGPT-User":["partial","named",2,0],"OAI-SearchBot":["partial","*",30,0],"ClaudeBot":["partial","named",2,0],"Claude-User":["partial","named",2,0],"Claude-SearchBot":["partial","named",2,0],"anthropic-ai":["partial","*",30,0],"PerplexityBot":["partial","named",2,0],"Perplexity-User":["partial","named",2,0],"Google-Extended":["partial","named",2,0],"Applebot-Extended":["partial","*",30,0],"CCBot":["partial","*",30,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["partial","named",2,0],"meta-externalagent":["partial","*",30,0],"cohere-ai":["partial","*",30,0],"Diffbot":["partial","*",30,0],"ContextFetchBot":["partial","*",30,0]},"llms":{"status":"missing","http":404,"bytes":3528},"home":{"status":"ok","http":200,"url":"https://www.which.co.uk/","html_tokens":100829,"markdown_tokens":128,"pruned_tokens":128}},
{"rank":2932,"domain":"wildberries.ru","origin":"https://wildberries.ru","robots":{"status":"refused","http":498,"url":"https://www.wildberries.ru/robots.txt","bytes":1130,"groups":0,"error":"HTTP 498"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":2933,"domain":"komoot.com","origin":"https://komoot.com","robots":{"status":"ok","http":200,"url":"https://www.komoot.com/robots.txt","bytes":10990,"groups":2},"bots":{"GPTBot":["partial","*",125,0],"ChatGPT-User":["partial","*",125,0],"OAI-SearchBot":["partial","*",125,0],"ClaudeBot":["partial","*",125,0],"Claude-User":["partial","*",125,0],"Claude-SearchBot":["partial","*",125,0],"anthropic-ai":["partial","*",125,0],"PerplexityBot":["partial","*",125,0],"Perplexity-User":["partial","*",125,0],"Google-Extended":["partial","*",125,0],"Applebot-Extended":["partial","*",125,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["partial","*",125,0],"meta-externalagent":["partial","*",125,0],"cohere-ai":["partial","*",125,0],"Diffbot":["partial","*",125,0],"ContextFetchBot":["partial","*",125,0]},"llms":{"status":"missing","http":404,"bytes":12993},"home":{"status":"ok","http":200,"url":"https://www.komoot.com/","html_tokens":189896,"markdown_tokens":393,"pruned_tokens":227}},
{"rank":2934,"domain":"adjust.com","origin":"https://adjust.com","robots":{"status":"refused","http":429,"url":"https://www.adjust.com/robots.txt","bytes":34228,"groups":0,"error":"HTTP 429"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":2935,"domain":"nybooks.com","origin":"https://nybooks.com","robots":{"status":"ok","http":200,"url":"https://www.nybooks.com/robots.txt","bytes":1418,"groups":3},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["partial","*",6,0],"OAI-SearchBot":["partial","*",6,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",6,0],"Claude-SearchBot":["partial","*",6,0],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["partial","*",6,0],"Perplexity-User":["partial","*",6,0],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["partial","*",6,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",6,0]},"llms":{"status":"missing","http":404,"bytes":84817},"home":{"status":"ok","http":200,"url":"https://www.nybooks.com/","html_tokens":70342,"markdown_tokens":74,"pruned_tokens":74}},
{"rank":2936,"domain":"georgia.gov","origin":"https://georgia.gov","robots":{"status":"ok","http":200,"url":"https://georgia.gov/robots.txt","bytes":1677,"groups":1},"bots":{"GPTBot":["partial","*",21,18],"ChatGPT-User":["partial","*",21,18],"OAI-SearchBot":["partial","*",21,18],"ClaudeBot":["partial","*",21,18],"Claude-User":["partial","*",21,18],"Claude-SearchBot":["partial","*",21,18],"anthropic-ai":["partial","*",21,18],"PerplexityBot":["partial","*",21,18],"Perplexity-User":["partial","*",21,18],"Google-Extended":["partial","*",21,18],"Applebot-Extended":["partial","*",21,18],"CCBot":["partial","*",21,18],"Bytespider":["partial","*",21,18],"Amazonbot":["partial","*",21,18],"meta-externalagent":["partial","*",21,18],"cohere-ai":["partial","*",21,18],"Diffbot":["partial","*",21,18],"ContextFetchBot":["partial","*",21,18]},"llms":{"status":"missing","http":404,"bytes":185},"home":{"status":"ok","http":200,"url":"https://georgia.gov/","html_tokens":25461,"markdown_tokens":335,"pruned_tokens":73}},
{"rank":2937,"domain":"aif.ru","origin":"https://aif.ru","robots":{"status":"ok","http":200,"url":"https://aif.ru/robots.txt","bytes":3168,"groups":3},"bots":{"GPTBot":["partial","*",38,3],"ChatGPT-User":["partial","*",38,3],"OAI-SearchBot":["partial","*",38,3],"ClaudeBot":["partial","*",38,3],"Claude-User":["partial","*",38,3],"Claude-SearchBot":["partial","*",38,3],"anthropic-ai":["partial","*",38,3],"PerplexityBot":["partial","*",38,3],"Perplexity-User":["partial","*",38,3],"Google-Extended":["partial","*",38,3],"Applebot-Extended":["partial","*",38,3],"CCBot":["partial","*",38,3],"Bytespider":["partial","*",38,3],"Amazonbot":["partial","*",38,3],"meta-externalagent":["partial","*",38,3],"cohere-ai":["partial","*",38,3],"Diffbot":["partial","*",38,3],"ContextFetchBot":["partial","*",38,3]},"llms":{"status":"missing","http":404,"bytes":93504},"home":{"status":"ok","http":200,"url":"https://aif.ru/","html_tokens":44332,"markdown_tokens":215,"pruned_tokens":100}},
{"rank":2938,"domain":"imo.org","origin":"https://imo.org","robots":{"status":"error","http":null,"url":"https://imo.org/robots.txt","bytes":0,"groups":0,"error":"more than 5 redirects"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":2939,"domain":"typescriptlang.org","origin":"https://typescriptlang.org","robots":{"status":"missing","http":404,"url":"https://www.typescriptlang.org/robots.txt","bytes":9379,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":9379},"home":{"status":"ok","http":200,"url":"https://www.typescriptlang.org/","html_tokens":64705,"markdown_tokens":798,"pruned_tokens":154}},
{"rank":2940,"domain":"2gis.ru","origin":"https://2gis.ru","robots":{"status":"ok","http":200,"url":"https://2gis.ru/robots.txt","bytes":1193,"groups":3},"bots":{"GPTBot":["partial","*",13,0],"ChatGPT-User":["partial","*",13,0],"OAI-SearchBot":["partial","*",13,0],"ClaudeBot":["partial","*",13,0],"Claude-User":["partial","*",13,0],"Claude-SearchBot":["partial","*",13,0],"anthropic-ai":["partial","*",13,0],"PerplexityBot":["partial","*",13,0],"Perplexity-User":["partial","*",13,0],"Google-Extended":["partial","*",13,0],"Applebot-Extended":["partial","*",13,0],"CCBot":["partial","*",13,0],"Bytespider":["partial","*",13,0],"Amazonbot":["partial","*",13,0],"meta-externalagent":["partial","*",13,0],"cohere-ai":["partial","*",13,0],"Diffbot":["partial","*",13,0],"ContextFetchBot":["partial","*",13,0]},"llms":{"status":"missing","http":404,"bytes":1265469},"home":{"status":"ok","http":200,"url":"https://2gis.ru/","html_tokens":168635,"markdown_tokens":278,"pruned_tokens":170}},
{"rank":2941,"domain":"ifttt.com","origin":"https://ifttt.com","robots":{"status":"ok","http":200,"url":"https://ifttt.com/robots.txt","bytes":1347,"groups":2},"bots":{"GPTBot":["partial","*",6,0],"ChatGPT-User":["partial","*",6,0],"OAI-SearchBot":["partial","*",6,0],"ClaudeBot":["partial","*",6,0],"Claude-User":["partial","*",6,0],"Claude-SearchBot":["partial","*",6,0],"anthropic-ai":["partial","*",6,0],"PerplexityBot":["partial","*",6,0],"Perplexity-User":["partial","*",6,0],"Google-Extended":["partial","*",6,0],"Applebot-Extended":["partial","*",6,0],"CCBot":["partial","*",6,0],"Bytespider":["partial","*",6,0],"Amazonbot":["partial","named",8,0],"meta-externalagent":["partial","*",6,0],"cohere-ai":["partial","*",6,0],"Diffbot":["partial","*",6,0],"ContextFetchBot":["partial","*",6,0]},"llms":{"status":"present","http":200,"bytes":6813},"home":{"status":"ok","http":200,"url":"https://ifttt.com/","html_tokens":114207,"markdown_tokens":749,"pruned_tokens":170}},
{"rank":2942,"domain":"srf.ch","origin":"https://srf.ch","robots":{"status":"ok","http":200,"url":"https://www.srf.ch/robots.txt","bytes":2772,"groups":23},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["partial","*",15,0],"OAI-SearchBot":["partial","*",15,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",15,0],"Claude-SearchBot":["partial","*",15,0],"anthropic-ai":["partial","*",15,0],"PerplexityBot":["partial","*",15,0],"Perplexity-User":["partial","*",15,0],"Google-Extended":["partial","*",15,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["partial","*",15,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",15,0]},"llms":{"status":"missing","http":404,"bytes":90706},"home":{"status":"ok","http":200,"url":"https://www.srf.ch","html_tokens":87152,"markdown_tokens":65,"pruned_tokens":26}},
{"rank":2943,"domain":"gawker.com","origin":"https://gawker.com","robots":{"status":"missing","http":404,"url":"https://gawker.com/robots.txt","bytes":10942,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":10942},"home":{"status":"ok","http":200,"url":"https://gawker.com/","html_tokens":4040,"markdown_tokens":12,"pruned_tokens":12}},
{"rank":2944,"domain":"opengroup.org","origin":"https://opengroup.org","robots":{"status":"html","http":200,"url":"https://www.opengroup.org/","bytes":86681,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":2945,"domain":"reviewjournal.com","origin":"https://reviewjournal.com","robots":{"status":"ok","http":200,"url":"https://www.reviewjournal.com/robots.txt","bytes":716,"groups":9},"bots":{"GPTBot":["partial","*",1,1],"ChatGPT-User":["partial","*",1,1],"OAI-SearchBot":["partial","*",1,1],"ClaudeBot":["partial","*",1,1],"Claude-User":["partial","*",1,1],"Claude-SearchBot":["partial","*",1,1],"anthropic-ai":["partial","*",1,1],"PerplexityBot":["partial","*",1,1],"Perplexity-User":["partial","*",1,1],"Google-Extended":["partial","*",1,1],"Applebot-Extended":["partial","*",1,1],"CCBot":["partial","*",1,1],"Bytespider":["blocked","named",1,0],"Amazonbot":["partial","*",1,1],"meta-externalagent":["partial","*",1,1],"cohere-ai":["partial","*",1,1],"Diffbot":["partial","*",1,1],"ContextFetchBot":["partial","*",1,1]},"llms":{"status":"http_error","http":403,"bytes":5508},"home":{"status":"http_error","http":403,"url":"https://www.reviewjournal.com/"}},
{"rank":2946,"domain":"blackmagicdesign.com","origin":"https://blackmagicdesign.com","robots":{"status":"ok","http":200,"url":"https://www.blackmagicdesign.com/robots.txt","bytes":1745,"groups":30},"bots":{"GPTBot":["partial","*",8,10],"ChatGPT-User":["partial","*",8,10],"OAI-SearchBot":["partial","*",8,10],"ClaudeBot":["partial","*",8,10],"Claude-User":["partial","*",8,10],"Claude-SearchBot":["partial","*",8,10],"anthropic-ai":["partial","*",8,10],"PerplexityBot":["partial","*",8,10],"Perplexity-User":["partial","*",8,10],"Google-Extended":["partial","*",8,10],"Applebot-Extended":["partial","*",8,10],"CCBot":["partial","*",8,10],"Bytespider":["partial","*",8,10],"Amazonbot":["partial","*",8,10],"meta-externalagent":["partial","*",8,10],"cohere-ai":["partial","*",8,10],"Diffbot":["partial","*",8,10],"ContextFetchBot":["partial","*",8,10]},"llms":{"status":"present","http":200,"bytes":27753},"home":{"status":"ok","http":200,"url":"https://www.blackmagicdesign.com/","html_tokens":37170,"markdown_tokens":58,"pruned_tokens":58}},
{"rank":2947,"domain":"uni-muenster.de","origin":"https://uni-muenster.de","robots":{"status":"ok","http":200,"url":"https://www.uni-muenster.de/robots.txt","bytes":308,"groups":1},"bots":{"GPTBot":["partial","*",11,0],"ChatGPT-User":["partial","*",11,0],"OAI-SearchBot":["partial","*",11,0],"ClaudeBot":["partial","*",11,0],"Claude-User":["partial","*",11,0],"Claude-SearchBot":["partial","*",11,0],"anthropic-ai":["partial","*",11,0],"PerplexityBot":["partial","*",11,0],"Perplexity-User":["partial","*",11,0],"Google-Extended":["partial","*",11,0],"Applebot-Extended":["partial","*",11,0],"CCBot":["partial","*",11,0],"Bytespider":["partial","*",11,0],"Amazonbot":["partial","*",11,0],"meta-externalagent":["partial","*",11,0],"cohere-ai":["partial","*",11,0],"Diffbot":["partial","*",11,0],"ContextFetchBot":["partial","*",11,0]},"llms":{"status":"missing","http":404,"bytes":16811},"home":{"status":"ok","http":200,"url":"https://www.uni-muenster.de/de/","html_tokens":8722,"markdown_tokens":274,"pruned_tokens":173}},
{"rank":2948,"domain":"36kr.com","origin":"https://36kr.com","robots":{"status":"html","http":200,"url":"https://36kr.com/robots.txt","bytes":25361,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":2949,"domain":"nh.gov","origin":"https://nh.gov","robots":{"status":"refused","http":403,"url":"https://www.nh.gov/robots.txt","bytes":376,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":2950,"domain":"glamour.com","origin":"https://glamour.com","robots":{"status":"ok","http":200,"url":"https://www.glamour.com/robots.txt","bytes":2239,"groups":3},"bots":{"GPTBot":["partial","*",15,14],"ChatGPT-User":["partial","*",15,14],"OAI-SearchBot":["partial","*",15,14],"ClaudeBot":["blocked","named",1,0],"Claude-User":["blocked","named",1,0],"Claude-SearchBot":["blocked","named",1,0],"anthropic-ai":["partial","*",15,14],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["blocked","named",1,0],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",15,14]},"llms":{"status":"missing","http":404,"bytes":735303},"home":{"status":"ok","http":200,"url":"https://www.glamour.com/","html_tokens":363492,"markdown_tokens":21,"pruned_tokens":21}},
{"rank":2951,"domain":"puma.com","origin":"https://puma.com","robots":{"status":"html","http":200,"url":"https://us.puma.com/us/en","bytes":881575,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":2952,"domain":"utm.edu","origin":"https://utm.edu","robots":{"status":"ok","http":200,"url":"https://utm.edu/robots.txt","bytes":275,"groups":1},"bots":{"GPTBot":["partial","*",12,0],"ChatGPT-User":["partial","*",12,0],"OAI-SearchBot":["partial","*",12,0],"ClaudeBot":["partial","*",12,0],"Claude-User":["partial","*",12,0],"Claude-SearchBot":["partial","*",12,0],"anthropic-ai":["partial","*",12,0],"PerplexityBot":["partial","*",12,0],"Perplexity-User":["partial","*",12,0],"Google-Extended":["partial","*",12,0],"Applebot-Extended":["partial","*",12,0],"CCBot":["partial","*",12,0],"Bytespider":["partial","*",12,0],"Amazonbot":["partial","*",12,0],"meta-externalagent":["partial","*",12,0],"cohere-ai":["partial","*",12,0],"Diffbot":["partial","*",12,0],"ContextFetchBot":["partial","*",12,0]},"llms":{"status":"missing","http":404,"bytes":102291},"home":{"status":"ok","http":200,"url":"https://utm.edu/","html_tokens":26497,"markdown_tokens":299,"pruned_tokens":147}},
{"rank":2953,"domain":"elperiodico.com","origin":"https://www.elperiodico.com","robots":{"status":"refused","http":406,"url":"https://www.elperiodico.com/robots.txt","bytes":437,"groups":0,"error":"HTTP 406"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":2954,"domain":"9to5google.com","origin":"https://9to5google.com","robots":{"status":"ok","http":200,"url":"https://9to5google.com/robots.txt","bytes":914,"groups":4},"bots":{"GPTBot":["partial","*",4,0],"ChatGPT-User":["partial","*",4,0],"OAI-SearchBot":["partial","*",4,0],"ClaudeBot":["partial","*",4,0],"Claude-User":["partial","*",4,0],"Claude-SearchBot":["partial","*",4,0],"anthropic-ai":["partial","*",4,0],"PerplexityBot":["partial","*",4,0],"Perplexity-User":["partial","*",4,0],"Google-Extended":["partial","*",4,0],"Applebot-Extended":["partial","*",4,0],"CCBot":["partial","*",4,0],"Bytespider":["partial","*",4,0],"Amazonbot":["partial","*",4,0],"meta-externalagent":["partial","*",4,0],"cohere-ai":["partial","*",4,0],"Diffbot":["partial","*",4,0],"ContextFetchBot":["partial","*",4,0]},"llms":{"status":"missing","http":404,"bytes":116229},"home":{"status":"ok","http":200,"url":"https://9to5google.com/","html_tokens":67687,"markdown_tokens":1213,"pruned_tokens":85}},
{"rank":2955,"domain":"dailystar.co.uk","origin":"https://dailystar.co.uk","robots":{"status":"ok","http":200,"url":"https://www.dailystar.co.uk/robots.txt","bytes":1199,"groups":18},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["partial","*",10,0],"OAI-SearchBot":["blocked","named",1,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",10,0],"Claude-SearchBot":["partial","*",10,0],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["partial","*",10,0],"Google-Extended":["partial","*",10,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["partial","*",10,0],"Amazonbot":["partial","*",10,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["partial","*",10,0],"Diffbot":["partial","*",10,0],"ContextFetchBot":["partial","*",10,0]},"llms":{"status":"missing","http":404,"bytes":442880},"home":{"status":"ok","http":200,"url":"https://www.dailystar.co.uk/","html_tokens":408158,"markdown_tokens":153,"pruned_tokens":153}},
{"rank":2956,"domain":"nola.com","origin":"https://nola.com","robots":{"status":"ok","http":200,"url":"https://nola.com/robots.txt","bytes":8136,"groups":31},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["blocked","named",1,0],"OAI-SearchBot":["blocked","named",1,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",113,0],"Claude-SearchBot":["partial","*",113,0],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["partial","*",113,0],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["partial","*",113,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",113,0]},"llms":{"status":"http_error","http":429,"bytes":64},"home":{"status":"ok","http":200,"url":"https://www.nola.com/","html_tokens":148308,"markdown_tokens":114,"pruned_tokens":114}},
{"rank":2957,"domain":"anyflip.com","origin":"https://anyflip.com","robots":{"status":"ok","http":200,"url":"https://anyflip.com/robots.txt","bytes":8527,"groups":150},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["partial","*",12,0],"OAI-SearchBot":["partial","*",12,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",12,0],"Claude-SearchBot":["partial","*",12,0],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["partial","*",12,0],"Perplexity-User":["partial","*",12,0],"Google-Extended":["partial","*",12,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",2,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",12,0]},"llms":{"status":"missing","http":404,"bytes":146},"home":{"status":"ok","http":200,"url":"https://anyflip.com/","html_tokens":17528,"markdown_tokens":316,"pruned_tokens":232}},
{"rank":2958,"domain":"newswire.ca","origin":"https://newswire.ca","robots":{"status":"ok","http":200,"url":"https://www.newswire.ca/robots.txt","bytes":250,"groups":1},"bots":{"GPTBot":["partial","*",1,0],"ChatGPT-User":["partial","*",1,0],"OAI-SearchBot":["partial","*",1,0],"ClaudeBot":["partial","*",1,0],"Claude-User":["partial","*",1,0],"Claude-SearchBot":["partial","*",1,0],"anthropic-ai":["partial","*",1,0],"PerplexityBot":["partial","*",1,0],"Perplexity-User":["partial","*",1,0],"Google-Extended":["partial","*",1,0],"Applebot-Extended":["partial","*",1,0],"CCBot":["partial","*",1,0],"Bytespider":["partial","*",1,0],"Amazonbot":["partial","*",1,0],"meta-externalagent":["partial","*",1,0],"cohere-ai":["partial","*",1,0],"Diffbot":["partial","*",1,0],"ContextFetchBot":["partial","*",1,0]},"llms":{"status":"missing","http":404,"bytes":621},"home":{"status":"ok","http":200,"url":"https://www.newswire.ca/","html_tokens":57610,"markdown_tokens":120,"pruned_tokens":71}},
{"rank":2959,"domain":"genial.ly","origin":"https://genial.ly","robots":{"status":"ok","http":200,"url":"https://genially.com/robots.txt","bytes":68,"groups":1},"bots":{"GPTBot":["allowed","*",0,1],"ChatGPT-User":["allowed","*",0,1],"OAI-SearchBot":["allowed","*",0,1],"ClaudeBot":["allowed","*",0,1],"Claude-User":["allowed","*",0,1],"Claude-SearchBot":["allowed","*",0,1],"anthropic-ai":["allowed","*",0,1],"PerplexityBot":["allowed","*",0,1],"Perplexity-User":["allowed","*",0,1],"Google-Extended":["allowed","*",0,1],"Applebot-Extended":["allowed","*",0,1],"CCBot":["allowed","*",0,1],"Bytespider":["allowed","*",0,1],"Amazonbot":["allowed","*",0,1],"meta-externalagent":["allowed","*",0,1],"cohere-ai":["allowed","*",0,1],"Diffbot":["allowed","*",0,1],"ContextFetchBot":["allowed","*",0,1]},"llms":{"status":"present","http":200,"bytes":7501},"home":{"status":"ok","http":200,"url":"https://genially.com/","html_tokens":44736,"markdown_tokens":129,"pruned_tokens":39}},
{"rank":2960,"domain":"nv.gov","origin":"https://nv.gov","robots":{"status":"ok","http":200,"url":"https://www.nv.gov/robots.txt","bytes":140,"groups":4},"bots":{"GPTBot":["blocked","*",1,0],"ChatGPT-User":["blocked","*",1,0],"OAI-SearchBot":["blocked","*",1,0],"ClaudeBot":["blocked","*",1,0],"Claude-User":["blocked","*",1,0],"Claude-SearchBot":["blocked","*",1,0],"anthropic-ai":["blocked","*",1,0],"PerplexityBot":["blocked","*",1,0],"Perplexity-User":["blocked","*",1,0],"Google-Extended":["blocked","*",1,0],"Applebot-Extended":["blocked","*",1,0],"CCBot":["blocked","*",1,0],"Bytespider":["blocked","*",1,0],"Amazonbot":["blocked","*",1,0],"meta-externalagent":["blocked","*",1,0],"cohere-ai":["blocked","*",1,0],"Diffbot":["blocked","*",1,0],"ContextFetchBot":["blocked","*",1,0]},"llms":{"status":"disallowed"},"home":{"status":"disallowed"}},
{"rank":2961,"domain":"england.nhs.uk","origin":"https://england.nhs.uk","robots":{"status":"ok","http":200,"url":"https://england.nhs.uk/robots.txt","bytes":26,"groups":1},"bots":{"GPTBot":["blocked","*",1,0],"ChatGPT-User":["blocked","*",1,0],"OAI-SearchBot":["blocked","*",1,0],"ClaudeBot":["blocked","*",1,0],"Claude-User":["blocked","*",1,0],"Claude-SearchBot":["blocked","*",1,0],"anthropic-ai":["blocked","*",1,0],"PerplexityBot":["blocked","*",1,0],"Perplexity-User":["blocked","*",1,0],"Google-Extended":["blocked","*",1,0],"Applebot-Extended":["blocked","*",1,0],"CCBot":["blocked","*",1,0],"Bytespider":["blocked","*",1,0],"Amazonbot":["blocked","*",1,0],"meta-externalagent":["blocked","*",1,0],"cohere-ai":["blocked","*",1,0],"Diffbot":["blocked","*",1,0],"ContextFetchBot":["blocked","*",1,0]},"llms":{"status":"disallowed"},"home":{"status":"disallowed"}},
{"rank":2962,"domain":"ritzcarlton.com","origin":"https://ritzcarlton.com","robots":{"status":"ok","http":200,"url":"https://www.ritzcarlton.com/robots.txt","bytes":187,"groups":2},"bots":{"GPTBot":["partial","*",1,0],"ChatGPT-User":["partial","*",1,0],"OAI-SearchBot":["partial","*",1,0],"ClaudeBot":["partial","*",1,0],"Claude-User":["partial","*",1,0],"Claude-SearchBot":["partial","*",1,0],"anthropic-ai":["partial","*",1,0],"PerplexityBot":["partial","*",1,0],"Perplexity-User":["partial","*",1,0],"Google-Extended":["partial","*",1,0],"Applebot-Extended":["partial","*",1,0],"CCBot":["partial","*",1,0],"Bytespider":["partial","*",1,0],"Amazonbot":["partial","*",1,0],"meta-externalagent":["partial","*",1,0],"cohere-ai":["partial","*",1,0],"Diffbot":["partial","*",1,0],"ContextFetchBot":["partial","*",1,0]},"llms":{"status":"missing","http":404,"bytes":196},"home":{"status":"ok","http":200,"url":"https://www.ritzcarlton.com/","html_tokens":65756,"markdown_tokens":93,"pruned_tokens":93}},
{"rank":2963,"domain":"simplecast.com","origin":"https://simplecast.com","robots":{"status":"ok","http":200,"url":"https://www.simplecast.com/robots.txt","bytes":210,"groups":1},"bots":{"GPTBot":["partial","*",7,0],"ChatGPT-User":["partial","*",7,0],"OAI-SearchBot":["partial","*",7,0],"ClaudeBot":["partial","*",7,0],"Claude-User":["partial","*",7,0],"Claude-SearchBot":["partial","*",7,0],"anthropic-ai":["partial","*",7,0],"PerplexityBot":["partial","*",7,0],"Perplexity-User":["partial","*",7,0],"Google-Extended":["partial","*",7,0],"Applebot-Extended":["partial","*",7,0],"CCBot":["partial","*",7,0],"Bytespider":["partial","*",7,0],"Amazonbot":["partial","*",7,0],"meta-externalagent":["partial","*",7,0],"cohere-ai":["partial","*",7,0],"Diffbot":["partial","*",7,0],"ContextFetchBot":["partial","*",7,0]},"llms":{"status":"missing","http":404,"bytes":4762},"home":{"status":"ok","http":200,"url":"https://www.simplecast.com/","html_tokens":23176,"markdown_tokens":578,"pruned_tokens":149}},
{"rank":2964,"domain":"ademe.fr","origin":"https://ademe.fr","robots":{"status":"refused","http":403,"url":"https://www.ademe.fr/robots.txt","bytes":5507,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":2965,"domain":"kia.com","origin":"https://kia.com","robots":{"status":"ok","http":200,"url":"https://www.kia.com/robots.txt","bytes":4915,"groups":2},"bots":{"GPTBot":["partial","*",21,4],"ChatGPT-User":["partial","*",21,4],"OAI-SearchBot":["partial","*",21,4],"ClaudeBot":["partial","*",21,4],"Claude-User":["partial","*",21,4],"Claude-SearchBot":["partial","*",21,4],"anthropic-ai":["partial","*",21,4],"PerplexityBot":["partial","*",21,4],"Perplexity-User":["partial","*",21,4],"Google-Extended":["partial","*",21,4],"Applebot-Extended":["partial","*",21,4],"CCBot":["partial","*",21,4],"Bytespider":["partial","*",21,4],"Amazonbot":["partial","*",21,4],"meta-externalagent":["partial","*",21,4],"cohere-ai":["partial","*",21,4],"Diffbot":["partial","*",21,4],"ContextFetchBot":["partial","*",21,4]},"llms":{"status":"present","http":200,"bytes":3237},"home":{"status":"ok","http":200,"url":"https://www.kia.com/us/en","html_tokens":104404,"markdown_tokens":8027,"pruned_tokens":310}},
{"rank":2966,"domain":"pinterest.ca","origin":"https://pinterest.ca","robots":{"status":"ok","http":200,"url":"https://ca.pinterest.com/robots.txt","bytes":29433,"groups":8},"bots":{"GPTBot":["blocked","*",1,0],"ChatGPT-User":["blocked","*",1,0],"OAI-SearchBot":["blocked","*",1,0],"ClaudeBot":["blocked","*",1,0],"Claude-User":["blocked","*",1,0],"Claude-SearchBot":["blocked","*",1,0],"anthropic-ai":["blocked","*",1,0],"PerplexityBot":["blocked","*",1,0],"Perplexity-User":["blocked","*",1,0],"Google-Extended":["blocked","*",1,0],"Applebot-Extended":["blocked","*",1,0],"CCBot":["blocked","*",1,0],"Bytespider":["blocked","*",1,0],"Amazonbot":["blocked","*",1,0],"meta-externalagent":["blocked","*",1,0],"cohere-ai":["blocked","*",1,0],"Diffbot":["blocked","*",1,0],"ContextFetchBot":["blocked","*",1,0]},"llms":{"status":"disallowed"},"home":{"status":"disallowed"}},
{"rank":2967,"domain":"politifact.com","origin":"https://politifact.com","robots":{"status":"ok","http":200,"url":"https://politifact.com/robots.txt","bytes":133,"groups":1},"bots":{"GPTBot":["partial","*",1,1],"ChatGPT-User":["partial","*",1,1],"OAI-SearchBot":["partial","*",1,1],"ClaudeBot":["partial","*",1,1],"Claude-User":["partial","*",1,1],"Claude-SearchBot":["partial","*",1,1],"anthropic-ai":["partial","*",1,1],"PerplexityBot":["partial","*",1,1],"Perplexity-User":["partial","*",1,1],"Google-Extended":["partial","*",1,1],"Applebot-Extended":["partial","*",1,1],"CCBot":["partial","*",1,1],"Bytespider":["partial","*",1,1],"Amazonbot":["partial","*",1,1],"meta-externalagent":["partial","*",1,1],"cohere-ai":["partial","*",1,1],"Diffbot":["partial","*",1,1],"ContextFetchBot":["partial","*",1,1]},"llms":{"status":"http_error","http":403,"bytes":5545},"home":{"status":"http_error","http":403,"url":"https://politifact.com/"}},
{"rank":2968,"domain":"donorbox.org","origin":"https://donorbox.org","robots":{"status":"ok","http":200,"url":"https://donorbox.org/robots.txt","bytes":268,"groups":1},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"present","http":200,"bytes":19518},"home":{"status":"ok","http":200,"url":"https://donorbox.org/","html_tokens":298834,"markdown_tokens":1655,"pruned_tokens":510}},
{"rank":2969,"domain":"google.co.th","origin":"https://google.co.th","robots":{"status":"ok","http":200,"url":"https://www.google.co.th/robots.txt","bytes":6624,"groups":4},"bots":{"GPTBot":["partial","*",169,64],"ChatGPT-User":["partial","*",169,64],"OAI-SearchBot":["partial","*",169,64],"ClaudeBot":["partial","*",169,64],"Claude-User":["partial","*",169,64],"Claude-SearchBot":["partial","*",169,64],"anthropic-ai":["partial","*",169,64],"PerplexityBot":["partial","*",169,64],"Perplexity-User":["partial","*",169,64],"Google-Extended":["partial","*",169,64],"Applebot-Extended":["partial","*",169,64],"CCBot":["partial","*",169,64],"Bytespider":["partial","*",169,64],"Amazonbot":["partial","*",169,64],"meta-externalagent":["partial","*",169,64],"cohere-ai":["partial","*",169,64],"Diffbot":["partial","*",169,64],"ContextFetchBot":["partial","*",169,64]},"llms":{"status":"missing","http":404,"bytes":1569},"home":{"status":"ok","http":200,"url":"https://www.google.com/","html_tokens":21504,"markdown_tokens":30,"pruned_tokens":30}},
{"rank":2970,"domain":"tau.ac.il","origin":"https://tau.ac.il","robots":{"status":"html","http":200,"url":"https://tau.ac.il/robots.txt","bytes":269,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":2971,"domain":"hostgator.com","origin":"https://hostgator.com","robots":{"status":"refused","http":403,"url":"https://www.hostgator.com/robots.txt","bytes":387762,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":2972,"domain":"bpb.de","origin":"https://bpb.de","robots":{"status":"ok","http":200,"url":"https://www.bpb.de/robots.txt","bytes":1701,"groups":1},"bots":{"GPTBot":["partial","*",22,18],"ChatGPT-User":["partial","*",22,18],"OAI-SearchBot":["partial","*",22,18],"ClaudeBot":["partial","*",22,18],"Claude-User":["partial","*",22,18],"Claude-SearchBot":["partial","*",22,18],"anthropic-ai":["partial","*",22,18],"PerplexityBot":["partial","*",22,18],"Perplexity-User":["partial","*",22,18],"Google-Extended":["partial","*",22,18],"Applebot-Extended":["partial","*",22,18],"CCBot":["partial","*",22,18],"Bytespider":["partial","*",22,18],"Amazonbot":["partial","*",22,18],"meta-externalagent":["partial","*",22,18],"cohere-ai":["partial","*",22,18],"Diffbot":["partial","*",22,18],"ContextFetchBot":["partial","*",22,18]},"llms":{"status":"missing","http":404,"bytes":184},"home":{"status":"ok","http":200,"url":"https://www.bpb.de/","html_tokens":106308,"markdown_tokens":2514,"pruned_tokens":100}},
{"rank":2973,"domain":"nouvelobs.com","origin":"https://nouvelobs.com","robots":{"status":"ok","http":200,"url":"https://nouvelobs.com/robots.txt","bytes":8475,"groups":5},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["blocked","named",1,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":118507},"home":{"status":"ok","http":200,"url":"https://www.nouvelobs.com/?_staled_=","html_tokens":33282,"markdown_tokens":539,"pruned_tokens":115}},
{"rank":2974,"domain":"autoblog.com","origin":"https://autoblog.com","robots":{"status":"ok","http":200,"url":"https://www.autoblog.com/robots.txt","bytes":186,"groups":1},"bots":{"GPTBot":["partial","*",4,0],"ChatGPT-User":["partial","*",4,0],"OAI-SearchBot":["partial","*",4,0],"ClaudeBot":["partial","*",4,0],"Claude-User":["partial","*",4,0],"Claude-SearchBot":["partial","*",4,0],"anthropic-ai":["partial","*",4,0],"PerplexityBot":["partial","*",4,0],"Perplexity-User":["partial","*",4,0],"Google-Extended":["partial","*",4,0],"Applebot-Extended":["partial","*",4,0],"CCBot":["partial","*",4,0],"Bytespider":["partial","*",4,0],"Amazonbot":["partial","*",4,0],"meta-externalagent":["partial","*",4,0],"cohere-ai":["partial","*",4,0],"Diffbot":["partial","*",4,0],"ContextFetchBot":["partial","*",4,0]},"llms":{"status":"http_error","http":403,"bytes":772},"home":{"status":"http_error","http":403,"url":"https://autoblog.com/"}},
{"rank":2975,"domain":"pitchbook.com","origin":"https://pitchbook.com","robots":{"status":"ok","http":200,"url":"https://pitchbook.com/robots.txt","bytes":329,"groups":1},"bots":{"GPTBot":["partial","*",7,0],"ChatGPT-User":["partial","*",7,0],"OAI-SearchBot":["partial","*",7,0],"ClaudeBot":["partial","*",7,0],"Claude-User":["partial","*",7,0],"Claude-SearchBot":["partial","*",7,0],"anthropic-ai":["partial","*",7,0],"PerplexityBot":["partial","*",7,0],"Perplexity-User":["partial","*",7,0],"Google-Extended":["partial","*",7,0],"Applebot-Extended":["partial","*",7,0],"CCBot":["partial","*",7,0],"Bytespider":["partial","*",7,0],"Amazonbot":["partial","*",7,0],"meta-externalagent":["partial","*",7,0],"cohere-ai":["partial","*",7,0],"Diffbot":["partial","*",7,0],"ContextFetchBot":["partial","*",7,0]},"llms":{"status":"http_error","http":403,"bytes":5480},"home":{"status":"http_error","http":403,"url":"https://pitchbook.com/"}},
{"rank":2976,"domain":"cadenaser.com","origin":"https://cadenaser.com","robots":{"status":"ok","http":200,"url":"https://cadenaser.com/robots.txt","bytes":5686,"groups":101},"bots":{"GPTBot":["partial","*",42,7],"ChatGPT-User":["partial","*",42,7],"OAI-SearchBot":["partial","*",42,7],"ClaudeBot":["blocked","named",1,0],"Claude-User":["blocked","named",1,0],"Claude-SearchBot":["blocked","named",1,0],"anthropic-ai":["partial","*",42,7],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["blocked","named",1,0],"Google-Extended":["partial","*",42,7],"Applebot-Extended":["partial","*",42,7],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["partial","*",42,7],"cohere-ai":["partial","*",42,7],"Diffbot":["partial","*",42,7],"ContextFetchBot":["partial","*",42,7]},"llms":{"status":"missing","http":404,"bytes":144218},"home":{"status":"ok","http":200,"url":"https://cadenaser.com/","html_tokens":90250,"markdown_tokens":217,"pruned_tokens":174}},
{"rank":2977,"domain":"1wcasino.com","origin":"https://1wcasino.com","robots":{"status":"ok","http":200,"url":"https://1wcasino.com/robots.txt","bytes":25,"groups":1},"bots":{"GPTBot":["blocked","*",1,0],"ChatGPT-User":["blocked","*",1,0],"OAI-SearchBot":["blocked","*",1,0],"ClaudeBot":["blocked","*",1,0],"Claude-User":["blocked","*",1,0],"Claude-SearchBot":["blocked","*",1,0],"anthropic-ai":["blocked","*",1,0],"PerplexityBot":["blocked","*",1,0],"Perplexity-User":["blocked","*",1,0],"Google-Extended":["blocked","*",1,0],"Applebot-Extended":["blocked","*",1,0],"CCBot":["blocked","*",1,0],"Bytespider":["blocked","*",1,0],"Amazonbot":["blocked","*",1,0],"meta-externalagent":["blocked","*",1,0],"cohere-ai":["blocked","*",1,0],"Diffbot":["blocked","*",1,0],"ContextFetchBot":["blocked","*",1,0]},"llms":{"status":"disallowed"},"home":{"status":"disallowed"}},
{"rank":2978,"domain":"lmgtfy.com","origin":"https://lmgtfy.com","robots":{"status":"ok","http":200,"url":"https://lmgtfy.com/robots.txt","bytes":392,"groups":1},"bots":{"GPTBot":["partial","*",1,0],"ChatGPT-User":["partial","*",1,0],"OAI-SearchBot":["partial","*",1,0],"ClaudeBot":["partial","*",1,0],"Claude-User":["partial","*",1,0],"Claude-SearchBot":["partial","*",1,0],"anthropic-ai":["partial","*",1,0],"PerplexityBot":["partial","*",1,0],"Perplexity-User":["partial","*",1,0],"Google-Extended":["partial","*",1,0],"Applebot-Extended":["partial","*",1,0],"CCBot":["partial","*",1,0],"Bytespider":["partial","*",1,0],"Amazonbot":["partial","*",1,0],"meta-externalagent":["partial","*",1,0],"cohere-ai":["partial","*",1,0],"Diffbot":["partial","*",1,0],"ContextFetchBot":["partial","*",1,0]},"llms":{"status":"missing","http":404,"bytes":1564},"home":{"status":"ok","http":200,"url":"https://lmgtfy.com/","html_tokens":19114,"markdown_tokens":142,"pruned_tokens":142}},
{"rank":2979,"domain":"jma.go.jp","origin":"https://www.jma.go.jp","robots":{"status":"missing","http":404,"url":"https://www.jma.go.jp/robots.txt","bytes":2203,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":2203},"home":{"status":"ok","http":200,"url":"https://www.jma.go.jp/jma/index.html","html_tokens":3350,"markdown_tokens":6,"pruned_tokens":6}},
{"rank":2980,"domain":"masterclass.com","origin":"https://masterclass.com","robots":{"status":"ok","http":200,"url":"https://www.masterclass.com/robots.txt","bytes":724,"groups":5},"bots":{"GPTBot":["allowed","named",0,1],"ChatGPT-User":["partial","*",7,1],"OAI-SearchBot":["partial","*",7,1],"ClaudeBot":["allowed","named",0,1],"Claude-User":["partial","*",7,1],"Claude-SearchBot":["partial","*",7,1],"anthropic-ai":["partial","*",7,1],"PerplexityBot":["partial","*",7,1],"Perplexity-User":["partial","*",7,1],"Google-Extended":["allowed","named",0,1],"Applebot-Extended":["partial","*",7,1],"CCBot":["allowed","named",0,1],"Bytespider":["partial","*",7,1],"Amazonbot":["partial","*",7,1],"meta-externalagent":["partial","*",7,1],"cohere-ai":["partial","*",7,1],"Diffbot":["partial","*",7,1],"ContextFetchBot":["partial","*",7,1]},"llms":{"status":"present","http":200,"bytes":2292},"home":{"status":"http_error","http":403,"url":"https://www.masterclass.com/"}},
{"rank":2981,"domain":"opencart.com","origin":"https://opencart.com","robots":{"status":"ok","http":200,"url":"https://www.opencart.com/robots.txt","bytes":93,"groups":1},"bots":{"GPTBot":["partial","*",3,0],"ChatGPT-User":["partial","*",3,0],"OAI-SearchBot":["partial","*",3,0],"ClaudeBot":["partial","*",3,0],"Claude-User":["partial","*",3,0],"Claude-SearchBot":["partial","*",3,0],"anthropic-ai":["partial","*",3,0],"PerplexityBot":["partial","*",3,0],"Perplexity-User":["partial","*",3,0],"Google-Extended":["partial","*",3,0],"Applebot-Extended":["partial","*",3,0],"CCBot":["partial","*",3,0],"Bytespider":["partial","*",3,0],"Amazonbot":["partial","*",3,0],"meta-externalagent":["partial","*",3,0],"cohere-ai":["partial","*",3,0],"Diffbot":["partial","*",3,0],"ContextFetchBot":["partial","*",3,0]},"llms":{"status":"html","http":200,"bytes":13905},"home":{"status":"ok","http":200,"url":"https://www.opencart.com/","html_tokens":7219,"markdown_tokens":389,"pruned_tokens":366}},
{"rank":2982,"domain":"oaic.gov.au","origin":"https://oaic.gov.au","robots":{"status":"ok","http":200,"url":"https://www.oaic.gov.au/robots.txt","bytes":360,"groups":1},"bots":{"GPTBot":["partial","*",14,0],"ChatGPT-User":["partial","*",14,0],"OAI-SearchBot":["partial","*",14,0],"ClaudeBot":["partial","*",14,0],"Claude-User":["partial","*",14,0],"Claude-SearchBot":["partial","*",14,0],"anthropic-ai":["partial","*",14,0],"PerplexityBot":["partial","*",14,0],"Perplexity-User":["partial","*",14,0],"Google-Extended":["partial","*",14,0],"Applebot-Extended":["partial","*",14,0],"CCBot":["partial","*",14,0],"Bytespider":["partial","*",14,0],"Amazonbot":["partial","*",14,0],"meta-externalagent":["partial","*",14,0],"cohere-ai":["partial","*",14,0],"Diffbot":["partial","*",14,0],"ContextFetchBot":["partial","*",14,0]},"llms":{"status":"missing","http":404,"bytes":56372},"home":{"status":"ok","http":200,"url":"https://www.oaic.gov.au/","html_tokens":22089,"markdown_tokens":46,"pruned_tokens":46}},
{"rank":2983,"domain":"meltwater.com","origin":"https://meltwater.com","robots":{"status":"ok","http":200,"url":"https://www.meltwater.com/robots.txt","bytes":90,"groups":1},"bots":{"GPTBot":["partial","*",1,1],"ChatGPT-User":["partial","*",1,1],"OAI-SearchBot":["partial","*",1,1],"ClaudeBot":["partial","*",1,1],"Claude-User":["partial","*",1,1],"Claude-SearchBot":["partial","*",1,1],"anthropic-ai":["partial","*",1,1],"PerplexityBot":["partial","*",1,1],"Perplexity-User":["partial","*",1,1],"Google-Extended":["partial","*",1,1],"Applebot-Extended":["partial","*",1,1],"CCBot":["partial","*",1,1],"Bytespider":["partial","*",1,1],"Amazonbot":["partial","*",1,1],"meta-externalagent":["partial","*",1,1],"cohere-ai":["partial","*",1,1],"Diffbot":["partial","*",1,1],"ContextFetchBot":["partial","*",1,1]},"llms":{"status":"present","http":200,"bytes":9321},"home":{"status":"ok","http":200,"url":"https://www.meltwater.com/en","html_tokens":44869,"markdown_tokens":640,"pruned_tokens":281}},
{"rank":2984,"domain":"cc.com","origin":"https://cc.com","robots":{"status":"ok","http":200,"url":"https://www.cc.com/robots.txt","bytes":253,"groups":2},"bots":{"GPTBot":["partial","*",1,0],"ChatGPT-User":["partial","*",1,0],"OAI-SearchBot":["partial","*",1,0],"ClaudeBot":["partial","*",1,0],"Claude-User":["partial","*",1,0],"Claude-SearchBot":["partial","*",1,0],"anthropic-ai":["partial","*",1,0],"PerplexityBot":["partial","*",1,0],"Perplexity-User":["partial","*",1,0],"Google-Extended":["partial","*",1,0],"Applebot-Extended":["partial","*",1,0],"CCBot":["partial","*",1,0],"Bytespider":["partial","*",1,0],"Amazonbot":["partial","*",1,0],"meta-externalagent":["partial","*",1,0],"cohere-ai":["partial","*",1,0],"Diffbot":["partial","*",1,0],"ContextFetchBot":["partial","*",1,0]},"llms":{"status":"missing","http":404,"bytes":794765},"home":{"status":"ok","http":200,"url":"https://www.cc.com","html_tokens":787828,"markdown_tokens":1369,"pruned_tokens":157}},
{"rank":2985,"domain":"www.info.gouv.fr","origin":"https://www.info.gouv.fr","robots":{"status":"ok","http":200,"url":"https://www.info.gouv.fr/robots.txt","bytes":1688,"groups":1},"bots":{"GPTBot":["partial","*",30,0],"ChatGPT-User":["partial","*",30,0],"OAI-SearchBot":["partial","*",30,0],"ClaudeBot":["partial","*",30,0],"Claude-User":["partial","*",30,0],"Claude-SearchBot":["partial","*",30,0],"anthropic-ai":["partial","*",30,0],"PerplexityBot":["partial","*",30,0],"Perplexity-User":["partial","*",30,0],"Google-Extended":["partial","*",30,0],"Applebot-Extended":["partial","*",30,0],"CCBot":["partial","*",30,0],"Bytespider":["partial","*",30,0],"Amazonbot":["partial","*",30,0],"meta-externalagent":["partial","*",30,0],"cohere-ai":["partial","*",30,0],"Diffbot":["partial","*",30,0],"ContextFetchBot":["partial","*",30,0]},"llms":{"status":"missing","http":404,"bytes":65256},"home":{"status":"http_error","http":403,"url":"https://www.info.gouv.fr/"}},
{"rank":2986,"domain":"odysee.com","origin":"https://odysee.com","robots":{"status":"ok","http":200,"url":"https://odysee.com/robots.txt","bytes":23,"groups":1},"bots":{"GPTBot":["allowed","*",0,1],"ChatGPT-User":["allowed","*",0,1],"OAI-SearchBot":["allowed","*",0,1],"ClaudeBot":["allowed","*",0,1],"Claude-User":["allowed","*",0,1],"Claude-SearchBot":["allowed","*",0,1],"anthropic-ai":["allowed","*",0,1],"PerplexityBot":["allowed","*",0,1],"Perplexity-User":["allowed","*",0,1],"Google-Extended":["allowed","*",0,1],"Applebot-Extended":["allowed","*",0,1],"CCBot":["allowed","*",0,1],"Bytespider":["allowed","*",0,1],"Amazonbot":["allowed","*",0,1],"meta-externalagent":["allowed","*",0,1],"cohere-ai":["allowed","*",0,1],"Diffbot":["allowed","*",0,1],"ContextFetchBot":["allowed","*",0,1]},"llms":{"status":"html","http":200,"bytes":11432},"home":{"status":"ok","http":200,"url":"https://odysee.com/","html_tokens":3023,"markdown_tokens":0,"pruned_tokens":0}},
{"rank":2987,"domain":"wpconsent.com","origin":"https://wpconsent.com","robots":{"status":"ok","http":200,"url":"https://wpconsent.com/robots.txt","bytes":199,"groups":1},"bots":{"GPTBot":["partial","*",2,1],"ChatGPT-User":["partial","*",2,1],"OAI-SearchBot":["partial","*",2,1],"ClaudeBot":["partial","*",2,1],"Claude-User":["partial","*",2,1],"Claude-SearchBot":["partial","*",2,1],"anthropic-ai":["partial","*",2,1],"PerplexityBot":["partial","*",2,1],"Perplexity-User":["partial","*",2,1],"Google-Extended":["partial","*",2,1],"Applebot-Extended":["partial","*",2,1],"CCBot":["partial","*",2,1],"Bytespider":["partial","*",2,1],"Amazonbot":["partial","*",2,1],"meta-externalagent":["partial","*",2,1],"cohere-ai":["partial","*",2,1],"Diffbot":["partial","*",2,1],"ContextFetchBot":["partial","*",2,1]},"llms":{"status":"other_text","http":200,"bytes":61334},"home":{"status":"ok","http":200,"url":"https://wpconsent.com/","html_tokens":80912,"markdown_tokens":1287,"pruned_tokens":683}},
{"rank":2988,"domain":"graph.org","origin":"https://graph.org","robots":{"status":"missing","http":404,"url":"https://graph.org/robots.txt","bytes":951,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":951},"home":{"status":"ok","http":200,"url":"https://graph.org/","html_tokens":1144,"markdown_tokens":0,"pruned_tokens":0}},
{"rank":2989,"domain":"vim.org","origin":"https://www.vim.org","robots":{"status":"ok","http":200,"url":"https://www.vim.org/robots.txt","bytes":41,"groups":1},"bots":{"GPTBot":["partial","*",1,1],"ChatGPT-User":["partial","*",1,1],"OAI-SearchBot":["partial","*",1,1],"ClaudeBot":["partial","*",1,1],"Claude-User":["partial","*",1,1],"Claude-SearchBot":["partial","*",1,1],"anthropic-ai":["partial","*",1,1],"PerplexityBot":["partial","*",1,1],"Perplexity-User":["partial","*",1,1],"Google-Extended":["partial","*",1,1],"Applebot-Extended":["partial","*",1,1],"CCBot":["partial","*",1,1],"Bytespider":["partial","*",1,1],"Amazonbot":["partial","*",1,1],"meta-externalagent":["partial","*",1,1],"cohere-ai":["partial","*",1,1],"Diffbot":["partial","*",1,1],"ContextFetchBot":["partial","*",1,1]},"llms":{"status":"missing","http":404,"bytes":196},"home":{"status":"ok","http":200,"url":"https://www.vim.org/","html_tokens":4572,"markdown_tokens":959,"pruned_tokens":398}},
{"rank":2990,"domain":"so-net.ne.jp","origin":"https://so-net.ne.jp","robots":{"status":"ok","http":200,"url":"https://www.so-net.ne.jp/robots.txt","bytes":1063,"groups":1},"bots":{"GPTBot":["partial","*",32,0],"ChatGPT-User":["partial","*",32,0],"OAI-SearchBot":["partial","*",32,0],"ClaudeBot":["partial","*",32,0],"Claude-User":["partial","*",32,0],"Claude-SearchBot":["partial","*",32,0],"anthropic-ai":["partial","*",32,0],"PerplexityBot":["partial","*",32,0],"Perplexity-User":["partial","*",32,0],"Google-Extended":["partial","*",32,0],"Applebot-Extended":["partial","*",32,0],"CCBot":["partial","*",32,0],"Bytespider":["partial","*",32,0],"Amazonbot":["partial","*",32,0],"meta-externalagent":["partial","*",32,0],"cohere-ai":["partial","*",32,0],"Diffbot":["partial","*",32,0],"ContextFetchBot":["partial","*",32,0]},"llms":{"status":"missing","http":404,"bytes":2813},"home":{"status":"ok","http":200,"url":"https://www.so-net.ne.jp/","html_tokens":28546,"markdown_tokens":216,"pruned_tokens":49}},
{"rank":2991,"domain":"e-monsite.com","origin":"https://e-monsite.com","robots":{"status":"ok","http":200,"url":"https://www.e-monsite.com/robots.txt","bytes":389,"groups":5},"bots":{"GPTBot":["partial","*",1,1],"ChatGPT-User":["partial","*",1,1],"OAI-SearchBot":["partial","*",1,1],"ClaudeBot":["partial","*",1,1],"Claude-User":["partial","*",1,1],"Claude-SearchBot":["partial","*",1,1],"anthropic-ai":["partial","*",1,1],"PerplexityBot":["partial","*",1,1],"Perplexity-User":["partial","*",1,1],"Google-Extended":["partial","*",1,1],"Applebot-Extended":["partial","*",1,1],"CCBot":["partial","*",1,1],"Bytespider":["partial","*",1,1],"Amazonbot":["partial","*",1,1],"meta-externalagent":["partial","*",1,1],"cohere-ai":["partial","*",1,1],"Diffbot":["partial","*",1,1],"ContextFetchBot":["partial","*",1,1]},"llms":{"status":"present","http":200,"bytes":334339},"home":{"status":"ok","http":200,"url":"https://www.e-monsite.com/","html_tokens":38612,"markdown_tokens":2728,"pruned_tokens":594}},
{"rank":2992,"domain":"usmagazine.com","origin":"https://usmagazine.com","robots":{"status":"ok","http":200,"url":"https://www.usmagazine.com/robots.txt","bytes":5510,"groups":101},"bots":{"GPTBot":["partial","*",10,0],"ChatGPT-User":["partial","*",10,0],"OAI-SearchBot":["partial","*",10,0],"ClaudeBot":["partial","*",10,0],"Claude-User":["partial","*",10,0],"Claude-SearchBot":["partial","*",10,0],"anthropic-ai":["partial","*",10,0],"PerplexityBot":["partial","*",10,0],"Perplexity-User":["partial","*",10,0],"Google-Extended":["partial","*",10,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["partial","*",10,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",10,0]},"llms":{"status":"missing","http":404,"bytes":272903},"home":{"status":"ok","http":200,"url":"https://www.usmagazine.com/","html_tokens":118279,"markdown_tokens":42,"pruned_tokens":42}},
{"rank":2993,"domain":"infoseek.co.jp","origin":"https://infoseek.co.jp","robots":{"status":"html","http":200,"url":"https://www.infoseek.co.jp","bytes":228056,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":2994,"domain":"yourdomain.com","origin":"https://yourdomain.com","robots":{"status":"unreachable","http":null,"url":"https://yourdomain.com/robots.txt","bytes":0,"groups":0,"error":"[SSL: CERTIFICATE_VERIFY_FAILED] certificate verify failed: Hostname mismatch, certificate is not valid for 'yourdomain.com'. (_ssl.c:1028)"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":2995,"domain":"worldatlas.com","origin":"https://worldatlas.com","robots":{"status":"ok","http":200,"url":"https://www.worldatlas.com/robots.txt","bytes":810,"groups":2},"bots":{"GPTBot":["partial","*",15,0],"ChatGPT-User":["partial","*",15,0],"OAI-SearchBot":["partial","*",15,0],"ClaudeBot":["partial","*",15,0],"Claude-User":["partial","*",15,0],"Claude-SearchBot":["partial","*",15,0],"anthropic-ai":["partial","*",15,0],"PerplexityBot":["partial","*",15,0],"Perplexity-User":["partial","*",15,0],"Google-Extended":["partial","*",15,0],"Applebot-Extended":["partial","*",15,0],"CCBot":["partial","*",15,0],"Bytespider":["partial","*",15,0],"Amazonbot":["partial","*",15,0],"meta-externalagent":["partial","*",15,0],"cohere-ai":["partial","*",15,0],"Diffbot":["partial","*",15,0],"ContextFetchBot":["partial","*",15,0]},"llms":{"status":"missing","http":404,"bytes":34357},"home":{"status":"ok","http":200,"url":"https://www.worldatlas.com/","html_tokens":149450,"markdown_tokens":14,"pruned_tokens":14}},
{"rank":2996,"domain":"ripe.net","origin":"https://ripe.net","robots":{"status":"ok","http":200,"url":"https://www.ripe.net/robots.txt","bytes":234,"groups":1},"bots":{"GPTBot":["partial","*",4,0],"ChatGPT-User":["partial","*",4,0],"OAI-SearchBot":["partial","*",4,0],"ClaudeBot":["partial","*",4,0],"Claude-User":["partial","*",4,0],"Claude-SearchBot":["partial","*",4,0],"anthropic-ai":["partial","*",4,0],"PerplexityBot":["partial","*",4,0],"Perplexity-User":["partial","*",4,0],"Google-Extended":["partial","*",4,0],"Applebot-Extended":["partial","*",4,0],"CCBot":["partial","*",4,0],"Bytespider":["partial","*",4,0],"Amazonbot":["partial","*",4,0],"meta-externalagent":["partial","*",4,0],"cohere-ai":["partial","*",4,0],"Diffbot":["partial","*",4,0],"ContextFetchBot":["partial","*",4,0]},"llms":{"status":"missing","http":404,"bytes":107440},"home":{"status":"ok","http":200,"url":"https://www.ripe.net/","html_tokens":34265,"markdown_tokens":693,"pruned_tokens":477}},
{"rank":2997,"domain":"orlandosentinel.com","origin":"https://orlandosentinel.com","robots":{"status":"ok","http":200,"url":"https://www.orlandosentinel.com/robots.txt","bytes":4496,"groups":68},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["blocked","named",1,0],"OAI-SearchBot":["blocked","named",1,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["blocked","named",1,0],"Claude-SearchBot":["blocked","named",1,0],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["blocked","named",1,0],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["partial","*",8,1],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",8,1]},"llms":{"status":"missing","http":404,"bytes":110208},"home":{"status":"ok","http":200,"url":"https://www.orlandosentinel.com/","html_tokens":68259,"markdown_tokens":484,"pruned_tokens":105}},
{"rank":2998,"domain":"runwayml.com","origin":"https://runwayml.com","robots":{"status":"ok","http":200,"url":"https://runway.com/robots.txt","bytes":666,"groups":1},"bots":{"GPTBot":["partial","*",1,1],"ChatGPT-User":["partial","*",1,1],"OAI-SearchBot":["partial","*",1,1],"ClaudeBot":["partial","*",1,1],"Claude-User":["partial","*",1,1],"Claude-SearchBot":["partial","*",1,1],"anthropic-ai":["partial","*",1,1],"PerplexityBot":["partial","*",1,1],"Perplexity-User":["partial","*",1,1],"Google-Extended":["partial","*",1,1],"Applebot-Extended":["partial","*",1,1],"CCBot":["partial","*",1,1],"Bytespider":["partial","*",1,1],"Amazonbot":["partial","*",1,1],"meta-externalagent":["partial","*",1,1],"cohere-ai":["partial","*",1,1],"Diffbot":["partial","*",1,1],"ContextFetchBot":["partial","*",1,1]},"llms":{"status":"present","http":200,"bytes":8035},"home":{"status":"ok","http":200,"url":"https://runway.com/","html_tokens":296335,"markdown_tokens":336,"pruned_tokens":68}},
{"rank":2999,"domain":"basecamp.com","origin":"https://basecamp.com","robots":{"status":"ok","http":200,"url":"https://basecamp.com/robots.txt","bytes":105,"groups":1},"bots":{"GPTBot":["partial","*",2,1],"ChatGPT-User":["partial","*",2,1],"OAI-SearchBot":["partial","*",2,1],"ClaudeBot":["partial","*",2,1],"Claude-User":["partial","*",2,1],"Claude-SearchBot":["partial","*",2,1],"anthropic-ai":["partial","*",2,1],"PerplexityBot":["partial","*",2,1],"Perplexity-User":["partial","*",2,1],"Google-Extended":["partial","*",2,1],"Applebot-Extended":["partial","*",2,1],"CCBot":["partial","*",2,1],"Bytespider":["partial","*",2,1],"Amazonbot":["partial","*",2,1],"meta-externalagent":["partial","*",2,1],"cohere-ai":["partial","*",2,1],"Diffbot":["partial","*",2,1],"ContextFetchBot":["partial","*",2,1]},"llms":{"status":"missing","http":404,"bytes":22185},"home":{"status":"ok","http":200,"url":"https://basecamp.com/","html_tokens":29627,"markdown_tokens":305,"pruned_tokens":163}},
{"rank":3000,"domain":"groupon.com","origin":"https://groupon.com","robots":{"status":"ok","http":200,"url":"https://www.groupon.com/robots.txt","bytes":5431,"groups":13},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["partial","*",114,5],"OAI-SearchBot":["partial","*",114,5],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",114,5],"Claude-SearchBot":["partial","*",114,5],"anthropic-ai":["partial","*",114,5],"PerplexityBot":["partial","*",114,5],"Perplexity-User":["partial","*",114,5],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["partial","*",114,5],"Diffbot":["partial","*",114,5],"ContextFetchBot":["partial","*",114,5]},"llms":{"status":"present","http":200,"bytes":3859},"home":{"status":"http_error","http":403,"url":"https://www.groupon.com/"}},
{"rank":3001,"domain":"maff.go.jp","origin":"https://www.maff.go.jp","robots":{"status":"refused","http":403,"url":"https://www.maff.go.jp/robots.txt","bytes":37408,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":3002,"domain":"blogspot.com.au","origin":"https://blogspot.com.au","robots":{"status":"missing","http":404,"url":"https://blogspot.com.au/robots.txt","bytes":1571,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":1569},"home":{"status":"http_error","http":404,"url":"https://blogspot.com.au/"}},
{"rank":3003,"domain":"doe.gov","origin":"http://doe.gov","robots":{"status":"ok","http":200,"url":"https://www.energy.gov/robots.txt","bytes":2141,"groups":1},"bots":{"GPTBot":["partial","*",35,22],"ChatGPT-User":["partial","*",35,22],"OAI-SearchBot":["partial","*",35,22],"ClaudeBot":["partial","*",35,22],"Claude-User":["partial","*",35,22],"Claude-SearchBot":["partial","*",35,22],"anthropic-ai":["partial","*",35,22],"PerplexityBot":["partial","*",35,22],"Perplexity-User":["partial","*",35,22],"Google-Extended":["partial","*",35,22],"Applebot-Extended":["partial","*",35,22],"CCBot":["partial","*",35,22],"Bytespider":["partial","*",35,22],"Amazonbot":["partial","*",35,22],"meta-externalagent":["partial","*",35,22],"cohere-ai":["partial","*",35,22],"Diffbot":["partial","*",35,22],"ContextFetchBot":["partial","*",35,22]},"llms":{"status":"missing","http":404,"bytes":35385},"home":{"status":"ok","http":200,"url":"https://www.energy.gov/","html_tokens":32660,"markdown_tokens":194,"pruned_tokens":80}},
{"rank":3004,"domain":"hachettebookgroup.com","origin":"https://hachettebookgroup.com","robots":{"status":"ok","http":200,"url":"https://www.hachettebookgroup.com/robots.txt","bytes":141,"groups":1},"bots":{"GPTBot":["partial","*",1,1],"ChatGPT-User":["partial","*",1,1],"OAI-SearchBot":["partial","*",1,1],"ClaudeBot":["partial","*",1,1],"Claude-User":["partial","*",1,1],"Claude-SearchBot":["partial","*",1,1],"anthropic-ai":["partial","*",1,1],"PerplexityBot":["partial","*",1,1],"Perplexity-User":["partial","*",1,1],"Google-Extended":["partial","*",1,1],"Applebot-Extended":["partial","*",1,1],"CCBot":["partial","*",1,1],"Bytespider":["partial","*",1,1],"Amazonbot":["partial","*",1,1],"meta-externalagent":["partial","*",1,1],"cohere-ai":["partial","*",1,1],"Diffbot":["partial","*",1,1],"ContextFetchBot":["partial","*",1,1]},"llms":{"status":"missing","http":404,"bytes":269884},"home":{"status":"ok","http":200,"url":"https://www.hachettebookgroup.com/","html_tokens":137826,"markdown_tokens":114,"pruned_tokens":114}},
{"rank":3005,"domain":"extremetech.com","origin":"https://extremetech.com","robots":{"status":"refused","http":403,"url":"https://www.extremetech.com/robots.txt","bytes":5536,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":3006,"domain":"asme.org","origin":"https://asme.org","robots":{"status":"ok","http":200,"url":"https://www.asme.org/robots.txt","bytes":124,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":119594},"home":{"status":"ok","http":200,"url":"https://www.asme.org/","html_tokens":43762,"markdown_tokens":244,"pruned_tokens":157}},
{"rank":3007,"domain":"unimi.it","origin":"https://unimi.it","robots":{"status":"ok","http":200,"url":"https://www.unimi.it//robots.txt","bytes":2149,"groups":2},"bots":{"GPTBot":["partial","*",34,18],"ChatGPT-User":["partial","*",34,18],"OAI-SearchBot":["partial","*",34,18],"ClaudeBot":["partial","*",34,18],"Claude-User":["partial","*",34,18],"Claude-SearchBot":["partial","*",34,18],"anthropic-ai":["partial","*",34,18],"PerplexityBot":["partial","*",34,18],"Perplexity-User":["partial","*",34,18],"Google-Extended":["partial","*",34,18],"Applebot-Extended":["partial","*",34,18],"CCBot":["partial","*",34,18],"Bytespider":["partial","*",34,18],"Amazonbot":["partial","*",34,18],"meta-externalagent":["partial","*",34,18],"cohere-ai":["partial","*",34,18],"Diffbot":["partial","*",34,18],"ContextFetchBot":["partial","*",34,18]},"llms":{"status":"missing","http":404,"bytes":547},"home":{"status":"ok","http":200,"url":"https://www.unimi.it/it","html_tokens":27969,"markdown_tokens":46,"pruned_tokens":46}},
{"rank":3008,"domain":"cityu.edu.hk","origin":"https://cityu.edu.hk","robots":{"status":"ok","http":200,"url":"https://www.cityu.edu.hk/robots.txt","bytes":302,"groups":1},"bots":{"GPTBot":["partial","*",10,0],"ChatGPT-User":["partial","*",10,0],"OAI-SearchBot":["partial","*",10,0],"ClaudeBot":["partial","*",10,0],"Claude-User":["partial","*",10,0],"Claude-SearchBot":["partial","*",10,0],"anthropic-ai":["partial","*",10,0],"PerplexityBot":["partial","*",10,0],"Perplexity-User":["partial","*",10,0],"Google-Extended":["partial","*",10,0],"Applebot-Extended":["partial","*",10,0],"CCBot":["partial","*",10,0],"Bytespider":["partial","*",10,0],"Amazonbot":["partial","*",10,0],"meta-externalagent":["partial","*",10,0],"cohere-ai":["partial","*",10,0],"Diffbot":["partial","*",10,0],"ContextFetchBot":["partial","*",10,0]},"llms":{"status":"missing","http":404,"bytes":7211},"home":{"status":"ok","http":200,"url":"https://cityu.edu.hk/","html_tokens":53,"markdown_tokens":0,"pruned_tokens":0}},
{"rank":3009,"domain":"vogue.co.uk","origin":"https://vogue.co.uk","robots":{"status":"ok","http":200,"url":"https://www.vogue.co.uk/robots.txt","bytes":2170,"groups":3},"bots":{"GPTBot":["partial","*",14,12],"ChatGPT-User":["partial","*",14,12],"OAI-SearchBot":["partial","*",14,12],"ClaudeBot":["blocked","named",1,0],"Claude-User":["blocked","named",1,0],"Claude-SearchBot":["blocked","named",1,0],"anthropic-ai":["partial","*",14,12],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["blocked","named",1,0],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",14,12]},"llms":{"status":"missing","http":404,"bytes":472545},"home":{"status":"ok","http":200,"url":"https://www.vogue.co.uk/","html_tokens":304693,"markdown_tokens":37,"pruned_tokens":37}},
{"rank":3010,"domain":"ggpht.com","origin":"https://ggpht.com","robots":{"status":"unreachable","http":null,"url":"https://ggpht.com/robots.txt","bytes":0,"groups":0,"error":"cannot resolve ggpht.com"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":3011,"domain":"avma.org","origin":"https://avma.org","robots":{"status":"html","http":200,"url":"https://www.avma.org/robots.txt","bytes":954,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":3012,"domain":"fortunecity.com","origin":"http://fortunecity.com","robots":{"status":"ok","http":200,"url":"http://fortunecity.com/robots.txt","bytes":0,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":867},"home":{"status":"not_html","error":"unsupported content type application/x-httpd-lsphp"}},
{"rank":3013,"domain":"wsimg.com","origin":"https://wsimg.com","robots":{"status":"unreachable","http":null,"url":"https://wsimg.com/robots.txt","bytes":0,"groups":0,"error":"cannot resolve wsimg.com"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":3014,"domain":"texastribune.org","origin":"https://texastribune.org","robots":{"status":"ok","http":200,"url":"https://www.texastribune.org/robots.txt","bytes":256,"groups":1},"bots":{"GPTBot":["partial","*",1,0],"ChatGPT-User":["partial","*",1,0],"OAI-SearchBot":["partial","*",1,0],"ClaudeBot":["partial","*",1,0],"Claude-User":["partial","*",1,0],"Claude-SearchBot":["partial","*",1,0],"anthropic-ai":["partial","*",1,0],"PerplexityBot":["partial","*",1,0],"Perplexity-User":["partial","*",1,0],"Google-Extended":["partial","*",1,0],"Applebot-Extended":["partial","*",1,0],"CCBot":["partial","*",1,0],"Bytespider":["partial","*",1,0],"Amazonbot":["partial","*",1,0],"meta-externalagent":["partial","*",1,0],"cohere-ai":["partial","*",1,0],"Diffbot":["partial","*",1,0],"ContextFetchBot":["partial","*",1,0]},"llms":{"status":"missing","http":404,"bytes":195156},"home":{"status":"ok","http":200,"url":"https://www.texastribune.org/","html_tokens":95182,"markdown_tokens":169,"pruned_tokens":56}},
{"rank":3015,"domain":"coub.com","origin":"https://coub.com","robots":{"status":"ok","http":200,"url":"https://coub.com/robots.txt","bytes":2158,"groups":1},"bots":{"GPTBot":["partial","*",103,2],"ChatGPT-User":["partial","*",103,2],"OAI-SearchBot":["partial","*",103,2],"ClaudeBot":["partial","*",103,2],"Claude-User":["partial","*",103,2],"Claude-SearchBot":["partial","*",103,2],"anthropic-ai":["partial","*",103,2],"PerplexityBot":["partial","*",103,2],"Perplexity-User":["partial","*",103,2],"Google-Extended":["partial","*",103,2],"Applebot-Extended":["partial","*",103,2],"CCBot":["partial","*",103,2],"Bytespider":["partial","*",103,2],"Amazonbot":["partial","*",103,2],"meta-externalagent":["partial","*",103,2],"cohere-ai":["partial","*",103,2],"Diffbot":["partial","*",103,2],"ContextFetchBot":["partial","*",103,2]},"llms":{"status":"missing","http":404,"bytes":147571},"home":{"status":"ok","http":200,"url":"https://coub.com/featured/creators","html_tokens":237666,"markdown_tokens":37,"pruned_tokens":37}},
{"rank":3016,"domain":"udacity.com","origin":"https://udacity.com","robots":{"status":"ok","http":200,"url":"https://www.udacity.com/robots.txt","bytes":1173,"groups":2},"bots":{"GPTBot":["partial","*",33,1],"ChatGPT-User":["partial","*",33,1],"OAI-SearchBot":["partial","*",33,1],"ClaudeBot":["partial","*",33,1],"Claude-User":["partial","*",33,1],"Claude-SearchBot":["partial","*",33,1],"anthropic-ai":["partial","*",33,1],"PerplexityBot":["partial","*",33,1],"Perplexity-User":["partial","*",33,1],"Google-Extended":["partial","*",33,1],"Applebot-Extended":["blocked","named",1,0],"CCBot":["partial","*",33,1],"Bytespider":["partial","*",33,1],"Amazonbot":["partial","*",33,1],"meta-externalagent":["partial","*",33,1],"cohere-ai":["partial","*",33,1],"Diffbot":["partial","*",33,1],"ContextFetchBot":["partial","*",33,1]},"llms":{"status":"missing","http":404,"bytes":142891},"home":{"status":"ok","http":200,"url":"https://www.udacity.com/","html_tokens":157586,"markdown_tokens":135,"pruned_tokens":135}},
{"rank":3017,"domain":"mapsengine.google.com","origin":"https://mapsengine.google.com","robots":{"status":"html","http":200,"url":"https://mapsengine.google.com/about/index.html","bytes":3195,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":3018,"domain":"uniqlo.com","origin":"https://uniqlo.com","robots":{"status":"unreachable","http":null,"url":"https://uniqlo.com/robots.txt","bytes":0,"groups":0,"error":"<StreamReset stream_id:1, error_code:2, remote_reset:True>"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":3019,"domain":"chess.com","origin":"https://chess.com","robots":{"status":"ok","http":200,"url":"https://www.chess.com/robots.txt","bytes":3507,"groups":8},"bots":{"GPTBot":["partial","*",40,0],"ChatGPT-User":["partial","*",40,0],"OAI-SearchBot":["partial","*",40,0],"ClaudeBot":["partial","*",40,0],"Claude-User":["partial","*",40,0],"Claude-SearchBot":["partial","*",40,0],"anthropic-ai":["partial","*",40,0],"PerplexityBot":["partial","*",40,0],"Perplexity-User":["partial","*",40,0],"Google-Extended":["partial","*",40,0],"Applebot-Extended":["partial","*",40,0],"CCBot":["partial","*",40,0],"Bytespider":["partial","*",40,0],"Amazonbot":["partial","*",40,0],"meta-externalagent":["partial","*",40,0],"cohere-ai":["partial","*",40,0],"Diffbot":["partial","*",40,0],"ContextFetchBot":["partial","*",40,0]},"llms":{"status":"missing","http":404,"bytes":48709},"home":{"status":"ok","http":200,"url":"https://www.chess.com/","html_tokens":30313,"markdown_tokens":8,"pruned_tokens":8}},
{"rank":3020,"domain":"gamedeveloper.com","origin":"https://gamedeveloper.com","robots":{"status":"ok","http":200,"url":"https://www.gamedeveloper.com/robots.txt","bytes":1559,"groups":2},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["partial","*",6,0],"OAI-SearchBot":["partial","*",6,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",6,0],"Claude-SearchBot":["partial","*",6,0],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["partial","*",6,0],"Perplexity-User":["partial","*",6,0],"Google-Extended":["partial","*",6,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",6,0]},"llms":{"status":"missing","http":404,"bytes":332194},"home":{"status":"ok","http":200,"url":"https://www.gamedeveloper.com/","html_tokens":128264,"markdown_tokens":92,"pruned_tokens":92}},
{"rank":3021,"domain":"philly.com","origin":"https://philly.com","robots":{"status":"ok","http":200,"url":"https://www.inquirer.com/robots.txt","bytes":547,"groups":1},"bots":{"GPTBot":["partial","*",10,0],"ChatGPT-User":["partial","*",10,0],"OAI-SearchBot":["partial","*",10,0],"ClaudeBot":["partial","*",10,0],"Claude-User":["partial","*",10,0],"Claude-SearchBot":["partial","*",10,0],"anthropic-ai":["partial","*",10,0],"PerplexityBot":["partial","*",10,0],"Perplexity-User":["partial","*",10,0],"Google-Extended":["partial","*",10,0],"Applebot-Extended":["partial","*",10,0],"CCBot":["partial","*",10,0],"Bytespider":["partial","*",10,0],"Amazonbot":["partial","*",10,0],"meta-externalagent":["partial","*",10,0],"cohere-ai":["partial","*",10,0],"Diffbot":["partial","*",10,0],"ContextFetchBot":["partial","*",10,0]},"llms":{"status":"missing","http":404,"bytes":324910},"home":{"status":"ok","http":200,"url":"https://www.inquirer.com/","html_tokens":790013,"markdown_tokens":3,"pruned_tokens":3}},
{"rank":3022,"domain":"carnegieendowment.org","origin":"https://carnegieendowment.org","robots":{"status":"ok","http":200,"url":"https://carnegieendowment.org/robots.txt","bytes":93,"groups":1},"bots":{"GPTBot":["partial","*",1,1],"ChatGPT-User":["partial","*",1,1],"OAI-SearchBot":["partial","*",1,1],"ClaudeBot":["partial","*",1,1],"Claude-User":["partial","*",1,1],"Claude-SearchBot":["partial","*",1,1],"anthropic-ai":["partial","*",1,1],"PerplexityBot":["partial","*",1,1],"Perplexity-User":["partial","*",1,1],"Google-Extended":["partial","*",1,1],"Applebot-Extended":["partial","*",1,1],"CCBot":["partial","*",1,1],"Bytespider":["partial","*",1,1],"Amazonbot":["partial","*",1,1],"meta-externalagent":["partial","*",1,1],"cohere-ai":["partial","*",1,1],"Diffbot":["partial","*",1,1],"ContextFetchBot":["partial","*",1,1]},"llms":{"status":"missing","http":404,"bytes":12096},"home":{"status":"ok","http":200,"url":"https://carnegieendowment.org/","html_tokens":86285,"markdown_tokens":586,"pruned_tokens":142}},
{"rank":3023,"domain":"google.gr","origin":"https://google.gr","robots":{"status":"ok","http":200,"url":"https://www.google.gr/robots.txt","bytes":6624,"groups":4},"bots":{"GPTBot":["partial","*",169,64],"ChatGPT-User":["partial","*",169,64],"OAI-SearchBot":["partial","*",169,64],"ClaudeBot":["partial","*",169,64],"Claude-User":["partial","*",169,64],"Claude-SearchBot":["partial","*",169,64],"anthropic-ai":["partial","*",169,64],"PerplexityBot":["partial","*",169,64],"Perplexity-User":["partial","*",169,64],"Google-Extended":["partial","*",169,64],"Applebot-Extended":["partial","*",169,64],"CCBot":["partial","*",169,64],"Bytespider":["partial","*",169,64],"Amazonbot":["partial","*",169,64],"meta-externalagent":["partial","*",169,64],"cohere-ai":["partial","*",169,64],"Diffbot":["partial","*",169,64],"ContextFetchBot":["partial","*",169,64]},"llms":{"status":"missing","http":404,"bytes":1569},"home":{"status":"ok","http":200,"url":"https://www.google.com/","html_tokens":21350,"markdown_tokens":30,"pruned_tokens":30}},
{"rank":3024,"domain":"sportingnews.com","origin":"https://sportingnews.com","robots":{"status":"ok","http":200,"url":"https://www.sportingnews.com/robots.txt","bytes":4482,"groups":50},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["allowed","named",0,1],"OAI-SearchBot":["allowed","named",0,1],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",15,0],"Claude-SearchBot":["partial","*",15,0],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["allowed","named",0,1],"Perplexity-User":["partial","*",15,0],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",2,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",15,0]},"llms":{"status":"missing","http":404,"bytes":795950},"home":{"status":"ok","http":200,"url":"https://www.sportingnews.com/us","html_tokens":247906,"markdown_tokens":41,"pruned_tokens":41}},
{"rank":3025,"domain":"roku.com","origin":"https://roku.com","robots":{"status":"ok","http":200,"url":"https://www.roku.com/robots.txt","bytes":303,"groups":1},"bots":{"GPTBot":["partial","*",1,1],"ChatGPT-User":["partial","*",1,1],"OAI-SearchBot":["partial","*",1,1],"ClaudeBot":["partial","*",1,1],"Claude-User":["partial","*",1,1],"Claude-SearchBot":["partial","*",1,1],"anthropic-ai":["partial","*",1,1],"PerplexityBot":["partial","*",1,1],"Perplexity-User":["partial","*",1,1],"Google-Extended":["partial","*",1,1],"Applebot-Extended":["partial","*",1,1],"CCBot":["partial","*",1,1],"Bytespider":["partial","*",1,1],"Amazonbot":["partial","*",1,1],"meta-externalagent":["partial","*",1,1],"cohere-ai":["partial","*",1,1],"Diffbot":["partial","*",1,1],"ContextFetchBot":["partial","*",1,1]},"llms":{"status":"present","http":200,"bytes":15683},"home":{"status":"ok","http":200,"url":"https://www.roku.com/","html_tokens":72427,"markdown_tokens":19,"pruned_tokens":19}},
{"rank":3026,"domain":"synology.com","origin":"https://synology.com","robots":{"status":"ok","http":200,"url":"https://www.synology.com/robots.txt","bytes":127,"groups":1},"bots":{"GPTBot":["partial","*",3,0],"ChatGPT-User":["partial","*",3,0],"OAI-SearchBot":["partial","*",3,0],"ClaudeBot":["partial","*",3,0],"Claude-User":["partial","*",3,0],"Claude-SearchBot":["partial","*",3,0],"anthropic-ai":["partial","*",3,0],"PerplexityBot":["partial","*",3,0],"Perplexity-User":["partial","*",3,0],"Google-Extended":["partial","*",3,0],"Applebot-Extended":["partial","*",3,0],"CCBot":["partial","*",3,0],"Bytespider":["partial","*",3,0],"Amazonbot":["partial","*",3,0],"meta-externalagent":["partial","*",3,0],"cohere-ai":["partial","*",3,0],"Diffbot":["partial","*",3,0],"ContextFetchBot":["partial","*",3,0]},"llms":{"status":"html","http":200,"bytes":166532},"home":{"status":"ok","http":200,"url":"https://www.synology.com/en-us","html_tokens":41606,"markdown_tokens":812,"pruned_tokens":326}},
{"rank":3027,"domain":"joins.com","origin":"https://joins.com","robots":{"status":"ok","http":200,"url":"https://joins.com/robots.txt","bytes":55,"groups":1},"bots":{"GPTBot":["partial","*",1,2],"ChatGPT-User":["partial","*",1,2],"OAI-SearchBot":["partial","*",1,2],"ClaudeBot":["partial","*",1,2],"Claude-User":["partial","*",1,2],"Claude-SearchBot":["partial","*",1,2],"anthropic-ai":["partial","*",1,2],"PerplexityBot":["partial","*",1,2],"Perplexity-User":["partial","*",1,2],"Google-Extended":["partial","*",1,2],"Applebot-Extended":["partial","*",1,2],"CCBot":["partial","*",1,2],"Bytespider":["partial","*",1,2],"Amazonbot":["partial","*",1,2],"meta-externalagent":["partial","*",1,2],"cohere-ai":["partial","*",1,2],"Diffbot":["partial","*",1,2],"ContextFetchBot":["partial","*",1,2]},"llms":{"status":"disallowed"},"home":{"status":"ok","http":200,"url":"https://joins.com/","html_tokens":236,"markdown_tokens":26,"pruned_tokens":19}},
{"rank":3028,"domain":"ustr.gov","origin":"https://ustr.gov","robots":{"status":"ok","http":200,"url":"https://ustr.gov/robots.txt","bytes":2027,"groups":1},"bots":{"GPTBot":["partial","*",34,18],"ChatGPT-User":["partial","*",34,18],"OAI-SearchBot":["partial","*",34,18],"ClaudeBot":["partial","*",34,18],"Claude-User":["partial","*",34,18],"Claude-SearchBot":["partial","*",34,18],"anthropic-ai":["partial","*",34,18],"PerplexityBot":["partial","*",34,18],"Perplexity-User":["partial","*",34,18],"Google-Extended":["partial","*",34,18],"Applebot-Extended":["partial","*",34,18],"CCBot":["partial","*",34,18],"Bytespider":["partial","*",34,18],"Amazonbot":["partial","*",34,18],"meta-externalagent":["partial","*",34,18],"cohere-ai":["partial","*",34,18],"Diffbot":["partial","*",34,18],"ContextFetchBot":["partial","*",34,18]},"llms":{"status":"missing","http":404,"bytes":182},"home":{"status":"ok","http":200,"url":"https://ustr.gov/","html_tokens":13357,"markdown_tokens":384,"pruned_tokens":112}},
{"rank":3029,"domain":"tinypng.com","origin":"https://tinypng.com","robots":{"status":"ok","http":200,"url":"https://tinypng.com/robots.txt","bytes":244,"groups":1},"bots":{"GPTBot":["partial","*",8,0],"ChatGPT-User":["partial","*",8,0],"OAI-SearchBot":["partial","*",8,0],"ClaudeBot":["partial","*",8,0],"Claude-User":["partial","*",8,0],"Claude-SearchBot":["partial","*",8,0],"anthropic-ai":["partial","*",8,0],"PerplexityBot":["partial","*",8,0],"Perplexity-User":["partial","*",8,0],"Google-Extended":["partial","*",8,0],"Applebot-Extended":["partial","*",8,0],"CCBot":["partial","*",8,0],"Bytespider":["partial","*",8,0],"Amazonbot":["partial","*",8,0],"meta-externalagent":["partial","*",8,0],"cohere-ai":["partial","*",8,0],"Diffbot":["partial","*",8,0],"ContextFetchBot":["partial","*",8,0]},"llms":{"status":"missing","http":404,"bytes":3843},"home":{"status":"ok","http":200,"url":"https://tinypng.com/","html_tokens":57479,"markdown_tokens":2955,"pruned_tokens":492}},
{"rank":3030,"domain":"edmunds.com","origin":"https://edmunds.com","robots":{"status":"ok","http":200,"url":"https://www.edmunds.com/robots.txt","bytes":6186,"groups":17},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["partial","*",88,1],"OAI-SearchBot":["partial","*",88,1],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",88,1],"Claude-SearchBot":["partial","*",88,1],"anthropic-ai":["partial","*",88,1],"PerplexityBot":["partial","*",88,1],"Perplexity-User":["partial","*",88,1],"Google-Extended":["partial","*",88,1],"Applebot-Extended":["partial","*",88,1],"CCBot":["partial","*",88,1],"Bytespider":["blocked","named",1,0],"Amazonbot":["partial","*",88,1],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["partial","*",88,1],"Diffbot":["partial","*",88,1],"ContextFetchBot":["partial","*",88,1]},"llms":{"status":"missing","http":404,"bytes":74273},"home":{"status":"http_error","http":403,"url":"https://www.edmunds.com/"}},
{"rank":3031,"domain":"cochranelibrary.com","origin":"https://cochranelibrary.com","robots":{"status":"refused","http":403,"url":"https://cochranelibrary.com/robots.txt","bytes":5536,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":3032,"domain":"jalopnik.com","origin":"https://jalopnik.com","robots":{"status":"ok","http":200,"url":"https://jalopnik.com/robots.txt","bytes":255,"groups":2},"bots":{"GPTBot":["partial","*",4,0],"ChatGPT-User":["partial","*",4,0],"OAI-SearchBot":["partial","*",4,0],"ClaudeBot":["partial","*",4,0],"Claude-User":["partial","*",4,0],"Claude-SearchBot":["partial","*",4,0],"anthropic-ai":["partial","*",4,0],"PerplexityBot":["partial","*",4,0],"Perplexity-User":["partial","*",4,0],"Google-Extended":["partial","*",4,0],"Applebot-Extended":["partial","*",4,0],"CCBot":["partial","*",4,0],"Bytespider":["partial","*",4,0],"Amazonbot":["partial","*",4,0],"meta-externalagent":["partial","*",4,0],"cohere-ai":["partial","*",4,0],"Diffbot":["partial","*",4,0],"ContextFetchBot":["partial","*",4,0]},"llms":{"status":"html","http":200,"bytes":103352},"home":{"status":"ok","http":200,"url":"https://www.jalopnik.com/","html_tokens":25835,"markdown_tokens":1190,"pruned_tokens":122}},
{"rank":3033,"domain":"marinetraffic.com","origin":"https://marinetraffic.com","robots":{"status":"refused","http":403,"url":"https://www.marinetraffic.com/robots.txt","bytes":4911,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":3034,"domain":"qmul.ac.uk","origin":"https://qmul.ac.uk","robots":{"status":"ok","http":200,"url":"https://www.qmul.ac.uk/robots.txt","bytes":485,"groups":2},"bots":{"GPTBot":["partial","*",9,0],"ChatGPT-User":["partial","*",9,0],"OAI-SearchBot":["partial","*",9,0],"ClaudeBot":["partial","*",9,0],"Claude-User":["partial","*",9,0],"Claude-SearchBot":["partial","*",9,0],"anthropic-ai":["partial","*",9,0],"PerplexityBot":["partial","*",9,0],"Perplexity-User":["partial","*",9,0],"Google-Extended":["partial","*",9,0],"Applebot-Extended":["partial","*",9,0],"CCBot":["partial","*",9,0],"Bytespider":["partial","*",9,0],"Amazonbot":["partial","*",9,0],"meta-externalagent":["partial","*",9,0],"cohere-ai":["partial","*",9,0],"Diffbot":["partial","*",9,0],"ContextFetchBot":["partial","*",9,0]},"llms":{"status":"missing","http":404,"bytes":37654},"home":{"status":"ok","http":200,"url":"https://www.qmul.ac.uk/","html_tokens":18941,"markdown_tokens":152,"pruned_tokens":152}},
{"rank":3035,"domain":"parents.com","origin":"https://parents.com","robots":{"status":"ok","http":200,"url":"https://www.parents.com/robots.txt","bytes":3467,"groups":6},"bots":{"GPTBot":["partial","named",1,0],"ChatGPT-User":["partial","named",1,0],"OAI-SearchBot":["partial","named",1,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["blocked","named",1,0],"Claude-SearchBot":["partial","named",1,2],"anthropic-ai":["partial","named",1,2],"PerplexityBot":["partial","named",1,2],"Perplexity-User":["blocked","named",1,0],"Google-Extended":["partial","named",1,2],"Applebot-Extended":["blocked","named",1,0],"CCBot":["partial","named",1,2],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["partial","named",1,2],"cohere-ai":["partial","named",1,2],"Diffbot":["partial","*",2,0],"ContextFetchBot":["partial","*",2,0]},"llms":{"status":"http_error","http":402,"bytes":612},"home":{"status":"http_error","http":402,"url":"https://www.parents.com/"}},
{"rank":3036,"domain":"mix.com","origin":"https://mix.com","robots":{"status":"ok","http":200,"url":"https://mix.com/robots.txt","bytes":1347,"groups":3},"bots":{"GPTBot":["partial","named",1,2],"ChatGPT-User":["partial","named",1,2],"OAI-SearchBot":["partial","named",1,2],"ClaudeBot":["partial","named",1,2],"Claude-User":["partial","named",1,2],"Claude-SearchBot":["partial","named",1,2],"anthropic-ai":["partial","*",29,0],"PerplexityBot":["partial","named",1,2],"Perplexity-User":["partial","named",1,2],"Google-Extended":["partial","named",1,2],"Applebot-Extended":["partial","named",1,2],"CCBot":["partial","*",29,0],"Bytespider":["partial","*",29,0],"Amazonbot":["partial","*",29,0],"meta-externalagent":["partial","*",29,0],"cohere-ai":["partial","*",29,0],"Diffbot":["partial","*",29,0],"ContextFetchBot":["partial","*",29,0]},"llms":{"status":"missing","http":404,"bytes":29303},"home":{"status":"ok","http":200,"url":"https://mix.com/","html_tokens":9745,"markdown_tokens":49,"pruned_tokens":17}},
{"rank":3037,"domain":"ou.edu","origin":"https://ou.edu","robots":{"status":"missing","http":404,"url":"https://ou.edu/robots.txt","bytes":69725,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":1044},"home":{"status":"ok","http":200,"url":"https://www.ou.edu/","html_tokens":29861,"markdown_tokens":471,"pruned_tokens":138}},
{"rank":3038,"domain":"sourceware.org","origin":"https://sourceware.org","robots":{"status":"ok","http":200,"url":"https://sourceware.org/robots.txt","bytes":963,"groups":6},"bots":{"GPTBot":["partial","*",20,0],"ChatGPT-User":["partial","*",20,0],"OAI-SearchBot":["partial","*",20,0],"ClaudeBot":["partial","*",20,0],"Claude-User":["partial","*",20,0],"Claude-SearchBot":["partial","*",20,0],"anthropic-ai":["partial","*",20,0],"PerplexityBot":["partial","*",20,0],"Perplexity-User":["partial","*",20,0],"Google-Extended":["partial","*",20,0],"Applebot-Extended":["partial","*",20,0],"CCBot":["partial","*",20,0],"Bytespider":["partial","*",20,0],"Amazonbot":["partial","*",20,0],"meta-externalagent":["partial","*",20,0],"cohere-ai":["partial","*",20,0],"Diffbot":["partial","*",20,0],"ContextFetchBot":["partial","*",20,0]},"llms":{"status":"missing","http":404,"bytes":196},"home":{"status":"ok","http":200,"url":"https://sourceware.org/","html_tokens":2960,"markdown_tokens":1130,"pruned_tokens":463}},
{"rank":3039,"domain":"mind.org.uk","origin":"https://mind.org.uk","robots":{"status":"ok","http":200,"url":"https://www.mind.org.uk/robots.txt","bytes":259,"groups":1},"bots":{"GPTBot":["partial","*",7,1],"ChatGPT-User":["partial","*",7,1],"OAI-SearchBot":["partial","*",7,1],"ClaudeBot":["partial","*",7,1],"Claude-User":["partial","*",7,1],"Claude-SearchBot":["partial","*",7,1],"anthropic-ai":["partial","*",7,1],"PerplexityBot":["partial","*",7,1],"Perplexity-User":["partial","*",7,1],"Google-Extended":["partial","*",7,1],"Applebot-Extended":["partial","*",7,1],"CCBot":["partial","*",7,1],"Bytespider":["partial","*",7,1],"Amazonbot":["partial","*",7,1],"meta-externalagent":["partial","*",7,1],"cohere-ai":["partial","*",7,1],"Diffbot":["partial","*",7,1],"ContextFetchBot":["partial","*",7,1]},"llms":{"status":"missing","http":404,"bytes":0},"home":{"status":"http_error","http":403,"url":"https://mind.org.uk/"}},
{"rank":3040,"domain":"choicehotels.com","origin":"https://choicehotels.com","robots":{"status":"unreachable","http":null,"url":"https://choicehotels.com/robots.txt","bytes":0,"groups":0,"error":"<StreamReset stream_id:1, error_code:2, remote_reset:True>"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":3041,"domain":"tenor.com","origin":"https://tenor.com","robots":{"status":"ok","http":200,"url":"https://tenor.com/robots.txt","bytes":25,"groups":1},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"missing","http":404,"bytes":16025},"home":{"status":"ok","http":200,"url":"https://tenor.com/","html_tokens":68668,"markdown_tokens":563,"pruned_tokens":174}},
{"rank":3042,"domain":"homeadvisor.com","origin":"https://homeadvisor.com","robots":{"status":"ok","http":200,"url":"https://www.homeadvisor.com/robots.txt","bytes":752,"groups":5},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["partial","*",23,0],"OAI-SearchBot":["partial","*",23,0],"ClaudeBot":["partial","*",23,0],"Claude-User":["partial","*",23,0],"Claude-SearchBot":["partial","*",23,0],"anthropic-ai":["partial","*",23,0],"PerplexityBot":["partial","*",23,0],"Perplexity-User":["partial","*",23,0],"Google-Extended":["partial","*",23,0],"Applebot-Extended":["partial","*",23,0],"CCBot":["partial","*",23,0],"Bytespider":["partial","*",23,0],"Amazonbot":["partial","*",23,0],"meta-externalagent":["partial","*",23,0],"cohere-ai":["partial","*",23,0],"Diffbot":["partial","*",23,0],"ContextFetchBot":["partial","*",23,0]},"llms":{"status":"missing","http":404,"bytes":153556},"home":{"status":"http_error","http":403,"url":"https://www.homeadvisor.com/"}},
{"rank":3043,"domain":"csoonline.com","origin":"https://csoonline.com","robots":{"status":"ok","http":200,"url":"https://www.csoonline.com/robots.txt","bytes":3627,"groups":37},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["partial","*",24,0],"OAI-SearchBot":["partial","*",24,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",24,0],"Claude-SearchBot":["partial","*",24,0],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["partial","*",24,0],"Perplexity-User":["partial","*",24,0],"Google-Extended":["partial","*",24,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["partial","*",24,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",24,0]},"llms":{"status":"missing","http":404,"bytes":177077},"home":{"status":"ok","http":200,"url":"https://www.csoonline.com","html_tokens":67695,"markdown_tokens":83,"pruned_tokens":83}},
{"rank":3044,"domain":"honda.com","origin":"https://honda.com","robots":{"status":"refused","http":403,"url":"https://www.honda.com/robots.txt","bytes":381,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":3045,"domain":"follow.it","origin":"https://follow.it","robots":{"status":"ok","http":200,"url":"https://follow.it/robots.txt","bytes":294,"groups":1},"bots":{"GPTBot":["partial","*",5,0],"ChatGPT-User":["partial","*",5,0],"OAI-SearchBot":["partial","*",5,0],"ClaudeBot":["partial","*",5,0],"Claude-User":["partial","*",5,0],"Claude-SearchBot":["partial","*",5,0],"anthropic-ai":["partial","*",5,0],"PerplexityBot":["partial","*",5,0],"Perplexity-User":["partial","*",5,0],"Google-Extended":["partial","*",5,0],"Applebot-Extended":["partial","*",5,0],"CCBot":["partial","*",5,0],"Bytespider":["partial","*",5,0],"Amazonbot":["partial","*",5,0],"meta-externalagent":["partial","*",5,0],"cohere-ai":["partial","*",5,0],"Diffbot":["partial","*",5,0],"ContextFetchBot":["partial","*",5,0]},"llms":{"status":"html","http":200,"bytes":334615},"home":{"status":"ok","http":200,"url":"https://follow.it/","html_tokens":46298,"markdown_tokens":337,"pruned_tokens":134}},
{"rank":3046,"domain":"fed.us","origin":"https://fed.us","robots":{"status":"unreachable","http":null,"url":"https://fed.us/robots.txt","bytes":0,"groups":0,"error":"cannot resolve fed.us"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":3047,"domain":"aeaweb.org","origin":"https://aeaweb.org","robots":{"status":"ok","http":200,"url":"https://www.aeaweb.org/robots.txt","bytes":65,"groups":2},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":12259},"home":{"status":"ok","http":200,"url":"https://www.aeaweb.org/","html_tokens":10296,"markdown_tokens":127,"pruned_tokens":127}},
{"rank":3048,"domain":"iec.ch","origin":"https://iec.ch","robots":{"status":"refused","http":403,"url":"https://iec.ch/robots.txt","bytes":919,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":3049,"domain":"workable.com","origin":"https://workable.com","robots":{"status":"ok","http":200,"url":"https://workable.com/robots.txt","bytes":698,"groups":1},"bots":{"GPTBot":["partial","*",4,0],"ChatGPT-User":["partial","*",4,0],"OAI-SearchBot":["partial","*",4,0],"ClaudeBot":["partial","*",4,0],"Claude-User":["partial","*",4,0],"Claude-SearchBot":["partial","*",4,0],"anthropic-ai":["partial","*",4,0],"PerplexityBot":["partial","*",4,0],"Perplexity-User":["partial","*",4,0],"Google-Extended":["partial","*",4,0],"Applebot-Extended":["partial","*",4,0],"CCBot":["partial","*",4,0],"Bytespider":["partial","*",4,0],"Amazonbot":["partial","*",4,0],"meta-externalagent":["partial","*",4,0],"cohere-ai":["partial","*",4,0],"Diffbot":["partial","*",4,0],"ContextFetchBot":["partial","*",4,0]},"llms":{"status":"other_text","http":200,"bytes":6681},"home":{"status":"ok","http":200,"url":"https://www.workable.com/","html_tokens":43366,"markdown_tokens":1786,"pruned_tokens":528}},
{"rank":3050,"domain":"europa.eu.int","origin":"https://europa.eu.int","robots":{"status":"unreachable","http":null,"url":"https://europa.eu.int/robots.txt","bytes":0,"groups":0,"error":"cannot resolve europa.eu.int"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":3051,"domain":"iucnredlist.org","origin":"https://iucnredlist.org","robots":{"status":"ok","http":200,"url":"https://www.iucnredlist.org/robots.txt","bytes":185,"groups":1},"bots":{"GPTBot":["partial","*",6,0],"ChatGPT-User":["partial","*",6,0],"OAI-SearchBot":["partial","*",6,0],"ClaudeBot":["partial","*",6,0],"Claude-User":["partial","*",6,0],"Claude-SearchBot":["partial","*",6,0],"anthropic-ai":["partial","*",6,0],"PerplexityBot":["partial","*",6,0],"Perplexity-User":["partial","*",6,0],"Google-Extended":["partial","*",6,0],"Applebot-Extended":["partial","*",6,0],"CCBot":["partial","*",6,0],"Bytespider":["partial","*",6,0],"Amazonbot":["partial","*",6,0],"meta-externalagent":["partial","*",6,0],"cohere-ai":["partial","*",6,0],"Diffbot":["partial","*",6,0],"ContextFetchBot":["partial","*",6,0]},"llms":{"status":"missing","http":404,"bytes":26220},"home":{"status":"http_error","http":403,"url":"https://iucnredlist.org/"}},
{"rank":3052,"domain":"eventbrite.ca","origin":"https://eventbrite.ca","robots":{"status":"ok","http":200,"url":"https://eventbrite.ca/robots.txt","bytes":7499,"groups":135},"bots":{"GPTBot":["partial","*",51,2],"ChatGPT-User":["partial","*",51,2],"OAI-SearchBot":["partial","*",51,2],"ClaudeBot":["partial","*",51,2],"Claude-User":["partial","*",51,2],"Claude-SearchBot":["partial","*",51,2],"anthropic-ai":["partial","*",51,2],"PerplexityBot":["partial","*",51,2],"Perplexity-User":["partial","*",51,2],"Google-Extended":["partial","*",51,2],"Applebot-Extended":["partial","*",51,2],"CCBot":["blocked","named",1,0],"Bytespider":["partial","*",51,2],"Amazonbot":["partial","*",51,2],"meta-externalagent":["partial","*",51,2],"cohere-ai":["partial","*",51,2],"Diffbot":["partial","*",51,2],"ContextFetchBot":["partial","*",51,2]},"llms":{"status":"missing","http":404,"bytes":79167},"home":{"status":"ok","http":200,"url":"https://www.eventbrite.ca","html_tokens":50455,"markdown_tokens":60,"pruned_tokens":60}},
{"rank":3053,"domain":"informa.com","origin":"https://informa.com","robots":{"status":"ok","http":200,"url":"https://www.informa.com/robots.txt","bytes":136,"groups":1},"bots":{"GPTBot":["partial","*",3,1],"ChatGPT-User":["partial","*",3,1],"OAI-SearchBot":["partial","*",3,1],"ClaudeBot":["partial","*",3,1],"Claude-User":["partial","*",3,1],"Claude-SearchBot":["partial","*",3,1],"anthropic-ai":["partial","*",3,1],"PerplexityBot":["partial","*",3,1],"Perplexity-User":["partial","*",3,1],"Google-Extended":["partial","*",3,1],"Applebot-Extended":["partial","*",3,1],"CCBot":["partial","*",3,1],"Bytespider":["partial","*",3,1],"Amazonbot":["partial","*",3,1],"meta-externalagent":["partial","*",3,1],"cohere-ai":["partial","*",3,1],"Diffbot":["partial","*",3,1],"ContextFetchBot":["partial","*",3,1]},"llms":{"status":"missing","http":404,"bytes":0},"home":{"status":"ok","http":200,"url":"https://www.informa.com/","html_tokens":14400,"markdown_tokens":976,"pruned_tokens":436}},
{"rank":3054,"domain":"apartmenttherapy.com","origin":"https://apartmenttherapy.com","robots":{"status":"ok","http":200,"url":"https://www.apartmenttherapy.com/robots.txt","bytes":187,"groups":2},"bots":{"GPTBot":["partial","*",1,0],"ChatGPT-User":["partial","*",1,0],"OAI-SearchBot":["partial","*",1,0],"ClaudeBot":["partial","*",1,0],"Claude-User":["partial","*",1,0],"Claude-SearchBot":["partial","*",1,0],"anthropic-ai":["partial","*",1,0],"PerplexityBot":["partial","*",1,0],"Perplexity-User":["partial","*",1,0],"Google-Extended":["partial","*",1,0],"Applebot-Extended":["partial","*",1,0],"CCBot":["partial","*",1,0],"Bytespider":["partial","*",1,0],"Amazonbot":["partial","*",1,0],"meta-externalagent":["partial","*",1,0],"cohere-ai":["partial","*",1,0],"Diffbot":["partial","*",1,0],"ContextFetchBot":["partial","*",1,0]},"llms":{"status":"http_error","http":403,"bytes":3264},"home":{"status":"http_error","http":403,"url":"https://www.apartmenttherapy.com/"}},
{"rank":3055,"domain":"linode.com","origin":"https://linode.com","robots":{"status":"ok","http":200,"url":"https://www.linode.com/robots.txt","bytes":499,"groups":2},"bots":{"GPTBot":["partial","*",7,0],"ChatGPT-User":["partial","*",7,0],"OAI-SearchBot":["partial","*",7,0],"ClaudeBot":["partial","*",7,0],"Claude-User":["partial","*",7,0],"Claude-SearchBot":["partial","*",7,0],"anthropic-ai":["partial","*",7,0],"PerplexityBot":["partial","*",7,0],"Perplexity-User":["partial","*",7,0],"Google-Extended":["partial","*",7,0],"Applebot-Extended":["partial","*",7,0],"CCBot":["partial","*",7,0],"Bytespider":["partial","*",7,0],"Amazonbot":["partial","*",7,0],"meta-externalagent":["partial","*",7,0],"cohere-ai":["partial","*",7,0],"Diffbot":["partial","*",7,0],"ContextFetchBot":["partial","*",7,0]},"llms":{"status":"timeout","error":"timed out after 10 s"},"home":{"status":"ok","http":200,"url":"https://www.linode.com/","html_tokens":40426,"markdown_tokens":165,"pruned_tokens":165}},
{"rank":3056,"domain":"n8n.io","origin":"https://n8n.io","robots":{"status":"ok","http":200,"url":"https://n8n.io/robots.txt","bytes":76,"groups":1},"bots":{"GPTBot":["partial","*",1,0],"ChatGPT-User":["partial","*",1,0],"OAI-SearchBot":["partial","*",1,0],"ClaudeBot":["partial","*",1,0],"Claude-User":["partial","*",1,0],"Claude-SearchBot":["partial","*",1,0],"anthropic-ai":["partial","*",1,0],"PerplexityBot":["partial","*",1,0],"Perplexity-User":["partial","*",1,0],"Google-Extended":["partial","*",1,0],"Applebot-Extended":["partial","*",1,0],"CCBot":["partial","*",1,0],"Bytespider":["partial","*",1,0],"Amazonbot":["partial","*",1,0],"meta-externalagent":["partial","*",1,0],"cohere-ai":["partial","*",1,0],"Diffbot":["partial","*",1,0],"ContextFetchBot":["partial","*",1,0]},"llms":{"status":"present","http":200,"bytes":1160},"home":{"status":"ok","http":200,"url":"https://n8n.io/","html_tokens":87959,"markdown_tokens":161,"pruned_tokens":45}},
{"rank":3057,"domain":"sothebys.com","origin":"https://sothebys.com","robots":{"status":"ok","http":200,"url":"https://www.sothebys.com/robots.txt","bytes":2081,"groups":6},"bots":{"GPTBot":["partial","*",11,0],"ChatGPT-User":["partial","*",11,0],"OAI-SearchBot":["partial","*",11,0],"ClaudeBot":["partial","*",11,0],"Claude-User":["partial","*",11,0],"Claude-SearchBot":["partial","*",11,0],"anthropic-ai":["partial","*",11,0],"PerplexityBot":["partial","*",11,0],"Perplexity-User":["partial","*",11,0],"Google-Extended":["partial","*",11,0],"Applebot-Extended":["partial","*",11,0],"CCBot":["partial","*",11,0],"Bytespider":["partial","*",11,0],"Amazonbot":["partial","*",11,0],"meta-externalagent":["partial","*",11,0],"cohere-ai":["partial","*",11,0],"Diffbot":["partial","*",11,0],"ContextFetchBot":["partial","*",11,0]},"llms":{"status":"missing","http":404,"bytes":104507},"home":{"status":"timeout","error":"timed out after 15 s"}},
{"rank":3058,"domain":"blogspot.jp","origin":"https://blogspot.jp","robots":{"status":"ok","http":200,"url":"https://blogspot.jp/robots.txt","bytes":6624,"groups":4},"bots":{"GPTBot":["partial","*",169,64],"ChatGPT-User":["partial","*",169,64],"OAI-SearchBot":["partial","*",169,64],"ClaudeBot":["partial","*",169,64],"Claude-User":["partial","*",169,64],"Claude-SearchBot":["partial","*",169,64],"anthropic-ai":["partial","*",169,64],"PerplexityBot":["partial","*",169,64],"Perplexity-User":["partial","*",169,64],"Google-Extended":["partial","*",169,64],"Applebot-Extended":["partial","*",169,64],"CCBot":["partial","*",169,64],"Bytespider":["partial","*",169,64],"Amazonbot":["partial","*",169,64],"meta-externalagent":["partial","*",169,64],"cohere-ai":["partial","*",169,64],"Diffbot":["partial","*",169,64],"ContextFetchBot":["partial","*",169,64]},"llms":{"status":"missing","http":404,"bytes":1569},"home":{"status":"ok","http":200,"url":"https://www.google.com/","html_tokens":21295,"markdown_tokens":30,"pruned_tokens":30}},
{"rank":3059,"domain":"mt.gov","origin":"https://mt.gov","robots":{"status":"missing","http":404,"url":"https://mt.gov/robots.txt","bytes":43727,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":43727},"home":{"status":"ok","http":200,"url":"https://mt.gov/","html_tokens":21498,"markdown_tokens":299,"pruned_tokens":98}},
{"rank":3060,"domain":"keio.ac.jp","origin":"https://www.keio.ac.jp","robots":{"status":"missing","http":404,"url":"https://www.keio.ac.jp/robots.txt","bytes":126524,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":126516},"home":{"status":"ok","http":200,"url":"https://www.keio.ac.jp/ja/","html_tokens":88579,"markdown_tokens":23,"pruned_tokens":23}},
{"rank":3061,"domain":"china.org.cn","origin":"http://china.org.cn","robots":{"status":"ok","http":200,"url":"http://www.china.org.cn/robots.txt","bytes":23,"groups":1},"bots":{"GPTBot":["allowed","*",0,1],"ChatGPT-User":["allowed","*",0,1],"OAI-SearchBot":["allowed","*",0,1],"ClaudeBot":["allowed","*",0,1],"Claude-User":["allowed","*",0,1],"Claude-SearchBot":["allowed","*",0,1],"anthropic-ai":["allowed","*",0,1],"PerplexityBot":["allowed","*",0,1],"Perplexity-User":["allowed","*",0,1],"Google-Extended":["allowed","*",0,1],"Applebot-Extended":["allowed","*",0,1],"CCBot":["allowed","*",0,1],"Bytespider":["allowed","*",0,1],"Amazonbot":["allowed","*",0,1],"meta-externalagent":["allowed","*",0,1],"cohere-ai":["allowed","*",0,1],"Diffbot":["allowed","*",0,1],"ContextFetchBot":["allowed","*",0,1]},"llms":{"status":"missing","http":404,"bytes":206},"home":{"status":"ok","http":200,"url":"http://www.china.org.cn/","html_tokens":8669,"markdown_tokens":35,"pruned_tokens":35}},
{"rank":3062,"domain":"crunchyroll.com","origin":"https://crunchyroll.com","robots":{"status":"ok","http":200,"url":"https://www.crunchyroll.com/robots.txt","bytes":829,"groups":1},"bots":{"GPTBot":["partial","*",36,0],"ChatGPT-User":["partial","*",36,0],"OAI-SearchBot":["partial","*",36,0],"ClaudeBot":["partial","*",36,0],"Claude-User":["partial","*",36,0],"Claude-SearchBot":["partial","*",36,0],"anthropic-ai":["partial","*",36,0],"PerplexityBot":["partial","*",36,0],"Perplexity-User":["partial","*",36,0],"Google-Extended":["partial","*",36,0],"Applebot-Extended":["partial","*",36,0],"CCBot":["partial","*",36,0],"Bytespider":["partial","*",36,0],"Amazonbot":["partial","*",36,0],"meta-externalagent":["partial","*",36,0],"cohere-ai":["partial","*",36,0],"Diffbot":["partial","*",36,0],"ContextFetchBot":["partial","*",36,0]},"llms":{"status":"missing","http":404,"bytes":286209},"home":{"status":"http_error","http":403,"url":"https://www.crunchyroll.com/"}},
{"rank":3063,"domain":"lung.org","origin":"https://lung.org","robots":{"status":"ok","http":200,"url":"https://www.lung.org/robots.txt","bytes":160,"groups":2},"bots":{"GPTBot":["partial","*",2,1],"ChatGPT-User":["partial","*",2,1],"OAI-SearchBot":["partial","*",2,1],"ClaudeBot":["partial","*",2,1],"Claude-User":["partial","*",2,1],"Claude-SearchBot":["partial","*",2,1],"anthropic-ai":["partial","*",2,1],"PerplexityBot":["partial","*",2,1],"Perplexity-User":["partial","*",2,1],"Google-Extended":["partial","*",2,1],"Applebot-Extended":["partial","*",2,1],"CCBot":["partial","*",2,1],"Bytespider":["partial","*",2,1],"Amazonbot":["partial","*",2,1],"meta-externalagent":["partial","*",2,1],"cohere-ai":["partial","*",2,1],"Diffbot":["partial","*",2,1],"ContextFetchBot":["partial","*",2,1]},"llms":{"status":"missing","http":404,"bytes":44436},"home":{"status":"ok","http":200,"url":"https://www.lung.org/","html_tokens":33487,"markdown_tokens":151,"pruned_tokens":91}},
{"rank":3064,"domain":"forecast7.com","origin":"https://forecast7.com","robots":{"status":"ok","http":200,"url":"https://forecast7.com/robots.txt","bytes":1101,"groups":4},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"missing","http":404,"bytes":1487},"home":{"status":"ok","http":200,"url":"https://forecast7.com/","html_tokens":1307,"markdown_tokens":0,"pruned_tokens":0}},
{"rank":3065,"domain":"collider.com","origin":"https://collider.com","robots":{"status":"ok","http":200,"url":"https://collider.com/robots.txt","bytes":2029,"groups":2},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["partial","*",22,0],"OAI-SearchBot":["partial","*",22,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",22,0],"Claude-SearchBot":["partial","*",22,0],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["partial","*",22,0],"Perplexity-User":["partial","*",22,0],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",22,0]},"llms":{"status":"missing","http":404,"bytes":109844},"home":{"status":"ok","http":200,"url":"https://collider.com/","html_tokens":226522,"markdown_tokens":27,"pruned_tokens":27}},
{"rank":3066,"domain":"ghostery.com","origin":"https://ghostery.com","robots":{"status":"ok","http":200,"url":"https://www.ghostery.com/robots.txt","bytes":10961,"groups":1},"bots":{"GPTBot":["partial","*",229,0],"ChatGPT-User":["partial","*",229,0],"OAI-SearchBot":["partial","*",229,0],"ClaudeBot":["partial","*",229,0],"Claude-User":["partial","*",229,0],"Claude-SearchBot":["partial","*",229,0],"anthropic-ai":["partial","*",229,0],"PerplexityBot":["partial","*",229,0],"Perplexity-User":["partial","*",229,0],"Google-Extended":["partial","*",229,0],"Applebot-Extended":["partial","*",229,0],"CCBot":["partial","*",229,0],"Bytespider":["partial","*",229,0],"Amazonbot":["partial","*",229,0],"meta-externalagent":["partial","*",229,0],"cohere-ai":["partial","*",229,0],"Diffbot":["partial","*",229,0],"ContextFetchBot":["partial","*",229,0]},"llms":{"status":"missing","http":404,"bytes":15476},"home":{"status":"ok","http":200,"url":"https://www.ghostery.com/","html_tokens":59035,"markdown_tokens":1481,"pruned_tokens":603}},
{"rank":3067,"domain":"hypotheses.org","origin":"https://hypotheses.org","robots":{"status":"ok","http":200,"url":"https://hypotheses.org/robots.txt","bytes":36,"groups":1},"bots":{"GPTBot":["partial","*",1,0],"ChatGPT-User":["partial","*",1,0],"OAI-SearchBot":["partial","*",1,0],"ClaudeBot":["partial","*",1,0],"Claude-User":["partial","*",1,0],"Claude-SearchBot":["partial","*",1,0],"anthropic-ai":["partial","*",1,0],"PerplexityBot":["partial","*",1,0],"Perplexity-User":["partial","*",1,0],"Google-Extended":["partial","*",1,0],"Applebot-Extended":["partial","*",1,0],"CCBot":["partial","*",1,0],"Bytespider":["partial","*",1,0],"Amazonbot":["partial","*",1,0],"meta-externalagent":["partial","*",1,0],"cohere-ai":["partial","*",1,0],"Diffbot":["partial","*",1,0],"ContextFetchBot":["partial","*",1,0]},"llms":{"status":"missing","http":404,"bytes":89749},"home":{"status":"ok","http":200,"url":"https://hypotheses.org/","html_tokens":30317,"markdown_tokens":106,"pruned_tokens":86}},
{"rank":3068,"domain":"nine.com.au","origin":"https://nine.com.au","robots":{"status":"ok","http":200,"url":"https://www.nine.com.au/robots.txt","bytes":3985,"groups":51},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["blocked","named",1,0],"OAI-SearchBot":["blocked","named",1,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",4,1],"Claude-SearchBot":["partial","*",4,1],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["partial","*",4,1],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",4,1]},"llms":{"status":"missing","http":404,"bytes":107446},"home":{"status":"ok","http":200,"url":"https://www.nine.com.au/","html_tokens":349531,"markdown_tokens":115,"pruned_tokens":115}},
{"rank":3069,"domain":"hubspotusercontent-na1.net","origin":"https://hubspotusercontent-na1.net","robots":{"status":"unreachable","http":null,"url":"https://hubspotusercontent-na1.net/robots.txt","bytes":0,"groups":0,"error":"cannot resolve hubspotusercontent-na1.net"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":3070,"domain":"freshworks.com","origin":"https://freshworks.com","robots":{"status":"ok","http":200,"url":"https://www.freshworks.com/robots.txt","bytes":420,"groups":1},"bots":{"GPTBot":["partial","*",10,0],"ChatGPT-User":["partial","*",10,0],"OAI-SearchBot":["partial","*",10,0],"ClaudeBot":["partial","*",10,0],"Claude-User":["partial","*",10,0],"Claude-SearchBot":["partial","*",10,0],"anthropic-ai":["partial","*",10,0],"PerplexityBot":["partial","*",10,0],"Perplexity-User":["partial","*",10,0],"Google-Extended":["partial","*",10,0],"Applebot-Extended":["partial","*",10,0],"CCBot":["partial","*",10,0],"Bytespider":["partial","*",10,0],"Amazonbot":["partial","*",10,0],"meta-externalagent":["partial","*",10,0],"cohere-ai":["partial","*",10,0],"Diffbot":["partial","*",10,0],"ContextFetchBot":["partial","*",10,0]},"llms":{"status":"present","http":200,"bytes":10111},"home":{"status":"http_error","http":403,"url":"https://www.freshworks.com/"}},
{"rank":3071,"domain":"zju.edu.cn","origin":"https://www.zju.edu.cn","robots":{"status":"missing","http":404,"url":"https://www.zju.edu.cn/robots.txt","bytes":196,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":196},"home":{"status":"ok","http":200,"url":"https://www.zju.edu.cn/","html_tokens":22513,"markdown_tokens":0,"pruned_tokens":0}},
{"rank":3072,"domain":"att.net","origin":"https://att.net","robots":{"status":"ok","http":200,"url":"https://www.att.net/robots.txt","bytes":250,"groups":1},"bots":{"GPTBot":["partial","*",4,0],"ChatGPT-User":["partial","*",4,0],"OAI-SearchBot":["partial","*",4,0],"ClaudeBot":["partial","*",4,0],"Claude-User":["partial","*",4,0],"Claude-SearchBot":["partial","*",4,0],"anthropic-ai":["partial","*",4,0],"PerplexityBot":["partial","*",4,0],"Perplexity-User":["partial","*",4,0],"Google-Extended":["partial","*",4,0],"Applebot-Extended":["partial","*",4,0],"CCBot":["partial","*",4,0],"Bytespider":["partial","*",4,0],"Amazonbot":["partial","*",4,0],"meta-externalagent":["partial","*",4,0],"cohere-ai":["partial","*",4,0],"Diffbot":["partial","*",4,0],"ContextFetchBot":["partial","*",4,0]},"llms":{"status":"timeout","error":"timed out after 10 s"},"home":{"status":"disallowed","error":"robots.txt of start.att.net not readable (refused)"}},
{"rank":3073,"domain":"guidestar.org","origin":"https://guidestar.org","robots":{"status":"ok","http":200,"url":"https://www.guidestar.org/robots.txt","bytes":767,"groups":6},"bots":{"GPTBot":["partial","*",19,0],"ChatGPT-User":["partial","*",19,0],"OAI-SearchBot":["partial","*",19,0],"ClaudeBot":["partial","*",19,0],"Claude-User":["partial","*",19,0],"Claude-SearchBot":["partial","*",19,0],"anthropic-ai":["partial","*",19,0],"PerplexityBot":["partial","*",19,0],"Perplexity-User":["partial","*",19,0],"Google-Extended":["partial","*",19,0],"Applebot-Extended":["partial","*",19,0],"CCBot":["partial","*",19,0],"Bytespider":["partial","*",19,0],"Amazonbot":["partial","*",19,0],"meta-externalagent":["partial","*",19,0],"cohere-ai":["partial","*",19,0],"Diffbot":["partial","*",19,0],"ContextFetchBot":["partial","*",19,0]},"llms":{"status":"missing","http":404,"bytes":16147},"home":{"status":"ok","http":200,"url":"https://www.guidestar.org/","html_tokens":6619,"markdown_tokens":271,"pruned_tokens":170}},
{"rank":3074,"domain":"bloglovin.com","origin":"https://bloglovin.com","robots":{"status":"html","http":200,"url":"https://www.bloglovin.com/robots.txt","bytes":1859,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":3075,"domain":"commonsensemedia.org","origin":"https://commonsensemedia.org","robots":{"status":"ok","http":200,"url":"https://www.commonsensemedia.org/robots.txt","bytes":4693,"groups":12},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["blocked","named",1,0],"OAI-SearchBot":["partial","*",94,24],"ClaudeBot":["partial","*",94,24],"Claude-User":["partial","*",94,24],"Claude-SearchBot":["partial","*",94,24],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["partial","*",94,24],"Perplexity-User":["partial","*",94,24],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["partial","*",94,24],"CCBot":["blocked","named",1,0],"Bytespider":["partial","*",94,24],"Amazonbot":["partial","*",94,24],"meta-externalagent":["partial","*",94,24],"cohere-ai":["blocked","named",1,0],"Diffbot":["partial","*",94,24],"ContextFetchBot":["partial","*",94,24]},"llms":{"status":"missing","http":404,"bytes":559},"home":{"status":"ok","http":200,"url":"https://www.commonsensemedia.org/","html_tokens":25024,"markdown_tokens":161,"pruned_tokens":161}},
{"rank":3076,"domain":"rp-online.de","origin":"https://rp-online.de","robots":{"status":"ok","http":200,"url":"https://rp-online.de/robots.txt","bytes":20367,"groups":3},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["blocked","named",1,0],"OAI-SearchBot":["blocked","named",1,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["blocked","named",1,0],"Claude-SearchBot":["blocked","named",1,0],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["blocked","named",1,0],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",343,0]},"llms":{"status":"missing","http":404,"bytes":1485749},"home":{"status":"ok","http":200,"url":"https://rp-online.de/","html_tokens":251643,"markdown_tokens":919,"pruned_tokens":151}},
{"rank":3077,"domain":"uv.es","origin":"https://www.uv.es","robots":{"status":"ok","http":200,"url":"https://www.uv.es/robots.txt","bytes":472,"groups":1},"bots":{"GPTBot":["partial","*",14,0],"ChatGPT-User":["partial","*",14,0],"OAI-SearchBot":["partial","*",14,0],"ClaudeBot":["partial","*",14,0],"Claude-User":["partial","*",14,0],"Claude-SearchBot":["partial","*",14,0],"anthropic-ai":["partial","*",14,0],"PerplexityBot":["partial","*",14,0],"Perplexity-User":["partial","*",14,0],"Google-Extended":["partial","*",14,0],"Applebot-Extended":["partial","*",14,0],"CCBot":["partial","*",14,0],"Bytespider":["partial","*",14,0],"Amazonbot":["partial","*",14,0],"meta-externalagent":["partial","*",14,0],"cohere-ai":["partial","*",14,0],"Diffbot":["partial","*",14,0],"ContextFetchBot":["partial","*",14,0]},"llms":{"status":"missing","http":404,"bytes":196},"home":{"status":"ok","http":200,"url":"https://www.uv.es/","html_tokens":273006,"markdown_tokens":13,"pruned_tokens":13}},
{"rank":3078,"domain":"tailwindcss.com","origin":"https://tailwindcss.com","robots":{"status":"missing","http":404,"url":"https://tailwindcss.com/robots.txt","bytes":133683,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":133679},"home":{"status":"ok","http":200,"url":"https://tailwindcss.com/","html_tokens":219435,"markdown_tokens":1101,"pruned_tokens":75}},
{"rank":3079,"domain":"opencollective.com","origin":"https://opencollective.com","robots":{"status":"ok","http":200,"url":"https://opencollective.com/robots.txt","bytes":424,"groups":11},"bots":{"GPTBot":["partial","*",2,0],"ChatGPT-User":["partial","*",2,0],"OAI-SearchBot":["partial","*",2,0],"ClaudeBot":["partial","*",2,0],"Claude-User":["partial","*",2,0],"Claude-SearchBot":["partial","*",2,0],"anthropic-ai":["partial","*",2,0],"PerplexityBot":["partial","*",2,0],"Perplexity-User":["partial","*",2,0],"Google-Extended":["partial","*",2,0],"Applebot-Extended":["partial","*",2,0],"CCBot":["partial","*",2,0],"Bytespider":["partial","*",2,0],"Amazonbot":["partial","*",2,0],"meta-externalagent":["partial","*",2,0],"cohere-ai":["partial","*",2,0],"Diffbot":["partial","*",2,0],"ContextFetchBot":["partial","*",2,0]},"llms":{"status":"missing","http":404,"bytes":30553},"home":{"status":"ok","http":200,"url":"https://opencollective.com/","html_tokens":13359,"markdown_tokens":338,"pruned_tokens":102}},
{"rank":3080,"domain":"mozillazine.org","origin":"https://mozillazine.org","robots":{"status":"missing","http":404,"url":"https://mozillazine.org/robots.txt","bytes":1623,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":1623},"home":{"status":"ok","http":200,"url":"https://mozillazine.org/","html_tokens":1201,"markdown_tokens":70,"pruned_tokens":56}},
{"rank":3081,"domain":"niedersachsen.de","origin":"https://niedersachsen.de","robots":{"status":"ok","http":200,"url":"https://niedersachsen.de/robots.txt","bytes":299,"groups":1},"bots":{"GPTBot":["partial","*",5,0],"ChatGPT-User":["partial","*",5,0],"OAI-SearchBot":["partial","*",5,0],"ClaudeBot":["partial","*",5,0],"Claude-User":["partial","*",5,0],"Claude-SearchBot":["partial","*",5,0],"anthropic-ai":["partial","*",5,0],"PerplexityBot":["partial","*",5,0],"Perplexity-User":["partial","*",5,0],"Google-Extended":["partial","*",5,0],"Applebot-Extended":["partial","*",5,0],"CCBot":["partial","*",5,0],"Bytespider":["partial","*",5,0],"Amazonbot":["partial","*",5,0],"meta-externalagent":["partial","*",5,0],"cohere-ai":["partial","*",5,0],"Diffbot":["partial","*",5,0],"ContextFetchBot":["partial","*",5,0]},"llms":{"status":"missing","http":404,"bytes":75948},"home":{"status":"ok","http":200,"url":"https://www.niedersachsen.de/startseite/","html_tokens":31364,"markdown_tokens":959,"pruned_tokens":150}},
{"rank":3082,"domain":"upm.es","origin":"https://upm.es","robots":{"status":"ok","http":200,"url":"https://www.upm.es/robots.txt","bytes":24,"groups":1},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"missing","http":404,"bytes":317029},"home":{"status":"ok","http":200,"url":"https://www.upm.es/","html_tokens":8134,"markdown_tokens":211,"pruned_tokens":68}},
{"rank":3083,"domain":"matrix.to","origin":"https://matrix.to","robots":{"status":"html","http":200,"url":"https://matrix.to/robots.txt","bytes":1836,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":3084,"domain":"uwa.edu.au","origin":"https://uwa.edu.au","robots":{"status":"ok","http":200,"url":"https://www.uwa.edu.au/robots.txt","bytes":3806,"groups":1},"bots":{"GPTBot":["partial","*",13,1],"ChatGPT-User":["partial","*",13,1],"OAI-SearchBot":["partial","*",13,1],"ClaudeBot":["partial","*",13,1],"Claude-User":["partial","*",13,1],"Claude-SearchBot":["partial","*",13,1],"anthropic-ai":["partial","*",13,1],"PerplexityBot":["partial","*",13,1],"Perplexity-User":["partial","*",13,1],"Google-Extended":["partial","*",13,1],"Applebot-Extended":["partial","*",13,1],"CCBot":["partial","*",13,1],"Bytespider":["partial","*",13,1],"Amazonbot":["partial","*",13,1],"meta-externalagent":["partial","*",13,1],"cohere-ai":["partial","*",13,1],"Diffbot":["partial","*",13,1],"ContextFetchBot":["partial","*",13,1]},"llms":{"status":"missing","http":404,"bytes":1245},"home":{"status":"ok","http":200,"url":"https://www.uwa.edu.au/home","html_tokens":58539,"markdown_tokens":159,"pruned_tokens":159}},
{"rank":3085,"domain":"thekitchn.com","origin":"https://thekitchn.com","robots":{"status":"ok","http":200,"url":"https://www.thekitchn.com/robots.txt","bytes":173,"groups":2},"bots":{"GPTBot":["partial","*",1,0],"ChatGPT-User":["partial","*",1,0],"OAI-SearchBot":["partial","*",1,0],"ClaudeBot":["partial","*",1,0],"Claude-User":["partial","*",1,0],"Claude-SearchBot":["partial","*",1,0],"anthropic-ai":["partial","*",1,0],"PerplexityBot":["partial","*",1,0],"Perplexity-User":["partial","*",1,0],"Google-Extended":["partial","*",1,0],"Applebot-Extended":["partial","*",1,0],"CCBot":["partial","*",1,0],"Bytespider":["partial","*",1,0],"Amazonbot":["partial","*",1,0],"meta-externalagent":["partial","*",1,0],"cohere-ai":["partial","*",1,0],"Diffbot":["partial","*",1,0],"ContextFetchBot":["partial","*",1,0]},"llms":{"status":"http_error","http":403,"bytes":3264},"home":{"status":"http_error","http":403,"url":"https://www.thekitchn.com/"}},
{"rank":3086,"domain":"mk.ru","origin":"https://mk.ru","robots":{"status":"ok","http":200,"url":"https://www.mk.ru/robots.txt","bytes":1383,"groups":2},"bots":{"GPTBot":["partial","*",15,0],"ChatGPT-User":["partial","*",15,0],"OAI-SearchBot":["partial","*",15,0],"ClaudeBot":["partial","*",15,0],"Claude-User":["partial","*",15,0],"Claude-SearchBot":["partial","*",15,0],"anthropic-ai":["partial","*",15,0],"PerplexityBot":["partial","*",15,0],"Perplexity-User":["partial","*",15,0],"Google-Extended":["partial","*",15,0],"Applebot-Extended":["partial","*",15,0],"CCBot":["partial","*",15,0],"Bytespider":["partial","*",15,0],"Amazonbot":["partial","*",15,0],"meta-externalagent":["partial","*",15,0],"cohere-ai":["partial","*",15,0],"Diffbot":["partial","*",15,0],"ContextFetchBot":["partial","*",15,0]},"llms":{"status":"missing","http":404,"bytes":158083},"home":{"status":"ok","http":200,"url":"https://www.mk.ru/","html_tokens":114563,"markdown_tokens":2121,"pruned_tokens":135}},
{"rank":3087,"domain":"ivoox.com","origin":"https://ivoox.com","robots":{"status":"ok","http":200,"url":"https://www.ivoox.com/robots.txt","bytes":1015,"groups":4},"bots":{"GPTBot":["partial","*",36,0],"ChatGPT-User":["partial","*",36,0],"OAI-SearchBot":["partial","*",36,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",36,0],"Claude-SearchBot":["partial","*",36,0],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["partial","*",36,0],"Perplexity-User":["partial","*",36,0],"Google-Extended":["partial","*",36,0],"Applebot-Extended":["partial","*",36,0],"CCBot":["partial","*",36,0],"Bytespider":["partial","*",36,0],"Amazonbot":["partial","*",36,0],"meta-externalagent":["partial","*",36,0],"cohere-ai":["partial","*",36,0],"Diffbot":["partial","*",36,0],"ContextFetchBot":["partial","*",36,0]},"llms":{"status":"missing","http":404,"bytes":80241},"home":{"status":"ok","http":200,"url":"https://www.ivoox.com/en/","html_tokens":78416,"markdown_tokens":1125,"pruned_tokens":320}},
{"rank":3088,"domain":"assemblee-nationale.fr","origin":"https://www.assemblee-nationale.fr","robots":{"status":"ok","http":200,"url":"https://www.assemblee-nationale.fr/robots.txt","bytes":1286,"groups":1},"bots":{"GPTBot":["partial","*",33,0],"ChatGPT-User":["partial","*",33,0],"OAI-SearchBot":["partial","*",33,0],"ClaudeBot":["partial","*",33,0],"Claude-User":["partial","*",33,0],"Claude-SearchBot":["partial","*",33,0],"anthropic-ai":["partial","*",33,0],"PerplexityBot":["partial","*",33,0],"Perplexity-User":["partial","*",33,0],"Google-Extended":["partial","*",33,0],"Applebot-Extended":["partial","*",33,0],"CCBot":["partial","*",33,0],"Bytespider":["partial","*",33,0],"Amazonbot":["partial","*",33,0],"meta-externalagent":["partial","*",33,0],"cohere-ai":["partial","*",33,0],"Diffbot":["partial","*",33,0],"ContextFetchBot":["partial","*",33,0]},"llms":{"status":"missing","http":404,"bytes":2725},"home":{"status":"ok","http":200,"url":"https://www.assemblee-nationale.fr/","html_tokens":29394,"markdown_tokens":148,"pruned_tokens":148}},
{"rank":3089,"domain":"muckrack.com","origin":"https://muckrack.com","robots":{"status":"refused","http":403,"url":"https://muckrack.com/robots.txt","bytes":5507,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":3090,"domain":"india.com","origin":"https://india.com","robots":{"status":"ok","http":200,"url":"https://www.india.com/robots.txt","bytes":1793,"groups":2},"bots":{"GPTBot":["allowed","named",0,1],"ChatGPT-User":["allowed","named",0,1],"OAI-SearchBot":["partial","*",23,2],"ClaudeBot":["allowed","named",0,1],"Claude-User":["partial","*",23,2],"Claude-SearchBot":["partial","*",23,2],"anthropic-ai":["partial","*",23,2],"PerplexityBot":["allowed","named",0,1],"Perplexity-User":["allowed","named",0,1],"Google-Extended":["allowed","named",0,1],"Applebot-Extended":["partial","*",23,2],"CCBot":["partial","*",23,2],"Bytespider":["partial","*",23,2],"Amazonbot":["partial","*",23,2],"meta-externalagent":["partial","*",23,2],"cohere-ai":["partial","*",23,2],"Diffbot":["partial","*",23,2],"ContextFetchBot":["partial","*",23,2]},"llms":{"status":"missing","http":404,"bytes":120897},"home":{"status":"ok","http":200,"url":"https://www.india.com/","html_tokens":70023,"markdown_tokens":49,"pruned_tokens":49}},
{"rank":3091,"domain":"chalmers.se","origin":"https://chalmers.se","robots":{"status":"ok","http":200,"url":"https://www.chalmers.se/robots.txt","bytes":265,"groups":1},"bots":{"GPTBot":["partial","*",2,1],"ChatGPT-User":["partial","*",2,1],"OAI-SearchBot":["partial","*",2,1],"ClaudeBot":["partial","*",2,1],"Claude-User":["partial","*",2,1],"Claude-SearchBot":["partial","*",2,1],"anthropic-ai":["partial","*",2,1],"PerplexityBot":["partial","*",2,1],"Perplexity-User":["partial","*",2,1],"Google-Extended":["partial","*",2,1],"Applebot-Extended":["partial","*",2,1],"CCBot":["partial","*",2,1],"Bytespider":["partial","*",2,1],"Amazonbot":["partial","*",2,1],"meta-externalagent":["partial","*",2,1],"cohere-ai":["partial","*",2,1],"Diffbot":["partial","*",2,1],"ContextFetchBot":["partial","*",2,1]},"llms":{"status":"missing","http":404,"bytes":262633},"home":{"status":"ok","http":200,"url":"https://www.chalmers.se/","html_tokens":99518,"markdown_tokens":569,"pruned_tokens":87}},
{"rank":3092,"domain":"jalbum.net","origin":"https://jalbum.net","robots":{"status":"ok","http":200,"url":"https://jalbum.net/robots.txt","bytes":3969,"groups":2},"bots":{"GPTBot":["partial","*",119,0],"ChatGPT-User":["partial","*",119,0],"OAI-SearchBot":["partial","*",119,0],"ClaudeBot":["partial","*",119,0],"Claude-User":["partial","*",119,0],"Claude-SearchBot":["partial","*",119,0],"anthropic-ai":["partial","*",119,0],"PerplexityBot":["partial","*",119,0],"Perplexity-User":["partial","*",119,0],"Google-Extended":["partial","*",119,0],"Applebot-Extended":["partial","*",119,0],"CCBot":["partial","*",119,0],"Bytespider":["partial","*",119,0],"Amazonbot":["partial","*",119,0],"meta-externalagent":["partial","*",119,0],"cohere-ai":["partial","*",119,0],"Diffbot":["partial","*",119,0],"ContextFetchBot":["partial","*",119,0]},"llms":{"status":"present","http":200,"bytes":5502},"home":{"status":"ok","http":200,"url":"https://jalbum.net/en/","html_tokens":9803,"markdown_tokens":603,"pruned_tokens":194}},
{"rank":3093,"domain":"blackrock.com","origin":"https://blackrock.com","robots":{"status":"ok","http":200,"url":"https://www.blackrock.com/robots.txt","bytes":5506,"groups":2},"bots":{"GPTBot":["partial","*",51,0],"ChatGPT-User":["partial","*",51,0],"OAI-SearchBot":["partial","*",51,0],"ClaudeBot":["partial","*",51,0],"Claude-User":["partial","*",51,0],"Claude-SearchBot":["partial","*",51,0],"anthropic-ai":["partial","*",51,0],"PerplexityBot":["partial","*",51,0],"Perplexity-User":["partial","*",51,0],"Google-Extended":["partial","*",51,0],"Applebot-Extended":["partial","*",51,0],"CCBot":["partial","*",51,0],"Bytespider":["partial","*",51,0],"Amazonbot":["partial","*",51,0],"meta-externalagent":["partial","*",51,0],"cohere-ai":["partial","*",51,0],"Diffbot":["partial","*",51,0],"ContextFetchBot":["partial","*",51,0]},"llms":{"status":"missing","http":404,"bytes":239781},"home":{"status":"ok","http":200,"url":"https://www.blackrock.com/us/individual","html_tokens":32105,"markdown_tokens":64,"pruned_tokens":64}},
{"rank":3094,"domain":"vogueplay.com","origin":"https://vogueplay.com","robots":{"status":"ok","http":200,"url":"https://vogueplay.com/robots.txt","bytes":1169,"groups":2},"bots":{"GPTBot":["partial","*",20,0],"ChatGPT-User":["partial","*",20,0],"OAI-SearchBot":["partial","*",20,0],"ClaudeBot":["partial","*",20,0],"Claude-User":["partial","*",20,0],"Claude-SearchBot":["partial","*",20,0],"anthropic-ai":["partial","*",20,0],"PerplexityBot":["partial","*",20,0],"Perplexity-User":["partial","*",20,0],"Google-Extended":["partial","*",20,0],"Applebot-Extended":["partial","*",20,0],"CCBot":["partial","*",20,0],"Bytespider":["partial","*",20,0],"Amazonbot":["partial","*",20,0],"meta-externalagent":["partial","*",20,0],"cohere-ai":["partial","*",20,0],"Diffbot":["partial","*",20,0],"ContextFetchBot":["partial","*",20,0]},"llms":{"status":"missing","http":404,"bytes":72776},"home":{"status":"ok","http":200,"url":"https://vogueplay.com/","html_tokens":67669,"markdown_tokens":2992,"pruned_tokens":151}},
{"rank":3095,"domain":"business.com","origin":"https://business.com","robots":{"status":"ok","http":200,"url":"https://www.business.com/robots.txt","bytes":633,"groups":2},"bots":{"GPTBot":["allowed","named",0,1],"ChatGPT-User":["allowed","named",0,1],"OAI-SearchBot":["allowed","named",0,1],"ClaudeBot":["allowed","named",0,1],"Claude-User":["partial","*",4,0],"Claude-SearchBot":["partial","*",4,0],"anthropic-ai":["allowed","named",0,1],"PerplexityBot":["allowed","named",0,1],"Perplexity-User":["allowed","named",0,1],"Google-Extended":["allowed","named",0,1],"Applebot-Extended":["allowed","named",0,1],"CCBot":["partial","*",4,0],"Bytespider":["partial","*",4,0],"Amazonbot":["allowed","named",0,1],"meta-externalagent":["allowed","named",0,1],"cohere-ai":["partial","*",4,0],"Diffbot":["partial","*",4,0],"ContextFetchBot":["partial","*",4,0]},"llms":{"status":"missing","http":404,"bytes":121152},"home":{"status":"ok","http":200,"url":"https://www.business.com/","html_tokens":85145,"markdown_tokens":144,"pruned_tokens":66}},
{"rank":3096,"domain":"sltrib.com","origin":"https://sltrib.com","robots":{"status":"ok","http":200,"url":"https://www.sltrib.com/robots.txt","bytes":582,"groups":13},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["blocked","named",1,0],"OAI-SearchBot":["partial","*",1,0],"ClaudeBot":["partial","*",1,0],"Claude-User":["partial","*",1,0],"Claude-SearchBot":["partial","*",1,0],"anthropic-ai":["partial","*",1,0],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["partial","*",1,0],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["partial","*",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["partial","*",1,0],"Amazonbot":["partial","*",1,0],"meta-externalagent":["partial","*",1,0],"cohere-ai":["partial","*",1,0],"Diffbot":["partial","*",1,0],"ContextFetchBot":["partial","*",1,0]},"llms":{"status":"missing","http":404,"bytes":113350},"home":{"status":"ok","http":200,"url":"https://www.sltrib.com/","html_tokens":282299,"markdown_tokens":88,"pruned_tokens":88}},
{"rank":3097,"domain":"rackcdn.com","origin":"https://rackcdn.com","robots":{"status":"unreachable","http":null,"url":"https://rackcdn.com/robots.txt","bytes":0,"groups":0,"error":"cannot resolve rackcdn.com"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":3098,"domain":"wizards.com","origin":"https://wizards.com","robots":{"status":"html","http":200,"url":"https://company.wizards.com/en","bytes":331512,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":3099,"domain":"nxp.com","origin":"https://nxp.com","robots":{"status":"ok","http":200,"url":"https://www.nxp.com/robots.txt","bytes":665,"groups":2},"bots":{"GPTBot":["partial","*",6,0],"ChatGPT-User":["partial","*",6,0],"OAI-SearchBot":["partial","*",6,0],"ClaudeBot":["partial","*",6,0],"Claude-User":["partial","*",6,0],"Claude-SearchBot":["partial","*",6,0],"anthropic-ai":["partial","*",6,0],"PerplexityBot":["partial","*",6,0],"Perplexity-User":["partial","*",6,0],"Google-Extended":["partial","*",6,0],"Applebot-Extended":["partial","*",6,0],"CCBot":["partial","*",6,0],"Bytespider":["partial","*",6,0],"Amazonbot":["partial","*",6,0],"meta-externalagent":["partial","*",6,0],"cohere-ai":["partial","*",6,0],"Diffbot":["partial","*",6,0],"ContextFetchBot":["partial","*",6,0]},"llms":{"status":"missing","http":404,"bytes":36129},"home":{"status":"ok","http":200,"url":"https://www.nxp.com/","html_tokens":20676,"markdown_tokens":238,"pruned_tokens":113}},
{"rank":3100,"domain":"investors.com","origin":"https://investors.com","robots":{"status":"ok","http":200,"url":"https://www.investors.com/robots.txt","bytes":2509,"groups":2},"bots":{"GPTBot":["partial","named",21,1],"ChatGPT-User":["partial","named",21,1],"OAI-SearchBot":["partial","named",21,1],"ClaudeBot":["blocked","*",1,0],"Claude-User":["blocked","*",1,0],"Claude-SearchBot":["blocked","*",1,0],"anthropic-ai":["blocked","*",1,0],"PerplexityBot":["blocked","*",1,0],"Perplexity-User":["blocked","*",1,0],"Google-Extended":["blocked","*",1,0],"Applebot-Extended":["blocked","*",1,0],"CCBot":["blocked","*",1,0],"Bytespider":["blocked","*",1,0],"Amazonbot":["blocked","*",1,0],"meta-externalagent":["blocked","*",1,0],"cohere-ai":["blocked","*",1,0],"Diffbot":["blocked","*",1,0],"ContextFetchBot":["blocked","*",1,0]},"llms":{"status":"disallowed"},"home":{"status":"disallowed"}},
{"rank":3101,"domain":"google.pt","origin":"https://google.pt","robots":{"status":"ok","http":200,"url":"https://www.google.pt/robots.txt","bytes":6624,"groups":4},"bots":{"GPTBot":["partial","*",169,64],"ChatGPT-User":["partial","*",169,64],"OAI-SearchBot":["partial","*",169,64],"ClaudeBot":["partial","*",169,64],"Claude-User":["partial","*",169,64],"Claude-SearchBot":["partial","*",169,64],"anthropic-ai":["partial","*",169,64],"PerplexityBot":["partial","*",169,64],"Perplexity-User":["partial","*",169,64],"Google-Extended":["partial","*",169,64],"Applebot-Extended":["partial","*",169,64],"CCBot":["partial","*",169,64],"Bytespider":["partial","*",169,64],"Amazonbot":["partial","*",169,64],"meta-externalagent":["partial","*",169,64],"cohere-ai":["partial","*",169,64],"Diffbot":["partial","*",169,64],"ContextFetchBot":["partial","*",169,64]},"llms":{"status":"missing","http":404,"bytes":1569},"home":{"status":"ok","http":200,"url":"https://www.google.com/","html_tokens":21300,"markdown_tokens":30,"pruned_tokens":30}},
{"rank":3102,"domain":"google.com.ar","origin":"https://google.com.ar","robots":{"status":"ok","http":200,"url":"https://www.google.com.ar/robots.txt","bytes":6624,"groups":4},"bots":{"GPTBot":["partial","*",169,64],"ChatGPT-User":["partial","*",169,64],"OAI-SearchBot":["partial","*",169,64],"ClaudeBot":["partial","*",169,64],"Claude-User":["partial","*",169,64],"Claude-SearchBot":["partial","*",169,64],"anthropic-ai":["partial","*",169,64],"PerplexityBot":["partial","*",169,64],"Perplexity-User":["partial","*",169,64],"Google-Extended":["partial","*",169,64],"Applebot-Extended":["partial","*",169,64],"CCBot":["partial","*",169,64],"Bytespider":["partial","*",169,64],"Amazonbot":["partial","*",169,64],"meta-externalagent":["partial","*",169,64],"cohere-ai":["partial","*",169,64],"Diffbot":["partial","*",169,64],"ContextFetchBot":["partial","*",169,64]},"llms":{"status":"missing","http":404,"bytes":1569},"home":{"status":"ok","http":200,"url":"https://www.google.com/","html_tokens":21470,"markdown_tokens":30,"pruned_tokens":30}},
{"rank":3103,"domain":"fmprc.gov.cn","origin":"https://www.fmprc.gov.cn","robots":{"status":"html","http":200,"url":"https://www.mfa.gov.cn/web/system/index_17321.shtml","bytes":54156,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":3104,"domain":"nwsource.com","origin":"https://nwsource.com","robots":{"status":"html","http":200,"url":"https://www.seattletimes.com","bytes":786284,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":3105,"domain":"mapbox.com","origin":"https://mapbox.com","robots":{"status":"ok","http":200,"url":"https://www.mapbox.com/robots.txt","bytes":126,"groups":2},"bots":{"GPTBot":["partial","*",1,0],"ChatGPT-User":["partial","*",1,0],"OAI-SearchBot":["partial","*",1,0],"ClaudeBot":["partial","*",1,0],"Claude-User":["partial","*",1,0],"Claude-SearchBot":["partial","*",1,0],"anthropic-ai":["partial","*",1,0],"PerplexityBot":["partial","*",1,0],"Perplexity-User":["partial","*",1,0],"Google-Extended":["partial","*",1,0],"Applebot-Extended":["partial","*",1,0],"CCBot":["partial","*",1,0],"Bytespider":["partial","*",1,0],"Amazonbot":["partial","*",1,0],"meta-externalagent":["partial","*",1,0],"cohere-ai":["partial","*",1,0],"Diffbot":["partial","*",1,0],"ContextFetchBot":["partial","*",1,0]},"llms":{"status":"missing","http":404,"bytes":11608},"home":{"status":"ok","http":200,"url":"https://www.mapbox.com/","html_tokens":174966,"markdown_tokens":762,"pruned_tokens":252}},
{"rank":3106,"domain":"onlyfans.com","origin":"https://onlyfans.com","robots":{"status":"ok","http":200,"url":"https://onlyfans.com/robots.txt","bytes":162,"groups":1},"bots":{"GPTBot":["partial","*",7,0],"ChatGPT-User":["partial","*",7,0],"OAI-SearchBot":["partial","*",7,0],"ClaudeBot":["partial","*",7,0],"Claude-User":["partial","*",7,0],"Claude-SearchBot":["partial","*",7,0],"anthropic-ai":["partial","*",7,0],"PerplexityBot":["partial","*",7,0],"Perplexity-User":["partial","*",7,0],"Google-Extended":["partial","*",7,0],"Applebot-Extended":["partial","*",7,0],"CCBot":["partial","*",7,0],"Bytespider":["partial","*",7,0],"Amazonbot":["partial","*",7,0],"meta-externalagent":["partial","*",7,0],"cohere-ai":["partial","*",7,0],"Diffbot":["partial","*",7,0],"ContextFetchBot":["partial","*",7,0]},"llms":{"status":"html","http":200,"bytes":17669},"home":{"status":"http_error","http":403,"url":"https://onlyfans.com/"}},
{"rank":3107,"domain":"mid.ru","origin":"https://mid.ru","robots":{"status":"ok","http":200,"url":"https://mid.ru/robots.txt","bytes":1959,"groups":5},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["partial","*",66,0],"OAI-SearchBot":["partial","*",66,0],"ClaudeBot":["partial","*",66,0],"Claude-User":["partial","*",66,0],"Claude-SearchBot":["partial","*",66,0],"anthropic-ai":["partial","*",66,0],"PerplexityBot":["partial","*",66,0],"Perplexity-User":["partial","*",66,0],"Google-Extended":["partial","*",66,0],"Applebot-Extended":["partial","*",66,0],"CCBot":["partial","*",66,0],"Bytespider":["partial","*",66,0],"Amazonbot":["partial","*",66,0],"meta-externalagent":["partial","*",66,0],"cohere-ai":["partial","*",66,0],"Diffbot":["partial","*",66,0],"ContextFetchBot":["partial","*",66,0]},"llms":{"status":"html","http":200,"bytes":302},"home":{"status":"ok","http":200,"url":"https://mid.ru/","html_tokens":76,"markdown_tokens":33,"pruned_tokens":32}},
{"rank":3108,"domain":"ua.edu","origin":"https://ua.edu","robots":{"status":"refused","http":403,"url":"https://ua.edu/robots.txt","bytes":560,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":3109,"domain":"rae.es","origin":"https://rae.es","robots":{"status":"ok","http":200,"url":"https://www.rae.es/robots.txt","bytes":2043,"groups":1},"bots":{"GPTBot":["partial","*",34,18],"ChatGPT-User":["partial","*",34,18],"OAI-SearchBot":["partial","*",34,18],"ClaudeBot":["partial","*",34,18],"Claude-User":["partial","*",34,18],"Claude-SearchBot":["partial","*",34,18],"anthropic-ai":["partial","*",34,18],"PerplexityBot":["partial","*",34,18],"Perplexity-User":["partial","*",34,18],"Google-Extended":["partial","*",34,18],"Applebot-Extended":["partial","*",34,18],"CCBot":["partial","*",34,18],"Bytespider":["partial","*",34,18],"Amazonbot":["partial","*",34,18],"meta-externalagent":["partial","*",34,18],"cohere-ai":["partial","*",34,18],"Diffbot":["partial","*",34,18],"ContextFetchBot":["partial","*",34,18]},"llms":{"status":"missing","http":404,"bytes":184},"home":{"status":"http_error","http":403,"url":"https://rae.es/"}},
{"rank":3110,"domain":"adb.org","origin":"https://adb.org","robots":{"status":"html","http":200,"url":"https://www.adb.org/","bytes":224713,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":3111,"domain":"uni-goettingen.de","origin":"https://uni-goettingen.de","robots":{"status":"ok","http":200,"url":"https://uni-goettingen.de/robots.txt","bytes":577,"groups":2},"bots":{"GPTBot":["partial","*",13,0],"ChatGPT-User":["partial","*",13,0],"OAI-SearchBot":["partial","*",13,0],"ClaudeBot":["partial","*",13,0],"Claude-User":["partial","*",13,0],"Claude-SearchBot":["partial","*",13,0],"anthropic-ai":["partial","*",13,0],"PerplexityBot":["partial","*",13,0],"Perplexity-User":["partial","*",13,0],"Google-Extended":["partial","*",13,0],"Applebot-Extended":["partial","*",13,0],"CCBot":["partial","*",13,0],"Bytespider":["partial","*",13,0],"Amazonbot":["partial","*",13,0],"meta-externalagent":["partial","*",13,0],"cohere-ai":["partial","*",13,0],"Diffbot":["partial","*",13,0],"ContextFetchBot":["partial","*",13,0]},"llms":{"status":"missing","http":404,"bytes":2997},"home":{"status":"ok","http":200,"url":"https://uni-goettingen.de/","html_tokens":17945,"markdown_tokens":839,"pruned_tokens":154}},
{"rank":3112,"domain":"gopro.com","origin":"https://gopro.com","robots":{"status":"refused","http":403,"url":"https://gopro.com/robots.txt","bytes":769,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":3113,"domain":"bsigroup.com","origin":"https://bsigroup.com","robots":{"status":"ok","http":200,"url":"https://www.bsigroup.com/robots.txt","bytes":2044,"groups":3},"bots":{"GPTBot":["partial","*",38,0],"ChatGPT-User":["partial","*",38,0],"OAI-SearchBot":["partial","*",38,0],"ClaudeBot":["partial","*",38,0],"Claude-User":["partial","*",38,0],"Claude-SearchBot":["partial","*",38,0],"anthropic-ai":["partial","*",38,0],"PerplexityBot":["partial","*",38,0],"Perplexity-User":["partial","*",38,0],"Google-Extended":["allowed","named",0,1],"Applebot-Extended":["partial","*",38,0],"CCBot":["partial","*",38,0],"Bytespider":["partial","*",38,0],"Amazonbot":["partial","*",38,0],"meta-externalagent":["partial","*",38,0],"cohere-ai":["partial","*",38,0],"Diffbot":["partial","*",38,0],"ContextFetchBot":["partial","*",38,0]},"llms":{"status":"missing","http":404,"bytes":184943},"home":{"status":"ok","http":200,"url":"https://www.bsigroup.com/","html_tokens":67041,"markdown_tokens":453,"pruned_tokens":138}},
{"rank":3114,"domain":"awin.com","origin":"https://awin.com","robots":{"status":"ok","http":200,"url":"https://www.awin.com/robots.txt","bytes":165,"groups":1},"bots":{"GPTBot":["allowed","*",1,0],"ChatGPT-User":["allowed","*",1,0],"OAI-SearchBot":["allowed","*",1,0],"ClaudeBot":["allowed","*",1,0],"Claude-User":["allowed","*",1,0],"Claude-SearchBot":["allowed","*",1,0],"anthropic-ai":["allowed","*",1,0],"PerplexityBot":["allowed","*",1,0],"Perplexity-User":["allowed","*",1,0],"Google-Extended":["allowed","*",1,0],"Applebot-Extended":["allowed","*",1,0],"CCBot":["allowed","*",1,0],"Bytespider":["allowed","*",1,0],"Amazonbot":["allowed","*",1,0],"meta-externalagent":["allowed","*",1,0],"cohere-ai":["allowed","*",1,0],"Diffbot":["allowed","*",1,0],"ContextFetchBot":["allowed","*",1,0]},"llms":{"status":"missing","http":404,"bytes":85560},"home":{"status":"ok","http":200,"url":"https://www.awin.com/","html_tokens":74125,"markdown_tokens":946,"pruned_tokens":70}},
{"rank":3115,"domain":"diabetesjournals.org","origin":"https://diabetesjournals.org","robots":{"status":"ok","http":200,"url":"https://diabetesjournals.org/robots.txt","bytes":1272,"groups":2},"bots":{"GPTBot":["partial","*",36,3],"ChatGPT-User":["partial","*",36,3],"OAI-SearchBot":["partial","*",36,3],"ClaudeBot":["partial","*",36,3],"Claude-User":["partial","*",36,3],"Claude-SearchBot":["partial","*",36,3],"anthropic-ai":["partial","*",36,3],"PerplexityBot":["partial","*",36,3],"Perplexity-User":["partial","*",36,3],"Google-Extended":["partial","*",36,3],"Applebot-Extended":["partial","*",36,3],"CCBot":["partial","*",36,3],"Bytespider":["partial","*",36,3],"Amazonbot":["partial","*",36,3],"meta-externalagent":["partial","*",36,3],"cohere-ai":["partial","*",36,3],"Diffbot":["partial","*",36,3],"ContextFetchBot":["partial","*",36,3]},"llms":{"status":"http_error","http":403,"bytes":5531},"home":{"status":"http_error","http":403,"url":"https://diabetesjournals.org/"}},
{"rank":3116,"domain":"jasper.ai","origin":"https://jasper.ai","robots":{"status":"ok","http":200,"url":"https://www.jasper.ai/robots.txt","bytes":665,"groups":7},"bots":{"GPTBot":["allowed","named",0,1],"ChatGPT-User":["allowed","named",0,1],"OAI-SearchBot":["allowed","named",0,1],"ClaudeBot":["allowed","named",0,1],"Claude-User":["partial","*",1,0],"Claude-SearchBot":["partial","*",1,0],"anthropic-ai":["allowed","named",0,1],"PerplexityBot":["allowed","named",0,1],"Perplexity-User":["partial","*",1,0],"Google-Extended":["allowed","named",0,1],"Applebot-Extended":["allowed","named",0,1],"CCBot":["allowed","named",0,1],"Bytespider":["allowed","named",0,1],"Amazonbot":["partial","*",1,0],"meta-externalagent":["allowed","named",0,1],"cohere-ai":["allowed","named",0,1],"Diffbot":["allowed","named",0,1],"ContextFetchBot":["partial","*",1,0]},"llms":{"status":"present","http":200,"bytes":12237},"home":{"status":"ok","http":200,"url":"https://www.jasper.ai/","html_tokens":121497,"markdown_tokens":1706,"pruned_tokens":653}},
{"rank":3117,"domain":"inverse.com","origin":"https://inverse.com","robots":{"status":"ok","http":200,"url":"https://www.inverse.com/robots.txt","bytes":420,"groups":2},"bots":{"GPTBot":["partial","*",1,0],"ChatGPT-User":["partial","*",1,0],"OAI-SearchBot":["partial","*",1,0],"ClaudeBot":["partial","*",1,0],"Claude-User":["partial","*",1,0],"Claude-SearchBot":["partial","*",1,0],"anthropic-ai":["partial","*",1,0],"PerplexityBot":["partial","*",1,0],"Perplexity-User":["partial","*",1,0],"Google-Extended":["partial","*",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["partial","*",1,0],"Bytespider":["partial","*",1,0],"Amazonbot":["partial","*",1,0],"meta-externalagent":["partial","*",1,0],"cohere-ai":["partial","*",1,0],"Diffbot":["partial","*",1,0],"ContextFetchBot":["partial","*",1,0]},"llms":{"status":"missing","http":404,"bytes":35451},"home":{"status":"ok","http":200,"url":"https://www.inverse.com","html_tokens":102549,"markdown_tokens":734,"pruned_tokens":79}},
{"rank":3118,"domain":"hamburg.de","origin":"https://hamburg.de","robots":{"status":"ok","http":200,"url":"https://www.hamburg.de/robots.txt","bytes":1979,"groups":42},"bots":{"GPTBot":["partial","*",15,0],"ChatGPT-User":["partial","*",15,0],"OAI-SearchBot":["partial","*",15,0],"ClaudeBot":["partial","*",15,0],"Claude-User":["partial","*",15,0],"Claude-SearchBot":["partial","*",15,0],"anthropic-ai":["partial","*",15,0],"PerplexityBot":["partial","*",15,0],"Perplexity-User":["partial","*",15,0],"Google-Extended":["partial","*",15,0],"Applebot-Extended":["partial","*",15,0],"CCBot":["partial","*",15,0],"Bytespider":["partial","*",15,0],"Amazonbot":["partial","*",15,0],"meta-externalagent":["partial","*",15,0],"cohere-ai":["partial","*",15,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",15,0]},"llms":{"status":"missing","http":404,"bytes":251097},"home":{"status":"ok","http":200,"url":"https://www.hamburg.de/","html_tokens":76150,"markdown_tokens":64,"pruned_tokens":64}},
{"rank":3119,"domain":"sgs.com","origin":"https://sgs.com","robots":{"status":"refused","http":403,"url":"https://www.sgs.com/robots.txt","bytes":379,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":3120,"domain":"mynewsdesk.com","origin":"https://mynewsdesk.com","robots":{"status":"ok","http":200,"url":"https://www.mynewsdesk.com/robots.txt","bytes":315,"groups":1},"bots":{"GPTBot":["partial","*",4,0],"ChatGPT-User":["partial","*",4,0],"OAI-SearchBot":["partial","*",4,0],"ClaudeBot":["partial","*",4,0],"Claude-User":["partial","*",4,0],"Claude-SearchBot":["partial","*",4,0],"anthropic-ai":["partial","*",4,0],"PerplexityBot":["partial","*",4,0],"Perplexity-User":["partial","*",4,0],"Google-Extended":["partial","*",4,0],"Applebot-Extended":["partial","*",4,0],"CCBot":["partial","*",4,0],"Bytespider":["partial","*",4,0],"Amazonbot":["partial","*",4,0],"meta-externalagent":["partial","*",4,0],"cohere-ai":["partial","*",4,0],"Diffbot":["partial","*",4,0],"ContextFetchBot":["partial","*",4,0]},"llms":{"status":"missing","http":404,"bytes":10537},"home":{"status":"ok","http":200,"url":"https://www.mynewsdesk.com/en/","html_tokens":47174,"markdown_tokens":131,"pruned_tokens":131}},
{"rank":3121,"domain":"begambleaware.org","origin":"https://begambleaware.org","robots":{"status":"ok","http":200,"url":"https://www.gambleaware.org/robots.txt/","bytes":71,"groups":1},"bots":{"GPTBot":["allowed","*",0,1],"ChatGPT-User":["allowed","*",0,1],"OAI-SearchBot":["allowed","*",0,1],"ClaudeBot":["allowed","*",0,1],"Claude-User":["allowed","*",0,1],"Claude-SearchBot":["allowed","*",0,1],"anthropic-ai":["allowed","*",0,1],"PerplexityBot":["allowed","*",0,1],"Perplexity-User":["allowed","*",0,1],"Google-Extended":["allowed","*",0,1],"Applebot-Extended":["allowed","*",0,1],"CCBot":["allowed","*",0,1],"Bytespider":["allowed","*",0,1],"Amazonbot":["allowed","*",0,1],"meta-externalagent":["allowed","*",0,1],"cohere-ai":["allowed","*",0,1],"Diffbot":["allowed","*",0,1],"ContextFetchBot":["allowed","*",0,1]},"llms":{"status":"missing","http":404,"bytes":27242},"home":{"status":"ok","http":200,"url":"https://www.gambleaware.org/","html_tokens":11228,"markdown_tokens":281,"pruned_tokens":130}},
{"rank":3122,"domain":"cbinsights.com","origin":"https://cbinsights.com","robots":{"status":"ok","http":200,"url":"https://www.cbinsights.com/robots.txt","bytes":1352,"groups":6},"bots":{"GPTBot":["partial","named",5,0],"ChatGPT-User":["partial","*",23,0],"OAI-SearchBot":["partial","*",23,0],"ClaudeBot":["partial","named",5,0],"Claude-User":["partial","*",23,0],"Claude-SearchBot":["partial","*",23,0],"anthropic-ai":["partial","*",23,0],"PerplexityBot":["partial","named",5,0],"Perplexity-User":["partial","*",23,0],"Google-Extended":["partial","named",5,0],"Applebot-Extended":["partial","*",23,0],"CCBot":["partial","named",5,0],"Bytespider":["partial","*",23,0],"Amazonbot":["partial","*",23,0],"meta-externalagent":["partial","*",23,0],"cohere-ai":["partial","*",23,0],"Diffbot":["partial","*",23,0],"ContextFetchBot":["partial","*",23,0]},"llms":{"status":"missing","http":404,"bytes":7362},"home":{"status":"ok","http":200,"url":"https://www.cbinsights.com/","html_tokens":170044,"markdown_tokens":413,"pruned_tokens":100}},
{"rank":3123,"domain":"skillshare.com","origin":"https://skillshare.com","robots":{"status":"ok","http":200,"url":"https://www.skillshare.com/robots.txt","bytes":609,"groups":1},"bots":{"GPTBot":["partial","*",11,1],"ChatGPT-User":["partial","*",11,1],"OAI-SearchBot":["partial","*",11,1],"ClaudeBot":["partial","*",11,1],"Claude-User":["partial","*",11,1],"Claude-SearchBot":["partial","*",11,1],"anthropic-ai":["partial","*",11,1],"PerplexityBot":["partial","*",11,1],"Perplexity-User":["partial","*",11,1],"Google-Extended":["partial","*",11,1],"Applebot-Extended":["partial","*",11,1],"CCBot":["partial","*",11,1],"Bytespider":["partial","*",11,1],"Amazonbot":["partial","*",11,1],"meta-externalagent":["partial","*",11,1],"cohere-ai":["partial","*",11,1],"Diffbot":["partial","*",11,1],"ContextFetchBot":["partial","*",11,1]},"llms":{"status":"missing","http":404,"bytes":7126},"home":{"status":"http_error","http":403,"url":"https://www.skillshare.com/"}},
{"rank":3124,"domain":"abc7.com","origin":"https://abc7.com","robots":{"status":"ok","http":200,"url":"https://abc7.com/robots.txt","bytes":1123,"groups":26},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["blocked","named",1,0],"OAI-SearchBot":["partial","*",7,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",7,0],"Claude-SearchBot":["partial","*",7,0],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["partial","*",7,0],"Perplexity-User":["partial","*",7,0],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["partial","*",7,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",7,0]},"llms":{"status":"html","http":200,"bytes":73743},"home":{"status":"ok","http":200,"url":"https://abc7.com/","html_tokens":150330,"markdown_tokens":88,"pruned_tokens":88}},
{"rank":3125,"domain":"sun-sentinel.com","origin":"https://sun-sentinel.com","robots":{"status":"ok","http":200,"url":"https://www.sun-sentinel.com/robots.txt","bytes":4487,"groups":68},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["blocked","named",1,0],"OAI-SearchBot":["blocked","named",1,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["blocked","named",1,0],"Claude-SearchBot":["blocked","named",1,0],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["blocked","named",1,0],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["partial","*",8,1],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",8,1]},"llms":{"status":"missing","http":404,"bytes":91411},"home":{"status":"ok","http":200,"url":"https://www.sun-sentinel.com/","html_tokens":68942,"markdown_tokens":407,"pruned_tokens":159}},
{"rank":3126,"domain":"winehq.org","origin":"https://winehq.org","robots":{"status":"refused","http":403,"url":"https://www.winehq.org/robots.txt","bytes":5424,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":3127,"domain":"macys.com","origin":"https://macys.com","robots":{"status":"refused","http":403,"url":"https://www.macys.com/robots.txt","bytes":9900,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":3128,"domain":"or.ke","origin":"https://or.ke","robots":{"status":"unreachable","http":null,"url":"https://or.ke/robots.txt","bytes":0,"groups":0,"error":"cannot resolve or.ke"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":3129,"domain":"bath.ac.uk","origin":"https://bath.ac.uk","robots":{"status":"ok","http":200,"url":"https://www.bath.ac.uk/robots.txt","bytes":528,"groups":1},"bots":{"GPTBot":["partial","*",15,0],"ChatGPT-User":["partial","*",15,0],"OAI-SearchBot":["partial","*",15,0],"ClaudeBot":["partial","*",15,0],"Claude-User":["partial","*",15,0],"Claude-SearchBot":["partial","*",15,0],"anthropic-ai":["partial","*",15,0],"PerplexityBot":["partial","*",15,0],"Perplexity-User":["partial","*",15,0],"Google-Extended":["partial","*",15,0],"Applebot-Extended":["partial","*",15,0],"CCBot":["partial","*",15,0],"Bytespider":["partial","*",15,0],"Amazonbot":["partial","*",15,0],"meta-externalagent":["partial","*",15,0],"cohere-ai":["partial","*",15,0],"Diffbot":["partial","*",15,0],"ContextFetchBot":["partial","*",15,0]},"llms":{"status":"missing","http":404,"bytes":30480},"home":{"status":"ok","http":200,"url":"https://www.bath.ac.uk/","html_tokens":13921,"markdown_tokens":308,"pruned_tokens":220}},
{"rank":3130,"domain":"br.linkedin.com","origin":"https://br.linkedin.com","robots":{"status":"ok","http":200,"url":"https://br.linkedin.com/robots.txt","bytes":120190,"groups":77},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["blocked","named",1,0],"OAI-SearchBot":["partial","named",111,9],"ClaudeBot":["blocked","named",1,0],"Claude-User":["blocked","named",1,0],"Claude-SearchBot":["partial","named",111,9],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["blocked","named",1,0],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","*",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","*",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["blocked","*",1,0]},"llms":{"status":"disallowed"},"home":{"status":"disallowed"}},
{"rank":3131,"domain":"theme-fusion.com","origin":"https://theme-fusion.com","robots":{"status":"refused","http":403,"url":"https://avada.com","bytes":5495,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":3132,"domain":"ugr.es","origin":"https://ugr.es","robots":{"status":"ok","http":200,"url":"https://www.ugr.es/robots.txt","bytes":1665,"groups":1},"bots":{"GPTBot":["partial","*",26,18],"ChatGPT-User":["partial","*",26,18],"OAI-SearchBot":["partial","*",26,18],"ClaudeBot":["partial","*",26,18],"Claude-User":["partial","*",26,18],"Claude-SearchBot":["partial","*",26,18],"anthropic-ai":["partial","*",26,18],"PerplexityBot":["partial","*",26,18],"Perplexity-User":["partial","*",26,18],"Google-Extended":["partial","*",26,18],"Applebot-Extended":["partial","*",26,18],"CCBot":["partial","*",26,18],"Bytespider":["partial","*",26,18],"Amazonbot":["partial","*",26,18],"meta-externalagent":["partial","*",26,18],"cohere-ai":["partial","*",26,18],"Diffbot":["partial","*",26,18],"ContextFetchBot":["partial","*",26,18]},"llms":{"status":"missing","http":404,"bytes":183},"home":{"status":"ok","http":200,"url":"https://www.ugr.es/","html_tokens":27371,"markdown_tokens":86,"pruned_tokens":86}},
{"rank":3133,"domain":"pytorch.org","origin":"https://pytorch.org","robots":{"status":"refused","http":403,"url":"https://pytorch.org/robots.txt","bytes":560,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":3134,"domain":"anandtech.com","origin":"https://anandtech.com","robots":{"status":"html","http":200,"url":"https://forums.anandtech.com/.stile/challenge?rung=nojs&band=watch&return=%2F&sig=15351159399c9169e0d21d48877aa22ee6bca7442798acf1144ba9dcf49d755f&aud=forums.anandtech.com&aid=37af1cb7c686bd6f78ebca02b998131c","bytes":2859,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":3135,"domain":"lwshosting.name","origin":"https://lwshosting.name","robots":{"status":"ok","http":200,"url":"https://lwshosting.name/robots.txt","bytes":1248,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":2173},"home":{"status":"http_error","http":403,"url":"https://lwshosting.name/"}},
{"rank":3136,"domain":"domain.com","origin":"https://domain.com","robots":{"status":"refused","http":403,"url":"https://www.domain.com/robots.txt","bytes":5551,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":3137,"domain":"oxfam.org","origin":"https://oxfam.org","robots":{"status":"ok","http":200,"url":"https://www.oxfam.org/robots.txt","bytes":1732,"groups":1},"bots":{"GPTBot":["partial","*",22,18],"ChatGPT-User":["partial","*",22,18],"OAI-SearchBot":["partial","*",22,18],"ClaudeBot":["partial","*",22,18],"Claude-User":["partial","*",22,18],"Claude-SearchBot":["partial","*",22,18],"anthropic-ai":["partial","*",22,18],"PerplexityBot":["partial","*",22,18],"Perplexity-User":["partial","*",22,18],"Google-Extended":["partial","*",22,18],"Applebot-Extended":["partial","*",22,18],"CCBot":["partial","*",22,18],"Bytespider":["partial","*",22,18],"Amazonbot":["partial","*",22,18],"meta-externalagent":["partial","*",22,18],"cohere-ai":["partial","*",22,18],"Diffbot":["partial","*",22,18],"ContextFetchBot":["partial","*",22,18]},"llms":{"status":"http_error","http":403,"bytes":153},"home":{"status":"timeout","error":"timed out after 15 s"}},
{"rank":3138,"domain":"sendinblue.com","origin":"https://sendinblue.com","robots":{"status":"ok","http":200,"url":"https://www.brevo.com/robots.txt","bytes":1142,"groups":5},"bots":{"GPTBot":["partial","*",10,0],"ChatGPT-User":["partial","*",10,0],"OAI-SearchBot":["partial","*",10,0],"ClaudeBot":["partial","*",10,0],"Claude-User":["partial","*",10,0],"Claude-SearchBot":["partial","*",10,0],"anthropic-ai":["partial","*",10,0],"PerplexityBot":["partial","*",10,0],"Perplexity-User":["partial","*",10,0],"Google-Extended":["partial","*",10,0],"Applebot-Extended":["partial","*",10,0],"CCBot":["partial","*",10,0],"Bytespider":["partial","*",10,0],"Amazonbot":["partial","*",10,0],"meta-externalagent":["partial","*",10,0],"cohere-ai":["partial","*",10,0],"Diffbot":["partial","*",10,0],"ContextFetchBot":["partial","*",10,0]},"llms":{"status":"present","http":200,"bytes":22165},"home":{"status":"ok","http":200,"url":"https://www.brevo.com/","html_tokens":137677,"markdown_tokens":310,"pruned_tokens":102}},
{"rank":3139,"domain":"uni-ulm.de","origin":"https://uni-ulm.de","robots":{"status":"ok","http":200,"url":"https://www.uni-ulm.de/robots.txt","bytes":32,"groups":1},"bots":{"GPTBot":["partial","*",1,0],"ChatGPT-User":["partial","*",1,0],"OAI-SearchBot":["partial","*",1,0],"ClaudeBot":["partial","*",1,0],"Claude-User":["partial","*",1,0],"Claude-SearchBot":["partial","*",1,0],"anthropic-ai":["partial","*",1,0],"PerplexityBot":["partial","*",1,0],"Perplexity-User":["partial","*",1,0],"Google-Extended":["partial","*",1,0],"Applebot-Extended":["partial","*",1,0],"CCBot":["partial","*",1,0],"Bytespider":["partial","*",1,0],"Amazonbot":["partial","*",1,0],"meta-externalagent":["partial","*",1,0],"cohere-ai":["partial","*",1,0],"Diffbot":["partial","*",1,0],"ContextFetchBot":["partial","*",1,0]},"llms":{"status":"missing","http":404,"bytes":8827},"home":{"status":"ok","http":200,"url":"https://www.uni-ulm.de/","html_tokens":21025,"markdown_tokens":773,"pruned_tokens":76}},
{"rank":3140,"domain":"sephora.com","origin":"https://sephora.com","robots":{"status":"refused","http":403,"url":"https://www.sephora.com/robots.txt","bytes":379,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":3141,"domain":"pagesix.com","origin":"https://pagesix.com","robots":{"status":"ok","http":200,"url":"https://pagesix.com/robots.txt","bytes":6626,"groups":65},"bots":{"GPTBot":["partial","*",92,2],"ChatGPT-User":["partial","*",92,2],"OAI-SearchBot":["partial","*",92,2],"ClaudeBot":["blocked","named",1,0],"Claude-User":["blocked","named",1,0],"Claude-SearchBot":["blocked","named",1,0],"anthropic-ai":["blocked","named",2,0],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["partial","*",92,2],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",3,0],"cohere-ai":["blocked","named",2,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",92,2]},"llms":{"status":"missing","http":404,"bytes":376436},"home":{"status":"ok","http":200,"url":"https://pagesix.com/","html_tokens":210243,"markdown_tokens":381,"pruned_tokens":69}},
{"rank":3142,"domain":"opendemocracy.net","origin":"https://opendemocracy.net","robots":{"status":"ok","http":200,"url":"https://www.opendemocracy.net/robots.txt","bytes":220,"groups":1},"bots":{"GPTBot":["partial","*",6,0],"ChatGPT-User":["partial","*",6,0],"OAI-SearchBot":["partial","*",6,0],"ClaudeBot":["partial","*",6,0],"Claude-User":["partial","*",6,0],"Claude-SearchBot":["partial","*",6,0],"anthropic-ai":["partial","*",6,0],"PerplexityBot":["partial","*",6,0],"Perplexity-User":["partial","*",6,0],"Google-Extended":["partial","*",6,0],"Applebot-Extended":["partial","*",6,0],"CCBot":["partial","*",6,0],"Bytespider":["partial","*",6,0],"Amazonbot":["partial","*",6,0],"meta-externalagent":["partial","*",6,0],"cohere-ai":["partial","*",6,0],"Diffbot":["partial","*",6,0],"ContextFetchBot":["partial","*",6,0]},"llms":{"status":"present","http":200,"bytes":51116},"home":{"status":"ok","http":200,"url":"https://www.opendemocracy.net/","html_tokens":63503,"markdown_tokens":80,"pruned_tokens":80}},
{"rank":3143,"domain":"svt.se","origin":"https://svt.se","robots":{"status":"ok","http":200,"url":"https://www.svt.se/robots.txt","bytes":7294,"groups":58},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["allowed","named",0,1],"OAI-SearchBot":["allowed","named",0,1],"ClaudeBot":["blocked","named",1,0],"Claude-User":["allowed","named",0,1],"Claude-SearchBot":["allowed","named",0,1],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["allowed","named",0,1],"Perplexity-User":["allowed","*",0,1],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["allowed","*",0,1]},"llms":{"status":"missing","http":404,"bytes":133496},"home":{"status":"ok","http":200,"url":"https://www.svt.se/","html_tokens":124021,"markdown_tokens":54,"pruned_tokens":21}},
{"rank":3144,"domain":"fitbit.com","origin":"https://fitbit.com","robots":{"status":"ok","http":200,"url":"https://www.fitbit.com/robots.txt","bytes":322,"groups":1},"bots":{"GPTBot":["partial","*",14,0],"ChatGPT-User":["partial","*",14,0],"OAI-SearchBot":["partial","*",14,0],"ClaudeBot":["partial","*",14,0],"Claude-User":["partial","*",14,0],"Claude-SearchBot":["partial","*",14,0],"anthropic-ai":["partial","*",14,0],"PerplexityBot":["partial","*",14,0],"Perplexity-User":["partial","*",14,0],"Google-Extended":["partial","*",14,0],"Applebot-Extended":["partial","*",14,0],"CCBot":["partial","*",14,0],"Bytespider":["partial","*",14,0],"Amazonbot":["partial","*",14,0],"meta-externalagent":["partial","*",14,0],"cohere-ai":["partial","*",14,0],"Diffbot":["partial","*",14,0],"ContextFetchBot":["partial","*",14,0]},"llms":{"status":"missing","http":404,"bytes":782},"home":{"status":"ok","http":200,"url":"https://store.google.com/category/watches_trackers?hl=en-US&utm_source=fitbit_redirect&utm_medium=google_ooo&utm_campaign=category","html_tokens":749430,"markdown_tokens":197,"pruned_tokens":151}},
{"rank":3145,"domain":"meduza.io","origin":"https://meduza.io","robots":{"status":"ok","http":200,"url":"https://meduza.io/robots.txt","bytes":75,"groups":1},"bots":{"GPTBot":["partial","*",2,0],"ChatGPT-User":["partial","*",2,0],"OAI-SearchBot":["partial","*",2,0],"ClaudeBot":["partial","*",2,0],"Claude-User":["partial","*",2,0],"Claude-SearchBot":["partial","*",2,0],"anthropic-ai":["partial","*",2,0],"PerplexityBot":["partial","*",2,0],"Perplexity-User":["partial","*",2,0],"Google-Extended":["partial","*",2,0],"Applebot-Extended":["partial","*",2,0],"CCBot":["partial","*",2,0],"Bytespider":["partial","*",2,0],"Amazonbot":["partial","*",2,0],"meta-externalagent":["partial","*",2,0],"cohere-ai":["partial","*",2,0],"Diffbot":["partial","*",2,0],"ContextFetchBot":["partial","*",2,0]},"llms":{"status":"http_error","http":502,"bytes":6372},"home":{"status":"ok","http":200,"url":"https://meduza.io/","html_tokens":80633,"markdown_tokens":256,"pruned_tokens":170}},
{"rank":3146,"domain":"hardrock.com","origin":"https://hardrock.com","robots":{"status":"ok","http":200,"url":"https://www.hardrock.com/robots.txt","bytes":277,"groups":1},"bots":{"GPTBot":["allowed","*",0,4],"ChatGPT-User":["allowed","*",0,4],"OAI-SearchBot":["allowed","*",0,4],"ClaudeBot":["allowed","*",0,4],"Claude-User":["allowed","*",0,4],"Claude-SearchBot":["allowed","*",0,4],"anthropic-ai":["allowed","*",0,4],"PerplexityBot":["allowed","*",0,4],"Perplexity-User":["allowed","*",0,4],"Google-Extended":["allowed","*",0,4],"Applebot-Extended":["allowed","*",0,4],"CCBot":["allowed","*",0,4],"Bytespider":["allowed","*",0,4],"Amazonbot":["allowed","*",0,4],"meta-externalagent":["allowed","*",0,4],"cohere-ai":["allowed","*",0,4],"Diffbot":["allowed","*",0,4],"ContextFetchBot":["allowed","*",0,4]},"llms":{"status":"missing","http":404,"bytes":54444},"home":{"status":"ok","http":200,"url":"https://www.hardrock.com/","html_tokens":35711,"markdown_tokens":2399,"pruned_tokens":369}},
{"rank":3147,"domain":"google.co.id","origin":"https://google.co.id","robots":{"status":"ok","http":200,"url":"https://www.google.co.id/robots.txt","bytes":6624,"groups":4},"bots":{"GPTBot":["partial","*",169,64],"ChatGPT-User":["partial","*",169,64],"OAI-SearchBot":["partial","*",169,64],"ClaudeBot":["partial","*",169,64],"Claude-User":["partial","*",169,64],"Claude-SearchBot":["partial","*",169,64],"anthropic-ai":["partial","*",169,64],"PerplexityBot":["partial","*",169,64],"Perplexity-User":["partial","*",169,64],"Google-Extended":["partial","*",169,64],"Applebot-Extended":["partial","*",169,64],"CCBot":["partial","*",169,64],"Bytespider":["partial","*",169,64],"Amazonbot":["partial","*",169,64],"meta-externalagent":["partial","*",169,64],"cohere-ai":["partial","*",169,64],"Diffbot":["partial","*",169,64],"ContextFetchBot":["partial","*",169,64]},"llms":{"status":"missing","http":404,"bytes":1569},"home":{"status":"ok","http":200,"url":"https://www.google.com/","html_tokens":21309,"markdown_tokens":30,"pruned_tokens":30}},
{"rank":3148,"domain":"parade.com","origin":"https://parade.com","robots":{"status":"ok","http":200,"url":"https://parade.com/robots.txt","bytes":229,"groups":1},"bots":{"GPTBot":["partial","*",6,0],"ChatGPT-User":["partial","*",6,0],"OAI-SearchBot":["partial","*",6,0],"ClaudeBot":["partial","*",6,0],"Claude-User":["partial","*",6,0],"Claude-SearchBot":["partial","*",6,0],"anthropic-ai":["partial","*",6,0],"PerplexityBot":["partial","*",6,0],"Perplexity-User":["partial","*",6,0],"Google-Extended":["partial","*",6,0],"Applebot-Extended":["partial","*",6,0],"CCBot":["partial","*",6,0],"Bytespider":["partial","*",6,0],"Amazonbot":["partial","*",6,0],"meta-externalagent":["partial","*",6,0],"cohere-ai":["partial","*",6,0],"Diffbot":["partial","*",6,0],"ContextFetchBot":["partial","*",6,0]},"llms":{"status":"http_error","http":403,"bytes":770},"home":{"status":"http_error","http":403,"url":"https://parade.com/"}},
{"rank":3149,"domain":"taylorandfrancis.com","origin":"https://taylorandfrancis.com","robots":{"status":"ok","http":200,"url":"https://taylorandfrancis.com/robots.txt","bytes":132,"groups":1},"bots":{"GPTBot":["partial","*",2,1],"ChatGPT-User":["partial","*",2,1],"OAI-SearchBot":["partial","*",2,1],"ClaudeBot":["partial","*",2,1],"Claude-User":["partial","*",2,1],"Claude-SearchBot":["partial","*",2,1],"anthropic-ai":["partial","*",2,1],"PerplexityBot":["partial","*",2,1],"Perplexity-User":["partial","*",2,1],"Google-Extended":["partial","*",2,1],"Applebot-Extended":["partial","*",2,1],"CCBot":["partial","*",2,1],"Bytespider":["partial","*",2,1],"Amazonbot":["partial","*",2,1],"meta-externalagent":["partial","*",2,1],"cohere-ai":["partial","*",2,1],"Diffbot":["partial","*",2,1],"ContextFetchBot":["partial","*",2,1]},"llms":{"status":"http_error","http":403,"bytes":6264},"home":{"status":"ok","http":200,"url":"https://taylorandfrancis.com/","html_tokens":37753,"markdown_tokens":88,"pruned_tokens":76}},
{"rank":3150,"domain":"regulations.gov","origin":"https://regulations.gov","robots":{"status":"refused","http":403,"url":"https://www.regulations.gov/robots.txt","bytes":919,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":3151,"domain":"etherscan.io","origin":"https://etherscan.io","robots":{"status":"ok","http":200,"url":"https://etherscan.io/robots.txt","bytes":514,"groups":1},"bots":{"GPTBot":["partial","*",8,0],"ChatGPT-User":["partial","*",8,0],"OAI-SearchBot":["partial","*",8,0],"ClaudeBot":["partial","*",8,0],"Claude-User":["partial","*",8,0],"Claude-SearchBot":["partial","*",8,0],"anthropic-ai":["partial","*",8,0],"PerplexityBot":["partial","*",8,0],"Perplexity-User":["partial","*",8,0],"Google-Extended":["partial","*",8,0],"Applebot-Extended":["partial","*",8,0],"CCBot":["partial","*",8,0],"Bytespider":["partial","*",8,0],"Amazonbot":["partial","*",8,0],"meta-externalagent":["partial","*",8,0],"cohere-ai":["partial","*",8,0],"Diffbot":["partial","*",8,0],"ContextFetchBot":["partial","*",8,0]},"llms":{"status":"present","http":200,"bytes":11020},"home":{"status":"ok","http":200,"url":"https://etherscan.io/","html_tokens":35269,"markdown_tokens":45,"pruned_tokens":45}},
{"rank":3152,"domain":"theinformation.com","origin":"https://theinformation.com","robots":{"status":"refused","http":403,"url":"https://theinformation.com/robots.txt","bytes":4912,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":3153,"domain":"alamy.com","origin":"https://alamy.com","robots":{"status":"ok","http":200,"url":"https://www.alamy.com/robots.txt","bytes":11791,"groups":3},"bots":{"GPTBot":["partial","named",555,13],"ChatGPT-User":["partial","*",548,13],"OAI-SearchBot":["partial","*",548,13],"ClaudeBot":["partial","named",555,13],"Claude-User":["partial","*",548,13],"Claude-SearchBot":["partial","*",548,13],"anthropic-ai":["partial","*",548,13],"PerplexityBot":["partial","named",548,13],"Perplexity-User":["partial","named",548,13],"Google-Extended":["partial","*",548,13],"Applebot-Extended":["partial","named",548,13],"CCBot":["partial","named",548,13],"Bytespider":["partial","named",548,13],"Amazonbot":["partial","*",548,13],"meta-externalagent":["partial","named",548,13],"cohere-ai":["partial","*",548,13],"Diffbot":["partial","*",548,13],"ContextFetchBot":["partial","*",548,13]},"llms":{"status":"missing","http":404,"bytes":229323},"home":{"status":"ok","http":200,"url":"https://www.alamy.com/","html_tokens":106949,"markdown_tokens":42,"pruned_tokens":42}},
{"rank":3154,"domain":"sparkfun.com","origin":"https://sparkfun.com","robots":{"status":"ok","http":200,"url":"https://www.sparkfun.com/robots.txt","bytes":4791,"groups":13},"bots":{"GPTBot":["partial","*",74,1],"ChatGPT-User":["partial","*",74,1],"OAI-SearchBot":["partial","*",74,1],"ClaudeBot":["partial","*",74,1],"Claude-User":["partial","*",74,1],"Claude-SearchBot":["partial","*",74,1],"anthropic-ai":["partial","*",74,1],"PerplexityBot":["partial","*",74,1],"Perplexity-User":["partial","*",74,1],"Google-Extended":["partial","*",74,1],"Applebot-Extended":["partial","*",74,1],"CCBot":["partial","*",74,1],"Bytespider":["partial","*",74,1],"Amazonbot":["partial","*",74,1],"meta-externalagent":["partial","named",75,1],"cohere-ai":["partial","*",74,1],"Diffbot":["partial","*",74,1],"ContextFetchBot":["partial","*",74,1]},"llms":{"status":"missing","http":404,"bytes":645160},"home":{"status":"ok","http":200,"url":"https://www.sparkfun.com/","html_tokens":210126,"markdown_tokens":212,"pruned_tokens":56}},
{"rank":3155,"domain":"plosone.org","origin":"https://plosone.org","robots":{"status":"html","http":200,"url":"https://journals.plos.org/plosone/","bytes":71780,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":3156,"domain":"pewinternet.org","origin":"https://pewinternet.org","robots":{"status":"missing","http":404,"url":"https://www.pewresearch.org/internet/robots.txt","bytes":287679,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":287673},"home":{"status":"ok","http":200,"url":"https://www.pewresearch.org/topic/internet-technology/","html_tokens":139576,"markdown_tokens":213,"pruned_tokens":166}},
{"rank":3157,"domain":"houstonchronicle.com","origin":"https://houstonchronicle.com","robots":{"status":"ok","http":200,"url":"https://www.houstonchronicle.com/robots.txt","bytes":1197,"groups":3},"bots":{"GPTBot":["partial","*",10,0],"ChatGPT-User":["partial","*",10,0],"OAI-SearchBot":["partial","*",10,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["blocked","named",1,0],"Claude-SearchBot":["partial","*",10,0],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["blocked","named",1,0],"Google-Extended":["partial","*",10,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["partial","*",10,0],"meta-externalagent":["partial","*",10,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",10,0]},"llms":{"status":"missing","http":404,"bytes":61875},"home":{"status":"ok","http":200,"url":"https://www.houstonchronicle.com/","html_tokens":283177,"markdown_tokens":24,"pruned_tokens":24}},
{"rank":3158,"domain":"sante.fr","origin":"https://sante.fr","robots":{"status":"ok","http":200,"url":"https://sante.fr/robots.txt","bytes":81,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":280},"home":{"status":"ok","http":200,"url":"https://www.sante.fr/","html_tokens":54881,"markdown_tokens":785,"pruned_tokens":235}},
{"rank":3159,"domain":"omny.fm","origin":"https://omny.fm","robots":{"status":"ok","http":200,"url":"https://omny.fm/robots.txt","bytes":22,"groups":1},"bots":{"GPTBot":["allowed","*",0,1],"ChatGPT-User":["allowed","*",0,1],"OAI-SearchBot":["allowed","*",0,1],"ClaudeBot":["allowed","*",0,1],"Claude-User":["allowed","*",0,1],"Claude-SearchBot":["allowed","*",0,1],"anthropic-ai":["allowed","*",0,1],"PerplexityBot":["allowed","*",0,1],"Perplexity-User":["allowed","*",0,1],"Google-Extended":["allowed","*",0,1],"Applebot-Extended":["allowed","*",0,1],"CCBot":["allowed","*",0,1],"Bytespider":["allowed","*",0,1],"Amazonbot":["allowed","*",0,1],"meta-externalagent":["allowed","*",0,1],"cohere-ai":["allowed","*",0,1],"Diffbot":["allowed","*",0,1],"ContextFetchBot":["allowed","*",0,1]},"llms":{"status":"missing","http":404,"bytes":10554},"home":{"status":"ok","http":200,"url":"https://omny.fm/","html_tokens":6029,"markdown_tokens":17,"pruned_tokens":17}},
{"rank":3160,"domain":"aao.org","origin":"https://aao.org","robots":{"status":"ok","http":200,"url":"https://www.aao.org/robots.txt","bytes":656,"groups":2},"bots":{"GPTBot":["partial","*",7,0],"ChatGPT-User":["partial","*",7,0],"OAI-SearchBot":["partial","*",7,0],"ClaudeBot":["partial","*",7,0],"Claude-User":["partial","*",7,0],"Claude-SearchBot":["partial","*",7,0],"anthropic-ai":["partial","*",7,0],"PerplexityBot":["partial","*",7,0],"Perplexity-User":["partial","*",7,0],"Google-Extended":["partial","*",7,0],"Applebot-Extended":["partial","*",7,0],"CCBot":["partial","*",7,0],"Bytespider":["partial","*",7,0],"Amazonbot":["partial","*",7,0],"meta-externalagent":["partial","*",7,0],"cohere-ai":["partial","*",7,0],"Diffbot":["partial","*",7,0],"ContextFetchBot":["partial","*",7,0]},"llms":{"status":"present","http":200,"bytes":9828},"home":{"status":"ok","http":200,"url":"https://www.aao.org/","html_tokens":29019,"markdown_tokens":130,"pruned_tokens":130}},
{"rank":3161,"domain":"gu.se","origin":"https://gu.se","robots":{"status":"ok","http":200,"url":"https://www.gu.se/robots.txt","bytes":2045,"groups":1},"bots":{"GPTBot":["partial","*",33,18],"ChatGPT-User":["partial","*",33,18],"OAI-SearchBot":["partial","*",33,18],"ClaudeBot":["partial","*",33,18],"Claude-User":["partial","*",33,18],"Claude-SearchBot":["partial","*",33,18],"anthropic-ai":["partial","*",33,18],"PerplexityBot":["partial","*",33,18],"Perplexity-User":["partial","*",33,18],"Google-Extended":["partial","*",33,18],"Applebot-Extended":["partial","*",33,18],"CCBot":["partial","*",33,18],"Bytespider":["partial","*",33,18],"Amazonbot":["partial","*",33,18],"meta-externalagent":["partial","*",33,18],"cohere-ai":["partial","*",33,18],"Diffbot":["partial","*",33,18],"ContextFetchBot":["partial","*",33,18]},"llms":{"status":"http_error","http":403,"bytes":153},"home":{"status":"ok","http":200,"url":"https://www.gu.se/","html_tokens":53277,"markdown_tokens":564,"pruned_tokens":162}},
{"rank":3162,"domain":"filmfreeway.com","origin":"https://filmfreeway.com","robots":{"status":"ok","http":200,"url":"https://filmfreeway.com/robots.txt","bytes":494,"groups":1},"bots":{"GPTBot":["partial","*",7,0],"ChatGPT-User":["partial","*",7,0],"OAI-SearchBot":["partial","*",7,0],"ClaudeBot":["partial","*",7,0],"Claude-User":["partial","*",7,0],"Claude-SearchBot":["partial","*",7,0],"anthropic-ai":["partial","*",7,0],"PerplexityBot":["partial","*",7,0],"Perplexity-User":["partial","*",7,0],"Google-Extended":["partial","*",7,0],"Applebot-Extended":["partial","*",7,0],"CCBot":["partial","*",7,0],"Bytespider":["partial","*",7,0],"Amazonbot":["partial","*",7,0],"meta-externalagent":["partial","*",7,0],"cohere-ai":["partial","*",7,0],"Diffbot":["partial","*",7,0],"ContextFetchBot":["partial","*",7,0]},"llms":{"status":"missing","http":404,"bytes":3079},"home":{"status":"http_error","http":403,"url":"https://filmfreeway.com/"}},
{"rank":3163,"domain":"nordstrom.com","origin":"https://nordstrom.com","robots":{"status":"ok","http":200,"url":"https://www.nordstrom.com/robots.txt","bytes":2685,"groups":6},"bots":{"GPTBot":["partial","*",45,3],"ChatGPT-User":["partial","*",45,3],"OAI-SearchBot":["partial","*",45,3],"ClaudeBot":["partial","*",45,3],"Claude-User":["partial","*",45,3],"Claude-SearchBot":["partial","*",45,3],"anthropic-ai":["partial","*",45,3],"PerplexityBot":["partial","*",45,3],"Perplexity-User":["partial","*",45,3],"Google-Extended":["partial","*",45,3],"Applebot-Extended":["partial","*",45,3],"CCBot":["partial","*",45,3],"Bytespider":["partial","*",45,3],"Amazonbot":["partial","*",45,3],"meta-externalagent":["partial","named",1,0],"cohere-ai":["partial","*",45,3],"Diffbot":["partial","*",45,3],"ContextFetchBot":["partial","*",45,3]},"llms":{"status":"missing","http":404,"bytes":305915},"home":{"status":"ok","http":200,"url":"https://www.nordstrom.com/","html_tokens":64039,"markdown_tokens":0,"pruned_tokens":0}},
{"rank":3164,"domain":"cs.wikipedia.org","origin":"https://cs.wikipedia.org","robots":{"status":"ok","http":200,"url":"https://cs.wikipedia.org/robots.txt","bytes":15253,"groups":33},"bots":{"GPTBot":["partial","*",200,4],"ChatGPT-User":["partial","*",200,4],"OAI-SearchBot":["partial","*",200,4],"ClaudeBot":["partial","*",200,4],"Claude-User":["partial","*",200,4],"Claude-SearchBot":["partial","*",200,4],"anthropic-ai":["partial","*",200,4],"PerplexityBot":["partial","*",200,4],"Perplexity-User":["partial","*",200,4],"Google-Extended":["partial","*",200,4],"Applebot-Extended":["partial","*",200,4],"CCBot":["partial","*",200,4],"Bytespider":["partial","*",200,4],"Amazonbot":["partial","*",200,4],"meta-externalagent":["partial","*",200,4],"cohere-ai":["partial","*",200,4],"Diffbot":["partial","*",200,4],"ContextFetchBot":["partial","*",200,4]},"llms":{"status":"missing","http":404,"bytes":1868},"home":{"status":"ok","http":200,"url":"https://cs.wikipedia.org/wiki/Hlavn%C3%AD_strana","html_tokens":63282,"markdown_tokens":192,"pruned_tokens":57}},
{"rank":3165,"domain":"syfy.com","origin":"https://syfy.com","robots":{"status":"ok","http":200,"url":"https://www.syfy.com/robots.txt","bytes":3189,"groups":23},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["partial","*",31,19],"OAI-SearchBot":["partial","*",31,19],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",31,19],"Claude-SearchBot":["partial","*",31,19],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["partial","*",31,19],"Perplexity-User":["partial","*",31,19],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["partial","*",31,19],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",31,19]},"llms":{"status":"missing","http":404,"bytes":186},"home":{"status":"ok","http":200,"url":"https://www.syfy.com/","html_tokens":26105,"markdown_tokens":5,"pruned_tokens":5}},
{"rank":3166,"domain":"pixlr.com","origin":"https://pixlr.com","robots":{"status":"ok","http":200,"url":"https://pixlr.com/robots.txt","bytes":121,"groups":1},"bots":{"GPTBot":["partial","*",3,1],"ChatGPT-User":["partial","*",3,1],"OAI-SearchBot":["partial","*",3,1],"ClaudeBot":["partial","*",3,1],"Claude-User":["partial","*",3,1],"Claude-SearchBot":["partial","*",3,1],"anthropic-ai":["partial","*",3,1],"PerplexityBot":["partial","*",3,1],"Perplexity-User":["partial","*",3,1],"Google-Extended":["partial","*",3,1],"Applebot-Extended":["partial","*",3,1],"CCBot":["partial","*",3,1],"Bytespider":["partial","*",3,1],"Amazonbot":["partial","*",3,1],"meta-externalagent":["partial","*",3,1],"cohere-ai":["partial","*",3,1],"Diffbot":["partial","*",3,1],"ContextFetchBot":["partial","*",3,1]},"llms":{"status":"missing","http":404,"bytes":344855},"home":{"status":"ok","http":200,"url":"https://pixlr.com/","html_tokens":102052,"markdown_tokens":1913,"pruned_tokens":1490}},
{"rank":3167,"domain":"iowa.gov","origin":"https://iowa.gov","robots":{"status":"ok","http":200,"url":"https://www.iowa.gov/robots.txt","bytes":2182,"groups":1},"bots":{"GPTBot":["partial","*",35,22],"ChatGPT-User":["partial","*",35,22],"OAI-SearchBot":["partial","*",35,22],"ClaudeBot":["partial","*",35,22],"Claude-User":["partial","*",35,22],"Claude-SearchBot":["partial","*",35,22],"anthropic-ai":["partial","*",35,22],"PerplexityBot":["partial","*",35,22],"Perplexity-User":["partial","*",35,22],"Google-Extended":["partial","*",35,22],"Applebot-Extended":["partial","*",35,22],"CCBot":["partial","*",35,22],"Bytespider":["partial","*",35,22],"Amazonbot":["partial","*",35,22],"meta-externalagent":["partial","*",35,22],"cohere-ai":["partial","*",35,22],"Diffbot":["partial","*",35,22],"ContextFetchBot":["partial","*",35,22]},"llms":{"status":"missing","http":404,"bytes":186},"home":{"status":"ok","http":200,"url":"https://www.iowa.gov/","html_tokens":43744,"markdown_tokens":171,"pruned_tokens":171}},
{"rank":3168,"domain":"crowdstrike.com","origin":"https://crowdstrike.com","robots":{"status":"ok","http":200,"url":"https://www.crowdstrike.com/robots.txt","bytes":1822,"groups":1},"bots":{"GPTBot":["partial","*",28,1],"ChatGPT-User":["partial","*",28,1],"OAI-SearchBot":["partial","*",28,1],"ClaudeBot":["partial","*",28,1],"Claude-User":["partial","*",28,1],"Claude-SearchBot":["partial","*",28,1],"anthropic-ai":["partial","*",28,1],"PerplexityBot":["partial","*",28,1],"Perplexity-User":["partial","*",28,1],"Google-Extended":["partial","*",28,1],"Applebot-Extended":["partial","*",28,1],"CCBot":["partial","*",28,1],"Bytespider":["partial","*",28,1],"Amazonbot":["partial","*",28,1],"meta-externalagent":["partial","*",28,1],"cohere-ai":["partial","*",28,1],"Diffbot":["partial","*",28,1],"ContextFetchBot":["partial","*",28,1]},"llms":{"status":"present","http":200,"bytes":3248},"home":{"status":"ok","http":200,"url":"https://www.crowdstrike.com/en-us/","html_tokens":150074,"markdown_tokens":2560,"pruned_tokens":96}},
{"rank":3169,"domain":"oraclecloud.com","origin":"https://oraclecloud.com","robots":{"status":"refused","http":403,"url":"https://www.oracle.com/cloud/","bytes":1448,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":3170,"domain":"fgov.be","origin":"https://fgov.be","robots":{"status":"html","http":200,"url":"https://fgov.be/robots.txt","bytes":47875,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":3171,"domain":"leetcode.com","origin":"https://leetcode.com","robots":{"status":"ok","http":200,"url":"https://leetcode.com/robots.txt","bytes":514,"groups":1},"bots":{"GPTBot":["partial","*",19,0],"ChatGPT-User":["partial","*",19,0],"OAI-SearchBot":["partial","*",19,0],"ClaudeBot":["partial","*",19,0],"Claude-User":["partial","*",19,0],"Claude-SearchBot":["partial","*",19,0],"anthropic-ai":["partial","*",19,0],"PerplexityBot":["partial","*",19,0],"Perplexity-User":["partial","*",19,0],"Google-Extended":["partial","*",19,0],"Applebot-Extended":["partial","*",19,0],"CCBot":["partial","*",19,0],"Bytespider":["partial","*",19,0],"Amazonbot":["partial","*",19,0],"meta-externalagent":["partial","*",19,0],"cohere-ai":["partial","*",19,0],"Diffbot":["partial","*",19,0],"ContextFetchBot":["partial","*",19,0]},"llms":{"status":"missing","http":404,"bytes":21859},"home":{"status":"http_error","http":403,"url":"https://leetcode.com/"}},
{"rank":3172,"domain":"bufferapp.com","origin":"https://bufferapp.com","robots":{"status":"ok","http":200,"url":"https://buffer.com/robots.txt","bytes":498,"groups":2},"bots":{"GPTBot":["partial","*",5,0],"ChatGPT-User":["partial","*",5,0],"OAI-SearchBot":["partial","*",5,0],"ClaudeBot":["partial","*",5,0],"Claude-User":["partial","*",5,0],"Claude-SearchBot":["partial","*",5,0],"anthropic-ai":["partial","*",5,0],"PerplexityBot":["partial","*",5,0],"Perplexity-User":["partial","*",5,0],"Google-Extended":["partial","*",5,0],"Applebot-Extended":["partial","*",5,0],"CCBot":["partial","*",5,0],"Bytespider":["partial","*",5,0],"Amazonbot":["partial","*",5,0],"meta-externalagent":["partial","*",5,0],"cohere-ai":["partial","*",5,0],"Diffbot":["partial","*",5,0],"ContextFetchBot":["partial","*",5,0]},"llms":{"status":"present","http":200,"bytes":10846},"home":{"status":"ok","http":200,"url":"https://buffer.com/","html_tokens":153803,"markdown_tokens":576,"pruned_tokens":109}},
{"rank":3173,"domain":"codeproject.com","origin":"https://codeproject.com","robots":{"status":"ok","http":200,"url":"https://codeproject.com/robots.txt","bytes":66,"groups":1},"bots":{"GPTBot":["allowed","*",0,1],"ChatGPT-User":["allowed","*",0,1],"OAI-SearchBot":["allowed","*",0,1],"ClaudeBot":["allowed","*",0,1],"Claude-User":["allowed","*",0,1],"Claude-SearchBot":["allowed","*",0,1],"anthropic-ai":["allowed","*",0,1],"PerplexityBot":["allowed","*",0,1],"Perplexity-User":["allowed","*",0,1],"Google-Extended":["allowed","*",0,1],"Applebot-Extended":["allowed","*",0,1],"CCBot":["allowed","*",0,1],"Bytespider":["allowed","*",0,1],"Amazonbot":["allowed","*",0,1],"meta-externalagent":["allowed","*",0,1],"cohere-ai":["allowed","*",0,1],"Diffbot":["allowed","*",0,1],"ContextFetchBot":["allowed","*",0,1]},"llms":{"status":"other_text","http":200,"bytes":65},"home":{"status":"ok","http":200,"url":"https://codeproject.com/","html_tokens":29,"markdown_tokens":0,"pruned_tokens":0}},
{"rank":3174,"domain":"chase.com","origin":"https://chase.com","robots":{"status":"ok","http":200,"url":"https://www.chase.com/robots.txt","bytes":8484,"groups":1},"bots":{"GPTBot":["partial","*",136,7],"ChatGPT-User":["partial","*",136,7],"OAI-SearchBot":["partial","*",136,7],"ClaudeBot":["partial","*",136,7],"Claude-User":["partial","*",136,7],"Claude-SearchBot":["partial","*",136,7],"anthropic-ai":["partial","*",136,7],"PerplexityBot":["partial","*",136,7],"Perplexity-User":["partial","*",136,7],"Google-Extended":["partial","*",136,7],"Applebot-Extended":["partial","*",136,7],"CCBot":["partial","*",136,7],"Bytespider":["partial","*",136,7],"Amazonbot":["partial","*",136,7],"meta-externalagent":["partial","*",136,7],"cohere-ai":["partial","*",136,7],"Diffbot":["partial","*",136,7],"ContextFetchBot":["partial","*",136,7]},"llms":{"status":"missing","http":404,"bytes":231677},"home":{"status":"ok","http":200,"url":"https://www.chase.com/","html_tokens":78993,"markdown_tokens":96,"pruned_tokens":96}},
{"rank":3175,"domain":"allafrica.com","origin":"https://allafrica.com","robots":{"status":"ok","http":200,"url":"https://allafrica.com/robots.txt","bytes":1365,"groups":5},"bots":{"GPTBot":["partial","*",22,0],"ChatGPT-User":["partial","*",22,0],"OAI-SearchBot":["partial","*",22,0],"ClaudeBot":["partial","*",22,0],"Claude-User":["partial","*",22,0],"Claude-SearchBot":["partial","*",22,0],"anthropic-ai":["partial","*",22,0],"PerplexityBot":["partial","*",22,0],"Perplexity-User":["partial","*",22,0],"Google-Extended":["partial","*",22,0],"Applebot-Extended":["partial","*",22,0],"CCBot":["partial","*",22,0],"Bytespider":["partial","*",22,0],"Amazonbot":["partial","*",22,0],"meta-externalagent":["partial","*",22,0],"cohere-ai":["partial","*",22,0],"Diffbot":["partial","*",22,0],"ContextFetchBot":["partial","*",22,0]},"llms":{"status":"missing","http":404,"bytes":71268},"home":{"status":"ok","http":200,"url":"https://allafrica.com/","html_tokens":66088,"markdown_tokens":153,"pruned_tokens":42}},
{"rank":3176,"domain":"jus.br","origin":"https://www.jus.br","robots":{"status":"html","http":200,"url":"https://www.jus.br/robots.txt","bytes":28105,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":3177,"domain":"flagcounter.com","origin":"https://flagcounter.com","robots":{"status":"ok","http":200,"url":"https://flagcounter.com/robots.txt","bytes":547,"groups":1},"bots":{"GPTBot":["partial","*",23,0],"ChatGPT-User":["partial","*",23,0],"OAI-SearchBot":["partial","*",23,0],"ClaudeBot":["partial","*",23,0],"Claude-User":["partial","*",23,0],"Claude-SearchBot":["partial","*",23,0],"anthropic-ai":["partial","*",23,0],"PerplexityBot":["partial","*",23,0],"Perplexity-User":["partial","*",23,0],"Google-Extended":["partial","*",23,0],"Applebot-Extended":["partial","*",23,0],"CCBot":["partial","*",23,0],"Bytespider":["partial","*",23,0],"Amazonbot":["partial","*",23,0],"meta-externalagent":["partial","*",23,0],"cohere-ai":["partial","*",23,0],"Diffbot":["partial","*",23,0],"ContextFetchBot":["partial","*",23,0]},"llms":{"status":"missing","http":404,"bytes":196},"home":{"status":"ok","http":200,"url":"https://flagcounter.com/","html_tokens":6202,"markdown_tokens":128,"pruned_tokens":128}},
{"rank":3178,"domain":"newsmax.com","origin":"https://newsmax.com","robots":{"status":"unreachable","http":null,"url":"https://newsmax.com/robots.txt","bytes":0,"groups":0,"error":"<StreamReset stream_id:1, error_code:2, remote_reset:True>"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":3179,"domain":"sans.org","origin":"https://sans.org","robots":{"status":"ok","http":200,"url":"https://www.sans.org/robots.txt","bytes":9808,"groups":1},"bots":{"GPTBot":["partial","*",235,0],"ChatGPT-User":["partial","*",235,0],"OAI-SearchBot":["partial","*",235,0],"ClaudeBot":["partial","*",235,0],"Claude-User":["partial","*",235,0],"Claude-SearchBot":["partial","*",235,0],"anthropic-ai":["partial","*",235,0],"PerplexityBot":["partial","*",235,0],"Perplexity-User":["partial","*",235,0],"Google-Extended":["partial","*",235,0],"Applebot-Extended":["partial","*",235,0],"CCBot":["partial","*",235,0],"Bytespider":["partial","*",235,0],"Amazonbot":["partial","*",235,0],"meta-externalagent":["partial","*",235,0],"cohere-ai":["partial","*",235,0],"Diffbot":["partial","*",235,0],"ContextFetchBot":["partial","*",235,0]},"llms":{"status":"http_error","http":500,"bytes":52375},"home":{"status":"ok","http":200,"url":"https://www.sans.org/","html_tokens":128295,"markdown_tokens":1556,"pruned_tokens":333}},
{"rank":3180,"domain":"estadao.com.br","origin":"https://estadao.com.br","robots":{"status":"refused","http":403,"url":"https://estadao.com.br/robots.txt","bytes":382,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":3181,"domain":"wpml.org","origin":"https://wpml.org","robots":{"status":"ok","http":200,"url":"https://wpml.org/robots.txt","bytes":81,"groups":1},"bots":{"GPTBot":["partial","*",1,1],"ChatGPT-User":["partial","*",1,1],"OAI-SearchBot":["partial","*",1,1],"ClaudeBot":["partial","*",1,1],"Claude-User":["partial","*",1,1],"Claude-SearchBot":["partial","*",1,1],"anthropic-ai":["partial","*",1,1],"PerplexityBot":["partial","*",1,1],"Perplexity-User":["partial","*",1,1],"Google-Extended":["partial","*",1,1],"Applebot-Extended":["partial","*",1,1],"CCBot":["partial","*",1,1],"Bytespider":["partial","*",1,1],"Amazonbot":["partial","*",1,1],"meta-externalagent":["partial","*",1,1],"cohere-ai":["partial","*",1,1],"Diffbot":["partial","*",1,1],"ContextFetchBot":["partial","*",1,1]},"llms":{"status":"missing","http":404,"bytes":108222},"home":{"status":"ok","http":200,"url":"https://wpml.org/","html_tokens":23678,"markdown_tokens":430,"pruned_tokens":430}},
{"rank":3182,"domain":"lexology.com","origin":"https://lexology.com","robots":{"status":"refused","http":403,"url":"https://lexology.com/robots.txt","bytes":5550,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":3183,"domain":"rediff.com","origin":"https://rediff.com","robots":{"status":"ok","http":200,"url":"http://www.rediff.com/robots.txt","bytes":608,"groups":1},"bots":{"GPTBot":["partial","*",5,0],"ChatGPT-User":["partial","*",5,0],"OAI-SearchBot":["partial","*",5,0],"ClaudeBot":["partial","*",5,0],"Claude-User":["partial","*",5,0],"Claude-SearchBot":["partial","*",5,0],"anthropic-ai":["partial","*",5,0],"PerplexityBot":["partial","*",5,0],"Perplexity-User":["partial","*",5,0],"Google-Extended":["partial","*",5,0],"Applebot-Extended":["partial","*",5,0],"CCBot":["partial","*",5,0],"Bytespider":["partial","*",5,0],"Amazonbot":["partial","*",5,0],"meta-externalagent":["partial","*",5,0],"cohere-ai":["partial","*",5,0],"Diffbot":["partial","*",5,0],"ContextFetchBot":["partial","*",5,0]},"llms":{"status":"present","http":200,"bytes":6838},"home":{"status":"ok","http":200,"url":"https://www.rediff.com/","html_tokens":36179,"markdown_tokens":20,"pruned_tokens":20}},
{"rank":3184,"domain":"themoviedb.org","origin":"https://themoviedb.org","robots":{"status":"ok","http":200,"url":"https://www.themoviedb.org/robots.txt","bytes":2382,"groups":1},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["blocked","named",1,0],"OAI-SearchBot":["blocked","named",1,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["blocked","named",1,0],"Claude-SearchBot":["blocked","named",1,0],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["blocked","named",1,0],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":83319},"home":{"status":"ok","http":200,"url":"https://www.themoviedb.org/","html_tokens":30526,"markdown_tokens":13,"pruned_tokens":13}},
{"rank":3185,"domain":"cybernews.com","origin":"https://cybernews.com","robots":{"status":"ok","http":200,"url":"https://cybernews.com/robots.txt","bytes":795,"groups":4},"bots":{"GPTBot":["partial","*",6,1],"ChatGPT-User":["partial","*",6,1],"OAI-SearchBot":["partial","*",6,1],"ClaudeBot":["partial","*",6,1],"Claude-User":["partial","*",6,1],"Claude-SearchBot":["partial","*",6,1],"anthropic-ai":["partial","*",6,1],"PerplexityBot":["partial","*",6,1],"Perplexity-User":["partial","*",6,1],"Google-Extended":["partial","*",6,1],"Applebot-Extended":["partial","*",6,1],"CCBot":["partial","*",6,1],"Bytespider":["blocked","named",1,0],"Amazonbot":["partial","*",6,1],"meta-externalagent":["partial","*",6,1],"cohere-ai":["partial","*",6,1],"Diffbot":["partial","*",6,1],"ContextFetchBot":["partial","*",6,1]},"llms":{"status":"http_error","http":403,"bytes":5553},"home":{"status":"http_error","http":403,"url":"https://cybernews.com/"}},
{"rank":3186,"domain":"bgr.com","origin":"https://bgr.com","robots":{"status":"ok","http":200,"url":"https://bgr.com/robots.txt","bytes":245,"groups":2},"bots":{"GPTBot":["partial","*",4,0],"ChatGPT-User":["partial","*",4,0],"OAI-SearchBot":["partial","*",4,0],"ClaudeBot":["partial","*",4,0],"Claude-User":["partial","*",4,0],"Claude-SearchBot":["partial","*",4,0],"anthropic-ai":["partial","*",4,0],"PerplexityBot":["partial","*",4,0],"Perplexity-User":["partial","*",4,0],"Google-Extended":["partial","*",4,0],"Applebot-Extended":["partial","*",4,0],"CCBot":["partial","*",4,0],"Bytespider":["partial","*",4,0],"Amazonbot":["partial","*",4,0],"meta-externalagent":["partial","*",4,0],"cohere-ai":["partial","*",4,0],"Diffbot":["partial","*",4,0],"ContextFetchBot":["partial","*",4,0]},"llms":{"status":"html","http":200,"bytes":96516},"home":{"status":"ok","http":200,"url":"https://www.bgr.com/","html_tokens":24129,"markdown_tokens":1147,"pruned_tokens":165}},
{"rank":3187,"domain":"iflscience.com","origin":"https://iflscience.com","robots":{"status":"refused","http":403,"url":"https://www.iflscience.com/robots.txt","bytes":919,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":3188,"domain":"uni-leipzig.de","origin":"https://uni-leipzig.de","robots":{"status":"ok","http":200,"url":"https://www.uni-leipzig.de/robots.txt","bytes":1719,"groups":1},"bots":{"GPTBot":["partial","*",21,27],"ChatGPT-User":["partial","*",21,27],"OAI-SearchBot":["partial","*",21,27],"ClaudeBot":["partial","*",21,27],"Claude-User":["partial","*",21,27],"Claude-SearchBot":["partial","*",21,27],"anthropic-ai":["partial","*",21,27],"PerplexityBot":["partial","*",21,27],"Perplexity-User":["partial","*",21,27],"Google-Extended":["partial","*",21,27],"Applebot-Extended":["partial","*",21,27],"CCBot":["partial","*",21,27],"Bytespider":["partial","*",21,27],"Amazonbot":["partial","*",21,27],"meta-externalagent":["partial","*",21,27],"cohere-ai":["partial","*",21,27],"Diffbot":["partial","*",21,27],"ContextFetchBot":["partial","*",21,27]},"llms":{"status":"timeout","error":"timed out after 10 s"},"home":{"status":"ok","http":200,"url":"https://www.uni-leipzig.de/","html_tokens":80670,"markdown_tokens":230,"pruned_tokens":110}},
{"rank":3189,"domain":"ahrq.gov","origin":"https://ahrq.gov","robots":{"status":"ok","http":200,"url":"https://www.ahrq.gov/robots.txt","bytes":4031,"groups":6},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["blocked","named",1,0],"OAI-SearchBot":["partial","*",44,18],"ClaudeBot":["partial","*",44,18],"Claude-User":["partial","*",44,18],"Claude-SearchBot":["partial","*",44,18],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["partial","*",44,18],"Perplexity-User":["partial","*",44,18],"Google-Extended":["partial","*",44,18],"Applebot-Extended":["partial","*",44,18],"CCBot":["blocked","named",1,0],"Bytespider":["partial","*",44,18],"Amazonbot":["partial","*",44,18],"meta-externalagent":["partial","*",44,18],"cohere-ai":["partial","*",44,18],"Diffbot":["partial","*",44,18],"ContextFetchBot":["partial","*",44,18]},"llms":{"status":"missing","http":404,"bytes":186},"home":{"status":"ok","http":200,"url":"https://www.ahrq.gov/","html_tokens":21514,"markdown_tokens":111,"pruned_tokens":110}},
{"rank":3190,"domain":"dwcdn.net","origin":"https://dwcdn.net","robots":{"status":"unreachable","http":null,"url":"https://dwcdn.net/robots.txt","bytes":0,"groups":0,"error":"cannot resolve dwcdn.net"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":3191,"domain":"sjsu.edu","origin":"https://sjsu.edu","robots":{"status":"ok","http":200,"url":"https://sjsu.edu/robots.txt","bytes":679,"groups":1},"bots":{"GPTBot":["partial","*",18,0],"ChatGPT-User":["partial","*",18,0],"OAI-SearchBot":["partial","*",18,0],"ClaudeBot":["partial","*",18,0],"Claude-User":["partial","*",18,0],"Claude-SearchBot":["partial","*",18,0],"anthropic-ai":["partial","*",18,0],"PerplexityBot":["partial","*",18,0],"Perplexity-User":["partial","*",18,0],"Google-Extended":["partial","*",18,0],"Applebot-Extended":["partial","*",18,0],"CCBot":["partial","*",18,0],"Bytespider":["partial","*",18,0],"Amazonbot":["partial","*",18,0],"meta-externalagent":["partial","*",18,0],"cohere-ai":["partial","*",18,0],"Diffbot":["partial","*",18,0],"ContextFetchBot":["partial","*",18,0]},"llms":{"status":"missing","http":404,"bytes":255},"home":{"status":"ok","http":200,"url":"https://sjsu.edu/","html_tokens":18479,"markdown_tokens":129,"pruned_tokens":49}},
{"rank":3192,"domain":"macmillan.com","origin":"https://macmillan.com","robots":{"status":"unreachable","http":null,"url":"https://macmillan.com/robots.txt","bytes":0,"groups":0,"error":"[SSL: CERTIFICATE_VERIFY_FAILED] certificate verify failed: unable to get local issuer certificate (_ssl.c:1028)"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":3193,"domain":"nutrition.org","origin":"https://nutrition.org","robots":{"status":"ok","http":200,"url":"https://nutrition.org/robots.txt","bytes":129,"groups":1},"bots":{"GPTBot":["partial","*",1,1],"ChatGPT-User":["partial","*",1,1],"OAI-SearchBot":["partial","*",1,1],"ClaudeBot":["partial","*",1,1],"Claude-User":["partial","*",1,1],"Claude-SearchBot":["partial","*",1,1],"anthropic-ai":["partial","*",1,1],"PerplexityBot":["partial","*",1,1],"Perplexity-User":["partial","*",1,1],"Google-Extended":["partial","*",1,1],"Applebot-Extended":["partial","*",1,1],"CCBot":["partial","*",1,1],"Bytespider":["partial","*",1,1],"Amazonbot":["partial","*",1,1],"meta-externalagent":["partial","*",1,1],"cohere-ai":["partial","*",1,1],"Diffbot":["partial","*",1,1],"ContextFetchBot":["partial","*",1,1]},"llms":{"status":"http_error","http":403,"bytes":5566},"home":{"status":"http_error","http":403,"url":"https://nutrition.org/"}},
{"rank":3194,"domain":"openculture.com","origin":"https://openculture.com","robots":{"status":"ok","http":200,"url":"https://www.openculture.com/robots.txt","bytes":336,"groups":3},"bots":{"GPTBot":["partial","*",4,0],"ChatGPT-User":["partial","*",4,0],"OAI-SearchBot":["partial","*",4,0],"ClaudeBot":["partial","*",4,0],"Claude-User":["partial","*",4,0],"Claude-SearchBot":["partial","*",4,0],"anthropic-ai":["partial","*",4,0],"PerplexityBot":["partial","*",4,0],"Perplexity-User":["partial","*",4,0],"Google-Extended":["partial","*",4,0],"Applebot-Extended":["partial","*",4,0],"CCBot":["partial","*",4,0],"Bytespider":["partial","*",4,0],"Amazonbot":["partial","*",4,0],"meta-externalagent":["partial","*",4,0],"cohere-ai":["partial","*",4,0],"Diffbot":["partial","*",4,0],"ContextFetchBot":["partial","*",4,0]},"llms":{"status":"missing","http":404,"bytes":101463},"home":{"status":"ok","http":200,"url":"https://www.openculture.com/","html_tokens":78275,"markdown_tokens":659,"pruned_tokens":136}},
{"rank":3195,"domain":"uottawa.ca","origin":"https://uottawa.ca","robots":{"status":"ok","http":200,"url":"https://www.uottawa.ca/robots.txt","bytes":2875,"groups":1},"bots":{"GPTBot":["partial","*",28,18],"ChatGPT-User":["partial","*",28,18],"OAI-SearchBot":["partial","*",28,18],"ClaudeBot":["partial","*",28,18],"Claude-User":["partial","*",28,18],"Claude-SearchBot":["partial","*",28,18],"anthropic-ai":["partial","*",28,18],"PerplexityBot":["partial","*",28,18],"Perplexity-User":["partial","*",28,18],"Google-Extended":["partial","*",28,18],"Applebot-Extended":["partial","*",28,18],"CCBot":["partial","*",28,18],"Bytespider":["partial","*",28,18],"Amazonbot":["partial","*",28,18],"meta-externalagent":["partial","*",28,18],"cohere-ai":["partial","*",28,18],"Diffbot":["partial","*",28,18],"ContextFetchBot":["partial","*",28,18]},"llms":{"status":"missing","http":404,"bytes":188},"home":{"status":"ok","http":200,"url":"https://www.uottawa.ca/en","html_tokens":21548,"markdown_tokens":290,"pruned_tokens":162}},
{"rank":3196,"domain":"bitcoin.com","origin":"https://bitcoin.com","robots":{"status":"ok","http":200,"url":"https://www.bitcoin.com/robots.txt","bytes":4357,"groups":21},"bots":{"GPTBot":["partial","named",7,1],"ChatGPT-User":["partial","named",7,1],"OAI-SearchBot":["partial","named",7,1],"ClaudeBot":["blocked","named",1,0],"Claude-User":["allowed","named",0,1],"Claude-SearchBot":["allowed","named",0,1],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["partial","named",7,1],"Perplexity-User":["partial","*",24,9],"Google-Extended":["partial","named",7,1],"Applebot-Extended":["partial","named",7,1],"CCBot":["partial","named",7,1],"Bytespider":["partial","named",7,1],"Amazonbot":["partial","*",24,9],"meta-externalagent":["partial","named",7,1],"cohere-ai":["partial","*",24,9],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",24,9]},"llms":{"status":"present","http":200,"bytes":9019},"home":{"status":"ok","http":200,"url":"https://www.bitcoin.com/","html_tokens":288113,"markdown_tokens":150,"pruned_tokens":114}},
{"rank":3197,"domain":"nbcnewyork.com","origin":"https://nbcnewyork.com","robots":{"status":"ok","http":200,"url":"https://www.nbcnewyork.com/robots.txt","bytes":9602,"groups":27},"bots":{"GPTBot":["partial","named",1,6],"ChatGPT-User":["partial","named",1,6],"OAI-SearchBot":["partial","*",13,3],"ClaudeBot":["partial","named",1,6],"Claude-User":["partial","*",13,3],"Claude-SearchBot":["partial","*",13,3],"anthropic-ai":["partial","named",1,6],"PerplexityBot":["partial","named",1,6],"Perplexity-User":["partial","*",13,3],"Google-Extended":["partial","named",1,6],"Applebot-Extended":["partial","*",13,3],"CCBot":["partial","named",1,6],"Bytespider":["partial","named",1,6],"Amazonbot":["partial","named",1,8],"meta-externalagent":["partial","*",13,3],"cohere-ai":["partial","*",13,3],"Diffbot":["partial","named",1,6],"ContextFetchBot":["partial","*",13,3]},"llms":{"status":"html","http":200,"bytes":555011},"home":{"status":"ok","http":200,"url":"https://www.nbcnewyork.com/","html_tokens":138739,"markdown_tokens":3,"pruned_tokens":3}},
{"rank":3198,"domain":"middleeasteye.net","origin":"https://middleeasteye.net","robots":{"status":"ok","http":200,"url":"https://www.middleeasteye.net/robots.txt","bytes":1799,"groups":1},"bots":{"GPTBot":["partial","*",30,18],"ChatGPT-User":["partial","*",30,18],"OAI-SearchBot":["partial","*",30,18],"ClaudeBot":["partial","*",30,18],"Claude-User":["partial","*",30,18],"Claude-SearchBot":["partial","*",30,18],"anthropic-ai":["partial","*",30,18],"PerplexityBot":["partial","*",30,18],"Perplexity-User":["partial","*",30,18],"Google-Extended":["partial","*",30,18],"Applebot-Extended":["partial","*",30,18],"CCBot":["partial","*",30,18],"Bytespider":["partial","*",30,18],"Amazonbot":["partial","*",30,18],"meta-externalagent":["partial","*",30,18],"cohere-ai":["partial","*",30,18],"Diffbot":["partial","*",30,18],"ContextFetchBot":["partial","*",30,18]},"llms":{"status":"http_error","http":403,"bytes":150},"home":{"status":"ok","http":200,"url":"https://www.middleeasteye.net/","html_tokens":54664,"markdown_tokens":18,"pruned_tokens":18}},
{"rank":3199,"domain":"dailydot.com","origin":"https://dailydot.com","robots":{"status":"ok","http":200,"url":"https://dailydot.com/robots.txt","bytes":352,"groups":2},"bots":{"GPTBot":["partial","*",7,0],"ChatGPT-User":["partial","*",7,0],"OAI-SearchBot":["partial","*",7,0],"ClaudeBot":["partial","*",7,0],"Claude-User":["partial","*",7,0],"Claude-SearchBot":["partial","*",7,0],"anthropic-ai":["partial","*",7,0],"PerplexityBot":["partial","*",7,0],"Perplexity-User":["partial","*",7,0],"Google-Extended":["partial","*",7,0],"Applebot-Extended":["partial","*",7,0],"CCBot":["partial","*",7,0],"Bytespider":["partial","*",7,0],"Amazonbot":["partial","*",7,0],"meta-externalagent":["partial","*",7,0],"cohere-ai":["partial","*",7,0],"Diffbot":["partial","*",7,0],"ContextFetchBot":["partial","*",7,0]},"llms":{"status":"missing","http":404,"bytes":9},"home":{"status":"ok","http":200,"url":"https://dailydot.com/","html_tokens":124331,"markdown_tokens":49,"pruned_tokens":49}},
{"rank":3200,"domain":"stats.gov.cn","origin":"https://www.stats.gov.cn","robots":{"status":"missing","http":404,"url":"https://www.stats.gov.cn/robots.txt","bytes":555,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":153},"home":{"status":"ok","http":200,"url":"https://www.stats.gov.cn/","html_tokens":31881,"markdown_tokens":22,"pruned_tokens":22}},
{"rank":3201,"domain":"muni.cz","origin":"https://muni.cz","robots":{"status":"ok","http":200,"url":"https://www.muni.cz/robots.txt","bytes":132,"groups":2},"bots":{"GPTBot":["allowed","*",0,1],"ChatGPT-User":["allowed","*",0,1],"OAI-SearchBot":["allowed","*",0,1],"ClaudeBot":["allowed","*",0,1],"Claude-User":["allowed","*",0,1],"Claude-SearchBot":["allowed","*",0,1],"anthropic-ai":["allowed","*",0,1],"PerplexityBot":["allowed","*",0,1],"Perplexity-User":["allowed","*",0,1],"Google-Extended":["allowed","*",0,1],"Applebot-Extended":["allowed","*",0,1],"CCBot":["allowed","*",0,1],"Bytespider":["allowed","*",0,1],"Amazonbot":["allowed","*",0,1],"meta-externalagent":["allowed","*",0,1],"cohere-ai":["allowed","*",0,1],"Diffbot":["allowed","*",0,1],"ContextFetchBot":["allowed","*",0,1]},"llms":{"status":"missing","http":404,"bytes":0},"home":{"status":"ok","http":200,"url":"https://www.muni.cz/","html_tokens":24165,"markdown_tokens":78,"pruned_tokens":78}},
{"rank":3202,"domain":"crossref.org","origin":"https://crossref.org","robots":{"status":"ok","http":200,"url":"https://www.crossref.org/robots.txt","bytes":124,"groups":1},"bots":{"GPTBot":["partial","*",3,1],"ChatGPT-User":["partial","*",3,1],"OAI-SearchBot":["partial","*",3,1],"ClaudeBot":["partial","*",3,1],"Claude-User":["partial","*",3,1],"Claude-SearchBot":["partial","*",3,1],"anthropic-ai":["partial","*",3,1],"PerplexityBot":["partial","*",3,1],"Perplexity-User":["partial","*",3,1],"Google-Extended":["partial","*",3,1],"Applebot-Extended":["partial","*",3,1],"CCBot":["partial","*",3,1],"Bytespider":["partial","*",3,1],"Amazonbot":["partial","*",3,1],"meta-externalagent":["partial","*",3,1],"cohere-ai":["partial","*",3,1],"Diffbot":["partial","*",3,1],"ContextFetchBot":["partial","*",3,1]},"llms":{"status":"missing","http":404,"bytes":286},"home":{"status":"ok","http":200,"url":"https://www.crossref.org/","html_tokens":8802,"markdown_tokens":186,"pruned_tokens":129}},
{"rank":3203,"domain":"techdirt.com","origin":"https://techdirt.com","robots":{"status":"ok","http":200,"url":"https://www.techdirt.com/robots.txt","bytes":199,"groups":2},"bots":{"GPTBot":["partial","*",1,1],"ChatGPT-User":["partial","*",1,1],"OAI-SearchBot":["partial","*",1,1],"ClaudeBot":["partial","*",1,1],"Claude-User":["partial","*",1,1],"Claude-SearchBot":["partial","*",1,1],"anthropic-ai":["partial","*",1,1],"PerplexityBot":["partial","*",1,1],"Perplexity-User":["partial","*",1,1],"Google-Extended":["partial","*",1,1],"Applebot-Extended":["partial","*",1,1],"CCBot":["partial","*",1,1],"Bytespider":["partial","*",1,1],"Amazonbot":["partial","*",1,1],"meta-externalagent":["partial","*",1,1],"cohere-ai":["partial","*",1,1],"Diffbot":["partial","*",1,1],"ContextFetchBot":["partial","*",1,1]},"llms":{"status":"http_error","http":403,"bytes":5544},"home":{"status":"http_error","http":403,"url":"https://techdirt.com/"}},
{"rank":3204,"domain":"jdpower.com","origin":"https://jdpower.com","robots":{"status":"ok","http":200,"url":"https://www.jdpower.com/robots.txt","bytes":5127,"groups":1},"bots":{"GPTBot":["partial","*",92,48],"ChatGPT-User":["partial","*",92,48],"OAI-SearchBot":["partial","*",92,48],"ClaudeBot":["partial","*",92,48],"Claude-User":["partial","*",92,48],"Claude-SearchBot":["partial","*",92,48],"anthropic-ai":["partial","*",92,48],"PerplexityBot":["partial","*",92,48],"Perplexity-User":["partial","*",92,48],"Google-Extended":["partial","*",92,48],"Applebot-Extended":["partial","*",92,48],"CCBot":["partial","*",92,48],"Bytespider":["partial","*",92,48],"Amazonbot":["partial","*",92,48],"meta-externalagent":["partial","*",92,48],"cohere-ai":["partial","*",92,48],"Diffbot":["partial","*",92,48],"ContextFetchBot":["partial","*",92,48]},"llms":{"status":"http_error","http":403,"bytes":5542},"home":{"status":"http_error","http":403,"url":"https://jdpower.com/"}},
{"rank":3205,"domain":"wral.com","origin":"https://wral.com","robots":{"status":"ok","http":200,"url":"https://www.wral.com/robots.txt","bytes":4939,"groups":5},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["partial","named",9,0],"OAI-SearchBot":["partial","named",9,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","named",9,0],"Claude-SearchBot":["partial","named",9,0],"anthropic-ai":["partial","*",9,0],"PerplexityBot":["partial","named",9,0],"Perplexity-User":["partial","named",9,0],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",9,0]},"llms":{"status":"missing","http":404,"bytes":5054},"home":{"status":"ok","http":200,"url":"https://www.wral.com/","html_tokens":34904,"markdown_tokens":53,"pruned_tokens":53}},
{"rank":3206,"domain":"courthousenews.com","origin":"https://courthousenews.com","robots":{"status":"ok","http":200,"url":"https://courthousenews.com/robots.txt","bytes":119,"groups":1},"bots":{"GPTBot":["partial","*",1,1],"ChatGPT-User":["partial","*",1,1],"OAI-SearchBot":["partial","*",1,1],"ClaudeBot":["partial","*",1,1],"Claude-User":["partial","*",1,1],"Claude-SearchBot":["partial","*",1,1],"anthropic-ai":["partial","*",1,1],"PerplexityBot":["partial","*",1,1],"Perplexity-User":["partial","*",1,1],"Google-Extended":["partial","*",1,1],"Applebot-Extended":["partial","*",1,1],"CCBot":["partial","*",1,1],"Bytespider":["partial","*",1,1],"Amazonbot":["partial","*",1,1],"meta-externalagent":["partial","*",1,1],"cohere-ai":["partial","*",1,1],"Diffbot":["partial","*",1,1],"ContextFetchBot":["partial","*",1,1]},"llms":{"status":"missing","http":404,"bytes":165903},"home":{"status":"http_error","http":403,"url":"https://courthousenews.com/"}},
{"rank":3207,"domain":"audubon.org","origin":"https://audubon.org","robots":{"status":"refused","http":403,"url":"https://audubon.org/robots.txt","bytes":560,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":3208,"domain":"transip.nl","origin":"https://transip.nl","robots":{"status":"ok","http":200,"url":"https://www.transip.nl/robots.txt","bytes":132,"groups":1},"bots":{"GPTBot":["partial","*",2,1],"ChatGPT-User":["partial","*",2,1],"OAI-SearchBot":["partial","*",2,1],"ClaudeBot":["partial","*",2,1],"Claude-User":["partial","*",2,1],"Claude-SearchBot":["partial","*",2,1],"anthropic-ai":["partial","*",2,1],"PerplexityBot":["partial","*",2,1],"Perplexity-User":["partial","*",2,1],"Google-Extended":["partial","*",2,1],"Applebot-Extended":["partial","*",2,1],"CCBot":["partial","*",2,1],"Bytespider":["partial","*",2,1],"Amazonbot":["partial","*",2,1],"meta-externalagent":["partial","*",2,1],"cohere-ai":["partial","*",2,1],"Diffbot":["partial","*",2,1],"ContextFetchBot":["partial","*",2,1]},"llms":{"status":"missing","http":404,"bytes":266744},"home":{"status":"ok","http":200,"url":"https://www.transip.nl/","html_tokens":91133,"markdown_tokens":468,"pruned_tokens":172}},
{"rank":3209,"domain":"contentmarketinginstitute.com","origin":"https://contentmarketinginstitute.com","robots":{"status":"ok","http":200,"url":"https://contentmarketinginstitute.com/robots.txt","bytes":1525,"groups":2},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["partial","*",6,0],"OAI-SearchBot":["partial","*",6,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",6,0],"Claude-SearchBot":["partial","*",6,0],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["partial","*",6,0],"Perplexity-User":["partial","*",6,0],"Google-Extended":["partial","*",6,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",6,0]},"llms":{"status":"missing","http":404,"bytes":352042},"home":{"status":"ok","http":200,"url":"https://contentmarketinginstitute.com/","html_tokens":110942,"markdown_tokens":240,"pruned_tokens":67}},
{"rank":3210,"domain":"pixieset.com","origin":"https://pixieset.com","robots":{"status":"ok","http":200,"url":"https://pixieset.com/robots.txt","bytes":339,"groups":3},"bots":{"GPTBot":["partial","*",3,0],"ChatGPT-User":["partial","*",3,0],"OAI-SearchBot":["partial","*",3,0],"ClaudeBot":["partial","*",3,0],"Claude-User":["partial","*",3,0],"Claude-SearchBot":["partial","*",3,0],"anthropic-ai":["partial","*",3,0],"PerplexityBot":["partial","*",3,0],"Perplexity-User":["partial","*",3,0],"Google-Extended":["partial","*",3,0],"Applebot-Extended":["partial","*",3,0],"CCBot":["partial","*",3,0],"Bytespider":["partial","*",3,0],"Amazonbot":["partial","*",3,0],"meta-externalagent":["partial","*",3,0],"cohere-ai":["partial","*",3,0],"Diffbot":["partial","*",3,0],"ContextFetchBot":["partial","*",3,0]},"llms":{"status":"missing","http":404,"bytes":12334},"home":{"status":"ok","http":200,"url":"https://pixieset.com/","html_tokens":183768,"markdown_tokens":725,"pruned_tokens":328}},
{"rank":3211,"domain":"awwwards.com","origin":"https://awwwards.com","robots":{"status":"ok","http":200,"url":"https://www.awwwards.com/robots.txt","bytes":935,"groups":7},"bots":{"GPTBot":["partial","*",27,0],"ChatGPT-User":["partial","*",27,0],"OAI-SearchBot":["partial","*",27,0],"ClaudeBot":["partial","*",27,0],"Claude-User":["partial","*",27,0],"Claude-SearchBot":["partial","*",27,0],"anthropic-ai":["partial","*",27,0],"PerplexityBot":["partial","*",27,0],"Perplexity-User":["partial","*",27,0],"Google-Extended":["partial","*",27,0],"Applebot-Extended":["partial","*",27,0],"CCBot":["partial","*",27,0],"Bytespider":["partial","*",27,0],"Amazonbot":["partial","*",27,0],"meta-externalagent":["partial","*",27,0],"cohere-ai":["partial","*",27,0],"Diffbot":["partial","*",27,0],"ContextFetchBot":["partial","*",27,0]},"llms":{"status":"present","http":200,"bytes":1870},"home":{"status":"ok","http":200,"url":"https://www.awwwards.com/","html_tokens":84312,"markdown_tokens":37,"pruned_tokens":37}},
{"rank":3212,"domain":"runnersworld.com","origin":"https://runnersworld.com","robots":{"status":"ok","http":200,"url":"https://runnersworld.com/robots.txt","bytes":954,"groups":3},"bots":{"GPTBot":["partial","named",31,0],"ChatGPT-User":["partial","named",31,0],"OAI-SearchBot":["partial","named",31,0],"ClaudeBot":["partial","*",19,0],"Claude-User":["partial","*",19,0],"Claude-SearchBot":["partial","*",19,0],"anthropic-ai":["partial","*",19,0],"PerplexityBot":["partial","*",19,0],"Perplexity-User":["partial","*",19,0],"Google-Extended":["partial","*",19,0],"Applebot-Extended":["partial","*",19,0],"CCBot":["partial","*",19,0],"Bytespider":["partial","*",19,0],"Amazonbot":["partial","*",19,0],"meta-externalagent":["partial","*",19,0],"cohere-ai":["partial","*",19,0],"Diffbot":["partial","*",19,0],"ContextFetchBot":["partial","*",19,0]},"llms":{"status":"missing","http":404,"bytes":84},"home":{"status":"ok","http":200,"url":"https://www.runnersworld.com/","html_tokens":295128,"markdown_tokens":240,"pruned_tokens":138}},
{"rank":3213,"domain":"penzu.com","origin":"https://penzu.com","robots":{"status":"ok","http":200,"url":"https://penzu.com/robots.txt","bytes":736,"groups":1},"bots":{"GPTBot":["partial","*",29,1],"ChatGPT-User":["partial","*",29,1],"OAI-SearchBot":["partial","*",29,1],"ClaudeBot":["partial","*",29,1],"Claude-User":["partial","*",29,1],"Claude-SearchBot":["partial","*",29,1],"anthropic-ai":["partial","*",29,1],"PerplexityBot":["partial","*",29,1],"Perplexity-User":["partial","*",29,1],"Google-Extended":["partial","*",29,1],"Applebot-Extended":["partial","*",29,1],"CCBot":["partial","*",29,1],"Bytespider":["partial","*",29,1],"Amazonbot":["partial","*",29,1],"meta-externalagent":["partial","*",29,1],"cohere-ai":["partial","*",29,1],"Diffbot":["partial","*",29,1],"ContextFetchBot":["partial","*",29,1]},"llms":{"status":"missing","http":404,"bytes":56286},"home":{"status":"ok","http":200,"url":"https://penzu.com/","html_tokens":43076,"markdown_tokens":522,"pruned_tokens":131}},
{"rank":3214,"domain":"curbed.com","origin":"https://curbed.com","robots":{"status":"ok","http":200,"url":"https://curbed.com/robots.txt","bytes":7479,"groups":107},"bots":{"GPTBot":["allowed","named",0,1],"ChatGPT-User":["partial","named",1,2],"OAI-SearchBot":["partial","*",5,0],"ClaudeBot":["partial","named",1,2],"Claude-User":["partial","named",1,2],"Claude-SearchBot":["partial","named",1,2],"anthropic-ai":["partial","named",1,2],"PerplexityBot":["partial","named",1,2],"Perplexity-User":["partial","named",1,2],"Google-Extended":["partial","named",1,2],"Applebot-Extended":["partial","named",1,2],"CCBot":["partial","named",1,2],"Bytespider":["partial","named",1,2],"Amazonbot":["allowed","named",0,1],"meta-externalagent":["partial","named",1,2],"cohere-ai":["partial","named",1,2],"Diffbot":["partial","named",1,2],"ContextFetchBot":["partial","*",5,0]},"llms":{"status":"missing","http":404,"bytes":47002},"home":{"status":"ok","http":200,"url":"https://www.curbed.com/","html_tokens":83209,"markdown_tokens":360,"pruned_tokens":100}},
{"rank":3215,"domain":"interpol.int","origin":"https://interpol.int","robots":{"status":"html","http":200,"url":"https://www.interpol.int/","bytes":73720,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":3216,"domain":"fastly.com","origin":"https://fastly.com","robots":{"status":"ok","http":200,"url":"https://www.fastly.com/robots.txt","bytes":398,"groups":1},"bots":{"GPTBot":["partial","*",11,0],"ChatGPT-User":["partial","*",11,0],"OAI-SearchBot":["partial","*",11,0],"ClaudeBot":["partial","*",11,0],"Claude-User":["partial","*",11,0],"Claude-SearchBot":["partial","*",11,0],"anthropic-ai":["partial","*",11,0],"PerplexityBot":["partial","*",11,0],"Perplexity-User":["partial","*",11,0],"Google-Extended":["partial","*",11,0],"Applebot-Extended":["partial","*",11,0],"CCBot":["partial","*",11,0],"Bytespider":["partial","*",11,0],"Amazonbot":["partial","*",11,0],"meta-externalagent":["partial","*",11,0],"cohere-ai":["partial","*",11,0],"Diffbot":["partial","*",11,0],"ContextFetchBot":["partial","*",11,0]},"llms":{"status":"present","http":200,"bytes":16618},"home":{"status":"ok","http":200,"url":"https://www.fastly.com/","html_tokens":55078,"markdown_tokens":1149,"pruned_tokens":297}},
{"rank":3217,"domain":"catb.org","origin":"http://catb.org","robots":{"status":"missing","http":404,"url":"http://catb.org/robots.txt","bytes":11166,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":11166},"home":{"status":"ok","http":200,"url":"http://catb.org/","html_tokens":373,"markdown_tokens":58,"pruned_tokens":49}},
{"rank":3218,"domain":"sxsw.com","origin":"https://sxsw.com","robots":{"status":"ok","http":200,"url":"https://sxsw.com/robots.txt","bytes":213,"groups":1},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"missing","http":404,"bytes":0},"home":{"status":"ok","http":200,"url":"https://sxsw.com/","html_tokens":65586,"markdown_tokens":379,"pruned_tokens":25}},
{"rank":3219,"domain":"synopsys.com","origin":"https://synopsys.com","robots":{"status":"ok","http":200,"url":"https://www.synopsys.com/robots.txt","bytes":650,"groups":1},"bots":{"GPTBot":["partial","*",5,1],"ChatGPT-User":["partial","*",5,1],"OAI-SearchBot":["partial","*",5,1],"ClaudeBot":["partial","*",5,1],"Claude-User":["partial","*",5,1],"Claude-SearchBot":["partial","*",5,1],"anthropic-ai":["partial","*",5,1],"PerplexityBot":["partial","*",5,1],"Perplexity-User":["partial","*",5,1],"Google-Extended":["partial","*",5,1],"Applebot-Extended":["partial","*",5,1],"CCBot":["partial","*",5,1],"Bytespider":["partial","*",5,1],"Amazonbot":["partial","*",5,1],"meta-externalagent":["partial","*",5,1],"cohere-ai":["partial","*",5,1],"Diffbot":["partial","*",5,1],"ContextFetchBot":["partial","*",5,1]},"llms":{"status":"present","http":200,"bytes":5644},"home":{"status":"ok","http":200,"url":"https://www.synopsys.com/","html_tokens":55739,"markdown_tokens":89,"pruned_tokens":46}},
{"rank":3220,"domain":"nolo.com","origin":"https://nolo.com","robots":{"status":"refused","http":403,"url":"https://www.nolo.com/robots.txt","bytes":5529,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":3221,"domain":"linuxmint.com","origin":"https://linuxmint.com","robots":{"status":"missing","http":404,"url":"https://linuxmint.com/robots.txt","bytes":18624,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":18624},"home":{"status":"ok","http":200,"url":"https://linuxmint.com/","html_tokens":9620,"markdown_tokens":78,"pruned_tokens":78}},
{"rank":3222,"domain":"anl.gov","origin":"https://www.anl.gov","robots":{"status":"refused","http":403,"url":"https://www.anl.gov/robots.txt","bytes":5506,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":3223,"domain":"lanl.gov","origin":"https://lanl.gov","robots":{"status":"ok","http":200,"url":"https://www.lanl.gov/robots.txt","bytes":120,"groups":2},"bots":{"GPTBot":["allowed","*",0,1],"ChatGPT-User":["allowed","*",0,1],"OAI-SearchBot":["allowed","*",0,1],"ClaudeBot":["allowed","*",0,1],"Claude-User":["allowed","*",0,1],"Claude-SearchBot":["allowed","*",0,1],"anthropic-ai":["allowed","*",0,1],"PerplexityBot":["allowed","*",0,1],"Perplexity-User":["allowed","*",0,1],"Google-Extended":["allowed","*",0,1],"Applebot-Extended":["allowed","*",0,1],"CCBot":["allowed","*",0,1],"Bytespider":["allowed","*",0,1],"Amazonbot":["allowed","*",0,1],"meta-externalagent":["allowed","*",0,1],"cohere-ai":["allowed","*",0,1],"Diffbot":["allowed","*",0,1],"ContextFetchBot":["allowed","*",0,1]},"llms":{"status":"missing","http":404,"bytes":237593},"home":{"status":"ok","http":200,"url":"https://www.lanl.gov/","html_tokens":106846,"markdown_tokens":333,"pruned_tokens":19}},
{"rank":3224,"domain":"dtu.dk","origin":"https://dtu.dk","robots":{"status":"ok","http":200,"url":"https://dtu.dk/robots.txt","bytes":1855,"groups":4},"bots":{"GPTBot":["partial","named",16,0],"ChatGPT-User":["partial","*",16,0],"OAI-SearchBot":["partial","*",16,0],"ClaudeBot":["partial","named",16,0],"Claude-User":["partial","*",16,0],"Claude-SearchBot":["partial","*",16,0],"anthropic-ai":["partial","*",16,0],"PerplexityBot":["partial","*",16,0],"Perplexity-User":["partial","*",16,0],"Google-Extended":["partial","*",16,0],"Applebot-Extended":["partial","*",16,0],"CCBot":["partial","*",16,0],"Bytespider":["partial","*",16,0],"Amazonbot":["partial","*",16,0],"meta-externalagent":["partial","*",16,0],"cohere-ai":["partial","*",16,0],"Diffbot":["partial","*",16,0],"ContextFetchBot":["partial","*",16,0]},"llms":{"status":"missing","http":404,"bytes":103},"home":{"status":"ok","http":200,"url":"https://dtu.dk/","html_tokens":51958,"markdown_tokens":110,"pruned_tokens":84}},
{"rank":3225,"domain":"acronis.com","origin":"https://acronis.com","robots":{"status":"ok","http":200,"url":"https://www.acronis.com/robots.txt","bytes":1479,"groups":3},"bots":{"GPTBot":["partial","*",23,0],"ChatGPT-User":["partial","*",23,0],"OAI-SearchBot":["partial","*",23,0],"ClaudeBot":["partial","*",23,0],"Claude-User":["partial","*",23,0],"Claude-SearchBot":["partial","*",23,0],"anthropic-ai":["partial","*",23,0],"PerplexityBot":["partial","*",23,0],"Perplexity-User":["partial","*",23,0],"Google-Extended":["partial","*",23,0],"Applebot-Extended":["partial","*",23,0],"CCBot":["partial","*",23,0],"Bytespider":["partial","*",23,0],"Amazonbot":["partial","*",23,0],"meta-externalagent":["partial","*",23,0],"cohere-ai":["partial","*",23,0],"Diffbot":["partial","*",23,0],"ContextFetchBot":["partial","*",23,0]},"llms":{"status":"present","http":200,"bytes":11713},"home":{"status":"ok","http":200,"url":"https://www.acronis.com/en/","html_tokens":131015,"markdown_tokens":323,"pruned_tokens":173}},
{"rank":3226,"domain":"auburn.edu","origin":"https://auburn.edu","robots":{"status":"ok","http":200,"url":"https://auburn.edu/robots.txt","bytes":155,"groups":1},"bots":{"GPTBot":["partial","*",2,0],"ChatGPT-User":["partial","*",2,0],"OAI-SearchBot":["partial","*",2,0],"ClaudeBot":["partial","*",2,0],"Claude-User":["partial","*",2,0],"Claude-SearchBot":["partial","*",2,0],"anthropic-ai":["partial","*",2,0],"PerplexityBot":["partial","*",2,0],"Perplexity-User":["partial","*",2,0],"Google-Extended":["partial","*",2,0],"Applebot-Extended":["partial","*",2,0],"CCBot":["partial","*",2,0],"Bytespider":["partial","*",2,0],"Amazonbot":["partial","*",2,0],"meta-externalagent":["partial","*",2,0],"cohere-ai":["partial","*",2,0],"Diffbot":["partial","*",2,0],"ContextFetchBot":["partial","*",2,0]},"llms":{"status":"missing","http":404,"bytes":24043},"home":{"status":"ok","http":200,"url":"https://auburn.edu/","html_tokens":21680,"markdown_tokens":244,"pruned_tokens":146}},
{"rank":3227,"domain":"bonappetit.com","origin":"https://bonappetit.com","robots":{"status":"ok","http":200,"url":"https://www.bonappetit.com/robots.txt","bytes":2221,"groups":3},"bots":{"GPTBot":["partial","*",16,15],"ChatGPT-User":["partial","*",16,15],"OAI-SearchBot":["partial","*",16,15],"ClaudeBot":["blocked","named",1,0],"Claude-User":["blocked","named",1,0],"Claude-SearchBot":["blocked","named",1,0],"anthropic-ai":["partial","*",16,15],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["blocked","named",1,0],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",16,15]},"llms":{"status":"missing","http":404,"bytes":772298},"home":{"status":"ok","http":200,"url":"https://www.bonappetit.com/","html_tokens":428196,"markdown_tokens":177,"pruned_tokens":39}},
{"rank":3228,"domain":"ameli.fr","origin":"https://ameli.fr","robots":{"status":"refused","http":403,"url":"https://www.ameli.fr/robots.txt","bytes":36503,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":3229,"domain":"studentaid.gov","origin":"https://studentaid.gov","robots":{"status":"unreachable","http":null,"url":"https://studentaid.gov/robots.txt","bytes":0,"groups":0,"error":"<StreamReset stream_id:1, error_code:2, remote_reset:True>"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":3230,"domain":"ui.com","origin":"https://ui.com","robots":{"status":"ok","http":200,"url":"https://ui.com/robots.txt","bytes":367,"groups":10},"bots":{"GPTBot":["allowed","named",0,1],"ChatGPT-User":["allowed","named",0,1],"OAI-SearchBot":["allowed","named",0,1],"ClaudeBot":["allowed","named",0,1],"Claude-User":["allowed","*",0,1],"Claude-SearchBot":["allowed","*",0,1],"anthropic-ai":["allowed","*",0,1],"PerplexityBot":["allowed","named",0,1],"Perplexity-User":["allowed","*",0,1],"Google-Extended":["allowed","named",0,1],"Applebot-Extended":["allowed","named",0,1],"CCBot":["allowed","named",0,1],"Bytespider":["allowed","*",0,1],"Amazonbot":["allowed","*",0,1],"meta-externalagent":["allowed","*",0,1],"cohere-ai":["allowed","*",0,1],"Diffbot":["allowed","*",0,1],"ContextFetchBot":["allowed","*",0,1]},"llms":{"status":"missing","http":404,"bytes":272717},"home":{"status":"ok","http":200,"url":"https://ui.com/","html_tokens":105014,"markdown_tokens":182,"pruned_tokens":151}},
{"rank":3231,"domain":"sentry.io","origin":"https://sentry.io","robots":{"status":"ok","http":200,"url":"https://sentry.io/robots.txt","bytes":976,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"present","http":200,"bytes":245108},"home":{"status":"not_html","error":"unsupported content type text/markdown"}},
{"rank":3232,"domain":"aftonbladet.se","origin":"https://aftonbladet.se","robots":{"status":"ok","http":200,"url":"https://www.aftonbladet.se/robots.txt","bytes":1704,"groups":2},"bots":{"GPTBot":["partial","*",5,1],"ChatGPT-User":["partial","*",5,1],"OAI-SearchBot":["partial","named",5,1],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",5,1],"Claude-SearchBot":["blocked","named",1,0],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["partial","*",5,1],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",5,1]},"llms":{"status":"missing","http":404,"bytes":83310},"home":{"status":"ok","http":200,"url":"https://www.aftonbladet.se","html_tokens":130243,"markdown_tokens":19,"pruned_tokens":19}},
{"rank":3233,"domain":"google.com.sg","origin":"https://google.com.sg","robots":{"status":"ok","http":200,"url":"https://www.google.com.sg/robots.txt","bytes":6624,"groups":4},"bots":{"GPTBot":["partial","*",169,64],"ChatGPT-User":["partial","*",169,64],"OAI-SearchBot":["partial","*",169,64],"ClaudeBot":["partial","*",169,64],"Claude-User":["partial","*",169,64],"Claude-SearchBot":["partial","*",169,64],"anthropic-ai":["partial","*",169,64],"PerplexityBot":["partial","*",169,64],"Perplexity-User":["partial","*",169,64],"Google-Extended":["partial","*",169,64],"Applebot-Extended":["partial","*",169,64],"CCBot":["partial","*",169,64],"Bytespider":["partial","*",169,64],"Amazonbot":["partial","*",169,64],"meta-externalagent":["partial","*",169,64],"cohere-ai":["partial","*",169,64],"Diffbot":["partial","*",169,64],"ContextFetchBot":["partial","*",169,64]},"llms":{"status":"missing","http":404,"bytes":1569},"home":{"status":"ok","http":200,"url":"https://www.google.com/","html_tokens":21310,"markdown_tokens":30,"pruned_tokens":30}},
{"rank":3234,"domain":"iol.co.za","origin":"https://iol.co.za","robots":{"status":"ok","http":200,"url":"https://iol.co.za/robots.txt","bytes":1754,"groups":26},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["blocked","named",1,0],"OAI-SearchBot":["partial","*",5,2],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",5,2],"Claude-SearchBot":["partial","*",5,2],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["partial","*",5,2],"Perplexity-User":["partial","*",5,2],"Google-Extended":["allowed","named",0,1],"Applebot-Extended":["allowed","named",0,1],"CCBot":["allowed","named",0,1],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["partial","*",5,2],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",5,2]},"llms":{"status":"missing","http":404,"bytes":150535},"home":{"status":"http_error","http":403,"url":"https://iol.co.za/"}},
{"rank":3235,"domain":"about.google","origin":"https://about.google","robots":{"status":"ok","http":200,"url":"https://about.google/robots.txt","bytes":66,"groups":1},"bots":{"GPTBot":["allowed","*",0,1],"ChatGPT-User":["allowed","*",0,1],"OAI-SearchBot":["allowed","*",0,1],"ClaudeBot":["allowed","*",0,1],"Claude-User":["allowed","*",0,1],"Claude-SearchBot":["allowed","*",0,1],"anthropic-ai":["allowed","*",0,1],"PerplexityBot":["allowed","*",0,1],"Perplexity-User":["allowed","*",0,1],"Google-Extended":["allowed","*",0,1],"Applebot-Extended":["allowed","*",0,1],"CCBot":["allowed","*",0,1],"Bytespider":["allowed","*",0,1],"Amazonbot":["allowed","*",0,1],"meta-externalagent":["allowed","*",0,1],"cohere-ai":["allowed","*",0,1],"Diffbot":["allowed","*",0,1],"ContextFetchBot":["allowed","*",0,1]},"llms":{"status":"missing","http":404,"bytes":1569},"home":{"status":"ok","http":200,"url":"https://about.google/","html_tokens":33300,"markdown_tokens":19,"pruned_tokens":19}},
{"rank":3236,"domain":"webfx.com","origin":"https://webfx.com","robots":{"status":"ok","http":200,"url":"https://www.webfx.com/robots.txt","bytes":692,"groups":2},"bots":{"GPTBot":["partial","*",21,1],"ChatGPT-User":["partial","*",21,1],"OAI-SearchBot":["partial","*",21,1],"ClaudeBot":["partial","*",21,1],"Claude-User":["partial","*",21,1],"Claude-SearchBot":["partial","*",21,1],"anthropic-ai":["partial","*",21,1],"PerplexityBot":["partial","*",21,1],"Perplexity-User":["partial","*",21,1],"Google-Extended":["partial","*",21,1],"Applebot-Extended":["partial","*",21,1],"CCBot":["partial","*",21,1],"Bytespider":["partial","*",21,1],"Amazonbot":["partial","*",21,1],"meta-externalagent":["partial","*",21,1],"cohere-ai":["partial","*",21,1],"Diffbot":["partial","*",21,1],"ContextFetchBot":["partial","*",21,1]},"llms":{"status":"present","http":200,"bytes":5424},"home":{"status":"ok","http":200,"url":"https://www.webfx.com/","html_tokens":335352,"markdown_tokens":953,"pruned_tokens":356}},
{"rank":3237,"domain":"publishersweekly.com","origin":"https://publishersweekly.com","robots":{"status":"ok","http":200,"url":"https://www.publishersweekly.com/robots.txt","bytes":656,"groups":2},"bots":{"GPTBot":["blocked","*",1,0],"ChatGPT-User":["blocked","*",1,0],"OAI-SearchBot":["blocked","*",1,0],"ClaudeBot":["blocked","*",1,0],"Claude-User":["blocked","*",1,0],"Claude-SearchBot":["blocked","*",1,0],"anthropic-ai":["blocked","*",1,0],"PerplexityBot":["blocked","*",1,0],"Perplexity-User":["blocked","*",1,0],"Google-Extended":["blocked","*",1,0],"Applebot-Extended":["blocked","*",1,0],"CCBot":["blocked","*",1,0],"Bytespider":["blocked","*",1,0],"Amazonbot":["blocked","*",1,0],"meta-externalagent":["blocked","*",1,0],"cohere-ai":["blocked","*",1,0],"Diffbot":["blocked","*",1,0],"ContextFetchBot":["blocked","*",1,0]},"llms":{"status":"disallowed"},"home":{"status":"disallowed"}},
{"rank":3238,"domain":"ntv.co.jp","origin":"https://ntv.co.jp","robots":{"status":"ok","http":200,"url":"https://www.ntv.co.jp/robots.txt","bytes":306,"groups":3},"bots":{"GPTBot":["allowed","*",0,1],"ChatGPT-User":["allowed","*",0,1],"OAI-SearchBot":["allowed","*",0,1],"ClaudeBot":["allowed","*",0,1],"Claude-User":["allowed","*",0,1],"Claude-SearchBot":["allowed","*",0,1],"anthropic-ai":["allowed","*",0,1],"PerplexityBot":["allowed","*",0,1],"Perplexity-User":["allowed","*",0,1],"Google-Extended":["allowed","*",0,1],"Applebot-Extended":["allowed","*",0,1],"CCBot":["allowed","*",0,1],"Bytespider":["allowed","*",0,1],"Amazonbot":["allowed","*",0,1],"meta-externalagent":["allowed","*",0,1],"cohere-ai":["allowed","*",0,1],"Diffbot":["allowed","*",0,1],"ContextFetchBot":["allowed","*",0,1]},"llms":{"status":"missing","http":404,"bytes":1651},"home":{"status":"ok","http":200,"url":"https://www.ntv.co.jp/","html_tokens":14842,"markdown_tokens":7,"pruned_tokens":7}},
{"rank":3239,"domain":"digikey.com","origin":"https://digikey.com","robots":{"status":"ok","http":200,"url":"https://www.digikey.com/robots.txt","bytes":1361,"groups":12},"bots":{"GPTBot":["allowed","named",0,1],"ChatGPT-User":["allowed","named",0,1],"OAI-SearchBot":["allowed","named",0,1],"ClaudeBot":["allowed","named",0,1],"Claude-User":["allowed","named",0,1],"Claude-SearchBot":["allowed","named",0,1],"anthropic-ai":["partial","*",7,0],"PerplexityBot":["allowed","named",0,1],"Perplexity-User":["allowed","named",0,1],"Google-Extended":["allowed","named",0,1],"Applebot-Extended":["partial","*",7,0],"CCBot":["partial","*",7,0],"Bytespider":["partial","*",7,0],"Amazonbot":["partial","*",7,0],"meta-externalagent":["partial","*",7,0],"cohere-ai":["partial","*",7,0],"Diffbot":["partial","*",7,0],"ContextFetchBot":["partial","*",7,0]},"llms":{"status":"http_error","http":403,"bytes":5526},"home":{"status":"http_error","http":403,"url":"https://www.digikey.com/"}},
{"rank":3240,"domain":"cmegroup.com","origin":"https://cmegroup.com","robots":{"status":"timeout","http":null,"url":"https://cmegroup.com/robots.txt","bytes":0,"groups":0,"error":"timed out after 10 s"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":3241,"domain":"yaml.org","origin":"https://yaml.org","robots":{"status":"missing","http":404,"url":"https://yaml.org/robots.txt","bytes":21358,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":21358},"home":{"status":"ok","http":200,"url":"https://yaml.org/","html_tokens":6288,"markdown_tokens":246,"pruned_tokens":246}},
{"rank":3242,"domain":"amazon.com.br","origin":"https://amazon.com.br","robots":{"status":"ok","http":200,"url":"https://www.amazon.com.br/robots.txt","bytes":5764,"groups":101},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["blocked","named",1,0],"OAI-SearchBot":["blocked","named",1,0],"ClaudeBot":["blocked","named",2,0],"Claude-User":["blocked","named",1,0],"Claude-SearchBot":["blocked","named",1,0],"anthropic-ai":["partial","*",21,47],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["blocked","named",1,0],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["partial","*",21,47],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["partial","*",21,47],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",21,47]},"llms":{"status":"http_error","http":503,"bytes":1644},"home":{"status":"ok","http":200,"url":"https://www.amazon.com.br/","html_tokens":324093,"markdown_tokens":94,"pruned_tokens":94}},
{"rank":3243,"domain":"timesunion.com","origin":"https://timesunion.com","robots":{"status":"ok","http":200,"url":"https://www.timesunion.com/robots.txt","bytes":1082,"groups":3},"bots":{"GPTBot":["partial","*",9,0],"ChatGPT-User":["partial","*",9,0],"OAI-SearchBot":["partial","*",9,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["blocked","named",1,0],"Claude-SearchBot":["partial","*",9,0],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["blocked","named",1,0],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["partial","*",9,0],"meta-externalagent":["partial","*",9,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",9,0]},"llms":{"status":"missing","http":404,"bytes":635893},"home":{"status":"ok","http":200,"url":"https://www.timesunion.com/","html_tokens":377871,"markdown_tokens":19,"pruned_tokens":19}},
{"rank":3244,"domain":"makezine.com","origin":"https://makezine.com","robots":{"status":"ok","http":200,"url":"https://makezine.com/robots.txt","bytes":145,"groups":1},"bots":{"GPTBot":["partial","*",2,0],"ChatGPT-User":["partial","*",2,0],"OAI-SearchBot":["partial","*",2,0],"ClaudeBot":["partial","*",2,0],"Claude-User":["partial","*",2,0],"Claude-SearchBot":["partial","*",2,0],"anthropic-ai":["partial","*",2,0],"PerplexityBot":["partial","*",2,0],"Perplexity-User":["partial","*",2,0],"Google-Extended":["partial","*",2,0],"Applebot-Extended":["partial","*",2,0],"CCBot":["partial","*",2,0],"Bytespider":["partial","*",2,0],"Amazonbot":["partial","*",2,0],"meta-externalagent":["partial","*",2,0],"cohere-ai":["partial","*",2,0],"Diffbot":["partial","*",2,0],"ContextFetchBot":["partial","*",2,0]},"llms":{"status":"http_error","http":403,"bytes":5565},"home":{"status":"http_error","http":403,"url":"https://makezine.com/"}},
{"rank":3245,"domain":"nongnu.org","origin":"https://www.nongnu.org","robots":{"status":"missing","http":404,"url":"https://www.nongnu.org/robots.txt","bytes":984,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":984},"home":{"status":"ok","http":200,"url":"http://savannah.nongnu.org/","html_tokens":5184,"markdown_tokens":1246,"pruned_tokens":54}},
{"rank":3246,"domain":"ad.nl","origin":"https://ad.nl","robots":{"status":"ok","http":200,"url":"https://www.ad.nl/robots.txt","bytes":2860,"groups":30},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["blocked","named",1,0],"OAI-SearchBot":["blocked","named",1,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",16,0],"Claude-SearchBot":["partial","*",16,0],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["blocked","named",1,0],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["partial","*",16,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",16,0]},"llms":{"status":"missing","http":404,"bytes":161689},"home":{"status":"ok","http":200,"url":"https://www.ad.nl/","html_tokens":148314,"markdown_tokens":5389,"pruned_tokens":173}},
{"rank":3247,"domain":"senat.fr","origin":"https://senat.fr","robots":{"status":"ok","http":200,"url":"https://www.senat.fr/robots.txt","bytes":916,"groups":1},"bots":{"GPTBot":["partial","*",17,3],"ChatGPT-User":["partial","*",17,3],"OAI-SearchBot":["partial","*",17,3],"ClaudeBot":["partial","*",17,3],"Claude-User":["partial","*",17,3],"Claude-SearchBot":["partial","*",17,3],"anthropic-ai":["partial","*",17,3],"PerplexityBot":["partial","*",17,3],"Perplexity-User":["partial","*",17,3],"Google-Extended":["partial","*",17,3],"Applebot-Extended":["partial","*",17,3],"CCBot":["partial","*",17,3],"Bytespider":["partial","*",17,3],"Amazonbot":["partial","*",17,3],"meta-externalagent":["partial","*",17,3],"cohere-ai":["partial","*",17,3],"Diffbot":["partial","*",17,3],"ContextFetchBot":["partial","*",17,3]},"llms":{"status":"missing","http":404,"bytes":137760},"home":{"status":"ok","http":200,"url":"https://www.senat.fr/","html_tokens":52069,"markdown_tokens":95,"pruned_tokens":95}},
{"rank":3248,"domain":"se.net","origin":"https://se.net","robots":{"status":"ok","http":200,"url":"https://se.net/robots.txt","bytes":0,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":7603},"home":{"status":"ok","http":200,"url":"https://se.net/","html_tokens":8930,"markdown_tokens":807,"pruned_tokens":791}},
{"rank":3249,"domain":"expressvpn.com","origin":"https://expressvpn.com","robots":{"status":"ok","http":200,"url":"https://www.expressvpn.com/robots.txt","bytes":1136,"groups":6},"bots":{"GPTBot":["partial","*",19,0],"ChatGPT-User":["partial","*",19,0],"OAI-SearchBot":["partial","*",19,0],"ClaudeBot":["partial","*",19,0],"Claude-User":["partial","*",19,0],"Claude-SearchBot":["partial","*",19,0],"anthropic-ai":["partial","*",19,0],"PerplexityBot":["partial","*",19,0],"Perplexity-User":["partial","*",19,0],"Google-Extended":["partial","*",19,0],"Applebot-Extended":["partial","*",19,0],"CCBot":["partial","*",19,0],"Bytespider":["partial","*",19,0],"Amazonbot":["partial","*",19,0],"meta-externalagent":["partial","*",19,0],"cohere-ai":["partial","*",19,0],"Diffbot":["partial","*",19,0],"ContextFetchBot":["partial","*",19,0]},"llms":{"status":"missing","http":404,"bytes":94007},"home":{"status":"ok","http":200,"url":"https://www.expressvpn.com/","html_tokens":159744,"markdown_tokens":1637,"pruned_tokens":369}},
{"rank":3250,"domain":"rai.it","origin":"https://rai.it","robots":{"status":"ok","http":200,"url":"https://www.rai.it/robots.txt","bytes":5815,"groups":1},"bots":{"GPTBot":["partial","*",74,0],"ChatGPT-User":["partial","*",74,0],"OAI-SearchBot":["partial","*",74,0],"ClaudeBot":["partial","*",74,0],"Claude-User":["partial","*",74,0],"Claude-SearchBot":["partial","*",74,0],"anthropic-ai":["partial","*",74,0],"PerplexityBot":["partial","*",74,0],"Perplexity-User":["partial","*",74,0],"Google-Extended":["partial","*",74,0],"Applebot-Extended":["partial","*",74,0],"CCBot":["partial","*",74,0],"Bytespider":["partial","*",74,0],"Amazonbot":["partial","*",74,0],"meta-externalagent":["partial","*",74,0],"cohere-ai":["partial","*",74,0],"Diffbot":["partial","*",74,0],"ContextFetchBot":["partial","*",74,0]},"llms":{"status":"missing","http":404,"bytes":7255},"home":{"status":"ok","http":200,"url":"https://www.rai.it/","html_tokens":6079,"markdown_tokens":29,"pruned_tokens":29}},
{"rank":3251,"domain":"payscale.com","origin":"https://payscale.com","robots":{"status":"ok","http":200,"url":"https://www.payscale.com/robots.txt","bytes":1136,"groups":3},"bots":{"GPTBot":["partial","*",33,0],"ChatGPT-User":["partial","*",33,0],"OAI-SearchBot":["partial","*",33,0],"ClaudeBot":["partial","*",33,0],"Claude-User":["partial","*",33,0],"Claude-SearchBot":["partial","*",33,0],"anthropic-ai":["partial","*",33,0],"PerplexityBot":["partial","*",33,0],"Perplexity-User":["partial","*",33,0],"Google-Extended":["partial","*",33,0],"Applebot-Extended":["partial","*",33,0],"CCBot":["partial","*",33,0],"Bytespider":["partial","*",33,0],"Amazonbot":["partial","*",33,0],"meta-externalagent":["partial","*",33,0],"cohere-ai":["partial","*",33,0],"Diffbot":["partial","*",33,0],"ContextFetchBot":["partial","*",33,0]},"llms":{"status":"missing","http":404,"bytes":351},"home":{"status":"ok","http":200,"url":"https://www.payscale.com/","html_tokens":55480,"markdown_tokens":2344,"pruned_tokens":573}},
{"rank":3252,"domain":"zippia.com","origin":"https://zippia.com","robots":{"status":"ok","http":200,"url":"https://www.zippia.com/robots.txt","bytes":621,"groups":2},"bots":{"GPTBot":["partial","*",21,1],"ChatGPT-User":["partial","*",21,1],"OAI-SearchBot":["partial","*",21,1],"ClaudeBot":["partial","*",21,1],"Claude-User":["partial","*",21,1],"Claude-SearchBot":["partial","*",21,1],"anthropic-ai":["partial","*",21,1],"PerplexityBot":["partial","*",21,1],"Perplexity-User":["partial","*",21,1],"Google-Extended":["partial","*",21,1],"Applebot-Extended":["partial","*",21,1],"CCBot":["partial","*",21,1],"Bytespider":["partial","*",21,1],"Amazonbot":["partial","*",21,1],"meta-externalagent":["partial","*",21,1],"cohere-ai":["partial","*",21,1],"Diffbot":["partial","*",21,1],"ContextFetchBot":["partial","*",21,1]},"llms":{"status":"html","http":200,"bytes":72477},"home":{"status":"ok","http":200,"url":"https://www.zippia.com/","html_tokens":49344,"markdown_tokens":362,"pruned_tokens":160}},
{"rank":3253,"domain":"vocal.media","origin":"https://vocal.media","robots":{"status":"ok","http":200,"url":"https://vocal.media/robots.txt","bytes":63,"groups":1},"bots":{"GPTBot":["partial","*",2,0],"ChatGPT-User":["partial","*",2,0],"OAI-SearchBot":["partial","*",2,0],"ClaudeBot":["partial","*",2,0],"Claude-User":["partial","*",2,0],"Claude-SearchBot":["partial","*",2,0],"anthropic-ai":["partial","*",2,0],"PerplexityBot":["partial","*",2,0],"Perplexity-User":["partial","*",2,0],"Google-Extended":["partial","*",2,0],"Applebot-Extended":["partial","*",2,0],"CCBot":["partial","*",2,0],"Bytespider":["partial","*",2,0],"Amazonbot":["partial","*",2,0],"meta-externalagent":["partial","*",2,0],"cohere-ai":["partial","*",2,0],"Diffbot":["partial","*",2,0],"ContextFetchBot":["partial","*",2,0]},"llms":{"status":"missing","http":404,"bytes":57976},"home":{"status":"ok","http":200,"url":"https://vocal.media/","html_tokens":45930,"markdown_tokens":134,"pruned_tokens":134}},
{"rank":3254,"domain":"netgear.com","origin":"https://netgear.com","robots":{"status":"ok","http":200,"url":"https://www.netgear.com/robots.txt","bytes":2122,"groups":1},"bots":{"GPTBot":["partial","*",4,0],"ChatGPT-User":["partial","*",4,0],"OAI-SearchBot":["partial","*",4,0],"ClaudeBot":["partial","*",4,0],"Claude-User":["partial","*",4,0],"Claude-SearchBot":["partial","*",4,0],"anthropic-ai":["partial","*",4,0],"PerplexityBot":["partial","*",4,0],"Perplexity-User":["partial","*",4,0],"Google-Extended":["partial","*",4,0],"Applebot-Extended":["partial","*",4,0],"CCBot":["partial","*",4,0],"Bytespider":["partial","*",4,0],"Amazonbot":["partial","*",4,0],"meta-externalagent":["partial","*",4,0],"cohere-ai":["partial","*",4,0],"Diffbot":["partial","*",4,0],"ContextFetchBot":["partial","*",4,0]},"llms":{"status":"present","http":200,"bytes":17122},"home":{"status":"ok","http":200,"url":"https://www.netgear.com/","html_tokens":125357,"markdown_tokens":165,"pruned_tokens":165}},
{"rank":3255,"domain":"lilly.com","origin":"https://lilly.com","robots":{"status":"ok","http":200,"url":"https://www.lilly.com/robots.txt","bytes":1977,"groups":1},"bots":{"GPTBot":["partial","*",21,0],"ChatGPT-User":["partial","*",21,0],"OAI-SearchBot":["partial","*",21,0],"ClaudeBot":["partial","*",21,0],"Claude-User":["partial","*",21,0],"Claude-SearchBot":["partial","*",21,0],"anthropic-ai":["partial","*",21,0],"PerplexityBot":["partial","*",21,0],"Perplexity-User":["partial","*",21,0],"Google-Extended":["partial","*",21,0],"Applebot-Extended":["partial","*",21,0],"CCBot":["partial","*",21,0],"Bytespider":["partial","*",21,0],"Amazonbot":["partial","*",21,0],"meta-externalagent":["partial","*",21,0],"cohere-ai":["partial","*",21,0],"Diffbot":["partial","*",21,0],"ContextFetchBot":["partial","*",21,0]},"llms":{"status":"missing","http":404,"bytes":758},"home":{"status":"http_error","http":403,"url":"https://www.lilly.com/"}},
{"rank":3256,"domain":"on24.com","origin":"https://on24.com","robots":{"status":"refused","http":403,"url":"https://on24.com/robots.txt","bytes":103,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":3257,"domain":"uchile.cl","origin":"https://uchile.cl","robots":{"status":"ok","http":200,"url":"https://uchile.cl/robots.txt","bytes":163,"groups":2},"bots":{"GPTBot":["partial","*",4,1],"ChatGPT-User":["partial","*",4,1],"OAI-SearchBot":["partial","*",4,1],"ClaudeBot":["partial","*",4,1],"Claude-User":["partial","*",4,1],"Claude-SearchBot":["partial","*",4,1],"anthropic-ai":["partial","*",4,1],"PerplexityBot":["partial","*",4,1],"Perplexity-User":["partial","*",4,1],"Google-Extended":["partial","*",4,1],"Applebot-Extended":["partial","*",4,1],"CCBot":["partial","*",4,1],"Bytespider":["partial","*",4,1],"Amazonbot":["partial","*",4,1],"meta-externalagent":["partial","*",4,1],"cohere-ai":["partial","*",4,1],"Diffbot":["partial","*",4,1],"ContextFetchBot":["partial","*",4,1]},"llms":{"status":"missing","http":404,"bytes":11},"home":{"status":"ok","http":200,"url":"https://uchile.cl/","html_tokens":22450,"markdown_tokens":128,"pruned_tokens":128}},
{"rank":3258,"domain":"steemit.com","origin":"https://steemit.com","robots":{"status":"ok","http":200,"url":"https://steemit.com/robots.txt","bytes":22,"groups":1},"bots":{"GPTBot":["allowed","*",0,1],"ChatGPT-User":["allowed","*",0,1],"OAI-SearchBot":["allowed","*",0,1],"ClaudeBot":["allowed","*",0,1],"Claude-User":["allowed","*",0,1],"Claude-SearchBot":["allowed","*",0,1],"anthropic-ai":["allowed","*",0,1],"PerplexityBot":["allowed","*",0,1],"Perplexity-User":["allowed","*",0,1],"Google-Extended":["allowed","*",0,1],"Applebot-Extended":["allowed","*",0,1],"CCBot":["allowed","*",0,1],"Bytespider":["allowed","*",0,1],"Amazonbot":["allowed","*",0,1],"meta-externalagent":["allowed","*",0,1],"cohere-ai":["allowed","*",0,1],"Diffbot":["allowed","*",0,1],"ContextFetchBot":["allowed","*",0,1]},"llms":{"status":"html","http":200,"bytes":58751},"home":{"status":"ok","http":200,"url":"https://steemit.com/","html_tokens":137140,"markdown_tokens":1395,"pruned_tokens":606}},
{"rank":3259,"domain":"allegro.pl","origin":"https://allegro.pl","robots":{"status":"ok","http":200,"url":"https://allegro.pl/robots.txt","bytes":1363,"groups":3},"bots":{"GPTBot":["partial","*",33,3],"ChatGPT-User":["partial","*",33,3],"OAI-SearchBot":["partial","*",33,3],"ClaudeBot":["partial","*",33,3],"Claude-User":["partial","*",33,3],"Claude-SearchBot":["partial","*",33,3],"anthropic-ai":["partial","*",33,3],"PerplexityBot":["partial","*",33,3],"Perplexity-User":["partial","*",33,3],"Google-Extended":["partial","*",33,3],"Applebot-Extended":["partial","*",33,3],"CCBot":["partial","named",8,1],"Bytespider":["partial","*",33,3],"Amazonbot":["partial","*",33,3],"meta-externalagent":["partial","*",33,3],"cohere-ai":["partial","*",33,3],"Diffbot":["partial","*",33,3],"ContextFetchBot":["partial","*",33,3]},"llms":{"status":"present","http":200,"bytes":14037},"home":{"status":"http_error","http":403,"url":"https://allegro.pl/"}},
{"rank":3260,"domain":"etymonline.com","origin":"https://etymonline.com","robots":{"status":"ok","http":200,"url":"https://www.etymonline.com/robots.txt","bytes":1410,"groups":23},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["blocked","named",1,0],"OAI-SearchBot":["partial","*",6,2],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",6,2],"Claude-SearchBot":["partial","*",6,2],"anthropic-ai":["partial","*",6,2],"PerplexityBot":["partial","*",6,2],"Perplexity-User":["partial","*",6,2],"Google-Extended":["partial","*",6,2],"Applebot-Extended":["partial","*",6,2],"CCBot":["blocked","named",1,0],"Bytespider":["partial","*",6,2],"Amazonbot":["partial","*",6,2],"meta-externalagent":["partial","*",6,2],"cohere-ai":["partial","*",6,2],"Diffbot":["partial","*",6,2],"ContextFetchBot":["partial","*",6,2]},"llms":{"status":"missing","http":404,"bytes":101},"home":{"status":"http_error","http":403,"url":"https://www.etymonline.com/"}},
{"rank":3261,"domain":"penguin.co.uk","origin":"https://penguin.co.uk","robots":{"status":"ok","http":200,"url":"https://www.penguin.co.uk/robots.txt","bytes":308,"groups":2},"bots":{"GPTBot":["partial","*",7,1],"ChatGPT-User":["partial","*",7,1],"OAI-SearchBot":["partial","*",7,1],"ClaudeBot":["partial","*",7,1],"Claude-User":["partial","*",7,1],"Claude-SearchBot":["partial","*",7,1],"anthropic-ai":["partial","*",7,1],"PerplexityBot":["partial","*",7,1],"Perplexity-User":["partial","*",7,1],"Google-Extended":["partial","*",7,1],"Applebot-Extended":["partial","*",7,1],"CCBot":["partial","*",7,1],"Bytespider":["partial","*",7,1],"Amazonbot":["partial","*",7,1],"meta-externalagent":["partial","*",7,1],"cohere-ai":["partial","*",7,1],"Diffbot":["partial","*",7,1],"ContextFetchBot":["partial","*",7,1]},"llms":{"status":"html","http":200,"bytes":518702},"home":{"status":"ok","http":200,"url":"https://www.penguin.co.uk","html_tokens":129643,"markdown_tokens":214,"pruned_tokens":38}},
{"rank":3262,"domain":"onetrust.com","origin":"https://onetrust.com","robots":{"status":"ok","http":200,"url":"https://www.onetrust.com/robots.txt","bytes":396,"groups":1},"bots":{"GPTBot":["partial","*",16,0],"ChatGPT-User":["partial","*",16,0],"OAI-SearchBot":["partial","*",16,0],"ClaudeBot":["partial","*",16,0],"Claude-User":["partial","*",16,0],"Claude-SearchBot":["partial","*",16,0],"anthropic-ai":["partial","*",16,0],"PerplexityBot":["partial","*",16,0],"Perplexity-User":["partial","*",16,0],"Google-Extended":["partial","*",16,0],"Applebot-Extended":["partial","*",16,0],"CCBot":["partial","*",16,0],"Bytespider":["partial","*",16,0],"Amazonbot":["partial","*",16,0],"meta-externalagent":["partial","*",16,0],"cohere-ai":["partial","*",16,0],"Diffbot":["partial","*",16,0],"ContextFetchBot":["partial","*",16,0]},"llms":{"status":"present","http":200,"bytes":6859},"home":{"status":"ok","http":200,"url":"https://www.onetrust.com/","html_tokens":64072,"markdown_tokens":403,"pruned_tokens":160}},
{"rank":3263,"domain":"vnexpress.net","origin":"https://vnexpress.net","robots":{"status":"missing","http":404,"url":"https://vnexpress.net/robots.txt","bytes":2888,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"http_error","http":406,"bytes":27},"home":{"status":"http_error","http":404,"url":"https://vnexpress.net/"}},
{"rank":3264,"domain":"720yun.com","origin":"https://720yun.com","robots":{"status":"ok","http":200,"url":"https://720yun.com/robots.txt","bytes":587,"groups":7},"bots":{"GPTBot":["allowed","*",0,1],"ChatGPT-User":["allowed","named",0,1],"OAI-SearchBot":["allowed","named",0,1],"ClaudeBot":["allowed","*",0,1],"Claude-User":["allowed","named",0,1],"Claude-SearchBot":["allowed","named",0,1],"anthropic-ai":["allowed","*",0,1],"PerplexityBot":["allowed","named",0,1],"Perplexity-User":["allowed","named",0,1],"Google-Extended":["allowed","*",0,1],"Applebot-Extended":["allowed","*",0,1],"CCBot":["allowed","*",0,1],"Bytespider":["allowed","*",0,1],"Amazonbot":["allowed","*",0,1],"meta-externalagent":["allowed","*",0,1],"cohere-ai":["allowed","*",0,1],"Diffbot":["allowed","*",0,1],"ContextFetchBot":["allowed","*",0,1]},"llms":{"status":"present","http":200,"bytes":1648},"home":{"status":"ok","http":200,"url":"https://www.720yun.com/","html_tokens":56564,"markdown_tokens":165,"pruned_tokens":165}},
{"rank":3265,"domain":"trezor.io","origin":"https://trezor.io","robots":{"status":"ok","http":200,"url":"https://trezor.io/robots.txt","bytes":902,"groups":1},"bots":{"GPTBot":["partial","*",8,0],"ChatGPT-User":["partial","*",8,0],"OAI-SearchBot":["partial","*",8,0],"ClaudeBot":["partial","*",8,0],"Claude-User":["partial","*",8,0],"Claude-SearchBot":["partial","*",8,0],"anthropic-ai":["partial","*",8,0],"PerplexityBot":["partial","*",8,0],"Perplexity-User":["partial","*",8,0],"Google-Extended":["partial","*",8,0],"Applebot-Extended":["partial","*",8,0],"CCBot":["partial","*",8,0],"Bytespider":["partial","*",8,0],"Amazonbot":["partial","*",8,0],"meta-externalagent":["partial","*",8,0],"cohere-ai":["partial","*",8,0],"Diffbot":["partial","*",8,0],"ContextFetchBot":["partial","*",8,0]},"llms":{"status":"present","http":200,"bytes":2938},"home":{"status":"ok","http":200,"url":"https://trezor.io/","html_tokens":172128,"markdown_tokens":972,"pruned_tokens":378}},
{"rank":3266,"domain":"tldp.org","origin":"https://tldp.org","robots":{"status":"ok","http":200,"url":"https://tldp.org/robots.txt","bytes":31,"groups":1},"bots":{"GPTBot":["partial","*",1,0],"ChatGPT-User":["partial","*",1,0],"OAI-SearchBot":["partial","*",1,0],"ClaudeBot":["partial","*",1,0],"Claude-User":["partial","*",1,0],"Claude-SearchBot":["partial","*",1,0],"anthropic-ai":["partial","*",1,0],"PerplexityBot":["partial","*",1,0],"Perplexity-User":["partial","*",1,0],"Google-Extended":["partial","*",1,0],"Applebot-Extended":["partial","*",1,0],"CCBot":["partial","*",1,0],"Bytespider":["partial","*",1,0],"Amazonbot":["partial","*",1,0],"meta-externalagent":["partial","*",1,0],"cohere-ai":["partial","*",1,0],"Diffbot":["partial","*",1,0],"ContextFetchBot":["partial","*",1,0]},"llms":{"status":"missing","http":404,"bytes":0},"home":{"status":"ok","http":200,"url":"https://tldp.org/","html_tokens":2522,"markdown_tokens":134,"pruned_tokens":134}},
{"rank":3267,"domain":"imgbb.com","origin":"https://imgbb.com","robots":{"status":"ok","http":200,"url":"https://imgbb.com/robots.txt","bytes":0,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":150},"home":{"status":"ok","http":200,"url":"https://imgbb.com/","html_tokens":10679,"markdown_tokens":85,"pruned_tokens":19}},
{"rank":3268,"domain":"vivo.com.cn","origin":"https://vivo.com.cn","robots":{"status":"ok","http":200,"url":"https://www.vivo.com.cn/robots.txt","bytes":602,"groups":1},"bots":{"GPTBot":["partial","*",17,0],"ChatGPT-User":["partial","*",17,0],"OAI-SearchBot":["partial","*",17,0],"ClaudeBot":["partial","*",17,0],"Claude-User":["partial","*",17,0],"Claude-SearchBot":["partial","*",17,0],"anthropic-ai":["partial","*",17,0],"PerplexityBot":["partial","*",17,0],"Perplexity-User":["partial","*",17,0],"Google-Extended":["partial","*",17,0],"Applebot-Extended":["partial","*",17,0],"CCBot":["partial","*",17,0],"Bytespider":["partial","*",17,0],"Amazonbot":["partial","*",17,0],"meta-externalagent":["partial","*",17,0],"cohere-ai":["partial","*",17,0],"Diffbot":["partial","*",17,0],"ContextFetchBot":["partial","*",17,0]},"llms":{"status":"missing","http":404,"bytes":0},"home":{"status":"ok","http":200,"url":"https://www.vivo.com.cn/","html_tokens":16654,"markdown_tokens":45,"pruned_tokens":45}},
{"rank":3269,"domain":"dailykos.com","origin":"https://dailykos.com","robots":{"status":"ok","http":200,"url":"https://www.dailykos.com/robots.txt","bytes":710,"groups":3},"bots":{"GPTBot":["partial","named",1,1],"ChatGPT-User":["partial","*",2,2],"OAI-SearchBot":["partial","*",2,2],"ClaudeBot":["partial","named",1,1],"Claude-User":["partial","*",2,2],"Claude-SearchBot":["partial","*",2,2],"anthropic-ai":["partial","*",2,2],"PerplexityBot":["partial","*",2,2],"Perplexity-User":["partial","*",2,2],"Google-Extended":["partial","named",1,1],"Applebot-Extended":["partial","named",1,1],"CCBot":["partial","named",1,1],"Bytespider":["partial","*",2,2],"Amazonbot":["partial","named",1,1],"meta-externalagent":["partial","named",1,1],"cohere-ai":["partial","*",2,2],"Diffbot":["partial","*",2,2],"ContextFetchBot":["partial","*",2,2]},"llms":{"status":"missing","http":404,"bytes":180747},"home":{"status":"ok","http":200,"url":"https://www.dailykos.com/","html_tokens":101057,"markdown_tokens":91,"pruned_tokens":76}},
{"rank":3270,"domain":"hh.ru","origin":"https://hh.ru","robots":{"status":"ok","http":200,"url":"https://hh.ru/robots.txt","bytes":10235,"groups":20},"bots":{"GPTBot":["partial","named",3,0],"ChatGPT-User":["partial","named",3,0],"OAI-SearchBot":["partial","named",3,0],"ClaudeBot":["partial","named",3,0],"Claude-User":["partial","*",14,3],"Claude-SearchBot":["partial","*",14,3],"anthropic-ai":["partial","named",3,0],"PerplexityBot":["partial","named",3,0],"Perplexity-User":["partial","*",14,3],"Google-Extended":["partial","named",3,0],"Applebot-Extended":["partial","*",14,3],"CCBot":["partial","named",3,0],"Bytespider":["partial","*",14,3],"Amazonbot":["partial","*",14,3],"meta-externalagent":["partial","*",14,3],"cohere-ai":["partial","*",14,3],"Diffbot":["partial","*",14,3],"ContextFetchBot":["partial","*",14,3]},"llms":{"status":"missing","http":404,"bytes":379265},"home":{"status":"ok","http":200,"url":"https://hh.ru/","html_tokens":436038,"markdown_tokens":21,"pruned_tokens":21}},
{"rank":3271,"domain":"weblate.org","origin":"https://weblate.org","robots":{"status":"ok","http":200,"url":"https://weblate.org/robots.txt","bytes":159,"groups":1},"bots":{"GPTBot":["partial","*",3,0],"ChatGPT-User":["partial","*",3,0],"OAI-SearchBot":["partial","*",3,0],"ClaudeBot":["partial","*",3,0],"Claude-User":["partial","*",3,0],"Claude-SearchBot":["partial","*",3,0],"anthropic-ai":["partial","*",3,0],"PerplexityBot":["partial","*",3,0],"Perplexity-User":["partial","*",3,0],"Google-Extended":["partial","*",3,0],"Applebot-Extended":["partial","*",3,0],"CCBot":["partial","*",3,0],"Bytespider":["partial","*",3,0],"Amazonbot":["partial","*",3,0],"meta-externalagent":["partial","*",3,0],"cohere-ai":["partial","*",3,0],"Diffbot":["partial","*",3,0],"ContextFetchBot":["partial","*",3,0]},"llms":{"status":"present","http":200,"bytes":3226},"home":{"status":"ok","http":200,"url":"https://weblate.org/en/","html_tokens":10860,"markdown_tokens":252,"pruned_tokens":177}},
{"rank":3272,"domain":"decrypt.co","origin":"https://decrypt.co","robots":{"status":"ok","http":200,"url":"https://decrypt.co/robots.txt","bytes":160,"groups":1},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"missing","http":404,"bytes":458613},"home":{"status":"ok","http":200,"url":"https://decrypt.co/","html_tokens":662276,"markdown_tokens":602,"pruned_tokens":58}},
{"rank":3273,"domain":"ofcom.org.uk","origin":"https://ofcom.org.uk","robots":{"status":"refused","http":403,"url":"https://www.ofcom.org.uk/robots.txt","bytes":5575,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":3274,"domain":"louisvuitton.com","origin":"https://www.louisvuitton.com","robots":{"status":"ok","http":200,"url":"https://www.louisvuitton.com/robots.txt","bytes":317,"groups":2},"bots":{"GPTBot":["partial","named",1,0],"ChatGPT-User":["partial","*",3,0],"OAI-SearchBot":["partial","*",3,0],"ClaudeBot":["partial","named",1,0],"Claude-User":["partial","*",3,0],"Claude-SearchBot":["partial","*",3,0],"anthropic-ai":["partial","*",3,0],"PerplexityBot":["partial","named",1,0],"Perplexity-User":["partial","*",3,0],"Google-Extended":["partial","named",1,0],"Applebot-Extended":["partial","named",1,0],"CCBot":["partial","named",1,0],"Bytespider":["partial","*",3,0],"Amazonbot":["partial","*",3,0],"meta-externalagent":["partial","*",3,0],"cohere-ai":["partial","*",3,0],"Diffbot":["partial","*",3,0],"ContextFetchBot":["partial","*",3,0]},"llms":{"status":"missing","http":404,"bytes":12},"home":{"status":"timeout","error":"timed out after 15 s"}},
{"rank":3275,"domain":"uni-hannover.de","origin":"https://uni-hannover.de","robots":{"status":"ok","http":200,"url":"https://www.uni-hannover.de/robots.php","bytes":330,"groups":1},"bots":{"GPTBot":["partial","*",7,0],"ChatGPT-User":["partial","*",7,0],"OAI-SearchBot":["partial","*",7,0],"ClaudeBot":["partial","*",7,0],"Claude-User":["partial","*",7,0],"Claude-SearchBot":["partial","*",7,0],"anthropic-ai":["partial","*",7,0],"PerplexityBot":["partial","*",7,0],"Perplexity-User":["partial","*",7,0],"Google-Extended":["partial","*",7,0],"Applebot-Extended":["partial","*",7,0],"CCBot":["partial","*",7,0],"Bytespider":["partial","*",7,0],"Amazonbot":["partial","*",7,0],"meta-externalagent":["partial","*",7,0],"cohere-ai":["partial","*",7,0],"Diffbot":["partial","*",7,0],"ContextFetchBot":["partial","*",7,0]},"llms":{"status":"missing","http":404,"bytes":55048},"home":{"status":"ok","http":200,"url":"https://www.uni-hannover.de/","html_tokens":32863,"markdown_tokens":137,"pruned_tokens":127}},
{"rank":3276,"domain":"tulane.edu","origin":"https://tulane.edu","robots":{"status":"refused","http":403,"url":"https://tulane.edu/robots.txt","bytes":560,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":3277,"domain":"campaign-archive1.com","origin":"https://campaign-archive1.com","robots":{"status":"unreachable","http":null,"url":"https://campaign-archive1.com/robots.txt","bytes":0,"groups":0,"error":"[SSL: CERTIFICATE_VERIFY_FAILED] certificate verify failed: Hostname mismatch, certificate is not valid for 'campaign-archive1.com'. (_ssl.c:1028)"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":3278,"domain":"chanel.com","origin":"https://chanel.com","robots":{"status":"refused","http":403,"url":"https://chanel.com/robots.txt","bytes":374,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":3279,"domain":"manchestereveningnews.co.uk","origin":"https://manchestereveningnews.co.uk","robots":{"status":"ok","http":200,"url":"https://www.manchestereveningnews.co.uk/robots.txt","bytes":996,"groups":13},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["partial","*",10,0],"OAI-SearchBot":["blocked","named",1,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",10,0],"Claude-SearchBot":["partial","*",10,0],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["partial","*",10,0],"Google-Extended":["partial","*",10,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["partial","*",10,0],"Amazonbot":["partial","*",10,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["partial","*",10,0],"Diffbot":["partial","*",10,0],"ContextFetchBot":["partial","*",10,0]},"llms":{"status":"missing","http":404,"bytes":807382},"home":{"status":"ok","http":200,"url":"https://www.manchestereveningnews.co.uk/","html_tokens":291399,"markdown_tokens":153,"pruned_tokens":153}},
{"rank":3280,"domain":"slashgear.com","origin":"https://slashgear.com","robots":{"status":"ok","http":200,"url":"https://slashgear.com/robots.txt","bytes":283,"groups":2},"bots":{"GPTBot":["partial","*",6,0],"ChatGPT-User":["partial","*",6,0],"OAI-SearchBot":["partial","*",6,0],"ClaudeBot":["partial","*",6,0],"Claude-User":["partial","*",6,0],"Claude-SearchBot":["partial","*",6,0],"anthropic-ai":["partial","*",6,0],"PerplexityBot":["partial","*",6,0],"Perplexity-User":["partial","*",6,0],"Google-Extended":["partial","*",6,0],"Applebot-Extended":["partial","*",6,0],"CCBot":["partial","*",6,0],"Bytespider":["partial","*",6,0],"Amazonbot":["partial","*",6,0],"meta-externalagent":["partial","*",6,0],"cohere-ai":["partial","*",6,0],"Diffbot":["partial","*",6,0],"ContextFetchBot":["partial","*",6,0]},"llms":{"status":"html","http":200,"bytes":102613},"home":{"status":"ok","http":200,"url":"https://www.slashgear.com/","html_tokens":25649,"markdown_tokens":1240,"pruned_tokens":126}},
{"rank":3281,"domain":"oanda.com","origin":"https://oanda.com","robots":{"status":"ok","http":200,"url":"https://www.oanda.com/robots.txt","bytes":378,"groups":2},"bots":{"GPTBot":["partial","*",8,0],"ChatGPT-User":["partial","*",8,0],"OAI-SearchBot":["partial","*",8,0],"ClaudeBot":["partial","*",8,0],"Claude-User":["partial","*",8,0],"Claude-SearchBot":["partial","*",8,0],"anthropic-ai":["partial","*",8,0],"PerplexityBot":["partial","*",8,0],"Perplexity-User":["partial","*",8,0],"Google-Extended":["partial","*",8,0],"Applebot-Extended":["partial","*",8,0],"CCBot":["partial","*",8,0],"Bytespider":["partial","*",8,0],"Amazonbot":["partial","*",8,0],"meta-externalagent":["partial","*",8,0],"cohere-ai":["partial","*",8,0],"Diffbot":["partial","*",8,0],"ContextFetchBot":["partial","*",8,0]},"llms":{"status":"missing","http":404,"bytes":12914},"home":{"status":"ok","http":200,"url":"https://www.oanda.com/us-en/","html_tokens":60734,"markdown_tokens":198,"pruned_tokens":134}},
{"rank":3282,"domain":"kansascity.com","origin":"https://kansascity.com","robots":{"status":"unreachable","http":null,"url":"https://kansascity.com/robots.txt","bytes":0,"groups":0,"error":"<StreamReset stream_id:1, error_code:2, remote_reset:True>"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":3283,"domain":"mindtools.com","origin":"https://mindtools.com","robots":{"status":"ok","http":200,"url":"https://www.mindtools.com/robots.txt","bytes":187,"groups":1},"bots":{"GPTBot":["partial","*",3,0],"ChatGPT-User":["partial","*",3,0],"OAI-SearchBot":["partial","*",3,0],"ClaudeBot":["partial","*",3,0],"Claude-User":["partial","*",3,0],"Claude-SearchBot":["partial","*",3,0],"anthropic-ai":["partial","*",3,0],"PerplexityBot":["partial","*",3,0],"Perplexity-User":["partial","*",3,0],"Google-Extended":["partial","*",3,0],"Applebot-Extended":["partial","*",3,0],"CCBot":["partial","*",3,0],"Bytespider":["partial","*",3,0],"Amazonbot":["partial","*",3,0],"meta-externalagent":["partial","*",3,0],"cohere-ai":["partial","*",3,0],"Diffbot":["partial","*",3,0],"ContextFetchBot":["partial","*",3,0]},"llms":{"status":"missing","http":404,"bytes":60},"home":{"status":"ok","http":200,"url":"https://www.mindtools.com/","html_tokens":30039,"markdown_tokens":629,"pruned_tokens":270}},
{"rank":3284,"domain":"gucci.com","origin":"https://gucci.com","robots":{"status":"unreachable","http":null,"url":"https://gucci.com/robots.txt","bytes":0,"groups":0,"error":"<StreamReset stream_id:1, error_code:2, remote_reset:True>"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":3285,"domain":"laweekly.com","origin":"https://laweekly.com","robots":{"status":"ok","http":200,"url":"https://www.laweekly.com/robots.txt","bytes":419,"groups":2},"bots":{"GPTBot":["partial","*",11,1],"ChatGPT-User":["partial","*",11,1],"OAI-SearchBot":["partial","*",11,1],"ClaudeBot":["partial","*",11,1],"Claude-User":["partial","*",11,1],"Claude-SearchBot":["partial","*",11,1],"anthropic-ai":["partial","*",11,1],"PerplexityBot":["partial","*",11,1],"Perplexity-User":["partial","*",11,1],"Google-Extended":["partial","*",11,1],"Applebot-Extended":["partial","*",11,1],"CCBot":["partial","*",11,1],"Bytespider":["partial","*",11,1],"Amazonbot":["partial","*",11,1],"meta-externalagent":["partial","*",11,1],"cohere-ai":["partial","*",11,1],"Diffbot":["partial","*",11,1],"ContextFetchBot":["partial","*",11,1]},"llms":{"status":"missing","http":404,"bytes":416991},"home":{"status":"ok","http":200,"url":"https://www.laweekly.com/","html_tokens":132213,"markdown_tokens":745,"pruned_tokens":126}},
{"rank":3286,"domain":"lodz.pl","origin":"https://lodz.pl","robots":{"status":"ok","http":200,"url":"https://lodz.pl/robots.txt","bytes":22,"groups":1},"bots":{"GPTBot":["allowed","*",0,1],"ChatGPT-User":["allowed","*",0,1],"OAI-SearchBot":["allowed","*",0,1],"ClaudeBot":["allowed","*",0,1],"Claude-User":["allowed","*",0,1],"Claude-SearchBot":["allowed","*",0,1],"anthropic-ai":["allowed","*",0,1],"PerplexityBot":["allowed","*",0,1],"Perplexity-User":["allowed","*",0,1],"Google-Extended":["allowed","*",0,1],"Applebot-Extended":["allowed","*",0,1],"CCBot":["allowed","*",0,1],"Bytespider":["allowed","*",0,1],"Amazonbot":["allowed","*",0,1],"meta-externalagent":["allowed","*",0,1],"cohere-ai":["allowed","*",0,1],"Diffbot":["allowed","*",0,1],"ContextFetchBot":["allowed","*",0,1]},"llms":{"status":"missing","http":404,"bytes":20611},"home":{"status":"ok","http":200,"url":"https://lodz.pl/","html_tokens":24986,"markdown_tokens":364,"pruned_tokens":121}},
{"rank":3287,"domain":"runway.com","origin":"https://runway.com","robots":{"status":"ok","http":200,"url":"https://runway.com/robots.txt","bytes":666,"groups":1},"bots":{"GPTBot":["partial","*",1,1],"ChatGPT-User":["partial","*",1,1],"OAI-SearchBot":["partial","*",1,1],"ClaudeBot":["partial","*",1,1],"Claude-User":["partial","*",1,1],"Claude-SearchBot":["partial","*",1,1],"anthropic-ai":["partial","*",1,1],"PerplexityBot":["partial","*",1,1],"Perplexity-User":["partial","*",1,1],"Google-Extended":["partial","*",1,1],"Applebot-Extended":["partial","*",1,1],"CCBot":["partial","*",1,1],"Bytespider":["partial","*",1,1],"Amazonbot":["partial","*",1,1],"meta-externalagent":["partial","*",1,1],"cohere-ai":["partial","*",1,1],"Diffbot":["partial","*",1,1],"ContextFetchBot":["partial","*",1,1]},"llms":{"status":"present","http":200,"bytes":8035},"home":{"status":"ok","http":200,"url":"https://runway.com/","html_tokens":296335,"markdown_tokens":336,"pruned_tokens":68}},
{"rank":3288,"domain":"francebleu.fr","origin":"https://francebleu.fr","robots":{"status":"ok","http":200,"url":"https://www.ici.fr/robots.txt","bytes":4252,"groups":73},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["blocked","named",1,0],"OAI-SearchBot":["blocked","named",1,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",20,0],"Claude-SearchBot":["partial","*",20,0],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["partial","*",20,0],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["partial","*",20,0],"ContextFetchBot":["partial","*",20,0]},"llms":{"status":"missing","http":404,"bytes":59209},"home":{"status":"ok","http":200,"url":"https://www.ici.fr/","html_tokens":246376,"markdown_tokens":5090,"pruned_tokens":173}},
{"rank":3289,"domain":"ruby-lang.org","origin":"https://ruby-lang.org","robots":{"status":"ok","http":200,"url":"https://www.ruby-lang.org/robots.txt","bytes":71,"groups":1},"bots":{"GPTBot":["allowed","*",0,1],"ChatGPT-User":["allowed","*",0,1],"OAI-SearchBot":["allowed","*",0,1],"ClaudeBot":["allowed","*",0,1],"Claude-User":["allowed","*",0,1],"Claude-SearchBot":["allowed","*",0,1],"anthropic-ai":["allowed","*",0,1],"PerplexityBot":["allowed","*",0,1],"Perplexity-User":["allowed","*",0,1],"Google-Extended":["allowed","*",0,1],"Applebot-Extended":["allowed","*",0,1],"CCBot":["allowed","*",0,1],"Bytespider":["allowed","*",0,1],"Amazonbot":["allowed","*",0,1],"meta-externalagent":["allowed","*",0,1],"cohere-ai":["allowed","*",0,1],"Diffbot":["allowed","*",0,1],"ContextFetchBot":["allowed","*",0,1]},"llms":{"status":"present","http":200,"bytes":9872},"home":{"status":"ok","http":200,"url":"https://www.ruby-lang.org/","html_tokens":287,"markdown_tokens":7,"pruned_tokens":7}},
{"rank":3290,"domain":"pconline.com.cn","origin":"https://pconline.com.cn","robots":{"status":"ok","http":200,"url":"https://www.pconline.com.cn/robots.txt","bytes":334,"groups":1},"bots":{"GPTBot":["partial","*",17,0],"ChatGPT-User":["partial","*",17,0],"OAI-SearchBot":["partial","*",17,0],"ClaudeBot":["partial","*",17,0],"Claude-User":["partial","*",17,0],"Claude-SearchBot":["partial","*",17,0],"anthropic-ai":["partial","*",17,0],"PerplexityBot":["partial","*",17,0],"Perplexity-User":["partial","*",17,0],"Google-Extended":["partial","*",17,0],"Applebot-Extended":["partial","*",17,0],"CCBot":["partial","*",17,0],"Bytespider":["partial","*",17,0],"Amazonbot":["partial","*",17,0],"meta-externalagent":["partial","*",17,0],"cohere-ai":["partial","*",17,0],"Diffbot":["partial","*",17,0],"ContextFetchBot":["partial","*",17,0]},"llms":{"status":"html","http":200,"bytes":4026},"home":{"status":"http_error","http":503,"url":"https://www.pconline.com.cn/"}},
{"rank":3291,"domain":"teenvogue.com","origin":"https://teenvogue.com","robots":{"status":"ok","http":200,"url":"https://www.teenvogue.com/robots.txt","bytes":2234,"groups":3},"bots":{"GPTBot":["partial","*",15,14],"ChatGPT-User":["partial","*",15,14],"OAI-SearchBot":["partial","*",15,14],"ClaudeBot":["blocked","named",1,0],"Claude-User":["blocked","named",1,0],"Claude-SearchBot":["blocked","named",1,0],"anthropic-ai":["partial","*",15,14],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["blocked","named",1,0],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",15,14]},"llms":{"status":"missing","http":404,"bytes":781388},"home":{"status":"ok","http":200,"url":"https://www.teenvogue.com/","html_tokens":487498,"markdown_tokens":117,"pruned_tokens":117}},
{"rank":3292,"domain":"oas.org","origin":"https://oas.org","robots":{"status":"ok","http":200,"url":"https://oas.org/robots.txt","bytes":4952,"groups":1},"bots":{"GPTBot":["partial","*",129,0],"ChatGPT-User":["partial","*",129,0],"OAI-SearchBot":["partial","*",129,0],"ClaudeBot":["partial","*",129,0],"Claude-User":["partial","*",129,0],"Claude-SearchBot":["partial","*",129,0],"anthropic-ai":["partial","*",129,0],"PerplexityBot":["partial","*",129,0],"Perplexity-User":["partial","*",129,0],"Google-Extended":["partial","*",129,0],"Applebot-Extended":["partial","*",129,0],"CCBot":["partial","*",129,0],"Bytespider":["partial","*",129,0],"Amazonbot":["partial","*",129,0],"meta-externalagent":["partial","*",129,0],"cohere-ai":["partial","*",129,0],"Diffbot":["partial","*",129,0],"ContextFetchBot":["partial","*",129,0]},"llms":{"status":"disallowed","error":"robots.txt of www.oas.org disallows /wearesorry.htm"},"home":{"status":"ok","http":200,"url":"https://www.oas.org/en/","html_tokens":16742,"markdown_tokens":672,"pruned_tokens":612}},
{"rank":3293,"domain":"vancouversun.com","origin":"https://vancouversun.com","robots":{"status":"ok","http":200,"url":"https://vancouversun.com/robots.txt","bytes":1227,"groups":15},"bots":{"GPTBot":["partial","*",3,6],"ChatGPT-User":["partial","*",3,6],"OAI-SearchBot":["partial","*",3,6],"ClaudeBot":["partial","*",3,6],"Claude-User":["partial","*",3,6],"Claude-SearchBot":["partial","*",3,6],"anthropic-ai":["partial","*",3,6],"PerplexityBot":["partial","*",3,6],"Perplexity-User":["partial","*",3,6],"Google-Extended":["partial","*",3,6],"Applebot-Extended":["partial","*",3,6],"CCBot":["partial","*",3,6],"Bytespider":["partial","*",3,6],"Amazonbot":["partial","*",3,6],"meta-externalagent":["partial","*",3,6],"cohere-ai":["partial","*",3,6],"Diffbot":["partial","*",3,6],"ContextFetchBot":["partial","*",3,6]},"llms":{"status":"missing","http":404,"bytes":312730},"home":{"status":"ok","http":200,"url":"https://vancouversun.com/","html_tokens":152198,"markdown_tokens":163,"pruned_tokens":163}},
{"rank":3294,"domain":"leo.org","origin":"https://leo.org","robots":{"status":"ok","http":200,"url":"https://www.leo.org/robots.txt","bytes":148,"groups":3},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["blocked","named",1,0],"OAI-SearchBot":["partial","*",1,0],"ClaudeBot":["partial","*",1,0],"Claude-User":["partial","*",1,0],"Claude-SearchBot":["partial","*",1,0],"anthropic-ai":["partial","*",1,0],"PerplexityBot":["partial","*",1,0],"Perplexity-User":["partial","*",1,0],"Google-Extended":["partial","*",1,0],"Applebot-Extended":["partial","*",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["partial","*",1,0],"Amazonbot":["partial","*",1,0],"meta-externalagent":["partial","*",1,0],"cohere-ai":["partial","*",1,0],"Diffbot":["partial","*",1,0],"ContextFetchBot":["partial","*",1,0]},"llms":{"status":"missing","http":404,"bytes":634},"home":{"status":"http_error","http":403,"url":"https://leo.org/"}},
{"rank":3295,"domain":"premierleague.com","origin":"https://premierleague.com","robots":{"status":"ok","http":200,"url":"https://www.premierleague.com/robots.txt","bytes":638,"groups":1},"bots":{"GPTBot":["partial","*",24,0],"ChatGPT-User":["partial","*",24,0],"OAI-SearchBot":["partial","*",24,0],"ClaudeBot":["partial","*",24,0],"Claude-User":["partial","*",24,0],"Claude-SearchBot":["partial","*",24,0],"anthropic-ai":["partial","*",24,0],"PerplexityBot":["partial","*",24,0],"Perplexity-User":["partial","*",24,0],"Google-Extended":["partial","*",24,0],"Applebot-Extended":["partial","*",24,0],"CCBot":["partial","*",24,0],"Bytespider":["partial","*",24,0],"Amazonbot":["partial","*",24,0],"meta-externalagent":["partial","*",24,0],"cohere-ai":["partial","*",24,0],"Diffbot":["partial","*",24,0],"ContextFetchBot":["partial","*",24,0]},"llms":{"status":"missing","http":404,"bytes":81850},"home":{"status":"ok","http":200,"url":"https://www.premierleague.com/","html_tokens":30801,"markdown_tokens":80,"pruned_tokens":80}},
{"rank":3296,"domain":"bostonherald.com","origin":"https://bostonherald.com","robots":{"status":"ok","http":200,"url":"https://www.bostonherald.com/robots.txt","bytes":4486,"groups":68},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["blocked","named",1,0],"OAI-SearchBot":["blocked","named",1,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["blocked","named",1,0],"Claude-SearchBot":["blocked","named",1,0],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["blocked","named",1,0],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["partial","*",8,1],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",8,1]},"llms":{"status":"missing","http":404,"bytes":97748},"home":{"status":"ok","http":200,"url":"https://www.bostonherald.com/","html_tokens":62960,"markdown_tokens":313,"pruned_tokens":155}},
{"rank":3297,"domain":"juntadeandalucia.es","origin":"https://juntadeandalucia.es","robots":{"status":"ok","http":200,"url":"https://www.juntadeandalucia.es/robots.txt","bytes":8823,"groups":2},"bots":{"GPTBot":["partial","*",273,1],"ChatGPT-User":["partial","*",273,1],"OAI-SearchBot":["partial","*",273,1],"ClaudeBot":["partial","*",273,1],"Claude-User":["partial","*",273,1],"Claude-SearchBot":["partial","*",273,1],"anthropic-ai":["partial","*",273,1],"PerplexityBot":["partial","*",273,1],"Perplexity-User":["partial","*",273,1],"Google-Extended":["partial","*",273,1],"Applebot-Extended":["partial","*",273,1],"CCBot":["partial","*",273,1],"Bytespider":["partial","*",273,1],"Amazonbot":["partial","*",273,1],"meta-externalagent":["partial","*",273,1],"cohere-ai":["partial","*",273,1],"Diffbot":["partial","*",273,1],"ContextFetchBot":["partial","*",273,1]},"llms":{"status":"missing","http":404,"bytes":33402},"home":{"status":"ok","http":200,"url":"https://www.juntadeandalucia.es/","html_tokens":49281,"markdown_tokens":278,"pruned_tokens":56}},
{"rank":3298,"domain":"zhaopin.com","origin":"https://zhaopin.com","robots":{"status":"ok","http":200,"url":"https://zhaopin.com/robots.txt","bytes":114,"groups":1},"bots":{"GPTBot":["partial","*",4,1],"ChatGPT-User":["partial","*",4,1],"OAI-SearchBot":["partial","*",4,1],"ClaudeBot":["partial","*",4,1],"Claude-User":["partial","*",4,1],"Claude-SearchBot":["partial","*",4,1],"anthropic-ai":["partial","*",4,1],"PerplexityBot":["partial","*",4,1],"Perplexity-User":["partial","*",4,1],"Google-Extended":["partial","*",4,1],"Applebot-Extended":["partial","*",4,1],"CCBot":["partial","*",4,1],"Bytespider":["partial","*",4,1],"Amazonbot":["partial","*",4,1],"meta-externalagent":["partial","*",4,1],"cohere-ai":["partial","*",4,1],"Diffbot":["partial","*",4,1],"ContextFetchBot":["partial","*",4,1]},"llms":{"status":"missing","http":404,"bytes":573},"home":{"status":"disallowed","error":"robots.txt of www.zhaopin.com not readable (html)"}},
{"rank":3299,"domain":"freelancer.com","origin":"https://freelancer.com","robots":{"status":"ok","http":200,"url":"https://www.freelancer.com/robots.txt","bytes":2620,"groups":4},"bots":{"GPTBot":["partial","*",38,5],"ChatGPT-User":["partial","*",38,5],"OAI-SearchBot":["partial","*",38,5],"ClaudeBot":["partial","*",38,5],"Claude-User":["partial","*",38,5],"Claude-SearchBot":["partial","*",38,5],"anthropic-ai":["partial","*",38,5],"PerplexityBot":["partial","*",38,5],"Perplexity-User":["partial","*",38,5],"Google-Extended":["partial","*",38,5],"Applebot-Extended":["partial","*",38,5],"CCBot":["partial","*",38,5],"Bytespider":["partial","*",38,5],"Amazonbot":["partial","*",38,5],"meta-externalagent":["partial","*",38,5],"cohere-ai":["partial","*",38,5],"Diffbot":["partial","*",38,5],"ContextFetchBot":["partial","*",38,5]},"llms":{"status":"missing","http":404,"bytes":129704},"home":{"status":"ok","http":200,"url":"https://www.freelancer.com/","html_tokens":273858,"markdown_tokens":551,"pruned_tokens":194}},
{"rank":3300,"domain":"grist.org","origin":"https://grist.org","robots":{"status":"ok","http":200,"url":"https://grist.org/robots.txt","bytes":360,"groups":2},"bots":{"GPTBot":["partial","*",1,0],"ChatGPT-User":["partial","*",1,0],"OAI-SearchBot":["partial","*",1,0],"ClaudeBot":["partial","*",1,0],"Claude-User":["partial","*",1,0],"Claude-SearchBot":["partial","*",1,0],"anthropic-ai":["partial","*",1,0],"PerplexityBot":["partial","*",1,0],"Perplexity-User":["partial","*",1,0],"Google-Extended":["partial","*",1,0],"Applebot-Extended":["partial","*",1,0],"CCBot":["partial","*",1,0],"Bytespider":["partial","*",1,0],"Amazonbot":["partial","*",1,0],"meta-externalagent":["partial","*",1,0],"cohere-ai":["partial","*",1,0],"Diffbot":["partial","*",1,0],"ContextFetchBot":["partial","*",1,0]},"llms":{"status":"missing","http":404,"bytes":154870},"home":{"status":"ok","http":200,"url":"https://grist.org/","html_tokens":78804,"markdown_tokens":304,"pruned_tokens":108}},
{"rank":3301,"domain":"rubyonrails.org","origin":"https://rubyonrails.org","robots":{"status":"ok","http":200,"url":"https://rubyonrails.org/robots.txt","bytes":45,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":5321},"home":{"status":"ok","http":200,"url":"https://rubyonrails.org/","html_tokens":6264,"markdown_tokens":541,"pruned_tokens":162}},
{"rank":3302,"domain":"uw.edu.pl","origin":"https://uw.edu.pl","robots":{"status":"ok","http":200,"url":"https://uw.edu.pl/robots.txt","bytes":206,"groups":1},"bots":{"GPTBot":["partial","*",6,1],"ChatGPT-User":["partial","*",6,1],"OAI-SearchBot":["partial","*",6,1],"ClaudeBot":["partial","*",6,1],"Claude-User":["partial","*",6,1],"Claude-SearchBot":["partial","*",6,1],"anthropic-ai":["partial","*",6,1],"PerplexityBot":["partial","*",6,1],"Perplexity-User":["partial","*",6,1],"Google-Extended":["partial","*",6,1],"Applebot-Extended":["partial","*",6,1],"CCBot":["partial","*",6,1],"Bytespider":["partial","*",6,1],"Amazonbot":["partial","*",6,1],"meta-externalagent":["partial","*",6,1],"cohere-ai":["partial","*",6,1],"Diffbot":["partial","*",6,1],"ContextFetchBot":["partial","*",6,1]},"llms":{"status":"missing","http":404,"bytes":88617},"home":{"status":"ok","http":200,"url":"https://www.uw.edu.pl/","html_tokens":67522,"markdown_tokens":84,"pruned_tokens":84}},
{"rank":3303,"domain":"techpowerup.com","origin":"https://techpowerup.com","robots":{"status":"ok","http":200,"url":"https://www.techpowerup.com/robots.txt","bytes":2211,"groups":10},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["blocked","named",1,0],"OAI-SearchBot":["blocked","named",1,0],"ClaudeBot":["blocked","named",5,0],"Claude-User":["blocked","named",1,0],"Claude-SearchBot":["blocked","named",1,0],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["blocked","named",1,0],"Google-Extended":["partial","*",2,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["partial","*",2,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["partial","*",2,0],"ContextFetchBot":["partial","*",2,0]},"llms":{"status":"missing","http":404,"bytes":20},"home":{"status":"ok","http":200,"url":"https://www.techpowerup.com/","html_tokens":60422,"markdown_tokens":371,"pruned_tokens":167}},
{"rank":3304,"domain":"dlr.de","origin":"https://dlr.de","robots":{"status":"ok","http":200,"url":"https://www.dlr.de/robots.txt","bytes":315,"groups":2},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"present","http":200,"bytes":58641},"home":{"status":"ok","http":200,"url":"https://www.dlr.de/de","html_tokens":138704,"markdown_tokens":21,"pruned_tokens":21}},
{"rank":3305,"domain":"searchenginewatch.com","origin":"https://searchenginewatch.com","robots":{"status":"refused","http":403,"url":"https://searchenginewatch.com/robots.txt","bytes":103,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":3306,"domain":"uptodate.com","origin":"https://uptodate.com","robots":{"status":"ok","http":200,"url":"https://www.uptodate.com/robots.txt","bytes":3177,"groups":26},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["blocked","named",1,0],"OAI-SearchBot":["partial","*",37,32],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",37,32],"Claude-SearchBot":["partial","*",37,32],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["partial","*",37,32],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["partial","*",37,32],"ContextFetchBot":["partial","*",37,32]},"llms":{"status":"timeout","error":"timed out after 10 s"},"home":{"status":"timeout","error":"timed out after 15 s"}},
{"rank":3307,"domain":"gsk.com","origin":"https://gsk.com","robots":{"status":"ok","http":200,"url":"https://gsk.com/robots.txt","bytes":81,"groups":1},"bots":{"GPTBot":["partial","*",1,1],"ChatGPT-User":["partial","*",1,1],"OAI-SearchBot":["partial","*",1,1],"ClaudeBot":["partial","*",1,1],"Claude-User":["partial","*",1,1],"Claude-SearchBot":["partial","*",1,1],"anthropic-ai":["partial","*",1,1],"PerplexityBot":["partial","*",1,1],"Perplexity-User":["partial","*",1,1],"Google-Extended":["partial","*",1,1],"Applebot-Extended":["partial","*",1,1],"CCBot":["partial","*",1,1],"Bytespider":["partial","*",1,1],"Amazonbot":["partial","*",1,1],"meta-externalagent":["partial","*",1,1],"cohere-ai":["partial","*",1,1],"Diffbot":["partial","*",1,1],"ContextFetchBot":["partial","*",1,1]},"llms":{"status":"empty","http":200,"bytes":0},"home":{"status":"ok","http":200,"url":"https://www.gsk.com/en-gb/","html_tokens":106521,"markdown_tokens":152,"pruned_tokens":61}},
{"rank":3308,"domain":"blossomthemes.com","origin":"https://blossomthemes.com","robots":{"status":"ok","http":200,"url":"https://blossomthemes.com/robots.txt","bytes":304,"groups":1},"bots":{"GPTBot":["partial","*",9,0],"ChatGPT-User":["partial","*",9,0],"OAI-SearchBot":["partial","*",9,0],"ClaudeBot":["partial","*",9,0],"Claude-User":["partial","*",9,0],"Claude-SearchBot":["partial","*",9,0],"anthropic-ai":["partial","*",9,0],"PerplexityBot":["partial","*",9,0],"Perplexity-User":["partial","*",9,0],"Google-Extended":["partial","*",9,0],"Applebot-Extended":["partial","*",9,0],"CCBot":["partial","*",9,0],"Bytespider":["partial","*",9,0],"Amazonbot":["partial","*",9,0],"meta-externalagent":["partial","*",9,0],"cohere-ai":["partial","*",9,0],"Diffbot":["partial","*",9,0],"ContextFetchBot":["partial","*",9,0]},"llms":{"status":"missing","http":404,"bytes":118034},"home":{"status":"ok","http":200,"url":"https://blossomthemes.com/","html_tokens":101753,"markdown_tokens":708,"pruned_tokens":119}},
{"rank":3309,"domain":"oberlo.com","origin":"https://oberlo.com","robots":{"status":"ok","http":200,"url":"https://www.oberlo.com/robots.txt","bytes":100,"groups":1},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"present","http":200,"bytes":5044},"home":{"status":"ok","http":200,"url":"https://www.oberlo.com/","html_tokens":125964,"markdown_tokens":267,"pruned_tokens":42}},
{"rank":3310,"domain":"ferrari.com","origin":"https://ferrari.com","robots":{"status":"refused","http":403,"url":"https://ferrari.com/robots.txt","bytes":919,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":3311,"domain":"publications.parliament.uk","origin":"https://publications.parliament.uk","robots":{"status":"refused","http":403,"url":"https://publications.parliament.uk/robots.txt","bytes":5564,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":3312,"domain":"tapatalk.com","origin":"https://tapatalk.com","robots":{"status":"refused","http":403,"url":"https://tapatalk.com/robots.txt","bytes":4545,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":3313,"domain":"templatemonster.com","origin":"https://templatemonster.com","robots":{"status":"refused","http":403,"url":"https://www.templatemonster.com/robots.txt","bytes":5561,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":3314,"domain":"court.gov.cn","origin":"https://www.court.gov.cn","robots":{"status":"html","http":200,"url":"https://www.court.gov.cn/robots.txt","bytes":3144,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":3315,"domain":"winzip.com","origin":"https://winzip.com","robots":{"status":"ok","http":200,"url":"https://www.winzip.com/robots.txt","bytes":287,"groups":4},"bots":{"GPTBot":["partial","*",4,0],"ChatGPT-User":["partial","*",4,0],"OAI-SearchBot":["partial","*",4,0],"ClaudeBot":["partial","*",4,0],"Claude-User":["partial","*",4,0],"Claude-SearchBot":["partial","*",4,0],"anthropic-ai":["partial","*",4,0],"PerplexityBot":["partial","*",4,0],"Perplexity-User":["partial","*",4,0],"Google-Extended":["partial","*",4,0],"Applebot-Extended":["partial","*",4,0],"CCBot":["partial","*",4,0],"Bytespider":["partial","*",4,0],"Amazonbot":["partial","*",4,0],"meta-externalagent":["partial","*",4,0],"cohere-ai":["partial","*",4,0],"Diffbot":["partial","*",4,0],"ContextFetchBot":["partial","*",4,0]},"llms":{"status":"present","http":200,"bytes":2597},"home":{"status":"ok","http":200,"url":"https://www.winzip.com/en/","html_tokens":18998,"markdown_tokens":59,"pruned_tokens":59}},
{"rank":3316,"domain":"spectator.co.uk","origin":"https://spectator.co.uk","robots":{"status":"ok","http":200,"url":"https://spectator.com/robots.txt","bytes":182,"groups":1},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"missing","http":404,"bytes":150945},"home":{"status":"ok","http":200,"url":"https://spectator.com/","html_tokens":69063,"markdown_tokens":169,"pruned_tokens":169}},
{"rank":3317,"domain":"cleverreach.com","origin":"https://cleverreach.com","robots":{"status":"ok","http":200,"url":"https://www.cleverreach.com/robots.txt","bytes":129,"groups":1},"bots":{"GPTBot":["partial","*",1,1],"ChatGPT-User":["partial","*",1,1],"OAI-SearchBot":["partial","*",1,1],"ClaudeBot":["partial","*",1,1],"Claude-User":["partial","*",1,1],"Claude-SearchBot":["partial","*",1,1],"anthropic-ai":["partial","*",1,1],"PerplexityBot":["partial","*",1,1],"Perplexity-User":["partial","*",1,1],"Google-Extended":["partial","*",1,1],"Applebot-Extended":["partial","*",1,1],"CCBot":["partial","*",1,1],"Bytespider":["partial","*",1,1],"Amazonbot":["partial","*",1,1],"meta-externalagent":["partial","*",1,1],"cohere-ai":["partial","*",1,1],"Diffbot":["partial","*",1,1],"ContextFetchBot":["partial","*",1,1]},"llms":{"status":"present","http":200,"bytes":10941},"home":{"status":"ok","http":200,"url":"https://www.cleverreach.com/","html_tokens":2990,"markdown_tokens":64,"pruned_tokens":62}},
{"rank":3318,"domain":"panoramio.com","origin":"https://panoramio.com","robots":{"status":"missing","http":404,"url":"https://panoramio.com/robots.txt","bytes":1571,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":1569},"home":{"status":"ok","http":200,"url":"https://panoramio.com/","html_tokens":1078,"markdown_tokens":324,"pruned_tokens":289}},
{"rank":3319,"domain":"worldhistory.org","origin":"https://worldhistory.org","robots":{"status":"ok","http":200,"url":"https://www.worldhistory.org/robots.txt","bytes":1650,"groups":2},"bots":{"GPTBot":["partial","*",34,4],"ChatGPT-User":["partial","*",34,4],"OAI-SearchBot":["partial","*",34,4],"ClaudeBot":["partial","*",34,4],"Claude-User":["partial","*",34,4],"Claude-SearchBot":["partial","*",34,4],"anthropic-ai":["partial","*",34,4],"PerplexityBot":["partial","*",34,4],"Perplexity-User":["partial","*",34,4],"Google-Extended":["partial","*",34,4],"Applebot-Extended":["partial","*",34,4],"CCBot":["partial","*",34,4],"Bytespider":["partial","*",34,4],"Amazonbot":["partial","*",34,4],"meta-externalagent":["partial","*",34,4],"cohere-ai":["partial","*",34,4],"Diffbot":["partial","*",34,4],"ContextFetchBot":["partial","*",34,4]},"llms":{"status":"present","http":200,"bytes":6415},"home":{"status":"ok","http":200,"url":"https://www.worldhistory.org/","html_tokens":53953,"markdown_tokens":477,"pruned_tokens":106}},
{"rank":3320,"domain":"whmcs.com","origin":"https://whmcs.com","robots":{"status":"ok","http":200,"url":"https://www.whmcs.com/robots.txt","bytes":77,"groups":1},"bots":{"GPTBot":["partial","*",1,0],"ChatGPT-User":["partial","*",1,0],"OAI-SearchBot":["partial","*",1,0],"ClaudeBot":["partial","*",1,0],"Claude-User":["partial","*",1,0],"Claude-SearchBot":["partial","*",1,0],"anthropic-ai":["partial","*",1,0],"PerplexityBot":["partial","*",1,0],"Perplexity-User":["partial","*",1,0],"Google-Extended":["partial","*",1,0],"Applebot-Extended":["partial","*",1,0],"CCBot":["partial","*",1,0],"Bytespider":["partial","*",1,0],"Amazonbot":["partial","*",1,0],"meta-externalagent":["partial","*",1,0],"cohere-ai":["partial","*",1,0],"Diffbot":["partial","*",1,0],"ContextFetchBot":["partial","*",1,0]},"llms":{"status":"missing","http":404,"bytes":104130},"home":{"status":"ok","http":200,"url":"https://www.whmcs.com/","html_tokens":55747,"markdown_tokens":866,"pruned_tokens":264}},
{"rank":3321,"domain":"mindbodygreen.com","origin":"https://mindbodygreen.com","robots":{"status":"ok","http":200,"url":"https://www.mindbodygreen.com/robots.txt","bytes":1051,"groups":5},"bots":{"GPTBot":["allowed","named",0,1],"ChatGPT-User":["allowed","named",0,1],"OAI-SearchBot":["partial","*",8,1],"ClaudeBot":["allowed","named",0,1],"Claude-User":["allowed","named",0,1],"Claude-SearchBot":["partial","*",8,1],"anthropic-ai":["allowed","named",0,1],"PerplexityBot":["allowed","named",0,1],"Perplexity-User":["allowed","named",0,1],"Google-Extended":["allowed","named",0,1],"Applebot-Extended":["partial","*",8,1],"CCBot":["partial","*",8,1],"Bytespider":["partial","*",8,1],"Amazonbot":["partial","*",8,1],"meta-externalagent":["partial","*",8,1],"cohere-ai":["partial","*",8,1],"Diffbot":["partial","*",8,1],"ContextFetchBot":["partial","*",8,1]},"llms":{"status":"missing","http":404,"bytes":62214},"home":{"status":"ok","http":200,"url":"https://www.mindbodygreen.com/","html_tokens":186143,"markdown_tokens":1585,"pruned_tokens":214}},
{"rank":3322,"domain":"reverso.net","origin":"https://reverso.net","robots":{"status":"refused","http":403,"url":"https://reverso.net/robots.txt","bytes":5570,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":3323,"domain":"ballotpedia.org","origin":"https://ballotpedia.org","robots":{"status":"html","http":202,"url":"https://ballotpedia.org/robots.txt","bytes":2447,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":3324,"domain":"ams.org","origin":"https://ams.org","robots":{"status":"refused","http":403,"url":"https://ams.org/robots.txt","bytes":28801,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":3325,"domain":"samba.org","origin":"https://samba.org","robots":{"status":"ok","http":200,"url":"https://www.samba.org/robots.txt","bytes":957,"groups":2},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":346},"home":{"status":"ok","http":200,"url":"https://www.samba.org/","html_tokens":2922,"markdown_tokens":402,"pruned_tokens":261}},
{"rank":3326,"domain":"alternet.org","origin":"https://alternet.org","robots":{"status":"ok","http":200,"url":"https://www.alternet.org/robots.txt","bytes":470,"groups":6},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["partial","*",3,1],"OAI-SearchBot":["partial","*",3,1],"ClaudeBot":["partial","*",3,1],"Claude-User":["partial","*",3,1],"Claude-SearchBot":["partial","*",3,1],"anthropic-ai":["partial","*",3,1],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["partial","*",3,1],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["partial","*",3,1],"CCBot":["blocked","named",1,0],"Bytespider":["partial","*",3,1],"Amazonbot":["partial","*",3,1],"meta-externalagent":["partial","*",3,1],"cohere-ai":["partial","*",3,1],"Diffbot":["partial","*",3,1],"ContextFetchBot":["partial","*",3,1]},"llms":{"status":"missing","http":404,"bytes":219838},"home":{"status":"ok","http":200,"url":"https://www.alternet.org/","html_tokens":99842,"markdown_tokens":238,"pruned_tokens":60}},
{"rank":3327,"domain":"ulaval.ca","origin":"https://ulaval.ca","robots":{"status":"unreachable","http":null,"url":"https://ulaval.ca/robots.txt","bytes":0,"groups":0,"error":"ReadError"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":3328,"domain":"list-manage2.com","origin":"https://list-manage2.com","robots":{"status":"timeout","http":null,"url":"https://list-manage2.com/robots.txt","bytes":0,"groups":0,"error":"ConnectTimeout"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":3329,"domain":"xsrv.jp","origin":"https://xsrv.jp","robots":{"status":"timeout","http":null,"url":"https://xsrv.jp/robots.txt","bytes":0,"groups":0,"error":"ConnectTimeout"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":3330,"domain":"tailscale.com","origin":"https://tailscale.com","robots":{"status":"ok","http":200,"url":"https://tailscale.com/robots.txt","bytes":109,"groups":2},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"present","http":200,"bytes":415},"home":{"status":"ok","http":200,"url":"https://tailscale.com/","html_tokens":72200,"markdown_tokens":800,"pruned_tokens":309}},
{"rank":3331,"domain":"health.gov.au","origin":"https://health.gov.au","robots":{"status":"unreachable","http":null,"url":"https://health.gov.au/robots.txt","bytes":0,"groups":0,"error":"<StreamReset stream_id:1, error_code:2, remote_reset:True>"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":3332,"domain":"novartis.com","origin":"https://novartis.com","robots":{"status":"ok","http":200,"url":"https://www.novartis.com/robots.txt","bytes":1764,"groups":2},"bots":{"GPTBot":["partial","named",1,0],"ChatGPT-User":["partial","*",18,18],"OAI-SearchBot":["partial","*",18,18],"ClaudeBot":["partial","*",18,18],"Claude-User":["partial","*",18,18],"Claude-SearchBot":["partial","*",18,18],"anthropic-ai":["partial","*",18,18],"PerplexityBot":["partial","*",18,18],"Perplexity-User":["partial","*",18,18],"Google-Extended":["partial","*",18,18],"Applebot-Extended":["partial","*",18,18],"CCBot":["partial","*",18,18],"Bytespider":["partial","*",18,18],"Amazonbot":["partial","*",18,18],"meta-externalagent":["partial","*",18,18],"cohere-ai":["partial","*",18,18],"Diffbot":["partial","*",18,18],"ContextFetchBot":["partial","*",18,18]},"llms":{"status":"missing","http":404,"bytes":551},"home":{"status":"ok","http":200,"url":"https://www.novartis.com/","html_tokens":52862,"markdown_tokens":255,"pruned_tokens":140}},
{"rank":3333,"domain":"pcisecuritystandards.org","origin":"https://pcisecuritystandards.org","robots":{"status":"ok","http":200,"url":"https://www.pcisecuritystandards.org/robots.txt","bytes":264,"groups":1},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"missing","http":404,"bytes":647388},"home":{"status":"ok","http":200,"url":"https://www.pcisecuritystandards.org/","html_tokens":198151,"markdown_tokens":147,"pruned_tokens":147}},
{"rank":3334,"domain":"databricks.com","origin":"https://databricks.com","robots":{"status":"ok","http":200,"url":"https://www.databricks.com/robots.txt","bytes":129,"groups":1},"bots":{"GPTBot":["allowed","*",0,1],"ChatGPT-User":["allowed","*",0,1],"OAI-SearchBot":["allowed","*",0,1],"ClaudeBot":["allowed","*",0,1],"Claude-User":["allowed","*",0,1],"Claude-SearchBot":["allowed","*",0,1],"anthropic-ai":["allowed","*",0,1],"PerplexityBot":["allowed","*",0,1],"Perplexity-User":["allowed","*",0,1],"Google-Extended":["allowed","*",0,1],"Applebot-Extended":["allowed","*",0,1],"CCBot":["allowed","*",0,1],"Bytespider":["allowed","*",0,1],"Amazonbot":["allowed","*",0,1],"meta-externalagent":["allowed","*",0,1],"cohere-ai":["allowed","*",0,1],"Diffbot":["allowed","*",0,1],"ContextFetchBot":["allowed","*",0,1]},"llms":{"status":"present","http":200,"bytes":12873},"home":{"status":"ok","http":200,"url":"https://www.databricks.com/","html_tokens":216965,"markdown_tokens":174,"pruned_tokens":74}},
{"rank":3335,"domain":"doc.gov","origin":"https://doc.gov","robots":{"status":"refused","http":403,"url":"https://www.commerce.gov/","bytes":5503,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":3336,"domain":"campaign-archive2.com","origin":"https://campaign-archive2.com","robots":{"status":"unreachable","http":null,"url":"https://campaign-archive2.com/robots.txt","bytes":0,"groups":0,"error":"[SSL: CERTIFICATE_VERIFY_FAILED] certificate verify failed: Hostname mismatch, certificate is not valid for 'campaign-archive2.com'. (_ssl.c:1028)"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":3337,"domain":"uba.ar","origin":"https://uba.ar","robots":{"status":"ok","http":200,"url":"https://uba.ar/robots.txt","bytes":26,"groups":1},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"html","http":200,"bytes":15738},"home":{"status":"ok","http":200,"url":"https://uba.ar/","html_tokens":25061,"markdown_tokens":221,"pruned_tokens":79}},
{"rank":3338,"domain":"uptodown.com","origin":"https://uptodown.com","robots":{"status":"ok","http":200,"url":"https://uptodown.com/robots.txt","bytes":268,"groups":2},"bots":{"GPTBot":["partial","*",7,3],"ChatGPT-User":["partial","*",7,3],"OAI-SearchBot":["partial","*",7,3],"ClaudeBot":["partial","*",7,3],"Claude-User":["partial","*",7,3],"Claude-SearchBot":["partial","*",7,3],"anthropic-ai":["partial","*",7,3],"PerplexityBot":["partial","*",7,3],"Perplexity-User":["partial","*",7,3],"Google-Extended":["partial","*",7,3],"Applebot-Extended":["partial","*",7,3],"CCBot":["partial","*",7,3],"Bytespider":["partial","*",7,3],"Amazonbot":["partial","*",7,3],"meta-externalagent":["partial","*",7,3],"cohere-ai":["partial","*",7,3],"Diffbot":["partial","*",7,3],"ContextFetchBot":["partial","*",7,3]},"llms":{"status":"missing","http":404,"bytes":15930},"home":{"status":"ok","http":200,"url":"https://www.uptodown.com/","html_tokens":89562,"markdown_tokens":329,"pruned_tokens":169}},
{"rank":3339,"domain":"wildapricot.org","origin":"https://wildapricot.org","robots":{"status":"ok","http":200,"url":"https://wildapricot.org/robots.txt","bytes":707,"groups":1},"bots":{"GPTBot":["partial","*",19,0],"ChatGPT-User":["partial","*",19,0],"OAI-SearchBot":["partial","*",19,0],"ClaudeBot":["partial","*",19,0],"Claude-User":["partial","*",19,0],"Claude-SearchBot":["partial","*",19,0],"anthropic-ai":["partial","*",19,0],"PerplexityBot":["partial","*",19,0],"Perplexity-User":["partial","*",19,0],"Google-Extended":["partial","*",19,0],"Applebot-Extended":["partial","*",19,0],"CCBot":["partial","*",19,0],"Bytespider":["partial","*",19,0],"Amazonbot":["partial","*",19,0],"meta-externalagent":["partial","*",19,0],"cohere-ai":["partial","*",19,0],"Diffbot":["partial","*",19,0],"ContextFetchBot":["partial","*",19,0]},"llms":{"status":"timeout","error":"timed out after 10 s"},"home":{"status":"ok","http":200,"url":"https://wildapricot.org/","html_tokens":6846,"markdown_tokens":45,"pruned_tokens":45}},
{"rank":3340,"domain":"amtrak.com","origin":"https://amtrak.com","robots":{"status":"ok","http":200,"url":"https://www.amtrak.com/robots.txt","bytes":427,"groups":1},"bots":{"GPTBot":["partial","*",15,0],"ChatGPT-User":["partial","*",15,0],"OAI-SearchBot":["partial","*",15,0],"ClaudeBot":["partial","*",15,0],"Claude-User":["partial","*",15,0],"Claude-SearchBot":["partial","*",15,0],"anthropic-ai":["partial","*",15,0],"PerplexityBot":["partial","*",15,0],"Perplexity-User":["partial","*",15,0],"Google-Extended":["partial","*",15,0],"Applebot-Extended":["partial","*",15,0],"CCBot":["partial","*",15,0],"Bytespider":["partial","*",15,0],"Amazonbot":["partial","*",15,0],"meta-externalagent":["partial","*",15,0],"cohere-ai":["partial","*",15,0],"Diffbot":["partial","*",15,0],"ContextFetchBot":["partial","*",15,0]},"llms":{"status":"missing","http":404,"bytes":10647},"home":{"status":"ok","http":200,"url":"https://www.amtrak.com/home","html_tokens":59825,"markdown_tokens":97,"pruned_tokens":68}},
{"rank":3341,"domain":"splunk.com","origin":"https://splunk.com","robots":{"status":"ok","http":200,"url":"https://www.splunk.com/robots.txt","bytes":7689,"groups":1},"bots":{"GPTBot":["partial","*",30,0],"ChatGPT-User":["partial","*",30,0],"OAI-SearchBot":["partial","*",30,0],"ClaudeBot":["partial","*",30,0],"Claude-User":["partial","*",30,0],"Claude-SearchBot":["partial","*",30,0],"anthropic-ai":["partial","*",30,0],"PerplexityBot":["partial","*",30,0],"Perplexity-User":["partial","*",30,0],"Google-Extended":["partial","*",30,0],"Applebot-Extended":["partial","*",30,0],"CCBot":["partial","*",30,0],"Bytespider":["partial","*",30,0],"Amazonbot":["partial","*",30,0],"meta-externalagent":["partial","*",30,0],"cohere-ai":["partial","*",30,0],"Diffbot":["partial","*",30,0],"ContextFetchBot":["partial","*",30,0]},"llms":{"status":"missing","http":404,"bytes":140739},"home":{"status":"ok","http":200,"url":"https://www.splunk.com/","html_tokens":175855,"markdown_tokens":788,"pruned_tokens":180}},
{"rank":3342,"domain":"dailycaller.com","origin":"https://dailycaller.com","robots":{"status":"ok","http":200,"url":"https://dailycaller.com/robots.txt","bytes":603,"groups":14},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["blocked","named",1,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"missing","http":404,"bytes":332259},"home":{"status":"ok","http":200,"url":"https://dailycaller.com/","html_tokens":108618,"markdown_tokens":16,"pruned_tokens":16}},
{"rank":3343,"domain":"theculturetrip.com","origin":"https://theculturetrip.com","robots":{"status":"ok","http":200,"url":"https://theculturetrip.com/robots.txt","bytes":1475,"groups":27},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["allowed","named",0,1],"OAI-SearchBot":["allowed","named",0,1],"ClaudeBot":["blocked","named",1,0],"Claude-User":["allowed","named",0,1],"Claude-SearchBot":["allowed","named",0,1],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["allowed","named",0,1],"Perplexity-User":["allowed","named",0,1],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["partial","*",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["partial","*",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",1,0]},"llms":{"status":"unreachable","error":"<StreamReset stream_id:3, error_code:2, remote_reset:True>"},"home":{"status":"unreachable","error":"<StreamReset stream_id:5, error_code:2, remote_reset:True>"}},
{"rank":3344,"domain":"koreaherald.com","origin":"https://koreaherald.com","robots":{"status":"ok","http":200,"url":"https://www.koreaherald.com/robots.txt","bytes":986,"groups":2},"bots":{"GPTBot":["partial","*",5,1],"ChatGPT-User":["partial","*",5,1],"OAI-SearchBot":["partial","*",5,1],"ClaudeBot":["partial","*",5,1],"Claude-User":["partial","*",5,1],"Claude-SearchBot":["partial","*",5,1],"anthropic-ai":["partial","*",5,1],"PerplexityBot":["partial","*",5,1],"Perplexity-User":["partial","*",5,1],"Google-Extended":["partial","*",5,1],"Applebot-Extended":["partial","*",5,1],"CCBot":["partial","*",5,1],"Bytespider":["partial","*",5,1],"Amazonbot":["partial","*",5,1],"meta-externalagent":["partial","*",5,1],"cohere-ai":["partial","*",5,1],"Diffbot":["partial","*",5,1],"ContextFetchBot":["partial","*",5,1]},"llms":{"status":"missing","http":404,"bytes":43318},"home":{"status":"ok","http":200,"url":"https://www.koreaherald.com/","html_tokens":24107,"markdown_tokens":249,"pruned_tokens":121}},
{"rank":3345,"domain":"polimi.it","origin":"https://polimi.it","robots":{"status":"html","http":200,"url":"https://www.polimi.it/","bytes":716342,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":3346,"domain":"thehindubusinessline.com","origin":"https://thehindubusinessline.com","robots":{"status":"ok","http":200,"url":"https://www.thehindubusinessline.com/robots.txt","bytes":2209,"groups":18},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["blocked","named",1,0],"OAI-SearchBot":["blocked","named",1,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",44,3],"Claude-SearchBot":["partial","*",44,3],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["partial","*",44,3],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["partial","*",44,3],"Diffbot":["partial","*",44,3],"ContextFetchBot":["partial","*",44,3]},"llms":{"status":"missing","http":404,"bytes":25434},"home":{"status":"ok","http":200,"url":"https://www.thehindubusinessline.com/","html_tokens":53158,"markdown_tokens":53,"pruned_tokens":53}},
{"rank":3347,"domain":"dal.ca","origin":"https://dal.ca","robots":{"status":"ok","http":200,"url":"https://www.dal.ca/robots.txt","bytes":44,"groups":1},"bots":{"GPTBot":["partial","*",1,0],"ChatGPT-User":["partial","*",1,0],"OAI-SearchBot":["partial","*",1,0],"ClaudeBot":["partial","*",1,0],"Claude-User":["partial","*",1,0],"Claude-SearchBot":["partial","*",1,0],"anthropic-ai":["partial","*",1,0],"PerplexityBot":["partial","*",1,0],"Perplexity-User":["partial","*",1,0],"Google-Extended":["partial","*",1,0],"Applebot-Extended":["partial","*",1,0],"CCBot":["partial","*",1,0],"Bytespider":["partial","*",1,0],"Amazonbot":["partial","*",1,0],"meta-externalagent":["partial","*",1,0],"cohere-ai":["partial","*",1,0],"Diffbot":["partial","*",1,0],"ContextFetchBot":["partial","*",1,0]},"llms":{"status":"missing","http":404,"bytes":10579},"home":{"status":"ok","http":200,"url":"https://www.dal.ca/","html_tokens":36776,"markdown_tokens":147,"pruned_tokens":40}},
{"rank":3348,"domain":"brightcove.com","origin":"https://brightcove.com","robots":{"status":"missing","http":404,"url":"https://www.brightcove.com/robots.txt","bytes":73047,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":73041},"home":{"status":"ok","http":200,"url":"https://www.brightcove.com/","html_tokens":266078,"markdown_tokens":224,"pruned_tokens":83}},
{"rank":3349,"domain":"hackmd.io","origin":"https://hackmd.io","robots":{"status":"ok","http":200,"url":"https://hackmd.io/robots.txt","bytes":1843,"groups":14},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["allowed","*",0,1],"OAI-SearchBot":["allowed","*",0,1],"ClaudeBot":["blocked","named",1,0],"Claude-User":["allowed","*",0,1],"Claude-SearchBot":["allowed","*",0,1],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["allowed","*",0,1],"Perplexity-User":["allowed","*",0,1],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["allowed","*",0,1]},"llms":{"status":"present","http":200,"bytes":8129},"home":{"status":"ok","http":200,"url":"https://hackmd.io/","html_tokens":199714,"markdown_tokens":882,"pruned_tokens":388}},
{"rank":3350,"domain":"france.tv","origin":"https://france.tv","robots":{"status":"ok","http":200,"url":"https://www.france.tv/robots.txt","bytes":1246,"groups":2},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["blocked","named",1,0],"OAI-SearchBot":["blocked","named",1,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["blocked","named",1,0],"Claude-SearchBot":["partial","*",12,6],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["partial","*",12,6],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["partial","*",12,6],"ContextFetchBot":["partial","*",12,6]},"llms":{"status":"missing","http":404,"bytes":45016},"home":{"status":"ok","http":200,"url":"https://www.france.tv/","html_tokens":356269,"markdown_tokens":40,"pruned_tokens":40}},
{"rank":3351,"domain":"osce.org","origin":"https://www.osce.org","robots":{"status":"ok","http":200,"url":"https://www.osce.org/robots.txt","bytes":2141,"groups":1},"bots":{"GPTBot":["partial","*",35,22],"ChatGPT-User":["partial","*",35,22],"OAI-SearchBot":["partial","*",35,22],"ClaudeBot":["partial","*",35,22],"Claude-User":["partial","*",35,22],"Claude-SearchBot":["partial","*",35,22],"anthropic-ai":["partial","*",35,22],"PerplexityBot":["partial","*",35,22],"Perplexity-User":["partial","*",35,22],"Google-Extended":["partial","*",35,22],"Applebot-Extended":["partial","*",35,22],"CCBot":["partial","*",35,22],"Bytespider":["partial","*",35,22],"Amazonbot":["partial","*",35,22],"meta-externalagent":["partial","*",35,22],"cohere-ai":["partial","*",35,22],"Diffbot":["partial","*",35,22],"ContextFetchBot":["partial","*",35,22]},"llms":{"status":"missing","http":404,"bytes":186},"home":{"status":"ok","http":200,"url":"https://www.osce.org/","html_tokens":45204,"markdown_tokens":328,"pruned_tokens":104}},
{"rank":3352,"domain":"sublimetext.com","origin":"https://sublimetext.com","robots":{"status":"missing","http":404,"url":"https://www.sublimetext.com/robots.txt","bytes":1210,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":1210},"home":{"status":"ok","http":200,"url":"https://www.sublimetext.com/","html_tokens":7775,"markdown_tokens":574,"pruned_tokens":141}},
{"rank":3353,"domain":"lockheedmartin.com","origin":"https://lockheedmartin.com","robots":{"status":"ok","http":200,"url":"https://lockheedmartin.com/robots.txt","bytes":4222,"groups":1},"bots":{"GPTBot":["partial","*",40,34],"ChatGPT-User":["partial","*",40,34],"OAI-SearchBot":["partial","*",40,34],"ClaudeBot":["partial","*",40,34],"Claude-User":["partial","*",40,34],"Claude-SearchBot":["partial","*",40,34],"anthropic-ai":["partial","*",40,34],"PerplexityBot":["partial","*",40,34],"Perplexity-User":["partial","*",40,34],"Google-Extended":["partial","*",40,34],"Applebot-Extended":["partial","*",40,34],"CCBot":["partial","*",40,34],"Bytespider":["partial","*",40,34],"Amazonbot":["partial","*",40,34],"meta-externalagent":["partial","*",40,34],"cohere-ai":["partial","*",40,34],"Diffbot":["partial","*",40,34],"ContextFetchBot":["partial","*",40,34]},"llms":{"status":"missing","http":404,"bytes":63423},"home":{"status":"ok","http":200,"url":"https://lockheedmartin.com/","html_tokens":24484,"markdown_tokens":16,"pruned_tokens":16}},
{"rank":3354,"domain":"terra.com.br","origin":"https://terra.com.br","robots":{"status":"ok","http":200,"url":"https://www.terra.com.br/robots.txt","bytes":92,"groups":1},"bots":{"GPTBot":["partial","*",3,0],"ChatGPT-User":["partial","*",3,0],"OAI-SearchBot":["partial","*",3,0],"ClaudeBot":["partial","*",3,0],"Claude-User":["partial","*",3,0],"Claude-SearchBot":["partial","*",3,0],"anthropic-ai":["partial","*",3,0],"PerplexityBot":["partial","*",3,0],"Perplexity-User":["partial","*",3,0],"Google-Extended":["partial","*",3,0],"Applebot-Extended":["partial","*",3,0],"CCBot":["partial","*",3,0],"Bytespider":["partial","*",3,0],"Amazonbot":["partial","*",3,0],"meta-externalagent":["partial","*",3,0],"cohere-ai":["partial","*",3,0],"Diffbot":["partial","*",3,0],"ContextFetchBot":["partial","*",3,0]},"llms":{"status":"missing","http":404,"bytes":303},"home":{"status":"ok","http":200,"url":"https://www.terra.com.br/","html_tokens":89100,"markdown_tokens":169,"pruned_tokens":95}},
{"rank":3355,"domain":"bamboohr.com","origin":"https://bamboohr.com","robots":{"status":"ok","http":200,"url":"https://www.bamboohr.com/robots.txt","bytes":2891,"groups":11},"bots":{"GPTBot":["partial","named",5,7],"ChatGPT-User":["partial","*",5,8],"OAI-SearchBot":["partial","named",5,8],"ClaudeBot":["partial","named",5,7],"Claude-User":["partial","*",5,8],"Claude-SearchBot":["partial","*",5,8],"anthropic-ai":["partial","named",5,7],"PerplexityBot":["partial","named",5,8],"Perplexity-User":["partial","*",5,8],"Google-Extended":["partial","named",5,7],"Applebot-Extended":["partial","named",5,7],"CCBot":["partial","named",5,7],"Bytespider":["partial","named",5,7],"Amazonbot":["partial","*",5,8],"meta-externalagent":["partial","named",5,7],"cohere-ai":["partial","*",5,8],"Diffbot":["partial","*",5,8],"ContextFetchBot":["partial","*",5,8]},"llms":{"status":"present","http":200,"bytes":6301},"home":{"status":"ok","http":200,"url":"https://www.bamboohr.com/","html_tokens":11390,"markdown_tokens":1406,"pruned_tokens":806}},
{"rank":3356,"domain":"unglobalcompact.org","origin":"https://unglobalcompact.org","robots":{"status":"ok","http":200,"url":"https://unglobalcompact.org/robots.txt","bytes":1091,"groups":3},"bots":{"GPTBot":["partial","*",14,0],"ChatGPT-User":["partial","*",14,0],"OAI-SearchBot":["partial","*",14,0],"ClaudeBot":["partial","*",14,0],"Claude-User":["partial","*",14,0],"Claude-SearchBot":["partial","*",14,0],"anthropic-ai":["partial","*",14,0],"PerplexityBot":["partial","*",14,0],"Perplexity-User":["partial","*",14,0],"Google-Extended":["partial","*",14,0],"Applebot-Extended":["partial","*",14,0],"CCBot":["partial","*",14,0],"Bytespider":["partial","*",14,0],"Amazonbot":["partial","*",14,0],"meta-externalagent":["partial","*",14,0],"cohere-ai":["partial","*",14,0],"Diffbot":["partial","*",14,0],"ContextFetchBot":["partial","*",14,0]},"llms":{"status":"missing","http":404,"bytes":0},"home":{"status":"ok","http":200,"url":"https://unglobalcompact.org/","html_tokens":5230,"markdown_tokens":135,"pruned_tokens":135}},
{"rank":3357,"domain":"cao.go.jp","origin":"https://www.cao.go.jp","robots":{"status":"missing","http":404,"url":"https://www.cao.go.jp/robots.txt","bytes":5452,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":5452},"home":{"status":"ok","http":200,"url":"https://www.cao.go.jp/","html_tokens":3170,"markdown_tokens":41,"pruned_tokens":41}},
{"rank":3358,"domain":"home.kpmg","origin":"https://home.kpmg","robots":{"status":"ok","http":200,"url":"https://kpmg.com/robots.txt","bytes":672,"groups":3},"bots":{"GPTBot":["partial","*",14,1],"ChatGPT-User":["partial","*",14,1],"OAI-SearchBot":["partial","*",14,1],"ClaudeBot":["partial","*",14,1],"Claude-User":["partial","*",14,1],"Claude-SearchBot":["partial","*",14,1],"anthropic-ai":["partial","*",14,1],"PerplexityBot":["partial","*",14,1],"Perplexity-User":["partial","*",14,1],"Google-Extended":["partial","*",14,1],"Applebot-Extended":["partial","*",14,1],"CCBot":["partial","*",14,1],"Bytespider":["partial","*",14,1],"Amazonbot":["partial","*",14,1],"meta-externalagent":["partial","*",14,1],"cohere-ai":["partial","*",14,1],"Diffbot":["partial","*",14,1],"ContextFetchBot":["partial","*",14,1]},"llms":{"status":"present","http":200,"bytes":6837},"home":{"status":"ok","http":200,"url":"https://kpmg.com/xx/en.html","html_tokens":53191,"markdown_tokens":70,"pruned_tokens":62}},
{"rank":3359,"domain":"findarticles.com","origin":"https://findarticles.com","robots":{"status":"ok","http":200,"url":"https://findarticles.com/robots.txt","bytes":220,"groups":1},"bots":{"GPTBot":["partial","*",2,0],"ChatGPT-User":["partial","*",2,0],"OAI-SearchBot":["partial","*",2,0],"ClaudeBot":["partial","*",2,0],"Claude-User":["partial","*",2,0],"Claude-SearchBot":["partial","*",2,0],"anthropic-ai":["partial","*",2,0],"PerplexityBot":["partial","*",2,0],"Perplexity-User":["partial","*",2,0],"Google-Extended":["partial","*",2,0],"Applebot-Extended":["partial","*",2,0],"CCBot":["partial","*",2,0],"Bytespider":["partial","*",2,0],"Amazonbot":["partial","*",2,0],"meta-externalagent":["partial","*",2,0],"cohere-ai":["partial","*",2,0],"Diffbot":["partial","*",2,0],"ContextFetchBot":["partial","*",2,0]},"llms":{"status":"other_text","http":200,"bytes":2214},"home":{"status":"ok","http":200,"url":"https://www.findarticles.com/","html_tokens":83327,"markdown_tokens":44,"pruned_tokens":44}},
{"rank":3360,"domain":"thedrive.com","origin":"https://thedrive.com","robots":{"status":"ok","http":200,"url":"https://www.thedrive.com/robots.txt","bytes":368,"groups":2},"bots":{"GPTBot":["partial","*",6,1],"ChatGPT-User":["partial","*",6,1],"OAI-SearchBot":["partial","*",6,1],"ClaudeBot":["partial","*",6,1],"Claude-User":["partial","*",6,1],"Claude-SearchBot":["partial","*",6,1],"anthropic-ai":["partial","*",6,1],"PerplexityBot":["partial","*",6,1],"Perplexity-User":["partial","*",6,1],"Google-Extended":["partial","*",6,1],"Applebot-Extended":["partial","*",6,1],"CCBot":["partial","*",6,1],"Bytespider":["partial","*",6,1],"Amazonbot":["partial","*",6,1],"meta-externalagent":["partial","*",6,1],"cohere-ai":["partial","*",6,1],"Diffbot":["partial","*",6,1],"ContextFetchBot":["partial","*",6,1]},"llms":{"status":"missing","http":404,"bytes":97580},"home":{"status":"ok","http":200,"url":"https://www.thedrive.com/","html_tokens":41042,"markdown_tokens":158,"pruned_tokens":157}},
{"rank":3361,"domain":"canva.link","origin":"https://canva.link","robots":{"status":"refused","http":403,"url":"https://www.canva.com/404","bytes":754830,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":3362,"domain":"cafe24.com","origin":"https://cafe24.com","robots":{"status":"html","http":200,"url":"https://www.cafe24.com/","bytes":210998,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":3363,"domain":"libera.chat","origin":"https://libera.chat","robots":{"status":"ok","http":200,"url":"https://libera.chat/robots.txt","bytes":41,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":17230},"home":{"status":"ok","http":200,"url":"https://libera.chat/","html_tokens":7067,"markdown_tokens":1726,"pruned_tokens":567}},
{"rank":3364,"domain":"google.hu","origin":"https://google.hu","robots":{"status":"ok","http":200,"url":"https://www.google.hu/robots.txt","bytes":6624,"groups":4},"bots":{"GPTBot":["partial","*",169,64],"ChatGPT-User":["partial","*",169,64],"OAI-SearchBot":["partial","*",169,64],"ClaudeBot":["partial","*",169,64],"Claude-User":["partial","*",169,64],"Claude-SearchBot":["partial","*",169,64],"anthropic-ai":["partial","*",169,64],"PerplexityBot":["partial","*",169,64],"Perplexity-User":["partial","*",169,64],"Google-Extended":["partial","*",169,64],"Applebot-Extended":["partial","*",169,64],"CCBot":["partial","*",169,64],"Bytespider":["partial","*",169,64],"Amazonbot":["partial","*",169,64],"meta-externalagent":["partial","*",169,64],"cohere-ai":["partial","*",169,64],"Diffbot":["partial","*",169,64],"ContextFetchBot":["partial","*",169,64]},"llms":{"status":"missing","http":404,"bytes":1569},"home":{"status":"ok","http":200,"url":"https://www.google.com/","html_tokens":21449,"markdown_tokens":30,"pruned_tokens":30}},
{"rank":3365,"domain":"newindianexpress.com","origin":"https://newindianexpress.com","robots":{"status":"ok","http":200,"url":"https://www.newindianexpress.com/robots.txt","bytes":135,"groups":1},"bots":{"GPTBot":["allowed","*",0,1],"ChatGPT-User":["allowed","*",0,1],"OAI-SearchBot":["allowed","*",0,1],"ClaudeBot":["allowed","*",0,1],"Claude-User":["allowed","*",0,1],"Claude-SearchBot":["allowed","*",0,1],"anthropic-ai":["allowed","*",0,1],"PerplexityBot":["allowed","*",0,1],"Perplexity-User":["allowed","*",0,1],"Google-Extended":["allowed","*",0,1],"Applebot-Extended":["allowed","*",0,1],"CCBot":["allowed","*",0,1],"Bytespider":["allowed","*",0,1],"Amazonbot":["allowed","*",0,1],"meta-externalagent":["allowed","*",0,1],"cohere-ai":["allowed","*",0,1],"Diffbot":["allowed","*",0,1],"ContextFetchBot":["allowed","*",0,1]},"llms":{"status":"missing","http":404,"bytes":1577540},"home":{"status":"ok","http":200,"url":"https://www.newindianexpress.com/","html_tokens":368805,"markdown_tokens":44,"pruned_tokens":44}},
{"rank":3366,"domain":"marketscreener.com","origin":"https://marketscreener.com","robots":{"status":"ok","http":200,"url":"https://www.marketscreener.com/robots.txt","bytes":362,"groups":1},"bots":{"GPTBot":["partial","*",16,0],"ChatGPT-User":["partial","*",16,0],"OAI-SearchBot":["partial","*",16,0],"ClaudeBot":["partial","*",16,0],"Claude-User":["partial","*",16,0],"Claude-SearchBot":["partial","*",16,0],"anthropic-ai":["partial","*",16,0],"PerplexityBot":["partial","*",16,0],"Perplexity-User":["partial","*",16,0],"Google-Extended":["partial","*",16,0],"Applebot-Extended":["partial","*",16,0],"CCBot":["partial","*",16,0],"Bytespider":["partial","*",16,0],"Amazonbot":["partial","*",16,0],"meta-externalagent":["partial","*",16,0],"cohere-ai":["partial","*",16,0],"Diffbot":["partial","*",16,0],"ContextFetchBot":["partial","*",16,0]},"llms":{"status":"missing","http":404,"bytes":168600},"home":{"status":"ok","http":200,"url":"https://www.marketscreener.com/","html_tokens":120390,"markdown_tokens":429,"pruned_tokens":21}},
{"rank":3367,"domain":"hypebeast.com","origin":"https://hypebeast.com","robots":{"status":"ok","http":200,"url":"https://hypebeast.com/robots.txt","bytes":1843,"groups":17},"bots":{"GPTBot":["partial","*",22,2],"ChatGPT-User":["partial","*",22,2],"OAI-SearchBot":["partial","*",22,2],"ClaudeBot":["allowed","named",0,1],"Claude-User":["allowed","named",0,1],"Claude-SearchBot":["allowed","named",0,1],"anthropic-ai":["partial","*",22,2],"PerplexityBot":["partial","*",22,2],"Perplexity-User":["partial","*",22,2],"Google-Extended":["partial","*",22,2],"Applebot-Extended":["partial","*",22,2],"CCBot":["partial","*",22,2],"Bytespider":["partial","*",22,2],"Amazonbot":["partial","*",22,2],"meta-externalagent":["allowed","named",0,1],"cohere-ai":["partial","*",22,2],"Diffbot":["partial","*",22,2],"ContextFetchBot":["partial","*",22,2]},"llms":{"status":"present","http":200,"bytes":4436},"home":{"status":"ok","http":200,"url":"https://hypebeast.com/","html_tokens":98720,"markdown_tokens":1380,"pruned_tokens":139}},
{"rank":3368,"domain":"payhip.com","origin":"https://payhip.com","robots":{"status":"ok","http":200,"url":"https://payhip.com/robots.txt","bytes":407,"groups":1},"bots":{"GPTBot":["partial","*",15,0],"ChatGPT-User":["partial","*",15,0],"OAI-SearchBot":["partial","*",15,0],"ClaudeBot":["partial","*",15,0],"Claude-User":["partial","*",15,0],"Claude-SearchBot":["partial","*",15,0],"anthropic-ai":["partial","*",15,0],"PerplexityBot":["partial","*",15,0],"Perplexity-User":["partial","*",15,0],"Google-Extended":["partial","*",15,0],"Applebot-Extended":["partial","*",15,0],"CCBot":["partial","*",15,0],"Bytespider":["partial","*",15,0],"Amazonbot":["partial","*",15,0],"meta-externalagent":["partial","*",15,0],"cohere-ai":["partial","*",15,0],"Diffbot":["partial","*",15,0],"ContextFetchBot":["partial","*",15,0]},"llms":{"status":"missing","http":404,"bytes":1932},"home":{"status":"http_error","http":403,"url":"https://payhip.com/"}},
{"rank":3369,"domain":"quantcast.com","origin":"https://quantcast.com","robots":{"status":"ok","http":200,"url":"https://www.quantcast.com/robots.txt","bytes":352,"groups":1},"bots":{"GPTBot":["partial","*",2,0],"ChatGPT-User":["partial","*",2,0],"OAI-SearchBot":["partial","*",2,0],"ClaudeBot":["partial","*",2,0],"Claude-User":["partial","*",2,0],"Claude-SearchBot":["partial","*",2,0],"anthropic-ai":["partial","*",2,0],"PerplexityBot":["partial","*",2,0],"Perplexity-User":["partial","*",2,0],"Google-Extended":["partial","*",2,0],"Applebot-Extended":["partial","*",2,0],"CCBot":["partial","*",2,0],"Bytespider":["partial","*",2,0],"Amazonbot":["partial","*",2,0],"meta-externalagent":["partial","*",2,0],"cohere-ai":["partial","*",2,0],"Diffbot":["partial","*",2,0],"ContextFetchBot":["partial","*",2,0]},"llms":{"status":"missing","http":404,"bytes":32157},"home":{"status":"ok","http":200,"url":"https://www.quantcast.com/home","html_tokens":119341,"markdown_tokens":228,"pruned_tokens":103}},
{"rank":3370,"domain":"webopedia.com","origin":"https://webopedia.com","robots":{"status":"ok","http":200,"url":"https://www.webopedia.com/robots.txt","bytes":1683,"groups":2},"bots":{"GPTBot":["partial","*",15,8],"ChatGPT-User":["partial","*",15,8],"OAI-SearchBot":["partial","*",15,8],"ClaudeBot":["partial","*",15,8],"Claude-User":["partial","*",15,8],"Claude-SearchBot":["partial","*",15,8],"anthropic-ai":["partial","*",15,8],"PerplexityBot":["partial","*",15,8],"Perplexity-User":["partial","*",15,8],"Google-Extended":["partial","*",15,8],"Applebot-Extended":["partial","*",15,8],"CCBot":["partial","*",15,8],"Bytespider":["partial","*",15,8],"Amazonbot":["partial","*",15,8],"meta-externalagent":["partial","*",15,8],"cohere-ai":["partial","*",15,8],"Diffbot":["partial","*",15,8],"ContextFetchBot":["partial","*",15,8]},"llms":{"status":"present","http":200,"bytes":38842},"home":{"status":"http_error","http":403,"url":"https://webopedia.com/"}},
{"rank":3371,"domain":"as.me","origin":"https://as.me","robots":{"status":"ok","http":200,"url":"https://as.me/robots.txt","bytes":941,"groups":1},"bots":{"GPTBot":["partial","*",20,0],"ChatGPT-User":["partial","*",20,0],"OAI-SearchBot":["partial","*",20,0],"ClaudeBot":["partial","*",20,0],"Claude-User":["partial","*",20,0],"Claude-SearchBot":["partial","*",20,0],"anthropic-ai":["partial","*",20,0],"PerplexityBot":["partial","*",20,0],"Perplexity-User":["partial","*",20,0],"Google-Extended":["partial","*",20,0],"Applebot-Extended":["partial","*",20,0],"CCBot":["partial","*",20,0],"Bytespider":["partial","*",20,0],"Amazonbot":["partial","*",20,0],"meta-externalagent":["partial","*",20,0],"cohere-ai":["partial","*",20,0],"Diffbot":["partial","*",20,0],"ContextFetchBot":["partial","*",20,0]},"llms":{"status":"missing","http":404,"bytes":59785},"home":{"status":"ok","http":200,"url":"https://acuityscheduling.com","html_tokens":23879,"markdown_tokens":95,"pruned_tokens":95}},
{"rank":3372,"domain":"xfinity.com","origin":"https://xfinity.com","robots":{"status":"ok","http":200,"url":"https://www.xfinity.com/robots.txt","bytes":792,"groups":1},"bots":{"GPTBot":["partial","*",12,0],"ChatGPT-User":["partial","*",12,0],"OAI-SearchBot":["partial","*",12,0],"ClaudeBot":["partial","*",12,0],"Claude-User":["partial","*",12,0],"Claude-SearchBot":["partial","*",12,0],"anthropic-ai":["partial","*",12,0],"PerplexityBot":["partial","*",12,0],"Perplexity-User":["partial","*",12,0],"Google-Extended":["partial","*",12,0],"Applebot-Extended":["partial","*",12,0],"CCBot":["partial","*",12,0],"Bytespider":["partial","*",12,0],"Amazonbot":["partial","*",12,0],"meta-externalagent":["partial","*",12,0],"cohere-ai":["partial","*",12,0],"Diffbot":["partial","*",12,0],"ContextFetchBot":["partial","*",12,0]},"llms":{"status":"present","http":200,"bytes":40769},"home":{"status":"http_error","http":403,"url":"https://www.xfinity.com/"}},
{"rank":3373,"domain":"mediaroom.com","origin":"https://mediaroom.com","robots":{"status":"html","http":200,"url":"https://www.prnewswire.com/mediaroom/","bytes":211345,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":3374,"domain":"rnz.co.nz","origin":"https://rnz.co.nz","robots":{"status":"refused","http":403,"url":"https://rnz.co.nz/robots.txt","bytes":425,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":3375,"domain":"aacrjournals.org","origin":"https://aacrjournals.org","robots":{"status":"ok","http":200,"url":"https://aacrjournals.org/robots.txt","bytes":1612,"groups":3},"bots":{"GPTBot":["partial","*",37,3],"ChatGPT-User":["partial","*",37,3],"OAI-SearchBot":["partial","*",37,3],"ClaudeBot":["partial","*",37,3],"Claude-User":["partial","*",37,3],"Claude-SearchBot":["partial","*",37,3],"anthropic-ai":["partial","*",37,3],"PerplexityBot":["partial","*",37,3],"Perplexity-User":["partial","*",37,3],"Google-Extended":["partial","*",37,3],"Applebot-Extended":["blocked","named",1,0],"CCBot":["partial","*",37,3],"Bytespider":["partial","*",37,3],"Amazonbot":["partial","*",37,3],"meta-externalagent":["partial","*",37,3],"cohere-ai":["partial","*",37,3],"Diffbot":["partial","*",37,3],"ContextFetchBot":["partial","*",37,3]},"llms":{"status":"http_error","http":403,"bytes":5527},"home":{"status":"http_error","http":403,"url":"https://aacrjournals.org/"}},
{"rank":3376,"domain":"punchng.com","origin":"https://punchng.com","robots":{"status":"ok","http":200,"url":"https://punchng.com/robots.txt","bytes":407,"groups":2},"bots":{"GPTBot":["partial","*",6,0],"ChatGPT-User":["partial","*",6,0],"OAI-SearchBot":["partial","*",6,0],"ClaudeBot":["partial","*",6,0],"Claude-User":["partial","*",6,0],"Claude-SearchBot":["partial","*",6,0],"anthropic-ai":["partial","*",6,0],"PerplexityBot":["partial","*",6,0],"Perplexity-User":["partial","*",6,0],"Google-Extended":["partial","*",6,0],"Applebot-Extended":["partial","*",6,0],"CCBot":["partial","*",6,0],"Bytespider":["partial","*",6,0],"Amazonbot":["partial","*",6,0],"meta-externalagent":["partial","*",6,0],"cohere-ai":["partial","*",6,0],"Diffbot":["partial","*",6,0],"ContextFetchBot":["partial","*",6,0]},"llms":{"status":"missing","http":404,"bytes":76569},"home":{"status":"ok","http":200,"url":"https://punchng.com/","html_tokens":80673,"markdown_tokens":77,"pruned_tokens":77}},
{"rank":3377,"domain":"revolut.com","origin":"https://revolut.com","robots":{"status":"ok","http":200,"url":"https://www.revolut.com/robots.txt","bytes":2496,"groups":1},"bots":{"GPTBot":["partial","*",57,3],"ChatGPT-User":["partial","*",57,3],"OAI-SearchBot":["partial","*",57,3],"ClaudeBot":["partial","*",57,3],"Claude-User":["partial","*",57,3],"Claude-SearchBot":["partial","*",57,3],"anthropic-ai":["partial","*",57,3],"PerplexityBot":["partial","*",57,3],"Perplexity-User":["partial","*",57,3],"Google-Extended":["partial","*",57,3],"Applebot-Extended":["partial","*",57,3],"CCBot":["partial","*",57,3],"Bytespider":["partial","*",57,3],"Amazonbot":["partial","*",57,3],"meta-externalagent":["partial","*",57,3],"cohere-ai":["partial","*",57,3],"Diffbot":["partial","*",57,3],"ContextFetchBot":["partial","*",57,3]},"llms":{"status":"http_error","http":403,"bytes":872921},"home":{"status":"http_error","http":403,"url":"https://www.revolut.com/"}},
{"rank":3378,"domain":"suse.com","origin":"https://suse.com","robots":{"status":"ok","http":200,"url":"https://www.suse.com/robots.txt","bytes":940,"groups":3},"bots":{"GPTBot":["partial","*",28,1],"ChatGPT-User":["partial","*",28,1],"OAI-SearchBot":["partial","*",28,1],"ClaudeBot":["partial","*",28,1],"Claude-User":["partial","*",28,1],"Claude-SearchBot":["partial","*",28,1],"anthropic-ai":["partial","*",28,1],"PerplexityBot":["partial","*",28,1],"Perplexity-User":["partial","*",28,1],"Google-Extended":["partial","*",28,1],"Applebot-Extended":["partial","*",28,1],"CCBot":["partial","*",28,1],"Bytespider":["partial","*",28,1],"Amazonbot":["partial","*",28,1],"meta-externalagent":["partial","*",28,1],"cohere-ai":["partial","*",28,1],"Diffbot":["partial","*",28,1],"ContextFetchBot":["partial","*",28,1]},"llms":{"status":"present","http":200,"bytes":13104},"home":{"status":"ok","http":200,"url":"https://www.suse.com/","html_tokens":39757,"markdown_tokens":336,"pruned_tokens":98}},
{"rank":3379,"domain":"myftpupload.com","origin":"https://myftpupload.com","robots":{"status":"unreachable","http":null,"url":"https://myftpupload.com/robots.txt","bytes":0,"groups":0,"error":"cannot resolve myftpupload.com"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":3380,"domain":"eatingwell.com","origin":"https://eatingwell.com","robots":{"status":"ok","http":200,"url":"https://www.eatingwell.com/robots.txt","bytes":3492,"groups":6},"bots":{"GPTBot":["partial","named",1,0],"ChatGPT-User":["partial","named",1,0],"OAI-SearchBot":["partial","named",1,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["blocked","named",1,0],"Claude-SearchBot":["partial","named",1,2],"anthropic-ai":["partial","named",1,2],"PerplexityBot":["partial","named",1,2],"Perplexity-User":["blocked","named",1,0],"Google-Extended":["partial","named",1,2],"Applebot-Extended":["blocked","named",1,0],"CCBot":["partial","named",1,2],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["partial","named",1,2],"cohere-ai":["partial","named",1,2],"Diffbot":["partial","*",3,0],"ContextFetchBot":["partial","*",3,0]},"llms":{"status":"http_error","http":402,"bytes":612},"home":{"status":"http_error","http":402,"url":"https://www.eatingwell.com/"}},
{"rank":3381,"domain":"hrsa.gov","origin":"https://hrsa.gov","robots":{"status":"refused","http":403,"url":"https://www.hrsa.gov/robots.txt","bytes":380,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":3382,"domain":"hse.ru","origin":"https://hse.ru","robots":{"status":"ok","http":200,"url":"https://www.hse.ru/robots.txt","bytes":3412,"groups":4},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["partial","*",45,0],"OAI-SearchBot":["partial","*",45,0],"ClaudeBot":["partial","*",45,0],"Claude-User":["partial","*",45,0],"Claude-SearchBot":["partial","*",45,0],"anthropic-ai":["partial","*",45,0],"PerplexityBot":["partial","*",45,0],"Perplexity-User":["partial","*",45,0],"Google-Extended":["partial","*",45,0],"Applebot-Extended":["partial","*",45,0],"CCBot":["partial","*",45,0],"Bytespider":["partial","*",45,0],"Amazonbot":["partial","*",45,0],"meta-externalagent":["partial","*",45,0],"cohere-ai":["partial","*",45,0],"Diffbot":["partial","*",45,0],"ContextFetchBot":["partial","*",45,0]},"llms":{"status":"missing","http":404,"bytes":33066},"home":{"status":"ok","http":200,"url":"https://www.hse.ru/en/","html_tokens":27798,"markdown_tokens":954,"pruned_tokens":430}},
{"rank":3383,"domain":"aalto.fi","origin":"https://aalto.fi","robots":{"status":"timeout","http":null,"url":"https://aalto.fi/robots.txt","bytes":0,"groups":0,"error":"timed out after 10 s"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":3384,"domain":"rarathemes.com","origin":"https://rarathemes.com","robots":{"status":"ok","http":200,"url":"https://rarathemes.com/robots.txt","bytes":136,"groups":1},"bots":{"GPTBot":["partial","*",2,1],"ChatGPT-User":["partial","*",2,1],"OAI-SearchBot":["partial","*",2,1],"ClaudeBot":["partial","*",2,1],"Claude-User":["partial","*",2,1],"Claude-SearchBot":["partial","*",2,1],"anthropic-ai":["partial","*",2,1],"PerplexityBot":["partial","*",2,1],"Perplexity-User":["partial","*",2,1],"Google-Extended":["partial","*",2,1],"Applebot-Extended":["partial","*",2,1],"CCBot":["partial","*",2,1],"Bytespider":["partial","*",2,1],"Amazonbot":["partial","*",2,1],"meta-externalagent":["partial","*",2,1],"cohere-ai":["partial","*",2,1],"Diffbot":["partial","*",2,1],"ContextFetchBot":["partial","*",2,1]},"llms":{"status":"missing","http":404,"bytes":61384},"home":{"status":"ok","http":200,"url":"https://rarathemes.com/","html_tokens":47463,"markdown_tokens":30,"pruned_tokens":30}},
{"rank":3385,"domain":"sachsen.de","origin":"https://sachsen.de","robots":{"status":"ok","http":200,"url":"https://www.sachsen.de/robots.txt","bytes":39,"groups":1},"bots":{"GPTBot":["partial","*",1,0],"ChatGPT-User":["partial","*",1,0],"OAI-SearchBot":["partial","*",1,0],"ClaudeBot":["partial","*",1,0],"Claude-User":["partial","*",1,0],"Claude-SearchBot":["partial","*",1,0],"anthropic-ai":["partial","*",1,0],"PerplexityBot":["partial","*",1,0],"Perplexity-User":["partial","*",1,0],"Google-Extended":["partial","*",1,0],"Applebot-Extended":["partial","*",1,0],"CCBot":["partial","*",1,0],"Bytespider":["partial","*",1,0],"Amazonbot":["partial","*",1,0],"meta-externalagent":["partial","*",1,0],"cohere-ai":["partial","*",1,0],"Diffbot":["partial","*",1,0],"ContextFetchBot":["partial","*",1,0]},"llms":{"status":"missing","http":404,"bytes":69606},"home":{"status":"ok","http":200,"url":"https://www.sachsen.de/","html_tokens":27328,"markdown_tokens":434,"pruned_tokens":41}},
{"rank":3386,"domain":"w3techs.com","origin":"https://w3techs.com","robots":{"status":"ok","http":200,"url":"https://w3techs.com/robots.txt","bytes":143,"groups":4},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":274},"home":{"status":"ok","http":200,"url":"https://w3techs.com/","html_tokens":14895,"markdown_tokens":5052,"pruned_tokens":716}},
{"rank":3387,"domain":"12371.cn","origin":"https://12371.cn","robots":{"status":"ok","http":200,"url":"http://www.12371.cn/robots.txt","bytes":77,"groups":1},"bots":{"GPTBot":["partial","*",2,0],"ChatGPT-User":["partial","*",2,0],"OAI-SearchBot":["partial","*",2,0],"ClaudeBot":["partial","*",2,0],"Claude-User":["partial","*",2,0],"Claude-SearchBot":["partial","*",2,0],"anthropic-ai":["partial","*",2,0],"PerplexityBot":["partial","*",2,0],"Perplexity-User":["partial","*",2,0],"Google-Extended":["partial","*",2,0],"Applebot-Extended":["partial","*",2,0],"CCBot":["partial","*",2,0],"Bytespider":["partial","*",2,0],"Amazonbot":["partial","*",2,0],"meta-externalagent":["partial","*",2,0],"cohere-ai":["partial","*",2,0],"Diffbot":["partial","*",2,0],"ContextFetchBot":["partial","*",2,0]},"llms":{"status":"html","http":200,"bytes":3425},"home":{"status":"ok","http":200,"url":"http://www.12371.cn/","html_tokens":34229,"markdown_tokens":32,"pruned_tokens":32}},
{"rank":3388,"domain":"mq.edu.au","origin":"https://mq.edu.au","robots":{"status":"refused","http":403,"url":"https://mq.edu.au/robots.txt","bytes":5504,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":3389,"domain":"bank.in","origin":"https://bank.in","robots":{"status":"unreachable","http":null,"url":"https://bank.in/robots.txt","bytes":0,"groups":0,"error":"cannot resolve bank.in"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":3390,"domain":"icao.int","origin":"https://icao.int","robots":{"status":"ok","http":200,"url":"https://www.icao.int/robots.txt","bytes":2141,"groups":1},"bots":{"GPTBot":["partial","*",35,22],"ChatGPT-User":["partial","*",35,22],"OAI-SearchBot":["partial","*",35,22],"ClaudeBot":["partial","*",35,22],"Claude-User":["partial","*",35,22],"Claude-SearchBot":["partial","*",35,22],"anthropic-ai":["partial","*",35,22],"PerplexityBot":["partial","*",35,22],"Perplexity-User":["partial","*",35,22],"Google-Extended":["partial","*",35,22],"Applebot-Extended":["partial","*",35,22],"CCBot":["partial","*",35,22],"Bytespider":["partial","*",35,22],"Amazonbot":["partial","*",35,22],"meta-externalagent":["partial","*",35,22],"cohere-ai":["partial","*",35,22],"Diffbot":["partial","*",35,22],"ContextFetchBot":["partial","*",35,22]},"llms":{"status":"missing","http":404,"bytes":186},"home":{"status":"ok","http":200,"url":"https://www.icao.int/","html_tokens":19172,"markdown_tokens":67,"pruned_tokens":67}},
{"rank":3391,"domain":"paulgraham.com","origin":"https://paulgraham.com","robots":{"status":"ok","http":200,"url":"https://paulgraham.com/robots.txt","bytes":265,"groups":2},"bots":{"GPTBot":["partial","*",12,0],"ChatGPT-User":["partial","*",12,0],"OAI-SearchBot":["partial","*",12,0],"ClaudeBot":["partial","*",12,0],"Claude-User":["partial","*",12,0],"Claude-SearchBot":["partial","*",12,0],"anthropic-ai":["partial","*",12,0],"PerplexityBot":["partial","*",12,0],"Perplexity-User":["partial","*",12,0],"Google-Extended":["partial","*",12,0],"Applebot-Extended":["partial","*",12,0],"CCBot":["partial","*",12,0],"Bytespider":["partial","*",12,0],"Amazonbot":["partial","*",12,0],"meta-externalagent":["partial","*",12,0],"cohere-ai":["partial","*",12,0],"Diffbot":["partial","*",12,0],"ContextFetchBot":["partial","*",12,0]},"llms":{"status":"missing","http":404,"bytes":0},"home":{"status":"ok","http":200,"url":"https://paulgraham.com/","html_tokens":1338,"markdown_tokens":34,"pruned_tokens":34}},
{"rank":3392,"domain":"pravda.ru","origin":"https://pravda.ru","robots":{"status":"ok","http":200,"url":"https://www.pravda.ru/robots.txt","bytes":1076,"groups":2},"bots":{"GPTBot":["partial","*",21,1],"ChatGPT-User":["partial","*",21,1],"OAI-SearchBot":["partial","*",21,1],"ClaudeBot":["partial","*",21,1],"Claude-User":["partial","*",21,1],"Claude-SearchBot":["partial","*",21,1],"anthropic-ai":["partial","*",21,1],"PerplexityBot":["partial","*",21,1],"Perplexity-User":["partial","*",21,1],"Google-Extended":["partial","*",21,1],"Applebot-Extended":["partial","*",21,1],"CCBot":["partial","*",21,1],"Bytespider":["partial","*",21,1],"Amazonbot":["partial","*",21,1],"meta-externalagent":["partial","*",21,1],"cohere-ai":["partial","*",21,1],"Diffbot":["partial","*",21,1],"ContextFetchBot":["partial","*",21,1]},"llms":{"status":"missing","http":404,"bytes":43},"home":{"status":"ok","http":200,"url":"https://www.pravda.ru/","html_tokens":28858,"markdown_tokens":629,"pruned_tokens":162}},
{"rank":3393,"domain":"freedomhouse.org","origin":"https://freedomhouse.org","robots":{"status":"ok","http":200,"url":"https://freedomhouse.org/robots.txt","bytes":3539,"groups":1},"bots":{"GPTBot":["partial","*",46,18],"ChatGPT-User":["partial","*",46,18],"OAI-SearchBot":["partial","*",46,18],"ClaudeBot":["partial","*",46,18],"Claude-User":["partial","*",46,18],"Claude-SearchBot":["partial","*",46,18],"anthropic-ai":["partial","*",46,18],"PerplexityBot":["partial","*",46,18],"Perplexity-User":["partial","*",46,18],"Google-Extended":["partial","*",46,18],"Applebot-Extended":["partial","*",46,18],"CCBot":["partial","*",46,18],"Bytespider":["partial","*",46,18],"Amazonbot":["partial","*",46,18],"meta-externalagent":["partial","*",46,18],"cohere-ai":["partial","*",46,18],"Diffbot":["partial","*",46,18],"ContextFetchBot":["partial","*",46,18]},"llms":{"status":"missing","http":404,"bytes":551},"home":{"status":"ok","http":200,"url":"https://freedomhouse.org/","html_tokens":28679,"markdown_tokens":230,"pruned_tokens":105}},
{"rank":3394,"domain":"cudasvc.com","origin":"https://cudasvc.com","robots":{"status":"unreachable","http":null,"url":"https://cudasvc.com/robots.txt","bytes":0,"groups":0,"error":"cannot resolve cudasvc.com"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":3395,"domain":"ustc.edu.cn","origin":"https://ustc.edu.cn","robots":{"status":"missing","http":404,"url":"https://ustc.edu.cn/robots.txt","bytes":1693,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":1693},"home":{"status":"ok","http":200,"url":"https://ustc.edu.cn/","html_tokens":9330,"markdown_tokens":226,"pruned_tokens":158}},
{"rank":3396,"domain":"jpn.org","origin":"https://jpn.org","robots":{"status":"unreachable","http":null,"url":"https://jpn.org/robots.txt","bytes":0,"groups":0,"error":"cannot resolve jpn.org"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":3397,"domain":"sage.com","origin":"https://sage.com","robots":{"status":"ok","http":200,"url":"https://www.sage.com/robots.txt","bytes":1607,"groups":3},"bots":{"GPTBot":["partial","*",50,4],"ChatGPT-User":["partial","*",50,4],"OAI-SearchBot":["partial","*",50,4],"ClaudeBot":["partial","*",50,4],"Claude-User":["partial","*",50,4],"Claude-SearchBot":["partial","*",50,4],"anthropic-ai":["partial","*",50,4],"PerplexityBot":["partial","*",50,4],"Perplexity-User":["partial","*",50,4],"Google-Extended":["partial","*",50,4],"Applebot-Extended":["partial","*",50,4],"CCBot":["partial","*",50,4],"Bytespider":["partial","*",50,4],"Amazonbot":["partial","*",50,4],"meta-externalagent":["partial","*",50,4],"cohere-ai":["partial","*",50,4],"Diffbot":["partial","*",50,4],"ContextFetchBot":["partial","*",50,4]},"llms":{"status":"http_error","http":403,"bytes":4545},"home":{"status":"http_error","http":403,"url":"https://www.sage.com/en-us/"}},
{"rank":3398,"domain":"coolors.co","origin":"https://coolors.co","robots":{"status":"ok","http":200,"url":"https://coolors.co/robots.txt","bytes":140,"groups":2},"bots":{"GPTBot":["partial","*",3,0],"ChatGPT-User":["partial","*",3,0],"OAI-SearchBot":["partial","*",3,0],"ClaudeBot":["partial","*",3,0],"Claude-User":["partial","*",3,0],"Claude-SearchBot":["partial","*",3,0],"anthropic-ai":["partial","*",3,0],"PerplexityBot":["partial","*",3,0],"Perplexity-User":["partial","*",3,0],"Google-Extended":["partial","*",3,0],"Applebot-Extended":["partial","*",3,0],"CCBot":["partial","*",3,0],"Bytespider":["partial","*",3,0],"Amazonbot":["partial","*",3,0],"meta-externalagent":["partial","*",3,0],"cohere-ai":["partial","*",3,0],"Diffbot":["partial","*",3,0],"ContextFetchBot":["partial","*",3,0]},"llms":{"status":"html","http":200,"bytes":553476},"home":{"status":"ok","http":200,"url":"https://coolors.co/","html_tokens":141426,"markdown_tokens":113,"pruned_tokens":40}},
{"rank":3399,"domain":"digiday.com","origin":"https://digiday.com","robots":{"status":"ok","http":200,"url":"https://digiday.com/robots.txt","bytes":326,"groups":1},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"missing","http":404,"bytes":43995},"home":{"status":"ok","http":200,"url":"https://digiday.com/","html_tokens":37729,"markdown_tokens":600,"pruned_tokens":161}},
{"rank":3400,"domain":"player.fm","origin":"https://player.fm","robots":{"status":"refused","http":403,"url":"https://player.fm/robots.txt","bytes":5504,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":3401,"domain":"supsystic.com","origin":"https://supsystic.com","robots":{"status":"ok","http":200,"url":"https://supsystic.com/robots.txt","bytes":684,"groups":1},"bots":{"GPTBot":["partial","*",31,0],"ChatGPT-User":["partial","*",31,0],"OAI-SearchBot":["partial","*",31,0],"ClaudeBot":["partial","*",31,0],"Claude-User":["partial","*",31,0],"Claude-SearchBot":["partial","*",31,0],"anthropic-ai":["partial","*",31,0],"PerplexityBot":["partial","*",31,0],"Perplexity-User":["partial","*",31,0],"Google-Extended":["partial","*",31,0],"Applebot-Extended":["partial","*",31,0],"CCBot":["partial","*",31,0],"Bytespider":["partial","*",31,0],"Amazonbot":["partial","*",31,0],"meta-externalagent":["partial","*",31,0],"cohere-ai":["partial","*",31,0],"Diffbot":["partial","*",31,0],"ContextFetchBot":["partial","*",31,0]},"llms":{"status":"missing","http":404,"bytes":73825},"home":{"status":"ok","http":200,"url":"https://supsystic.com/","html_tokens":27431,"markdown_tokens":141,"pruned_tokens":16}},
{"rank":3402,"domain":"unu.edu","origin":"https://unu.edu","robots":{"status":"refused","http":403,"url":"https://unu.edu/robots.txt","bytes":560,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":3403,"domain":"wvu.edu","origin":"https://wvu.edu","robots":{"status":"refused","http":403,"url":"https://www.wvu.edu/robots.txt","bytes":919,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":3404,"domain":"wayfair.com","origin":"https://wayfair.com","robots":{"status":"ok","http":200,"url":"https://www.wayfair.com/robots.txt","bytes":3224,"groups":1},"bots":{"GPTBot":["partial","*",51,12],"ChatGPT-User":["partial","*",51,12],"OAI-SearchBot":["partial","*",51,12],"ClaudeBot":["partial","*",51,12],"Claude-User":["partial","*",51,12],"Claude-SearchBot":["partial","*",51,12],"anthropic-ai":["partial","*",51,12],"PerplexityBot":["partial","*",51,12],"Perplexity-User":["partial","*",51,12],"Google-Extended":["partial","*",51,12],"Applebot-Extended":["partial","*",51,12],"CCBot":["partial","*",51,12],"Bytespider":["partial","*",51,12],"Amazonbot":["partial","*",51,12],"meta-externalagent":["partial","*",51,12],"cohere-ai":["partial","*",51,12],"Diffbot":["partial","*",51,12],"ContextFetchBot":["partial","*",51,12]},"llms":{"status":"missing","http":404,"bytes":860398},"home":{"status":"ok","http":200,"url":"https://www.wayfair.com/","html_tokens":223672,"markdown_tokens":9,"pruned_tokens":9}},
{"rank":3405,"domain":"devpost.com","origin":"https://devpost.com","robots":{"status":"ok","http":200,"url":"https://devpost.com/robots.txt","bytes":403,"groups":6},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"present","http":200,"bytes":1674},"home":{"status":"http_error","http":403,"url":"https://devpost.com/"}},
{"rank":3406,"domain":"sil.org","origin":"https://sil.org","robots":{"status":"ok","http":200,"url":"https://www.sil.org/robots.txt","bytes":5409,"groups":37},"bots":{"GPTBot":["partial","*",45,33],"ChatGPT-User":["partial","*",45,33],"OAI-SearchBot":["partial","*",45,33],"ClaudeBot":["partial","*",45,33],"Claude-User":["partial","*",45,33],"Claude-SearchBot":["partial","*",45,33],"anthropic-ai":["partial","*",45,33],"PerplexityBot":["partial","*",45,33],"Perplexity-User":["partial","*",45,33],"Google-Extended":["partial","*",45,33],"Applebot-Extended":["partial","*",45,33],"CCBot":["partial","*",45,33],"Bytespider":["blocked","named",1,0],"Amazonbot":["partial","*",45,33],"meta-externalagent":["partial","*",45,33],"cohere-ai":["partial","*",45,33],"Diffbot":["partial","*",45,33],"ContextFetchBot":["partial","*",45,33]},"llms":{"status":"missing","http":404,"bytes":29868},"home":{"status":"http_error","http":403,"url":"https://global.sil.org/"}},
{"rank":3407,"domain":"globalprivacycontrol.org","origin":"https://globalprivacycontrol.org","robots":{"status":"missing","http":404,"url":"https://globalprivacycontrol.org/robots.txt","bytes":3381,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":3381},"home":{"status":"ok","http":200,"url":"https://globalprivacycontrol.org/","html_tokens":17229,"markdown_tokens":1026,"pruned_tokens":170}},
{"rank":3408,"domain":"project-syndicate.org","origin":"https://project-syndicate.org","robots":{"status":"ok","http":200,"url":"https://www.project-syndicate.org/robots.txt","bytes":1504,"groups":23},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["blocked","named",1,0],"OAI-SearchBot":["partial","*",16,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",16,0],"Claude-SearchBot":["partial","*",16,0],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["partial","*",16,0],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["partial","*",16,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",16,0]},"llms":{"status":"missing","http":404,"bytes":9},"home":{"status":"ok","http":200,"url":"https://www.project-syndicate.org/","html_tokens":55782,"markdown_tokens":113,"pruned_tokens":113}},
{"rank":3409,"domain":"oscars.org","origin":"https://oscars.org","robots":{"status":"ok","http":200,"url":"https://www.oscars.org/robots.txt","bytes":2026,"groups":1},"bots":{"GPTBot":["partial","*",34,18],"ChatGPT-User":["partial","*",34,18],"OAI-SearchBot":["partial","*",34,18],"ClaudeBot":["partial","*",34,18],"Claude-User":["partial","*",34,18],"Claude-SearchBot":["partial","*",34,18],"anthropic-ai":["partial","*",34,18],"PerplexityBot":["partial","*",34,18],"Perplexity-User":["partial","*",34,18],"Google-Extended":["partial","*",34,18],"Applebot-Extended":["partial","*",34,18],"CCBot":["partial","*",34,18],"Bytespider":["partial","*",34,18],"Amazonbot":["partial","*",34,18],"meta-externalagent":["partial","*",34,18],"cohere-ai":["partial","*",34,18],"Diffbot":["partial","*",34,18],"ContextFetchBot":["partial","*",34,18]},"llms":{"status":"missing","http":404,"bytes":188},"home":{"status":"ok","http":200,"url":"https://www.oscars.org/","html_tokens":13323,"markdown_tokens":117,"pruned_tokens":47}},
{"rank":3410,"domain":"llnl.gov","origin":"https://llnl.gov","robots":{"status":"ok","http":200,"url":"https://www.llnl.gov/robots.txt","bytes":2027,"groups":1},"bots":{"GPTBot":["partial","*",34,18],"ChatGPT-User":["partial","*",34,18],"OAI-SearchBot":["partial","*",34,18],"ClaudeBot":["partial","*",34,18],"Claude-User":["partial","*",34,18],"Claude-SearchBot":["partial","*",34,18],"anthropic-ai":["partial","*",34,18],"PerplexityBot":["partial","*",34,18],"Perplexity-User":["partial","*",34,18],"Google-Extended":["partial","*",34,18],"Applebot-Extended":["partial","*",34,18],"CCBot":["partial","*",34,18],"Bytespider":["partial","*",34,18],"Amazonbot":["partial","*",34,18],"meta-externalagent":["partial","*",34,18],"cohere-ai":["partial","*",34,18],"Diffbot":["partial","*",34,18],"ContextFetchBot":["partial","*",34,18]},"llms":{"status":"missing","http":404,"bytes":547},"home":{"status":"ok","http":200,"url":"https://www.llnl.gov/","html_tokens":31733,"markdown_tokens":259,"pruned_tokens":224}},
{"rank":3411,"domain":"reactjs.org","origin":"https://reactjs.org","robots":{"status":"ok","http":200,"url":"https://legacy.reactjs.org/robots.txt","bytes":33,"groups":1},"bots":{"GPTBot":["partial","*",1,0],"ChatGPT-User":["partial","*",1,0],"OAI-SearchBot":["partial","*",1,0],"ClaudeBot":["partial","*",1,0],"Claude-User":["partial","*",1,0],"Claude-SearchBot":["partial","*",1,0],"anthropic-ai":["partial","*",1,0],"PerplexityBot":["partial","*",1,0],"Perplexity-User":["partial","*",1,0],"Google-Extended":["partial","*",1,0],"Applebot-Extended":["partial","*",1,0],"CCBot":["partial","*",1,0],"Bytespider":["partial","*",1,0],"Amazonbot":["partial","*",1,0],"meta-externalagent":["partial","*",1,0],"cohere-ai":["partial","*",1,0],"Diffbot":["partial","*",1,0],"ContextFetchBot":["partial","*",1,0]},"llms":{"status":"missing","http":404,"bytes":71232},"home":{"status":"http_error","http":404,"url":"https://react.dev/"}},
{"rank":3412,"domain":"up.pt","origin":"https://up.pt","robots":{"status":"ok","http":200,"url":"https://www.up.pt/robots.txt","bytes":68,"groups":1},"bots":{"GPTBot":["partial","*",2,0],"ChatGPT-User":["partial","*",2,0],"OAI-SearchBot":["partial","*",2,0],"ClaudeBot":["partial","*",2,0],"Claude-User":["partial","*",2,0],"Claude-SearchBot":["partial","*",2,0],"anthropic-ai":["partial","*",2,0],"PerplexityBot":["partial","*",2,0],"Perplexity-User":["partial","*",2,0],"Google-Extended":["partial","*",2,0],"Applebot-Extended":["partial","*",2,0],"CCBot":["partial","*",2,0],"Bytespider":["partial","*",2,0],"Amazonbot":["partial","*",2,0],"meta-externalagent":["partial","*",2,0],"cohere-ai":["partial","*",2,0],"Diffbot":["partial","*",2,0],"ContextFetchBot":["partial","*",2,0]},"llms":{"status":"missing","http":404,"bytes":557},"home":{"status":"ok","http":200,"url":"https://www.up.pt/portal/en/","html_tokens":71194,"markdown_tokens":540,"pruned_tokens":174}},
{"rank":3413,"domain":"nhc.gov.cn","origin":"https://www.nhc.gov.cn","robots":{"status":"missing","http":404,"url":"https://www.nhc.gov.cn/robots.txt","bytes":146,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":146},"home":{"status":"http_error","http":412,"url":"https://www.nhc.gov.cn/"}},
{"rank":3414,"domain":"camp-fire.jp","origin":"https://camp-fire.jp","robots":{"status":"ok","http":200,"url":"https://camp-fire.jp/robots.txt","bytes":1787,"groups":2},"bots":{"GPTBot":["partial","named",12,0],"ChatGPT-User":["partial","*",10,0],"OAI-SearchBot":["partial","*",10,0],"ClaudeBot":["partial","named",12,0],"Claude-User":["partial","*",10,0],"Claude-SearchBot":["partial","*",10,0],"anthropic-ai":["partial","*",10,0],"PerplexityBot":["partial","*",10,0],"Perplexity-User":["partial","*",10,0],"Google-Extended":["partial","named",12,0],"Applebot-Extended":["partial","named",12,0],"CCBot":["partial","named",12,0],"Bytespider":["partial","*",10,0],"Amazonbot":["partial","named",12,0],"meta-externalagent":["partial","named",12,0],"cohere-ai":["partial","*",10,0],"Diffbot":["partial","*",10,0],"ContextFetchBot":["partial","*",10,0]},"llms":{"status":"missing","http":404,"bytes":39035},"home":{"status":"ok","http":200,"url":"https://camp-fire.jp/","html_tokens":150451,"markdown_tokens":45,"pruned_tokens":45}},
{"rank":3415,"domain":"openclaw.ai","origin":"https://openclaw.ai","robots":{"status":"ok","http":200,"url":"https://openclaw.ai/robots.txt","bytes":71,"groups":1},"bots":{"GPTBot":["allowed","*",0,1],"ChatGPT-User":["allowed","*",0,1],"OAI-SearchBot":["allowed","*",0,1],"ClaudeBot":["allowed","*",0,1],"Claude-User":["allowed","*",0,1],"Claude-SearchBot":["allowed","*",0,1],"anthropic-ai":["allowed","*",0,1],"PerplexityBot":["allowed","*",0,1],"Perplexity-User":["allowed","*",0,1],"Google-Extended":["allowed","*",0,1],"Applebot-Extended":["allowed","*",0,1],"CCBot":["allowed","*",0,1],"Bytespider":["allowed","*",0,1],"Amazonbot":["allowed","*",0,1],"meta-externalagent":["allowed","*",0,1],"cohere-ai":["allowed","*",0,1],"Diffbot":["allowed","*",0,1],"ContextFetchBot":["allowed","*",0,1]},"llms":{"status":"present","http":200,"bytes":900},"home":{"status":"ok","http":200,"url":"https://openclaw.ai/","html_tokens":53203,"markdown_tokens":3676,"pruned_tokens":736}},
{"rank":3416,"domain":"redd.it","origin":"https://redd.it","robots":{"status":"ok","http":200,"url":"https://redd.it/robots.txt","bytes":24,"groups":1},"bots":{"GPTBot":["allowed","*",0,1],"ChatGPT-User":["allowed","*",0,1],"OAI-SearchBot":["allowed","*",0,1],"ClaudeBot":["allowed","*",0,1],"Claude-User":["allowed","*",0,1],"Claude-SearchBot":["allowed","*",0,1],"anthropic-ai":["allowed","*",0,1],"PerplexityBot":["allowed","*",0,1],"Perplexity-User":["allowed","*",0,1],"Google-Extended":["allowed","*",0,1],"Applebot-Extended":["allowed","*",0,1],"CCBot":["allowed","*",0,1],"Bytespider":["allowed","*",0,1],"Amazonbot":["allowed","*",0,1],"meta-externalagent":["allowed","*",0,1],"cohere-ai":["allowed","*",0,1],"Diffbot":["allowed","*",0,1],"ContextFetchBot":["allowed","*",0,1]},"llms":{"status":"missing","http":404,"bytes":13},"home":{"status":"disallowed","error":"robots.txt of www.reddit.com disallows /"}},
{"rank":3417,"domain":"computerhistory.org","origin":"https://computerhistory.org","robots":{"status":"ok","http":200,"url":"https://computerhistory.org/robots.txt","bytes":177,"groups":1},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"missing","http":404,"bytes":74955},"home":{"status":"ok","http":200,"url":"https://computerhistory.org/","html_tokens":25359,"markdown_tokens":550,"pruned_tokens":95}},
{"rank":3418,"domain":"phoca.cz","origin":"https://phoca.cz","robots":{"status":"ok","http":200,"url":"https://www.phoca.cz/robots.txt","bytes":1143,"groups":3},"bots":{"GPTBot":["partial","*",15,0],"ChatGPT-User":["partial","*",15,0],"OAI-SearchBot":["partial","*",15,0],"ClaudeBot":["partial","*",15,0],"Claude-User":["partial","*",15,0],"Claude-SearchBot":["partial","*",15,0],"anthropic-ai":["partial","*",15,0],"PerplexityBot":["partial","*",15,0],"Perplexity-User":["partial","*",15,0],"Google-Extended":["partial","*",15,0],"Applebot-Extended":["partial","*",15,0],"CCBot":["partial","*",15,0],"Bytespider":["partial","*",15,0],"Amazonbot":["partial","*",15,0],"meta-externalagent":["partial","*",15,0],"cohere-ai":["partial","*",15,0],"Diffbot":["partial","*",15,0],"ContextFetchBot":["partial","*",15,0]},"llms":{"status":"missing","http":404,"bytes":25395},"home":{"status":"ok","http":200,"url":"https://www.phoca.cz/","html_tokens":17328,"markdown_tokens":158,"pruned_tokens":158}},
{"rank":3419,"domain":"thedrum.com","origin":"https://thedrum.com","robots":{"status":"ok","http":200,"url":"https://www.thedrum.com/robots.txt","bytes":2840,"groups":45},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["allowed","named",0,1],"OAI-SearchBot":["allowed","named",0,1],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",38,0],"Claude-SearchBot":["partial","*",38,0],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["allowed","named",0,1],"Perplexity-User":["allowed","named",0,1],"Google-Extended":["partial","*",38,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",38,0]},"llms":{"status":"missing","http":404,"bytes":146},"home":{"status":"ok","http":200,"url":"https://www.thedrum.com/","html_tokens":61846,"markdown_tokens":123,"pruned_tokens":123}},
{"rank":3420,"domain":"ru.nl","origin":"https://ru.nl","robots":{"status":"missing","http":404,"url":"https://www.ru.nl/404.html","bytes":196,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":183},"home":{"status":"ok","http":200,"url":"https://www.ru.nl/","html_tokens":16361,"markdown_tokens":432,"pruned_tokens":167}},
{"rank":3421,"domain":"upc.edu","origin":"https://upc.edu","robots":{"status":"ok","http":200,"url":"https://www.upc.edu/robots.txt","bytes":1835,"groups":2},"bots":{"GPTBot":["partial","*",23,0],"ChatGPT-User":["partial","*",23,0],"OAI-SearchBot":["partial","*",23,0],"ClaudeBot":["partial","*",23,0],"Claude-User":["partial","*",23,0],"Claude-SearchBot":["partial","*",23,0],"anthropic-ai":["partial","*",23,0],"PerplexityBot":["partial","*",23,0],"Perplexity-User":["partial","*",23,0],"Google-Extended":["partial","*",23,0],"Applebot-Extended":["partial","*",23,0],"CCBot":["partial","*",23,0],"Bytespider":["partial","*",23,0],"Amazonbot":["partial","*",23,0],"meta-externalagent":["partial","*",23,0],"cohere-ai":["partial","*",23,0],"Diffbot":["partial","*",23,0],"ContextFetchBot":["partial","*",23,0]},"llms":{"status":"missing","http":404,"bytes":78430},"home":{"status":"ok","http":200,"url":"https://www.upc.edu/ca","html_tokens":127746,"markdown_tokens":188,"pruned_tokens":126}},
{"rank":3422,"domain":"iarc.fr","origin":"https://iarc.fr","robots":{"status":"ok","http":200,"url":"https://www.iarc.who.int/robots.txt","bytes":67,"groups":1},"bots":{"GPTBot":["partial","*",1,1],"ChatGPT-User":["partial","*",1,1],"OAI-SearchBot":["partial","*",1,1],"ClaudeBot":["partial","*",1,1],"Claude-User":["partial","*",1,1],"Claude-SearchBot":["partial","*",1,1],"anthropic-ai":["partial","*",1,1],"PerplexityBot":["partial","*",1,1],"Perplexity-User":["partial","*",1,1],"Google-Extended":["partial","*",1,1],"Applebot-Extended":["partial","*",1,1],"CCBot":["partial","*",1,1],"Bytespider":["partial","*",1,1],"Amazonbot":["partial","*",1,1],"meta-externalagent":["partial","*",1,1],"cohere-ai":["partial","*",1,1],"Diffbot":["partial","*",1,1],"ContextFetchBot":["partial","*",1,1]},"llms":{"status":"missing","http":404,"bytes":50934},"home":{"status":"ok","http":200,"url":"https://www.iarc.who.int/","html_tokens":30204,"markdown_tokens":24,"pruned_tokens":24}},
{"rank":3423,"domain":"upv.es","origin":"https://www.upv.es","robots":{"status":"ok","http":200,"url":"https://www.upv.es/robots.txt","bytes":1268,"groups":2},"bots":{"GPTBot":["partial","*",12,10],"ChatGPT-User":["partial","*",12,10],"OAI-SearchBot":["partial","*",12,10],"ClaudeBot":["partial","*",12,10],"Claude-User":["partial","*",12,10],"Claude-SearchBot":["partial","*",12,10],"anthropic-ai":["partial","*",12,10],"PerplexityBot":["partial","*",12,10],"Perplexity-User":["partial","*",12,10],"Google-Extended":["partial","*",12,10],"Applebot-Extended":["partial","*",12,10],"CCBot":["partial","*",12,10],"Bytespider":["partial","*",12,10],"Amazonbot":["partial","*",12,10],"meta-externalagent":["partial","*",12,10],"cohere-ai":["partial","*",12,10],"Diffbot":["partial","*",12,10],"ContextFetchBot":["partial","*",12,10]},"llms":{"status":"missing","http":404,"bytes":15472},"home":{"status":"ok","http":200,"url":"https://www.upv.es/","html_tokens":29908,"markdown_tokens":223,"pruned_tokens":107}},
{"rank":3424,"domain":"reurl.cc","origin":"https://reurl.cc","robots":{"status":"ok","http":200,"url":"https://reurl.cc/robots.txt","bytes":23,"groups":1},"bots":{"GPTBot":["allowed","*",0,1],"ChatGPT-User":["allowed","*",0,1],"OAI-SearchBot":["allowed","*",0,1],"ClaudeBot":["allowed","*",0,1],"Claude-User":["allowed","*",0,1],"Claude-SearchBot":["allowed","*",0,1],"anthropic-ai":["allowed","*",0,1],"PerplexityBot":["allowed","*",0,1],"Perplexity-User":["allowed","*",0,1],"Google-Extended":["allowed","*",0,1],"Applebot-Extended":["allowed","*",0,1],"CCBot":["allowed","*",0,1],"Bytespider":["allowed","*",0,1],"Amazonbot":["allowed","*",0,1],"meta-externalagent":["allowed","*",0,1],"cohere-ai":["allowed","*",0,1],"Diffbot":["allowed","*",0,1],"ContextFetchBot":["allowed","*",0,1]},"llms":{"status":"missing","http":404,"bytes":153},"home":{"status":"ok","http":200,"url":"https://reurl.cc/main/en","html_tokens":8883,"markdown_tokens":205,"pruned_tokens":128}},
{"rank":3425,"domain":"honor.com","origin":"https://honor.com","robots":{"status":"ok","http":200,"url":"https://www.honor.com/robots.txt","bytes":1921,"groups":1},"bots":{"GPTBot":["partial","*",66,2],"ChatGPT-User":["partial","*",66,2],"OAI-SearchBot":["partial","*",66,2],"ClaudeBot":["partial","*",66,2],"Claude-User":["partial","*",66,2],"Claude-SearchBot":["partial","*",66,2],"anthropic-ai":["partial","*",66,2],"PerplexityBot":["partial","*",66,2],"Perplexity-User":["partial","*",66,2],"Google-Extended":["partial","*",66,2],"Applebot-Extended":["partial","*",66,2],"CCBot":["partial","*",66,2],"Bytespider":["partial","*",66,2],"Amazonbot":["partial","*",66,2],"meta-externalagent":["partial","*",66,2],"cohere-ai":["partial","*",66,2],"Diffbot":["partial","*",66,2],"ContextFetchBot":["partial","*",66,2]},"llms":{"status":"present","http":200,"bytes":1960},"home":{"status":"timeout","error":"timed out after 15 s"}},
{"rank":3426,"domain":"kaltura.com","origin":"https://kaltura.com","robots":{"status":"ok","http":200,"url":"http://www.kaltura.com/robots.txt","bytes":398,"groups":2},"bots":{"GPTBot":["partial","*",5,2],"ChatGPT-User":["partial","*",5,2],"OAI-SearchBot":["partial","*",5,2],"ClaudeBot":["partial","*",5,2],"Claude-User":["partial","*",5,2],"Claude-SearchBot":["partial","*",5,2],"anthropic-ai":["partial","*",5,2],"PerplexityBot":["partial","*",5,2],"Perplexity-User":["partial","*",5,2],"Google-Extended":["partial","*",5,2],"Applebot-Extended":["partial","*",5,2],"CCBot":["partial","*",5,2],"Bytespider":["partial","*",5,2],"Amazonbot":["partial","*",5,2],"meta-externalagent":["partial","*",5,2],"cohere-ai":["partial","*",5,2],"Diffbot":["partial","*",5,2],"ContextFetchBot":["partial","*",5,2]},"llms":{"status":"missing","http":404,"bytes":261},"home":{"status":"not_html","error":"unsupported content type text/markdown"}},
{"rank":3427,"domain":"oprah.com","origin":"https://oprah.com","robots":{"status":"refused","http":403,"url":"https://www.oprah.com/robots.txt","bytes":381,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":3428,"domain":"iadb.org","origin":"https://iadb.org","robots":{"status":"ok","http":200,"url":"https://iadb.org/robots.txt","bytes":2725,"groups":1},"bots":{"GPTBot":["partial","*",54,18],"ChatGPT-User":["partial","*",54,18],"OAI-SearchBot":["partial","*",54,18],"ClaudeBot":["partial","*",54,18],"Claude-User":["partial","*",54,18],"Claude-SearchBot":["partial","*",54,18],"anthropic-ai":["partial","*",54,18],"PerplexityBot":["partial","*",54,18],"Perplexity-User":["partial","*",54,18],"Google-Extended":["partial","*",54,18],"Applebot-Extended":["partial","*",54,18],"CCBot":["partial","*",54,18],"Bytespider":["partial","*",54,18],"Amazonbot":["partial","*",54,18],"meta-externalagent":["partial","*",54,18],"cohere-ai":["partial","*",54,18],"Diffbot":["partial","*",54,18],"ContextFetchBot":["partial","*",54,18]},"llms":{"status":"disallowed","error":"robots.txt of www.iadb.org not readable (refused)"},"home":{"status":"disallowed","error":"robots.txt of www.iadb.org not readable (refused)"}},
{"rank":3429,"domain":"coocan.jp","origin":"https://coocan.jp","robots":{"status":"timeout","http":null,"url":"https://coocan.jp/robots.txt","bytes":0,"groups":0,"error":"ConnectTimeout"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":3430,"domain":"carleton.ca","origin":"https://carleton.ca","robots":{"status":"ok","http":200,"url":"https://carleton.ca/robots.txt","bytes":5817,"groups":1},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"missing","http":404,"bytes":137608},"home":{"status":"ok","http":200,"url":"https://carleton.ca/","html_tokens":48872,"markdown_tokens":500,"pruned_tokens":201}},
{"rank":3431,"domain":"sc.gov","origin":"https://sc.gov","robots":{"status":"missing","http":404,"url":"https://sc.gov/robots.txt","bytes":182,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":180},"home":{"status":"ok","http":200,"url":"https://sc.gov/","html_tokens":98262,"markdown_tokens":259,"pruned_tokens":66}},
{"rank":3432,"domain":"gdpr.eu","origin":"https://gdpr.eu","robots":{"status":"ok","http":200,"url":"https://gdpr.eu/robots.txt","bytes":165,"groups":1},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"missing","http":404,"bytes":30394},"home":{"status":"ok","http":200,"url":"https://gdpr.eu/","html_tokens":10965,"markdown_tokens":378,"pruned_tokens":129}},
{"rank":3433,"domain":"wroc.pl","origin":"https://wroc.pl","robots":{"status":"unreachable","http":null,"url":"https://wroc.pl/robots.txt","bytes":0,"groups":0,"error":"cannot resolve wroc.pl"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":3434,"domain":"yimg.com","origin":"https://yimg.com","robots":{"status":"html","http":200,"url":"https://yimg.com/robots.txt","bytes":194,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":3435,"domain":"nga.gov","origin":"https://www.nga.gov","robots":{"status":"ok","http":200,"url":"https://www.nga.gov/robots.txt","bytes":2366,"groups":1},"bots":{"GPTBot":["partial","*",42,18],"ChatGPT-User":["partial","*",42,18],"OAI-SearchBot":["partial","*",42,18],"ClaudeBot":["partial","*",42,18],"Claude-User":["partial","*",42,18],"Claude-SearchBot":["partial","*",42,18],"anthropic-ai":["partial","*",42,18],"PerplexityBot":["partial","*",42,18],"Perplexity-User":["partial","*",42,18],"Google-Extended":["partial","*",42,18],"Applebot-Extended":["partial","*",42,18],"CCBot":["partial","*",42,18],"Bytespider":["partial","*",42,18],"Amazonbot":["partial","*",42,18],"meta-externalagent":["partial","*",42,18],"cohere-ai":["partial","*",42,18],"Diffbot":["partial","*",42,18],"ContextFetchBot":["partial","*",42,18]},"llms":{"status":"missing","http":404,"bytes":546},"home":{"status":"http_error","http":403,"url":"https://www.nga.gov/"}},
{"rank":3436,"domain":"gotomeeting.com","origin":"https://gotomeeting.com","robots":{"status":"html","http":200,"url":"https://www.goto.com/meeting","bytes":322114,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":3437,"domain":"truthout.org","origin":"https://truthout.org","robots":{"status":"ok","http":200,"url":"https://truthout.org/robots.txt","bytes":850,"groups":4},"bots":{"GPTBot":["partial","*",2,1],"ChatGPT-User":["partial","*",2,1],"OAI-SearchBot":["partial","*",2,1],"ClaudeBot":["partial","*",2,1],"Claude-User":["partial","*",2,1],"Claude-SearchBot":["partial","*",2,1],"anthropic-ai":["partial","*",2,1],"PerplexityBot":["partial","*",2,1],"Perplexity-User":["partial","*",2,1],"Google-Extended":["partial","*",2,1],"Applebot-Extended":["partial","*",2,1],"CCBot":["partial","*",2,1],"Bytespider":["partial","*",2,1],"Amazonbot":["partial","*",2,1],"meta-externalagent":["partial","*",2,1],"cohere-ai":["partial","*",2,1],"Diffbot":["partial","*",2,1],"ContextFetchBot":["partial","*",2,1]},"llms":{"status":"missing","http":404,"bytes":189184},"home":{"status":"ok","http":200,"url":"https://truthout.org/","html_tokens":68059,"markdown_tokens":57,"pruned_tokens":53}},
{"rank":3438,"domain":"nsa.gov","origin":"https://nsa.gov","robots":{"status":"refused","http":403,"url":"https://nsa.gov/robots.txt","bytes":371,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":3439,"domain":"dailyrecord.co.uk","origin":"https://dailyrecord.co.uk","robots":{"status":"ok","http":200,"url":"https://www.dailyrecord.co.uk/robots.txt","bytes":1693,"groups":13},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["partial","*",27,0],"OAI-SearchBot":["blocked","named",1,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",27,0],"Claude-SearchBot":["partial","*",27,0],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["partial","*",27,0],"Google-Extended":["partial","*",27,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["partial","*",27,0],"Amazonbot":["partial","*",27,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["partial","*",27,0],"Diffbot":["partial","*",27,0],"ContextFetchBot":["partial","*",27,0]},"llms":{"status":"missing","http":404,"bytes":801988},"home":{"status":"ok","http":200,"url":"https://www.dailyrecord.co.uk/","html_tokens":208170,"markdown_tokens":153,"pruned_tokens":153}},
{"rank":3440,"domain":"grammy.com","origin":"https://grammy.com","robots":{"status":"ok","http":200,"url":"https://www.grammy.com/robots.txt","bytes":69,"groups":1},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"missing","http":404,"bytes":140718},"home":{"status":"ok","http":200,"url":"https://www.grammy.com/","html_tokens":49488,"markdown_tokens":245,"pruned_tokens":105}},
{"rank":3441,"domain":"pluralsight.com","origin":"https://pluralsight.com","robots":{"status":"ok","http":200,"url":"https://www.pluralsight.com/robots.txt","bytes":2739,"groups":32},"bots":{"GPTBot":["partial","*",34,0],"ChatGPT-User":["partial","*",34,0],"OAI-SearchBot":["partial","*",34,0],"ClaudeBot":["partial","*",34,0],"Claude-User":["partial","*",34,0],"Claude-SearchBot":["partial","*",34,0],"anthropic-ai":["partial","*",34,0],"PerplexityBot":["partial","*",34,0],"Perplexity-User":["partial","*",34,0],"Google-Extended":["partial","*",34,0],"Applebot-Extended":["partial","*",34,0],"CCBot":["partial","*",34,0],"Bytespider":["partial","*",34,0],"Amazonbot":["partial","*",34,0],"meta-externalagent":["partial","*",34,0],"cohere-ai":["partial","*",34,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",34,0]},"llms":{"status":"missing","http":404,"bytes":401224},"home":{"status":"ok","http":200,"url":"https://www.pluralsight.com/","html_tokens":114328,"markdown_tokens":531,"pruned_tokens":88}},
{"rank":3442,"domain":"photoshelter.com","origin":"https://photoshelter.com","robots":{"status":"ok","http":200,"url":"https://www.photoshelter.com/robots.txt","bytes":1281,"groups":9},"bots":{"GPTBot":["partial","*",14,0],"ChatGPT-User":["partial","*",14,0],"OAI-SearchBot":["partial","*",14,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",14,0],"Claude-SearchBot":["partial","*",14,0],"anthropic-ai":["partial","*",14,0],"PerplexityBot":["partial","*",14,0],"Perplexity-User":["partial","*",14,0],"Google-Extended":["partial","*",14,0],"Applebot-Extended":["partial","*",14,0],"CCBot":["partial","*",14,0],"Bytespider":["partial","*",14,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["partial","*",14,0],"cohere-ai":["partial","*",14,0],"Diffbot":["partial","*",14,0],"ContextFetchBot":["partial","*",14,0]},"llms":{"status":"missing","http":404,"bytes":31815},"home":{"status":"ok","http":200,"url":"https://www.photoshelter.com","html_tokens":80776,"markdown_tokens":310,"pruned_tokens":101}},
{"rank":3443,"domain":"khaleejtimes.com","origin":"https://khaleejtimes.com","robots":{"status":"ok","http":200,"url":"https://www.khaleejtimes.com/robots.txt","bytes":3698,"groups":11},"bots":{"GPTBot":["partial","*",58,0],"ChatGPT-User":["partial","*",58,0],"OAI-SearchBot":["partial","*",58,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",58,0],"Claude-SearchBot":["partial","*",58,0],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["partial","*",58,0],"Perplexity-User":["partial","*",58,0],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["partial","*",58,0],"Bytespider":["partial","*",58,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["partial","named",14,0],"cohere-ai":["partial","*",58,0],"Diffbot":["partial","*",58,0],"ContextFetchBot":["partial","*",58,0]},"llms":{"status":"missing","http":404,"bytes":328664},"home":{"status":"ok","http":200,"url":"https://www.khaleejtimes.com/","html_tokens":215269,"markdown_tokens":64,"pruned_tokens":64}},
{"rank":3444,"domain":"eso.org","origin":"https://eso.org","robots":{"status":"ok","http":200,"url":"https://eso.org/robots.txt","bytes":2277,"groups":3},"bots":{"GPTBot":["partial","*",67,0],"ChatGPT-User":["partial","*",67,0],"OAI-SearchBot":["partial","*",67,0],"ClaudeBot":["partial","*",67,0],"Claude-User":["partial","*",67,0],"Claude-SearchBot":["partial","*",67,0],"anthropic-ai":["partial","*",67,0],"PerplexityBot":["partial","*",67,0],"Perplexity-User":["partial","*",67,0],"Google-Extended":["partial","*",67,0],"Applebot-Extended":["partial","*",67,0],"CCBot":["partial","*",67,0],"Bytespider":["partial","*",67,0],"Amazonbot":["partial","*",67,0],"meta-externalagent":["partial","*",67,0],"cohere-ai":["partial","*",67,0],"Diffbot":["partial","*",67,0],"ContextFetchBot":["partial","*",67,0]},"llms":{"status":"missing","http":404,"bytes":206},"home":{"status":"ok","http":200,"url":"https://www.eso.org/public/","html_tokens":47099,"markdown_tokens":1350,"pruned_tokens":213}},
{"rank":3445,"domain":"presseportal.de","origin":"https://presseportal.de","robots":{"status":"server_error","http":503,"url":"https://presseportal.de/robots.txt","bytes":30610,"groups":0,"error":"HTTP 503"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":3446,"domain":"ma.us","origin":"https://ma.us","robots":{"status":"unreachable","http":null,"url":"https://ma.us/robots.txt","bytes":0,"groups":0,"error":"cannot resolve ma.us"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":3447,"domain":"google.dk","origin":"https://google.dk","robots":{"status":"ok","http":200,"url":"https://www.google.dk/robots.txt","bytes":6624,"groups":4},"bots":{"GPTBot":["partial","*",169,64],"ChatGPT-User":["partial","*",169,64],"OAI-SearchBot":["partial","*",169,64],"ClaudeBot":["partial","*",169,64],"Claude-User":["partial","*",169,64],"Claude-SearchBot":["partial","*",169,64],"anthropic-ai":["partial","*",169,64],"PerplexityBot":["partial","*",169,64],"Perplexity-User":["partial","*",169,64],"Google-Extended":["partial","*",169,64],"Applebot-Extended":["partial","*",169,64],"CCBot":["partial","*",169,64],"Bytespider":["partial","*",169,64],"Amazonbot":["partial","*",169,64],"meta-externalagent":["partial","*",169,64],"cohere-ai":["partial","*",169,64],"Diffbot":["partial","*",169,64],"ContextFetchBot":["partial","*",169,64]},"llms":{"status":"missing","http":404,"bytes":1569},"home":{"status":"ok","http":200,"url":"https://www.google.com/","html_tokens":21280,"markdown_tokens":30,"pruned_tokens":30}},
{"rank":3448,"domain":"ansi.org","origin":"https://ansi.org","robots":{"status":"refused","http":403,"url":"https://ansi.org/robots.txt","bytes":5481,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":3449,"domain":"eleconomista.es","origin":"https://eleconomista.es","robots":{"status":"refused","http":403,"url":"https://www.eleconomista.es/robots.txt","bytes":385,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":3450,"domain":"nrc.nl","origin":"https://nrc.nl","robots":{"status":"ok","http":200,"url":"https://www.nrc.nl/robots.txt","bytes":3502,"groups":3},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["blocked","named",1,0],"OAI-SearchBot":["blocked","named",1,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["blocked","named",1,0],"Claude-SearchBot":["blocked","named",1,0],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["blocked","named",1,0],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",9,0]},"llms":{"status":"missing","http":404,"bytes":46245},"home":{"status":"ok","http":200,"url":"https://www.nrc.nl/","html_tokens":79810,"markdown_tokens":132,"pruned_tokens":132}},
{"rank":3451,"domain":"thrillist.com","origin":"https://thrillist.com","robots":{"status":"ok","http":200,"url":"https://thrillist.com/robots.txt","bytes":5106,"groups":103},"bots":{"GPTBot":["allowed","named",0,1],"ChatGPT-User":["partial","named",1,1],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["partial","named",1,1],"Claude-User":["partial","named",1,1],"Claude-SearchBot":["partial","named",1,1],"anthropic-ai":["partial","named",1,1],"PerplexityBot":["partial","named",1,1],"Perplexity-User":["partial","named",1,1],"Google-Extended":["partial","named",1,1],"Applebot-Extended":["partial","named",1,1],"CCBot":["partial","named",1,1],"Bytespider":["partial","named",1,1],"Amazonbot":["allowed","named",0,1],"meta-externalagent":["partial","named",1,1],"cohere-ai":["partial","named",1,1],"Diffbot":["partial","named",1,1],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":361628},"home":{"status":"ok","http":200,"url":"https://www.thrillist.com/","html_tokens":226395,"markdown_tokens":9,"pruned_tokens":9}},
{"rank":3452,"domain":"semver.org","origin":"https://semver.org","robots":{"status":"ok","http":200,"url":"https://semver.org/robots.txt","bytes":40,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":9379},"home":{"status":"ok","http":200,"url":"https://semver.org/","html_tokens":7710,"markdown_tokens":3884,"pruned_tokens":546}},
{"rank":3453,"domain":"osmfoundation.org","origin":"https://osmfoundation.org","robots":{"status":"ok","http":200,"url":"https://osmfoundation.org/robots.txt","bytes":909,"groups":4},"bots":{"GPTBot":["partial","*",8,3],"ChatGPT-User":["partial","*",8,3],"OAI-SearchBot":["partial","*",8,3],"ClaudeBot":["partial","*",8,3],"Claude-User":["partial","*",8,3],"Claude-SearchBot":["partial","*",8,3],"anthropic-ai":["partial","*",8,3],"PerplexityBot":["partial","*",8,3],"Perplexity-User":["partial","*",8,3],"Google-Extended":["partial","*",8,3],"Applebot-Extended":["partial","*",8,3],"CCBot":["partial","*",8,3],"Bytespider":["partial","*",8,3],"Amazonbot":["partial","*",8,3],"meta-externalagent":["partial","named",17,1],"cohere-ai":["partial","*",8,3],"Diffbot":["partial","*",8,3],"ContextFetchBot":["partial","*",8,3]},"llms":{"status":"missing","http":404,"bytes":30194},"home":{"status":"ok","http":200,"url":"https://osmfoundation.org/","html_tokens":11118,"markdown_tokens":147,"pruned_tokens":147}},
{"rank":3454,"domain":"villagevoice.com","origin":"https://villagevoice.com","robots":{"status":"ok","http":200,"url":"https://villagevoice.com/robots.txt","bytes":135,"groups":1},"bots":{"GPTBot":["partial","*",1,1],"ChatGPT-User":["partial","*",1,1],"OAI-SearchBot":["partial","*",1,1],"ClaudeBot":["partial","*",1,1],"Claude-User":["partial","*",1,1],"Claude-SearchBot":["partial","*",1,1],"anthropic-ai":["partial","*",1,1],"PerplexityBot":["partial","*",1,1],"Perplexity-User":["partial","*",1,1],"Google-Extended":["partial","*",1,1],"Applebot-Extended":["partial","*",1,1],"CCBot":["partial","*",1,1],"Bytespider":["partial","*",1,1],"Amazonbot":["partial","*",1,1],"meta-externalagent":["partial","*",1,1],"cohere-ai":["partial","*",1,1],"Diffbot":["partial","*",1,1],"ContextFetchBot":["partial","*",1,1]},"llms":{"status":"missing","http":404,"bytes":422761},"home":{"status":"ok","http":200,"url":"https://www.villagevoice.com/","html_tokens":260570,"markdown_tokens":43,"pruned_tokens":43}},
{"rank":3455,"domain":"wikitravel.org","origin":"https://wikitravel.org","robots":{"status":"refused","http":403,"url":"https://wikitravel.org/robots.txt","bytes":5535,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":3456,"domain":"compuserve.com","origin":"https://compuserve.com","robots":{"status":"html","http":200,"url":"https://www.compuserve.com/","bytes":24711,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":3457,"domain":"skynet.be","origin":"https://skynet.be","robots":{"status":"html","http":200,"url":"https://www.pickx.be/","bytes":5551,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":3458,"domain":"ionos.fr","origin":"https://ionos.fr","robots":{"status":"ok","http":200,"url":"https://www.ionos.fr/robots.txt","bytes":449,"groups":2},"bots":{"GPTBot":["partial","*",8,1],"ChatGPT-User":["partial","*",8,1],"OAI-SearchBot":["partial","*",8,1],"ClaudeBot":["partial","*",8,1],"Claude-User":["partial","*",8,1],"Claude-SearchBot":["partial","*",8,1],"anthropic-ai":["partial","*",8,1],"PerplexityBot":["partial","*",8,1],"Perplexity-User":["partial","*",8,1],"Google-Extended":["partial","*",8,1],"Applebot-Extended":["partial","*",8,1],"CCBot":["partial","*",8,1],"Bytespider":["partial","*",8,1],"Amazonbot":["partial","*",8,1],"meta-externalagent":["partial","*",8,1],"cohere-ai":["partial","*",8,1],"Diffbot":["partial","*",8,1],"ContextFetchBot":["partial","*",8,1]},"llms":{"status":"missing","http":404,"bytes":658672},"home":{"status":"ok","http":200,"url":"https://www.ionos.fr/","html_tokens":208947,"markdown_tokens":154,"pruned_tokens":129}},
{"rank":3459,"domain":"fairmont.com","origin":"https://fairmont.com","robots":{"status":"ok","http":200,"url":"https://fairmont.com/robots.txt","bytes":815,"groups":2},"bots":{"GPTBot":["partial","*",34,0],"ChatGPT-User":["partial","*",34,0],"OAI-SearchBot":["partial","*",34,0],"ClaudeBot":["partial","*",34,0],"Claude-User":["partial","*",34,0],"Claude-SearchBot":["partial","*",34,0],"anthropic-ai":["partial","*",34,0],"PerplexityBot":["partial","*",34,0],"Perplexity-User":["partial","*",34,0],"Google-Extended":["partial","*",34,0],"Applebot-Extended":["partial","*",34,0],"CCBot":["partial","*",34,0],"Bytespider":["partial","*",34,0],"Amazonbot":["partial","*",34,0],"meta-externalagent":["partial","*",34,0],"cohere-ai":["partial","*",34,0],"Diffbot":["partial","*",34,0],"ContextFetchBot":["partial","*",34,0]},"llms":{"status":"missing","http":404,"bytes":764173},"home":{"status":"ok","http":200,"url":"https://www.fairmont.com/en.html","html_tokens":284618,"markdown_tokens":566,"pruned_tokens":277}},
{"rank":3460,"domain":"cloudflareaccess.com","origin":"https://cloudflareaccess.com","robots":{"status":"missing","http":404,"url":"https://cloudflareaccess.com/robots.txt","bytes":0,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":0},"home":{"status":"http_error","http":404,"url":"https://cloudflareaccess.com/"}},
{"rank":3461,"domain":"wiwo.de","origin":"https://wiwo.de","robots":{"status":"ok","http":200,"url":"https://www.wiwo.de/robots.txt","bytes":1430,"groups":2},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["partial","*",25,0],"OAI-SearchBot":["partial","*",25,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",25,0],"Claude-SearchBot":["blocked","named",1,0],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["partial","*",25,0],"Perplexity-User":["partial","*",25,0],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",25,0]},"llms":{"status":"missing","http":404,"bytes":321},"home":{"status":"ok","http":200,"url":"https://www.wiwo.de/","html_tokens":538943,"markdown_tokens":74,"pruned_tokens":74}},
{"rank":3462,"domain":"gfycat.com","origin":"https://gfycat.com","robots":{"status":"unreachable","http":null,"url":"https://gfycat.com/robots.txt","bytes":0,"groups":0,"error":"cannot resolve gfycat.com"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":3463,"domain":"blog.gov.uk","origin":"https://blog.gov.uk","robots":{"status":"ok","http":200,"url":"https://www.blog.gov.uk/robots.txt","bytes":320,"groups":3},"bots":{"GPTBot":["partial","*",1,1],"ChatGPT-User":["partial","*",1,1],"OAI-SearchBot":["partial","*",1,1],"ClaudeBot":["partial","*",1,1],"Claude-User":["partial","*",1,1],"Claude-SearchBot":["partial","*",1,1],"anthropic-ai":["partial","*",1,1],"PerplexityBot":["partial","*",1,1],"Perplexity-User":["partial","*",1,1],"Google-Extended":["partial","*",1,1],"Applebot-Extended":["partial","*",1,1],"CCBot":["partial","*",1,1],"Bytespider":["partial","*",1,1],"Amazonbot":["partial","*",1,1],"meta-externalagent":["partial","*",1,1],"cohere-ai":["partial","*",1,1],"Diffbot":["partial","*",1,1],"ContextFetchBot":["partial","*",1,1]},"llms":{"status":"missing","http":404,"bytes":32779},"home":{"status":"ok","http":200,"url":"https://www.blog.gov.uk/","html_tokens":33069,"markdown_tokens":2916,"pruned_tokens":165}},
{"rank":3464,"domain":"ziprecruiter.com","origin":"https://ziprecruiter.com","robots":{"status":"ok","http":200,"url":"https://www.ziprecruiter.com/robots.txt","bytes":2689,"groups":4},"bots":{"GPTBot":["partial","named",1,28],"ChatGPT-User":["partial","named",26,1],"OAI-SearchBot":["partial","named",26,1],"ClaudeBot":["partial","named",26,1],"Claude-User":["partial","named",26,1],"Claude-SearchBot":["partial","named",26,1],"anthropic-ai":["partial","*",1,1],"PerplexityBot":["partial","named",26,1],"Perplexity-User":["partial","*",1,1],"Google-Extended":["partial","named",26,1],"Applebot-Extended":["partial","named",1,28],"CCBot":["partial","named",1,28],"Bytespider":["partial","*",1,1],"Amazonbot":["partial","*",1,1],"meta-externalagent":["partial","named",1,28],"cohere-ai":["partial","*",1,1],"Diffbot":["partial","*",1,1],"ContextFetchBot":["partial","*",1,1]},"llms":{"status":"present","http":200,"bytes":5956},"home":{"status":"disallowed"}},
{"rank":3465,"domain":"deccanherald.com","origin":"https://deccanherald.com","robots":{"status":"ok","http":200,"url":"https://www.deccanherald.com/robots.txt","bytes":3666,"groups":17},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["blocked","named",1,0],"OAI-SearchBot":["blocked","named",1,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",97,1],"Claude-SearchBot":["partial","*",97,1],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["blocked","named",1,0],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["partial","*",97,1],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["partial","*",97,1],"cohere-ai":["partial","*",97,1],"Diffbot":["partial","*",97,1],"ContextFetchBot":["partial","*",97,1]},"llms":{"status":"missing","http":404,"bytes":62037},"home":{"status":"ok","http":200,"url":"https://www.deccanherald.com/","html_tokens":1060160,"markdown_tokens":614,"pruned_tokens":125}},
{"rank":3466,"domain":"scitation.org","origin":"https://scitation.org","robots":{"status":"ok","http":200,"url":"https://pubs.aip.org/robots.txt","bytes":1537,"groups":2},"bots":{"GPTBot":["partial","*",36,3],"ChatGPT-User":["partial","*",36,3],"OAI-SearchBot":["partial","*",36,3],"ClaudeBot":["partial","*",36,3],"Claude-User":["partial","*",36,3],"Claude-SearchBot":["partial","*",36,3],"anthropic-ai":["partial","*",36,3],"PerplexityBot":["partial","*",36,3],"Perplexity-User":["partial","*",36,3],"Google-Extended":["partial","*",36,3],"Applebot-Extended":["partial","*",36,3],"CCBot":["partial","*",36,3],"Bytespider":["partial","*",36,3],"Amazonbot":["partial","*",36,3],"meta-externalagent":["partial","*",36,3],"cohere-ai":["partial","*",36,3],"Diffbot":["partial","*",36,3],"ContextFetchBot":["partial","*",36,3]},"llms":{"status":"http_error","http":403,"bytes":5523},"home":{"status":"http_error","http":403,"url":"https://pubs.aip.org/"}},
{"rank":3467,"domain":"creativebloq.com","origin":"https://creativebloq.com","robots":{"status":"ok","http":200,"url":"https://www.creativebloq.com/robots.txt","bytes":1709,"groups":5},"bots":{"GPTBot":["partial","*",20,0],"ChatGPT-User":["partial","*",20,0],"OAI-SearchBot":["partial","*",20,0],"ClaudeBot":["partial","*",20,0],"Claude-User":["partial","*",20,0],"Claude-SearchBot":["partial","*",20,0],"anthropic-ai":["partial","*",20,0],"PerplexityBot":["partial","*",20,0],"Perplexity-User":["partial","*",20,0],"Google-Extended":["partial","*",20,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["partial","*",20,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["partial","*",20,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",20,0]},"llms":{"status":"missing","http":404,"bytes":1247681},"home":{"status":"ok","http":200,"url":"https://www.creativebloq.com/","html_tokens":420168,"markdown_tokens":41,"pruned_tokens":41}},
{"rank":3468,"domain":"androidpolice.com","origin":"https://androidpolice.com","robots":{"status":"ok","http":200,"url":"https://www.androidpolice.com/robots.txt","bytes":2047,"groups":2},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["partial","*",22,0],"OAI-SearchBot":["partial","*",22,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",22,0],"Claude-SearchBot":["partial","*",22,0],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["partial","*",22,0],"Perplexity-User":["partial","*",22,0],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",22,0]},"llms":{"status":"missing","http":404,"bytes":109146},"home":{"status":"ok","http":200,"url":"https://www.androidpolice.com/","html_tokens":231340,"markdown_tokens":29,"pruned_tokens":29}},
{"rank":3469,"domain":"a8.net","origin":"https://a8.net","robots":{"status":"ok","http":200,"url":"https://www.a8.net/robots.txt","bytes":434,"groups":1},"bots":{"GPTBot":["partial","*",11,4],"ChatGPT-User":["partial","*",11,4],"OAI-SearchBot":["partial","*",11,4],"ClaudeBot":["partial","*",11,4],"Claude-User":["partial","*",11,4],"Claude-SearchBot":["partial","*",11,4],"anthropic-ai":["partial","*",11,4],"PerplexityBot":["partial","*",11,4],"Perplexity-User":["partial","*",11,4],"Google-Extended":["partial","*",11,4],"Applebot-Extended":["partial","*",11,4],"CCBot":["partial","*",11,4],"Bytespider":["partial","*",11,4],"Amazonbot":["partial","*",11,4],"meta-externalagent":["partial","*",11,4],"cohere-ai":["partial","*",11,4],"Diffbot":["partial","*",11,4],"ContextFetchBot":["partial","*",11,4]},"llms":{"status":"http_error","http":403,"bytes":199},"home":{"status":"ok","http":200,"url":"https://www.a8.net/","html_tokens":10386,"markdown_tokens":489,"pruned_tokens":276}},
{"rank":3470,"domain":"ice.gov","origin":"https://ice.gov","robots":{"status":"refused","http":403,"url":"https://ice.gov/robots.txt","bytes":371,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":3471,"domain":"mail-archive.com","origin":"https://mail-archive.com","robots":{"status":"ok","http":200,"url":"https://mail-archive.com/robots.txt","bytes":714,"groups":1},"bots":{"GPTBot":["partial","*",6,0],"ChatGPT-User":["partial","*",6,0],"OAI-SearchBot":["partial","*",6,0],"ClaudeBot":["partial","*",6,0],"Claude-User":["partial","*",6,0],"Claude-SearchBot":["partial","*",6,0],"anthropic-ai":["partial","*",6,0],"PerplexityBot":["partial","*",6,0],"Perplexity-User":["partial","*",6,0],"Google-Extended":["partial","*",6,0],"Applebot-Extended":["partial","*",6,0],"CCBot":["partial","*",6,0],"Bytespider":["partial","*",6,0],"Amazonbot":["partial","*",6,0],"meta-externalagent":["partial","*",6,0],"cohere-ai":["partial","*",6,0],"Diffbot":["partial","*",6,0],"ContextFetchBot":["partial","*",6,0]},"llms":{"status":"missing","http":404,"bytes":1153},"home":{"status":"ok","http":200,"url":"https://mail-archive.com/","html_tokens":1933,"markdown_tokens":74,"pruned_tokens":50}},
{"rank":3472,"domain":"pinterest.es","origin":"https://pinterest.es","robots":{"status":"ok","http":200,"url":"https://es.pinterest.com/robots.txt","bytes":36884,"groups":8},"bots":{"GPTBot":["blocked","*",1,0],"ChatGPT-User":["blocked","*",1,0],"OAI-SearchBot":["blocked","*",1,0],"ClaudeBot":["blocked","*",1,0],"Claude-User":["blocked","*",1,0],"Claude-SearchBot":["blocked","*",1,0],"anthropic-ai":["blocked","*",1,0],"PerplexityBot":["blocked","*",1,0],"Perplexity-User":["blocked","*",1,0],"Google-Extended":["blocked","*",1,0],"Applebot-Extended":["blocked","*",1,0],"CCBot":["blocked","*",1,0],"Bytespider":["blocked","*",1,0],"Amazonbot":["blocked","*",1,0],"meta-externalagent":["blocked","*",1,0],"cohere-ai":["blocked","*",1,0],"Diffbot":["blocked","*",1,0],"ContextFetchBot":["blocked","*",1,0]},"llms":{"status":"disallowed"},"home":{"status":"disallowed"}},
{"rank":3473,"domain":"google.no","origin":"https://google.no","robots":{"status":"ok","http":200,"url":"https://www.google.no/robots.txt","bytes":6624,"groups":4},"bots":{"GPTBot":["partial","*",169,64],"ChatGPT-User":["partial","*",169,64],"OAI-SearchBot":["partial","*",169,64],"ClaudeBot":["partial","*",169,64],"Claude-User":["partial","*",169,64],"Claude-SearchBot":["partial","*",169,64],"anthropic-ai":["partial","*",169,64],"PerplexityBot":["partial","*",169,64],"Perplexity-User":["partial","*",169,64],"Google-Extended":["partial","*",169,64],"Applebot-Extended":["partial","*",169,64],"CCBot":["partial","*",169,64],"Bytespider":["partial","*",169,64],"Amazonbot":["partial","*",169,64],"meta-externalagent":["partial","*",169,64],"cohere-ai":["partial","*",169,64],"Diffbot":["partial","*",169,64],"ContextFetchBot":["partial","*",169,64]},"llms":{"status":"missing","http":404,"bytes":1569},"home":{"status":"ok","http":200,"url":"https://www.google.com/","html_tokens":21443,"markdown_tokens":30,"pruned_tokens":30}},
{"rank":3474,"domain":"ynetnews.com","origin":"https://ynetnews.com","robots":{"status":"ok","http":200,"url":"https://www.ynetnews.com/robots.txt","bytes":1471,"groups":1},"bots":{"GPTBot":["partial","*",33,0],"ChatGPT-User":["partial","*",33,0],"OAI-SearchBot":["partial","*",33,0],"ClaudeBot":["partial","*",33,0],"Claude-User":["partial","*",33,0],"Claude-SearchBot":["partial","*",33,0],"anthropic-ai":["partial","*",33,0],"PerplexityBot":["partial","*",33,0],"Perplexity-User":["partial","*",33,0],"Google-Extended":["partial","*",33,0],"Applebot-Extended":["partial","*",33,0],"CCBot":["partial","*",33,0],"Bytespider":["partial","*",33,0],"Amazonbot":["partial","*",33,0],"meta-externalagent":["partial","*",33,0],"cohere-ai":["partial","*",33,0],"Diffbot":["partial","*",33,0],"ContextFetchBot":["partial","*",33,0]},"llms":{"status":"missing","http":404,"bytes":56973},"home":{"status":"ok","http":200,"url":"https://www.ynetnews.com/category/3083","html_tokens":124751,"markdown_tokens":27,"pruned_tokens":27}},
{"rank":3475,"domain":"ufrj.br","origin":"https://ufrj.br","robots":{"status":"ok","http":200,"url":"https://ufrj.br/robots.txt","bytes":108,"groups":1},"bots":{"GPTBot":["partial","*",1,1],"ChatGPT-User":["partial","*",1,1],"OAI-SearchBot":["partial","*",1,1],"ClaudeBot":["partial","*",1,1],"Claude-User":["partial","*",1,1],"Claude-SearchBot":["partial","*",1,1],"anthropic-ai":["partial","*",1,1],"PerplexityBot":["partial","*",1,1],"Perplexity-User":["partial","*",1,1],"Google-Extended":["partial","*",1,1],"Applebot-Extended":["partial","*",1,1],"CCBot":["partial","*",1,1],"Bytespider":["partial","*",1,1],"Amazonbot":["partial","*",1,1],"meta-externalagent":["partial","*",1,1],"cohere-ai":["partial","*",1,1],"Diffbot":["partial","*",1,1],"ContextFetchBot":["partial","*",1,1]},"llms":{"status":"missing","http":404,"bytes":50037},"home":{"status":"ok","http":200,"url":"https://ufrj.br/en/","html_tokens":16994,"markdown_tokens":675,"pruned_tokens":304}},
{"rank":3476,"domain":"postimages.org","origin":"https://postimages.org","robots":{"status":"ok","http":200,"url":"https://postimages.org/robots.txt","bytes":0,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":28645},"home":{"status":"ok","http":200,"url":"https://postimages.org/","html_tokens":10448,"markdown_tokens":198,"pruned_tokens":25}},
{"rank":3477,"domain":"flipkart.com","origin":"https://flipkart.com","robots":{"status":"ok","http":200,"url":"https://www.flipkart.com/robots.txt","bytes":3111,"groups":15},"bots":{"GPTBot":["partial","*",65,0],"ChatGPT-User":["partial","*",65,0],"OAI-SearchBot":["partial","*",65,0],"ClaudeBot":["partial","*",65,0],"Claude-User":["partial","*",65,0],"Claude-SearchBot":["partial","*",65,0],"anthropic-ai":["partial","*",65,0],"PerplexityBot":["partial","*",65,0],"Perplexity-User":["partial","*",65,0],"Google-Extended":["partial","*",65,0],"Applebot-Extended":["partial","*",65,0],"CCBot":["partial","*",65,0],"Bytespider":["partial","*",65,0],"Amazonbot":["partial","*",65,0],"meta-externalagent":["partial","*",65,0],"cohere-ai":["partial","*",65,0],"Diffbot":["partial","*",65,0],"ContextFetchBot":["partial","*",65,0]},"llms":{"status":"missing","http":404,"bytes":1780},"home":{"status":"ok","http":200,"url":"https://www.flipkart.com/","html_tokens":330557,"markdown_tokens":4026,"pruned_tokens":433}},
{"rank":3478,"domain":"googlesyndication.com","origin":"https://googlesyndication.com","robots":{"status":"missing","http":404,"url":"https://googlesyndication.com/robots.txt","bytes":1571,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":1569},"home":{"status":"ok","http":200,"url":"https://www.google.com/","html_tokens":21280,"markdown_tokens":30,"pruned_tokens":30}},
{"rank":3479,"domain":"bestwestern.com","origin":"https://bestwestern.com","robots":{"status":"refused","http":403,"url":"https://www.bestwestern.com/robots.txt","bytes":1136,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":3480,"domain":"fotolia.com","origin":"https://fotolia.com","robots":{"status":"ok","http":200,"url":"https://www.fotolia.com/robots.txt","bytes":899,"groups":17},"bots":{"GPTBot":["partial","*",12,0],"ChatGPT-User":["partial","*",12,0],"OAI-SearchBot":["partial","*",12,0],"ClaudeBot":["partial","*",12,0],"Claude-User":["partial","*",12,0],"Claude-SearchBot":["partial","*",12,0],"anthropic-ai":["partial","*",12,0],"PerplexityBot":["partial","*",12,0],"Perplexity-User":["partial","*",12,0],"Google-Extended":["partial","*",12,0],"Applebot-Extended":["partial","*",12,0],"CCBot":["blocked","named",1,0],"Bytespider":["partial","*",12,0],"Amazonbot":["partial","*",12,0],"meta-externalagent":["partial","*",12,0],"cohere-ai":["partial","*",12,0],"Diffbot":["partial","*",12,0],"ContextFetchBot":["partial","*",12,0]},"llms":{"status":"html","http":200,"bytes":438189},"home":{"status":"ok","http":200,"url":"https://stock.adobe.com/?as_campaign=ftmigration2&as_channel=dpcft&as_campclass=brand&as_source=ft_web&as_camptype=acquisition&as_audience=users&as_content=closure_home-page","html_tokens":109478,"markdown_tokens":309,"pruned_tokens":162}},
{"rank":3481,"domain":"thesaurus.com","origin":"https://thesaurus.com","robots":{"status":"ok","http":200,"url":"https://www.thesaurus.com/robots.txt","bytes":565,"groups":7},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["partial","*",9,3],"OAI-SearchBot":["partial","*",9,3],"ClaudeBot":["partial","*",9,3],"Claude-User":["partial","*",9,3],"Claude-SearchBot":["partial","*",9,3],"anthropic-ai":["partial","*",9,3],"PerplexityBot":["partial","*",9,3],"Perplexity-User":["partial","*",9,3],"Google-Extended":["partial","*",9,3],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["partial","*",9,3],"Amazonbot":["partial","*",9,3],"meta-externalagent":["partial","*",9,3],"cohere-ai":["partial","*",9,3],"Diffbot":["partial","*",9,3],"ContextFetchBot":["partial","*",9,3]},"llms":{"status":"missing","http":404,"bytes":51804},"home":{"status":"ok","http":200,"url":"https://www.thesaurus.com/","html_tokens":15753,"markdown_tokens":47,"pruned_tokens":47}},
{"rank":3482,"domain":"writeablog.net","origin":"https://writeablog.net","robots":{"status":"missing","http":404,"url":"https://writeablog.net/robots.txt","bytes":19,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"other_text","http":200,"bytes":15},"home":{"status":"ok","http":200,"url":"https://writeablog.net/","html_tokens":3864,"markdown_tokens":240,"pruned_tokens":165}},
{"rank":3483,"domain":"corporatefinanceinstitute.com","origin":"https://corporatefinanceinstitute.com","robots":{"status":"ok","http":200,"url":"https://corporatefinanceinstitute.com/robots.txt","bytes":465,"groups":1},"bots":{"GPTBot":["partial","*",15,0],"ChatGPT-User":["partial","*",15,0],"OAI-SearchBot":["partial","*",15,0],"ClaudeBot":["partial","*",15,0],"Claude-User":["partial","*",15,0],"Claude-SearchBot":["partial","*",15,0],"anthropic-ai":["partial","*",15,0],"PerplexityBot":["partial","*",15,0],"Perplexity-User":["partial","*",15,0],"Google-Extended":["partial","*",15,0],"Applebot-Extended":["partial","*",15,0],"CCBot":["partial","*",15,0],"Bytespider":["partial","*",15,0],"Amazonbot":["partial","*",15,0],"meta-externalagent":["partial","*",15,0],"cohere-ai":["partial","*",15,0],"Diffbot":["partial","*",15,0],"ContextFetchBot":["partial","*",15,0]},"llms":{"status":"missing","http":404,"bytes":108019},"home":{"status":"ok","http":200,"url":"https://corporatefinanceinstitute.com/","html_tokens":142548,"markdown_tokens":5127,"pruned_tokens":146}},
{"rank":3484,"domain":"tripadvisor.fr","origin":"https://tripadvisor.fr","robots":{"status":"ok","http":200,"url":"https://www.tripadvisor.fr/robots.txt","bytes":22865,"groups":9},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["partial","named",623,8],"OAI-SearchBot":["partial","named",623,8],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",621,8],"Claude-SearchBot":["partial","*",621,8],"anthropic-ai":["partial","*",621,8],"PerplexityBot":["partial","named",625,8],"Perplexity-User":["partial","*",621,8],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["partial","*",621,8],"ContextFetchBot":["partial","*",621,8]},"llms":{"status":"http_error","http":403,"bytes":774},"home":{"status":"http_error","http":403,"url":"https://www.tripadvisor.fr/"}},
{"rank":3485,"domain":"amazon.cn","origin":"https://amazon.cn","robots":{"status":"html","http":200,"url":"https://www.amazon.cn/robots.txt","bytes":30777,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":3486,"domain":"marketplace.org","origin":"https://marketplace.org","robots":{"status":"refused","http":403,"url":"https://www.marketplace.org/robots.txt","bytes":919,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":3487,"domain":"huji.ac.il","origin":"https://huji.ac.il","robots":{"status":"html","http":200,"url":"https://huji.ac.il/robots.txt","bytes":581,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":3488,"domain":"cbd.int","origin":"https://cbd.int","robots":{"status":"missing","http":404,"url":"https://www.cbd.int/robots.txt","bytes":5091,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":5091},"home":{"status":"ok","http":200,"url":"https://www.cbd.int","html_tokens":39576,"markdown_tokens":544,"pruned_tokens":153}},
{"rank":3489,"domain":"bea.gov","origin":"https://bea.gov","robots":{"status":"ok","http":200,"url":"https://bea.gov/robots.txt","bytes":2027,"groups":1},"bots":{"GPTBot":["partial","*",34,18],"ChatGPT-User":["partial","*",34,18],"OAI-SearchBot":["partial","*",34,18],"ClaudeBot":["partial","*",34,18],"Claude-User":["partial","*",34,18],"Claude-SearchBot":["partial","*",34,18],"anthropic-ai":["partial","*",34,18],"PerplexityBot":["partial","*",34,18],"Perplexity-User":["partial","*",34,18],"Google-Extended":["partial","*",34,18],"Applebot-Extended":["partial","*",34,18],"CCBot":["partial","*",34,18],"Bytespider":["partial","*",34,18],"Amazonbot":["partial","*",34,18],"meta-externalagent":["partial","*",34,18],"cohere-ai":["partial","*",34,18],"Diffbot":["partial","*",34,18],"ContextFetchBot":["partial","*",34,18]},"llms":{"status":"missing","http":404,"bytes":184},"home":{"status":"ok","http":200,"url":"https://bea.gov/","html_tokens":14748,"markdown_tokens":1353,"pruned_tokens":123}},
{"rank":3490,"domain":"psychiatryonline.org","origin":"https://psychiatryonline.org","robots":{"status":"refused","http":403,"url":"https://psychiatryonline.org/robots.txt","bytes":5579,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":3491,"domain":"palletsprojects.com","origin":"https://palletsprojects.com","robots":{"status":"ok","http":200,"url":"https://palletsprojects.com/robots.txt","bytes":1451,"groups":14},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":207},"home":{"status":"ok","http":200,"url":"https://palletsprojects.com/","html_tokens":1020,"markdown_tokens":94,"pruned_tokens":94}},
{"rank":3492,"domain":"baylor.edu","origin":"https://baylor.edu","robots":{"status":"ok","http":200,"url":"https://www.baylor.edu/robots.txt","bytes":441,"groups":2},"bots":{"GPTBot":["partial","*",1,0],"ChatGPT-User":["partial","*",1,0],"OAI-SearchBot":["partial","*",1,0],"ClaudeBot":["partial","*",1,0],"Claude-User":["partial","*",1,0],"Claude-SearchBot":["partial","*",1,0],"anthropic-ai":["partial","*",1,0],"PerplexityBot":["partial","*",1,0],"Perplexity-User":["partial","*",1,0],"Google-Extended":["partial","*",1,0],"Applebot-Extended":["partial","*",1,0],"CCBot":["partial","*",1,0],"Bytespider":["partial","*",1,0],"Amazonbot":["partial","*",1,0],"meta-externalagent":["partial","*",1,0],"cohere-ai":["partial","*",1,0],"Diffbot":["partial","*",1,0],"ContextFetchBot":["partial","*",1,0]},"llms":{"status":"missing","http":404,"bytes":2603},"home":{"status":"ok","http":200,"url":"https://www.baylor.edu/","html_tokens":111705,"markdown_tokens":327,"pruned_tokens":327}},
{"rank":3493,"domain":"hathitrust.org","origin":"https://hathitrust.org","robots":{"status":"refused","http":403,"url":"https://www.hathitrust.org/robots.txt","bytes":5577,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":3494,"domain":"plannedparenthood.org","origin":"https://plannedparenthood.org","robots":{"status":"ok","http":200,"url":"https://www.plannedparenthood.org/robots.txt","bytes":229,"groups":1},"bots":{"GPTBot":["partial","*",8,0],"ChatGPT-User":["partial","*",8,0],"OAI-SearchBot":["partial","*",8,0],"ClaudeBot":["partial","*",8,0],"Claude-User":["partial","*",8,0],"Claude-SearchBot":["partial","*",8,0],"anthropic-ai":["partial","*",8,0],"PerplexityBot":["partial","*",8,0],"Perplexity-User":["partial","*",8,0],"Google-Extended":["partial","*",8,0],"Applebot-Extended":["partial","*",8,0],"CCBot":["partial","*",8,0],"Bytespider":["partial","*",8,0],"Amazonbot":["partial","*",8,0],"meta-externalagent":["partial","*",8,0],"cohere-ai":["partial","*",8,0],"Diffbot":["partial","*",8,0],"ContextFetchBot":["partial","*",8,0]},"llms":{"status":"http_error","http":403,"bytes":118},"home":{"status":"ok","http":200,"url":"https://www.plannedparenthood.org/","html_tokens":32009,"markdown_tokens":277,"pruned_tokens":66}},
{"rank":3495,"domain":"vermont.gov","origin":"https://vermont.gov","robots":{"status":"ok","http":200,"url":"https://www.vermont.gov/robots.txt","bytes":5268,"groups":22},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["partial","named",31,3],"OAI-SearchBot":["partial","named",31,3],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","named",31,3],"Claude-SearchBot":["partial","*",42,18],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["partial","*",42,18],"Google-Extended":["partial","*",42,18],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["partial","*",42,18],"meta-externalagent":["partial","*",42,18],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",42,18]},"llms":{"status":"missing","http":404,"bytes":189},"home":{"status":"ok","http":200,"url":"https://www.vermont.gov/","html_tokens":12637,"markdown_tokens":281,"pruned_tokens":39}},
{"rank":3496,"domain":"epicurious.com","origin":"https://epicurious.com","robots":{"status":"ok","http":200,"url":"https://www.epicurious.com/robots.txt","bytes":2216,"groups":3},"bots":{"GPTBot":["partial","*",16,14],"ChatGPT-User":["partial","*",16,14],"OAI-SearchBot":["partial","*",16,14],"ClaudeBot":["blocked","named",1,0],"Claude-User":["blocked","named",1,0],"Claude-SearchBot":["blocked","named",1,0],"anthropic-ai":["partial","*",16,14],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["blocked","named",1,0],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",16,14]},"llms":{"status":"missing","http":404,"bytes":685908},"home":{"status":"ok","http":200,"url":"https://www.epicurious.com/","html_tokens":1084743,"markdown_tokens":38,"pruned_tokens":38}},
{"rank":3497,"domain":"yarnpkg.com","origin":"https://yarnpkg.com","robots":{"status":"missing","http":404,"url":"https://yarnpkg.com/robots.txt","bytes":20585,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":20585},"home":{"status":"ok","http":200,"url":"https://yarnpkg.com/","html_tokens":6205,"markdown_tokens":252,"pruned_tokens":126}},
{"rank":3498,"domain":"pib.gov.in","origin":"https://pib.gov.in","robots":{"status":"refused","http":403,"url":"https://www.pib.gov.in/robots.txt","bytes":386,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":3499,"domain":"homeaffairs.gov.au","origin":"https://homeaffairs.gov.au","robots":{"status":"refused","http":403,"url":"https://homeaffairs.gov.au/robots.txt","bytes":386,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":3500,"domain":"rlp.de","origin":"https://rlp.de","robots":{"status":"ok","http":200,"url":"https://rlp.de/robots.txt","bytes":156,"groups":1},"bots":{"GPTBot":["partial","*",4,2],"ChatGPT-User":["partial","*",4,2],"OAI-SearchBot":["partial","*",4,2],"ClaudeBot":["partial","*",4,2],"Claude-User":["partial","*",4,2],"Claude-SearchBot":["partial","*",4,2],"anthropic-ai":["partial","*",4,2],"PerplexityBot":["partial","*",4,2],"Perplexity-User":["partial","*",4,2],"Google-Extended":["partial","*",4,2],"Applebot-Extended":["partial","*",4,2],"CCBot":["partial","*",4,2],"Bytespider":["partial","*",4,2],"Amazonbot":["partial","*",4,2],"meta-externalagent":["partial","*",4,2],"cohere-ai":["partial","*",4,2],"Diffbot":["partial","*",4,2],"ContextFetchBot":["partial","*",4,2]},"llms":{"status":"html","http":200,"bytes":102859},"home":{"status":"ok","http":200,"url":"https://www.rlp.de/","html_tokens":25662,"markdown_tokens":289,"pruned_tokens":10}},
{"rank":3501,"domain":"t.cn","origin":"https://t.cn","robots":{"status":"missing","http":404,"url":"https://t.cn/robots.txt","bytes":36,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":36},"home":{"status":"disallowed","error":"robots.txt of weibo.com disallows /"}},
{"rank":3502,"domain":"dtic.mil","origin":"https://dtic.mil","robots":{"status":"unreachable","http":null,"url":"https://dtic.mil/robots.txt","bytes":0,"groups":0,"error":"[SSL: CERTIFICATE_VERIFY_FAILED] certificate verify failed: unable to get local issuer certificate (_ssl.c:1028)"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":3503,"domain":"google.co.kr","origin":"https://google.co.kr","robots":{"status":"ok","http":200,"url":"https://www.google.co.kr/robots.txt","bytes":6624,"groups":4},"bots":{"GPTBot":["partial","*",169,64],"ChatGPT-User":["partial","*",169,64],"OAI-SearchBot":["partial","*",169,64],"ClaudeBot":["partial","*",169,64],"Claude-User":["partial","*",169,64],"Claude-SearchBot":["partial","*",169,64],"anthropic-ai":["partial","*",169,64],"PerplexityBot":["partial","*",169,64],"Perplexity-User":["partial","*",169,64],"Google-Extended":["partial","*",169,64],"Applebot-Extended":["partial","*",169,64],"CCBot":["partial","*",169,64],"Bytespider":["partial","*",169,64],"Amazonbot":["partial","*",169,64],"meta-externalagent":["partial","*",169,64],"cohere-ai":["partial","*",169,64],"Diffbot":["partial","*",169,64],"ContextFetchBot":["partial","*",169,64]},"llms":{"status":"missing","http":404,"bytes":1569},"home":{"status":"ok","http":200,"url":"https://www.google.com/","html_tokens":21364,"markdown_tokens":30,"pruned_tokens":30}},
{"rank":3504,"domain":"newyorkfed.org","origin":"https://newyorkfed.org","robots":{"status":"ok","http":200,"url":"https://www.newyorkfed.org/robots.txt","bytes":211,"groups":1},"bots":{"GPTBot":["partial","*",3,0],"ChatGPT-User":["partial","*",3,0],"OAI-SearchBot":["partial","*",3,0],"ClaudeBot":["partial","*",3,0],"Claude-User":["partial","*",3,0],"Claude-SearchBot":["partial","*",3,0],"anthropic-ai":["partial","*",3,0],"PerplexityBot":["partial","*",3,0],"Perplexity-User":["partial","*",3,0],"Google-Extended":["partial","*",3,0],"Applebot-Extended":["partial","*",3,0],"CCBot":["partial","*",3,0],"Bytespider":["partial","*",3,0],"Amazonbot":["partial","*",3,0],"meta-externalagent":["partial","*",3,0],"cohere-ai":["partial","*",3,0],"Diffbot":["partial","*",3,0],"ContextFetchBot":["partial","*",3,0]},"llms":{"status":"html","http":200,"bytes":97975},"home":{"status":"ok","http":200,"url":"https://www.newyorkfed.org/","html_tokens":27554,"markdown_tokens":204,"pruned_tokens":132}},
{"rank":3505,"domain":"eib.org","origin":"https://eib.org","robots":{"status":"ok","http":200,"url":"https://www.eib.org/robots.txt","bytes":1139,"groups":1},"bots":{"GPTBot":["partial","*",34,3],"ChatGPT-User":["partial","*",34,3],"OAI-SearchBot":["partial","*",34,3],"ClaudeBot":["partial","*",34,3],"Claude-User":["partial","*",34,3],"Claude-SearchBot":["partial","*",34,3],"anthropic-ai":["partial","*",34,3],"PerplexityBot":["partial","*",34,3],"Perplexity-User":["partial","*",34,3],"Google-Extended":["partial","*",34,3],"Applebot-Extended":["partial","*",34,3],"CCBot":["partial","*",34,3],"Bytespider":["partial","*",34,3],"Amazonbot":["partial","*",34,3],"meta-externalagent":["partial","*",34,3],"cohere-ai":["partial","*",34,3],"Diffbot":["partial","*",34,3],"ContextFetchBot":["partial","*",34,3]},"llms":{"status":"missing","http":404,"bytes":82146},"home":{"status":"ok","http":200,"url":"https://www.eib.org/en/index","html_tokens":26153,"markdown_tokens":17,"pruned_tokens":17}},
{"rank":3506,"domain":"jenkins.io","origin":"https://jenkins.io","robots":{"status":"missing","http":404,"url":"https://www.jenkins.io/robots.txt","bytes":6163,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":6163},"home":{"status":"ok","http":200,"url":"https://www.jenkins.io/","html_tokens":14840,"markdown_tokens":266,"pruned_tokens":174}},
{"rank":3507,"domain":"state.co.us","origin":"https://state.co.us","robots":{"status":"unreachable","http":null,"url":"https://state.co.us/robots.txt","bytes":0,"groups":0,"error":"All connection attempts failed"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":3508,"domain":"megaupload.com","origin":"https://megaupload.com","robots":{"status":"unreachable","http":null,"url":"https://megaupload.com/robots.txt","bytes":0,"groups":0,"error":"cannot resolve megaupload.com"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":3509,"domain":"opensecrets.org","origin":"https://opensecrets.org","robots":{"status":"ok","http":200,"url":"https://opensecrets.org/robots.txt","bytes":2676,"groups":32},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["allowed","*",0,1],"OAI-SearchBot":["allowed","*",0,1],"ClaudeBot":["blocked","named",1,0],"Claude-User":["allowed","*",0,1],"Claude-SearchBot":["allowed","*",0,1],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["allowed","*",0,1],"Perplexity-User":["allowed","*",0,1],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["allowed","*",0,1]},"llms":{"status":"missing","http":404,"bytes":175249},"home":{"status":"http_error","http":403,"url":"https://opensecrets.org/"}},
{"rank":3510,"domain":"ameba.jp","origin":"https://ameba.jp","robots":{"status":"ok","http":200,"url":"https://www.ameba.jp/robots.txt","bytes":337,"groups":1},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":90368},"home":{"status":"ok","http":200,"url":"https://www.ameba.jp/","html_tokens":47968,"markdown_tokens":49,"pruned_tokens":49}},
{"rank":3511,"domain":"vesti.ru","origin":"https://vesti.ru","robots":{"status":"ok","http":200,"url":"https://www.vesti.ru/robots.txt","bytes":5412,"groups":3},"bots":{"GPTBot":["partial","*",26,55],"ChatGPT-User":["partial","*",26,55],"OAI-SearchBot":["partial","*",26,55],"ClaudeBot":["partial","*",26,55],"Claude-User":["partial","*",26,55],"Claude-SearchBot":["partial","*",26,55],"anthropic-ai":["partial","*",26,55],"PerplexityBot":["partial","*",26,55],"Perplexity-User":["partial","*",26,55],"Google-Extended":["partial","*",26,55],"Applebot-Extended":["partial","*",26,55],"CCBot":["partial","*",26,55],"Bytespider":["partial","*",26,55],"Amazonbot":["partial","*",26,55],"meta-externalagent":["partial","*",26,55],"cohere-ai":["partial","*",26,55],"Diffbot":["partial","*",26,55],"ContextFetchBot":["partial","*",26,55]},"llms":{"status":"missing","http":404,"bytes":180028},"home":{"status":"ok","http":200,"url":"https://www.vesti.ru/","html_tokens":144351,"markdown_tokens":49,"pruned_tokens":49}},
{"rank":3512,"domain":"edelman.com","origin":"https://edelman.com","robots":{"status":"refused","http":403,"url":"https://www.edelman.com/robots.txt","bytes":383,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":3513,"domain":"gigaom.com","origin":"https://gigaom.com","robots":{"status":"ok","http":200,"url":"https://gigaom.com/robots.txt","bytes":54,"groups":1},"bots":{"GPTBot":["partial","*",1,0],"ChatGPT-User":["partial","*",1,0],"OAI-SearchBot":["partial","*",1,0],"ClaudeBot":["partial","*",1,0],"Claude-User":["partial","*",1,0],"Claude-SearchBot":["partial","*",1,0],"anthropic-ai":["partial","*",1,0],"PerplexityBot":["partial","*",1,0],"Perplexity-User":["partial","*",1,0],"Google-Extended":["partial","*",1,0],"Applebot-Extended":["partial","*",1,0],"CCBot":["partial","*",1,0],"Bytespider":["partial","*",1,0],"Amazonbot":["partial","*",1,0],"meta-externalagent":["partial","*",1,0],"cohere-ai":["partial","*",1,0],"Diffbot":["partial","*",1,0],"ContextFetchBot":["partial","*",1,0]},"llms":{"status":"http_error","http":403,"bytes":5541},"home":{"status":"http_error","http":403,"url":"https://gigaom.com/"}},
{"rank":3514,"domain":"calm.com","origin":"https://calm.com","robots":{"status":"ok","http":200,"url":"https://www.calm.com/robots.txt","bytes":4142,"groups":1},"bots":{"GPTBot":["partial","*",102,0],"ChatGPT-User":["partial","*",102,0],"OAI-SearchBot":["partial","*",102,0],"ClaudeBot":["partial","*",102,0],"Claude-User":["partial","*",102,0],"Claude-SearchBot":["partial","*",102,0],"anthropic-ai":["partial","*",102,0],"PerplexityBot":["partial","*",102,0],"Perplexity-User":["partial","*",102,0],"Google-Extended":["partial","*",102,0],"Applebot-Extended":["partial","*",102,0],"CCBot":["partial","*",102,0],"Bytespider":["partial","*",102,0],"Amazonbot":["partial","*",102,0],"meta-externalagent":["partial","*",102,0],"cohere-ai":["partial","*",102,0],"Diffbot":["partial","*",102,0],"ContextFetchBot":["partial","*",102,0]},"llms":{"status":"missing","http":404,"bytes":404988},"home":{"status":"ok","http":200,"url":"https://www.calm.com","html_tokens":141308,"markdown_tokens":2358,"pruned_tokens":165}},
{"rank":3515,"domain":"stores.jp","origin":"https://stores.jp","robots":{"status":"ok","http":200,"url":"https://stores.fun/robots.txt","bytes":498,"groups":7},"bots":{"GPTBot":["allowed","named",0,1],"ChatGPT-User":["allowed","named",0,1],"OAI-SearchBot":["partial","*",10,1],"ClaudeBot":["allowed","named",0,1],"Claude-User":["partial","*",10,1],"Claude-SearchBot":["partial","*",10,1],"anthropic-ai":["allowed","named",0,1],"PerplexityBot":["allowed","named",0,1],"Perplexity-User":["partial","*",10,1],"Google-Extended":["allowed","named",0,1],"Applebot-Extended":["partial","*",10,1],"CCBot":["partial","*",10,1],"Bytespider":["partial","*",10,1],"Amazonbot":["partial","*",10,1],"meta-externalagent":["partial","*",10,1],"cohere-ai":["partial","*",10,1],"Diffbot":["partial","*",10,1],"ContextFetchBot":["partial","*",10,1]},"llms":{"status":"missing","http":404,"bytes":3170},"home":{"status":"ok","http":200,"url":"https://stores.fun/","html_tokens":45317,"markdown_tokens":350,"pruned_tokens":96}},
{"rank":3516,"domain":"google.ro","origin":"https://google.ro","robots":{"status":"ok","http":200,"url":"https://www.google.ro/robots.txt","bytes":6624,"groups":4},"bots":{"GPTBot":["partial","*",169,64],"ChatGPT-User":["partial","*",169,64],"OAI-SearchBot":["partial","*",169,64],"ClaudeBot":["partial","*",169,64],"Claude-User":["partial","*",169,64],"Claude-SearchBot":["partial","*",169,64],"anthropic-ai":["partial","*",169,64],"PerplexityBot":["partial","*",169,64],"Perplexity-User":["partial","*",169,64],"Google-Extended":["partial","*",169,64],"Applebot-Extended":["partial","*",169,64],"CCBot":["partial","*",169,64],"Bytespider":["partial","*",169,64],"Amazonbot":["partial","*",169,64],"meta-externalagent":["partial","*",169,64],"cohere-ai":["partial","*",169,64],"Diffbot":["partial","*",169,64],"ContextFetchBot":["partial","*",169,64]},"llms":{"status":"missing","http":404,"bytes":1569},"home":{"status":"ok","http":200,"url":"https://www.google.com/","html_tokens":21444,"markdown_tokens":30,"pruned_tokens":30}},
{"rank":3517,"domain":"ucoz.com","origin":"https://ucoz.com","robots":{"status":"ok","http":200,"url":"https://ucoz.com/robots.txt","bytes":121,"groups":1},"bots":{"GPTBot":["partial","*",5,0],"ChatGPT-User":["partial","*",5,0],"OAI-SearchBot":["partial","*",5,0],"ClaudeBot":["partial","*",5,0],"Claude-User":["partial","*",5,0],"Claude-SearchBot":["partial","*",5,0],"anthropic-ai":["partial","*",5,0],"PerplexityBot":["partial","*",5,0],"Perplexity-User":["partial","*",5,0],"Google-Extended":["partial","*",5,0],"Applebot-Extended":["partial","*",5,0],"CCBot":["partial","*",5,0],"Bytespider":["partial","*",5,0],"Amazonbot":["partial","*",5,0],"meta-externalagent":["partial","*",5,0],"cohere-ai":["partial","*",5,0],"Diffbot":["partial","*",5,0],"ContextFetchBot":["partial","*",5,0]},"llms":{"status":"present","http":200,"bytes":19107},"home":{"status":"ok","http":200,"url":"https://www.ucoz.com/","html_tokens":12430,"markdown_tokens":184,"pruned_tokens":126}},
{"rank":3518,"domain":"consumeraffairs.com","origin":"https://consumeraffairs.com","robots":{"status":"ok","http":200,"url":"https://www.consumeraffairs.com/robots.txt","bytes":1183,"groups":3},"bots":{"GPTBot":["partial","*",36,0],"ChatGPT-User":["partial","*",36,0],"OAI-SearchBot":["partial","*",36,0],"ClaudeBot":["partial","*",36,0],"Claude-User":["partial","*",36,0],"Claude-SearchBot":["partial","*",36,0],"anthropic-ai":["partial","*",36,0],"PerplexityBot":["partial","*",36,0],"Perplexity-User":["partial","*",36,0],"Google-Extended":["partial","*",36,0],"Applebot-Extended":["partial","*",36,0],"CCBot":["partial","*",36,0],"Bytespider":["partial","*",36,0],"Amazonbot":["partial","*",36,0],"meta-externalagent":["partial","*",36,0],"cohere-ai":["partial","*",36,0],"Diffbot":["partial","*",36,0],"ContextFetchBot":["partial","*",36,0]},"llms":{"status":"present","http":200,"bytes":13231},"home":{"status":"ok","http":200,"url":"https://www.consumeraffairs.com/","html_tokens":135390,"markdown_tokens":1185,"pruned_tokens":57}},
{"rank":3519,"domain":"22.cn","origin":"https://22.cn","robots":{"status":"ok","http":200,"url":"https://www.22.cn/robots.txt","bytes":173,"groups":2},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["partial","*",2,0],"OAI-SearchBot":["partial","*",2,0],"ClaudeBot":["partial","*",2,0],"Claude-User":["partial","*",2,0],"Claude-SearchBot":["partial","*",2,0],"anthropic-ai":["partial","*",2,0],"PerplexityBot":["partial","*",2,0],"Perplexity-User":["partial","*",2,0],"Google-Extended":["partial","*",2,0],"Applebot-Extended":["partial","*",2,0],"CCBot":["partial","*",2,0],"Bytespider":["partial","*",2,0],"Amazonbot":["partial","*",2,0],"meta-externalagent":["partial","*",2,0],"cohere-ai":["partial","*",2,0],"Diffbot":["partial","*",2,0],"ContextFetchBot":["partial","*",2,0]},"llms":{"status":"html","http":200,"bytes":93955},"home":{"status":"ok","http":200,"url":"https://www.22.cn/","html_tokens":58475,"markdown_tokens":275,"pruned_tokens":41}},
{"rank":3520,"domain":"natlawreview.com","origin":"https://natlawreview.com","robots":{"status":"refused","http":403,"url":"https://natlawreview.com/robots.txt","bytes":560,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":3521,"domain":"google.fi","origin":"https://google.fi","robots":{"status":"ok","http":200,"url":"https://www.google.fi/robots.txt","bytes":6624,"groups":4},"bots":{"GPTBot":["partial","*",169,64],"ChatGPT-User":["partial","*",169,64],"OAI-SearchBot":["partial","*",169,64],"ClaudeBot":["partial","*",169,64],"Claude-User":["partial","*",169,64],"Claude-SearchBot":["partial","*",169,64],"anthropic-ai":["partial","*",169,64],"PerplexityBot":["partial","*",169,64],"Perplexity-User":["partial","*",169,64],"Google-Extended":["partial","*",169,64],"Applebot-Extended":["partial","*",169,64],"CCBot":["partial","*",169,64],"Bytespider":["partial","*",169,64],"Amazonbot":["partial","*",169,64],"meta-externalagent":["partial","*",169,64],"cohere-ai":["partial","*",169,64],"Diffbot":["partial","*",169,64],"ContextFetchBot":["partial","*",169,64]},"llms":{"status":"missing","http":404,"bytes":1569},"home":{"status":"ok","http":200,"url":"https://www.google.com/","html_tokens":21325,"markdown_tokens":30,"pruned_tokens":30}},
{"rank":3522,"domain":"nrf.com","origin":"https://nrf.com","robots":{"status":"ok","http":200,"url":"https://nrf.com/robots.txt","bytes":195,"groups":1},"bots":{"GPTBot":["partial","*",3,1],"ChatGPT-User":["partial","*",3,1],"OAI-SearchBot":["partial","*",3,1],"ClaudeBot":["partial","*",3,1],"Claude-User":["partial","*",3,1],"Claude-SearchBot":["partial","*",3,1],"anthropic-ai":["partial","*",3,1],"PerplexityBot":["partial","*",3,1],"Perplexity-User":["partial","*",3,1],"Google-Extended":["partial","*",3,1],"Applebot-Extended":["partial","*",3,1],"CCBot":["partial","*",3,1],"Bytespider":["partial","*",3,1],"Amazonbot":["partial","*",3,1],"meta-externalagent":["partial","*",3,1],"cohere-ai":["partial","*",3,1],"Diffbot":["partial","*",3,1],"ContextFetchBot":["partial","*",3,1]},"llms":{"status":"present","http":200,"bytes":5721},"home":{"status":"ok","http":200,"url":"https://nrf.com/","html_tokens":116246,"markdown_tokens":122,"pruned_tokens":122}},
{"rank":3523,"domain":"infoplease.com","origin":"https://infoplease.com","robots":{"status":"ok","http":200,"url":"https://www.infoplease.com/robots.txt","bytes":1813,"groups":9},"bots":{"GPTBot":["partial","*",22,9],"ChatGPT-User":["partial","*",22,9],"OAI-SearchBot":["partial","*",22,9],"ClaudeBot":["partial","*",22,9],"Claude-User":["partial","*",22,9],"Claude-SearchBot":["partial","*",22,9],"anthropic-ai":["partial","*",22,9],"PerplexityBot":["partial","*",22,9],"Perplexity-User":["partial","*",22,9],"Google-Extended":["partial","*",22,9],"Applebot-Extended":["partial","*",22,9],"CCBot":["partial","*",22,9],"Bytespider":["partial","*",22,9],"Amazonbot":["partial","*",22,9],"meta-externalagent":["partial","*",22,9],"cohere-ai":["partial","*",22,9],"Diffbot":["partial","*",22,9],"ContextFetchBot":["partial","*",22,9]},"llms":{"status":"missing","http":404,"bytes":291},"home":{"status":"http_error","http":403,"url":"https://infoplease.com/"}},
{"rank":3524,"domain":"npc.gov.cn","origin":"http://npc.gov.cn","robots":{"status":"refused","http":403,"url":"http://www.npc.gov.cn/robots.txt","bytes":288,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":3525,"domain":"newgrounds.com","origin":"https://newgrounds.com","robots":{"status":"refused","http":403,"url":"https://newgrounds.com/robots.txt","bytes":9,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":3526,"domain":"builtin.com","origin":"https://builtin.com","robots":{"status":"ok","http":200,"url":"https://builtin.com/robots.txt","bytes":7802,"groups":5},"bots":{"GPTBot":["partial","named",1,21],"ChatGPT-User":["partial","named",1,21],"OAI-SearchBot":["partial","named",1,21],"ClaudeBot":["partial","*",119,39],"Claude-User":["partial","*",119,39],"Claude-SearchBot":["partial","*",119,39],"anthropic-ai":["partial","*",119,39],"PerplexityBot":["partial","named",1,21],"Perplexity-User":["partial","*",119,39],"Google-Extended":["partial","*",119,39],"Applebot-Extended":["partial","*",119,39],"CCBot":["partial","*",119,39],"Bytespider":["partial","*",119,39],"Amazonbot":["partial","*",119,39],"meta-externalagent":["partial","*",119,39],"cohere-ai":["partial","*",119,39],"Diffbot":["partial","*",119,39],"ContextFetchBot":["partial","*",119,39]},"llms":{"status":"missing","http":404,"bytes":652},"home":{"status":"ok","http":200,"url":"https://builtin.com/","html_tokens":17096,"markdown_tokens":48,"pruned_tokens":48}},
{"rank":3527,"domain":"itemfix.com","origin":"https://itemfix.com","robots":{"status":"ok","http":200,"url":"https://itemfix.com/robots.txt","bytes":71,"groups":2},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"html","http":200,"bytes":7345},"home":{"status":"ok","http":200,"url":"https://itemfix.com/","html_tokens":11462,"markdown_tokens":42,"pruned_tokens":42}},
{"rank":3528,"domain":"tv-tokyo.co.jp","origin":"https://tv-tokyo.co.jp","robots":{"status":"html","http":200,"url":"https://www.tv-tokyo.co.jp/","bytes":58519,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":3529,"domain":"oppo.com","origin":"https://oppo.com","robots":{"status":"ok","http":200,"url":"https://www.oppo.com/robots.txt","bytes":70,"groups":1},"bots":{"GPTBot":["partial","*",1,0],"ChatGPT-User":["partial","*",1,0],"OAI-SearchBot":["partial","*",1,0],"ClaudeBot":["partial","*",1,0],"Claude-User":["partial","*",1,0],"Claude-SearchBot":["partial","*",1,0],"anthropic-ai":["partial","*",1,0],"PerplexityBot":["partial","*",1,0],"Perplexity-User":["partial","*",1,0],"Google-Extended":["partial","*",1,0],"Applebot-Extended":["partial","*",1,0],"CCBot":["partial","*",1,0],"Bytespider":["partial","*",1,0],"Amazonbot":["partial","*",1,0],"meta-externalagent":["partial","*",1,0],"cohere-ai":["partial","*",1,0],"Diffbot":["partial","*",1,0],"ContextFetchBot":["partial","*",1,0]},"llms":{"status":"present","http":200,"bytes":3906208},"home":{"status":"ok","http":200,"url":"https://www.oppo.com/en/","html_tokens":50959,"markdown_tokens":54,"pruned_tokens":54}},
{"rank":3530,"domain":"aircanada.com","origin":"https://aircanada.com","robots":{"status":"missing","http":404,"url":"https://www.aircanada.com/robots.txt","bytes":350,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":464980},"home":{"status":"ok","http":200,"url":"https://www.aircanada.com/home/us/en/aco/flights","html_tokens":17788,"markdown_tokens":0,"pruned_tokens":0}},
{"rank":3531,"domain":"kiplinger.com","origin":"https://kiplinger.com","robots":{"status":"ok","http":200,"url":"https://www.kiplinger.com/robots.txt","bytes":1845,"groups":5},"bots":{"GPTBot":["partial","*",25,0],"ChatGPT-User":["partial","*",25,0],"OAI-SearchBot":["partial","*",25,0],"ClaudeBot":["partial","*",25,0],"Claude-User":["partial","*",25,0],"Claude-SearchBot":["partial","*",25,0],"anthropic-ai":["partial","*",25,0],"PerplexityBot":["partial","*",25,0],"Perplexity-User":["partial","*",25,0],"Google-Extended":["partial","*",25,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["partial","*",25,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["partial","*",25,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",25,0]},"llms":{"status":"missing","http":404,"bytes":1622409},"home":{"status":"ok","http":200,"url":"https://www.kiplinger.com/","html_tokens":500219,"markdown_tokens":2652,"pruned_tokens":329}},
{"rank":3532,"domain":"mohrss.gov.cn","origin":"https://www.mohrss.gov.cn","robots":{"status":"html","http":200,"url":"https://www.mohrss.gov.cn/robots.txt","bytes":985,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":3533,"domain":"ebc.com.br","origin":"https://ebc.com.br","robots":{"status":"ok","http":200,"url":"https://www.ebc.com.br/robots.txt","bytes":870,"groups":2},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"missing","http":404,"bytes":76187},"home":{"status":"ok","http":200,"url":"https://www.ebc.com.br/","html_tokens":37251,"markdown_tokens":23,"pruned_tokens":23}},
{"rank":3534,"domain":"google.co.za","origin":"https://google.co.za","robots":{"status":"ok","http":200,"url":"https://www.google.co.za/robots.txt","bytes":6624,"groups":4},"bots":{"GPTBot":["partial","*",169,64],"ChatGPT-User":["partial","*",169,64],"OAI-SearchBot":["partial","*",169,64],"ClaudeBot":["partial","*",169,64],"Claude-User":["partial","*",169,64],"Claude-SearchBot":["partial","*",169,64],"anthropic-ai":["partial","*",169,64],"PerplexityBot":["partial","*",169,64],"Perplexity-User":["partial","*",169,64],"Google-Extended":["partial","*",169,64],"Applebot-Extended":["partial","*",169,64],"CCBot":["partial","*",169,64],"Bytespider":["partial","*",169,64],"Amazonbot":["partial","*",169,64],"meta-externalagent":["partial","*",169,64],"cohere-ai":["partial","*",169,64],"Diffbot":["partial","*",169,64],"ContextFetchBot":["partial","*",169,64]},"llms":{"status":"missing","http":404,"bytes":1569},"home":{"status":"ok","http":200,"url":"https://www.google.com/","html_tokens":21330,"markdown_tokens":30,"pruned_tokens":30}},
{"rank":3535,"domain":"proboards.com","origin":"https://proboards.com","robots":{"status":"missing","http":404,"url":"https://proboards.com/robots.txt","bytes":8071,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":8071},"home":{"status":"ok","http":200,"url":"https://proboards.com/","html_tokens":10289,"markdown_tokens":165,"pruned_tokens":165}},
{"rank":3536,"domain":"couchsurfing.com","origin":"https://couchsurfing.com","robots":{"status":"ok","http":200,"url":"https://www.couchsurfing.com/robots.txt","bytes":1969,"groups":6},"bots":{"GPTBot":["partial","*",13,1],"ChatGPT-User":["partial","*",13,1],"OAI-SearchBot":["partial","*",13,1],"ClaudeBot":["partial","named",13,1],"Claude-User":["partial","named",13,1],"Claude-SearchBot":["partial","*",13,1],"anthropic-ai":["partial","*",13,1],"PerplexityBot":["partial","named",13,1],"Perplexity-User":["partial","*",13,1],"Google-Extended":["partial","*",13,1],"Applebot-Extended":["partial","*",13,1],"CCBot":["partial","*",13,1],"Bytespider":["partial","*",13,1],"Amazonbot":["partial","*",13,1],"meta-externalagent":["partial","*",13,1],"cohere-ai":["partial","*",13,1],"Diffbot":["partial","*",13,1],"ContextFetchBot":["partial","*",13,1]},"llms":{"status":"missing","http":404,"bytes":430012},"home":{"status":"ok","http":200,"url":"https://www.couchsurfing.com/","html_tokens":184583,"markdown_tokens":128,"pruned_tokens":26}},
{"rank":3537,"domain":"bundesfinanzministerium.de","origin":"https://bundesfinanzministerium.de","robots":{"status":"server_error","http":503,"url":"https://www.bundesfinanzministerium.de/robots.txt","bytes":30610,"groups":0,"error":"HTTP 503"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":3538,"domain":"authy.com","origin":"https://authy.com","robots":{"status":"missing","http":404,"url":"https://www.authy.com/robots.txt","bytes":628740,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":83527},"home":{"status":"ok","http":200,"url":"https://www.authy.com/","html_tokens":27679,"markdown_tokens":208,"pruned_tokens":108}},
{"rank":3539,"domain":"if.ua","origin":"https://if.ua","robots":{"status":"unreachable","http":null,"url":"https://if.ua/robots.txt","bytes":0,"groups":0,"error":"cannot resolve if.ua"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":3540,"domain":"le.ac.uk","origin":"https://le.ac.uk","robots":{"status":"refused","http":403,"url":"https://le.ac.uk/robots.txt","bytes":4902,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":3541,"domain":"telegraaf.nl","origin":"https://telegraaf.nl","robots":{"status":"ok","http":200,"url":"https://www.telegraaf.nl/robots.txt","bytes":2823,"groups":40},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["blocked","named",1,0],"OAI-SearchBot":["blocked","named",1,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["blocked","named",1,0],"Claude-SearchBot":["blocked","named",1,0],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["blocked","named",1,0],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["partial","*",3,2],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",3,2]},"llms":{"status":"http_error","http":403,"bytes":4545},"home":{"status":"http_error","http":403,"url":"https://www.telegraaf.nl/"}},
{"rank":3542,"domain":"nsc.org","origin":"https://nsc.org","robots":{"status":"ok","http":200,"url":"https://www.nsc.org/robots.txt","bytes":101,"groups":1},"bots":{"GPTBot":["partial","*",2,0],"ChatGPT-User":["partial","*",2,0],"OAI-SearchBot":["partial","*",2,0],"ClaudeBot":["partial","*",2,0],"Claude-User":["partial","*",2,0],"Claude-SearchBot":["partial","*",2,0],"anthropic-ai":["partial","*",2,0],"PerplexityBot":["partial","*",2,0],"Perplexity-User":["partial","*",2,0],"Google-Extended":["partial","*",2,0],"Applebot-Extended":["partial","*",2,0],"CCBot":["partial","*",2,0],"Bytespider":["partial","*",2,0],"Amazonbot":["partial","*",2,0],"meta-externalagent":["partial","*",2,0],"cohere-ai":["partial","*",2,0],"Diffbot":["partial","*",2,0],"ContextFetchBot":["partial","*",2,0]},"llms":{"status":"html","http":200,"bytes":68817},"home":{"status":"http_error","http":403,"url":"https://nsc.org/"}},
{"rank":3543,"domain":"universiteitleiden.nl","origin":"https://universiteitleiden.nl","robots":{"status":"html","http":200,"url":"https://universiteitleiden.nl/robots.txt","bytes":21994,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":3544,"domain":"carbonbrief.org","origin":"https://carbonbrief.org","robots":{"status":"ok","http":200,"url":"https://www.carbonbrief.org/robots.txt","bytes":193,"groups":1},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"http_error","http":403,"bytes":5781},"home":{"status":"http_error","http":403,"url":"https://carbonbrief.org/"}},
{"rank":3545,"domain":"onrender.com","origin":"https://onrender.com","robots":{"status":"html","http":200,"url":"https://render.com","bytes":387018,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":3546,"domain":"googlesource.com","origin":"https://googlesource.com","robots":{"status":"missing","http":404,"url":"https://googlesource.com/robots.txt","bytes":1803,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":1801},"home":{"status":"ok","http":200,"url":"https://opensource.google","html_tokens":15212,"markdown_tokens":771,"pruned_tokens":158}},
{"rank":3547,"domain":"todoist.com","origin":"https://todoist.com","robots":{"status":"ok","http":200,"url":"https://todoist.com/robots.txt","bytes":521,"groups":1},"bots":{"GPTBot":["partial","*",22,2],"ChatGPT-User":["partial","*",22,2],"OAI-SearchBot":["partial","*",22,2],"ClaudeBot":["partial","*",22,2],"Claude-User":["partial","*",22,2],"Claude-SearchBot":["partial","*",22,2],"anthropic-ai":["partial","*",22,2],"PerplexityBot":["partial","*",22,2],"Perplexity-User":["partial","*",22,2],"Google-Extended":["partial","*",22,2],"Applebot-Extended":["partial","*",22,2],"CCBot":["partial","*",22,2],"Bytespider":["partial","*",22,2],"Amazonbot":["partial","*",22,2],"meta-externalagent":["partial","*",22,2],"cohere-ai":["partial","*",22,2],"Diffbot":["partial","*",22,2],"ContextFetchBot":["partial","*",22,2]},"llms":{"status":"missing","http":404,"bytes":90965},"home":{"status":"ok","http":200,"url":"https://www.todoist.com/","html_tokens":175550,"markdown_tokens":639,"pruned_tokens":382}},
{"rank":3548,"domain":"caddy.community","origin":"https://caddy.community","robots":{"status":"ok","http":200,"url":"https://caddy.community/robots.txt","bytes":996,"groups":7},"bots":{"GPTBot":["partial","*",15,0],"ChatGPT-User":["partial","*",15,0],"OAI-SearchBot":["partial","*",15,0],"ClaudeBot":["partial","*",15,0],"Claude-User":["partial","*",15,0],"Claude-SearchBot":["partial","*",15,0],"anthropic-ai":["partial","*",15,0],"PerplexityBot":["partial","*",15,0],"Perplexity-User":["partial","*",15,0],"Google-Extended":["partial","*",15,0],"Applebot-Extended":["partial","*",15,0],"CCBot":["partial","*",15,0],"Bytespider":["partial","*",15,0],"Amazonbot":["partial","*",15,0],"meta-externalagent":["partial","*",15,0],"cohere-ai":["partial","*",15,0],"Diffbot":["partial","*",15,0],"ContextFetchBot":["partial","*",15,0]},"llms":{"status":"missing","http":404,"bytes":0},"home":{"status":"ok","http":200,"url":"https://caddy.community/","html_tokens":17521,"markdown_tokens":851,"pruned_tokens":467}},
{"rank":3549,"domain":"rmit.edu.au","origin":"https://rmit.edu.au","robots":{"status":"ok","http":200,"url":"https://www.rmit.edu.au/robots.txt","bytes":462,"groups":1},"bots":{"GPTBot":["partial","*",8,0],"ChatGPT-User":["partial","*",8,0],"OAI-SearchBot":["partial","*",8,0],"ClaudeBot":["partial","*",8,0],"Claude-User":["partial","*",8,0],"Claude-SearchBot":["partial","*",8,0],"anthropic-ai":["partial","*",8,0],"PerplexityBot":["partial","*",8,0],"Perplexity-User":["partial","*",8,0],"Google-Extended":["partial","*",8,0],"Applebot-Extended":["partial","*",8,0],"CCBot":["partial","*",8,0],"Bytespider":["partial","*",8,0],"Amazonbot":["partial","*",8,0],"meta-externalagent":["partial","*",8,0],"cohere-ai":["partial","*",8,0],"Diffbot":["partial","*",8,0],"ContextFetchBot":["partial","*",8,0]},"llms":{"status":"missing","http":404,"bytes":207152},"home":{"status":"ok","http":200,"url":"https://www.rmit.edu.au/","html_tokens":79890,"markdown_tokens":188,"pruned_tokens":122}},
{"rank":3550,"domain":"huaweicloud.com","origin":"https://huaweicloud.com","robots":{"status":"ok","http":200,"url":"https://www.huaweicloud.com/robots.txt","bytes":241,"groups":1},"bots":{"GPTBot":["partial","*",10,0],"ChatGPT-User":["partial","*",10,0],"OAI-SearchBot":["partial","*",10,0],"ClaudeBot":["partial","*",10,0],"Claude-User":["partial","*",10,0],"Claude-SearchBot":["partial","*",10,0],"anthropic-ai":["partial","*",10,0],"PerplexityBot":["partial","*",10,0],"Perplexity-User":["partial","*",10,0],"Google-Extended":["partial","*",10,0],"Applebot-Extended":["partial","*",10,0],"CCBot":["partial","*",10,0],"Bytespider":["partial","*",10,0],"Amazonbot":["partial","*",10,0],"meta-externalagent":["partial","*",10,0],"cohere-ai":["partial","*",10,0],"Diffbot":["partial","*",10,0],"ContextFetchBot":["partial","*",10,0]},"llms":{"status":"http_error","http":403,"bytes":263},"home":{"status":"ok","http":200,"url":"https://www.huaweicloud.com/","html_tokens":49332,"markdown_tokens":12,"pruned_tokens":12}},
{"rank":3551,"domain":"weezevent.com","origin":"https://weezevent.com","robots":{"status":"ok","http":200,"url":"https://weezevent.com/robots.txt","bytes":160,"groups":1},"bots":{"GPTBot":["partial","*",1,1],"ChatGPT-User":["partial","*",1,1],"OAI-SearchBot":["partial","*",1,1],"ClaudeBot":["partial","*",1,1],"Claude-User":["partial","*",1,1],"Claude-SearchBot":["partial","*",1,1],"anthropic-ai":["partial","*",1,1],"PerplexityBot":["partial","*",1,1],"Perplexity-User":["partial","*",1,1],"Google-Extended":["partial","*",1,1],"Applebot-Extended":["partial","*",1,1],"CCBot":["partial","*",1,1],"Bytespider":["partial","*",1,1],"Amazonbot":["partial","*",1,1],"meta-externalagent":["partial","*",1,1],"cohere-ai":["partial","*",1,1],"Diffbot":["partial","*",1,1],"ContextFetchBot":["partial","*",1,1]},"llms":{"status":"missing","http":404,"bytes":73163},"home":{"status":"ok","http":200,"url":"https://weezevent.com/en-gb/","html_tokens":62439,"markdown_tokens":221,"pruned_tokens":151}},
{"rank":3552,"domain":"epson.com","origin":"https://epson.com","robots":{"status":"ok","http":200,"url":"https://epson.com/robots.txt","bytes":1643,"groups":18},"bots":{"GPTBot":["partial","*",22,0],"ChatGPT-User":["partial","*",22,0],"OAI-SearchBot":["partial","*",22,0],"ClaudeBot":["partial","*",22,0],"Claude-User":["partial","*",22,0],"Claude-SearchBot":["partial","*",22,0],"anthropic-ai":["partial","*",22,0],"PerplexityBot":["partial","*",22,0],"Perplexity-User":["partial","*",22,0],"Google-Extended":["partial","*",22,0],"Applebot-Extended":["partial","*",22,0],"CCBot":["partial","*",22,0],"Bytespider":["partial","*",22,0],"Amazonbot":["partial","*",22,0],"meta-externalagent":["partial","*",22,0],"cohere-ai":["partial","*",22,0],"Diffbot":["partial","*",22,0],"ContextFetchBot":["partial","*",22,0]},"llms":{"status":"missing","http":404,"bytes":312092},"home":{"status":"ok","http":200,"url":"https://epson.com/","html_tokens":4923,"markdown_tokens":10,"pruned_tokens":10}},
{"rank":3553,"domain":"ntnu.no","origin":"https://ntnu.no","robots":{"status":"ok","http":200,"url":"https://www.ntnu.no/robots.txt","bytes":2742,"groups":1},"bots":{"GPTBot":["partial","*",70,0],"ChatGPT-User":["partial","*",70,0],"OAI-SearchBot":["partial","*",70,0],"ClaudeBot":["partial","*",70,0],"Claude-User":["partial","*",70,0],"Claude-SearchBot":["partial","*",70,0],"anthropic-ai":["partial","*",70,0],"PerplexityBot":["partial","*",70,0],"Perplexity-User":["partial","*",70,0],"Google-Extended":["partial","*",70,0],"Applebot-Extended":["partial","*",70,0],"CCBot":["partial","*",70,0],"Bytespider":["partial","*",70,0],"Amazonbot":["partial","*",70,0],"meta-externalagent":["partial","*",70,0],"cohere-ai":["partial","*",70,0],"Diffbot":["partial","*",70,0],"ContextFetchBot":["partial","*",70,0]},"llms":{"status":"missing","http":404,"bytes":4435},"home":{"status":"ok","http":200,"url":"https://www.ntnu.no/","html_tokens":25950,"markdown_tokens":44,"pruned_tokens":44}},
{"rank":3554,"domain":"schwab.com","origin":"https://schwab.com","robots":{"status":"ok","http":200,"url":"https://www.schwab.com/robots.txt","bytes":502,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"html","http":200,"bytes":502},"home":{"status":"ok","http":200,"url":"https://www.schwab.com/","html_tokens":125,"markdown_tokens":51,"pruned_tokens":51}},
{"rank":3555,"domain":"copilot.com","origin":"https://copilot.com","robots":{"status":"html","http":200,"url":"https://login.microsoftonline.com/common/oauth2/v2.0/authorize?client_id=4765445b-32c6-49b0-83e6-1d93765276ca&redirect_uri=https%3A%2F%2Fcopilot.com%2Flandingv2&response_type=code%20id_token&scope=openid%20profile%20offline_access%20https%3A%2F%2Fwww.office.com%2Fv2%2FOfficeHome.All&response_mode=form_post&nonce=639279712368116328.NjRiNzFmMjUtYzYyMS00NjA3LWFjMTktZjU4ODJhMDczZGM0NTQ1ODIyZDYtNjFiNC00NGZlLWFiZDMtODM2ZjQyYzA0NDE4&client-request-id=82fcceb4-50d0-4b9a-b445-9c660b11a6b5&state=Ko00-Be2JcjGvl0Tm8-9KxkCm7II0MeXpECDnQE7MszAenlutQDiryJjgr7cICrOEnZi856Q1IMeZT9BtJNjmrSFOEnp1ggCbsP-KeD9eBOCR5XliGp2mo9vGV0YxKlRaU2z04ODWE_7rsfxE1mzNzh4MTplpjJtKQuI5rL-m4t-7BjJ3BtfbYzMDqv6mCWPvRwSTmz3pO50cWEKfvZUNQbtIN_kx0503xOTsG3ETgBLv_PNfQsD02yjN4vPlX4X99Ek2-1Lj5Xt66rVYR73UTw4o_Rgz2q9ZaVfK7bvWoRP8KAqsyYKduyGs9H6hRg9NK0CDjvyKHLc5xD1X2izvXZ7WLRNasGHJ1Mpms0RPO05OuyCNA-P3W_CYC6Juqth&x-client-SKU=ID_NET8_0&x-client-ver=8.14.0.0","bytes":53585,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":3556,"domain":"intel.de","origin":"https://intel.de","robots":{"status":"ok","http":200,"url":"https://www.intel.com/geo-robots/de/robots.txt","bytes":3392,"groups":3},"bots":{"GPTBot":["partial","*",63,2],"ChatGPT-User":["partial","*",63,2],"OAI-SearchBot":["partial","*",63,2],"ClaudeBot":["partial","*",63,2],"Claude-User":["partial","*",63,2],"Claude-SearchBot":["partial","*",63,2],"anthropic-ai":["partial","*",63,2],"PerplexityBot":["partial","*",63,2],"Perplexity-User":["partial","*",63,2],"Google-Extended":["partial","*",63,2],"Applebot-Extended":["partial","*",63,2],"CCBot":["partial","*",63,2],"Bytespider":["partial","*",63,2],"Amazonbot":["partial","*",63,2],"meta-externalagent":["partial","*",63,2],"cohere-ai":["partial","*",63,2],"Diffbot":["partial","*",63,2],"ContextFetchBot":["partial","*",63,2]},"llms":{"status":"missing","http":404,"bytes":69892},"home":{"status":"ok","http":200,"url":"https://www.intel.de/content/www/de/de/homepage.html","html_tokens":34652,"markdown_tokens":98,"pruned_tokens":14}},
{"rank":3557,"domain":"umanitoba.ca","origin":"https://umanitoba.ca","robots":{"status":"ok","http":200,"url":"https://umanitoba.ca/robots.txt","bytes":5598,"groups":1},"bots":{"GPTBot":["partial","*",35,18],"ChatGPT-User":["partial","*",35,18],"OAI-SearchBot":["partial","*",35,18],"ClaudeBot":["partial","*",35,18],"Claude-User":["partial","*",35,18],"Claude-SearchBot":["partial","*",35,18],"anthropic-ai":["partial","*",35,18],"PerplexityBot":["partial","*",35,18],"Perplexity-User":["partial","*",35,18],"Google-Extended":["partial","*",35,18],"Applebot-Extended":["partial","*",35,18],"CCBot":["partial","*",35,18],"Bytespider":["partial","*",35,18],"Amazonbot":["partial","*",35,18],"meta-externalagent":["partial","*",35,18],"cohere-ai":["partial","*",35,18],"Diffbot":["partial","*",35,18],"ContextFetchBot":["partial","*",35,18]},"llms":{"status":"missing","http":404,"bytes":186},"home":{"status":"ok","http":200,"url":"https://umanitoba.ca/","html_tokens":19086,"markdown_tokens":795,"pruned_tokens":123}},
{"rank":3558,"domain":"netsuite.com","origin":"https://netsuite.com","robots":{"status":"refused","http":403,"url":"https://netsuite.com/robots.txt","bytes":376,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":3559,"domain":"stltoday.com","origin":"https://stltoday.com","robots":{"status":"ok","http":200,"url":"https://stltoday.com/robots.txt","bytes":14920,"groups":31},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["blocked","named",1,0],"OAI-SearchBot":["blocked","named",1,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",179,0],"Claude-SearchBot":["partial","*",179,0],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["partial","*",179,0],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["partial","*",179,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",179,0]},"llms":{"status":"http_error","http":429,"bytes":64},"home":{"status":"ok","http":200,"url":"https://www.stltoday.com/","html_tokens":142909,"markdown_tokens":294,"pruned_tokens":151}},
{"rank":3560,"domain":"rhs.org.uk","origin":"https://rhs.org.uk","robots":{"status":"ok","http":200,"url":"https://www.rhs.org.uk/robots.txt","bytes":359,"groups":6},"bots":{"GPTBot":["partial","*",5,0],"ChatGPT-User":["partial","*",5,0],"OAI-SearchBot":["partial","*",5,0],"ClaudeBot":["partial","*",5,0],"Claude-User":["partial","*",5,0],"Claude-SearchBot":["partial","*",5,0],"anthropic-ai":["partial","*",5,0],"PerplexityBot":["partial","*",5,0],"Perplexity-User":["partial","*",5,0],"Google-Extended":["partial","*",5,0],"Applebot-Extended":["partial","*",5,0],"CCBot":["partial","*",5,0],"Bytespider":["partial","*",5,0],"Amazonbot":["partial","*",5,0],"meta-externalagent":["partial","*",5,0],"cohere-ai":["partial","*",5,0],"Diffbot":["partial","*",5,0],"ContextFetchBot":["partial","*",5,0]},"llms":{"status":"missing","http":404,"bytes":88605},"home":{"status":"ok","http":200,"url":"https://www.rhs.org.uk/","html_tokens":30130,"markdown_tokens":80,"pruned_tokens":59}},
{"rank":3561,"domain":"gigabyte.com","origin":"https://gigabyte.com","robots":{"status":"refused","http":403,"url":"https://www.gigabyte.com/robots.txt","bytes":384,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":3562,"domain":"shinyapps.io","origin":"https://shinyapps.io","robots":{"status":"missing","http":404,"url":"https://www.shinyapps.io/robots.txt","bytes":69,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":69},"home":{"status":"ok","http":200,"url":"https://www.shinyapps.io/","html_tokens":7414,"markdown_tokens":324,"pruned_tokens":96}},
{"rank":3563,"domain":"overheid.nl","origin":"https://overheid.nl","robots":{"status":"ok","http":200,"url":"https://overheid.nl/robots.txt","bytes":1921,"groups":20},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["partial","*",4,1],"OAI-SearchBot":["blocked","named",1,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",4,1],"Claude-SearchBot":["partial","*",4,1],"anthropic-ai":["partial","*",4,1],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["partial","*",4,1],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["partial","*",4,1],"meta-externalagent":["partial","*",4,1],"cohere-ai":["partial","*",4,1],"Diffbot":["partial","*",4,1],"ContextFetchBot":["partial","*",4,1]},"llms":{"status":"missing","http":404,"bytes":146},"home":{"status":"ok","http":200,"url":"https://overheid.nl/","html_tokens":4550,"markdown_tokens":144,"pruned_tokens":144}},
{"rank":3564,"domain":"bz.it","origin":"https://bz.it","robots":{"status":"unreachable","http":null,"url":"https://bz.it/robots.txt","bytes":0,"groups":0,"error":"cannot resolve bz.it"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":3565,"domain":"cac.gov.cn","origin":"https://www.cac.gov.cn","robots":{"status":"ok","http":200,"url":"https://www.cac.gov.cn/robots.txt","bytes":106,"groups":1},"bots":{"GPTBot":["partial","*",4,0],"ChatGPT-User":["partial","*",4,0],"OAI-SearchBot":["partial","*",4,0],"ClaudeBot":["partial","*",4,0],"Claude-User":["partial","*",4,0],"Claude-SearchBot":["partial","*",4,0],"anthropic-ai":["partial","*",4,0],"PerplexityBot":["partial","*",4,0],"Perplexity-User":["partial","*",4,0],"Google-Extended":["partial","*",4,0],"Applebot-Extended":["partial","*",4,0],"CCBot":["partial","*",4,0],"Bytespider":["partial","*",4,0],"Amazonbot":["partial","*",4,0],"meta-externalagent":["partial","*",4,0],"cohere-ai":["partial","*",4,0],"Diffbot":["partial","*",4,0],"ContextFetchBot":["partial","*",4,0]},"llms":{"status":"missing","http":404,"bytes":6968},"home":{"status":"ok","http":200,"url":"https://www.cac.gov.cn/","html_tokens":14873,"markdown_tokens":20,"pruned_tokens":20}},
{"rank":3566,"domain":"hetzner.de","origin":"https://hetzner.de","robots":{"status":"missing","http":404,"url":"https://www.hetzner.com/de/robots.txt","bytes":111806,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":111802},"home":{"status":"ok","http":200,"url":"https://www.hetzner.com/de/","html_tokens":25312,"markdown_tokens":456,"pruned_tokens":290}},
{"rank":3567,"domain":"thenationonlineng.net","origin":"https://thenationonlineng.net","robots":{"status":"ok","http":200,"url":"https://thenationonlineng.net/robots.txt","bytes":124,"groups":1},"bots":{"GPTBot":["partial","*",1,1],"ChatGPT-User":["partial","*",1,1],"OAI-SearchBot":["partial","*",1,1],"ClaudeBot":["partial","*",1,1],"Claude-User":["partial","*",1,1],"Claude-SearchBot":["partial","*",1,1],"anthropic-ai":["partial","*",1,1],"PerplexityBot":["partial","*",1,1],"Perplexity-User":["partial","*",1,1],"Google-Extended":["partial","*",1,1],"Applebot-Extended":["partial","*",1,1],"CCBot":["partial","*",1,1],"Bytespider":["partial","*",1,1],"Amazonbot":["partial","*",1,1],"meta-externalagent":["partial","*",1,1],"cohere-ai":["partial","*",1,1],"Diffbot":["partial","*",1,1],"ContextFetchBot":["partial","*",1,1]},"llms":{"status":"other_text","http":200,"bytes":2086},"home":{"status":"timeout","error":"timed out after 15 s"}},
{"rank":3568,"domain":"comodo.com","origin":"https://comodo.com","robots":{"status":"ok","http":200,"url":"https://www.comodo.com/robots.txt","bytes":27957,"groups":2},"bots":{"GPTBot":["partial","*",368,1],"ChatGPT-User":["partial","*",368,1],"OAI-SearchBot":["partial","*",368,1],"ClaudeBot":["partial","*",368,1],"Claude-User":["partial","*",368,1],"Claude-SearchBot":["partial","*",368,1],"anthropic-ai":["partial","*",368,1],"PerplexityBot":["partial","*",368,1],"Perplexity-User":["partial","*",368,1],"Google-Extended":["partial","*",368,1],"Applebot-Extended":["partial","*",368,1],"CCBot":["partial","*",368,1],"Bytespider":["partial","*",368,1],"Amazonbot":["partial","*",368,1],"meta-externalagent":["partial","*",368,1],"cohere-ai":["partial","*",368,1],"Diffbot":["partial","*",368,1],"ContextFetchBot":["partial","*",368,1]},"llms":{"status":"missing","http":404,"bytes":0},"home":{"status":"ok","http":200,"url":"https://www.comodo.com/","html_tokens":62099,"markdown_tokens":668,"pruned_tokens":116}},
{"rank":3569,"domain":"babycenter.com","origin":"https://babycenter.com","robots":{"status":"ok","http":200,"url":"https://www.babycenter.com/robots.txt","bytes":5521,"groups":3},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["blocked","named",1,0],"OAI-SearchBot":["blocked","named",1,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["blocked","named",1,0],"Claude-SearchBot":["blocked","named",1,0],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["blocked","named",1,0],"Google-Extended":["partial","*",4,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",4,0]},"llms":{"status":"html","http":202,"bytes":2439},"home":{"status":"ok","http":200,"url":"https://www.babycenter.com/","html_tokens":158560,"markdown_tokens":16,"pruned_tokens":16}},
{"rank":3570,"domain":"rospotrebnadzor.ru","origin":"https://rospotrebnadzor.ru","robots":{"status":"timeout","http":null,"url":"https://rospotrebnadzor.ru/robots.txt","bytes":0,"groups":0,"error":"ConnectTimeout"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":3571,"domain":"gsma.com","origin":"https://gsma.com","robots":{"status":"ok","http":200,"url":"https://www.gsma.com/robots.txt","bytes":92,"groups":1},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"missing","http":404,"bytes":61617},"home":{"status":"http_error","http":403,"url":"https://gsma.com/"}},
{"rank":3572,"domain":"gouvernement.fr","origin":"https://gouvernement.fr","robots":{"status":"ok","http":200,"url":"https://www.info.gouv.fr/robots.txt","bytes":1688,"groups":1},"bots":{"GPTBot":["partial","*",30,0],"ChatGPT-User":["partial","*",30,0],"OAI-SearchBot":["partial","*",30,0],"ClaudeBot":["partial","*",30,0],"Claude-User":["partial","*",30,0],"Claude-SearchBot":["partial","*",30,0],"anthropic-ai":["partial","*",30,0],"PerplexityBot":["partial","*",30,0],"Perplexity-User":["partial","*",30,0],"Google-Extended":["partial","*",30,0],"Applebot-Extended":["partial","*",30,0],"CCBot":["partial","*",30,0],"Bytespider":["partial","*",30,0],"Amazonbot":["partial","*",30,0],"meta-externalagent":["partial","*",30,0],"cohere-ai":["partial","*",30,0],"Diffbot":["partial","*",30,0],"ContextFetchBot":["partial","*",30,0]},"llms":{"status":"missing","http":404,"bytes":65256},"home":{"status":"http_error","http":403,"url":"https://www.info.gouv.fr/"}},
{"rank":3573,"domain":"taxfoundation.org","origin":"https://taxfoundation.org","robots":{"status":"ok","http":200,"url":"https://taxfoundation.org/robots.txt","bytes":1125,"groups":2},"bots":{"GPTBot":["partial","named",2,0],"ChatGPT-User":["partial","named",2,0],"OAI-SearchBot":["partial","*",7,0],"ClaudeBot":["partial","*",7,0],"Claude-User":["partial","*",7,0],"Claude-SearchBot":["partial","*",7,0],"anthropic-ai":["partial","named",2,0],"PerplexityBot":["partial","*",7,0],"Perplexity-User":["partial","*",7,0],"Google-Extended":["partial","*",7,0],"Applebot-Extended":["partial","*",7,0],"CCBot":["partial","named",2,0],"Bytespider":["partial","*",7,0],"Amazonbot":["partial","*",7,0],"meta-externalagent":["partial","*",7,0],"cohere-ai":["partial","*",7,0],"Diffbot":["partial","*",7,0],"ContextFetchBot":["partial","*",7,0]},"llms":{"status":"present","http":200,"bytes":13021},"home":{"status":"ok","http":200,"url":"https://taxfoundation.org/","html_tokens":32695,"markdown_tokens":1487,"pruned_tokens":135}},
{"rank":3574,"domain":"fr.de","origin":"https://fr.de","robots":{"status":"ok","http":200,"url":"https://fr.de/robots.txt","bytes":1530,"groups":2},"bots":{"GPTBot":["partial","*",1,2],"ChatGPT-User":["partial","named",11,1],"OAI-SearchBot":["partial","named",11,1],"ClaudeBot":["partial","*",1,2],"Claude-User":["partial","named",11,1],"Claude-SearchBot":["partial","named",11,1],"anthropic-ai":["partial","*",1,2],"PerplexityBot":["partial","named",11,1],"Perplexity-User":["partial","named",11,1],"Google-Extended":["partial","*",1,2],"Applebot-Extended":["partial","*",1,2],"CCBot":["partial","*",1,2],"Bytespider":["partial","*",1,2],"Amazonbot":["partial","*",1,2],"meta-externalagent":["partial","*",1,2],"cohere-ai":["partial","*",1,2],"Diffbot":["partial","*",1,2],"ContextFetchBot":["partial","*",1,2]},"llms":{"status":"disallowed"},"home":{"status":"disallowed"}},
{"rank":3575,"domain":"umbc.edu","origin":"https://umbc.edu","robots":{"status":"refused","http":403,"url":"https://umbc.edu/robots.txt","bytes":560,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":3576,"domain":"go.cr","origin":"https://go.cr","robots":{"status":"unreachable","http":null,"url":"https://go.cr/robots.txt","bytes":0,"groups":0,"error":"cannot resolve go.cr"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":3577,"domain":"myqcloud.com","origin":"https://myqcloud.com","robots":{"status":"unreachable","http":null,"url":"https://myqcloud.com/robots.txt","bytes":0,"groups":0,"error":"myqcloud.com is not a public address"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":3578,"domain":"blogspot.nl","origin":"https://blogspot.nl","robots":{"status":"ok","http":200,"url":"https://blogspot.nl/robots.txt","bytes":6624,"groups":4},"bots":{"GPTBot":["partial","*",169,64],"ChatGPT-User":["partial","*",169,64],"OAI-SearchBot":["partial","*",169,64],"ClaudeBot":["partial","*",169,64],"Claude-User":["partial","*",169,64],"Claude-SearchBot":["partial","*",169,64],"anthropic-ai":["partial","*",169,64],"PerplexityBot":["partial","*",169,64],"Perplexity-User":["partial","*",169,64],"Google-Extended":["partial","*",169,64],"Applebot-Extended":["partial","*",169,64],"CCBot":["partial","*",169,64],"Bytespider":["partial","*",169,64],"Amazonbot":["partial","*",169,64],"meta-externalagent":["partial","*",169,64],"cohere-ai":["partial","*",169,64],"Diffbot":["partial","*",169,64],"ContextFetchBot":["partial","*",169,64]},"llms":{"status":"missing","http":404,"bytes":1569},"home":{"status":"ok","http":200,"url":"https://www.google.com/","html_tokens":21289,"markdown_tokens":30,"pruned_tokens":30}},
{"rank":3579,"domain":"naver.jp","origin":"https://naver.jp","robots":{"status":"unreachable","http":null,"url":"https://naver.jp/robots.txt","bytes":0,"groups":0,"error":"cannot resolve naver.jp"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":3580,"domain":"magento.com","origin":"https://magento.com","robots":{"status":"ok","http":200,"url":"https://magento.com/robots.txt","bytes":553,"groups":1},"bots":{"GPTBot":["blocked","*",1,0],"ChatGPT-User":["blocked","*",1,0],"OAI-SearchBot":["blocked","*",1,0],"ClaudeBot":["blocked","*",1,0],"Claude-User":["blocked","*",1,0],"Claude-SearchBot":["blocked","*",1,0],"anthropic-ai":["blocked","*",1,0],"PerplexityBot":["blocked","*",1,0],"Perplexity-User":["blocked","*",1,0],"Google-Extended":["blocked","*",1,0],"Applebot-Extended":["blocked","*",1,0],"CCBot":["blocked","*",1,0],"Bytespider":["blocked","*",1,0],"Amazonbot":["blocked","*",1,0],"meta-externalagent":["blocked","*",1,0],"cohere-ai":["blocked","*",1,0],"Diffbot":["blocked","*",1,0],"ContextFetchBot":["blocked","*",1,0]},"llms":{"status":"disallowed"},"home":{"status":"disallowed"}},
{"rank":3581,"domain":"su.se","origin":"https://su.se","robots":{"status":"ok","http":200,"url":"https://www.su.se/robots.txt","bytes":492,"groups":1},"bots":{"GPTBot":["partial","*",14,0],"ChatGPT-User":["partial","*",14,0],"OAI-SearchBot":["partial","*",14,0],"ClaudeBot":["partial","*",14,0],"Claude-User":["partial","*",14,0],"Claude-SearchBot":["partial","*",14,0],"anthropic-ai":["partial","*",14,0],"PerplexityBot":["partial","*",14,0],"Perplexity-User":["partial","*",14,0],"Google-Extended":["partial","*",14,0],"Applebot-Extended":["partial","*",14,0],"CCBot":["partial","*",14,0],"Bytespider":["partial","*",14,0],"Amazonbot":["partial","*",14,0],"meta-externalagent":["partial","*",14,0],"cohere-ai":["partial","*",14,0],"Diffbot":["partial","*",14,0],"ContextFetchBot":["partial","*",14,0]},"llms":{"status":"missing","http":404,"bytes":87738},"home":{"status":"ok","http":200,"url":"https://www.su.se/","html_tokens":27146,"markdown_tokens":214,"pruned_tokens":108}},
{"rank":3582,"domain":"gazzettaufficiale.it","origin":"https://www.gazzettaufficiale.it","robots":{"status":"missing","http":404,"url":"https://www.gazzettaufficiale.it/robots.txt","bytes":6407,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"html","http":200,"bytes":246},"home":{"status":"ok","http":200,"url":"https://www.gazzettaufficiale.it/","html_tokens":10543,"markdown_tokens":1357,"pruned_tokens":71}},
{"rank":3583,"domain":"ufmg.br","origin":"https://ufmg.br","robots":{"status":"html","http":200,"url":"https://www.ufmg.br","bytes":184056,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":3584,"domain":"happy-gambler.com","origin":"https://happy-gambler.com","robots":{"status":"ok","http":200,"url":"https://happy-gambler.com/robots.txt","bytes":539,"groups":1},"bots":{"GPTBot":["partial","*",19,0],"ChatGPT-User":["partial","*",19,0],"OAI-SearchBot":["partial","*",19,0],"ClaudeBot":["partial","*",19,0],"Claude-User":["partial","*",19,0],"Claude-SearchBot":["partial","*",19,0],"anthropic-ai":["partial","*",19,0],"PerplexityBot":["partial","*",19,0],"Perplexity-User":["partial","*",19,0],"Google-Extended":["partial","*",19,0],"Applebot-Extended":["partial","*",19,0],"CCBot":["partial","*",19,0],"Bytespider":["partial","*",19,0],"Amazonbot":["partial","*",19,0],"meta-externalagent":["partial","*",19,0],"cohere-ai":["partial","*",19,0],"Diffbot":["partial","*",19,0],"ContextFetchBot":["partial","*",19,0]},"llms":{"status":"missing","http":404,"bytes":50676},"home":{"status":"ok","http":200,"url":"https://happy-gambler.com/","html_tokens":33949,"markdown_tokens":3877,"pruned_tokens":98}},
{"rank":3585,"domain":"pdx.edu","origin":"https://pdx.edu","robots":{"status":"ok","http":200,"url":"https://www.pdx.edu/robots.txt","bytes":2121,"groups":1},"bots":{"GPTBot":["partial","*",35,18],"ChatGPT-User":["partial","*",35,18],"OAI-SearchBot":["partial","*",35,18],"ClaudeBot":["partial","*",35,18],"Claude-User":["partial","*",35,18],"Claude-SearchBot":["partial","*",35,18],"anthropic-ai":["partial","*",35,18],"PerplexityBot":["partial","*",35,18],"Perplexity-User":["partial","*",35,18],"Google-Extended":["partial","*",35,18],"Applebot-Extended":["partial","*",35,18],"CCBot":["partial","*",35,18],"Bytespider":["partial","*",35,18],"Amazonbot":["partial","*",35,18],"meta-externalagent":["partial","*",35,18],"cohere-ai":["partial","*",35,18],"Diffbot":["partial","*",35,18],"ContextFetchBot":["partial","*",35,18]},"llms":{"status":"missing","http":404,"bytes":185},"home":{"status":"ok","http":200,"url":"https://www.pdx.edu/","html_tokens":40528,"markdown_tokens":479,"pruned_tokens":135}},
{"rank":3586,"domain":"volkskrant.nl","origin":"https://volkskrant.nl","robots":{"status":"ok","http":200,"url":"https://www.volkskrant.nl/robots.txt","bytes":3053,"groups":41},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["blocked","named",1,0],"OAI-SearchBot":["blocked","named",1,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",16,2],"Claude-SearchBot":["blocked","named",1,0],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["blocked","named",1,0],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["partial","*",16,2],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",16,2]},"llms":{"status":"missing","http":404,"bytes":87235},"home":{"status":"ok","http":200,"url":"https://www.volkskrant.nl/","html_tokens":249598,"markdown_tokens":484,"pruned_tokens":126}},
{"rank":3587,"domain":"loreal.com","origin":"https://loreal.com","robots":{"status":"ok","http":200,"url":"https://www.loreal.com/robots.txt","bytes":239,"groups":1},"bots":{"GPTBot":["partial","*",6,0],"ChatGPT-User":["partial","*",6,0],"OAI-SearchBot":["partial","*",6,0],"ClaudeBot":["partial","*",6,0],"Claude-User":["partial","*",6,0],"Claude-SearchBot":["partial","*",6,0],"anthropic-ai":["partial","*",6,0],"PerplexityBot":["partial","*",6,0],"Perplexity-User":["partial","*",6,0],"Google-Extended":["partial","*",6,0],"Applebot-Extended":["partial","*",6,0],"CCBot":["partial","*",6,0],"Bytespider":["partial","*",6,0],"Amazonbot":["partial","*",6,0],"meta-externalagent":["partial","*",6,0],"cohere-ai":["partial","*",6,0],"Diffbot":["partial","*",6,0],"ContextFetchBot":["partial","*",6,0]},"llms":{"status":"missing","http":404,"bytes":189026},"home":{"status":"ok","http":200,"url":"https://www.loreal.com/en/","html_tokens":51873,"markdown_tokens":72,"pruned_tokens":72}},
{"rank":3588,"domain":"data.gov.uk","origin":"https://data.gov.uk","robots":{"status":"ok","http":200,"url":"https://www.data.gov.uk/robots.txt","bytes":28,"groups":1},"bots":{"GPTBot":["partial","*",1,0],"ChatGPT-User":["partial","*",1,0],"OAI-SearchBot":["partial","*",1,0],"ClaudeBot":["partial","*",1,0],"Claude-User":["partial","*",1,0],"Claude-SearchBot":["partial","*",1,0],"anthropic-ai":["partial","*",1,0],"PerplexityBot":["partial","*",1,0],"Perplexity-User":["partial","*",1,0],"Google-Extended":["partial","*",1,0],"Applebot-Extended":["partial","*",1,0],"CCBot":["partial","*",1,0],"Bytespider":["partial","*",1,0],"Amazonbot":["partial","*",1,0],"meta-externalagent":["partial","*",1,0],"cohere-ai":["partial","*",1,0],"Diffbot":["partial","*",1,0],"ContextFetchBot":["partial","*",1,0]},"llms":{"status":"missing","http":404,"bytes":12169},"home":{"status":"ok","http":200,"url":"https://www.data.gov.uk/","html_tokens":5400,"markdown_tokens":35,"pruned_tokens":35}},
{"rank":3589,"domain":"oh.us","origin":"https://oh.us","robots":{"status":"unreachable","http":null,"url":"https://oh.us/robots.txt","bytes":0,"groups":0,"error":"cannot resolve oh.us"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":3590,"domain":"the-sun.com","origin":"https://the-sun.com","robots":{"status":"ok","http":200,"url":"https://www.the-sun.com/robots.txt","bytes":4694,"groups":2},"bots":{"GPTBot":["partial","named",99,3],"ChatGPT-User":["partial","named",99,3],"OAI-SearchBot":["partial","named",99,3],"ClaudeBot":["blocked","*",1,0],"Claude-User":["blocked","*",1,0],"Claude-SearchBot":["blocked","*",1,0],"anthropic-ai":["blocked","*",1,0],"PerplexityBot":["blocked","*",1,0],"Perplexity-User":["blocked","*",1,0],"Google-Extended":["blocked","*",1,0],"Applebot-Extended":["blocked","*",1,0],"CCBot":["blocked","*",1,0],"Bytespider":["blocked","*",1,0],"Amazonbot":["blocked","*",1,0],"meta-externalagent":["blocked","*",1,0],"cohere-ai":["blocked","*",1,0],"Diffbot":["blocked","*",1,0],"ContextFetchBot":["blocked","*",1,0]},"llms":{"status":"disallowed"},"home":{"status":"disallowed"}},
{"rank":3591,"domain":"casio.com","origin":"https://casio.com","robots":{"status":"refused","http":403,"url":"https://casio.com/robots.txt","bytes":118,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":3592,"domain":"vk.me","origin":"https://vk.me","robots":{"status":"refused","http":403,"url":"https://vk.me/robots.txt","bytes":148,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":3593,"domain":"unipd.it","origin":"https://unipd.it","robots":{"status":"ok","http":200,"url":"https://www.unipd.it/robots.txt","bytes":66,"groups":1},"bots":{"GPTBot":["allowed","*",0,1],"ChatGPT-User":["allowed","*",0,1],"OAI-SearchBot":["allowed","*",0,1],"ClaudeBot":["allowed","*",0,1],"Claude-User":["allowed","*",0,1],"Claude-SearchBot":["allowed","*",0,1],"anthropic-ai":["allowed","*",0,1],"PerplexityBot":["allowed","*",0,1],"Perplexity-User":["allowed","*",0,1],"Google-Extended":["allowed","*",0,1],"Applebot-Extended":["allowed","*",0,1],"CCBot":["allowed","*",0,1],"Bytespider":["allowed","*",0,1],"Amazonbot":["allowed","*",0,1],"meta-externalagent":["allowed","*",0,1],"cohere-ai":["allowed","*",0,1],"Diffbot":["allowed","*",0,1],"ContextFetchBot":["allowed","*",0,1]},"llms":{"status":"html","http":200,"bytes":2065352},"home":{"status":"ok","http":200,"url":"https://www.unipd.it/","html_tokens":515743,"markdown_tokens":555,"pruned_tokens":126}},
{"rank":3594,"domain":"wisconsin.gov","origin":"https://wisconsin.gov","robots":{"status":"html","http":200,"url":"https://www.wisconsin.gov/Pages/Home.aspx","bytes":113989,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":3595,"domain":"teletype.in","origin":"https://teletype.in","robots":{"status":"ok","http":200,"url":"https://teletype.in/robots.txt","bytes":275,"groups":1},"bots":{"GPTBot":["partial","*",6,0],"ChatGPT-User":["partial","*",6,0],"OAI-SearchBot":["partial","*",6,0],"ClaudeBot":["partial","*",6,0],"Claude-User":["partial","*",6,0],"Claude-SearchBot":["partial","*",6,0],"anthropic-ai":["partial","*",6,0],"PerplexityBot":["partial","*",6,0],"Perplexity-User":["partial","*",6,0],"Google-Extended":["partial","*",6,0],"Applebot-Extended":["partial","*",6,0],"CCBot":["partial","*",6,0],"Bytespider":["partial","*",6,0],"Amazonbot":["partial","*",6,0],"meta-externalagent":["partial","*",6,0],"cohere-ai":["partial","*",6,0],"Diffbot":["partial","*",6,0],"ContextFetchBot":["partial","*",6,0]},"llms":{"status":"missing","http":404,"bytes":21264},"home":{"status":"ok","http":200,"url":"https://teletype.in/","html_tokens":6414,"markdown_tokens":10,"pruned_tokens":10}},
{"rank":3596,"domain":"thetrevorproject.org","origin":"https://thetrevorproject.org","robots":{"status":"ok","http":200,"url":"https://www.thetrevorproject.org/robots.txt","bytes":182,"groups":1},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"missing","http":404,"bytes":70552},"home":{"status":"ok","http":200,"url":"https://www.thetrevorproject.org/","html_tokens":24905,"markdown_tokens":124,"pruned_tokens":124}},
{"rank":3597,"domain":"ucpress.edu","origin":"https://ucpress.edu","robots":{"status":"ok","http":200,"url":"https://www.ucpress.edu/robots.txt","bytes":24,"groups":1},"bots":{"GPTBot":["allowed","*",0,1],"ChatGPT-User":["allowed","*",0,1],"OAI-SearchBot":["allowed","*",0,1],"ClaudeBot":["allowed","*",0,1],"Claude-User":["allowed","*",0,1],"Claude-SearchBot":["allowed","*",0,1],"anthropic-ai":["allowed","*",0,1],"PerplexityBot":["allowed","*",0,1],"Perplexity-User":["allowed","*",0,1],"Google-Extended":["allowed","*",0,1],"Applebot-Extended":["allowed","*",0,1],"CCBot":["allowed","*",0,1],"Bytespider":["allowed","*",0,1],"Amazonbot":["allowed","*",0,1],"meta-externalagent":["allowed","*",0,1],"cohere-ai":["allowed","*",0,1],"Diffbot":["allowed","*",0,1],"ContextFetchBot":["allowed","*",0,1]},"llms":{"status":"missing","http":404,"bytes":6485},"home":{"status":"ok","http":200,"url":"https://www.ucpress.edu/","html_tokens":74558,"markdown_tokens":335,"pruned_tokens":164}},
{"rank":3598,"domain":"publicationethics.org","origin":"https://publicationethics.org","robots":{"status":"refused","http":403,"url":"https://publicationethics.org/robots.txt","bytes":4554,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":3599,"domain":"openwrt.org","origin":"https://openwrt.org","robots":{"status":"ok","http":200,"url":"https://openwrt.org/robots.txt","bytes":5882,"groups":9},"bots":{"GPTBot":["partial","*",117,0],"ChatGPT-User":["partial","*",117,0],"OAI-SearchBot":["partial","*",117,0],"ClaudeBot":["partial","*",117,0],"Claude-User":["partial","*",117,0],"Claude-SearchBot":["partial","*",117,0],"anthropic-ai":["partial","*",117,0],"PerplexityBot":["partial","*",117,0],"Perplexity-User":["partial","*",117,0],"Google-Extended":["partial","*",117,0],"Applebot-Extended":["partial","*",117,0],"CCBot":["partial","*",117,0],"Bytespider":["partial","*",117,0],"Amazonbot":["partial","*",117,0],"meta-externalagent":["partial","*",117,0],"cohere-ai":["partial","*",117,0],"Diffbot":["partial","*",117,0],"ContextFetchBot":["partial","*",117,0]},"llms":{"status":"html","http":200,"bytes":30272},"home":{"status":"ok","http":200,"url":"https://openwrt.org/","html_tokens":11871,"markdown_tokens":1456,"pruned_tokens":1148}},
{"rank":3600,"domain":"blm.gov","origin":"https://blm.gov","robots":{"status":"ok","http":200,"url":"https://www.blm.gov/robots.txt","bytes":2192,"groups":1},"bots":{"GPTBot":["partial","*",35,22],"ChatGPT-User":["partial","*",35,22],"OAI-SearchBot":["partial","*",35,22],"ClaudeBot":["partial","*",35,22],"Claude-User":["partial","*",35,22],"Claude-SearchBot":["partial","*",35,22],"anthropic-ai":["partial","*",35,22],"PerplexityBot":["partial","*",35,22],"Perplexity-User":["partial","*",35,22],"Google-Extended":["partial","*",35,22],"Applebot-Extended":["partial","*",35,22],"CCBot":["partial","*",35,22],"Bytespider":["partial","*",35,22],"Amazonbot":["partial","*",35,22],"meta-externalagent":["partial","*",35,22],"cohere-ai":["partial","*",35,22],"Diffbot":["partial","*",35,22],"ContextFetchBot":["partial","*",35,22]},"llms":{"status":"missing","http":404,"bytes":185},"home":{"status":"ok","http":200,"url":"https://www.blm.gov/","html_tokens":52169,"markdown_tokens":78,"pruned_tokens":51}},
{"rank":3601,"domain":"bris.ac.uk","origin":"https://bris.ac.uk","robots":{"status":"refused","http":403,"url":"https://www.bristol.ac.uk/robots.txt","bytes":3175,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":3602,"domain":"mixpanel.com","origin":"https://mixpanel.com","robots":{"status":"ok","http":200,"url":"https://mixpanel.com/robots.txt","bytes":314,"groups":1},"bots":{"GPTBot":["partial","*",8,0],"ChatGPT-User":["partial","*",8,0],"OAI-SearchBot":["partial","*",8,0],"ClaudeBot":["partial","*",8,0],"Claude-User":["partial","*",8,0],"Claude-SearchBot":["partial","*",8,0],"anthropic-ai":["partial","*",8,0],"PerplexityBot":["partial","*",8,0],"Perplexity-User":["partial","*",8,0],"Google-Extended":["partial","*",8,0],"Applebot-Extended":["partial","*",8,0],"CCBot":["partial","*",8,0],"Bytespider":["partial","*",8,0],"Amazonbot":["partial","*",8,0],"meta-externalagent":["partial","*",8,0],"cohere-ai":["partial","*",8,0],"Diffbot":["partial","*",8,0],"ContextFetchBot":["partial","*",8,0]},"llms":{"status":"missing","http":404,"bytes":3150},"home":{"status":"ok","http":200,"url":"https://mixpanel.com/home/","html_tokens":325686,"markdown_tokens":242,"pruned_tokens":132}},
{"rank":3603,"domain":"f-secure.com","origin":"https://f-secure.com","robots":{"status":"ok","http":200,"url":"https://www.f-secure.com/robots.txt","bytes":68,"groups":1},"bots":{"GPTBot":["allowed","*",0,1],"ChatGPT-User":["allowed","*",0,1],"OAI-SearchBot":["allowed","*",0,1],"ClaudeBot":["allowed","*",0,1],"Claude-User":["allowed","*",0,1],"Claude-SearchBot":["allowed","*",0,1],"anthropic-ai":["allowed","*",0,1],"PerplexityBot":["allowed","*",0,1],"Perplexity-User":["allowed","*",0,1],"Google-Extended":["allowed","*",0,1],"Applebot-Extended":["allowed","*",0,1],"CCBot":["allowed","*",0,1],"Bytespider":["allowed","*",0,1],"Amazonbot":["allowed","*",0,1],"meta-externalagent":["allowed","*",0,1],"cohere-ai":["allowed","*",0,1],"Diffbot":["allowed","*",0,1],"ContextFetchBot":["allowed","*",0,1]},"llms":{"status":"missing","http":404,"bytes":114587},"home":{"status":"ok","http":200,"url":"https://www.f-secure.com/en","html_tokens":67620,"markdown_tokens":720,"pruned_tokens":365}},
{"rank":3604,"domain":"dangdang.com","origin":"https://dangdang.com","robots":{"status":"refused","http":405,"url":"http://www.dangdang.com/robots.txt","bytes":2657,"groups":0,"error":"HTTP 405"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":3605,"domain":"aei.org","origin":"https://aei.org","robots":{"status":"ok","http":200,"url":"https://aei.org/robots.txt","bytes":1411,"groups":3},"bots":{"GPTBot":["partial","*",29,0],"ChatGPT-User":["partial","*",29,0],"OAI-SearchBot":["partial","*",29,0],"ClaudeBot":["partial","*",29,0],"Claude-User":["partial","*",29,0],"Claude-SearchBot":["partial","*",29,0],"anthropic-ai":["partial","*",29,0],"PerplexityBot":["partial","*",29,0],"Perplexity-User":["partial","*",29,0],"Google-Extended":["partial","*",29,0],"Applebot-Extended":["partial","*",29,0],"CCBot":["partial","*",29,0],"Bytespider":["partial","*",29,0],"Amazonbot":["partial","*",29,0],"meta-externalagent":["partial","*",29,0],"cohere-ai":["partial","*",29,0],"Diffbot":["partial","*",29,0],"ContextFetchBot":["partial","*",29,0]},"llms":{"status":"missing","http":404,"bytes":55726},"home":{"status":"ok","http":200,"url":"https://www.aei.org/","html_tokens":59821,"markdown_tokens":210,"pruned_tokens":105}},
{"rank":3606,"domain":"careerbuilder.com","origin":"https://careerbuilder.com","robots":{"status":"refused","http":403,"url":"https://www.careerbuilder.com/","bytes":777,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":3607,"domain":"jquery.org","origin":"https://jquery.org","robots":{"status":"ok","http":200,"url":"https://jquery.org/robots.txt","bytes":111,"groups":1},"bots":{"GPTBot":["partial","*",1,1],"ChatGPT-User":["partial","*",1,1],"OAI-SearchBot":["partial","*",1,1],"ClaudeBot":["partial","*",1,1],"Claude-User":["partial","*",1,1],"Claude-SearchBot":["partial","*",1,1],"anthropic-ai":["partial","*",1,1],"PerplexityBot":["partial","*",1,1],"Perplexity-User":["partial","*",1,1],"Google-Extended":["partial","*",1,1],"Applebot-Extended":["partial","*",1,1],"CCBot":["partial","*",1,1],"Bytespider":["partial","*",1,1],"Amazonbot":["partial","*",1,1],"meta-externalagent":["partial","*",1,1],"cohere-ai":["partial","*",1,1],"Diffbot":["partial","*",1,1],"ContextFetchBot":["partial","*",1,1]},"llms":{"status":"missing","http":404,"bytes":18480},"home":{"status":"ok","http":200,"url":"https://openjsf.org/","html_tokens":30838,"markdown_tokens":96,"pruned_tokens":96}},
{"rank":3608,"domain":"zdfheute.de","origin":"https://zdfheute.de","robots":{"status":"ok","http":200,"url":"https://www.zdfheute.de/robots.txt","bytes":619,"groups":1},"bots":{"GPTBot":["partial","named",2,0],"ChatGPT-User":["partial","named",2,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["partial","named",2,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["partial","named",2,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":99736},"home":{"status":"ok","http":200,"url":"https://www.zdfheute.de/","html_tokens":446029,"markdown_tokens":910,"pruned_tokens":460}},
{"rank":3609,"domain":"kraken.com","origin":"https://kraken.com","robots":{"status":"ok","http":200,"url":"https://www.kraken.com/robots.txt","bytes":2374,"groups":1},"bots":{"GPTBot":["partial","*",2,0],"ChatGPT-User":["partial","*",2,0],"OAI-SearchBot":["partial","*",2,0],"ClaudeBot":["partial","*",2,0],"Claude-User":["partial","*",2,0],"Claude-SearchBot":["partial","*",2,0],"anthropic-ai":["partial","*",2,0],"PerplexityBot":["partial","*",2,0],"Perplexity-User":["partial","*",2,0],"Google-Extended":["partial","*",2,0],"Applebot-Extended":["partial","*",2,0],"CCBot":["partial","*",2,0],"Bytespider":["partial","*",2,0],"Amazonbot":["partial","*",2,0],"meta-externalagent":["partial","*",2,0],"cohere-ai":["partial","*",2,0],"Diffbot":["partial","*",2,0],"ContextFetchBot":["partial","*",2,0]},"llms":{"status":"missing","http":404,"bytes":8531},"home":{"status":"ok","http":200,"url":"https://www.kraken.com/","html_tokens":352972,"markdown_tokens":434,"pruned_tokens":190}},
{"rank":3610,"domain":"myfonts.com","origin":"https://myfonts.com","robots":{"status":"ok","http":200,"url":"https://www.myfonts.com/robots.txt","bytes":293,"groups":1},"bots":{"GPTBot":["partial","*",12,0],"ChatGPT-User":["partial","*",12,0],"OAI-SearchBot":["partial","*",12,0],"ClaudeBot":["partial","*",12,0],"Claude-User":["partial","*",12,0],"Claude-SearchBot":["partial","*",12,0],"anthropic-ai":["partial","*",12,0],"PerplexityBot":["partial","*",12,0],"Perplexity-User":["partial","*",12,0],"Google-Extended":["partial","*",12,0],"Applebot-Extended":["partial","*",12,0],"CCBot":["partial","*",12,0],"Bytespider":["partial","*",12,0],"Amazonbot":["partial","*",12,0],"meta-externalagent":["partial","*",12,0],"cohere-ai":["partial","*",12,0],"Diffbot":["partial","*",12,0],"ContextFetchBot":["partial","*",12,0]},"llms":{"status":"present","http":200,"bytes":8528},"home":{"status":"ok","http":200,"url":"https://www.myfonts.com/","html_tokens":194837,"markdown_tokens":40,"pruned_tokens":39}},
{"rank":3611,"domain":"unwto.org","origin":"https://unwto.org","robots":{"status":"refused","http":403,"url":"https://www.untourism.int/robots.txt","bytes":5534,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":3612,"domain":"sae.org","origin":"https://sae.org","robots":{"status":"ok","http":200,"url":"https://www.sae.org/robots.txt","bytes":755,"groups":14},"bots":{"GPTBot":["partial","*",11,0],"ChatGPT-User":["partial","*",11,0],"OAI-SearchBot":["partial","*",11,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",11,0],"Claude-SearchBot":["partial","*",11,0],"anthropic-ai":["partial","*",11,0],"PerplexityBot":["partial","*",11,0],"Perplexity-User":["partial","*",11,0],"Google-Extended":["partial","*",11,0],"Applebot-Extended":["partial","*",11,0],"CCBot":["partial","*",11,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["partial","*",11,0],"meta-externalagent":["partial","*",11,0],"cohere-ai":["partial","*",11,0],"Diffbot":["partial","*",11,0],"ContextFetchBot":["partial","*",11,0]},"llms":{"status":"html","http":200,"bytes":15274},"home":{"status":"ok","http":200,"url":"https://www.sae.org/","html_tokens":3819,"markdown_tokens":0,"pruned_tokens":0}},
{"rank":3613,"domain":"ana.co.jp","origin":"https://ana.co.jp","robots":{"status":"timeout","http":null,"url":"https://ana.co.jp/robots.txt","bytes":0,"groups":0,"error":"timed out after 10 s"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":3614,"domain":"lapresse.ca","origin":"https://lapresse.ca","robots":{"status":"ok","http":200,"url":"https://www.lapresse.ca/robots.txt","bytes":2311,"groups":2},"bots":{"GPTBot":["partial","named",1,2],"ChatGPT-User":["partial","named",1,2],"OAI-SearchBot":["partial","named",1,2],"ClaudeBot":["partial","named",1,2],"Claude-User":["partial","*",13,0],"Claude-SearchBot":["partial","named",1,2],"anthropic-ai":["partial","named",1,2],"PerplexityBot":["partial","named",1,2],"Perplexity-User":["partial","*",13,0],"Google-Extended":["partial","named",1,2],"Applebot-Extended":["partial","named",1,2],"CCBot":["partial","named",1,2],"Bytespider":["partial","named",1,2],"Amazonbot":["partial","named",1,2],"meta-externalagent":["partial","named",1,2],"cohere-ai":["partial","*",13,0],"Diffbot":["partial","named",1,2],"ContextFetchBot":["partial","*",13,0]},"llms":{"status":"missing","http":404,"bytes":128308},"home":{"status":"ok","http":200,"url":"https://www.lapresse.ca/","html_tokens":174554,"markdown_tokens":59,"pruned_tokens":59}},
{"rank":3615,"domain":"spiceworks.com","origin":"https://spiceworks.com","robots":{"status":"ok","http":200,"url":"https://www.spiceworks.com/robots.txt","bytes":7784,"groups":18},"bots":{"GPTBot":["partial","named",1,25],"ChatGPT-User":["partial","named",1,25],"OAI-SearchBot":["partial","named",1,25],"ClaudeBot":["partial","named",1,25],"Claude-User":["partial","named",1,25],"Claude-SearchBot":["partial","named",1,25],"anthropic-ai":["partial","named",1,25],"PerplexityBot":["partial","named",1,25],"Perplexity-User":["partial","named",1,25],"Google-Extended":["blocked","*",8,0],"Applebot-Extended":["partial","named",1,25],"CCBot":["partial","named",1,25],"Bytespider":["partial","named",1,25],"Amazonbot":["partial","named",1,25],"meta-externalagent":["partial","named",1,25],"cohere-ai":["partial","named",1,25],"Diffbot":["partial","named",1,25],"ContextFetchBot":["blocked","*",8,0]},"llms":{"status":"disallowed"},"home":{"status":"disallowed"}},
{"rank":3616,"domain":"rdcu.be","origin":"https://rdcu.be","robots":{"status":"html","http":202,"url":"https://rdcu.be/robots.txt","bytes":1983,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":3617,"domain":"xuexi.cn","origin":"https://www.xuexi.cn","robots":{"status":"ok","http":200,"url":"https://www.xuexi.cn/robots.txt","bytes":3718003,"groups":1},"bots":{"GPTBot":["allowed","*",0,6481],"ChatGPT-User":["allowed","*",0,6481],"OAI-SearchBot":["allowed","*",0,6481],"ClaudeBot":["allowed","*",0,6481],"Claude-User":["allowed","*",0,6481],"Claude-SearchBot":["allowed","*",0,6481],"anthropic-ai":["allowed","*",0,6481],"PerplexityBot":["allowed","*",0,6481],"Perplexity-User":["allowed","*",0,6481],"Google-Extended":["allowed","*",0,6481],"Applebot-Extended":["allowed","*",0,6481],"CCBot":["allowed","*",0,6481],"Bytespider":["allowed","*",0,6481],"Amazonbot":["allowed","*",0,6481],"meta-externalagent":["allowed","*",0,6481],"cohere-ai":["allowed","*",0,6481],"Diffbot":["allowed","*",0,6481],"ContextFetchBot":["allowed","*",0,6481]},"llms":{"status":"html","http":200,"bytes":4979},"home":{"status":"ok","http":200,"url":"https://www.xuexi.cn/","html_tokens":7459,"markdown_tokens":0,"pruned_tokens":0}},
{"rank":3618,"domain":"serverfault.com","origin":"https://serverfault.com","robots":{"status":"refused","http":403,"url":"https://serverfault.com/robots.txt","bytes":5532,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":3619,"domain":"md.us","origin":"https://md.us","robots":{"status":"unreachable","http":null,"url":"https://md.us/robots.txt","bytes":0,"groups":0,"error":"cannot resolve md.us"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":3620,"domain":"canada.com","origin":"https://canada.com","robots":{"status":"ok","http":200,"url":"https://o.canada.com/robots.txt","bytes":997,"groups":15},"bots":{"GPTBot":["partial","*",2,6],"ChatGPT-User":["partial","*",2,6],"OAI-SearchBot":["partial","*",2,6],"ClaudeBot":["partial","*",2,6],"Claude-User":["partial","*",2,6],"Claude-SearchBot":["partial","*",2,6],"anthropic-ai":["partial","*",2,6],"PerplexityBot":["partial","*",2,6],"Perplexity-User":["partial","*",2,6],"Google-Extended":["partial","*",2,6],"Applebot-Extended":["partial","*",2,6],"CCBot":["partial","*",2,6],"Bytespider":["partial","*",2,6],"Amazonbot":["partial","*",2,6],"meta-externalagent":["partial","*",2,6],"cohere-ai":["partial","*",2,6],"Diffbot":["partial","*",2,6],"ContextFetchBot":["partial","*",2,6]},"llms":{"status":"missing","http":404,"bytes":137750},"home":{"status":"ok","http":200,"url":"https://o.canada.com/","html_tokens":92366,"markdown_tokens":156,"pruned_tokens":156}},
{"rank":3621,"domain":"flightaware.com","origin":"https://flightaware.com","robots":{"status":"ok","http":200,"url":"https://www.flightaware.com/robots.txt","bytes":2349,"groups":34},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["partial","*",17,0],"OAI-SearchBot":["partial","*",17,0],"ClaudeBot":["partial","*",17,0],"Claude-User":["partial","*",17,0],"Claude-SearchBot":["partial","*",17,0],"anthropic-ai":["partial","*",17,0],"PerplexityBot":["partial","*",17,0],"Perplexity-User":["partial","*",17,0],"Google-Extended":["partial","*",17,0],"Applebot-Extended":["partial","*",17,0],"CCBot":["partial","*",17,0],"Bytespider":["partial","*",17,0],"Amazonbot":["partial","*",17,0],"meta-externalagent":["partial","*",17,0],"cohere-ai":["partial","*",17,0],"Diffbot":["partial","*",17,0],"ContextFetchBot":["partial","*",17,0]},"llms":{"status":"missing","http":404,"bytes":2506},"home":{"status":"ok","http":200,"url":"https://www.flightaware.com/","html_tokens":33645,"markdown_tokens":164,"pruned_tokens":111}},
{"rank":3622,"domain":"walesonline.co.uk","origin":"https://walesonline.co.uk","robots":{"status":"ok","http":200,"url":"https://www.walesonline.co.uk/robots.txt","bytes":897,"groups":13},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["partial","*",9,0],"OAI-SearchBot":["blocked","named",1,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",9,0],"Claude-SearchBot":["partial","*",9,0],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["partial","*",9,0],"Google-Extended":["partial","*",9,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["partial","*",9,0],"Amazonbot":["partial","*",9,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["partial","*",9,0],"Diffbot":["partial","*",9,0],"ContextFetchBot":["partial","*",9,0]},"llms":{"status":"missing","http":404,"bytes":441944},"home":{"status":"ok","http":200,"url":"https://www.walesonline.co.uk/","html_tokens":207609,"markdown_tokens":153,"pruned_tokens":153}},
{"rank":3623,"domain":"nationalinterest.org","origin":"https://nationalinterest.org","robots":{"status":"ok","http":200,"url":"https://nationalinterest.org/robots.txt","bytes":194,"groups":1},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"http_error","http":403,"bytes":6200},"home":{"status":"http_error","http":403,"url":"https://nationalinterest.org/"}},
{"rank":3624,"domain":"sierraclub.org","origin":"https://sierraclub.org","robots":{"status":"html","http":200,"url":"https://sierraclub.org/robots.txt","bytes":212,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":3625,"domain":"wp-royal-themes.com","origin":"https://wp-royal-themes.com","robots":{"status":"refused","http":403,"url":"https://wp-royal-themes.com/robots.txt","bytes":103,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":3626,"domain":"siteground.com","origin":"https://siteground.com","robots":{"status":"ok","http":200,"url":"https://www.siteground.com/robots.txt","bytes":212,"groups":1},"bots":{"GPTBot":["partial","*",5,0],"ChatGPT-User":["partial","*",5,0],"OAI-SearchBot":["partial","*",5,0],"ClaudeBot":["partial","*",5,0],"Claude-User":["partial","*",5,0],"Claude-SearchBot":["partial","*",5,0],"anthropic-ai":["partial","*",5,0],"PerplexityBot":["partial","*",5,0],"Perplexity-User":["partial","*",5,0],"Google-Extended":["partial","*",5,0],"Applebot-Extended":["partial","*",5,0],"CCBot":["partial","*",5,0],"Bytespider":["partial","*",5,0],"Amazonbot":["partial","*",5,0],"meta-externalagent":["partial","*",5,0],"cohere-ai":["partial","*",5,0],"Diffbot":["partial","*",5,0],"ContextFetchBot":["partial","*",5,0]},"llms":{"status":"present","http":200,"bytes":31682},"home":{"status":"ok","http":200,"url":"https://www.siteground.com/","html_tokens":31469,"markdown_tokens":131,"pruned_tokens":103}},
{"rank":3627,"domain":"farfetch.com","origin":"https://farfetch.com","robots":{"status":"ok","http":200,"url":"https://www.farfetch.com/robots.txt","bytes":13357,"groups":3},"bots":{"GPTBot":["partial","*",253,84],"ChatGPT-User":["partial","*",253,84],"OAI-SearchBot":["partial","*",253,84],"ClaudeBot":["partial","*",253,84],"Claude-User":["partial","*",253,84],"Claude-SearchBot":["partial","*",253,84],"anthropic-ai":["partial","*",253,84],"PerplexityBot":["partial","*",253,84],"Perplexity-User":["partial","*",253,84],"Google-Extended":["partial","*",253,84],"Applebot-Extended":["partial","*",253,84],"CCBot":["partial","*",253,84],"Bytespider":["partial","*",253,84],"Amazonbot":["partial","*",253,84],"meta-externalagent":["partial","*",253,84],"cohere-ai":["partial","*",253,84],"Diffbot":["partial","*",253,84],"ContextFetchBot":["partial","*",253,84]},"llms":{"status":"timeout","error":"timed out after 10 s"},"home":{"status":"http_error","http":403,"url":"https://www.farfetch.com/"}},
{"rank":3628,"domain":"washu.edu","origin":"https://washu.edu","robots":{"status":"refused","http":403,"url":"https://washu.edu/robots.txt","bytes":560,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":3629,"domain":"splashthat.com","origin":"https://splashthat.com","robots":{"status":"refused","http":403,"url":"https://splashthat.com/robots.txt","bytes":774,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":3630,"domain":"secureserver.net","origin":"https://secureserver.net","robots":{"status":"ok","http":200,"url":"https://www.secureserver.net/robots.txt","bytes":26,"groups":1},"bots":{"GPTBot":["blocked","*",1,0],"ChatGPT-User":["blocked","*",1,0],"OAI-SearchBot":["blocked","*",1,0],"ClaudeBot":["blocked","*",1,0],"Claude-User":["blocked","*",1,0],"Claude-SearchBot":["blocked","*",1,0],"anthropic-ai":["blocked","*",1,0],"PerplexityBot":["blocked","*",1,0],"Perplexity-User":["blocked","*",1,0],"Google-Extended":["blocked","*",1,0],"Applebot-Extended":["blocked","*",1,0],"CCBot":["blocked","*",1,0],"Bytespider":["blocked","*",1,0],"Amazonbot":["blocked","*",1,0],"meta-externalagent":["blocked","*",1,0],"cohere-ai":["blocked","*",1,0],"Diffbot":["blocked","*",1,0],"ContextFetchBot":["blocked","*",1,0]},"llms":{"status":"disallowed"},"home":{"status":"disallowed"}},
{"rank":3631,"domain":"kbb.com","origin":"https://kbb.com","robots":{"status":"ok","http":200,"url":"https://www.kbb.com/robots.txt","bytes":4880,"groups":21},"bots":{"GPTBot":["partial","named",10,0],"ChatGPT-User":["partial","named",10,0],"OAI-SearchBot":["partial","named",10,0],"ClaudeBot":["partial","named",6,0],"Claude-User":["partial","named",10,0],"Claude-SearchBot":["partial","named",10,0],"anthropic-ai":["partial","*",34,0],"PerplexityBot":["partial","named",10,0],"Perplexity-User":["partial","named",10,0],"Google-Extended":["partial","*",34,0],"Applebot-Extended":["partial","*",34,0],"CCBot":["partial","named",10,0],"Bytespider":["partial","*",34,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["partial","*",34,0],"cohere-ai":["partial","*",34,0],"Diffbot":["partial","*",34,0],"ContextFetchBot":["partial","*",34,0]},"llms":{"status":"timeout","error":"timed out after 10 s"},"home":{"status":"timeout","error":"timed out after 15 s"}},
{"rank":3632,"domain":"oschina.net","origin":"https://oschina.net","robots":{"status":"ok","http":200,"url":"https://www.oschina.net/robots.txt","bytes":77369,"groups":1},"bots":{"GPTBot":["partial","*",5,0],"ChatGPT-User":["partial","*",5,0],"OAI-SearchBot":["partial","*",5,0],"ClaudeBot":["partial","*",5,0],"Claude-User":["partial","*",5,0],"Claude-SearchBot":["partial","*",5,0],"anthropic-ai":["partial","*",5,0],"PerplexityBot":["partial","*",5,0],"Perplexity-User":["partial","*",5,0],"Google-Extended":["partial","*",5,0],"Applebot-Extended":["partial","*",5,0],"CCBot":["partial","*",5,0],"Bytespider":["partial","*",5,0],"Amazonbot":["partial","*",5,0],"meta-externalagent":["partial","*",5,0],"cohere-ai":["partial","*",5,0],"Diffbot":["partial","*",5,0],"ContextFetchBot":["partial","*",5,0]},"llms":{"status":"missing","http":404,"bytes":1548},"home":{"status":"ok","http":200,"url":"https://www.oschina.net/","html_tokens":829,"markdown_tokens":0,"pruned_tokens":0}},
{"rank":3633,"domain":"miguvideo.com","origin":"https://miguvideo.com","robots":{"status":"refused","http":403,"url":"https://www.miguvideo.com/robots.txt","bytes":146,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":3634,"domain":"pg.com","origin":"https://pg.com","robots":{"status":"ok","http":200,"url":"https://www.pg.com/robots.txt","bytes":71,"groups":1},"bots":{"GPTBot":["allowed","*",0,1],"ChatGPT-User":["allowed","*",0,1],"OAI-SearchBot":["allowed","*",0,1],"ClaudeBot":["allowed","*",0,1],"Claude-User":["allowed","*",0,1],"Claude-SearchBot":["allowed","*",0,1],"anthropic-ai":["allowed","*",0,1],"PerplexityBot":["allowed","*",0,1],"Perplexity-User":["allowed","*",0,1],"Google-Extended":["allowed","*",0,1],"Applebot-Extended":["allowed","*",0,1],"CCBot":["allowed","*",0,1],"Bytespider":["allowed","*",0,1],"Amazonbot":["allowed","*",0,1],"meta-externalagent":["allowed","*",0,1],"cohere-ai":["allowed","*",0,1],"Diffbot":["allowed","*",0,1],"ContextFetchBot":["allowed","*",0,1]},"llms":{"status":"html","http":200,"bytes":9084},"home":{"status":"ok","http":200,"url":"https://us.pg.com/","html_tokens":73529,"markdown_tokens":254,"pruned_tokens":168}},
{"rank":3635,"domain":"bundeswirtschaftsministerium.de","origin":"https://bundeswirtschaftsministerium.de","robots":{"status":"missing","http":404,"url":"https://www.bundeswirtschaftsministerium.de/robots.txt","bytes":133596,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":133493},"home":{"status":"ok","http":200,"url":"https://www.bundeswirtschaftsministerium.de/Navigation/DE/Home/home.html","html_tokens":29604,"markdown_tokens":13,"pruned_tokens":13}},
{"rank":3636,"domain":"code.org","origin":"https://code.org","robots":{"status":"ok","http":200,"url":"https://code.org/robots.txt","bytes":37,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":130399},"home":{"status":"ok","http":200,"url":"https://code.org/en-US","html_tokens":302163,"markdown_tokens":722,"pruned_tokens":196}},
{"rank":3637,"domain":"linux.com","origin":"https://linux.com","robots":{"status":"refused","http":403,"url":"https://linux.com/robots.txt","bytes":560,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":3638,"domain":"poets.org","origin":"https://poets.org","robots":{"status":"ok","http":200,"url":"https://poets.org/robots.txt","bytes":1506,"groups":1},"bots":{"GPTBot":["partial","*",34,18],"ChatGPT-User":["partial","*",34,18],"OAI-SearchBot":["partial","*",34,18],"ClaudeBot":["partial","*",34,18],"Claude-User":["partial","*",34,18],"Claude-SearchBot":["partial","*",34,18],"anthropic-ai":["partial","*",34,18],"PerplexityBot":["partial","*",34,18],"Perplexity-User":["partial","*",34,18],"Google-Extended":["partial","*",34,18],"Applebot-Extended":["partial","*",34,18],"CCBot":["partial","*",34,18],"Bytespider":["partial","*",34,18],"Amazonbot":["partial","*",34,18],"meta-externalagent":["partial","*",34,18],"cohere-ai":["partial","*",34,18],"Diffbot":["partial","*",34,18],"ContextFetchBot":["partial","*",34,18]},"llms":{"status":"missing","http":404,"bytes":183},"home":{"status":"ok","http":200,"url":"https://poets.org/","html_tokens":17763,"markdown_tokens":490,"pruned_tokens":114}},
{"rank":3639,"domain":"usask.ca","origin":"https://usask.ca","robots":{"status":"ok","http":200,"url":"https://www.usask.ca/robots.txt","bytes":420,"groups":1},"bots":{"GPTBot":["partial","*",10,0],"ChatGPT-User":["partial","*",10,0],"OAI-SearchBot":["partial","*",10,0],"ClaudeBot":["partial","*",10,0],"Claude-User":["partial","*",10,0],"Claude-SearchBot":["partial","*",10,0],"anthropic-ai":["partial","*",10,0],"PerplexityBot":["partial","*",10,0],"Perplexity-User":["partial","*",10,0],"Google-Extended":["partial","*",10,0],"Applebot-Extended":["partial","*",10,0],"CCBot":["partial","*",10,0],"Bytespider":["partial","*",10,0],"Amazonbot":["partial","*",10,0],"meta-externalagent":["partial","*",10,0],"cohere-ai":["partial","*",10,0],"Diffbot":["partial","*",10,0],"ContextFetchBot":["partial","*",10,0]},"llms":{"status":"missing","http":404,"bytes":8607},"home":{"status":"ok","http":200,"url":"https://www.usask.ca/","html_tokens":36079,"markdown_tokens":99,"pruned_tokens":99}},
{"rank":3640,"domain":"activemind.de","origin":"https://activemind.de","robots":{"status":"ok","http":200,"url":"https://www.activemind.de/robots.txt","bytes":297,"groups":2},"bots":{"GPTBot":["partial","*",5,0],"ChatGPT-User":["partial","*",5,0],"OAI-SearchBot":["partial","*",5,0],"ClaudeBot":["partial","*",5,0],"Claude-User":["partial","*",5,0],"Claude-SearchBot":["partial","*",5,0],"anthropic-ai":["partial","*",5,0],"PerplexityBot":["partial","*",5,0],"Perplexity-User":["partial","*",5,0],"Google-Extended":["partial","*",5,0],"Applebot-Extended":["partial","*",5,0],"CCBot":["partial","*",5,0],"Bytespider":["partial","*",5,0],"Amazonbot":["partial","*",5,0],"meta-externalagent":["partial","*",5,0],"cohere-ai":["partial","*",5,0],"Diffbot":["partial","*",5,0],"ContextFetchBot":["partial","*",5,0]},"llms":{"status":"present","http":200,"bytes":6066},"home":{"status":"ok","http":200,"url":"https://www.activemind.de/","html_tokens":75047,"markdown_tokens":616,"pruned_tokens":139}},
{"rank":3641,"domain":"militarytimes.com","origin":"https://militarytimes.com","robots":{"status":"ok","http":200,"url":"https://www.militarytimes.com/robots.txt","bytes":258,"groups":1},"bots":{"GPTBot":["partial","*",3,0],"ChatGPT-User":["partial","*",3,0],"OAI-SearchBot":["partial","*",3,0],"ClaudeBot":["partial","*",3,0],"Claude-User":["partial","*",3,0],"Claude-SearchBot":["partial","*",3,0],"anthropic-ai":["partial","*",3,0],"PerplexityBot":["partial","*",3,0],"Perplexity-User":["partial","*",3,0],"Google-Extended":["partial","*",3,0],"Applebot-Extended":["partial","*",3,0],"CCBot":["partial","*",3,0],"Bytespider":["partial","*",3,0],"Amazonbot":["partial","*",3,0],"meta-externalagent":["partial","*",3,0],"cohere-ai":["partial","*",3,0],"Diffbot":["partial","*",3,0],"ContextFetchBot":["partial","*",3,0]},"llms":{"status":"missing","http":404,"bytes":67294},"home":{"status":"ok","http":200,"url":"https://www.militarytimes.com/","html_tokens":129551,"markdown_tokens":271,"pruned_tokens":116}},
{"rank":3642,"domain":"eweek.com","origin":"https://eweek.com","robots":{"status":"ok","http":200,"url":"https://www.eweek.com/robots.txt","bytes":131,"groups":1},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"present","http":200,"bytes":21034},"home":{"status":"http_error","http":403,"url":"https://www.eweek.com/"}},
{"rank":3643,"domain":"motortrend.com","origin":"https://motortrend.com","robots":{"status":"ok","http":200,"url":"https://motortrend.com/robots.txt","bytes":2463,"groups":4},"bots":{"GPTBot":["partial","named",1,0],"ChatGPT-User":["partial","named",1,0],"OAI-SearchBot":["partial","named",1,0],"ClaudeBot":["partial","*",64,0],"Claude-User":["partial","*",64,0],"Claude-SearchBot":["partial","*",64,0],"anthropic-ai":["partial","*",64,0],"PerplexityBot":["partial","*",64,0],"Perplexity-User":["partial","*",64,0],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["partial","*",64,0],"CCBot":["blocked","named",1,0],"Bytespider":["partial","*",64,0],"Amazonbot":["partial","*",64,0],"meta-externalagent":["partial","*",64,0],"cohere-ai":["partial","*",64,0],"Diffbot":["partial","*",64,0],"ContextFetchBot":["partial","*",64,0]},"llms":{"status":"missing","http":404,"bytes":28127},"home":{"status":"ok","http":200,"url":"https://www.motortrend.com/","html_tokens":361742,"markdown_tokens":27,"pruned_tokens":27}},
{"rank":3644,"domain":"turkishairlines.com","origin":"https://turkishairlines.com","robots":{"status":"timeout","http":null,"url":"https://turkishairlines.com/robots.txt","bytes":0,"groups":0,"error":"timed out after 10 s"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":3645,"domain":"neurology.org","origin":"https://neurology.org","robots":{"status":"ok","http":200,"url":"https://www.neurology.org/robots.txt","bytes":780,"groups":3},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["partial","*",18,6],"OAI-SearchBot":["partial","*",18,6],"ClaudeBot":["partial","*",18,6],"Claude-User":["partial","*",18,6],"Claude-SearchBot":["partial","*",18,6],"anthropic-ai":["partial","*",18,6],"PerplexityBot":["partial","*",18,6],"Perplexity-User":["partial","*",18,6],"Google-Extended":["partial","*",18,6],"Applebot-Extended":["partial","*",18,6],"CCBot":["partial","*",18,6],"Bytespider":["partial","*",18,6],"Amazonbot":["partial","*",18,6],"meta-externalagent":["partial","*",18,6],"cohere-ai":["partial","*",18,6],"Diffbot":["partial","*",18,6],"ContextFetchBot":["partial","*",18,6]},"llms":{"status":"http_error","http":403,"bytes":5545},"home":{"status":"http_error","http":403,"url":"https://neurology.org/"}},
{"rank":3646,"domain":"avclub.com","origin":"https://avclub.com","robots":{"status":"ok","http":200,"url":"https://avclub.com/robots.txt","bytes":576,"groups":4},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["blocked","named",1,0],"OAI-SearchBot":["partial","*",2,0],"ClaudeBot":["partial","*",2,0],"Claude-User":["partial","*",2,0],"Claude-SearchBot":["partial","*",2,0],"anthropic-ai":["partial","*",2,0],"PerplexityBot":["partial","*",2,0],"Perplexity-User":["partial","*",2,0],"Google-Extended":["partial","*",2,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["partial","*",2,0],"Bytespider":["partial","*",2,0],"Amazonbot":["partial","*",2,0],"meta-externalagent":["partial","*",2,0],"cohere-ai":["partial","*",2,0],"Diffbot":["partial","*",2,0],"ContextFetchBot":["partial","*",2,0]},"llms":{"status":"html","http":200,"bytes":207880},"home":{"status":"ok","http":200,"url":"https://www.avclub.com/","html_tokens":51956,"markdown_tokens":5,"pruned_tokens":5}},
{"rank":3647,"domain":"uab.edu","origin":"https://uab.edu","robots":{"status":"ok","http":200,"url":"https://www.uab.edu/robots.txt","bytes":11311,"groups":1},"bots":{"GPTBot":["partial","*",339,15],"ChatGPT-User":["partial","*",339,15],"OAI-SearchBot":["partial","*",339,15],"ClaudeBot":["partial","*",339,15],"Claude-User":["partial","*",339,15],"Claude-SearchBot":["partial","*",339,15],"anthropic-ai":["partial","*",339,15],"PerplexityBot":["partial","*",339,15],"Perplexity-User":["partial","*",339,15],"Google-Extended":["partial","*",339,15],"Applebot-Extended":["partial","*",339,15],"CCBot":["partial","*",339,15],"Bytespider":["partial","*",339,15],"Amazonbot":["partial","*",339,15],"meta-externalagent":["partial","*",339,15],"cohere-ai":["partial","*",339,15],"Diffbot":["partial","*",339,15],"ContextFetchBot":["partial","*",339,15]},"llms":{"status":"timeout","error":"timed out after 10 s"},"home":{"status":"timeout","error":"timed out after 15 s"}},
{"rank":3648,"domain":"download.com","origin":"https://download.com","robots":{"status":"refused","http":412,"url":"https://download.cnet.com","bytes":0,"groups":0,"error":"HTTP 412"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":3649,"domain":"cocolog-nifty.com","origin":"https://www.cocolog-nifty.com","robots":{"status":"missing","http":404,"url":"https://www.cocolog-nifty.com/robots.txt","bytes":2992,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":2992},"home":{"status":"ok","http":200,"url":"https://www.cocolog-nifty.com/","html_tokens":5078,"markdown_tokens":18,"pruned_tokens":18}},
{"rank":3650,"domain":"here.com","origin":"https://here.com","robots":{"status":"ok","http":200,"url":"https://www.here.com/robots.txt","bytes":1734,"groups":1},"bots":{"GPTBot":["partial","*",26,18],"ChatGPT-User":["partial","*",26,18],"OAI-SearchBot":["partial","*",26,18],"ClaudeBot":["partial","*",26,18],"Claude-User":["partial","*",26,18],"Claude-SearchBot":["partial","*",26,18],"anthropic-ai":["partial","*",26,18],"PerplexityBot":["partial","*",26,18],"Perplexity-User":["partial","*",26,18],"Google-Extended":["partial","*",26,18],"Applebot-Extended":["partial","*",26,18],"CCBot":["partial","*",26,18],"Bytespider":["partial","*",26,18],"Amazonbot":["partial","*",26,18],"meta-externalagent":["partial","*",26,18],"cohere-ai":["partial","*",26,18],"Diffbot":["partial","*",26,18],"ContextFetchBot":["partial","*",26,18]},"llms":{"status":"present","http":200,"bytes":10203},"home":{"status":"ok","http":200,"url":"https://www.here.com/","html_tokens":64288,"markdown_tokens":68,"pruned_tokens":68}},
{"rank":3651,"domain":"uniroma1.it","origin":"https://www.uniroma1.it","robots":{"status":"ok","http":200,"url":"https://www.uniroma1.it/robots.txt","bytes":2189,"groups":1},"bots":{"GPTBot":["partial","*",36,32],"ChatGPT-User":["partial","*",36,32],"OAI-SearchBot":["partial","*",36,32],"ClaudeBot":["partial","*",36,32],"Claude-User":["partial","*",36,32],"Claude-SearchBot":["partial","*",36,32],"anthropic-ai":["partial","*",36,32],"PerplexityBot":["partial","*",36,32],"Perplexity-User":["partial","*",36,32],"Google-Extended":["partial","*",36,32],"Applebot-Extended":["partial","*",36,32],"CCBot":["partial","*",36,32],"Bytespider":["partial","*",36,32],"Amazonbot":["partial","*",36,32],"meta-externalagent":["partial","*",36,32],"cohere-ai":["partial","*",36,32],"Diffbot":["partial","*",36,32],"ContextFetchBot":["partial","*",36,32]},"llms":{"status":"missing","http":404,"bytes":291},"home":{"status":"ok","http":200,"url":"https://www.uniroma1.it/it/","html_tokens":128072,"markdown_tokens":210,"pruned_tokens":166}},
{"rank":3652,"domain":"c2.com","origin":"https://c2.com","robots":{"status":"ok","http":200,"url":"https://c2.com/robots.txt","bytes":2843,"groups":1},"bots":{"GPTBot":["partial","*",103,0],"ChatGPT-User":["partial","*",103,0],"OAI-SearchBot":["partial","*",103,0],"ClaudeBot":["partial","*",103,0],"Claude-User":["partial","*",103,0],"Claude-SearchBot":["partial","*",103,0],"anthropic-ai":["partial","*",103,0],"PerplexityBot":["partial","*",103,0],"Perplexity-User":["partial","*",103,0],"Google-Extended":["partial","*",103,0],"Applebot-Extended":["partial","*",103,0],"CCBot":["partial","*",103,0],"Bytespider":["partial","*",103,0],"Amazonbot":["partial","*",103,0],"meta-externalagent":["partial","*",103,0],"cohere-ai":["partial","*",103,0],"Diffbot":["partial","*",103,0],"ContextFetchBot":["partial","*",103,0]},"llms":{"status":"missing","http":404,"bytes":196},"home":{"status":"ok","http":200,"url":"https://c2.com/","html_tokens":749,"markdown_tokens":281,"pruned_tokens":158}},
{"rank":3653,"domain":"bundesgesundheitsministerium.de","origin":"https://bundesgesundheitsministerium.de","robots":{"status":"ok","http":200,"url":"https://www.bundesgesundheitsministerium.de//robots.txt","bytes":815,"groups":1},"bots":{"GPTBot":["partial","*",14,3],"ChatGPT-User":["partial","*",14,3],"OAI-SearchBot":["partial","*",14,3],"ClaudeBot":["partial","*",14,3],"Claude-User":["partial","*",14,3],"Claude-SearchBot":["partial","*",14,3],"anthropic-ai":["partial","*",14,3],"PerplexityBot":["partial","*",14,3],"Perplexity-User":["partial","*",14,3],"Google-Extended":["partial","*",14,3],"Applebot-Extended":["partial","*",14,3],"CCBot":["partial","*",14,3],"Bytespider":["partial","*",14,3],"Amazonbot":["partial","*",14,3],"meta-externalagent":["partial","*",14,3],"cohere-ai":["partial","*",14,3],"Diffbot":["partial","*",14,3],"ContextFetchBot":["partial","*",14,3]},"llms":{"status":"missing","http":404,"bytes":296616},"home":{"status":"ok","http":200,"url":"https://www.bundesgesundheitsministerium.de//","html_tokens":123920,"markdown_tokens":99,"pruned_tokens":99}},
{"rank":3654,"domain":"countryliving.com","origin":"https://countryliving.com","robots":{"status":"ok","http":200,"url":"https://countryliving.com/robots.txt","bytes":1012,"groups":4},"bots":{"GPTBot":["partial","named",32,0],"ChatGPT-User":["partial","named",32,0],"OAI-SearchBot":["partial","named",32,0],"ClaudeBot":["partial","*",19,0],"Claude-User":["partial","*",19,0],"Claude-SearchBot":["partial","*",19,0],"anthropic-ai":["partial","*",19,0],"PerplexityBot":["partial","*",19,0],"Perplexity-User":["partial","*",19,0],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["partial","*",19,0],"CCBot":["partial","*",19,0],"Bytespider":["partial","*",19,0],"Amazonbot":["partial","*",19,0],"meta-externalagent":["partial","*",19,0],"cohere-ai":["partial","*",19,0],"Diffbot":["partial","*",19,0],"ContextFetchBot":["partial","*",19,0]},"llms":{"status":"missing","http":404,"bytes":84},"home":{"status":"ok","http":200,"url":"https://www.countryliving.com/","html_tokens":196661,"markdown_tokens":208,"pruned_tokens":98}},
{"rank":3655,"domain":"golem.de","origin":"https://golem.de","robots":{"status":"ok","http":200,"url":"https://www.golem.de/robots.txt","bytes":3647,"groups":6},"bots":{"GPTBot":["partial","named",1,9],"ChatGPT-User":["partial","*",5,0],"OAI-SearchBot":["partial","named",4,0],"ClaudeBot":["partial","named",1,9],"Claude-User":["partial","*",5,0],"Claude-SearchBot":["partial","named",4,0],"anthropic-ai":["partial","*",5,0],"PerplexityBot":["partial","named",1,9],"Perplexity-User":["partial","*",5,0],"Google-Extended":["partial","*",5,0],"Applebot-Extended":["partial","named",1,9],"CCBot":["partial","*",5,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["partial","named",1,9],"cohere-ai":["partial","*",5,0],"Diffbot":["partial","named",1,9],"ContextFetchBot":["partial","*",5,0]},"llms":{"status":"missing","http":404,"bytes":24758},"home":{"status":"ok","http":200,"url":"https://www.golem.de/","html_tokens":179404,"markdown_tokens":808,"pruned_tokens":39}},
{"rank":3656,"domain":"slides.com","origin":"https://slides.com","robots":{"status":"ok","http":200,"url":"https://slides.com/robots.txt","bytes":245,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"present","http":200,"bytes":1617},"home":{"status":"ok","http":200,"url":"https://slides.com/","html_tokens":82120,"markdown_tokens":391,"pruned_tokens":151}},
{"rank":3657,"domain":"ri.gov","origin":"https://ri.gov","robots":{"status":"ok","http":200,"url":"https://www.ri.gov/robots.txt","bytes":47,"groups":1},"bots":{"GPTBot":["partial","*",1,0],"ChatGPT-User":["partial","*",1,0],"OAI-SearchBot":["partial","*",1,0],"ClaudeBot":["partial","*",1,0],"Claude-User":["partial","*",1,0],"Claude-SearchBot":["partial","*",1,0],"anthropic-ai":["partial","*",1,0],"PerplexityBot":["partial","*",1,0],"Perplexity-User":["partial","*",1,0],"Google-Extended":["partial","*",1,0],"Applebot-Extended":["partial","*",1,0],"CCBot":["partial","*",1,0],"Bytespider":["partial","*",1,0],"Amazonbot":["partial","*",1,0],"meta-externalagent":["partial","*",1,0],"cohere-ai":["partial","*",1,0],"Diffbot":["partial","*",1,0],"ContextFetchBot":["partial","*",1,0]},"llms":{"status":"missing","http":404,"bytes":22305},"home":{"status":"ok","http":200,"url":"https://www.ri.gov/","html_tokens":6293,"markdown_tokens":1,"pruned_tokens":1}},
{"rank":3658,"domain":"lacounty.gov","origin":"https://lacounty.gov","robots":{"status":"ok","http":200,"url":"https://lacounty.gov/robots.txt","bytes":162,"groups":2},"bots":{"GPTBot":["partial","*",4,0],"ChatGPT-User":["partial","*",4,0],"OAI-SearchBot":["partial","*",4,0],"ClaudeBot":["partial","*",4,0],"Claude-User":["partial","*",4,0],"Claude-SearchBot":["partial","*",4,0],"anthropic-ai":["partial","*",4,0],"PerplexityBot":["partial","*",4,0],"Perplexity-User":["partial","*",4,0],"Google-Extended":["partial","*",4,0],"Applebot-Extended":["partial","*",4,0],"CCBot":["partial","*",4,0],"Bytespider":["partial","*",4,0],"Amazonbot":["partial","*",4,0],"meta-externalagent":["partial","*",4,0],"cohere-ai":["partial","*",4,0],"Diffbot":["partial","*",4,0],"ContextFetchBot":["partial","*",4,0]},"llms":{"status":"missing","http":404,"bytes":60},"home":{"status":"ok","http":200,"url":"https://lacounty.gov/","html_tokens":82737,"markdown_tokens":139,"pruned_tokens":139}},
{"rank":3659,"domain":"fes.de","origin":"https://fes.de","robots":{"status":"ok","http":200,"url":"https://www.fes.de/robots.txt","bytes":94,"groups":2},"bots":{"GPTBot":["partial","*",2,0],"ChatGPT-User":["partial","*",2,0],"OAI-SearchBot":["partial","*",2,0],"ClaudeBot":["partial","*",2,0],"Claude-User":["partial","*",2,0],"Claude-SearchBot":["partial","*",2,0],"anthropic-ai":["partial","*",2,0],"PerplexityBot":["partial","*",2,0],"Perplexity-User":["partial","*",2,0],"Google-Extended":["partial","*",2,0],"Applebot-Extended":["partial","*",2,0],"CCBot":["partial","*",2,0],"Bytespider":["partial","*",2,0],"Amazonbot":["partial","*",2,0],"meta-externalagent":["partial","*",2,0],"cohere-ai":["partial","*",2,0],"Diffbot":["partial","*",2,0],"ContextFetchBot":["partial","*",2,0]},"llms":{"status":"missing","http":404,"bytes":228616},"home":{"status":"ok","http":200,"url":"https://www.fes.de/","html_tokens":69018,"markdown_tokens":80,"pruned_tokens":80}},
{"rank":3660,"domain":"broadwayworld.com","origin":"https://broadwayworld.com","robots":{"status":"ok","http":200,"url":"https://www.broadwayworld.com/robots.txt","bytes":1368,"groups":1},"bots":{"GPTBot":["partial","*",44,1],"ChatGPT-User":["partial","*",44,1],"OAI-SearchBot":["partial","*",44,1],"ClaudeBot":["partial","*",44,1],"Claude-User":["partial","*",44,1],"Claude-SearchBot":["partial","*",44,1],"anthropic-ai":["partial","*",44,1],"PerplexityBot":["partial","*",44,1],"Perplexity-User":["partial","*",44,1],"Google-Extended":["partial","*",44,1],"Applebot-Extended":["partial","*",44,1],"CCBot":["partial","*",44,1],"Bytespider":["partial","*",44,1],"Amazonbot":["partial","*",44,1],"meta-externalagent":["partial","*",44,1],"cohere-ai":["partial","*",44,1],"Diffbot":["partial","*",44,1],"ContextFetchBot":["partial","*",44,1]},"llms":{"status":"missing","http":404,"bytes":113662},"home":{"status":"ok","http":200,"url":"https://www.broadwayworld.com/","html_tokens":68778,"markdown_tokens":39,"pruned_tokens":39}},
{"rank":3661,"domain":"optimizely.com","origin":"https://optimizely.com","robots":{"status":"ok","http":200,"url":"https://www.optimizely.com/robots.txt","bytes":207,"groups":1},"bots":{"GPTBot":["partial","*",7,0],"ChatGPT-User":["partial","*",7,0],"OAI-SearchBot":["partial","*",7,0],"ClaudeBot":["partial","*",7,0],"Claude-User":["partial","*",7,0],"Claude-SearchBot":["partial","*",7,0],"anthropic-ai":["partial","*",7,0],"PerplexityBot":["partial","*",7,0],"Perplexity-User":["partial","*",7,0],"Google-Extended":["partial","*",7,0],"Applebot-Extended":["partial","*",7,0],"CCBot":["partial","*",7,0],"Bytespider":["partial","*",7,0],"Amazonbot":["partial","*",7,0],"meta-externalagent":["partial","*",7,0],"cohere-ai":["partial","*",7,0],"Diffbot":["partial","*",7,0],"ContextFetchBot":["partial","*",7,0]},"llms":{"status":"present","http":200,"bytes":10751},"home":{"status":"ok","http":200,"url":"https://www.optimizely.com/","html_tokens":108961,"markdown_tokens":66,"pruned_tokens":66}},
{"rank":3662,"domain":"uark.edu","origin":"https://uark.edu","robots":{"status":"ok","http":200,"url":"https://www.uark.edu/robots.txt","bytes":59,"groups":1},"bots":{"GPTBot":["partial","*",2,0],"ChatGPT-User":["partial","*",2,0],"OAI-SearchBot":["partial","*",2,0],"ClaudeBot":["partial","*",2,0],"Claude-User":["partial","*",2,0],"Claude-SearchBot":["partial","*",2,0],"anthropic-ai":["partial","*",2,0],"PerplexityBot":["partial","*",2,0],"Perplexity-User":["partial","*",2,0],"Google-Extended":["partial","*",2,0],"Applebot-Extended":["partial","*",2,0],"CCBot":["partial","*",2,0],"Bytespider":["partial","*",2,0],"Amazonbot":["partial","*",2,0],"meta-externalagent":["partial","*",2,0],"cohere-ai":["partial","*",2,0],"Diffbot":["partial","*",2,0],"ContextFetchBot":["partial","*",2,0]},"llms":{"status":"present","http":200,"bytes":4958},"home":{"status":"ok","http":200,"url":"https://www.uark.edu/","html_tokens":22054,"markdown_tokens":239,"pruned_tokens":172}},
{"rank":3663,"domain":"elifesciences.org","origin":"https://elifesciences.org","robots":{"status":"ok","http":200,"url":"https://elifesciences.org/robots.txt","bytes":473,"groups":8},"bots":{"GPTBot":["partial","*",2,0],"ChatGPT-User":["partial","*",2,0],"OAI-SearchBot":["partial","*",2,0],"ClaudeBot":["partial","*",2,0],"Claude-User":["partial","*",2,0],"Claude-SearchBot":["partial","*",2,0],"anthropic-ai":["partial","*",2,0],"PerplexityBot":["partial","*",2,0],"Perplexity-User":["partial","*",2,0],"Google-Extended":["partial","*",2,0],"Applebot-Extended":["partial","*",2,0],"CCBot":["partial","*",2,0],"Bytespider":["partial","*",2,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["partial","*",2,0],"cohere-ai":["partial","*",2,0],"Diffbot":["partial","*",2,0],"ContextFetchBot":["partial","*",2,0]},"llms":{"status":"missing","http":404,"bytes":89769},"home":{"status":"ok","http":200,"url":"https://elifesciences.org/","html_tokens":37212,"markdown_tokens":83,"pruned_tokens":22}},
{"rank":3664,"domain":"ecosia.org","origin":"https://ecosia.org","robots":{"status":"ok","http":200,"url":"https://www.ecosia.org/robots.txt","bytes":518,"groups":3},"bots":{"GPTBot":["partial","*",19,0],"ChatGPT-User":["partial","*",19,0],"OAI-SearchBot":["partial","*",19,0],"ClaudeBot":["partial","*",19,0],"Claude-User":["partial","*",19,0],"Claude-SearchBot":["partial","*",19,0],"anthropic-ai":["partial","*",19,0],"PerplexityBot":["partial","*",19,0],"Perplexity-User":["partial","*",19,0],"Google-Extended":["partial","*",19,0],"Applebot-Extended":["partial","*",19,0],"CCBot":["partial","*",19,0],"Bytespider":["partial","*",19,0],"Amazonbot":["partial","*",19,0],"meta-externalagent":["partial","*",19,0],"cohere-ai":["partial","*",19,0],"Diffbot":["partial","*",19,0],"ContextFetchBot":["partial","*",19,0]},"llms":{"status":"http_error","http":403,"bytes":4401},"home":{"status":"http_error","http":403,"url":"https://www.ecosia.org/"}},
{"rank":3665,"domain":"uni-mainz.de","origin":"https://uni-mainz.de","robots":{"status":"html","http":200,"url":"https://www.uni-mainz.de","bytes":235918,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":3666,"domain":"centos-webpanel.com","origin":"https://centos-webpanel.com","robots":{"status":"ok","http":200,"url":"https://centos-webpanel.com/robots.txt","bytes":116,"groups":1},"bots":{"GPTBot":["partial","*",1,1],"ChatGPT-User":["partial","*",1,1],"OAI-SearchBot":["partial","*",1,1],"ClaudeBot":["partial","*",1,1],"Claude-User":["partial","*",1,1],"Claude-SearchBot":["partial","*",1,1],"anthropic-ai":["partial","*",1,1],"PerplexityBot":["partial","*",1,1],"Perplexity-User":["partial","*",1,1],"Google-Extended":["partial","*",1,1],"Applebot-Extended":["partial","*",1,1],"CCBot":["partial","*",1,1],"Bytespider":["partial","*",1,1],"Amazonbot":["partial","*",1,1],"meta-externalagent":["partial","*",1,1],"cohere-ai":["partial","*",1,1],"Diffbot":["partial","*",1,1],"ContextFetchBot":["partial","*",1,1]},"llms":{"status":"missing","http":404,"bytes":1172},"home":{"status":"ok","http":200,"url":"https://centos-webpanel.com/","html_tokens":7778,"markdown_tokens":373,"pruned_tokens":342}},
{"rank":3667,"domain":"1tv.ru","origin":"https://1tv.ru","robots":{"status":"ok","http":200,"url":"https://www.1tv.ru/robots.txt","bytes":234,"groups":2},"bots":{"GPTBot":["partial","*",5,0],"ChatGPT-User":["partial","*",5,0],"OAI-SearchBot":["partial","*",5,0],"ClaudeBot":["partial","*",5,0],"Claude-User":["partial","*",5,0],"Claude-SearchBot":["partial","*",5,0],"anthropic-ai":["partial","*",5,0],"PerplexityBot":["partial","*",5,0],"Perplexity-User":["partial","*",5,0],"Google-Extended":["partial","*",5,0],"Applebot-Extended":["partial","*",5,0],"CCBot":["partial","*",5,0],"Bytespider":["partial","*",5,0],"Amazonbot":["partial","*",5,0],"meta-externalagent":["partial","*",5,0],"cohere-ai":["partial","*",5,0],"Diffbot":["partial","*",5,0],"ContextFetchBot":["partial","*",5,0]},"llms":{"status":"html","http":200,"bytes":434626},"home":{"status":"ok","http":200,"url":"https://www.1tv.ru/","html_tokens":104503,"markdown_tokens":669,"pruned_tokens":124}},
{"rank":3668,"domain":"transip.eu","origin":"https://transip.eu","robots":{"status":"ok","http":200,"url":"https://www.transip.eu/robots.txt","bytes":132,"groups":1},"bots":{"GPTBot":["partial","*",2,1],"ChatGPT-User":["partial","*",2,1],"OAI-SearchBot":["partial","*",2,1],"ClaudeBot":["partial","*",2,1],"Claude-User":["partial","*",2,1],"Claude-SearchBot":["partial","*",2,1],"anthropic-ai":["partial","*",2,1],"PerplexityBot":["partial","*",2,1],"Perplexity-User":["partial","*",2,1],"Google-Extended":["partial","*",2,1],"Applebot-Extended":["partial","*",2,1],"CCBot":["partial","*",2,1],"Bytespider":["partial","*",2,1],"Amazonbot":["partial","*",2,1],"meta-externalagent":["partial","*",2,1],"cohere-ai":["partial","*",2,1],"Diffbot":["partial","*",2,1],"ContextFetchBot":["partial","*",2,1]},"llms":{"status":"missing","http":404,"bytes":212017},"home":{"status":"ok","http":200,"url":"https://www.transip.eu/","html_tokens":72736,"markdown_tokens":302,"pruned_tokens":88}},
{"rank":3669,"domain":"imaginemodels.com","origin":"https://imaginemodels.com","robots":{"status":"ok","http":200,"url":"https://imaginemodels.com/robots.txt","bytes":206,"groups":1},"bots":{"GPTBot":["partial","*",1,0],"ChatGPT-User":["partial","*",1,0],"OAI-SearchBot":["partial","*",1,0],"ClaudeBot":["partial","*",1,0],"Claude-User":["partial","*",1,0],"Claude-SearchBot":["partial","*",1,0],"anthropic-ai":["partial","*",1,0],"PerplexityBot":["partial","*",1,0],"Perplexity-User":["partial","*",1,0],"Google-Extended":["partial","*",1,0],"Applebot-Extended":["partial","*",1,0],"CCBot":["partial","*",1,0],"Bytespider":["partial","*",1,0],"Amazonbot":["partial","*",1,0],"meta-externalagent":["partial","*",1,0],"cohere-ai":["partial","*",1,0],"Diffbot":["partial","*",1,0],"ContextFetchBot":["partial","*",1,0]},"llms":{"status":"missing","http":404,"bytes":105970},"home":{"status":"ok","http":200,"url":"https://imaginemodels.com/","html_tokens":40144,"markdown_tokens":27,"pruned_tokens":27}},
{"rank":3670,"domain":"hrc.org","origin":"https://hrc.org","robots":{"status":"ok","http":200,"url":"https://www.hrc.org/robots.txt","bytes":705,"groups":4},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["partial","*",4,0],"OAI-SearchBot":["partial","*",4,0],"ClaudeBot":["partial","*",4,0],"Claude-User":["partial","*",4,0],"Claude-SearchBot":["partial","*",4,0],"anthropic-ai":["partial","*",4,0],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["partial","*",4,0],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["partial","*",4,0],"CCBot":["partial","*",4,0],"Bytespider":["partial","*",4,0],"Amazonbot":["partial","*",4,0],"meta-externalagent":["partial","*",4,0],"cohere-ai":["partial","*",4,0],"Diffbot":["partial","*",4,0],"ContextFetchBot":["partial","*",4,0]},"llms":{"status":"missing","http":404,"bytes":80357},"home":{"status":"ok","http":200,"url":"https://www.hrc.org/","html_tokens":44631,"markdown_tokens":62,"pruned_tokens":62}},
{"rank":3671,"domain":"accorhotels.com","origin":"https://accorhotels.com","robots":{"status":"ok","http":200,"url":"https://all.accor.com/robots.txt","bytes":600,"groups":1},"bots":{"GPTBot":["partial","*",22,0],"ChatGPT-User":["partial","*",22,0],"OAI-SearchBot":["partial","*",22,0],"ClaudeBot":["partial","*",22,0],"Claude-User":["partial","*",22,0],"Claude-SearchBot":["partial","*",22,0],"anthropic-ai":["partial","*",22,0],"PerplexityBot":["partial","*",22,0],"Perplexity-User":["partial","*",22,0],"Google-Extended":["partial","*",22,0],"Applebot-Extended":["partial","*",22,0],"CCBot":["partial","*",22,0],"Bytespider":["partial","*",22,0],"Amazonbot":["partial","*",22,0],"meta-externalagent":["partial","*",22,0],"cohere-ai":["partial","*",22,0],"Diffbot":["partial","*",22,0],"ContextFetchBot":["partial","*",22,0]},"llms":{"status":"present","http":200,"bytes":105718},"home":{"status":"ok","http":200,"url":"https://all.accor.com/a/en.html","html_tokens":286570,"markdown_tokens":14,"pruned_tokens":14}},
{"rank":3672,"domain":"a16z.com","origin":"https://a16z.com","robots":{"status":"ok","http":200,"url":"https://a16z.com/robots.txt","bytes":1207,"groups":1},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"missing","http":404,"bytes":128960},"home":{"status":"ok","http":200,"url":"https://a16z.com/","html_tokens":38059,"markdown_tokens":228,"pruned_tokens":88}},
{"rank":3673,"domain":"ici.fr","origin":"https://ici.fr","robots":{"status":"ok","http":200,"url":"https://www.ici.fr/robots.txt","bytes":4252,"groups":73},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["blocked","named",1,0],"OAI-SearchBot":["blocked","named",1,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",20,0],"Claude-SearchBot":["partial","*",20,0],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["partial","*",20,0],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["partial","*",20,0],"ContextFetchBot":["partial","*",20,0]},"llms":{"status":"missing","http":404,"bytes":59209},"home":{"status":"ok","http":200,"url":"https://www.ici.fr/","html_tokens":246376,"markdown_tokens":5090,"pruned_tokens":144}},
{"rank":3674,"domain":"knightlab.com","origin":"https://knightlab.com","robots":{"status":"html","http":200,"url":"https://knightlab.northwestern.edu/","bytes":74433,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":3675,"domain":"json.org","origin":"https://json.org","robots":{"status":"missing","http":404,"url":"https://json.org/robots.txt","bytes":412,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":412},"home":{"status":"ok","http":200,"url":"https://json.org/","html_tokens":66,"markdown_tokens":9,"pruned_tokens":9}},
{"rank":3676,"domain":"customs.gov.cn","origin":"https://customs.gov.cn","robots":{"status":"timeout","http":null,"url":"https://customs.gov.cn/robots.txt","bytes":0,"groups":0,"error":"ConnectTimeout"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":3677,"domain":"beniculturali.it","origin":"https://beniculturali.it","robots":{"status":"unreachable","http":null,"url":"https://beniculturali.it/robots.txt","bytes":0,"groups":0,"error":"cannot resolve beniculturali.it"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":3678,"domain":"kuula.co","origin":"https://kuula.co","robots":{"status":"ok","http":200,"url":"https://kuula.co/robots.txt","bytes":134,"groups":1},"bots":{"GPTBot":["partial","*",4,0],"ChatGPT-User":["partial","*",4,0],"OAI-SearchBot":["partial","*",4,0],"ClaudeBot":["partial","*",4,0],"Claude-User":["partial","*",4,0],"Claude-SearchBot":["partial","*",4,0],"anthropic-ai":["partial","*",4,0],"PerplexityBot":["partial","*",4,0],"Perplexity-User":["partial","*",4,0],"Google-Extended":["partial","*",4,0],"Applebot-Extended":["partial","*",4,0],"CCBot":["partial","*",4,0],"Bytespider":["partial","*",4,0],"Amazonbot":["partial","*",4,0],"meta-externalagent":["partial","*",4,0],"cohere-ai":["partial","*",4,0],"Diffbot":["partial","*",4,0],"ContextFetchBot":["partial","*",4,0]},"llms":{"status":"missing","http":404,"bytes":1462},"home":{"status":"ok","http":200,"url":"https://kuula.co/","html_tokens":9225,"markdown_tokens":123,"pruned_tokens":123}},
{"rank":3679,"domain":"darpa.mil","origin":"https://darpa.mil","robots":{"status":"ok","http":200,"url":"https://www.darpa.mil/robots.txt","bytes":2027,"groups":1},"bots":{"GPTBot":["partial","*",34,18],"ChatGPT-User":["partial","*",34,18],"OAI-SearchBot":["partial","*",34,18],"ClaudeBot":["partial","*",34,18],"Claude-User":["partial","*",34,18],"Claude-SearchBot":["partial","*",34,18],"anthropic-ai":["partial","*",34,18],"PerplexityBot":["partial","*",34,18],"Perplexity-User":["partial","*",34,18],"Google-Extended":["partial","*",34,18],"Applebot-Extended":["partial","*",34,18],"CCBot":["partial","*",34,18],"Bytespider":["partial","*",34,18],"Amazonbot":["partial","*",34,18],"meta-externalagent":["partial","*",34,18],"cohere-ai":["partial","*",34,18],"Diffbot":["partial","*",34,18],"ContextFetchBot":["partial","*",34,18]},"llms":{"status":"missing","http":404,"bytes":34202},"home":{"status":"ok","http":200,"url":"https://www.darpa.mil/","html_tokens":22826,"markdown_tokens":698,"pruned_tokens":161}},
{"rank":3680,"domain":"alistapart.com","origin":"https://alistapart.com","robots":{"status":"ok","http":200,"url":"https://alistapart.com/robots.txt","bytes":159,"groups":1},"bots":{"GPTBot":["partial","*",1,1],"ChatGPT-User":["partial","*",1,1],"OAI-SearchBot":["partial","*",1,1],"ClaudeBot":["partial","*",1,1],"Claude-User":["partial","*",1,1],"Claude-SearchBot":["partial","*",1,1],"anthropic-ai":["partial","*",1,1],"PerplexityBot":["partial","*",1,1],"Perplexity-User":["partial","*",1,1],"Google-Extended":["partial","*",1,1],"Applebot-Extended":["partial","*",1,1],"CCBot":["partial","*",1,1],"Bytespider":["partial","*",1,1],"Amazonbot":["partial","*",1,1],"meta-externalagent":["partial","*",1,1],"cohere-ai":["partial","*",1,1],"Diffbot":["partial","*",1,1],"ContextFetchBot":["partial","*",1,1]},"llms":{"status":"missing","http":404,"bytes":44692},"home":{"status":"ok","http":200,"url":"https://alistapart.com/","html_tokens":16287,"markdown_tokens":696,"pruned_tokens":160}},
{"rank":3681,"domain":"apkpure.com","origin":"https://apkpure.com","robots":{"status":"ok","http":200,"url":"https://apkpure.com/robots.txt","bytes":3386,"groups":3},"bots":{"GPTBot":["partial","*",52,0],"ChatGPT-User":["partial","*",52,0],"OAI-SearchBot":["partial","*",52,0],"ClaudeBot":["partial","*",52,0],"Claude-User":["partial","*",52,0],"Claude-SearchBot":["partial","*",52,0],"anthropic-ai":["partial","*",52,0],"PerplexityBot":["partial","*",52,0],"Perplexity-User":["partial","*",52,0],"Google-Extended":["partial","*",52,0],"Applebot-Extended":["partial","*",52,0],"CCBot":["partial","*",52,0],"Bytespider":["partial","*",52,0],"Amazonbot":["partial","*",52,0],"meta-externalagent":["partial","*",52,0],"cohere-ai":["partial","*",52,0],"Diffbot":["partial","*",52,0],"ContextFetchBot":["partial","*",52,0]},"llms":{"status":"present","http":200,"bytes":15310},"home":{"status":"http_error","http":403,"url":"https://apkpure.com/"}},
{"rank":3682,"domain":"extendthemes.com","origin":"https://extendthemes.com","robots":{"status":"ok","http":200,"url":"https://extendthemes.com/robots.txt","bytes":67,"groups":1},"bots":{"GPTBot":["partial","*",1,1],"ChatGPT-User":["partial","*",1,1],"OAI-SearchBot":["partial","*",1,1],"ClaudeBot":["partial","*",1,1],"Claude-User":["partial","*",1,1],"Claude-SearchBot":["partial","*",1,1],"anthropic-ai":["partial","*",1,1],"PerplexityBot":["partial","*",1,1],"Perplexity-User":["partial","*",1,1],"Google-Extended":["partial","*",1,1],"Applebot-Extended":["partial","*",1,1],"CCBot":["partial","*",1,1],"Bytespider":["partial","*",1,1],"Amazonbot":["partial","*",1,1],"meta-externalagent":["partial","*",1,1],"cohere-ai":["partial","*",1,1],"Diffbot":["partial","*",1,1],"ContextFetchBot":["partial","*",1,1]},"llms":{"status":"missing","http":404,"bytes":427540},"home":{"status":"ok","http":200,"url":"https://extendthemes.com/","html_tokens":110980,"markdown_tokens":259,"pruned_tokens":126}},
{"rank":3683,"domain":"excite.com","origin":"http://excite.com","robots":{"status":"html","http":200,"url":"http://comingsoon.markmonitor.com/","bytes":812,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":3684,"domain":"agpd.es","origin":"https://www.agpd.es","robots":{"status":"ok","http":200,"url":"https://www.aepd.es/robots.txt","bytes":2757,"groups":1},"bots":{"GPTBot":["partial","*",49,18],"ChatGPT-User":["partial","*",49,18],"OAI-SearchBot":["partial","*",49,18],"ClaudeBot":["partial","*",49,18],"Claude-User":["partial","*",49,18],"Claude-SearchBot":["partial","*",49,18],"anthropic-ai":["partial","*",49,18],"PerplexityBot":["partial","*",49,18],"Perplexity-User":["partial","*",49,18],"Google-Extended":["partial","*",49,18],"Applebot-Extended":["partial","*",49,18],"CCBot":["partial","*",49,18],"Bytespider":["partial","*",49,18],"Amazonbot":["partial","*",49,18],"meta-externalagent":["partial","*",49,18],"cohere-ai":["partial","*",49,18],"Diffbot":["partial","*",49,18],"ContextFetchBot":["partial","*",49,18]},"llms":{"status":"http_error","http":500,"bytes":1682},"home":{"status":"ok","http":200,"url":"https://www.aepd.es/","html_tokens":57768,"markdown_tokens":847,"pruned_tokens":161}},
{"rank":3685,"domain":"turnitin.com","origin":"https://turnitin.com","robots":{"status":"refused","http":403,"url":"https://www.turnitin.com/robots.txt","bytes":919,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":3686,"domain":"langchain.com","origin":"https://langchain.com","robots":{"status":"ok","http":200,"url":"https://www.langchain.com/robots.txt","bytes":46,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":62698},"home":{"status":"ok","http":200,"url":"https://www.langchain.com/","html_tokens":67937,"markdown_tokens":74,"pruned_tokens":74}},
{"rank":3687,"domain":"irena.org","origin":"https://www.irena.org","robots":{"status":"refused","http":403,"url":"https://www.irena.org/robots.txt","bytes":9404,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":3688,"domain":"alaska.edu","origin":"https://alaska.edu","robots":{"status":"ok","http":200,"url":"https://www.alaska.edu/robots.txt","bytes":1213,"groups":2},"bots":{"GPTBot":["partial","*",36,0],"ChatGPT-User":["partial","*",36,0],"OAI-SearchBot":["partial","*",36,0],"ClaudeBot":["partial","*",36,0],"Claude-User":["partial","*",36,0],"Claude-SearchBot":["partial","*",36,0],"anthropic-ai":["partial","*",36,0],"PerplexityBot":["partial","*",36,0],"Perplexity-User":["partial","*",36,0],"Google-Extended":["partial","*",36,0],"Applebot-Extended":["partial","*",36,0],"CCBot":["partial","*",36,0],"Bytespider":["partial","*",36,0],"Amazonbot":["partial","*",36,0],"meta-externalagent":["partial","*",36,0],"cohere-ai":["partial","*",36,0],"Diffbot":["partial","*",36,0],"ContextFetchBot":["partial","*",36,0]},"llms":{"status":"missing","http":404,"bytes":16709},"home":{"status":"ok","http":200,"url":"https://www.alaska.edu/alaska/","html_tokens":12283,"markdown_tokens":784,"pruned_tokens":119}},
{"rank":3689,"domain":"unipi.it","origin":"https://www.unipi.it","robots":{"status":"ok","http":200,"url":"https://www.unipi.it/robots.txt","bytes":202,"groups":2},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"missing","http":404,"bytes":893054},"home":{"status":"ok","http":200,"url":"https://www.unipi.it/","html_tokens":274029,"markdown_tokens":839,"pruned_tokens":131}},
{"rank":3690,"domain":"jal.co.jp","origin":"https://jal.co.jp","robots":{"status":"refused","http":403,"url":"https://www.jal.co.jp/robots.txt","bytes":385,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":3691,"domain":"mn.co","origin":"https://mn.co","robots":{"status":"ok","http":200,"url":"https://mightynetworks.com/robots.txt","bytes":225,"groups":2},"bots":{"GPTBot":["partial","*",6,0],"ChatGPT-User":["partial","*",6,0],"OAI-SearchBot":["partial","*",6,0],"ClaudeBot":["partial","*",6,0],"Claude-User":["partial","*",6,0],"Claude-SearchBot":["partial","*",6,0],"anthropic-ai":["partial","*",6,0],"PerplexityBot":["partial","*",6,0],"Perplexity-User":["partial","*",6,0],"Google-Extended":["partial","*",6,0],"Applebot-Extended":["partial","*",6,0],"CCBot":["partial","*",6,0],"Bytespider":["partial","*",6,0],"Amazonbot":["partial","*",6,0],"meta-externalagent":["partial","*",6,0],"cohere-ai":["partial","*",6,0],"Diffbot":["partial","*",6,0],"ContextFetchBot":["partial","*",6,0]},"llms":{"status":"http_error","http":403,"bytes":81656},"home":{"status":"http_error","http":403,"url":"https://mn.co/"}},
{"rank":3692,"domain":"jacobin.com","origin":"https://jacobin.com","robots":{"status":"ok","http":200,"url":"https://jacobin.com/robots.txt","bytes":1248,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":2319},"home":{"status":"ok","http":200,"url":"https://jacobin.com/","html_tokens":15954,"markdown_tokens":814,"pruned_tokens":126}},
{"rank":3693,"domain":"torrentfreak.com","origin":"https://torrentfreak.com","robots":{"status":"ok","http":200,"url":"https://torrentfreak.com/robots.txt","bytes":159,"groups":2},"bots":{"GPTBot":["partial","*",2,1],"ChatGPT-User":["partial","*",2,1],"OAI-SearchBot":["partial","*",2,1],"ClaudeBot":["partial","*",2,1],"Claude-User":["partial","*",2,1],"Claude-SearchBot":["partial","*",2,1],"anthropic-ai":["partial","*",2,1],"PerplexityBot":["partial","*",2,1],"Perplexity-User":["partial","*",2,1],"Google-Extended":["partial","*",2,1],"Applebot-Extended":["partial","*",2,1],"CCBot":["partial","*",2,1],"Bytespider":["partial","*",2,1],"Amazonbot":["partial","*",2,1],"meta-externalagent":["partial","*",2,1],"cohere-ai":["partial","*",2,1],"Diffbot":["partial","*",2,1],"ContextFetchBot":["partial","*",2,1]},"llms":{"status":"other_text","http":200,"bytes":3691},"home":{"status":"ok","http":200,"url":"https://torrentfreak.com/","html_tokens":11446,"markdown_tokens":123,"pruned_tokens":123}},
{"rank":3694,"domain":"sussex.ac.uk","origin":"https://sussex.ac.uk","robots":{"status":"ok","http":200,"url":"https://www.sussex.ac.uk/robots.txt","bytes":5117,"groups":1},"bots":{"GPTBot":["partial","*",106,2],"ChatGPT-User":["partial","*",106,2],"OAI-SearchBot":["partial","*",106,2],"ClaudeBot":["partial","*",106,2],"Claude-User":["partial","*",106,2],"Claude-SearchBot":["partial","*",106,2],"anthropic-ai":["partial","*",106,2],"PerplexityBot":["partial","*",106,2],"Perplexity-User":["partial","*",106,2],"Google-Extended":["partial","*",106,2],"Applebot-Extended":["partial","*",106,2],"CCBot":["partial","*",106,2],"Bytespider":["partial","*",106,2],"Amazonbot":["partial","*",106,2],"meta-externalagent":["partial","*",106,2],"cohere-ai":["partial","*",106,2],"Diffbot":["partial","*",106,2],"ContextFetchBot":["partial","*",106,2]},"llms":{"status":"missing","http":404,"bytes":19840},"home":{"status":"ok","http":200,"url":"https://www.sussex.ac.uk/","html_tokens":10066,"markdown_tokens":432,"pruned_tokens":82}},
{"rank":3695,"domain":"blog.com","origin":"https://blog.com","robots":{"status":"missing","http":404,"url":"https://blog.com/robots.txt","bytes":146,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":146},"home":{"status":"ok","http":200,"url":"https://blog.com/","html_tokens":21,"markdown_tokens":0,"pruned_tokens":0}},
{"rank":3696,"domain":"uclahealth.org","origin":"https://uclahealth.org","robots":{"status":"ok","http":200,"url":"https://www.uclahealth.org/robots.txt","bytes":5237,"groups":9},"bots":{"GPTBot":["partial","*",71,70],"ChatGPT-User":["partial","*",71,70],"OAI-SearchBot":["partial","*",71,70],"ClaudeBot":["partial","*",71,70],"Claude-User":["partial","*",71,70],"Claude-SearchBot":["partial","*",71,70],"anthropic-ai":["partial","*",71,70],"PerplexityBot":["partial","*",71,70],"Perplexity-User":["partial","*",71,70],"Google-Extended":["partial","*",71,70],"Applebot-Extended":["partial","*",71,70],"CCBot":["partial","*",71,70],"Bytespider":["partial","*",71,70],"Amazonbot":["partial","*",71,70],"meta-externalagent":["partial","*",71,70],"cohere-ai":["partial","*",71,70],"Diffbot":["partial","*",71,70],"ContextFetchBot":["partial","*",71,70]},"llms":{"status":"present","http":200,"bytes":9993},"home":{"status":"ok","http":200,"url":"https://www.uclahealth.org/","html_tokens":30935,"markdown_tokens":108,"pruned_tokens":108}},
{"rank":3697,"domain":"msstate.edu","origin":"https://msstate.edu","robots":{"status":"ok","http":200,"url":"https://www.msstate.edu/robots.txt","bytes":1594,"groups":1},"bots":{"GPTBot":["partial","*",22,18],"ChatGPT-User":["partial","*",22,18],"OAI-SearchBot":["partial","*",22,18],"ClaudeBot":["partial","*",22,18],"Claude-User":["partial","*",22,18],"Claude-SearchBot":["partial","*",22,18],"anthropic-ai":["partial","*",22,18],"PerplexityBot":["partial","*",22,18],"Perplexity-User":["partial","*",22,18],"Google-Extended":["partial","*",22,18],"Applebot-Extended":["partial","*",22,18],"CCBot":["partial","*",22,18],"Bytespider":["partial","*",22,18],"Amazonbot":["partial","*",22,18],"meta-externalagent":["partial","*",22,18],"cohere-ai":["partial","*",22,18],"Diffbot":["partial","*",22,18],"ContextFetchBot":["partial","*",22,18]},"llms":{"status":"missing","http":404,"bytes":188},"home":{"status":"ok","http":200,"url":"https://www.msstate.edu/","html_tokens":23650,"markdown_tokens":246,"pruned_tokens":141}},
{"rank":3698,"domain":"exploratorium.edu","origin":"https://exploratorium.edu","robots":{"status":"refused","http":403,"url":"https://exploratorium.edu/robots.txt","bytes":560,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":3699,"domain":"blockchain.com","origin":"https://blockchain.com","robots":{"status":"ok","http":200,"url":"https://www.blockchain.com/robots.txt","bytes":703,"groups":2},"bots":{"GPTBot":["partial","*",6,3],"ChatGPT-User":["partial","*",6,3],"OAI-SearchBot":["partial","*",6,3],"ClaudeBot":["partial","*",6,3],"Claude-User":["partial","*",6,3],"Claude-SearchBot":["partial","*",6,3],"anthropic-ai":["partial","*",6,3],"PerplexityBot":["partial","*",6,3],"Perplexity-User":["partial","*",6,3],"Google-Extended":["partial","*",6,3],"Applebot-Extended":["partial","*",6,3],"CCBot":["partial","*",6,3],"Bytespider":["partial","*",6,3],"Amazonbot":["partial","*",6,3],"meta-externalagent":["partial","*",6,3],"cohere-ai":["partial","*",6,3],"Diffbot":["partial","*",6,3],"ContextFetchBot":["partial","*",6,3]},"llms":{"status":"missing","http":404,"bytes":650177},"home":{"status":"http_error","http":403,"url":"https://blockchain.com/"}},
{"rank":3700,"domain":"synthesia.io","origin":"https://synthesia.io","robots":{"status":"ok","http":200,"url":"https://www.synthesia.io/robots.txt","bytes":183,"groups":4},"bots":{"GPTBot":["allowed","*",0,1],"ChatGPT-User":["allowed","*",0,1],"OAI-SearchBot":["allowed","*",0,1],"ClaudeBot":["allowed","*",0,1],"Claude-User":["allowed","*",0,1],"Claude-SearchBot":["allowed","*",0,1],"anthropic-ai":["allowed","*",0,1],"PerplexityBot":["allowed","*",0,1],"Perplexity-User":["allowed","*",0,1],"Google-Extended":["allowed","*",0,1],"Applebot-Extended":["allowed","*",0,1],"CCBot":["allowed","*",0,1],"Bytespider":["allowed","*",0,1],"Amazonbot":["allowed","*",0,1],"meta-externalagent":["allowed","*",0,1],"cohere-ai":["allowed","*",0,1],"Diffbot":["allowed","*",0,1],"ContextFetchBot":["allowed","*",0,1]},"llms":{"status":"present","http":200,"bytes":13218},"home":{"status":"ok","http":200,"url":"https://www.synthesia.io/","html_tokens":165424,"markdown_tokens":1720,"pruned_tokens":174}},
{"rank":3701,"domain":"heroku.com","origin":"https://heroku.com","robots":{"status":"ok","http":200,"url":"https://www.heroku.com/robots.txt","bytes":375,"groups":2},"bots":{"GPTBot":["partial","*",5,0],"ChatGPT-User":["partial","*",5,0],"OAI-SearchBot":["partial","*",5,0],"ClaudeBot":["partial","*",5,0],"Claude-User":["partial","*",5,0],"Claude-SearchBot":["partial","*",5,0],"anthropic-ai":["partial","*",5,0],"PerplexityBot":["partial","*",5,0],"Perplexity-User":["partial","*",5,0],"Google-Extended":["partial","*",5,0],"Applebot-Extended":["partial","*",5,0],"CCBot":["partial","*",5,0],"Bytespider":["partial","*",5,0],"Amazonbot":["partial","*",5,0],"meta-externalagent":["partial","*",5,0],"cohere-ai":["partial","*",5,0],"Diffbot":["partial","*",5,0],"ContextFetchBot":["partial","*",5,0]},"llms":{"status":"other_text","http":200,"bytes":8787},"home":{"status":"ok","http":200,"url":"https://www.heroku.com/","html_tokens":108443,"markdown_tokens":1060,"pruned_tokens":464}},
{"rank":3702,"domain":"winscp.net","origin":"https://winscp.net","robots":{"status":"ok","http":200,"url":"https://winscp.net/robots.txt","bytes":279,"groups":5},"bots":{"GPTBot":["partial","*",3,0],"ChatGPT-User":["partial","*",3,0],"OAI-SearchBot":["partial","*",3,0],"ClaudeBot":["partial","*",3,0],"Claude-User":["partial","*",3,0],"Claude-SearchBot":["partial","*",3,0],"anthropic-ai":["partial","*",3,0],"PerplexityBot":["partial","*",3,0],"Perplexity-User":["partial","*",3,0],"Google-Extended":["partial","*",3,0],"Applebot-Extended":["partial","*",3,0],"CCBot":["partial","*",3,0],"Bytespider":["partial","*",3,0],"Amazonbot":["partial","*",3,0],"meta-externalagent":["partial","*",3,0],"cohere-ai":["partial","*",3,0],"Diffbot":["partial","*",3,0],"ContextFetchBot":["partial","*",3,0]},"llms":{"status":"missing","http":404,"bytes":19094},"home":{"status":"ok","http":200,"url":"https://winscp.net/eng/index.php","html_tokens":6289,"markdown_tokens":157,"pruned_tokens":157}},
{"rank":3703,"domain":"loopia.se","origin":"https://loopia.se","robots":{"status":"ok","http":200,"url":"https://www.loopia.se/robots.txt","bytes":217,"groups":1},"bots":{"GPTBot":["partial","*",7,0],"ChatGPT-User":["partial","*",7,0],"OAI-SearchBot":["partial","*",7,0],"ClaudeBot":["partial","*",7,0],"Claude-User":["partial","*",7,0],"Claude-SearchBot":["partial","*",7,0],"anthropic-ai":["partial","*",7,0],"PerplexityBot":["partial","*",7,0],"Perplexity-User":["partial","*",7,0],"Google-Extended":["partial","*",7,0],"Applebot-Extended":["partial","*",7,0],"CCBot":["partial","*",7,0],"Bytespider":["partial","*",7,0],"Amazonbot":["partial","*",7,0],"meta-externalagent":["partial","*",7,0],"cohere-ai":["partial","*",7,0],"Diffbot":["partial","*",7,0],"ContextFetchBot":["partial","*",7,0]},"llms":{"status":"present","http":200,"bytes":12574},"home":{"status":"ok","http":200,"url":"https://www.loopia.se/","html_tokens":13091,"markdown_tokens":1047,"pruned_tokens":129}},
{"rank":3704,"domain":"realclearpolitics.com","origin":"https://realclearpolitics.com","robots":{"status":"ok","http":200,"url":"https://www.realclearpolitics.com/robots.txt","bytes":313,"groups":3},"bots":{"GPTBot":["partial","*",9,0],"ChatGPT-User":["partial","*",9,0],"OAI-SearchBot":["partial","*",9,0],"ClaudeBot":["partial","*",9,0],"Claude-User":["partial","*",9,0],"Claude-SearchBot":["partial","*",9,0],"anthropic-ai":["partial","*",9,0],"PerplexityBot":["partial","*",9,0],"Perplexity-User":["partial","*",9,0],"Google-Extended":["partial","*",9,0],"Applebot-Extended":["partial","*",9,0],"CCBot":["partial","*",9,0],"Bytespider":["partial","*",9,0],"Amazonbot":["partial","*",9,0],"meta-externalagent":["partial","*",9,0],"cohere-ai":["partial","*",9,0],"Diffbot":["partial","*",9,0],"ContextFetchBot":["partial","*",9,0]},"llms":{"status":"http_error","http":403,"bytes":11540},"home":{"status":"http_error","http":403,"url":"https://www.realclearpolitics.com/"}},
{"rank":3705,"domain":"futura-sciences.com","origin":"https://futura-sciences.com","robots":{"status":"ok","http":200,"url":"https://www.futura-sciences.com/robots.txt","bytes":2894,"groups":3},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["allowed","named",0,1],"OAI-SearchBot":["allowed","named",0,1],"ClaudeBot":["blocked","named",1,0],"Claude-User":["allowed","named",0,1],"Claude-SearchBot":["allowed","named",0,1],"anthropic-ai":["partial","*",34,0],"PerplexityBot":["allowed","named",0,1],"Perplexity-User":["partial","*",34,0],"Google-Extended":["partial","*",34,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["partial","*",34,0],"cohere-ai":["partial","*",34,0],"Diffbot":["partial","*",34,0],"ContextFetchBot":["partial","*",34,0]},"llms":{"status":"missing","http":404,"bytes":96726},"home":{"status":"ok","http":200,"url":"https://www.futura-sciences.com/","html_tokens":38632,"markdown_tokens":99,"pruned_tokens":71}},
{"rank":3706,"domain":"jetpack.com","origin":"https://jetpack.com","robots":{"status":"ok","http":200,"url":"https://jetpack.com/robots.txt","bytes":676,"groups":1},"bots":{"GPTBot":["partial","*",10,1],"ChatGPT-User":["partial","*",10,1],"OAI-SearchBot":["partial","*",10,1],"ClaudeBot":["partial","*",10,1],"Claude-User":["partial","*",10,1],"Claude-SearchBot":["partial","*",10,1],"anthropic-ai":["partial","*",10,1],"PerplexityBot":["partial","*",10,1],"Perplexity-User":["partial","*",10,1],"Google-Extended":["partial","*",10,1],"Applebot-Extended":["partial","*",10,1],"CCBot":["partial","*",10,1],"Bytespider":["partial","*",10,1],"Amazonbot":["partial","*",10,1],"meta-externalagent":["partial","*",10,1],"cohere-ai":["partial","*",10,1],"Diffbot":["partial","*",10,1],"ContextFetchBot":["partial","*",10,1]},"llms":{"status":"missing","http":404,"bytes":201712},"home":{"status":"ok","http":200,"url":"https://jetpack.com/","html_tokens":70076,"markdown_tokens":830,"pruned_tokens":101}},
{"rank":3707,"domain":"yourdictionary.com","origin":"https://yourdictionary.com","robots":{"status":"ok","http":200,"url":"https://www.yourdictionary.com/robots.txt","bytes":173,"groups":2},"bots":{"GPTBot":["allowed","*",0,1],"ChatGPT-User":["blocked","named",1,0],"OAI-SearchBot":["allowed","*",0,1],"ClaudeBot":["allowed","*",0,1],"Claude-User":["allowed","*",0,1],"Claude-SearchBot":["allowed","*",0,1],"anthropic-ai":["allowed","*",0,1],"PerplexityBot":["allowed","*",0,1],"Perplexity-User":["allowed","*",0,1],"Google-Extended":["allowed","*",0,1],"Applebot-Extended":["allowed","*",0,1],"CCBot":["allowed","*",0,1],"Bytespider":["allowed","*",0,1],"Amazonbot":["allowed","*",0,1],"meta-externalagent":["allowed","*",0,1],"cohere-ai":["allowed","*",0,1],"Diffbot":["allowed","*",0,1],"ContextFetchBot":["allowed","*",0,1]},"llms":{"status":"missing","http":404,"bytes":0},"home":{"status":"ok","http":200,"url":"https://www.yourdictionary.com/","html_tokens":102257,"markdown_tokens":125,"pruned_tokens":125}},
{"rank":3708,"domain":"google.ie","origin":"https://google.ie","robots":{"status":"ok","http":200,"url":"https://www.google.ie/robots.txt","bytes":6624,"groups":4},"bots":{"GPTBot":["partial","*",169,64],"ChatGPT-User":["partial","*",169,64],"OAI-SearchBot":["partial","*",169,64],"ClaudeBot":["partial","*",169,64],"Claude-User":["partial","*",169,64],"Claude-SearchBot":["partial","*",169,64],"anthropic-ai":["partial","*",169,64],"PerplexityBot":["partial","*",169,64],"Perplexity-User":["partial","*",169,64],"Google-Extended":["partial","*",169,64],"Applebot-Extended":["partial","*",169,64],"CCBot":["partial","*",169,64],"Bytespider":["partial","*",169,64],"Amazonbot":["partial","*",169,64],"meta-externalagent":["partial","*",169,64],"cohere-ai":["partial","*",169,64],"Diffbot":["partial","*",169,64],"ContextFetchBot":["partial","*",169,64]},"llms":{"status":"missing","http":404,"bytes":1569},"home":{"status":"ok","http":200,"url":"https://www.google.com/","html_tokens":21471,"markdown_tokens":30,"pruned_tokens":30}},
{"rank":3709,"domain":"gothamist.com","origin":"https://gothamist.com","robots":{"status":"ok","http":200,"url":"https://gothamist.com/robots.txt","bytes":316,"groups":5},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["partial","*",4,0],"OAI-SearchBot":["partial","*",4,0],"ClaudeBot":["partial","*",4,0],"Claude-User":["partial","*",4,0],"Claude-SearchBot":["partial","*",4,0],"anthropic-ai":["partial","*",4,0],"PerplexityBot":["partial","*",4,0],"Perplexity-User":["partial","*",4,0],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["partial","*",4,0],"CCBot":["blocked","named",1,0],"Bytespider":["partial","*",4,0],"Amazonbot":["partial","*",4,0],"meta-externalagent":["partial","*",4,0],"cohere-ai":["partial","*",4,0],"Diffbot":["partial","*",4,0],"ContextFetchBot":["partial","*",4,0]},"llms":{"status":"missing","http":404,"bytes":656224},"home":{"status":"ok","http":200,"url":"https://gothamist.com/","html_tokens":198142,"markdown_tokens":241,"pruned_tokens":134}},
{"rank":3710,"domain":"huffingtonpost.ca","origin":"https://huffingtonpost.ca","robots":{"status":"missing","http":404,"url":"https://www.huffpost.com/archive/ca/vertical/robots.txt","bytes":115284,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":115275},"home":{"status":"ok","http":200,"url":"https://www.huffpost.com/","html_tokens":300629,"markdown_tokens":10,"pruned_tokens":10}},
{"rank":3711,"domain":"ukri.org","origin":"https://ukri.org","robots":{"status":"ok","http":200,"url":"https://www.ukri.org/robots.txt","bytes":156,"groups":1},"bots":{"GPTBot":["partial","*",1,1],"ChatGPT-User":["partial","*",1,1],"OAI-SearchBot":["partial","*",1,1],"ClaudeBot":["partial","*",1,1],"Claude-User":["partial","*",1,1],"Claude-SearchBot":["partial","*",1,1],"anthropic-ai":["partial","*",1,1],"PerplexityBot":["partial","*",1,1],"Perplexity-User":["partial","*",1,1],"Google-Extended":["partial","*",1,1],"Applebot-Extended":["partial","*",1,1],"CCBot":["partial","*",1,1],"Bytespider":["partial","*",1,1],"Amazonbot":["partial","*",1,1],"meta-externalagent":["partial","*",1,1],"cohere-ai":["partial","*",1,1],"Diffbot":["partial","*",1,1],"ContextFetchBot":["partial","*",1,1]},"llms":{"status":"missing","http":404,"bytes":62539},"home":{"status":"ok","http":200,"url":"https://www.ukri.org/","html_tokens":21065,"markdown_tokens":79,"pruned_tokens":79}},
{"rank":3712,"domain":"tesco.com","origin":"https://tesco.com","robots":{"status":"refused","http":403,"url":"https://tesco.com/robots.txt","bytes":371,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":3713,"domain":"oxforddictionaries.com","origin":"https://oxforddictionaries.com","robots":{"status":"refused","http":403,"url":"https://languages.oup.com/","bytes":5524,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":3714,"domain":"tennessean.com","origin":"https://tennessean.com","robots":{"status":"ok","http":200,"url":"https://www.tennessean.com/robots.txt","bytes":14686,"groups":288},"bots":{"GPTBot":["partial","named",1,6],"ChatGPT-User":["partial","named",1,6],"OAI-SearchBot":["partial","named",1,6],"ClaudeBot":["blocked","named",1,0],"Claude-User":["blocked","named",1,0],"Claude-SearchBot":["blocked","named",1,0],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["partial","named",1,6],"Perplexity-User":["blocked","named",1,0],"Google-Extended":["partial","named",1,6],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",2,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",34,0]},"llms":{"status":"http_error","http":402,"bytes":254},"home":{"status":"http_error","http":402,"url":"https://www.tennessean.com/"}},
{"rank":3715,"domain":"pravda.com.ua","origin":"https://pravda.com.ua","robots":{"status":"ok","http":200,"url":"https://www.pravda.com.ua/robots.txt","bytes":757,"groups":2},"bots":{"GPTBot":["partial","*",15,1],"ChatGPT-User":["partial","*",15,1],"OAI-SearchBot":["partial","*",15,1],"ClaudeBot":["partial","*",15,1],"Claude-User":["partial","*",15,1],"Claude-SearchBot":["partial","*",15,1],"anthropic-ai":["partial","*",15,1],"PerplexityBot":["partial","*",15,1],"Perplexity-User":["partial","*",15,1],"Google-Extended":["partial","*",15,1],"Applebot-Extended":["partial","*",15,1],"CCBot":["partial","*",15,1],"Bytespider":["partial","*",15,1],"Amazonbot":["partial","*",15,1],"meta-externalagent":["partial","*",15,1],"cohere-ai":["partial","*",15,1],"Diffbot":["partial","*",15,1],"ContextFetchBot":["partial","*",15,1]},"llms":{"status":"missing","http":404,"bytes":117875},"home":{"status":"ok","http":200,"url":"https://www.pravda.com.ua/","html_tokens":59405,"markdown_tokens":38,"pruned_tokens":38}},
{"rank":3716,"domain":"zoominfo.com","origin":"https://zoominfo.com","robots":{"status":"ok","http":200,"url":"https://www.zoominfo.com/robots.txt","bytes":5332,"groups":6},"bots":{"GPTBot":["partial","named",55,1],"ChatGPT-User":["partial","named",55,1],"OAI-SearchBot":["partial","named",55,1],"ClaudeBot":["partial","named",55,1],"Claude-User":["partial","named",55,1],"Claude-SearchBot":["partial","named",55,1],"anthropic-ai":["partial","*",55,1],"PerplexityBot":["partial","named",55,1],"Perplexity-User":["partial","named",55,1],"Google-Extended":["partial","named",55,1],"Applebot-Extended":["partial","named",55,1],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["partial","*",55,1],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",55,1]},"llms":{"status":"present","http":200,"bytes":11440},"home":{"status":"ok","http":200,"url":"https://www.zoominfo.com/","html_tokens":88823,"markdown_tokens":1636,"pruned_tokens":914}},
{"rank":3717,"domain":"ntu.edu.sg","origin":"https://ntu.edu.sg","robots":{"status":"ok","http":200,"url":"https://www.ntu.edu.sg/robots.txt","bytes":124,"groups":1},"bots":{"GPTBot":["partial","*",2,0],"ChatGPT-User":["partial","*",2,0],"OAI-SearchBot":["partial","*",2,0],"ClaudeBot":["partial","*",2,0],"Claude-User":["partial","*",2,0],"Claude-SearchBot":["partial","*",2,0],"anthropic-ai":["partial","*",2,0],"PerplexityBot":["partial","*",2,0],"Perplexity-User":["partial","*",2,0],"Google-Extended":["partial","*",2,0],"Applebot-Extended":["partial","*",2,0],"CCBot":["partial","*",2,0],"Bytespider":["partial","*",2,0],"Amazonbot":["partial","*",2,0],"meta-externalagent":["partial","*",2,0],"cohere-ai":["partial","*",2,0],"Diffbot":["partial","*",2,0],"ContextFetchBot":["partial","*",2,0]},"llms":{"status":"missing","http":404,"bytes":123245},"home":{"status":"ok","http":200,"url":"https://www.ntu.edu.sg/","html_tokens":34322,"markdown_tokens":76,"pruned_tokens":76}},
{"rank":3718,"domain":"barcelona.cat","origin":"https://barcelona.cat","robots":{"status":"ok","http":200,"url":"https://www.barcelona.cat/robots.txt","bytes":7201,"groups":2},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["blocked","named",1,0],"OAI-SearchBot":["blocked","named",1,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["blocked","named",1,0],"Claude-SearchBot":["blocked","named",1,0],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["blocked","named",1,0],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["partial","*",103,51],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",103,51]},"llms":{"status":"missing","http":404,"bytes":3361},"home":{"status":"ok","http":200,"url":"https://www.barcelona.cat/ca","html_tokens":20800,"markdown_tokens":76,"pruned_tokens":76}},
{"rank":3719,"domain":"notebookcheck.net","origin":"https://notebookcheck.net","robots":{"status":"refused","http":403,"url":"https://notebookcheck.net/robots.txt","bytes":5576,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":3720,"domain":"mg.gov.br","origin":"https://www.mg.gov.br","robots":{"status":"ok","http":200,"url":"https://www.mg.gov.br/robots.txt","bytes":2027,"groups":1},"bots":{"GPTBot":["partial","*",34,18],"ChatGPT-User":["partial","*",34,18],"OAI-SearchBot":["partial","*",34,18],"ClaudeBot":["partial","*",34,18],"Claude-User":["partial","*",34,18],"Claude-SearchBot":["partial","*",34,18],"anthropic-ai":["partial","*",34,18],"PerplexityBot":["partial","*",34,18],"Perplexity-User":["partial","*",34,18],"Google-Extended":["partial","*",34,18],"Applebot-Extended":["partial","*",34,18],"CCBot":["partial","*",34,18],"Bytespider":["partial","*",34,18],"Amazonbot":["partial","*",34,18],"meta-externalagent":["partial","*",34,18],"cohere-ai":["partial","*",34,18],"Diffbot":["partial","*",34,18],"ContextFetchBot":["partial","*",34,18]},"llms":{"status":"missing","http":404,"bytes":208},"home":{"status":"ok","http":200,"url":"https://www.mg.gov.br/","html_tokens":34476,"markdown_tokens":434,"pruned_tokens":175}},
{"rank":3721,"domain":"bnnbloomberg.ca","origin":"https://bnnbloomberg.ca","robots":{"status":"ok","http":200,"url":"https://www.bnnbloomberg.ca/robots.txt","bytes":1685,"groups":12},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["blocked","named",1,0],"OAI-SearchBot":["blocked","named",1,0],"ClaudeBot":["partial","*",30,1],"Claude-User":["partial","*",30,1],"Claude-SearchBot":["partial","*",30,1],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["partial","*",30,1],"Google-Extended":["partial","*",30,1],"Applebot-Extended":["partial","*",30,1],"CCBot":["blocked","named",1,0],"Bytespider":["partial","*",30,1],"Amazonbot":["partial","*",30,1],"meta-externalagent":["partial","*",30,1],"cohere-ai":["partial","*",30,1],"Diffbot":["partial","*",30,1],"ContextFetchBot":["partial","*",30,1]},"llms":{"status":"missing","http":404,"bytes":85633},"home":{"status":"ok","http":200,"url":"https://www.bnnbloomberg.ca/","html_tokens":81491,"markdown_tokens":160,"pruned_tokens":160}},
{"rank":3722,"domain":"pxhere.com","origin":"https://pxhere.com","robots":{"status":"ok","http":200,"url":"https://pxhere.com/robots.txt","bytes":598,"groups":3},"bots":{"GPTBot":["partial","*",2,1],"ChatGPT-User":["partial","*",2,1],"OAI-SearchBot":["partial","*",2,1],"ClaudeBot":["partial","*",2,1],"Claude-User":["partial","*",2,1],"Claude-SearchBot":["partial","*",2,1],"anthropic-ai":["partial","*",2,1],"PerplexityBot":["partial","*",2,1],"Perplexity-User":["partial","*",2,1],"Google-Extended":["partial","*",2,1],"Applebot-Extended":["partial","*",2,1],"CCBot":["blocked","named",1,0],"Bytespider":["partial","*",2,1],"Amazonbot":["partial","*",2,1],"meta-externalagent":["partial","*",2,1],"cohere-ai":["partial","*",2,1],"Diffbot":["partial","*",2,1],"ContextFetchBot":["partial","*",2,1]},"llms":{"status":"missing","http":404,"bytes":24647},"home":{"status":"ok","http":200,"url":"https://pxhere.com/","html_tokens":18953,"markdown_tokens":49,"pruned_tokens":16}},
{"rank":3723,"domain":"mtu.edu","origin":"https://mtu.edu","robots":{"status":"ok","http":200,"url":"https://www.mtu.edu/robots.txt","bytes":7376,"groups":1},"bots":{"GPTBot":["partial","*",96,54],"ChatGPT-User":["partial","*",96,54],"OAI-SearchBot":["partial","*",96,54],"ClaudeBot":["partial","*",96,54],"Claude-User":["partial","*",96,54],"Claude-SearchBot":["partial","*",96,54],"anthropic-ai":["partial","*",96,54],"PerplexityBot":["partial","*",96,54],"Perplexity-User":["partial","*",96,54],"Google-Extended":["partial","*",96,54],"Applebot-Extended":["partial","*",96,54],"CCBot":["partial","*",96,54],"Bytespider":["partial","*",96,54],"Amazonbot":["partial","*",96,54],"meta-externalagent":["partial","*",96,54],"cohere-ai":["partial","*",96,54],"Diffbot":["partial","*",96,54],"ContextFetchBot":["partial","*",96,54]},"llms":{"status":"missing","http":404,"bytes":47733},"home":{"status":"ok","http":200,"url":"https://www.mtu.edu/","html_tokens":32988,"markdown_tokens":383,"pruned_tokens":106}},
{"rank":3724,"domain":"bitwarden.com","origin":"https://bitwarden.com","robots":{"status":"ok","http":200,"url":"https://bitwarden.com/robots.txt","bytes":526,"groups":2},"bots":{"GPTBot":["partial","named",3,5],"ChatGPT-User":["partial","*",2,1],"OAI-SearchBot":["partial","named",3,5],"ClaudeBot":["partial","named",3,5],"Claude-User":["partial","*",2,1],"Claude-SearchBot":["partial","named",3,5],"anthropic-ai":["partial","named",3,5],"PerplexityBot":["partial","named",3,5],"Perplexity-User":["partial","*",2,1],"Google-Extended":["partial","*",2,1],"Applebot-Extended":["partial","*",2,1],"CCBot":["partial","named",3,5],"Bytespider":["partial","named",3,5],"Amazonbot":["partial","named",3,5],"meta-externalagent":["partial","*",2,1],"cohere-ai":["partial","*",2,1],"Diffbot":["partial","*",2,1],"ContextFetchBot":["partial","*",2,1]},"llms":{"status":"present","http":200,"bytes":8331},"home":{"status":"ok","http":200,"url":"https://bitwarden.com/","html_tokens":181150,"markdown_tokens":386,"pruned_tokens":200}},
{"rank":3725,"domain":"home-assistant.io","origin":"https://home-assistant.io","robots":{"status":"ok","http":200,"url":"https://www.home-assistant.io/robots.txt","bytes":76,"groups":1},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"missing","http":404,"bytes":34416},"home":{"status":"ok","http":200,"url":"https://www.home-assistant.io/","html_tokens":19288,"markdown_tokens":834,"pruned_tokens":499}},
{"rank":3726,"domain":"batmanapollo.ru","origin":"https://batmanapollo.ru","robots":{"status":"refused","http":403,"url":"https://opsuimolog.ru/robots.txt","bytes":5508,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":3727,"domain":"kinsta.com","origin":"https://kinsta.com","robots":{"status":"ok","http":200,"url":"https://kinsta.com/robots.txt","bytes":523,"groups":1},"bots":{"GPTBot":["partial","*",1,1],"ChatGPT-User":["partial","*",1,1],"OAI-SearchBot":["partial","*",1,1],"ClaudeBot":["partial","*",1,1],"Claude-User":["partial","*",1,1],"Claude-SearchBot":["partial","*",1,1],"anthropic-ai":["partial","*",1,1],"PerplexityBot":["partial","*",1,1],"Perplexity-User":["partial","*",1,1],"Google-Extended":["partial","*",1,1],"Applebot-Extended":["partial","*",1,1],"CCBot":["partial","*",1,1],"Bytespider":["partial","*",1,1],"Amazonbot":["partial","*",1,1],"meta-externalagent":["partial","*",1,1],"cohere-ai":["partial","*",1,1],"Diffbot":["partial","*",1,1],"ContextFetchBot":["partial","*",1,1]},"llms":{"status":"present","http":200,"bytes":9497},"home":{"status":"ok","http":200,"url":"https://kinsta.com/","html_tokens":32177,"markdown_tokens":932,"pruned_tokens":277}},
{"rank":3728,"domain":"ucs.org","origin":"https://ucs.org","robots":{"status":"ok","http":200,"url":"https://www.ucs.org/robots.txt","bytes":2188,"groups":1},"bots":{"GPTBot":["partial","*",33,18],"ChatGPT-User":["partial","*",33,18],"OAI-SearchBot":["partial","*",33,18],"ClaudeBot":["partial","*",33,18],"Claude-User":["partial","*",33,18],"Claude-SearchBot":["partial","*",33,18],"anthropic-ai":["partial","*",33,18],"PerplexityBot":["partial","*",33,18],"Perplexity-User":["partial","*",33,18],"Google-Extended":["partial","*",33,18],"Applebot-Extended":["partial","*",33,18],"CCBot":["partial","*",33,18],"Bytespider":["partial","*",33,18],"Amazonbot":["partial","*",33,18],"meta-externalagent":["partial","*",33,18],"cohere-ai":["partial","*",33,18],"Diffbot":["partial","*",33,18],"ContextFetchBot":["partial","*",33,18]},"llms":{"status":"missing","http":404,"bytes":185},"home":{"status":"ok","http":200,"url":"https://www.ucs.org/","html_tokens":36048,"markdown_tokens":65,"pruned_tokens":65}},
{"rank":3729,"domain":"rainews.it","origin":"https://rainews.it","robots":{"status":"ok","http":200,"url":"https://www.rainews.it/robots.txt","bytes":3185,"groups":3},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["blocked","named",1,0],"OAI-SearchBot":["partial","*",54,5],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",54,5],"Claude-SearchBot":["partial","*",54,5],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["partial","*",54,5],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["partial","*",54,5],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["partial","*",54,5],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",54,5]},"llms":{"status":"missing","http":404,"bytes":259507},"home":{"status":"ok","http":200,"url":"https://www.rainews.it/","html_tokens":520739,"markdown_tokens":17,"pruned_tokens":17}},
{"rank":3730,"domain":"pydata.org","origin":"https://pydata.org","robots":{"status":"ok","http":200,"url":"https://pydata.org/robots.txt","bytes":126,"groups":1},"bots":{"GPTBot":["partial","*",1,1],"ChatGPT-User":["partial","*",1,1],"OAI-SearchBot":["partial","*",1,1],"ClaudeBot":["partial","*",1,1],"Claude-User":["partial","*",1,1],"Claude-SearchBot":["partial","*",1,1],"anthropic-ai":["partial","*",1,1],"PerplexityBot":["partial","*",1,1],"Perplexity-User":["partial","*",1,1],"Google-Extended":["partial","*",1,1],"Applebot-Extended":["partial","*",1,1],"CCBot":["partial","*",1,1],"Bytespider":["partial","*",1,1],"Amazonbot":["partial","*",1,1],"meta-externalagent":["partial","*",1,1],"cohere-ai":["partial","*",1,1],"Diffbot":["partial","*",1,1],"ContextFetchBot":["partial","*",1,1]},"llms":{"status":"missing","http":404,"bytes":60},"home":{"status":"ok","http":200,"url":"https://pydata.org/","html_tokens":54238,"markdown_tokens":20,"pruned_tokens":20}},
{"rank":3731,"domain":"smotrim.ru","origin":"https://smotrim.ru","robots":{"status":"ok","http":200,"url":"https://smotrim.ru/robots.txt","bytes":6921,"groups":1},"bots":{"GPTBot":["partial","*",30,2],"ChatGPT-User":["partial","*",30,2],"OAI-SearchBot":["partial","*",30,2],"ClaudeBot":["partial","*",30,2],"Claude-User":["partial","*",30,2],"Claude-SearchBot":["partial","*",30,2],"anthropic-ai":["partial","*",30,2],"PerplexityBot":["partial","*",30,2],"Perplexity-User":["partial","*",30,2],"Google-Extended":["partial","*",30,2],"Applebot-Extended":["partial","*",30,2],"CCBot":["partial","*",30,2],"Bytespider":["partial","*",30,2],"Amazonbot":["partial","*",30,2],"meta-externalagent":["partial","*",30,2],"cohere-ai":["partial","*",30,2],"Diffbot":["partial","*",30,2],"ContextFetchBot":["partial","*",30,2]},"llms":{"status":"missing","http":404,"bytes":3052},"home":{"status":"ok","http":200,"url":"https://smotrim.ru/","html_tokens":39758,"markdown_tokens":196,"pruned_tokens":173}},
{"rank":3732,"domain":"tu-darmstadt.de","origin":"https://tu-darmstadt.de","robots":{"status":"ok","http":200,"url":"https://www.tu-darmstadt.de/robots.txt","bytes":281,"groups":1},"bots":{"GPTBot":["partial","*",11,0],"ChatGPT-User":["partial","*",11,0],"OAI-SearchBot":["partial","*",11,0],"ClaudeBot":["partial","*",11,0],"Claude-User":["partial","*",11,0],"Claude-SearchBot":["partial","*",11,0],"anthropic-ai":["partial","*",11,0],"PerplexityBot":["partial","*",11,0],"Perplexity-User":["partial","*",11,0],"Google-Extended":["partial","*",11,0],"Applebot-Extended":["partial","*",11,0],"CCBot":["partial","*",11,0],"Bytespider":["partial","*",11,0],"Amazonbot":["partial","*",11,0],"meta-externalagent":["partial","*",11,0],"cohere-ai":["partial","*",11,0],"Diffbot":["partial","*",11,0],"ContextFetchBot":["partial","*",11,0]},"llms":{"status":"missing","http":404,"bytes":44448},"home":{"status":"ok","http":200,"url":"https://www.tu-darmstadt.de/","html_tokens":31471,"markdown_tokens":396,"pruned_tokens":275}},
{"rank":3733,"domain":"regjeringen.no","origin":"https://regjeringen.no","robots":{"status":"refused","http":403,"url":"https://www.regjeringen.no/robots.txt","bytes":5577,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":3734,"domain":"boxofficemojo.com","origin":"https://boxofficemojo.com","robots":{"status":"ok","http":200,"url":"https://www.boxofficemojo.com/robots.txt","bytes":1471,"groups":2},"bots":{"GPTBot":["blocked","*",1,0],"ChatGPT-User":["blocked","*",1,0],"OAI-SearchBot":["blocked","*",1,0],"ClaudeBot":["blocked","*",1,0],"Claude-User":["blocked","*",1,0],"Claude-SearchBot":["blocked","*",1,0],"anthropic-ai":["blocked","*",1,0],"PerplexityBot":["blocked","*",1,0],"Perplexity-User":["blocked","*",1,0],"Google-Extended":["blocked","*",1,0],"Applebot-Extended":["blocked","*",1,0],"CCBot":["blocked","*",1,0],"Bytespider":["blocked","*",1,0],"Amazonbot":["partial","named",3,1],"meta-externalagent":["blocked","*",1,0],"cohere-ai":["blocked","*",1,0],"Diffbot":["blocked","*",1,0],"ContextFetchBot":["blocked","*",1,0]},"llms":{"status":"disallowed"},"home":{"status":"disallowed"}},
{"rank":3735,"domain":"nexusmods.com","origin":"https://nexusmods.com","robots":{"status":"refused","http":403,"url":"https://www.nexusmods.com/robots.txt","bytes":5534,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":3736,"domain":"ansible.com","origin":"https://ansible.com","robots":{"status":"html","http":200,"url":"https://www.redhat.com/en/ansible-collaborative?intcmp=7015Y000003t7aWQAQ","bytes":243696,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":3737,"domain":"nu.nl","origin":"https://nu.nl","robots":{"status":"ok","http":200,"url":"https://www.nu.nl/robots.txt","bytes":2486,"groups":3},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["blocked","named",1,0],"OAI-SearchBot":["blocked","named",1,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",6,0],"Claude-SearchBot":["partial","*",6,0],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["blocked","named",1,0],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["partial","*",6,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",6,0]},"llms":{"status":"missing","http":404,"bytes":75782},"home":{"status":"ok","http":200,"url":"https://www.nu.nl/","html_tokens":411714,"markdown_tokens":29,"pruned_tokens":29}},
{"rank":3738,"domain":"jcp.org","origin":"https://jcp.org","robots":{"status":"missing","http":404,"url":"https://jcp.org/robots.txt","bytes":11409,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":11409},"home":{"status":"ok","http":200,"url":"https://jcp.org/en/home/index","html_tokens":19362,"markdown_tokens":262,"pruned_tokens":131}},
{"rank":3739,"domain":"mjt.lu","origin":"https://mjt.lu","robots":{"status":"ok","http":200,"url":"https://mjt.lu/robots.txt","bytes":25,"groups":1},"bots":{"GPTBot":["blocked","*",1,0],"ChatGPT-User":["blocked","*",1,0],"OAI-SearchBot":["blocked","*",1,0],"ClaudeBot":["blocked","*",1,0],"Claude-User":["blocked","*",1,0],"Claude-SearchBot":["blocked","*",1,0],"anthropic-ai":["blocked","*",1,0],"PerplexityBot":["blocked","*",1,0],"Perplexity-User":["blocked","*",1,0],"Google-Extended":["blocked","*",1,0],"Applebot-Extended":["blocked","*",1,0],"CCBot":["blocked","*",1,0],"Bytespider":["blocked","*",1,0],"Amazonbot":["blocked","*",1,0],"meta-externalagent":["blocked","*",1,0],"cohere-ai":["blocked","*",1,0],"Diffbot":["blocked","*",1,0],"ContextFetchBot":["blocked","*",1,0]},"llms":{"status":"disallowed"},"home":{"status":"disallowed"}},
{"rank":3740,"domain":"forgejo.org","origin":"https://forgejo.org","robots":{"status":"ok","http":200,"url":"https://forgejo.org/robots.txt","bytes":23,"groups":1},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"missing","http":404,"bytes":25922},"home":{"status":"ok","http":200,"url":"https://forgejo.org/","html_tokens":9864,"markdown_tokens":530,"pruned_tokens":353}},
{"rank":3741,"domain":"muffingroup.com","origin":"https://muffingroup.com","robots":{"status":"ok","http":200,"url":"https://muffingroup.com/robots.txt","bytes":350,"groups":1},"bots":{"GPTBot":["partial","*",16,0],"ChatGPT-User":["partial","*",16,0],"OAI-SearchBot":["partial","*",16,0],"ClaudeBot":["partial","*",16,0],"Claude-User":["partial","*",16,0],"Claude-SearchBot":["partial","*",16,0],"anthropic-ai":["partial","*",16,0],"PerplexityBot":["partial","*",16,0],"Perplexity-User":["partial","*",16,0],"Google-Extended":["partial","*",16,0],"Applebot-Extended":["partial","*",16,0],"CCBot":["partial","*",16,0],"Bytespider":["partial","*",16,0],"Amazonbot":["partial","*",16,0],"meta-externalagent":["partial","*",16,0],"cohere-ai":["partial","*",16,0],"Diffbot":["partial","*",16,0],"ContextFetchBot":["partial","*",16,0]},"llms":{"status":"missing","http":404,"bytes":355},"home":{"status":"ok","http":200,"url":"https://muffingroup.com/","html_tokens":36930,"markdown_tokens":408,"pruned_tokens":118}},
{"rank":3742,"domain":"alibabacloud.com","origin":"https://alibabacloud.com","robots":{"status":"ok","http":200,"url":"https://www.alibabacloud.com/robots.txt","bytes":478,"groups":1},"bots":{"GPTBot":["partial","*",20,0],"ChatGPT-User":["partial","*",20,0],"OAI-SearchBot":["partial","*",20,0],"ClaudeBot":["partial","*",20,0],"Claude-User":["partial","*",20,0],"Claude-SearchBot":["partial","*",20,0],"anthropic-ai":["partial","*",20,0],"PerplexityBot":["partial","*",20,0],"Perplexity-User":["partial","*",20,0],"Google-Extended":["partial","*",20,0],"Applebot-Extended":["partial","*",20,0],"CCBot":["partial","*",20,0],"Bytespider":["partial","*",20,0],"Amazonbot":["partial","*",20,0],"meta-externalagent":["partial","*",20,0],"cohere-ai":["partial","*",20,0],"Diffbot":["partial","*",20,0],"ContextFetchBot":["partial","*",20,0]},"llms":{"status":"html","http":200,"bytes":53506},"home":{"status":"ok","http":200,"url":"https://www.alibabacloud.com/en?_p_lc=5","html_tokens":38584,"markdown_tokens":20,"pruned_tokens":20}},
{"rank":3743,"domain":"runoob.com","origin":"https://www.runoob.com","robots":{"status":"server_error","http":530,"url":"https://www.runoob.com/robots.txt","bytes":8238,"groups":0,"error":"HTTP 530"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":3744,"domain":"qatarairways.com","origin":"https://qatarairways.com","robots":{"status":"refused","http":403,"url":"https://qatarairways.com/robots.txt","bytes":376,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":3745,"domain":"kingston.com","origin":"https://kingston.com","robots":{"status":"ok","http":200,"url":"https://www.kingston.com/robots.txt","bytes":2653,"groups":6},"bots":{"GPTBot":["partial","*",12,0],"ChatGPT-User":["partial","*",12,0],"OAI-SearchBot":["partial","*",12,0],"ClaudeBot":["partial","*",12,0],"Claude-User":["partial","*",12,0],"Claude-SearchBot":["partial","*",12,0],"anthropic-ai":["partial","*",12,0],"PerplexityBot":["partial","*",12,0],"Perplexity-User":["partial","*",12,0],"Google-Extended":["partial","*",12,0],"Applebot-Extended":["partial","*",12,0],"CCBot":["partial","*",12,0],"Bytespider":["partial","*",12,0],"Amazonbot":["partial","*",12,0],"meta-externalagent":["partial","*",12,0],"cohere-ai":["partial","*",12,0],"Diffbot":["partial","*",12,0],"ContextFetchBot":["partial","*",12,0]},"llms":{"status":"missing","http":404,"bytes":403318},"home":{"status":"http_error","http":403,"url":"https://kingston.com/"}},
{"rank":3746,"domain":"kanagawa.jp","origin":"https://kanagawa.jp","robots":{"status":"unreachable","http":null,"url":"https://kanagawa.jp/robots.txt","bytes":0,"groups":0,"error":"cannot resolve kanagawa.jp"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":3747,"domain":"dummies.com","origin":"https://dummies.com","robots":{"status":"ok","http":200,"url":"https://www.dummies.com/robots/robots.txt","bytes":79,"groups":1},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"missing","http":404,"bytes":95095},"home":{"status":"ok","http":200,"url":"https://www.dummies.com/","html_tokens":158734,"markdown_tokens":9465,"pruned_tokens":1001}},
{"rank":3748,"domain":"blogtalkradio.com","origin":"https://blogtalkradio.com","robots":{"status":"unreachable","http":null,"url":"https://blogtalkradio.com/robots.txt","bytes":0,"groups":0,"error":"cannot resolve blogtalkradio.com"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":3749,"domain":"dior.com","origin":"https://dior.com","robots":{"status":"refused","http":403,"url":"https://www.dior.com/robots.txt","bytes":752,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":3750,"domain":"blip.tv","origin":"https://blip.tv","robots":{"status":"html","http":200,"url":"https://www.disney.com/","bytes":435238,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":3751,"domain":"bible.com","origin":"https://bible.com","robots":{"status":"ok","http":200,"url":"https://www.bible.com/robots.txt","bytes":695,"groups":1},"bots":{"GPTBot":["partial","*",15,0],"ChatGPT-User":["partial","*",15,0],"OAI-SearchBot":["partial","*",15,0],"ClaudeBot":["partial","*",15,0],"Claude-User":["partial","*",15,0],"Claude-SearchBot":["partial","*",15,0],"anthropic-ai":["partial","*",15,0],"PerplexityBot":["partial","*",15,0],"Perplexity-User":["partial","*",15,0],"Google-Extended":["partial","*",15,0],"Applebot-Extended":["partial","*",15,0],"CCBot":["partial","*",15,0],"Bytespider":["partial","*",15,0],"Amazonbot":["partial","*",15,0],"meta-externalagent":["partial","*",15,0],"cohere-ai":["partial","*",15,0],"Diffbot":["partial","*",15,0],"ContextFetchBot":["partial","*",15,0]},"llms":{"status":"missing","http":404,"bytes":75984},"home":{"status":"ok","http":200,"url":"https://www.bible.com/","html_tokens":105478,"markdown_tokens":4,"pruned_tokens":4}},
{"rank":3752,"domain":"nd.gov","origin":"https://nd.gov","robots":{"status":"ok","http":200,"url":"https://www.nd.gov/robots.txt","bytes":2141,"groups":1},"bots":{"GPTBot":["partial","*",35,22],"ChatGPT-User":["partial","*",35,22],"OAI-SearchBot":["partial","*",35,22],"ClaudeBot":["partial","*",35,22],"Claude-User":["partial","*",35,22],"Claude-SearchBot":["partial","*",35,22],"anthropic-ai":["partial","*",35,22],"PerplexityBot":["partial","*",35,22],"Perplexity-User":["partial","*",35,22],"Google-Extended":["partial","*",35,22],"Applebot-Extended":["partial","*",35,22],"CCBot":["partial","*",35,22],"Bytespider":["partial","*",35,22],"Amazonbot":["partial","*",35,22],"meta-externalagent":["partial","*",35,22],"cohere-ai":["partial","*",35,22],"Diffbot":["partial","*",35,22],"ContextFetchBot":["partial","*",35,22]},"llms":{"status":"missing","http":404,"bytes":31157},"home":{"status":"ok","http":200,"url":"https://www.nd.gov/","html_tokens":23938,"markdown_tokens":129,"pruned_tokens":129}},
{"rank":3753,"domain":"openstreetmap.fr","origin":"https://openstreetmap.fr","robots":{"status":"ok","http":200,"url":"https://openstreetmap.fr/robots.txt","bytes":70,"groups":1},"bots":{"GPTBot":["partial","*",1,1],"ChatGPT-User":["partial","*",1,1],"OAI-SearchBot":["partial","*",1,1],"ClaudeBot":["partial","*",1,1],"Claude-User":["partial","*",1,1],"Claude-SearchBot":["partial","*",1,1],"anthropic-ai":["partial","*",1,1],"PerplexityBot":["partial","*",1,1],"Perplexity-User":["partial","*",1,1],"Google-Extended":["partial","*",1,1],"Applebot-Extended":["partial","*",1,1],"CCBot":["partial","*",1,1],"Bytespider":["partial","*",1,1],"Amazonbot":["partial","*",1,1],"meta-externalagent":["partial","*",1,1],"cohere-ai":["partial","*",1,1],"Diffbot":["partial","*",1,1],"ContextFetchBot":["partial","*",1,1]},"llms":{"status":"missing","http":404,"bytes":29137},"home":{"status":"ok","http":200,"url":"https://www.openstreetmap.fr/","html_tokens":12041,"markdown_tokens":337,"pruned_tokens":193}},
{"rank":3754,"domain":"earthday.org","origin":"https://earthday.org","robots":{"status":"ok","http":200,"url":"https://earthday.org/robots.txt","bytes":66,"groups":1},"bots":{"GPTBot":["partial","*",1,1],"ChatGPT-User":["partial","*",1,1],"OAI-SearchBot":["partial","*",1,1],"ClaudeBot":["partial","*",1,1],"Claude-User":["partial","*",1,1],"Claude-SearchBot":["partial","*",1,1],"anthropic-ai":["partial","*",1,1],"PerplexityBot":["partial","*",1,1],"Perplexity-User":["partial","*",1,1],"Google-Extended":["partial","*",1,1],"Applebot-Extended":["partial","*",1,1],"CCBot":["partial","*",1,1],"Bytespider":["partial","*",1,1],"Amazonbot":["partial","*",1,1],"meta-externalagent":["partial","*",1,1],"cohere-ai":["partial","*",1,1],"Diffbot":["partial","*",1,1],"ContextFetchBot":["partial","*",1,1]},"llms":{"status":"other_text","http":200,"bytes":6021},"home":{"status":"ok","http":200,"url":"https://www.earthday.org/","html_tokens":108420,"markdown_tokens":1148,"pruned_tokens":276}},
{"rank":3755,"domain":"upf.edu","origin":"https://upf.edu","robots":{"status":"ok","http":200,"url":"https://www.upf.edu/robots.txt","bytes":23,"groups":1},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"missing","http":404,"bytes":160072},"home":{"status":"http_error","http":403,"url":"https://www.upf.edu/"}},
{"rank":3756,"domain":"ya.ru","origin":"https://ya.ru","robots":{"status":"ok","http":200,"url":"https://ya.ru/robots.txt","bytes":29944,"groups":5},"bots":{"GPTBot":["partial","*",221,49],"ChatGPT-User":["partial","*",221,49],"OAI-SearchBot":["partial","*",221,49],"ClaudeBot":["partial","*",221,49],"Claude-User":["partial","*",221,49],"Claude-SearchBot":["partial","*",221,49],"anthropic-ai":["partial","*",221,49],"PerplexityBot":["partial","*",221,49],"Perplexity-User":["partial","*",221,49],"Google-Extended":["partial","*",221,49],"Applebot-Extended":["partial","*",221,49],"CCBot":["partial","*",221,49],"Bytespider":["partial","*",221,49],"Amazonbot":["partial","*",221,49],"meta-externalagent":["partial","*",221,49],"cohere-ai":["partial","*",221,49],"Diffbot":["partial","*",221,49],"ContextFetchBot":["partial","*",221,49]},"llms":{"status":"missing","http":404,"bytes":11769},"home":{"status":"ok","http":200,"url":"https://ya.ru/","html_tokens":121352,"markdown_tokens":22,"pruned_tokens":22}},
{"rank":3757,"domain":"crypto.com","origin":"https://crypto.com","robots":{"status":"ok","http":200,"url":"https://crypto.com/robots.txt","bytes":606,"groups":3},"bots":{"GPTBot":["partial","*",11,0],"ChatGPT-User":["partial","*",11,0],"OAI-SearchBot":["partial","*",11,0],"ClaudeBot":["partial","*",11,0],"Claude-User":["partial","*",11,0],"Claude-SearchBot":["partial","*",11,0],"anthropic-ai":["partial","*",11,0],"PerplexityBot":["partial","*",11,0],"Perplexity-User":["partial","*",11,0],"Google-Extended":["partial","*",11,0],"Applebot-Extended":["partial","*",11,0],"CCBot":["partial","*",11,0],"Bytespider":["partial","*",11,0],"Amazonbot":["partial","*",11,0],"meta-externalagent":["partial","*",11,0],"cohere-ai":["partial","*",11,0],"Diffbot":["partial","*",11,0],"ContextFetchBot":["partial","*",11,0]},"llms":{"status":"present","http":200,"bytes":2610},"home":{"status":"ok","http":200,"url":"https://crypto.com/us","html_tokens":52847,"markdown_tokens":1331,"pruned_tokens":712}},
{"rank":3758,"domain":"eugdpr.org","origin":"http://eugdpr.org","robots":{"status":"missing","http":404,"url":"http://eugdpr.org/robots.txt","bytes":10370,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":10370},"home":{"status":"ok","http":200,"url":"http://eugdpr.org/","html_tokens":41,"markdown_tokens":0,"pruned_tokens":0}},
{"rank":3759,"domain":"worldscientific.com","origin":"https://worldscientific.com","robots":{"status":"ok","http":200,"url":"https://www.worldscientific.com/robots.txt","bytes":786,"groups":3},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["partial","*",18,6],"OAI-SearchBot":["partial","*",18,6],"ClaudeBot":["partial","*",18,6],"Claude-User":["partial","*",18,6],"Claude-SearchBot":["partial","*",18,6],"anthropic-ai":["partial","*",18,6],"PerplexityBot":["partial","*",18,6],"Perplexity-User":["partial","*",18,6],"Google-Extended":["partial","*",18,6],"Applebot-Extended":["partial","*",18,6],"CCBot":["partial","*",18,6],"Bytespider":["partial","*",18,6],"Amazonbot":["partial","*",18,6],"meta-externalagent":["partial","*",18,6],"cohere-ai":["partial","*",18,6],"Diffbot":["partial","*",18,6],"ContextFetchBot":["partial","*",18,6]},"llms":{"status":"missing","http":404,"bytes":26},"home":{"status":"http_error","http":403,"url":"https://www.worldscientific.com/"}},
{"rank":3760,"domain":"google.com.vn","origin":"https://google.com.vn","robots":{"status":"ok","http":200,"url":"https://www.google.com.vn/robots.txt","bytes":6624,"groups":4},"bots":{"GPTBot":["partial","*",169,64],"ChatGPT-User":["partial","*",169,64],"OAI-SearchBot":["partial","*",169,64],"ClaudeBot":["partial","*",169,64],"Claude-User":["partial","*",169,64],"Claude-SearchBot":["partial","*",169,64],"anthropic-ai":["partial","*",169,64],"PerplexityBot":["partial","*",169,64],"Perplexity-User":["partial","*",169,64],"Google-Extended":["partial","*",169,64],"Applebot-Extended":["partial","*",169,64],"CCBot":["partial","*",169,64],"Bytespider":["partial","*",169,64],"Amazonbot":["partial","*",169,64],"meta-externalagent":["partial","*",169,64],"cohere-ai":["partial","*",169,64],"Diffbot":["partial","*",169,64],"ContextFetchBot":["partial","*",169,64]},"llms":{"status":"missing","http":404,"bytes":1569},"home":{"status":"ok","http":200,"url":"https://www.google.com/","html_tokens":21277,"markdown_tokens":30,"pruned_tokens":30}},
{"rank":3761,"domain":"abs.gov.au","origin":"https://abs.gov.au","robots":{"status":"ok","http":200,"url":"https://www.abs.gov.au/robots.txt","bytes":641,"groups":1},"bots":{"GPTBot":["partial","*",5,18],"ChatGPT-User":["partial","*",5,18],"OAI-SearchBot":["partial","*",5,18],"ClaudeBot":["partial","*",5,18],"Claude-User":["partial","*",5,18],"Claude-SearchBot":["partial","*",5,18],"anthropic-ai":["partial","*",5,18],"PerplexityBot":["partial","*",5,18],"Perplexity-User":["partial","*",5,18],"Google-Extended":["partial","*",5,18],"Applebot-Extended":["partial","*",5,18],"CCBot":["partial","*",5,18],"Bytespider":["partial","*",5,18],"Amazonbot":["partial","*",5,18],"meta-externalagent":["partial","*",5,18],"cohere-ai":["partial","*",5,18],"Diffbot":["partial","*",5,18],"ContextFetchBot":["partial","*",5,18]},"llms":{"status":"missing","http":404,"bytes":66400},"home":{"status":"ok","http":200,"url":"https://www.abs.gov.au/","html_tokens":25975,"markdown_tokens":137,"pruned_tokens":137}},
{"rank":3762,"domain":"campaignmonitor.com","origin":"https://campaignmonitor.com","robots":{"status":"ok","http":200,"url":"https://campaignmonitor.com/robots.txt","bytes":591,"groups":1},"bots":{"GPTBot":["partial","*",3,1],"ChatGPT-User":["partial","*",3,1],"OAI-SearchBot":["partial","*",3,1],"ClaudeBot":["partial","*",3,1],"Claude-User":["partial","*",3,1],"Claude-SearchBot":["partial","*",3,1],"anthropic-ai":["partial","*",3,1],"PerplexityBot":["partial","*",3,1],"Perplexity-User":["partial","*",3,1],"Google-Extended":["partial","*",3,1],"Applebot-Extended":["partial","*",3,1],"CCBot":["partial","*",3,1],"Bytespider":["partial","*",3,1],"Amazonbot":["partial","*",3,1],"meta-externalagent":["partial","*",3,1],"cohere-ai":["partial","*",3,1],"Diffbot":["partial","*",3,1],"ContextFetchBot":["partial","*",3,1]},"llms":{"status":"http_error","http":403,"bytes":5572},"home":{"status":"http_error","http":403,"url":"https://campaignmonitor.com/"}},
{"rank":3763,"domain":"reading.ac.uk","origin":"https://reading.ac.uk","robots":{"status":"ok","http":200,"url":"https://reading.ac.uk/robots.txt","bytes":7640,"groups":1},"bots":{"GPTBot":["partial","*",27,0],"ChatGPT-User":["partial","*",27,0],"OAI-SearchBot":["partial","*",27,0],"ClaudeBot":["partial","*",27,0],"Claude-User":["partial","*",27,0],"Claude-SearchBot":["partial","*",27,0],"anthropic-ai":["partial","*",27,0],"PerplexityBot":["partial","*",27,0],"Perplexity-User":["partial","*",27,0],"Google-Extended":["partial","*",27,0],"Applebot-Extended":["partial","*",27,0],"CCBot":["partial","*",27,0],"Bytespider":["partial","*",27,0],"Amazonbot":["partial","*",27,0],"meta-externalagent":["partial","*",27,0],"cohere-ai":["partial","*",27,0],"Diffbot":["partial","*",27,0],"ContextFetchBot":["partial","*",27,0]},"llms":{"status":"html","http":200,"bytes":8135},"home":{"status":"ok","http":200,"url":"https://reading.ac.uk/","html_tokens":41523,"markdown_tokens":94,"pruned_tokens":43}},
{"rank":3764,"domain":"geogebra.org","origin":"https://geogebra.org","robots":{"status":"ok","http":200,"url":"https://geogebra.org/robots.txt","bytes":279,"groups":1},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"missing","http":404,"bytes":146585},"home":{"status":"ok","http":200,"url":"https://www.geogebra.org/","html_tokens":59646,"markdown_tokens":1100,"pruned_tokens":474}},
{"rank":3765,"domain":"darkreading.com","origin":"https://darkreading.com","robots":{"status":"ok","http":200,"url":"https://www.darkreading.com/robots.txt","bytes":1609,"groups":2},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["partial","*",6,0],"OAI-SearchBot":["partial","*",6,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",6,0],"Claude-SearchBot":["partial","*",6,0],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["partial","*",6,0],"Perplexity-User":["partial","*",6,0],"Google-Extended":["partial","*",6,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",6,0]},"llms":{"status":"missing","http":404,"bytes":311093},"home":{"status":"http_error","http":403,"url":"https://darkreading.com/"}},
{"rank":3766,"domain":"404media.co","origin":"https://404media.co","robots":{"status":"ok","http":200,"url":"https://www.404media.co/robots.txt","bytes":213,"groups":2},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["partial","*",5,0],"OAI-SearchBot":["partial","*",5,0],"ClaudeBot":["partial","*",5,0],"Claude-User":["partial","*",5,0],"Claude-SearchBot":["partial","*",5,0],"anthropic-ai":["partial","*",5,0],"PerplexityBot":["partial","*",5,0],"Perplexity-User":["partial","*",5,0],"Google-Extended":["partial","*",5,0],"Applebot-Extended":["partial","*",5,0],"CCBot":["partial","*",5,0],"Bytespider":["partial","*",5,0],"Amazonbot":["partial","*",5,0],"meta-externalagent":["partial","*",5,0],"cohere-ai":["partial","*",5,0],"Diffbot":["partial","*",5,0],"ContextFetchBot":["partial","*",5,0]},"llms":{"status":"html","http":200,"bytes":69130},"home":{"status":"ok","http":200,"url":"https://www.404media.co/","html_tokens":17269,"markdown_tokens":318,"pruned_tokens":150}},
{"rank":3767,"domain":"obsidian.md","origin":"https://obsidian.md","robots":{"status":"ok","http":200,"url":"https://obsidian.md/robots.txt","bytes":23,"groups":1},"bots":{"GPTBot":["allowed","*",0,1],"ChatGPT-User":["allowed","*",0,1],"OAI-SearchBot":["allowed","*",0,1],"ClaudeBot":["allowed","*",0,1],"Claude-User":["allowed","*",0,1],"Claude-SearchBot":["allowed","*",0,1],"anthropic-ai":["allowed","*",0,1],"PerplexityBot":["allowed","*",0,1],"Perplexity-User":["allowed","*",0,1],"Google-Extended":["allowed","*",0,1],"Applebot-Extended":["allowed","*",0,1],"CCBot":["allowed","*",0,1],"Bytespider":["allowed","*",0,1],"Amazonbot":["allowed","*",0,1],"meta-externalagent":["allowed","*",0,1],"cohere-ai":["allowed","*",0,1],"Diffbot":["allowed","*",0,1],"ContextFetchBot":["allowed","*",0,1]},"llms":{"status":"missing","http":404,"bytes":34071},"home":{"status":"ok","http":200,"url":"https://obsidian.md/","html_tokens":24187,"markdown_tokens":637,"pruned_tokens":60}},
{"rank":3768,"domain":"businesstoday.in","origin":"https://businesstoday.in","robots":{"status":"ok","http":200,"url":"https://www.businesstoday.in/robots.txt","bytes":881,"groups":2},"bots":{"GPTBot":["partial","*",17,1],"ChatGPT-User":["partial","*",17,1],"OAI-SearchBot":["partial","*",17,1],"ClaudeBot":["partial","*",17,1],"Claude-User":["partial","*",17,1],"Claude-SearchBot":["partial","*",17,1],"anthropic-ai":["partial","*",17,1],"PerplexityBot":["partial","*",17,1],"Perplexity-User":["partial","*",17,1],"Google-Extended":["partial","*",17,1],"Applebot-Extended":["partial","*",17,1],"CCBot":["partial","*",17,1],"Bytespider":["partial","*",17,1],"Amazonbot":["partial","*",17,1],"meta-externalagent":["partial","*",17,1],"cohere-ai":["partial","*",17,1],"Diffbot":["partial","*",17,1],"ContextFetchBot":["partial","*",17,1]},"llms":{"status":"missing","http":404,"bytes":226080},"home":{"status":"ok","http":200,"url":"https://www.businesstoday.in/","html_tokens":514691,"markdown_tokens":488,"pruned_tokens":173}},
{"rank":3769,"domain":"thinkgeek.com","origin":"https://thinkgeek.com","robots":{"status":"refused","http":403,"url":"https://www.gamestop.com/collectibles","bytes":5018,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":3770,"domain":"studiopress.com","origin":"https://studiopress.com","robots":{"status":"ok","http":200,"url":"https://www.studiopress.com/robots.txt","bytes":193,"groups":1},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"missing","http":404,"bytes":60},"home":{"status":"ok","http":200,"url":"https://www.studiopress.com/","html_tokens":32125,"markdown_tokens":368,"pruned_tokens":115}},
{"rank":3771,"domain":"simplypsychology.org","origin":"https://simplypsychology.org","robots":{"status":"ok","http":200,"url":"https://www.simplypsychology.org/robots.txt","bytes":2148,"groups":32},"bots":{"GPTBot":["allowed","named",0,1],"ChatGPT-User":["allowed","named",0,1],"OAI-SearchBot":["allowed","named",0,1],"ClaudeBot":["allowed","named",0,1],"Claude-User":["allowed","named",0,1],"Claude-SearchBot":["allowed","named",0,1],"anthropic-ai":["allowed","named",0,1],"PerplexityBot":["allowed","named",0,1],"Perplexity-User":["allowed","named",0,1],"Google-Extended":["allowed","named",0,1],"Applebot-Extended":["allowed","named",0,1],"CCBot":["allowed","named",0,1],"Bytespider":["partial","*",4,1],"Amazonbot":["partial","*",4,1],"meta-externalagent":["allowed","named",0,1],"cohere-ai":["partial","*",4,1],"Diffbot":["partial","*",4,1],"ContextFetchBot":["partial","*",4,1]},"llms":{"status":"missing","http":404,"bytes":174977},"home":{"status":"ok","http":200,"url":"https://www.simplypsychology.org/","html_tokens":56978,"markdown_tokens":482,"pruned_tokens":135}},
{"rank":3772,"domain":"gameinformer.com","origin":"https://gameinformer.com","robots":{"status":"ok","http":200,"url":"https://gameinformer.com/robots.txt","bytes":2726,"groups":1},"bots":{"GPTBot":["partial","*",45,18],"ChatGPT-User":["partial","*",45,18],"OAI-SearchBot":["partial","*",45,18],"ClaudeBot":["partial","*",45,18],"Claude-User":["partial","*",45,18],"Claude-SearchBot":["partial","*",45,18],"anthropic-ai":["partial","*",45,18],"PerplexityBot":["partial","*",45,18],"Perplexity-User":["partial","*",45,18],"Google-Extended":["partial","*",45,18],"Applebot-Extended":["partial","*",45,18],"CCBot":["partial","*",45,18],"Bytespider":["partial","*",45,18],"Amazonbot":["partial","*",45,18],"meta-externalagent":["partial","*",45,18],"cohere-ai":["partial","*",45,18],"Diffbot":["partial","*",45,18],"ContextFetchBot":["partial","*",45,18]},"llms":{"status":"present","http":200,"bytes":3564},"home":{"status":"ok","http":200,"url":"https://gameinformer.com/","html_tokens":64971,"markdown_tokens":98,"pruned_tokens":98}},
{"rank":3773,"domain":"hotmart.com","origin":"https://hotmart.com","robots":{"status":"ok","http":200,"url":"https://hotmart.com/robots.txt","bytes":1684,"groups":1},"bots":{"GPTBot":["partial","*",22,1],"ChatGPT-User":["partial","*",22,1],"OAI-SearchBot":["partial","*",22,1],"ClaudeBot":["partial","*",22,1],"Claude-User":["partial","*",22,1],"Claude-SearchBot":["partial","*",22,1],"anthropic-ai":["partial","*",22,1],"PerplexityBot":["partial","*",22,1],"Perplexity-User":["partial","*",22,1],"Google-Extended":["partial","*",22,1],"Applebot-Extended":["partial","*",22,1],"CCBot":["partial","*",22,1],"Bytespider":["partial","*",22,1],"Amazonbot":["partial","*",22,1],"meta-externalagent":["partial","*",22,1],"cohere-ai":["partial","*",22,1],"Diffbot":["partial","*",22,1],"ContextFetchBot":["partial","*",22,1]},"llms":{"status":"present","http":200,"bytes":15724},"home":{"status":"ok","http":200,"url":"https://hotmart.com/en","html_tokens":93909,"markdown_tokens":1435,"pruned_tokens":243}},
{"rank":3774,"domain":"indystar.com","origin":"https://indystar.com","robots":{"status":"ok","http":200,"url":"https://www.indystar.com/robots.txt","bytes":14297,"groups":288},"bots":{"GPTBot":["partial","named",1,6],"ChatGPT-User":["partial","named",1,6],"OAI-SearchBot":["partial","named",1,6],"ClaudeBot":["blocked","named",1,0],"Claude-User":["blocked","named",1,0],"Claude-SearchBot":["blocked","named",1,0],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["partial","named",1,6],"Perplexity-User":["blocked","named",1,0],"Google-Extended":["partial","named",1,6],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",2,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",33,0]},"llms":{"status":"http_error","http":402,"bytes":254},"home":{"status":"http_error","http":402,"url":"https://www.indystar.com/"}},
{"rank":3775,"domain":"english-heritage.org.uk","origin":"https://english-heritage.org.uk","robots":{"status":"ok","http":200,"url":"https://www.english-heritage.org.uk/robots.txt","bytes":1941,"groups":1},"bots":{"GPTBot":["partial","*",4,0],"ChatGPT-User":["partial","*",4,0],"OAI-SearchBot":["partial","*",4,0],"ClaudeBot":["partial","*",4,0],"Claude-User":["partial","*",4,0],"Claude-SearchBot":["partial","*",4,0],"anthropic-ai":["partial","*",4,0],"PerplexityBot":["partial","*",4,0],"Perplexity-User":["partial","*",4,0],"Google-Extended":["partial","*",4,0],"Applebot-Extended":["partial","*",4,0],"CCBot":["partial","*",4,0],"Bytespider":["partial","*",4,0],"Amazonbot":["partial","*",4,0],"meta-externalagent":["partial","*",4,0],"cohere-ai":["partial","*",4,0],"Diffbot":["partial","*",4,0],"ContextFetchBot":["partial","*",4,0]},"llms":{"status":"html","http":200,"bytes":3177},"home":{"status":"ok","http":200,"url":"https://www.english-heritage.org.uk/","html_tokens":24411,"markdown_tokens":540,"pruned_tokens":74}},
{"rank":3776,"domain":"winamp.com","origin":"https://winamp.com","robots":{"status":"ok","http":200,"url":"https://winamp.com/robots.txt","bytes":63,"groups":1},"bots":{"GPTBot":["allowed","*",0,1],"ChatGPT-User":["allowed","*",0,1],"OAI-SearchBot":["allowed","*",0,1],"ClaudeBot":["allowed","*",0,1],"Claude-User":["allowed","*",0,1],"Claude-SearchBot":["allowed","*",0,1],"anthropic-ai":["allowed","*",0,1],"PerplexityBot":["allowed","*",0,1],"Perplexity-User":["allowed","*",0,1],"Google-Extended":["allowed","*",0,1],"Applebot-Extended":["allowed","*",0,1],"CCBot":["allowed","*",0,1],"Bytespider":["allowed","*",0,1],"Amazonbot":["allowed","*",0,1],"meta-externalagent":["allowed","*",0,1],"cohere-ai":["allowed","*",0,1],"Diffbot":["allowed","*",0,1],"ContextFetchBot":["allowed","*",0,1]},"llms":{"status":"missing","http":404,"bytes":10},"home":{"status":"ok","http":200,"url":"https://winamp.com/","html_tokens":57542,"markdown_tokens":53,"pruned_tokens":53}},
{"rank":3777,"domain":"icims.com","origin":"https://icims.com","robots":{"status":"refused","http":403,"url":"https://www.icims.com/robots.txt","bytes":560,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":3778,"domain":"skool.com","origin":"https://skool.com","robots":{"status":"ok","http":200,"url":"https://www.skool.com/robots.txt","bytes":41,"groups":1},"bots":{"GPTBot":["partial","*",1,1],"ChatGPT-User":["partial","*",1,1],"OAI-SearchBot":["partial","*",1,1],"ClaudeBot":["partial","*",1,1],"Claude-User":["partial","*",1,1],"Claude-SearchBot":["partial","*",1,1],"anthropic-ai":["partial","*",1,1],"PerplexityBot":["partial","*",1,1],"Perplexity-User":["partial","*",1,1],"Google-Extended":["partial","*",1,1],"Applebot-Extended":["partial","*",1,1],"CCBot":["partial","*",1,1],"Bytespider":["partial","*",1,1],"Amazonbot":["partial","*",1,1],"meta-externalagent":["partial","*",1,1],"cohere-ai":["partial","*",1,1],"Diffbot":["partial","*",1,1],"ContextFetchBot":["partial","*",1,1]},"llms":{"status":"missing","http":404,"bytes":3576},"home":{"status":"ok","http":202,"url":"https://www.skool.com/","html_tokens":524,"markdown_tokens":40,"pruned_tokens":40}},
{"rank":3779,"domain":"bfi.org.uk","origin":"https://bfi.org.uk","robots":{"status":"missing","http":404,"url":"https://www.bfi.org.uk/robots.txt","bytes":42771,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":42773},"home":{"status":"ok","http":200,"url":"https://www.bfi.org.uk/","html_tokens":84964,"markdown_tokens":262,"pruned_tokens":61}},
{"rank":3780,"domain":"gale.com","origin":"https://gale.com","robots":{"status":"ok","http":200,"url":"https://gale.com/robots.txt","bytes":357,"groups":2},"bots":{"GPTBot":["partial","*",4,2],"ChatGPT-User":["partial","*",4,2],"OAI-SearchBot":["partial","*",4,2],"ClaudeBot":["partial","*",4,2],"Claude-User":["partial","*",4,2],"Claude-SearchBot":["partial","*",4,2],"anthropic-ai":["partial","*",4,2],"PerplexityBot":["partial","*",4,2],"Perplexity-User":["partial","*",4,2],"Google-Extended":["partial","*",4,2],"Applebot-Extended":["partial","*",4,2],"CCBot":["partial","*",4,2],"Bytespider":["partial","*",4,2],"Amazonbot":["partial","*",4,2],"meta-externalagent":["partial","*",4,2],"cohere-ai":["partial","*",4,2],"Diffbot":["partial","*",4,2],"ContextFetchBot":["partial","*",4,2]},"llms":{"status":"html","http":200,"bytes":154339},"home":{"status":"ok","http":200,"url":"https://gale.com/","html_tokens":76871,"markdown_tokens":18,"pruned_tokens":18}},
{"rank":3781,"domain":"e-gov.go.jp","origin":"https://www.e-gov.go.jp","robots":{"status":"ok","http":200,"url":"https://www.e-gov.go.jp/robots.txt","bytes":2027,"groups":1},"bots":{"GPTBot":["partial","*",34,18],"ChatGPT-User":["partial","*",34,18],"OAI-SearchBot":["partial","*",34,18],"ClaudeBot":["partial","*",34,18],"Claude-User":["partial","*",34,18],"Claude-SearchBot":["partial","*",34,18],"anthropic-ai":["partial","*",34,18],"PerplexityBot":["partial","*",34,18],"Perplexity-User":["partial","*",34,18],"Google-Extended":["partial","*",34,18],"Applebot-Extended":["partial","*",34,18],"CCBot":["partial","*",34,18],"Bytespider":["partial","*",34,18],"Amazonbot":["partial","*",34,18],"meta-externalagent":["partial","*",34,18],"cohere-ai":["partial","*",34,18],"Diffbot":["partial","*",34,18],"ContextFetchBot":["partial","*",34,18]},"llms":{"status":"missing","http":404,"bytes":188},"home":{"status":"ok","http":200,"url":"https://www.e-gov.go.jp/","html_tokens":7406,"markdown_tokens":111,"pruned_tokens":100}},
{"rank":3782,"domain":"recode.net","origin":"https://recode.net","robots":{"status":"server_error","http":500,"url":"https://recode.net/robots.txt","bytes":266,"groups":0,"error":"HTTP 500"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":3783,"domain":"spoti.fi","origin":"https://spoti.fi","robots":{"status":"ok","http":200,"url":"https://spoti.fi/robots.txt","bytes":75,"groups":1},"bots":{"GPTBot":["partial","*",1,0],"ChatGPT-User":["partial","*",1,0],"OAI-SearchBot":["partial","*",1,0],"ClaudeBot":["partial","*",1,0],"Claude-User":["partial","*",1,0],"Claude-SearchBot":["partial","*",1,0],"anthropic-ai":["partial","*",1,0],"PerplexityBot":["partial","*",1,0],"Perplexity-User":["partial","*",1,0],"Google-Extended":["partial","*",1,0],"Applebot-Extended":["partial","*",1,0],"CCBot":["partial","*",1,0],"Bytespider":["partial","*",1,0],"Amazonbot":["partial","*",1,0],"meta-externalagent":["partial","*",1,0],"cohere-ai":["partial","*",1,0],"Diffbot":["partial","*",1,0],"ContextFetchBot":["partial","*",1,0]},"llms":{"status":"html","http":200,"bytes":169313},"home":{"status":"ok","http":200,"url":"https://open.spotify.com/","html_tokens":42345,"markdown_tokens":0,"pruned_tokens":0}},
{"rank":3784,"domain":"adl.org","origin":"https://adl.org","robots":{"status":"ok","http":200,"url":"https://adl.org/robots.txt","bytes":7189,"groups":30},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["allowed","named",0,1],"OAI-SearchBot":["allowed","named",0,1],"ClaudeBot":["allowed","named",0,1],"Claude-User":["partial","*",62,42],"Claude-SearchBot":["partial","*",62,42],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["allowed","named",0,2],"Perplexity-User":["partial","*",62,42],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",2,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",62,42]},"llms":{"status":"missing","http":404,"bytes":546},"home":{"status":"ok","http":200,"url":"https://www.adl.org/","html_tokens":39903,"markdown_tokens":355,"pruned_tokens":248}},
{"rank":3785,"domain":"hasbro.com","origin":"https://hasbro.com","robots":{"status":"html","http":200,"url":"https://consumercare.hasbro.com/en-us/where-to-buy","bytes":132681,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":3786,"domain":"smarturl.it","origin":"https://smarturl.it","robots":{"status":"ok","http":200,"url":"https://smarturl.it/robots.txt","bytes":23,"groups":1},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"http_error","http":403,"bytes":118},"home":{"status":"http_error","http":404,"url":"https://www.linkfire.com/en-us/linkfire-acquired-smarturl"}},
{"rank":3787,"domain":"tv-asahi.co.jp","origin":"https://tv-asahi.co.jp","robots":{"status":"html","http":200,"url":"https://www.tv-asahi.co.jp/","bytes":9642,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":3788,"domain":"make.com","origin":"https://make.com","robots":{"status":"ok","http":200,"url":"https://www.make.com/robots.txt","bytes":499,"groups":1},"bots":{"GPTBot":["partial","*",18,3],"ChatGPT-User":["partial","*",18,3],"OAI-SearchBot":["partial","*",18,3],"ClaudeBot":["partial","*",18,3],"Claude-User":["partial","*",18,3],"Claude-SearchBot":["partial","*",18,3],"anthropic-ai":["partial","*",18,3],"PerplexityBot":["partial","*",18,3],"Perplexity-User":["partial","*",18,3],"Google-Extended":["partial","*",18,3],"Applebot-Extended":["partial","*",18,3],"CCBot":["partial","*",18,3],"Bytespider":["partial","*",18,3],"Amazonbot":["partial","*",18,3],"meta-externalagent":["partial","*",18,3],"cohere-ai":["partial","*",18,3],"Diffbot":["partial","*",18,3],"ContextFetchBot":["partial","*",18,3]},"llms":{"status":"present","http":200,"bytes":14363},"home":{"status":"ok","http":200,"url":"https://www.make.com/","html_tokens":87824,"markdown_tokens":735,"pruned_tokens":86}},
{"rank":3789,"domain":"labiennale.org","origin":"https://labiennale.org","robots":{"status":"ok","http":200,"url":"https://www.labiennale.org/robots.txt","bytes":2189,"groups":1},"bots":{"GPTBot":["partial","*",36,32],"ChatGPT-User":["partial","*",36,32],"OAI-SearchBot":["partial","*",36,32],"ClaudeBot":["partial","*",36,32],"Claude-User":["partial","*",36,32],"Claude-SearchBot":["partial","*",36,32],"anthropic-ai":["partial","*",36,32],"PerplexityBot":["partial","*",36,32],"Perplexity-User":["partial","*",36,32],"Google-Extended":["partial","*",36,32],"Applebot-Extended":["partial","*",36,32],"CCBot":["partial","*",36,32],"Bytespider":["partial","*",36,32],"Amazonbot":["partial","*",36,32],"meta-externalagent":["partial","*",36,32],"cohere-ai":["partial","*",36,32],"Diffbot":["partial","*",36,32],"ContextFetchBot":["partial","*",36,32]},"llms":{"status":"timeout","error":"timed out after 10 s"},"home":{"status":"timeout","error":"timed out after 15 s"}},
{"rank":3790,"domain":"aweber.com","origin":"https://aweber.com","robots":{"status":"ok","http":200,"url":"https://www.aweber.com/robots.txt","bytes":198,"groups":2},"bots":{"GPTBot":["partial","*",6,0],"ChatGPT-User":["partial","*",6,0],"OAI-SearchBot":["partial","*",6,0],"ClaudeBot":["partial","*",6,0],"Claude-User":["partial","*",6,0],"Claude-SearchBot":["partial","*",6,0],"anthropic-ai":["partial","*",6,0],"PerplexityBot":["partial","*",6,0],"Perplexity-User":["partial","*",6,0],"Google-Extended":["partial","*",6,0],"Applebot-Extended":["partial","*",6,0],"CCBot":["partial","*",6,0],"Bytespider":["partial","*",6,0],"Amazonbot":["partial","*",6,0],"meta-externalagent":["partial","*",6,0],"cohere-ai":["partial","*",6,0],"Diffbot":["partial","*",6,0],"ContextFetchBot":["partial","*",6,0]},"llms":{"status":"present","http":200,"bytes":6819},"home":{"status":"ok","http":200,"url":"https://www.aweber.com/","html_tokens":18332,"markdown_tokens":363,"pruned_tokens":64}},
{"rank":3791,"domain":"tripadvisor.in","origin":"https://tripadvisor.in","robots":{"status":"ok","http":200,"url":"https://www.tripadvisor.in/robots.txt","bytes":22907,"groups":9},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["partial","named",623,8],"OAI-SearchBot":["partial","named",623,8],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",621,8],"Claude-SearchBot":["partial","*",621,8],"anthropic-ai":["partial","*",621,8],"PerplexityBot":["partial","named",625,8],"Perplexity-User":["partial","*",621,8],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["partial","*",621,8],"ContextFetchBot":["partial","*",621,8]},"llms":{"status":"http_error","http":403,"bytes":774},"home":{"status":"http_error","http":403,"url":"https://www.tripadvisor.in/"}},
{"rank":3792,"domain":"scitechdaily.com","origin":"https://scitechdaily.com","robots":{"status":"ok","http":200,"url":"https://scitechdaily.com/robots.txt","bytes":112,"groups":1},"bots":{"GPTBot":["partial","*",2,0],"ChatGPT-User":["partial","*",2,0],"OAI-SearchBot":["partial","*",2,0],"ClaudeBot":["partial","*",2,0],"Claude-User":["partial","*",2,0],"Claude-SearchBot":["partial","*",2,0],"anthropic-ai":["partial","*",2,0],"PerplexityBot":["partial","*",2,0],"Perplexity-User":["partial","*",2,0],"Google-Extended":["partial","*",2,0],"Applebot-Extended":["partial","*",2,0],"CCBot":["partial","*",2,0],"Bytespider":["partial","*",2,0],"Amazonbot":["partial","*",2,0],"meta-externalagent":["partial","*",2,0],"cohere-ai":["partial","*",2,0],"Diffbot":["partial","*",2,0],"ContextFetchBot":["partial","*",2,0]},"llms":{"status":"missing","http":404,"bytes":44698},"home":{"status":"http_error","http":403,"url":"https://scitechdaily.com/"}},
{"rank":3793,"domain":"paramount.com","origin":"https://paramount.com","robots":{"status":"ok","http":200,"url":"https://www.paramount.com/robots.txt","bytes":2005,"groups":1},"bots":{"GPTBot":["partial","*",33,18],"ChatGPT-User":["partial","*",33,18],"OAI-SearchBot":["partial","*",33,18],"ClaudeBot":["partial","*",33,18],"Claude-User":["partial","*",33,18],"Claude-SearchBot":["partial","*",33,18],"anthropic-ai":["partial","*",33,18],"PerplexityBot":["partial","*",33,18],"Perplexity-User":["partial","*",33,18],"Google-Extended":["partial","*",33,18],"Applebot-Extended":["partial","*",33,18],"CCBot":["partial","*",33,18],"Bytespider":["partial","*",33,18],"Amazonbot":["partial","*",33,18],"meta-externalagent":["partial","*",33,18],"cohere-ai":["partial","*",33,18],"Diffbot":["partial","*",33,18],"ContextFetchBot":["partial","*",33,18]},"llms":{"status":"missing","http":404,"bytes":1907},"home":{"status":"ok","http":200,"url":"https://www.paramount.com/","html_tokens":27868,"markdown_tokens":349,"pruned_tokens":247}},
{"rank":3794,"domain":"salute.gov.it","origin":"https://www.salute.gov.it","robots":{"status":"html","http":200,"url":"https://www.salute.gov.it/robots.txt","bytes":27161,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":3795,"domain":"sendspace.com","origin":"https://sendspace.com","robots":{"status":"ok","http":200,"url":"https://www.sendspace.com/robots.txt","bytes":271,"groups":1},"bots":{"GPTBot":["partial","*",9,0],"ChatGPT-User":["partial","*",9,0],"OAI-SearchBot":["partial","*",9,0],"ClaudeBot":["partial","*",9,0],"Claude-User":["partial","*",9,0],"Claude-SearchBot":["partial","*",9,0],"anthropic-ai":["partial","*",9,0],"PerplexityBot":["partial","*",9,0],"Perplexity-User":["partial","*",9,0],"Google-Extended":["partial","*",9,0],"Applebot-Extended":["partial","*",9,0],"CCBot":["partial","*",9,0],"Bytespider":["partial","*",9,0],"Amazonbot":["partial","*",9,0],"meta-externalagent":["partial","*",9,0],"cohere-ai":["partial","*",9,0],"Diffbot":["partial","*",9,0],"ContextFetchBot":["partial","*",9,0]},"llms":{"status":"missing","http":404,"bytes":34069},"home":{"status":"ok","http":200,"url":"https://www.sendspace.com/","html_tokens":10772,"markdown_tokens":14,"pruned_tokens":14}},
{"rank":3796,"domain":"kodak.com","origin":"https://kodak.com","robots":{"status":"ok","http":200,"url":"https://www.kodak.com/robots.txt","bytes":718,"groups":2},"bots":{"GPTBot":["partial","*",6,0],"ChatGPT-User":["partial","*",6,0],"OAI-SearchBot":["partial","*",6,0],"ClaudeBot":["partial","*",6,0],"Claude-User":["partial","*",6,0],"Claude-SearchBot":["partial","*",6,0],"anthropic-ai":["partial","*",6,0],"PerplexityBot":["partial","*",6,0],"Perplexity-User":["partial","*",6,0],"Google-Extended":["partial","*",6,0],"Applebot-Extended":["partial","*",6,0],"CCBot":["partial","*",6,0],"Bytespider":["partial","*",6,0],"Amazonbot":["partial","*",6,0],"meta-externalagent":["partial","*",6,0],"cohere-ai":["partial","*",6,0],"Diffbot":["partial","*",6,0],"ContextFetchBot":["partial","*",6,0]},"llms":{"status":"missing","http":404,"bytes":649},"home":{"status":"ok","http":200,"url":"https://www.kodak.com/en/","html_tokens":18387,"markdown_tokens":30,"pruned_tokens":30}},
{"rank":3797,"domain":"salesforce-sites.com","origin":"https://salesforce-sites.com","robots":{"status":"ok","http":200,"url":"https://salesforce-sites.com/robots.txt","bytes":527,"groups":2},"bots":{"GPTBot":["partial","*",1,11],"ChatGPT-User":["partial","*",1,11],"OAI-SearchBot":["partial","*",1,11],"ClaudeBot":["partial","*",1,11],"Claude-User":["partial","*",1,11],"Claude-SearchBot":["partial","*",1,11],"anthropic-ai":["partial","*",1,11],"PerplexityBot":["partial","*",1,11],"Perplexity-User":["partial","*",1,11],"Google-Extended":["partial","*",1,11],"Applebot-Extended":["partial","*",1,11],"CCBot":["partial","*",1,11],"Bytespider":["partial","*",1,11],"Amazonbot":["partial","*",1,11],"meta-externalagent":["partial","*",1,11],"cohere-ai":["partial","*",1,11],"Diffbot":["partial","*",1,11],"ContextFetchBot":["partial","*",1,11]},"llms":{"status":"disallowed"},"home":{"status":"disallowed"}},
{"rank":3798,"domain":"smallbiztrends.com","origin":"https://smallbiztrends.com","robots":{"status":"ok","http":200,"url":"https://smallbiztrends.com/robots.txt","bytes":2179,"groups":4},"bots":{"GPTBot":["partial","*",11,1],"ChatGPT-User":["partial","*",11,1],"OAI-SearchBot":["partial","*",11,1],"ClaudeBot":["partial","*",11,1],"Claude-User":["partial","*",11,1],"Claude-SearchBot":["partial","*",11,1],"anthropic-ai":["partial","*",11,1],"PerplexityBot":["partial","*",11,1],"Perplexity-User":["partial","*",11,1],"Google-Extended":["partial","*",11,1],"Applebot-Extended":["partial","*",11,1],"CCBot":["partial","*",11,1],"Bytespider":["partial","*",11,1],"Amazonbot":["partial","named",6,0],"meta-externalagent":["partial","*",11,1],"cohere-ai":["partial","*",11,1],"Diffbot":["partial","*",11,1],"ContextFetchBot":["partial","*",11,1]},"llms":{"status":"missing","http":404,"bytes":45080},"home":{"status":"ok","http":200,"url":"https://smallbiztrends.com/","html_tokens":35523,"markdown_tokens":325,"pruned_tokens":165}},
{"rank":3799,"domain":"premiumtimesng.com","origin":"https://premiumtimesng.com","robots":{"status":"ok","http":200,"url":"https://www.premiumtimesng.com/robots.txt","bytes":273,"groups":2},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["partial","named",1,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"timeout","error":"timed out after 10 s"},"home":{"status":"ok","http":200,"url":"https://www.premiumtimesng.com/","html_tokens":214728,"markdown_tokens":1252,"pruned_tokens":59}},
{"rank":3800,"domain":"rainn.org","origin":"https://rainn.org","robots":{"status":"ok","http":200,"url":"https://rainn.org/robots.txt","bytes":167,"groups":1},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"missing","http":404,"bytes":299464},"home":{"status":"http_error","http":403,"url":"https://rainn.org/"}},
{"rank":3801,"domain":"postheaven.net","origin":"https://postheaven.net","robots":{"status":"missing","http":404,"url":"https://postheaven.net/robots.txt","bytes":19,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"other_text","http":200,"bytes":1549},"home":{"status":"ok","http":200,"url":"https://postheaven.net/","html_tokens":1082,"markdown_tokens":108,"pruned_tokens":105}},
{"rank":3802,"domain":"mobile.de","origin":"https://mobile.de","robots":{"status":"ok","http":200,"url":"https://www.mobile.de/robots.txt","bytes":1142,"groups":4},"bots":{"GPTBot":["partial","*",41,1],"ChatGPT-User":["partial","*",41,1],"OAI-SearchBot":["partial","*",41,1],"ClaudeBot":["partial","*",41,1],"Claude-User":["partial","*",41,1],"Claude-SearchBot":["partial","*",41,1],"anthropic-ai":["partial","*",41,1],"PerplexityBot":["partial","*",41,1],"Perplexity-User":["partial","*",41,1],"Google-Extended":["partial","*",41,1],"Applebot-Extended":["partial","*",41,1],"CCBot":["partial","*",41,1],"Bytespider":["partial","*",41,1],"Amazonbot":["partial","*",41,1],"meta-externalagent":["partial","*",41,1],"cohere-ai":["partial","*",41,1],"Diffbot":["partial","*",41,1],"ContextFetchBot":["partial","*",41,1]},"llms":{"status":"http_error","http":403,"bytes":7704},"home":{"status":"http_error","http":403,"url":"https://www.mobile.de/"}},
{"rank":3803,"domain":"art.pl","origin":"https://art.pl","robots":{"status":"timeout","http":null,"url":"https://art.pl/robots.txt","bytes":0,"groups":0,"error":"ConnectTimeout"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":3804,"domain":"mrbetlogin.com","origin":"https://mrbetlogin.com","robots":{"status":"ok","http":200,"url":"https://mrbetlogin.com/robots.txt","bytes":533,"groups":1},"bots":{"GPTBot":["partial","*",19,0],"ChatGPT-User":["partial","*",19,0],"OAI-SearchBot":["partial","*",19,0],"ClaudeBot":["partial","*",19,0],"Claude-User":["partial","*",19,0],"Claude-SearchBot":["partial","*",19,0],"anthropic-ai":["partial","*",19,0],"PerplexityBot":["partial","*",19,0],"Perplexity-User":["partial","*",19,0],"Google-Extended":["partial","*",19,0],"Applebot-Extended":["partial","*",19,0],"CCBot":["partial","*",19,0],"Bytespider":["partial","*",19,0],"Amazonbot":["partial","*",19,0],"meta-externalagent":["partial","*",19,0],"cohere-ai":["partial","*",19,0],"Diffbot":["partial","*",19,0],"ContextFetchBot":["partial","*",19,0]},"llms":{"status":"missing","http":404,"bytes":35089},"home":{"status":"ok","http":200,"url":"https://mrbetlogin.com/","html_tokens":24673,"markdown_tokens":2288,"pruned_tokens":136}},
{"rank":3805,"domain":"hubspot.net","origin":"https://hubspot.net","robots":{"status":"ok","http":200,"url":"https://www.hubspot.com/robots.txt","bytes":12606,"groups":1},"bots":{"GPTBot":["partial","*",356,1],"ChatGPT-User":["partial","*",356,1],"OAI-SearchBot":["partial","*",356,1],"ClaudeBot":["partial","*",356,1],"Claude-User":["partial","*",356,1],"Claude-SearchBot":["partial","*",356,1],"anthropic-ai":["partial","*",356,1],"PerplexityBot":["partial","*",356,1],"Perplexity-User":["partial","*",356,1],"Google-Extended":["partial","*",356,1],"Applebot-Extended":["partial","*",356,1],"CCBot":["partial","*",356,1],"Bytespider":["partial","*",356,1],"Amazonbot":["partial","*",356,1],"meta-externalagent":["partial","*",356,1],"cohere-ai":["partial","*",356,1],"Diffbot":["partial","*",356,1],"ContextFetchBot":["partial","*",356,1]},"llms":{"status":"present","http":200,"bytes":35564},"home":{"status":"ok","http":200,"url":"https://www.hubspot.com/","html_tokens":160793,"markdown_tokens":688,"pruned_tokens":226}},
{"rank":3806,"domain":"dur.ac.uk","origin":"https://dur.ac.uk","robots":{"status":"ok","http":200,"url":"https://dur.ac.uk/robots.txt","bytes":1823,"groups":2},"bots":{"GPTBot":["partial","*",39,0],"ChatGPT-User":["partial","*",39,0],"OAI-SearchBot":["partial","*",39,0],"ClaudeBot":["partial","*",39,0],"Claude-User":["partial","*",39,0],"Claude-SearchBot":["partial","*",39,0],"anthropic-ai":["partial","*",39,0],"PerplexityBot":["partial","*",39,0],"Perplexity-User":["partial","*",39,0],"Google-Extended":["partial","*",39,0],"Applebot-Extended":["partial","*",39,0],"CCBot":["partial","*",39,0],"Bytespider":["partial","*",39,0],"Amazonbot":["partial","*",39,0],"meta-externalagent":["partial","*",39,0],"cohere-ai":["partial","*",39,0],"Diffbot":["partial","*",39,0],"ContextFetchBot":["partial","*",39,0]},"llms":{"status":"present","http":200,"bytes":12736},"home":{"status":"ok","http":200,"url":"https://dur.ac.uk/","html_tokens":52920,"markdown_tokens":399,"pruned_tokens":115}},
{"rank":3807,"domain":"uibk.ac.at","origin":"https://uibk.ac.at","robots":{"status":"ok","http":200,"url":"https://www.uibk.ac.at/robots.txt","bytes":2233,"groups":8},"bots":{"GPTBot":["partial","*",20,0],"ChatGPT-User":["partial","*",20,0],"OAI-SearchBot":["partial","*",20,0],"ClaudeBot":["partial","*",20,0],"Claude-User":["partial","*",20,0],"Claude-SearchBot":["partial","*",20,0],"anthropic-ai":["partial","*",20,0],"PerplexityBot":["partial","*",20,0],"Perplexity-User":["partial","*",20,0],"Google-Extended":["partial","*",20,0],"Applebot-Extended":["partial","*",20,0],"CCBot":["partial","*",20,0],"Bytespider":["partial","*",20,0],"Amazonbot":["partial","*",20,0],"meta-externalagent":["partial","*",20,0],"cohere-ai":["partial","*",20,0],"Diffbot":["partial","*",20,0],"ContextFetchBot":["partial","*",20,0]},"llms":{"status":"missing","http":404,"bytes":13022},"home":{"status":"ok","http":200,"url":"https://www.uibk.ac.at/de/","html_tokens":37744,"markdown_tokens":778,"pruned_tokens":116}},
{"rank":3808,"domain":"quantamagazine.org","origin":"https://quantamagazine.org","robots":{"status":"ok","http":200,"url":"https://www.quantamagazine.org/robots.txt","bytes":121,"groups":1},"bots":{"GPTBot":["partial","*",1,1],"ChatGPT-User":["partial","*",1,1],"OAI-SearchBot":["partial","*",1,1],"ClaudeBot":["partial","*",1,1],"Claude-User":["partial","*",1,1],"Claude-SearchBot":["partial","*",1,1],"anthropic-ai":["partial","*",1,1],"PerplexityBot":["partial","*",1,1],"Perplexity-User":["partial","*",1,1],"Google-Extended":["partial","*",1,1],"Applebot-Extended":["partial","*",1,1],"CCBot":["partial","*",1,1],"Bytespider":["partial","*",1,1],"Amazonbot":["partial","*",1,1],"meta-externalagent":["partial","*",1,1],"cohere-ai":["partial","*",1,1],"Diffbot":["partial","*",1,1],"ContextFetchBot":["partial","*",1,1]},"llms":{"status":"missing","http":404,"bytes":116433},"home":{"status":"ok","http":200,"url":"https://www.quantamagazine.org/","html_tokens":60692,"markdown_tokens":569,"pruned_tokens":120}},
{"rank":3809,"domain":"ebi.ac.uk","origin":"https://ebi.ac.uk","robots":{"status":"ok","http":200,"url":"https://www.ebi.ac.uk/robots.txt","bytes":5301,"groups":10},"bots":{"GPTBot":["partial","*",170,0],"ChatGPT-User":["partial","*",170,0],"OAI-SearchBot":["partial","*",170,0],"ClaudeBot":["partial","*",170,0],"Claude-User":["partial","*",170,0],"Claude-SearchBot":["partial","*",170,0],"anthropic-ai":["partial","*",170,0],"PerplexityBot":["partial","*",170,0],"Perplexity-User":["partial","*",170,0],"Google-Extended":["partial","*",170,0],"Applebot-Extended":["partial","*",170,0],"CCBot":["partial","*",170,0],"Bytespider":["partial","*",170,0],"Amazonbot":["partial","*",170,0],"meta-externalagent":["partial","*",170,0],"cohere-ai":["partial","*",170,0],"Diffbot":["partial","*",170,0],"ContextFetchBot":["partial","*",170,0]},"llms":{"status":"missing","http":404,"bytes":20544},"home":{"status":"ok","http":200,"url":"https://www.ebi.ac.uk/","html_tokens":11949,"markdown_tokens":119,"pruned_tokens":119}},
{"rank":3810,"domain":"sennheiser.com","origin":"https://sennheiser.com","robots":{"status":"ok","http":200,"url":"https://www.sennheiser.com/robots.txt","bytes":133,"groups":1},"bots":{"GPTBot":["partial","*",3,0],"ChatGPT-User":["partial","*",3,0],"OAI-SearchBot":["partial","*",3,0],"ClaudeBot":["partial","*",3,0],"Claude-User":["partial","*",3,0],"Claude-SearchBot":["partial","*",3,0],"anthropic-ai":["partial","*",3,0],"PerplexityBot":["partial","*",3,0],"Perplexity-User":["partial","*",3,0],"Google-Extended":["partial","*",3,0],"Applebot-Extended":["partial","*",3,0],"CCBot":["partial","*",3,0],"Bytespider":["partial","*",3,0],"Amazonbot":["partial","*",3,0],"meta-externalagent":["partial","*",3,0],"cohere-ai":["partial","*",3,0],"Diffbot":["partial","*",3,0],"ContextFetchBot":["partial","*",3,0]},"llms":{"status":"missing","http":404,"bytes":149453},"home":{"status":"ok","http":200,"url":"https://www.sennheiser.com/en-us","html_tokens":110863,"markdown_tokens":15,"pruned_tokens":15}},
{"rank":3811,"domain":"veeam.com","origin":"https://veeam.com","robots":{"status":"ok","http":200,"url":"https://www.veeam.com/robots.txt","bytes":1846,"groups":1},"bots":{"GPTBot":["partial","*",41,0],"ChatGPT-User":["partial","*",41,0],"OAI-SearchBot":["partial","*",41,0],"ClaudeBot":["partial","*",41,0],"Claude-User":["partial","*",41,0],"Claude-SearchBot":["partial","*",41,0],"anthropic-ai":["partial","*",41,0],"PerplexityBot":["partial","*",41,0],"Perplexity-User":["partial","*",41,0],"Google-Extended":["partial","*",41,0],"Applebot-Extended":["partial","*",41,0],"CCBot":["partial","*",41,0],"Bytespider":["partial","*",41,0],"Amazonbot":["partial","*",41,0],"meta-externalagent":["partial","*",41,0],"cohere-ai":["partial","*",41,0],"Diffbot":["partial","*",41,0],"ContextFetchBot":["partial","*",41,0]},"llms":{"status":"present","http":200,"bytes":2069},"home":{"status":"ok","http":200,"url":"https://www.veeam.com/","html_tokens":65833,"markdown_tokens":891,"pruned_tokens":219}},
{"rank":3812,"domain":"lequipe.fr","origin":"https://lequipe.fr","robots":{"status":"refused","http":403,"url":"https://www.lequipe.fr/robots.txt","bytes":382,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":3813,"domain":"castbox.fm","origin":"https://castbox.fm","robots":{"status":"ok","http":200,"url":"https://castbox.fm/robots.txt","bytes":408,"groups":6},"bots":{"GPTBot":["partial","*",13,0],"ChatGPT-User":["partial","*",13,0],"OAI-SearchBot":["partial","*",13,0],"ClaudeBot":["partial","*",13,0],"Claude-User":["partial","*",13,0],"Claude-SearchBot":["partial","*",13,0],"anthropic-ai":["partial","*",13,0],"PerplexityBot":["partial","*",13,0],"Perplexity-User":["partial","*",13,0],"Google-Extended":["partial","*",13,0],"Applebot-Extended":["partial","*",13,0],"CCBot":["partial","*",13,0],"Bytespider":["partial","*",13,0],"Amazonbot":["partial","*",13,0],"meta-externalagent":["partial","*",13,0],"cohere-ai":["partial","*",13,0],"Diffbot":["partial","*",13,0],"ContextFetchBot":["partial","*",13,0]},"llms":{"status":"missing","http":404,"bytes":11113},"home":{"status":"ok","http":200,"url":"https://castbox.fm/","html_tokens":92795,"markdown_tokens":39,"pruned_tokens":39}},
{"rank":3814,"domain":"mercola.com","origin":"https://mercola.com","robots":{"status":"refused","http":403,"url":"https://mercola.com/robots.txt","bytes":139973,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":3815,"domain":"nocookie.net","origin":"https://www.nocookie.net","robots":{"status":"refused","http":403,"url":"https://www.nocookie.net/robots.txt","bytes":5533,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":3816,"domain":"translatewiki.net","origin":"https://translatewiki.net","robots":{"status":"ok","http":200,"url":"https://translatewiki.net/robots.txt","bytes":295,"groups":3},"bots":{"GPTBot":["partial","*",5,0],"ChatGPT-User":["partial","*",5,0],"OAI-SearchBot":["partial","*",5,0],"ClaudeBot":["partial","*",5,0],"Claude-User":["partial","*",5,0],"Claude-SearchBot":["partial","*",5,0],"anthropic-ai":["partial","*",5,0],"PerplexityBot":["partial","*",5,0],"Perplexity-User":["partial","*",5,0],"Google-Extended":["partial","*",5,0],"Applebot-Extended":["partial","*",5,0],"CCBot":["partial","*",5,0],"Bytespider":["partial","*",5,0],"Amazonbot":["partial","*",5,0],"meta-externalagent":["partial","*",5,0],"cohere-ai":["partial","*",5,0],"Diffbot":["partial","*",5,0],"ContextFetchBot":["partial","*",5,0]},"llms":{"status":"missing","http":404,"bytes":146},"home":{"status":"http_error","http":403,"url":"https://translatewiki.net/"}},
{"rank":3817,"domain":"techsmith.com","origin":"https://techsmith.com","robots":{"status":"ok","http":200,"url":"https://www.techsmith.com/robots.txt","bytes":175,"groups":1},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"missing","http":404,"bytes":397772},"home":{"status":"ok","http":200,"url":"https://www.techsmith.com/","html_tokens":120422,"markdown_tokens":461,"pruned_tokens":103}},
{"rank":3818,"domain":"google.co.il","origin":"https://google.co.il","robots":{"status":"ok","http":200,"url":"https://www.google.co.il/robots.txt","bytes":6624,"groups":4},"bots":{"GPTBot":["partial","*",169,64],"ChatGPT-User":["partial","*",169,64],"OAI-SearchBot":["partial","*",169,64],"ClaudeBot":["partial","*",169,64],"Claude-User":["partial","*",169,64],"Claude-SearchBot":["partial","*",169,64],"anthropic-ai":["partial","*",169,64],"PerplexityBot":["partial","*",169,64],"Perplexity-User":["partial","*",169,64],"Google-Extended":["partial","*",169,64],"Applebot-Extended":["partial","*",169,64],"CCBot":["partial","*",169,64],"Bytespider":["partial","*",169,64],"Amazonbot":["partial","*",169,64],"meta-externalagent":["partial","*",169,64],"cohere-ai":["partial","*",169,64],"Diffbot":["partial","*",169,64],"ContextFetchBot":["partial","*",169,64]},"llms":{"status":"missing","http":404,"bytes":1569},"home":{"status":"ok","http":200,"url":"https://www.google.com/","html_tokens":21333,"markdown_tokens":30,"pruned_tokens":30}},
{"rank":3819,"domain":"ces.tech","origin":"https://ces.tech","robots":{"status":"ok","http":200,"url":"https://www.ces.tech/robots.txt","bytes":216,"groups":1},"bots":{"GPTBot":["partial","*",8,0],"ChatGPT-User":["partial","*",8,0],"OAI-SearchBot":["partial","*",8,0],"ClaudeBot":["partial","*",8,0],"Claude-User":["partial","*",8,0],"Claude-SearchBot":["partial","*",8,0],"anthropic-ai":["partial","*",8,0],"PerplexityBot":["partial","*",8,0],"Perplexity-User":["partial","*",8,0],"Google-Extended":["partial","*",8,0],"Applebot-Extended":["partial","*",8,0],"CCBot":["partial","*",8,0],"Bytespider":["partial","*",8,0],"Amazonbot":["partial","*",8,0],"meta-externalagent":["partial","*",8,0],"cohere-ai":["partial","*",8,0],"Diffbot":["partial","*",8,0],"ContextFetchBot":["partial","*",8,0]},"llms":{"status":"missing","http":404,"bytes":0},"home":{"status":"ok","http":200,"url":"https://www.ces.tech/","html_tokens":88068,"markdown_tokens":490,"pruned_tokens":191}},
{"rank":3820,"domain":"dispatch.com","origin":"https://dispatch.com","robots":{"status":"ok","http":200,"url":"https://www.dispatch.com/robots.txt","bytes":14897,"groups":288},"bots":{"GPTBot":["partial","named",1,5],"ChatGPT-User":["partial","named",1,5],"OAI-SearchBot":["partial","named",1,5],"ClaudeBot":["blocked","named",1,0],"Claude-User":["blocked","named",1,0],"Claude-SearchBot":["blocked","named",1,0],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["partial","named",1,5],"Perplexity-User":["blocked","named",1,0],"Google-Extended":["partial","named",1,5],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",2,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",88,0]},"llms":{"status":"http_error","http":402,"bytes":254},"home":{"status":"http_error","http":402,"url":"https://www.dispatch.com/"}},
{"rank":3821,"domain":"commerce.gov","origin":"https://commerce.gov","robots":{"status":"refused","http":403,"url":"https://commerce.gov/robots.txt","bytes":5529,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":3822,"domain":"google.co.nz","origin":"https://google.co.nz","robots":{"status":"ok","http":200,"url":"https://www.google.co.nz/robots.txt","bytes":6624,"groups":4},"bots":{"GPTBot":["partial","*",169,64],"ChatGPT-User":["partial","*",169,64],"OAI-SearchBot":["partial","*",169,64],"ClaudeBot":["partial","*",169,64],"Claude-User":["partial","*",169,64],"Claude-SearchBot":["partial","*",169,64],"anthropic-ai":["partial","*",169,64],"PerplexityBot":["partial","*",169,64],"Perplexity-User":["partial","*",169,64],"Google-Extended":["partial","*",169,64],"Applebot-Extended":["partial","*",169,64],"CCBot":["partial","*",169,64],"Bytespider":["partial","*",169,64],"Amazonbot":["partial","*",169,64],"meta-externalagent":["partial","*",169,64],"cohere-ai":["partial","*",169,64],"Diffbot":["partial","*",169,64],"ContextFetchBot":["partial","*",169,64]},"llms":{"status":"missing","http":404,"bytes":1569},"home":{"status":"ok","http":200,"url":"https://www.google.com/","html_tokens":21293,"markdown_tokens":30,"pruned_tokens":30}},
{"rank":3823,"domain":"printables.com","origin":"https://printables.com","robots":{"status":"ok","http":200,"url":"https://www.printables.com/robots.txt","bytes":90,"groups":1},"bots":{"GPTBot":["partial","*",1,1],"ChatGPT-User":["partial","*",1,1],"OAI-SearchBot":["partial","*",1,1],"ClaudeBot":["partial","*",1,1],"Claude-User":["partial","*",1,1],"Claude-SearchBot":["partial","*",1,1],"anthropic-ai":["partial","*",1,1],"PerplexityBot":["partial","*",1,1],"Perplexity-User":["partial","*",1,1],"Google-Extended":["partial","*",1,1],"Applebot-Extended":["partial","*",1,1],"CCBot":["partial","*",1,1],"Bytespider":["partial","*",1,1],"Amazonbot":["partial","*",1,1],"meta-externalagent":["partial","*",1,1],"cohere-ai":["partial","*",1,1],"Diffbot":["partial","*",1,1],"ContextFetchBot":["partial","*",1,1]},"llms":{"status":"http_error","http":403,"bytes":5579},"home":{"status":"http_error","http":403,"url":"https://www.printables.com/"}},
{"rank":3824,"domain":"razer.com","origin":"https://razer.com","robots":{"status":"ok","http":200,"url":"https://www.razer.com/robots.txt","bytes":1485,"groups":7},"bots":{"GPTBot":["allowed","named",0,0],"ChatGPT-User":["partial","*",8,0],"OAI-SearchBot":["partial","*",8,0],"ClaudeBot":["allowed","named",0,0],"Claude-User":["partial","*",8,0],"Claude-SearchBot":["partial","*",8,0],"anthropic-ai":["partial","*",8,0],"PerplexityBot":["allowed","named",0,0],"Perplexity-User":["partial","*",8,0],"Google-Extended":["allowed","named",0,0],"Applebot-Extended":["partial","*",8,0],"CCBot":["partial","*",8,0],"Bytespider":["partial","*",8,0],"Amazonbot":["partial","*",8,0],"meta-externalagent":["partial","*",8,0],"cohere-ai":["partial","*",8,0],"Diffbot":["partial","*",8,0],"ContextFetchBot":["partial","*",8,0]},"llms":{"status":"present","http":200,"bytes":4075},"home":{"status":"ok","http":200,"url":"https://www.razer.com/","html_tokens":392541,"markdown_tokens":143,"pruned_tokens":117}},
{"rank":3825,"domain":"qt.io","origin":"https://qt.io","robots":{"status":"ok","http":200,"url":"https://www.qt.io/robots.txt","bytes":254,"groups":1},"bots":{"GPTBot":["partial","*",9,0],"ChatGPT-User":["partial","*",9,0],"OAI-SearchBot":["partial","*",9,0],"ClaudeBot":["partial","*",9,0],"Claude-User":["partial","*",9,0],"Claude-SearchBot":["partial","*",9,0],"anthropic-ai":["partial","*",9,0],"PerplexityBot":["partial","*",9,0],"Perplexity-User":["partial","*",9,0],"Google-Extended":["partial","*",9,0],"Applebot-Extended":["partial","*",9,0],"CCBot":["partial","*",9,0],"Bytespider":["partial","*",9,0],"Amazonbot":["partial","*",9,0],"meta-externalagent":["partial","*",9,0],"cohere-ai":["partial","*",9,0],"Diffbot":["partial","*",9,0],"ContextFetchBot":["partial","*",9,0]},"llms":{"status":"present","http":200,"bytes":3390},"home":{"status":"ok","http":200,"url":"https://www.qt.io/","html_tokens":57642,"markdown_tokens":267,"pruned_tokens":158}},
{"rank":3826,"domain":"allure.com","origin":"https://allure.com","robots":{"status":"ok","http":200,"url":"https://www.allure.com/robots.txt","bytes":2277,"groups":3},"bots":{"GPTBot":["partial","*",15,15],"ChatGPT-User":["partial","*",15,15],"OAI-SearchBot":["partial","*",15,15],"ClaudeBot":["blocked","named",1,0],"Claude-User":["blocked","named",1,0],"Claude-SearchBot":["blocked","named",1,0],"anthropic-ai":["partial","*",15,15],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["blocked","named",1,0],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",15,15]},"llms":{"status":"missing","http":404,"bytes":808202},"home":{"status":"ok","http":200,"url":"https://www.allure.com/","html_tokens":495229,"markdown_tokens":48,"pruned_tokens":20}},
{"rank":3827,"domain":"ffm.to","origin":"https://ffm.to","robots":{"status":"ok","http":200,"url":"https://ffm.to/robots.txt","bytes":59,"groups":1},"bots":{"GPTBot":["allowed","*",0,1],"ChatGPT-User":["allowed","*",0,1],"OAI-SearchBot":["allowed","*",0,1],"ClaudeBot":["allowed","*",0,1],"Claude-User":["allowed","*",0,1],"Claude-SearchBot":["allowed","*",0,1],"anthropic-ai":["allowed","*",0,1],"PerplexityBot":["allowed","*",0,1],"Perplexity-User":["allowed","*",0,1],"Google-Extended":["allowed","*",0,1],"Applebot-Extended":["allowed","*",0,1],"CCBot":["allowed","*",0,1],"Bytespider":["allowed","*",0,1],"Amazonbot":["allowed","*",0,1],"meta-externalagent":["allowed","*",0,1],"cohere-ai":["allowed","*",0,1],"Diffbot":["allowed","*",0,1],"ContextFetchBot":["allowed","*",0,1]},"llms":{"status":"missing","http":404,"bytes":0},"home":{"status":"http_error","http":404,"url":"https://asstes.ffm.to/404/index.html"}},
{"rank":3828,"domain":"latribune.fr","origin":"https://latribune.fr","robots":{"status":"ok","http":200,"url":"https://www.latribune.fr/robots.txt","bytes":6075,"groups":160},"bots":{"GPTBot":["partial","*",7,0],"ChatGPT-User":["partial","*",7,0],"OAI-SearchBot":["partial","*",7,0],"ClaudeBot":["partial","*",7,0],"Claude-User":["partial","*",7,0],"Claude-SearchBot":["partial","*",7,0],"anthropic-ai":["partial","*",7,0],"PerplexityBot":["partial","*",7,0],"Perplexity-User":["partial","*",7,0],"Google-Extended":["partial","*",7,0],"Applebot-Extended":["partial","*",7,0],"CCBot":["partial","*",7,0],"Bytespider":["partial","*",7,0],"Amazonbot":["partial","*",7,0],"meta-externalagent":["partial","*",7,0],"cohere-ai":["partial","*",7,0],"Diffbot":["partial","*",7,0],"ContextFetchBot":["partial","*",7,0]},"llms":{"status":"missing","http":404,"bytes":101989},"home":{"status":"ok","http":200,"url":"https://www.latribune.fr/","html_tokens":238768,"markdown_tokens":102,"pruned_tokens":102}},
{"rank":3829,"domain":"springernature.com","origin":"https://springernature.com","robots":{"status":"ok","http":200,"url":"https://www.springernature.com/robots.txt","bytes":35,"groups":1},"bots":{"GPTBot":["partial","*",1,0],"ChatGPT-User":["partial","*",1,0],"OAI-SearchBot":["partial","*",1,0],"ClaudeBot":["partial","*",1,0],"Claude-User":["partial","*",1,0],"Claude-SearchBot":["partial","*",1,0],"anthropic-ai":["partial","*",1,0],"PerplexityBot":["partial","*",1,0],"Perplexity-User":["partial","*",1,0],"Google-Extended":["partial","*",1,0],"Applebot-Extended":["partial","*",1,0],"CCBot":["partial","*",1,0],"Bytespider":["partial","*",1,0],"Amazonbot":["partial","*",1,0],"meta-externalagent":["partial","*",1,0],"cohere-ai":["partial","*",1,0],"Diffbot":["partial","*",1,0],"ContextFetchBot":["partial","*",1,0]},"llms":{"status":"missing","http":404,"bytes":21443},"home":{"status":"ok","http":200,"url":"https://www.springernature.com/gp","html_tokens":65128,"markdown_tokens":120,"pruned_tokens":120}},
{"rank":3830,"domain":"adr.org","origin":"https://adr.org","robots":{"status":"refused","http":403,"url":"https://adr.org/robots.txt","bytes":266,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":3831,"domain":"androidcentral.com","origin":"https://androidcentral.com","robots":{"status":"ok","http":200,"url":"https://www.androidcentral.com/robots.txt","bytes":1643,"groups":4},"bots":{"GPTBot":["partial","*",18,0],"ChatGPT-User":["partial","*",18,0],"OAI-SearchBot":["partial","*",18,0],"ClaudeBot":["partial","*",18,0],"Claude-User":["partial","*",18,0],"Claude-SearchBot":["partial","*",18,0],"anthropic-ai":["partial","*",18,0],"PerplexityBot":["partial","*",18,0],"Perplexity-User":["partial","*",18,0],"Google-Extended":["partial","*",18,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["partial","*",18,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["partial","*",18,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",18,0]},"llms":{"status":"missing","http":404,"bytes":1224636},"home":{"status":"ok","http":200,"url":"https://www.androidcentral.com/","html_tokens":430389,"markdown_tokens":45,"pruned_tokens":41}},
{"rank":3832,"domain":"band.us","origin":"https://band.us","robots":{"status":"ok","http":200,"url":"https://www.band.us/robots.txt","bytes":744,"groups":4},"bots":{"GPTBot":["partial","named",1,1],"ChatGPT-User":["partial","*",1,0],"OAI-SearchBot":["partial","named",1,1],"ClaudeBot":["partial","named",1,1],"Claude-User":["partial","*",1,0],"Claude-SearchBot":["partial","named",1,1],"anthropic-ai":["partial","*",1,0],"PerplexityBot":["partial","named",1,1],"Perplexity-User":["partial","*",1,0],"Google-Extended":["partial","named",1,1],"Applebot-Extended":["partial","named",1,1],"CCBot":["partial","named",1,1],"Bytespider":["partial","*",1,0],"Amazonbot":["partial","*",1,0],"meta-externalagent":["partial","named",1,1],"cohere-ai":["partial","*",1,0],"Diffbot":["partial","*",1,0],"ContextFetchBot":["partial","*",1,0]},"llms":{"status":"html","http":200,"bytes":2751},"home":{"status":"ok","http":200,"url":"https://www.band.us/","html_tokens":3929,"markdown_tokens":3,"pruned_tokens":3}},
{"rank":3833,"domain":"shop-pro.jp","origin":"https://shop-pro.jp","robots":{"status":"ok","http":200,"url":"https://shop-pro.jp/robots.txt","bytes":0,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"present","http":200,"bytes":3754},"home":{"status":"ok","http":200,"url":"https://shop-pro.jp/","html_tokens":44746,"markdown_tokens":150,"pruned_tokens":150}},
{"rank":3834,"domain":"20min.ch","origin":"https://20min.ch","robots":{"status":"ok","http":200,"url":"https://www.20min.ch/robots.txt","bytes":3859,"groups":34},"bots":{"GPTBot":["partial","named",3,1],"ChatGPT-User":["partial","*",6,0],"OAI-SearchBot":["partial","named",3,1],"ClaudeBot":["partial","named",3,1],"Claude-User":["partial","*",6,0],"Claude-SearchBot":["partial","*",6,0],"anthropic-ai":["partial","named",3,1],"PerplexityBot":["partial","named",3,1],"Perplexity-User":["partial","*",6,0],"Google-Extended":["partial","named",3,1],"Applebot-Extended":["partial","named",3,1],"CCBot":["partial","named",3,1],"Bytespider":["partial","named",3,1],"Amazonbot":["partial","named",3,1],"meta-externalagent":["partial","named",3,1],"cohere-ai":["partial","named",3,1],"Diffbot":["partial","named",3,1],"ContextFetchBot":["partial","*",6,0]},"llms":{"status":"missing","http":404,"bytes":41475},"home":{"status":"ok","http":200,"url":"https://www.20min.ch/","html_tokens":189226,"markdown_tokens":42,"pruned_tokens":42}},
{"rank":3835,"domain":"dupont.com","origin":"https://www.dupont.com","robots":{"status":"ok","http":200,"url":"https://www.dupont.com/robots.txt","bytes":13826,"groups":1},"bots":{"GPTBot":["partial","*",286,0],"ChatGPT-User":["partial","*",286,0],"OAI-SearchBot":["partial","*",286,0],"ClaudeBot":["partial","*",286,0],"Claude-User":["partial","*",286,0],"Claude-SearchBot":["partial","*",286,0],"anthropic-ai":["partial","*",286,0],"PerplexityBot":["partial","*",286,0],"Perplexity-User":["partial","*",286,0],"Google-Extended":["partial","*",286,0],"Applebot-Extended":["partial","*",286,0],"CCBot":["partial","*",286,0],"Bytespider":["partial","*",286,0],"Amazonbot":["partial","*",286,0],"meta-externalagent":["partial","*",286,0],"cohere-ai":["partial","*",286,0],"Diffbot":["partial","*",286,0],"ContextFetchBot":["partial","*",286,0]},"llms":{"status":"missing","http":404,"bytes":43738},"home":{"status":"ok","http":200,"url":"https://www.dupont.com/","html_tokens":18204,"markdown_tokens":292,"pruned_tokens":173}},
{"rank":3836,"domain":"uni-bremen.de","origin":"https://uni-bremen.de","robots":{"status":"ok","http":200,"url":"https://www.uni-bremen.de/robots.txt","bytes":852,"groups":2},"bots":{"GPTBot":["partial","*",17,0],"ChatGPT-User":["partial","*",17,0],"OAI-SearchBot":["partial","*",17,0],"ClaudeBot":["partial","*",17,0],"Claude-User":["partial","*",17,0],"Claude-SearchBot":["partial","*",17,0],"anthropic-ai":["partial","*",17,0],"PerplexityBot":["partial","*",17,0],"Perplexity-User":["partial","*",17,0],"Google-Extended":["partial","*",17,0],"Applebot-Extended":["partial","*",17,0],"CCBot":["partial","*",17,0],"Bytespider":["partial","*",17,0],"Amazonbot":["partial","*",17,0],"meta-externalagent":["partial","*",17,0],"cohere-ai":["partial","*",17,0],"Diffbot":["partial","*",17,0],"ContextFetchBot":["partial","*",17,0]},"llms":{"status":"missing","http":404,"bytes":204702},"home":{"status":"ok","http":200,"url":"https://www.uni-bremen.de/","html_tokens":79954,"markdown_tokens":476,"pruned_tokens":228}},
{"rank":3837,"domain":"chathamhouse.org","origin":"https://chathamhouse.org","robots":{"status":"ok","http":200,"url":"https://www.chathamhouse.org/robots.txt","bytes":1892,"groups":1},"bots":{"GPTBot":["partial","*",28,19],"ChatGPT-User":["partial","*",28,19],"OAI-SearchBot":["partial","*",28,19],"ClaudeBot":["partial","*",28,19],"Claude-User":["partial","*",28,19],"Claude-SearchBot":["partial","*",28,19],"anthropic-ai":["partial","*",28,19],"PerplexityBot":["partial","*",28,19],"Perplexity-User":["partial","*",28,19],"Google-Extended":["partial","*",28,19],"Applebot-Extended":["partial","*",28,19],"CCBot":["partial","*",28,19],"Bytespider":["partial","*",28,19],"Amazonbot":["partial","*",28,19],"meta-externalagent":["partial","*",28,19],"cohere-ai":["partial","*",28,19],"Diffbot":["partial","*",28,19],"ContextFetchBot":["partial","*",28,19]},"llms":{"status":"missing","http":404,"bytes":555},"home":{"status":"http_error","http":403,"url":"https://chathamhouse.org/"}},
{"rank":3838,"domain":"osticket.com","origin":"https://osticket.com","robots":{"status":"missing","http":404,"url":"https://osticket.com/robots.txt","bytes":113,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":60612},"home":{"status":"ok","http":200,"url":"https://osticket.com/","html_tokens":20446,"markdown_tokens":157,"pruned_tokens":157}},
{"rank":3839,"domain":"gmx.net","origin":"https://gmx.net","robots":{"status":"ok","http":200,"url":"https://www.gmx.net/robots.txt","bytes":1003,"groups":4},"bots":{"GPTBot":["partial","named",3,3],"ChatGPT-User":["partial","named",2,2],"OAI-SearchBot":["partial","named",2,2],"ClaudeBot":["partial","named",3,3],"Claude-User":["partial","named",2,2],"Claude-SearchBot":["partial","named",2,2],"anthropic-ai":["partial","*",2,0],"PerplexityBot":["partial","named",2,2],"Perplexity-User":["partial","named",2,2],"Google-Extended":["partial","named",3,3],"Applebot-Extended":["partial","named",3,3],"CCBot":["partial","named",3,3],"Bytespider":["partial","*",2,0],"Amazonbot":["partial","named",3,3],"meta-externalagent":["partial","named",3,3],"cohere-ai":["partial","named",3,3],"Diffbot":["partial","named",3,3],"ContextFetchBot":["partial","*",2,0]},"llms":{"status":"missing","http":404,"bytes":327174},"home":{"status":"ok","http":200,"url":"https://www.gmx.net/","html_tokens":82702,"markdown_tokens":14,"pruned_tokens":14}},
{"rank":3840,"domain":"audiomack.com","origin":"https://audiomack.com","robots":{"status":"ok","http":200,"url":"https://audiomack.com/robots.txt","bytes":385,"groups":1},"bots":{"GPTBot":["partial","*",16,0],"ChatGPT-User":["partial","*",16,0],"OAI-SearchBot":["partial","*",16,0],"ClaudeBot":["partial","*",16,0],"Claude-User":["partial","*",16,0],"Claude-SearchBot":["partial","*",16,0],"anthropic-ai":["partial","*",16,0],"PerplexityBot":["partial","*",16,0],"Perplexity-User":["partial","*",16,0],"Google-Extended":["partial","*",16,0],"Applebot-Extended":["partial","*",16,0],"CCBot":["partial","*",16,0],"Bytespider":["partial","*",16,0],"Amazonbot":["partial","*",16,0],"meta-externalagent":["partial","*",16,0],"cohere-ai":["partial","*",16,0],"Diffbot":["partial","*",16,0],"ContextFetchBot":["partial","*",16,0]},"llms":{"status":"html","http":200,"bytes":1584578},"home":{"status":"ok","http":200,"url":"https://audiomack.com/","html_tokens":395891,"markdown_tokens":80,"pruned_tokens":11}},
{"rank":3841,"domain":"infineon.com","origin":"https://infineon.com","robots":{"status":"ok","http":200,"url":"https://www.infineon.com/robots.txt","bytes":1582,"groups":1},"bots":{"GPTBot":["partial","*",45,11],"ChatGPT-User":["partial","*",45,11],"OAI-SearchBot":["partial","*",45,11],"ClaudeBot":["partial","*",45,11],"Claude-User":["partial","*",45,11],"Claude-SearchBot":["partial","*",45,11],"anthropic-ai":["partial","*",45,11],"PerplexityBot":["partial","*",45,11],"Perplexity-User":["partial","*",45,11],"Google-Extended":["partial","*",45,11],"Applebot-Extended":["partial","*",45,11],"CCBot":["partial","*",45,11],"Bytespider":["partial","*",45,11],"Amazonbot":["partial","*",45,11],"meta-externalagent":["partial","*",45,11],"cohere-ai":["partial","*",45,11],"Diffbot":["partial","*",45,11],"ContextFetchBot":["partial","*",45,11]},"llms":{"status":"missing","http":404,"bytes":1526821},"home":{"status":"ok","http":200,"url":"https://www.infineon.com","html_tokens":396857,"markdown_tokens":174,"pruned_tokens":72}},
{"rank":3842,"domain":"spectator.com","origin":"https://spectator.com","robots":{"status":"ok","http":200,"url":"https://spectator.com/robots.txt","bytes":182,"groups":1},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"missing","http":404,"bytes":150945},"home":{"status":"ok","http":200,"url":"https://spectator.com/","html_tokens":69063,"markdown_tokens":169,"pruned_tokens":169}},
{"rank":3843,"domain":"app-board.net","origin":"https://app-board.net","robots":{"status":"ok","http":200,"url":"https://app-board.net/robots.txt","bytes":795,"groups":5},"bots":{"GPTBot":["partial","*",17,0],"ChatGPT-User":["partial","*",17,0],"OAI-SearchBot":["partial","*",17,0],"ClaudeBot":["partial","*",17,0],"Claude-User":["partial","*",17,0],"Claude-SearchBot":["partial","*",17,0],"anthropic-ai":["partial","*",17,0],"PerplexityBot":["partial","*",17,0],"Perplexity-User":["partial","*",17,0],"Google-Extended":["partial","*",17,0],"Applebot-Extended":["partial","*",17,0],"CCBot":["partial","*",17,0],"Bytespider":["partial","*",17,0],"Amazonbot":["partial","*",17,0],"meta-externalagent":["partial","*",17,0],"cohere-ai":["partial","*",17,0],"Diffbot":["partial","*",17,0],"ContextFetchBot":["partial","*",17,0]},"llms":{"status":"missing","http":404,"bytes":0},"home":{"status":"ok","http":200,"url":"https://app-board.net/","html_tokens":23618,"markdown_tokens":830,"pruned_tokens":830}},
{"rank":3844,"domain":"wjx.cn","origin":"https://wjx.cn","robots":{"status":"ok","http":200,"url":"https://www.wjx.cn/robots.txt","bytes":1508,"groups":1},"bots":{"GPTBot":["partial","*",55,0],"ChatGPT-User":["partial","*",55,0],"OAI-SearchBot":["partial","*",55,0],"ClaudeBot":["partial","*",55,0],"Claude-User":["partial","*",55,0],"Claude-SearchBot":["partial","*",55,0],"anthropic-ai":["partial","*",55,0],"PerplexityBot":["partial","*",55,0],"Perplexity-User":["partial","*",55,0],"Google-Extended":["partial","*",55,0],"Applebot-Extended":["partial","*",55,0],"CCBot":["partial","*",55,0],"Bytespider":["partial","*",55,0],"Amazonbot":["partial","*",55,0],"meta-externalagent":["partial","*",55,0],"cohere-ai":["partial","*",55,0],"Diffbot":["partial","*",55,0],"ContextFetchBot":["partial","*",55,0]},"llms":{"status":"missing","http":404,"bytes":1183},"home":{"status":"ok","http":200,"url":"https://www.wjx.cn/","html_tokens":13981,"markdown_tokens":10,"pruned_tokens":10}},
{"rank":3845,"domain":"tomtom.com","origin":"https://tomtom.com","robots":{"status":"ok","http":200,"url":"https://www.tomtom.com/robots.txt","bytes":649,"groups":2},"bots":{"GPTBot":["partial","*",2,0],"ChatGPT-User":["allowed","named",0,1],"OAI-SearchBot":["allowed","named",0,1],"ClaudeBot":["allowed","named",0,1],"Claude-User":["allowed","named",0,1],"Claude-SearchBot":["allowed","named",0,1],"anthropic-ai":["partial","*",2,0],"PerplexityBot":["allowed","named",0,1],"Perplexity-User":["partial","*",2,0],"Google-Extended":["allowed","named",0,1],"Applebot-Extended":["partial","*",2,0],"CCBot":["allowed","named",0,1],"Bytespider":["partial","*",2,0],"Amazonbot":["allowed","named",0,1],"meta-externalagent":["partial","*",2,0],"cohere-ai":["partial","*",2,0],"Diffbot":["partial","*",2,0],"ContextFetchBot":["partial","*",2,0]},"llms":{"status":"missing","http":404,"bytes":69454},"home":{"status":"ok","http":200,"url":"https://www.tomtom.com/","html_tokens":82298,"markdown_tokens":275,"pruned_tokens":103}},
{"rank":3846,"domain":"rada.gov.ua","origin":"https://rada.gov.ua","robots":{"status":"html","http":200,"url":"https://www.rada.gov.ua","bytes":169056,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":3847,"domain":"kantei.go.jp","origin":"https://kantei.go.jp","robots":{"status":"missing","http":404,"url":"https://www.kantei.go.jp/robots.txt","bytes":4703,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":4703},"home":{"status":"ok","http":200,"url":"https://www.kantei.go.jp/","html_tokens":4440,"markdown_tokens":44,"pruned_tokens":44}},
{"rank":3848,"domain":"baden-wuerttemberg.de","origin":"https://baden-wuerttemberg.de","robots":{"status":"ok","http":200,"url":"https://www.baden-wuerttemberg.de/de/robots.txt","bytes":263,"groups":1},"bots":{"GPTBot":["partial","*",8,0],"ChatGPT-User":["partial","*",8,0],"OAI-SearchBot":["partial","*",8,0],"ClaudeBot":["partial","*",8,0],"Claude-User":["partial","*",8,0],"Claude-SearchBot":["partial","*",8,0],"anthropic-ai":["partial","*",8,0],"PerplexityBot":["partial","*",8,0],"Perplexity-User":["partial","*",8,0],"Google-Extended":["partial","*",8,0],"Applebot-Extended":["partial","*",8,0],"CCBot":["partial","*",8,0],"Bytespider":["partial","*",8,0],"Amazonbot":["partial","*",8,0],"meta-externalagent":["partial","*",8,0],"cohere-ai":["partial","*",8,0],"Diffbot":["partial","*",8,0],"ContextFetchBot":["partial","*",8,0]},"llms":{"status":"missing","http":404,"bytes":599404},"home":{"status":"ok","http":200,"url":"https://www.baden-wuerttemberg.de/de/startseite","html_tokens":160008,"markdown_tokens":366,"pruned_tokens":50}},
{"rank":3849,"domain":"lever.co","origin":"https://lever.co","robots":{"status":"ok","http":200,"url":"https://www.lever.co/robots.txt","bytes":320,"groups":2},"bots":{"GPTBot":["partial","named",2,1],"ChatGPT-User":["partial","*",4,1],"OAI-SearchBot":["partial","*",4,1],"ClaudeBot":["partial","*",4,1],"Claude-User":["partial","*",4,1],"Claude-SearchBot":["partial","*",4,1],"anthropic-ai":["partial","*",4,1],"PerplexityBot":["partial","named",2,1],"Perplexity-User":["partial","*",4,1],"Google-Extended":["partial","named",2,1],"Applebot-Extended":["partial","named",2,1],"CCBot":["partial","named",2,1],"Bytespider":["partial","*",4,1],"Amazonbot":["partial","*",4,1],"meta-externalagent":["partial","*",4,1],"cohere-ai":["partial","*",4,1],"Diffbot":["partial","*",4,1],"ContextFetchBot":["partial","*",4,1]},"llms":{"status":"missing","http":404,"bytes":47515},"home":{"status":"ok","http":200,"url":"https://www.lever.co/","html_tokens":34197,"markdown_tokens":1325,"pruned_tokens":153}},
{"rank":3850,"domain":"thejakartapost.com","origin":"https://thejakartapost.com","robots":{"status":"ok","http":200,"url":"https://www.thejakartapost.com/robots.txt","bytes":1566,"groups":1},"bots":{"GPTBot":["partial","*",7,1],"ChatGPT-User":["partial","*",7,1],"OAI-SearchBot":["partial","*",7,1],"ClaudeBot":["partial","*",7,1],"Claude-User":["partial","*",7,1],"Claude-SearchBot":["partial","*",7,1],"anthropic-ai":["partial","*",7,1],"PerplexityBot":["partial","*",7,1],"Perplexity-User":["partial","*",7,1],"Google-Extended":["partial","*",7,1],"Applebot-Extended":["partial","*",7,1],"CCBot":["partial","*",7,1],"Bytespider":["partial","*",7,1],"Amazonbot":["partial","*",7,1],"meta-externalagent":["partial","*",7,1],"cohere-ai":["partial","*",7,1],"Diffbot":["partial","*",7,1],"ContextFetchBot":["partial","*",7,1]},"llms":{"status":"missing","http":404,"bytes":7166},"home":{"status":"ok","http":200,"url":"https://www.thejakartapost.com/","html_tokens":39751,"markdown_tokens":85,"pruned_tokens":85}},
{"rank":3851,"domain":"cepr.org","origin":"https://cepr.org","robots":{"status":"refused","http":403,"url":"https://cepr.org/robots.txt","bytes":4541,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":3852,"domain":"wpenginepowered.com","origin":"https://wpenginepowered.com","robots":{"status":"missing","http":404,"url":"https://wpenginepowered.com/robots.txt","bytes":1957,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"http_error","http":403,"bytes":5572},"home":{"status":"http_error","http":403,"url":"https://wpenginepowered.com/"}},
{"rank":3853,"domain":"genome.gov","origin":"https://genome.gov","robots":{"status":"ok","http":200,"url":"https://www.genome.gov/robots.txt","bytes":2027,"groups":1},"bots":{"GPTBot":["partial","*",34,18],"ChatGPT-User":["partial","*",34,18],"OAI-SearchBot":["partial","*",34,18],"ClaudeBot":["partial","*",34,18],"Claude-User":["partial","*",34,18],"Claude-SearchBot":["partial","*",34,18],"anthropic-ai":["partial","*",34,18],"PerplexityBot":["partial","*",34,18],"Perplexity-User":["partial","*",34,18],"Google-Extended":["partial","*",34,18],"Applebot-Extended":["partial","*",34,18],"CCBot":["partial","*",34,18],"Bytespider":["partial","*",34,18],"Amazonbot":["partial","*",34,18],"meta-externalagent":["partial","*",34,18],"cohere-ai":["partial","*",34,18],"Diffbot":["partial","*",34,18],"ContextFetchBot":["partial","*",34,18]},"llms":{"status":"missing","http":404,"bytes":188},"home":{"status":"ok","http":200,"url":"https://www.genome.gov/","html_tokens":57628,"markdown_tokens":179,"pruned_tokens":155}},
{"rank":3854,"domain":"neowin.net","origin":"https://neowin.net","robots":{"status":"ok","http":200,"url":"https://neowin.net/robots.txt","bytes":1804,"groups":2},"bots":{"GPTBot":["partial","*",47,0],"ChatGPT-User":["partial","*",47,0],"OAI-SearchBot":["partial","*",47,0],"ClaudeBot":["partial","*",47,0],"Claude-User":["partial","*",47,0],"Claude-SearchBot":["partial","*",47,0],"anthropic-ai":["partial","*",47,0],"PerplexityBot":["partial","*",47,0],"Perplexity-User":["partial","*",47,0],"Google-Extended":["partial","*",47,0],"Applebot-Extended":["partial","*",47,0],"CCBot":["partial","*",47,0],"Bytespider":["partial","*",47,0],"Amazonbot":["partial","*",47,0],"meta-externalagent":["partial","*",47,0],"cohere-ai":["partial","*",47,0],"Diffbot":["partial","*",47,0],"ContextFetchBot":["partial","*",47,0]},"llms":{"status":"http_error","http":403,"bytes":5499},"home":{"status":"http_error","http":403,"url":"https://neowin.net/"}},
{"rank":3855,"domain":"icbc.com.cn","origin":"https://www.icbc.com.cn","robots":{"status":"missing","http":404,"url":"https://www.icbc.com.cn/robots.txt","bytes":176,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":176},"home":{"status":"ok","http":200,"url":"https://www.icbc.com.cn/","html_tokens":126490,"markdown_tokens":12,"pruned_tokens":12}},
{"rank":3856,"domain":"southernliving.com","origin":"https://southernliving.com","robots":{"status":"ok","http":200,"url":"https://www.southernliving.com/robots.txt","bytes":3481,"groups":6},"bots":{"GPTBot":["partial","named",1,0],"ChatGPT-User":["partial","named",1,0],"OAI-SearchBot":["partial","named",1,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["blocked","named",1,0],"Claude-SearchBot":["partial","named",1,2],"anthropic-ai":["partial","named",1,2],"PerplexityBot":["partial","named",1,2],"Perplexity-User":["blocked","named",1,0],"Google-Extended":["partial","named",1,2],"Applebot-Extended":["blocked","named",1,0],"CCBot":["partial","named",1,2],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["partial","named",1,2],"cohere-ai":["partial","named",1,2],"Diffbot":["partial","*",2,0],"ContextFetchBot":["partial","*",2,0]},"llms":{"status":"http_error","http":402,"bytes":612},"home":{"status":"http_error","http":402,"url":"https://www.southernliving.com/"}},
{"rank":3857,"domain":"merkur.de","origin":"https://merkur.de","robots":{"status":"ok","http":200,"url":"https://merkur.de/robots.txt","bytes":1550,"groups":2},"bots":{"GPTBot":["partial","*",1,2],"ChatGPT-User":["partial","named",11,1],"OAI-SearchBot":["partial","named",11,1],"ClaudeBot":["partial","*",1,2],"Claude-User":["partial","named",11,1],"Claude-SearchBot":["partial","named",11,1],"anthropic-ai":["partial","*",1,2],"PerplexityBot":["partial","named",11,1],"Perplexity-User":["partial","named",11,1],"Google-Extended":["partial","*",1,2],"Applebot-Extended":["partial","*",1,2],"CCBot":["partial","*",1,2],"Bytespider":["partial","*",1,2],"Amazonbot":["partial","*",1,2],"meta-externalagent":["partial","*",1,2],"cohere-ai":["partial","*",1,2],"Diffbot":["partial","*",1,2],"ContextFetchBot":["partial","*",1,2]},"llms":{"status":"disallowed"},"home":{"status":"disallowed"}},
{"rank":3858,"domain":"aeon.co","origin":"https://aeon.co","robots":{"status":"ok","http":200,"url":"https://aeon.co/robots.txt","bytes":645,"groups":2},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["partial","*",6,0],"OAI-SearchBot":["partial","*",6,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",6,0],"Claude-SearchBot":["partial","*",6,0],"anthropic-ai":["partial","*",6,0],"PerplexityBot":["partial","*",6,0],"Perplexity-User":["partial","*",6,0],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["partial","*",6,0],"meta-externalagent":["partial","*",6,0],"cohere-ai":["partial","*",6,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",6,0]},"llms":{"status":"missing","http":404,"bytes":37877},"home":{"status":"ok","http":200,"url":"https://aeon.co/","html_tokens":71112,"markdown_tokens":1098,"pruned_tokens":165}},
{"rank":3859,"domain":"mohurd.gov.cn","origin":"https://mohurd.gov.cn","robots":{"status":"timeout","http":null,"url":"https://mohurd.gov.cn/robots.txt","bytes":0,"groups":0,"error":"timed out after 10 s"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":3860,"domain":"ulisboa.pt","origin":"https://ulisboa.pt","robots":{"status":"refused","http":403,"url":"https://www.ulisboa.pt/robots.txt","bytes":390,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":3861,"domain":"unlv.edu","origin":"https://unlv.edu","robots":{"status":"refused","http":403,"url":"https://unlv.edu/robots.txt","bytes":560,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":3862,"domain":"counterpunch.org","origin":"https://counterpunch.org","robots":{"status":"refused","http":403,"url":"https://counterpunch.org/robots.txt","bytes":4910,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":3863,"domain":"ruhr-uni-bochum.de","origin":"https://ruhr-uni-bochum.de","robots":{"status":"ok","http":200,"url":"https://www.ruhr-uni-bochum.de/robots.txt","bytes":1618,"groups":1},"bots":{"GPTBot":["partial","*",23,18],"ChatGPT-User":["partial","*",23,18],"OAI-SearchBot":["partial","*",23,18],"ClaudeBot":["partial","*",23,18],"Claude-User":["partial","*",23,18],"Claude-SearchBot":["partial","*",23,18],"anthropic-ai":["partial","*",23,18],"PerplexityBot":["partial","*",23,18],"Perplexity-User":["partial","*",23,18],"Google-Extended":["partial","*",23,18],"Applebot-Extended":["partial","*",23,18],"CCBot":["partial","*",23,18],"Bytespider":["partial","*",23,18],"Amazonbot":["partial","*",23,18],"meta-externalagent":["partial","*",23,18],"cohere-ai":["partial","*",23,18],"Diffbot":["partial","*",23,18],"ContextFetchBot":["partial","*",23,18]},"llms":{"status":"missing","http":404,"bytes":6214},"home":{"status":"ok","http":200,"url":"https://www.ruhr-uni-bochum.de/de","html_tokens":30037,"markdown_tokens":206,"pruned_tokens":82}},
{"rank":3864,"domain":"westernunion.com","origin":"https://westernunion.com","robots":{"status":"ok","http":200,"url":"https://www.westernunion.com/robots.txt","bytes":1130,"groups":1},"bots":{"GPTBot":["partial","*",5,0],"ChatGPT-User":["partial","*",5,0],"OAI-SearchBot":["partial","*",5,0],"ClaudeBot":["partial","*",5,0],"Claude-User":["partial","*",5,0],"Claude-SearchBot":["partial","*",5,0],"anthropic-ai":["partial","*",5,0],"PerplexityBot":["partial","*",5,0],"Perplexity-User":["partial","*",5,0],"Google-Extended":["partial","*",5,0],"Applebot-Extended":["partial","*",5,0],"CCBot":["partial","*",5,0],"Bytespider":["partial","*",5,0],"Amazonbot":["partial","*",5,0],"meta-externalagent":["partial","*",5,0],"cohere-ai":["partial","*",5,0],"Diffbot":["partial","*",5,0],"ContextFetchBot":["partial","*",5,0]},"llms":{"status":"present","http":200,"bytes":879795},"home":{"status":"ok","http":200,"url":"https://www.westernunion.com/us/en/home.html","html_tokens":53438,"markdown_tokens":1314,"pruned_tokens":121}},
{"rank":3865,"domain":"globalresearch.ca","origin":"https://globalresearch.ca","robots":{"status":"ok","http":200,"url":"https://www.globalresearch.ca/robots.txt","bytes":286,"groups":2},"bots":{"GPTBot":["partial","*",2,0],"ChatGPT-User":["partial","*",2,0],"OAI-SearchBot":["partial","*",2,0],"ClaudeBot":["partial","*",2,0],"Claude-User":["partial","*",2,0],"Claude-SearchBot":["partial","*",2,0],"anthropic-ai":["partial","*",2,0],"PerplexityBot":["partial","*",2,0],"Perplexity-User":["partial","*",2,0],"Google-Extended":["partial","*",2,0],"Applebot-Extended":["partial","*",2,0],"CCBot":["partial","*",2,0],"Bytespider":["partial","*",2,0],"Amazonbot":["partial","*",2,0],"meta-externalagent":["partial","*",2,0],"cohere-ai":["partial","*",2,0],"Diffbot":["partial","*",2,0],"ContextFetchBot":["partial","*",2,0]},"llms":{"status":"missing","http":404,"bytes":69988},"home":{"status":"ok","http":200,"url":"https://www.globalresearch.ca/","html_tokens":37471,"markdown_tokens":1129,"pruned_tokens":140}},
{"rank":3866,"domain":"gamesindustry.biz","origin":"https://gamesindustry.biz","robots":{"status":"ok","http":200,"url":"https://www.gamesindustry.biz/robots.txt","bytes":5591,"groups":2},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["blocked","named",1,0],"OAI-SearchBot":["blocked","named",1,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["blocked","named",1,0],"Claude-SearchBot":["blocked","named",1,0],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["blocked","named",1,0],"Google-Extended":["partial","*",9,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",9,0]},"llms":{"status":"missing","http":404,"bytes":2295},"home":{"status":"ok","http":200,"url":"https://www.gamesindustry.biz/","html_tokens":47355,"markdown_tokens":584,"pruned_tokens":114}},
{"rank":3867,"domain":"servicenow.com","origin":"https://servicenow.com","robots":{"status":"refused","http":403,"url":"https://www.servicenow.com/robots.txt","bytes":386,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":3868,"domain":"rijksmuseum.nl","origin":"https://rijksmuseum.nl","robots":{"status":"ok","http":200,"url":"https://www.rijksmuseum.nl/robots.txt","bytes":143,"groups":1},"bots":{"GPTBot":["partial","*",2,1],"ChatGPT-User":["partial","*",2,1],"OAI-SearchBot":["partial","*",2,1],"ClaudeBot":["partial","*",2,1],"Claude-User":["partial","*",2,1],"Claude-SearchBot":["partial","*",2,1],"anthropic-ai":["partial","*",2,1],"PerplexityBot":["partial","*",2,1],"Perplexity-User":["partial","*",2,1],"Google-Extended":["partial","*",2,1],"Applebot-Extended":["partial","*",2,1],"CCBot":["partial","*",2,1],"Bytespider":["partial","*",2,1],"Amazonbot":["partial","*",2,1],"meta-externalagent":["partial","*",2,1],"cohere-ai":["partial","*",2,1],"Diffbot":["partial","*",2,1],"ContextFetchBot":["partial","*",2,1]},"llms":{"status":"missing","http":404,"bytes":51881},"home":{"status":"ok","http":200,"url":"https://www.rijksmuseum.nl/nl","html_tokens":27828,"markdown_tokens":19,"pruned_tokens":19}},
{"rank":3869,"domain":"bet365.com","origin":"https://bet365.com","robots":{"status":"refused","http":403,"url":"https://www.bet365.com/robots.txt","bytes":4543,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":3870,"domain":"medtronic.com","origin":"https://medtronic.com","robots":{"status":"ok","http":200,"url":"https://www.medtronic.com/robots.txt","bytes":3874,"groups":1},"bots":{"GPTBot":["partial","*",23,9],"ChatGPT-User":["partial","*",23,9],"OAI-SearchBot":["partial","*",23,9],"ClaudeBot":["partial","*",23,9],"Claude-User":["partial","*",23,9],"Claude-SearchBot":["partial","*",23,9],"anthropic-ai":["partial","*",23,9],"PerplexityBot":["partial","*",23,9],"Perplexity-User":["partial","*",23,9],"Google-Extended":["partial","*",23,9],"Applebot-Extended":["partial","*",23,9],"CCBot":["partial","*",23,9],"Bytespider":["partial","*",23,9],"Amazonbot":["partial","*",23,9],"meta-externalagent":["partial","*",23,9],"cohere-ai":["partial","*",23,9],"Diffbot":["partial","*",23,9],"ContextFetchBot":["partial","*",23,9]},"llms":{"status":"missing","http":404,"bytes":1885283},"home":{"status":"ok","http":200,"url":"https://www.medtronic.com/en-us/index.html","html_tokens":178701,"markdown_tokens":57,"pruned_tokens":28}},
{"rank":3871,"domain":"remove.bg","origin":"https://remove.bg","robots":{"status":"ok","http":200,"url":"https://www.remove.bg/robots.txt","bytes":115,"groups":2},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":52297},"home":{"status":"ok","http":200,"url":"https://www.remove.bg/","html_tokens":14979,"markdown_tokens":319,"pruned_tokens":303}},
{"rank":3872,"domain":"chng.it","origin":"https://chng.it","robots":{"status":"missing","http":404,"url":"https://chng.it/robots.txt","bytes":365,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":363},"home":{"status":"ok","http":200,"url":"https://www.change.org/","html_tokens":77114,"markdown_tokens":82,"pruned_tokens":82}},
{"rank":3873,"domain":"univision.com","origin":"https://univision.com","robots":{"status":"ok","http":200,"url":"https://www.univision.com/robots.txt","bytes":3013,"groups":28},"bots":{"GPTBot":["partial","*",15,1],"ChatGPT-User":["partial","*",15,1],"OAI-SearchBot":["partial","*",15,1],"ClaudeBot":["partial","*",15,1],"Claude-User":["partial","*",15,1],"Claude-SearchBot":["partial","*",15,1],"anthropic-ai":["partial","*",15,1],"PerplexityBot":["partial","*",15,1],"Perplexity-User":["partial","*",15,1],"Google-Extended":["partial","*",15,1],"Applebot-Extended":["partial","*",15,1],"CCBot":["partial","*",15,1],"Bytespider":["partial","*",15,1],"Amazonbot":["partial","*",15,1],"meta-externalagent":["partial","*",15,1],"cohere-ai":["partial","*",15,1],"Diffbot":["partial","*",15,1],"ContextFetchBot":["partial","*",15,1]},"llms":{"status":"missing","http":404,"bytes":143448},"home":{"status":"ok","http":200,"url":"https://www.univision.com/","html_tokens":549857,"markdown_tokens":2304,"pruned_tokens":120}},
{"rank":3874,"domain":"nmap.org","origin":"https://nmap.org","robots":{"status":"unreachable","http":null,"url":"https://nmap.org/robots.txt","bytes":0,"groups":0,"error":"[SSL: CERTIFICATE_VERIFY_FAILED] certificate verify failed: certificate has expired (_ssl.c:1028)"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":3875,"domain":"smu.edu","origin":"https://smu.edu","robots":{"status":"ok","http":200,"url":"https://www.smu.edu/robots.txt","bytes":4776,"groups":10},"bots":{"GPTBot":["partial","named",22,0],"ChatGPT-User":["partial","*",22,0],"OAI-SearchBot":["partial","*",22,0],"ClaudeBot":["partial","named",22,0],"Claude-User":["partial","*",22,0],"Claude-SearchBot":["partial","*",22,0],"anthropic-ai":["partial","named",22,0],"PerplexityBot":["partial","named",22,0],"Perplexity-User":["partial","*",22,0],"Google-Extended":["partial","named",22,0],"Applebot-Extended":["partial","*",22,0],"CCBot":["partial","*",22,0],"Bytespider":["partial","*",22,0],"Amazonbot":["partial","*",22,0],"meta-externalagent":["partial","named",22,0],"cohere-ai":["partial","named",22,0],"Diffbot":["partial","named",22,0],"ContextFetchBot":["partial","*",22,0]},"llms":{"status":"missing","http":404,"bytes":1245},"home":{"status":"ok","http":200,"url":"https://www.smu.edu/","html_tokens":49597,"markdown_tokens":317,"pruned_tokens":312}},
{"rank":3876,"domain":"thinkprogress.org","origin":"https://thinkprogress.org","robots":{"status":"refused","http":403,"url":"https://thinkprogress.org/robots.txt","bytes":243,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":3877,"domain":"ktla.com","origin":"https://ktla.com","robots":{"status":"ok","http":200,"url":"https://ktla.com/robots.txt","bytes":2294,"groups":40},"bots":{"GPTBot":["blocked","named",2,0],"ChatGPT-User":["blocked","named",1,0],"OAI-SearchBot":["blocked","named",1,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",12,1],"Claude-SearchBot":["partial","*",12,1],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["blocked","named",2,0],"Perplexity-User":["partial","*",12,1],"Google-Extended":["blocked","named",2,0],"Applebot-Extended":["blocked","named",2,0],"CCBot":["blocked","named",2,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["partial","*",12,1],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",12,1]},"llms":{"status":"http_error","http":403,"bytes":6514},"home":{"status":"http_error","http":403,"url":"https://ktla.com/"}},
{"rank":3878,"domain":"most.gov.cn","origin":"https://most.gov.cn","robots":{"status":"missing","http":404,"url":"https://most.gov.cn/robots.txt","bytes":7971,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":7971},"home":{"status":"ok","http":200,"url":"https://most.gov.cn/","html_tokens":40,"markdown_tokens":0,"pruned_tokens":0}},
{"rank":3879,"domain":"blog.youtube","origin":"https://blog.youtube","robots":{"status":"ok","http":200,"url":"https://blog.youtube/robots.txt","bytes":154,"groups":1},"bots":{"GPTBot":["partial","*",4,0],"ChatGPT-User":["partial","*",4,0],"OAI-SearchBot":["partial","*",4,0],"ClaudeBot":["partial","*",4,0],"Claude-User":["partial","*",4,0],"Claude-SearchBot":["partial","*",4,0],"anthropic-ai":["partial","*",4,0],"PerplexityBot":["partial","*",4,0],"Perplexity-User":["partial","*",4,0],"Google-Extended":["partial","*",4,0],"Applebot-Extended":["partial","*",4,0],"CCBot":["partial","*",4,0],"Bytespider":["partial","*",4,0],"Amazonbot":["partial","*",4,0],"meta-externalagent":["partial","*",4,0],"cohere-ai":["partial","*",4,0],"Diffbot":["partial","*",4,0],"ContextFetchBot":["partial","*",4,0]},"llms":{"status":"missing","http":404,"bytes":2429},"home":{"status":"ok","http":200,"url":"https://blog.youtube/","html_tokens":58134,"markdown_tokens":2290,"pruned_tokens":146}},
{"rank":3880,"domain":"cleantechnica.com","origin":"https://cleantechnica.com","robots":{"status":"ok","http":200,"url":"https://cleantechnica.com/robots.txt","bytes":300,"groups":1},"bots":{"GPTBot":["partial","*",5,1],"ChatGPT-User":["partial","*",5,1],"OAI-SearchBot":["partial","*",5,1],"ClaudeBot":["partial","*",5,1],"Claude-User":["partial","*",5,1],"Claude-SearchBot":["partial","*",5,1],"anthropic-ai":["partial","*",5,1],"PerplexityBot":["partial","*",5,1],"Perplexity-User":["partial","*",5,1],"Google-Extended":["partial","*",5,1],"Applebot-Extended":["partial","*",5,1],"CCBot":["partial","*",5,1],"Bytespider":["partial","*",5,1],"Amazonbot":["partial","*",5,1],"meta-externalagent":["partial","*",5,1],"cohere-ai":["partial","*",5,1],"Diffbot":["partial","*",5,1],"ContextFetchBot":["partial","*",5,1]},"llms":{"status":"crawl_delay","error":"Crawl-delay 600 s"},"home":{"status":"crawl_delay","error":"Crawl-delay 600 s"}},
{"rank":3881,"domain":"mouser.com","origin":"https://mouser.com","robots":{"status":"unreachable","http":null,"url":"https://mouser.com/robots.txt","bytes":0,"groups":0,"error":"<StreamReset stream_id:1, error_code:2, remote_reset:True>"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":3882,"domain":"humblebundle.com","origin":"https://humblebundle.com","robots":{"status":"ok","http":200,"url":"https://www.humblebundle.com/robots.txt","bytes":1256,"groups":1},"bots":{"GPTBot":["partial","*",13,0],"ChatGPT-User":["partial","*",13,0],"OAI-SearchBot":["partial","*",13,0],"ClaudeBot":["partial","*",13,0],"Claude-User":["partial","*",13,0],"Claude-SearchBot":["partial","*",13,0],"anthropic-ai":["partial","*",13,0],"PerplexityBot":["partial","*",13,0],"Perplexity-User":["partial","*",13,0],"Google-Extended":["partial","*",13,0],"Applebot-Extended":["partial","*",13,0],"CCBot":["partial","*",13,0],"Bytespider":["partial","*",13,0],"Amazonbot":["partial","*",13,0],"meta-externalagent":["partial","*",13,0],"cohere-ai":["partial","*",13,0],"Diffbot":["partial","*",13,0],"ContextFetchBot":["partial","*",13,0]},"llms":{"status":"missing","http":404,"bytes":69790},"home":{"status":"ok","http":200,"url":"https://www.humblebundle.com/","html_tokens":143851,"markdown_tokens":27,"pruned_tokens":27}},
{"rank":3883,"domain":"churchofjesuschrist.org","origin":"https://churchofjesuschrist.org","robots":{"status":"ok","http":200,"url":"https://www.churchofjesuschrist.org/robots.txt","bytes":15329,"groups":1},"bots":{"GPTBot":["partial","*",15,0],"ChatGPT-User":["partial","*",15,0],"OAI-SearchBot":["partial","*",15,0],"ClaudeBot":["partial","*",15,0],"Claude-User":["partial","*",15,0],"Claude-SearchBot":["partial","*",15,0],"anthropic-ai":["partial","*",15,0],"PerplexityBot":["partial","*",15,0],"Perplexity-User":["partial","*",15,0],"Google-Extended":["partial","*",15,0],"Applebot-Extended":["partial","*",15,0],"CCBot":["partial","*",15,0],"Bytespider":["partial","*",15,0],"Amazonbot":["partial","*",15,0],"meta-externalagent":["partial","*",15,0],"cohere-ai":["partial","*",15,0],"Diffbot":["partial","*",15,0],"ContextFetchBot":["partial","*",15,0]},"llms":{"status":"http_error","http":503,"bytes":221},"home":{"status":"ok","http":200,"url":"https://www.churchofjesuschrist.org/?lang=eng","html_tokens":20544,"markdown_tokens":1133,"pruned_tokens":69}},
{"rank":3884,"domain":"wayne.edu","origin":"https://wayne.edu","robots":{"status":"ok","http":200,"url":"https://wayne.edu/robots.txt","bytes":94,"groups":1},"bots":{"GPTBot":["partial","*",2,0],"ChatGPT-User":["partial","*",2,0],"OAI-SearchBot":["partial","*",2,0],"ClaudeBot":["partial","*",2,0],"Claude-User":["partial","*",2,0],"Claude-SearchBot":["partial","*",2,0],"anthropic-ai":["partial","*",2,0],"PerplexityBot":["partial","*",2,0],"Perplexity-User":["partial","*",2,0],"Google-Extended":["partial","*",2,0],"Applebot-Extended":["partial","*",2,0],"CCBot":["partial","*",2,0],"Bytespider":["partial","*",2,0],"Amazonbot":["partial","*",2,0],"meta-externalagent":["partial","*",2,0],"cohere-ai":["partial","*",2,0],"Diffbot":["partial","*",2,0],"ContextFetchBot":["partial","*",2,0]},"llms":{"status":"missing","http":404,"bytes":13838},"home":{"status":"ok","http":200,"url":"https://wayne.edu/","html_tokens":18552,"markdown_tokens":72,"pruned_tokens":72}},
{"rank":3885,"domain":"us.es","origin":"https://www.us.es","robots":{"status":"ok","http":200,"url":"https://www.us.es/robots.txt","bytes":23,"groups":1},"bots":{"GPTBot":["allowed","*",0,1],"ChatGPT-User":["allowed","*",0,1],"OAI-SearchBot":["allowed","*",0,1],"ClaudeBot":["allowed","*",0,1],"Claude-User":["allowed","*",0,1],"Claude-SearchBot":["allowed","*",0,1],"anthropic-ai":["allowed","*",0,1],"PerplexityBot":["allowed","*",0,1],"Perplexity-User":["allowed","*",0,1],"Google-Extended":["allowed","*",0,1],"Applebot-Extended":["allowed","*",0,1],"CCBot":["allowed","*",0,1],"Bytespider":["allowed","*",0,1],"Amazonbot":["allowed","*",0,1],"meta-externalagent":["allowed","*",0,1],"cohere-ai":["allowed","*",0,1],"Diffbot":["allowed","*",0,1],"ContextFetchBot":["allowed","*",0,1]},"llms":{"status":"missing","http":404,"bytes":183},"home":{"status":"ok","http":200,"url":"https://www.us.es/","html_tokens":58269,"markdown_tokens":160,"pruned_tokens":73}},
{"rank":3886,"domain":"mxtoolbox.com","origin":"https://mxtoolbox.com","robots":{"status":"ok","http":200,"url":"https://mxtoolbox.com/robots.txt","bytes":946,"groups":4},"bots":{"GPTBot":["partial","*",6,0],"ChatGPT-User":["partial","*",6,0],"OAI-SearchBot":["partial","*",6,0],"ClaudeBot":["partial","*",6,0],"Claude-User":["partial","*",6,0],"Claude-SearchBot":["partial","*",6,0],"anthropic-ai":["partial","*",6,0],"PerplexityBot":["partial","*",6,0],"Perplexity-User":["partial","*",6,0],"Google-Extended":["partial","*",6,0],"Applebot-Extended":["partial","*",6,0],"CCBot":["partial","*",6,0],"Bytespider":["partial","*",6,0],"Amazonbot":["partial","*",6,0],"meta-externalagent":["partial","named",1,0],"cohere-ai":["partial","*",6,0],"Diffbot":["partial","*",6,0],"ContextFetchBot":["partial","*",6,0]},"llms":{"status":"present","http":200,"bytes":2533},"home":{"status":"ok","http":200,"url":"https://mxtoolbox.com/","html_tokens":5285,"markdown_tokens":119,"pruned_tokens":119}},
{"rank":3887,"domain":"grok.com","origin":"https://grok.com","robots":{"status":"ok","http":200,"url":"https://grok.com/robots.txt","bytes":533,"groups":2},"bots":{"GPTBot":["partial","named",1,8],"ChatGPT-User":["partial","named",1,8],"OAI-SearchBot":["partial","*",6,5],"ClaudeBot":["partial","named",1,8],"Claude-User":["partial","*",6,5],"Claude-SearchBot":["partial","*",6,5],"anthropic-ai":["partial","*",6,5],"PerplexityBot":["partial","named",1,8],"Perplexity-User":["partial","*",6,5],"Google-Extended":["partial","named",1,8],"Applebot-Extended":["partial","named",1,8],"CCBot":["partial","*",6,5],"Bytespider":["partial","*",6,5],"Amazonbot":["partial","*",6,5],"meta-externalagent":["partial","*",6,5],"cohere-ai":["partial","*",6,5],"Diffbot":["partial","*",6,5],"ContextFetchBot":["partial","*",6,5]},"llms":{"status":"missing","http":404,"bytes":362374},"home":{"status":"ok","http":200,"url":"https://grok.com/","html_tokens":98678,"markdown_tokens":5,"pruned_tokens":5}},
{"rank":3888,"domain":"delish.com","origin":"https://delish.com","robots":{"status":"ok","http":200,"url":"https://delish.com/robots.txt","bytes":957,"groups":3},"bots":{"GPTBot":["partial","named",32,0],"ChatGPT-User":["partial","named",32,0],"OAI-SearchBot":["partial","named",32,0],"ClaudeBot":["partial","*",19,0],"Claude-User":["partial","*",19,0],"Claude-SearchBot":["partial","*",19,0],"anthropic-ai":["partial","*",19,0],"PerplexityBot":["partial","*",19,0],"Perplexity-User":["partial","*",19,0],"Google-Extended":["partial","*",19,0],"Applebot-Extended":["partial","*",19,0],"CCBot":["partial","*",19,0],"Bytespider":["partial","*",19,0],"Amazonbot":["partial","*",19,0],"meta-externalagent":["partial","*",19,0],"cohere-ai":["partial","*",19,0],"Diffbot":["partial","*",19,0],"ContextFetchBot":["partial","*",19,0]},"llms":{"status":"missing","http":404,"bytes":84},"home":{"status":"ok","http":200,"url":"https://www.delish.com/","html_tokens":184794,"markdown_tokens":51,"pruned_tokens":51}},
{"rank":3889,"domain":"cbo.gov","origin":"https://cbo.gov","robots":{"status":"refused","http":403,"url":"https://cbo.gov/robots.txt","bytes":767,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":3890,"domain":"uptimerobot.com","origin":"https://uptimerobot.com","robots":{"status":"ok","http":200,"url":"https://uptimerobot.com/robots.txt","bytes":126,"groups":1},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"present","http":200,"bytes":10956},"home":{"status":"ok","http":200,"url":"https://uptimerobot.com/","html_tokens":80312,"markdown_tokens":1350,"pruned_tokens":782}},
{"rank":3891,"domain":"naturalnews.com","origin":"https://naturalnews.com","robots":{"status":"ok","http":200,"url":"https://naturalnews.com/robots.txt","bytes":971,"groups":2},"bots":{"GPTBot":["partial","*",38,0],"ChatGPT-User":["partial","*",38,0],"OAI-SearchBot":["partial","*",38,0],"ClaudeBot":["partial","*",38,0],"Claude-User":["partial","*",38,0],"Claude-SearchBot":["partial","*",38,0],"anthropic-ai":["partial","*",38,0],"PerplexityBot":["partial","*",38,0],"Perplexity-User":["partial","*",38,0],"Google-Extended":["partial","*",38,0],"Applebot-Extended":["partial","*",38,0],"CCBot":["partial","*",38,0],"Bytespider":["partial","*",38,0],"Amazonbot":["partial","*",38,0],"meta-externalagent":["partial","*",38,0],"cohere-ai":["partial","*",38,0],"Diffbot":["partial","*",38,0],"ContextFetchBot":["partial","*",38,0]},"llms":{"status":"html","http":200,"bytes":18311},"home":{"status":"ok","http":200,"url":"https://naturalnews.com/","html_tokens":42247,"markdown_tokens":1219,"pruned_tokens":155}},
{"rank":3892,"domain":"haskell.org","origin":"https://haskell.org","robots":{"status":"ok","http":200,"url":"https://www.haskell.org/robots.txt","bytes":104,"groups":3},"bots":{"GPTBot":["partial","*",1,0],"ChatGPT-User":["partial","*",1,0],"OAI-SearchBot":["partial","*",1,0],"ClaudeBot":["partial","*",1,0],"Claude-User":["partial","*",1,0],"Claude-SearchBot":["partial","*",1,0],"anthropic-ai":["partial","*",1,0],"PerplexityBot":["partial","*",1,0],"Perplexity-User":["partial","*",1,0],"Google-Extended":["partial","*",1,0],"Applebot-Extended":["partial","*",1,0],"CCBot":["partial","*",1,0],"Bytespider":["partial","*",1,0],"Amazonbot":["partial","*",1,0],"meta-externalagent":["partial","*",1,0],"cohere-ai":["partial","*",1,0],"Diffbot":["partial","*",1,0],"ContextFetchBot":["partial","*",1,0]},"llms":{"status":"missing","http":404,"bytes":146},"home":{"status":"ok","http":200,"url":"https://www.haskell.org/","html_tokens":15752,"markdown_tokens":2321,"pruned_tokens":426}},
{"rank":3893,"domain":"at.ua","origin":"https://at.ua","robots":{"status":"timeout","http":null,"url":"https://at.ua/robots.txt","bytes":0,"groups":0,"error":"ConnectTimeout"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":3894,"domain":"hashnode.dev","origin":"https://hashnode.dev","robots":{"status":"refused","http":403,"url":"https://hashnode.dev/robots.txt","bytes":5507,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":3895,"domain":"usercentrics.com","origin":"https://usercentrics.com","robots":{"status":"ok","http":200,"url":"https://usercentrics.com/robots.txt","bytes":1135,"groups":1},"bots":{"GPTBot":["partial","*",24,1],"ChatGPT-User":["partial","*",24,1],"OAI-SearchBot":["partial","*",24,1],"ClaudeBot":["partial","*",24,1],"Claude-User":["partial","*",24,1],"Claude-SearchBot":["partial","*",24,1],"anthropic-ai":["partial","*",24,1],"PerplexityBot":["partial","*",24,1],"Perplexity-User":["partial","*",24,1],"Google-Extended":["partial","*",24,1],"Applebot-Extended":["partial","*",24,1],"CCBot":["partial","*",24,1],"Bytespider":["partial","*",24,1],"Amazonbot":["partial","*",24,1],"meta-externalagent":["partial","*",24,1],"cohere-ai":["partial","*",24,1],"Diffbot":["partial","*",24,1],"ContextFetchBot":["partial","*",24,1]},"llms":{"status":"http_error","http":403,"bytes":5569},"home":{"status":"http_error","http":403,"url":"https://usercentrics.com/"}},
{"rank":3896,"domain":"yellowpages.com","origin":"https://yellowpages.com","robots":{"status":"refused","http":403,"url":"https://www.yellowpages.com/","bytes":5851,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":3897,"domain":"kfw.de","origin":"https://kfw.de","robots":{"status":"ok","http":200,"url":"https://www.kfw.de/robots.txt","bytes":1352,"groups":1},"bots":{"GPTBot":["partial","*",23,6],"ChatGPT-User":["partial","*",23,6],"OAI-SearchBot":["partial","*",23,6],"ClaudeBot":["partial","*",23,6],"Claude-User":["partial","*",23,6],"Claude-SearchBot":["partial","*",23,6],"anthropic-ai":["partial","*",23,6],"PerplexityBot":["partial","*",23,6],"Perplexity-User":["partial","*",23,6],"Google-Extended":["partial","*",23,6],"Applebot-Extended":["partial","*",23,6],"CCBot":["partial","*",23,6],"Bytespider":["partial","*",23,6],"Amazonbot":["partial","*",23,6],"meta-externalagent":["partial","*",23,6],"cohere-ai":["partial","*",23,6],"Diffbot":["partial","*",23,6],"ContextFetchBot":["partial","*",23,6]},"llms":{"status":"missing","http":404,"bytes":31276},"home":{"status":"ok","http":200,"url":"https://www.kfw.de/kfw.de.html","html_tokens":20982,"markdown_tokens":228,"pruned_tokens":100}},
{"rank":3898,"domain":"web.skype.com","origin":"https://web.skype.com","robots":{"status":"html","http":200,"url":"https://teams.live.com/v2/unsupported-browser#env=life&browserOverrides={%22firefox%22:{%22isSupported%22:true,%22minVersion%22:%2297.0%22},%22safari%22:{%22isSupported%22:true,%22minVersion%22:%2216.0%22}}","bytes":231754,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":3899,"domain":"defensenews.com","origin":"https://defensenews.com","robots":{"status":"ok","http":200,"url":"https://www.defensenews.com/robots.txt","bytes":254,"groups":1},"bots":{"GPTBot":["partial","*",3,0],"ChatGPT-User":["partial","*",3,0],"OAI-SearchBot":["partial","*",3,0],"ClaudeBot":["partial","*",3,0],"Claude-User":["partial","*",3,0],"Claude-SearchBot":["partial","*",3,0],"anthropic-ai":["partial","*",3,0],"PerplexityBot":["partial","*",3,0],"Perplexity-User":["partial","*",3,0],"Google-Extended":["partial","*",3,0],"Applebot-Extended":["partial","*",3,0],"CCBot":["partial","*",3,0],"Bytespider":["partial","*",3,0],"Amazonbot":["partial","*",3,0],"meta-externalagent":["partial","*",3,0],"cohere-ai":["partial","*",3,0],"Diffbot":["partial","*",3,0],"ContextFetchBot":["partial","*",3,0]},"llms":{"status":"missing","http":404,"bytes":53070},"home":{"status":"ok","http":200,"url":"https://www.defensenews.com/","html_tokens":127814,"markdown_tokens":194,"pruned_tokens":78}},
{"rank":3900,"domain":"arkansas.gov","origin":"https://arkansas.gov","robots":{"status":"missing","http":410,"url":"https://arkansas.gov/robots.txt","bytes":47,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":410,"bytes":47},"home":{"status":"http_error","http":403,"url":"https://portal.arkansas.gov/"}},
{"rank":3901,"domain":"jugem.jp","origin":"https://jugem.jp","robots":{"status":"ok","http":200,"url":"https://jugem.jp/robots.txt","bytes":39,"groups":1},"bots":{"GPTBot":["partial","*",1,0],"ChatGPT-User":["partial","*",1,0],"OAI-SearchBot":["partial","*",1,0],"ClaudeBot":["partial","*",1,0],"Claude-User":["partial","*",1,0],"Claude-SearchBot":["partial","*",1,0],"anthropic-ai":["partial","*",1,0],"PerplexityBot":["partial","*",1,0],"Perplexity-User":["partial","*",1,0],"Google-Extended":["partial","*",1,0],"Applebot-Extended":["partial","*",1,0],"CCBot":["partial","*",1,0],"Bytespider":["partial","*",1,0],"Amazonbot":["partial","*",1,0],"meta-externalagent":["partial","*",1,0],"cohere-ai":["partial","*",1,0],"Diffbot":["partial","*",1,0],"ContextFetchBot":["partial","*",1,0]},"llms":{"status":"missing","http":404,"bytes":2719},"home":{"status":"ok","http":200,"url":"https://jugem.jp/","html_tokens":7226,"markdown_tokens":23,"pruned_tokens":23}},
{"rank":3902,"domain":"odnoklassniki.ru","origin":"https://odnoklassniki.ru","robots":{"status":"ok","http":200,"url":"https://ok.ru/robots.txt","bytes":6051,"groups":1},"bots":{"GPTBot":["partial","*",34,0],"ChatGPT-User":["partial","*",34,0],"OAI-SearchBot":["partial","*",34,0],"ClaudeBot":["partial","*",34,0],"Claude-User":["partial","*",34,0],"Claude-SearchBot":["partial","*",34,0],"anthropic-ai":["partial","*",34,0],"PerplexityBot":["partial","*",34,0],"Perplexity-User":["partial","*",34,0],"Google-Extended":["partial","*",34,0],"Applebot-Extended":["partial","*",34,0],"CCBot":["partial","*",34,0],"Bytespider":["partial","*",34,0],"Amazonbot":["partial","*",34,0],"meta-externalagent":["partial","*",34,0],"cohere-ai":["partial","*",34,0],"Diffbot":["partial","*",34,0],"ContextFetchBot":["partial","*",34,0]},"llms":{"status":"missing","http":404,"bytes":0},"home":{"status":"ok","http":200,"url":"https://ok.ru/","html_tokens":64469,"markdown_tokens":45,"pruned_tokens":45}},
{"rank":3903,"domain":"beatport.com","origin":"https://beatport.com","robots":{"status":"ok","http":200,"url":"https://www.beatport.com/robots.txt","bytes":170,"groups":2},"bots":{"GPTBot":["allowed","*",0,1],"ChatGPT-User":["allowed","*",0,1],"OAI-SearchBot":["allowed","*",0,1],"ClaudeBot":["allowed","*",0,1],"Claude-User":["allowed","*",0,1],"Claude-SearchBot":["allowed","*",0,1],"anthropic-ai":["allowed","*",0,1],"PerplexityBot":["allowed","*",0,1],"Perplexity-User":["allowed","*",0,1],"Google-Extended":["allowed","*",0,1],"Applebot-Extended":["allowed","*",0,1],"CCBot":["allowed","*",0,1],"Bytespider":["allowed","*",0,1],"Amazonbot":["allowed","*",0,1],"meta-externalagent":["allowed","*",0,1],"cohere-ai":["allowed","*",0,1],"Diffbot":["allowed","*",0,1],"ContextFetchBot":["allowed","*",0,1]},"llms":{"status":"missing","http":404,"bytes":66298},"home":{"status":"http_error","http":403,"url":"https://beatport.com/"}},
{"rank":3904,"domain":"mirniy-sp.ru","origin":"https://mirniy-sp.ru","robots":{"status":"refused","http":403,"url":"https://mirniy-sp.ru/robots.txt","bytes":4912,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":3905,"domain":"sonypictures.com","origin":"https://sonypictures.com","robots":{"status":"ok","http":200,"url":"https://www.sonypictures.com/robots.txt","bytes":1586,"groups":1},"bots":{"GPTBot":["partial","*",22,18],"ChatGPT-User":["partial","*",22,18],"OAI-SearchBot":["partial","*",22,18],"ClaudeBot":["partial","*",22,18],"Claude-User":["partial","*",22,18],"Claude-SearchBot":["partial","*",22,18],"anthropic-ai":["partial","*",22,18],"PerplexityBot":["partial","*",22,18],"Perplexity-User":["partial","*",22,18],"Google-Extended":["partial","*",22,18],"Applebot-Extended":["partial","*",22,18],"CCBot":["partial","*",22,18],"Bytespider":["partial","*",22,18],"Amazonbot":["partial","*",22,18],"meta-externalagent":["partial","*",22,18],"cohere-ai":["partial","*",22,18],"Diffbot":["partial","*",22,18],"ContextFetchBot":["partial","*",22,18]},"llms":{"status":"missing","http":404,"bytes":194},"home":{"status":"ok","http":200,"url":"https://www.sonypictures.com/","html_tokens":15040,"markdown_tokens":28,"pruned_tokens":28}},
{"rank":3906,"domain":"ndl.go.jp","origin":"https://ndl.go.jp","robots":{"status":"ok","http":200,"url":"https://www.ndl.go.jp/robots.txt","bytes":66,"groups":1},"bots":{"GPTBot":["allowed","*",0,1],"ChatGPT-User":["allowed","*",0,1],"OAI-SearchBot":["allowed","*",0,1],"ClaudeBot":["allowed","*",0,1],"Claude-User":["allowed","*",0,1],"Claude-SearchBot":["allowed","*",0,1],"anthropic-ai":["allowed","*",0,1],"PerplexityBot":["allowed","*",0,1],"Perplexity-User":["allowed","*",0,1],"Google-Extended":["allowed","*",0,1],"Applebot-Extended":["allowed","*",0,1],"CCBot":["allowed","*",0,1],"Bytespider":["allowed","*",0,1],"Amazonbot":["allowed","*",0,1],"meta-externalagent":["allowed","*",0,1],"cohere-ai":["allowed","*",0,1],"Diffbot":["allowed","*",0,1],"ContextFetchBot":["allowed","*",0,1]},"llms":{"status":"missing","http":404,"bytes":157763},"home":{"status":"ok","http":200,"url":"https://www.ndl.go.jp/","html_tokens":52809,"markdown_tokens":12,"pruned_tokens":12}},
{"rank":3907,"domain":"charitynavigator.org","origin":"https://charitynavigator.org","robots":{"status":"ok","http":200,"url":"https://www.charitynavigator.org/robots.txt","bytes":293,"groups":1},"bots":{"GPTBot":["partial","*",7,0],"ChatGPT-User":["partial","*",7,0],"OAI-SearchBot":["partial","*",7,0],"ClaudeBot":["partial","*",7,0],"Claude-User":["partial","*",7,0],"Claude-SearchBot":["partial","*",7,0],"anthropic-ai":["partial","*",7,0],"PerplexityBot":["partial","*",7,0],"Perplexity-User":["partial","*",7,0],"Google-Extended":["partial","*",7,0],"Applebot-Extended":["partial","*",7,0],"CCBot":["partial","*",7,0],"Bytespider":["partial","*",7,0],"Amazonbot":["partial","*",7,0],"meta-externalagent":["partial","*",7,0],"cohere-ai":["partial","*",7,0],"Diffbot":["partial","*",7,0],"ContextFetchBot":["partial","*",7,0]},"llms":{"status":"missing","http":404,"bytes":159655},"home":{"status":"ok","http":200,"url":"https://www.charitynavigator.org/","html_tokens":62571,"markdown_tokens":216,"pruned_tokens":146}},
{"rank":3908,"domain":"explodingtopics.com","origin":"https://explodingtopics.com","robots":{"status":"ok","http":200,"url":"https://explodingtopics.com/robots.txt","bytes":419,"groups":1},"bots":{"GPTBot":["allowed","*",0,1],"ChatGPT-User":["allowed","*",0,1],"OAI-SearchBot":["allowed","*",0,1],"ClaudeBot":["allowed","*",0,1],"Claude-User":["allowed","*",0,1],"Claude-SearchBot":["allowed","*",0,1],"anthropic-ai":["allowed","*",0,1],"PerplexityBot":["allowed","*",0,1],"Perplexity-User":["allowed","*",0,1],"Google-Extended":["allowed","*",0,1],"Applebot-Extended":["allowed","*",0,1],"CCBot":["allowed","*",0,1],"Bytespider":["allowed","*",0,1],"Amazonbot":["allowed","*",0,1],"meta-externalagent":["allowed","*",0,1],"cohere-ai":["allowed","*",0,1],"Diffbot":["allowed","*",0,1],"ContextFetchBot":["allowed","*",0,1]},"llms":{"status":"missing","http":404,"bytes":194724},"home":{"status":"ok","http":200,"url":"https://explodingtopics.com/","html_tokens":116295,"markdown_tokens":1860,"pruned_tokens":131}},
{"rank":3909,"domain":"youth.cn","origin":"https://youth.cn","robots":{"status":"ok","http":200,"url":"https://www.youth.cn/robots.txt","bytes":59,"groups":1},"bots":{"GPTBot":["partial","*",1,0],"ChatGPT-User":["partial","*",1,0],"OAI-SearchBot":["partial","*",1,0],"ClaudeBot":["partial","*",1,0],"Claude-User":["partial","*",1,0],"Claude-SearchBot":["partial","*",1,0],"anthropic-ai":["partial","*",1,0],"PerplexityBot":["partial","*",1,0],"Perplexity-User":["partial","*",1,0],"Google-Extended":["partial","*",1,0],"Applebot-Extended":["partial","*",1,0],"CCBot":["partial","*",1,0],"Bytespider":["partial","*",1,0],"Amazonbot":["partial","*",1,0],"meta-externalagent":["partial","*",1,0],"cohere-ai":["partial","*",1,0],"Diffbot":["partial","*",1,0],"ContextFetchBot":["partial","*",1,0]},"llms":{"status":"missing","http":404,"bytes":128},"home":{"status":"ok","http":200,"url":"https://www.youth.cn/","html_tokens":25205,"markdown_tokens":151,"pruned_tokens":151}},
{"rank":3910,"domain":"cambridgeenglish.org","origin":"https://cambridgeenglish.org","robots":{"status":"ok","http":200,"url":"https://www.cambridgeenglish.org/robots.txt","bytes":130,"groups":1},"bots":{"GPTBot":["partial","*",1,1],"ChatGPT-User":["partial","*",1,1],"OAI-SearchBot":["partial","*",1,1],"ClaudeBot":["partial","*",1,1],"Claude-User":["partial","*",1,1],"Claude-SearchBot":["partial","*",1,1],"anthropic-ai":["partial","*",1,1],"PerplexityBot":["partial","*",1,1],"Perplexity-User":["partial","*",1,1],"Google-Extended":["partial","*",1,1],"Applebot-Extended":["partial","*",1,1],"CCBot":["partial","*",1,1],"Bytespider":["partial","*",1,1],"Amazonbot":["partial","*",1,1],"meta-externalagent":["partial","*",1,1],"cohere-ai":["partial","*",1,1],"Diffbot":["partial","*",1,1],"ContextFetchBot":["partial","*",1,1]},"llms":{"status":"missing","http":404,"bytes":62549},"home":{"status":"ok","http":200,"url":"https://www.cambridgeenglish.org/","html_tokens":20388,"markdown_tokens":309,"pruned_tokens":42}},
{"rank":3911,"domain":"snu.ac.kr","origin":"https://snu.ac.kr","robots":{"status":"ok","http":200,"url":"https://snu.ac.kr/robots.txt","bytes":1791,"groups":40},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["allowed","*",0,1],"OAI-SearchBot":["allowed","*",0,1],"ClaudeBot":["blocked","named",2,0],"Claude-User":["allowed","*",0,1],"Claude-SearchBot":["allowed","*",0,1],"anthropic-ai":["allowed","*",0,1],"PerplexityBot":["allowed","*",0,1],"Perplexity-User":["allowed","*",0,1],"Google-Extended":["allowed","*",0,1],"Applebot-Extended":["allowed","*",0,1],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",3,0],"meta-externalagent":["allowed","*",0,1],"cohere-ai":["allowed","*",0,1],"Diffbot":["allowed","*",0,1],"ContextFetchBot":["allowed","*",0,1]},"llms":{"status":"missing","http":404,"bytes":2046},"home":{"status":"ok","http":200,"url":"https://snu.ac.kr/index.html","html_tokens":32301,"markdown_tokens":958,"pruned_tokens":32}},
{"rank":3912,"domain":"edutopia.org","origin":"https://edutopia.org","robots":{"status":"ok","http":200,"url":"https://www.edutopia.org/robots.txt","bytes":163,"groups":1},"bots":{"GPTBot":["partial","*",5,0],"ChatGPT-User":["partial","*",5,0],"OAI-SearchBot":["partial","*",5,0],"ClaudeBot":["partial","*",5,0],"Claude-User":["partial","*",5,0],"Claude-SearchBot":["partial","*",5,0],"anthropic-ai":["partial","*",5,0],"PerplexityBot":["partial","*",5,0],"Perplexity-User":["partial","*",5,0],"Google-Extended":["partial","*",5,0],"Applebot-Extended":["partial","*",5,0],"CCBot":["partial","*",5,0],"Bytespider":["partial","*",5,0],"Amazonbot":["partial","*",5,0],"meta-externalagent":["partial","*",5,0],"cohere-ai":["partial","*",5,0],"Diffbot":["partial","*",5,0],"ContextFetchBot":["partial","*",5,0]},"llms":{"status":"missing","http":404,"bytes":73521},"home":{"status":"ok","http":200,"url":"https://www.edutopia.org/","html_tokens":86313,"markdown_tokens":720,"pruned_tokens":168}},
{"rank":3913,"domain":"google.com.my","origin":"https://google.com.my","robots":{"status":"ok","http":200,"url":"https://www.google.com.my/robots.txt","bytes":6624,"groups":4},"bots":{"GPTBot":["partial","*",169,64],"ChatGPT-User":["partial","*",169,64],"OAI-SearchBot":["partial","*",169,64],"ClaudeBot":["partial","*",169,64],"Claude-User":["partial","*",169,64],"Claude-SearchBot":["partial","*",169,64],"anthropic-ai":["partial","*",169,64],"PerplexityBot":["partial","*",169,64],"Perplexity-User":["partial","*",169,64],"Google-Extended":["partial","*",169,64],"Applebot-Extended":["partial","*",169,64],"CCBot":["partial","*",169,64],"Bytespider":["partial","*",169,64],"Amazonbot":["partial","*",169,64],"meta-externalagent":["partial","*",169,64],"cohere-ai":["partial","*",169,64],"Diffbot":["partial","*",169,64],"ContextFetchBot":["partial","*",169,64]},"llms":{"status":"missing","http":404,"bytes":1569},"home":{"status":"ok","http":200,"url":"https://www.google.com/","html_tokens":21355,"markdown_tokens":30,"pruned_tokens":30}},
{"rank":3914,"domain":"torontosun.com","origin":"https://torontosun.com","robots":{"status":"ok","http":200,"url":"https://torontosun.com/robots.txt","bytes":998,"groups":15},"bots":{"GPTBot":["partial","*",4,5],"ChatGPT-User":["partial","*",4,5],"OAI-SearchBot":["partial","*",4,5],"ClaudeBot":["partial","*",4,5],"Claude-User":["partial","*",4,5],"Claude-SearchBot":["partial","*",4,5],"anthropic-ai":["partial","*",4,5],"PerplexityBot":["partial","*",4,5],"Perplexity-User":["partial","*",4,5],"Google-Extended":["partial","*",4,5],"Applebot-Extended":["partial","*",4,5],"CCBot":["partial","*",4,5],"Bytespider":["partial","*",4,5],"Amazonbot":["partial","*",4,5],"meta-externalagent":["partial","*",4,5],"cohere-ai":["partial","*",4,5],"Diffbot":["partial","*",4,5],"ContextFetchBot":["partial","*",4,5]},"llms":{"status":"missing","http":404,"bytes":268761},"home":{"status":"ok","http":200,"url":"https://torontosun.com/","html_tokens":135858,"markdown_tokens":158,"pruned_tokens":158}},
{"rank":3915,"domain":"allocine.fr","origin":"https://allocine.fr","robots":{"status":"ok","http":200,"url":"https://www.allocine.fr/robots.txt","bytes":1654,"groups":15},"bots":{"GPTBot":["partial","*",19,0],"ChatGPT-User":["partial","*",19,0],"OAI-SearchBot":["partial","*",19,0],"ClaudeBot":["partial","*",19,0],"Claude-User":["partial","*",19,0],"Claude-SearchBot":["partial","*",19,0],"anthropic-ai":["partial","*",19,0],"PerplexityBot":["partial","*",19,0],"Perplexity-User":["partial","*",19,0],"Google-Extended":["partial","*",19,0],"Applebot-Extended":["partial","*",19,0],"CCBot":["partial","*",19,0],"Bytespider":["partial","*",19,0],"Amazonbot":["partial","*",19,0],"meta-externalagent":["partial","*",19,0],"cohere-ai":["partial","*",19,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",19,0]},"llms":{"status":"missing","http":404,"bytes":105262},"home":{"status":"ok","http":200,"url":"https://www.allocine.fr/","html_tokens":98899,"markdown_tokens":189,"pruned_tokens":30}},
{"rank":3916,"domain":"west.cn","origin":"https://west.cn","robots":{"status":"ok","http":200,"url":"https://www.west.cn/robots.txt","bytes":97,"groups":1},"bots":{"GPTBot":["partial","*",3,0],"ChatGPT-User":["partial","*",3,0],"OAI-SearchBot":["partial","*",3,0],"ClaudeBot":["partial","*",3,0],"Claude-User":["partial","*",3,0],"Claude-SearchBot":["partial","*",3,0],"anthropic-ai":["partial","*",3,0],"PerplexityBot":["partial","*",3,0],"Perplexity-User":["partial","*",3,0],"Google-Extended":["partial","*",3,0],"Applebot-Extended":["partial","*",3,0],"CCBot":["partial","*",3,0],"Bytespider":["partial","*",3,0],"Amazonbot":["partial","*",3,0],"meta-externalagent":["partial","*",3,0],"cohere-ai":["partial","*",3,0],"Diffbot":["partial","*",3,0],"ContextFetchBot":["partial","*",3,0]},"llms":{"status":"missing","http":404,"bytes":124712},"home":{"status":"ok","http":200,"url":"https://www.west.cn/","html_tokens":61607,"markdown_tokens":807,"pruned_tokens":169}},
{"rank":3917,"domain":"dismall.com","origin":"https://www.dismall.com","robots":{"status":"ok","http":200,"url":"https://www.dismall.com/robots.txt","bytes":623,"groups":1},"bots":{"GPTBot":["partial","*",20,4],"ChatGPT-User":["partial","*",20,4],"OAI-SearchBot":["partial","*",20,4],"ClaudeBot":["partial","*",20,4],"Claude-User":["partial","*",20,4],"Claude-SearchBot":["partial","*",20,4],"anthropic-ai":["partial","*",20,4],"PerplexityBot":["partial","*",20,4],"Perplexity-User":["partial","*",20,4],"Google-Extended":["partial","*",20,4],"Applebot-Extended":["partial","*",20,4],"CCBot":["partial","*",20,4],"Bytespider":["partial","*",20,4],"Amazonbot":["partial","*",20,4],"meta-externalagent":["partial","*",20,4],"cohere-ai":["partial","*",20,4],"Diffbot":["partial","*",20,4],"ContextFetchBot":["partial","*",20,4]},"llms":{"status":"missing","http":404,"bytes":146},"home":{"status":"http_error","http":403,"url":"https://www.dismall.com/"}},
{"rank":3918,"domain":"splcenter.org","origin":"https://splcenter.org","robots":{"status":"ok","http":200,"url":"https://www.splcenter.org/robots.txt","bytes":191,"groups":1},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"http_error","http":403,"bytes":5566},"home":{"status":"http_error","http":403,"url":"https://splcenter.org/"}},
{"rank":3919,"domain":"influencermarketinghub.com","origin":"https://influencermarketinghub.com","robots":{"status":"ok","http":200,"url":"https://influencermarketinghub.com/robots.txt","bytes":562,"groups":6},"bots":{"GPTBot":["partial","*",8,1],"ChatGPT-User":["partial","*",8,1],"OAI-SearchBot":["partial","*",8,1],"ClaudeBot":["partial","*",8,1],"Claude-User":["partial","*",8,1],"Claude-SearchBot":["partial","*",8,1],"anthropic-ai":["partial","*",8,1],"PerplexityBot":["partial","*",8,1],"Perplexity-User":["partial","*",8,1],"Google-Extended":["partial","*",8,1],"Applebot-Extended":["partial","*",8,1],"CCBot":["partial","*",8,1],"Bytespider":["partial","*",8,1],"Amazonbot":["partial","*",8,1],"meta-externalagent":["partial","*",8,1],"cohere-ai":["partial","*",8,1],"Diffbot":["partial","*",8,1],"ContextFetchBot":["partial","*",8,1]},"llms":{"status":"other_text","http":200,"bytes":14706},"home":{"status":"ok","http":200,"url":"https://influencermarketinghub.com/","html_tokens":46715,"markdown_tokens":106,"pruned_tokens":106}},
{"rank":3920,"domain":"gambling.com","origin":"https://gambling.com","robots":{"status":"ok","http":200,"url":"https://www.gambling.com/robots.txt","bytes":38663,"groups":33},"bots":{"GPTBot":["partial","*",911,1],"ChatGPT-User":["partial","*",911,1],"OAI-SearchBot":["partial","*",911,1],"ClaudeBot":["partial","*",911,1],"Claude-User":["partial","*",911,1],"Claude-SearchBot":["partial","*",911,1],"anthropic-ai":["partial","*",911,1],"PerplexityBot":["partial","*",911,1],"Perplexity-User":["partial","*",911,1],"Google-Extended":["partial","*",911,1],"Applebot-Extended":["partial","*",911,1],"CCBot":["partial","*",911,1],"Bytespider":["partial","*",911,1],"Amazonbot":["partial","*",911,1],"meta-externalagent":["partial","*",911,1],"cohere-ai":["partial","*",911,1],"Diffbot":["partial","*",911,1],"ContextFetchBot":["partial","*",911,1]},"llms":{"status":"present","http":200,"bytes":8060},"home":{"status":"ok","http":200,"url":"https://www.gambling.com/","html_tokens":450904,"markdown_tokens":4702,"pruned_tokens":169}},
{"rank":3921,"domain":"lightwidget.com","origin":"https://lightwidget.com","robots":{"status":"ok","http":200,"url":"https://lightwidget.com/robots.txt","bytes":137,"groups":1},"bots":{"GPTBot":["partial","*",2,1],"ChatGPT-User":["partial","*",2,1],"OAI-SearchBot":["partial","*",2,1],"ClaudeBot":["partial","*",2,1],"Claude-User":["partial","*",2,1],"Claude-SearchBot":["partial","*",2,1],"anthropic-ai":["partial","*",2,1],"PerplexityBot":["partial","*",2,1],"Perplexity-User":["partial","*",2,1],"Google-Extended":["partial","*",2,1],"Applebot-Extended":["partial","*",2,1],"CCBot":["partial","*",2,1],"Bytespider":["partial","*",2,1],"Amazonbot":["partial","*",2,1],"meta-externalagent":["partial","*",2,1],"cohere-ai":["partial","*",2,1],"Diffbot":["partial","*",2,1],"ContextFetchBot":["partial","*",2,1]},"llms":{"status":"missing","http":404,"bytes":9893},"home":{"status":"ok","http":200,"url":"https://lightwidget.com/","html_tokens":11151,"markdown_tokens":692,"pruned_tokens":120}},
{"rank":3922,"domain":"proxmox.com","origin":"https://proxmox.com","robots":{"status":"ok","http":200,"url":"https://proxmox.com/robots.txt","bytes":1021,"groups":1},"bots":{"GPTBot":["partial","*",15,0],"ChatGPT-User":["partial","*",15,0],"OAI-SearchBot":["partial","*",15,0],"ClaudeBot":["partial","*",15,0],"Claude-User":["partial","*",15,0],"Claude-SearchBot":["partial","*",15,0],"anthropic-ai":["partial","*",15,0],"PerplexityBot":["partial","*",15,0],"Perplexity-User":["partial","*",15,0],"Google-Extended":["partial","*",15,0],"Applebot-Extended":["partial","*",15,0],"CCBot":["partial","*",15,0],"Bytespider":["partial","*",15,0],"Amazonbot":["partial","*",15,0],"meta-externalagent":["partial","*",15,0],"cohere-ai":["partial","*",15,0],"Diffbot":["partial","*",15,0],"ContextFetchBot":["partial","*",15,0]},"llms":{"status":"missing","http":404,"bytes":146},"home":{"status":"ok","http":200,"url":"https://proxmox.com/en/","html_tokens":17258,"markdown_tokens":283,"pruned_tokens":205}},
{"rank":3923,"domain":"wholefoodsmarket.com","origin":"https://wholefoodsmarket.com","robots":{"status":"ok","http":200,"url":"https://www.wholefoodsmarket.com/robots.txt","bytes":247,"groups":1},"bots":{"GPTBot":["partial","*",9,0],"ChatGPT-User":["partial","*",9,0],"OAI-SearchBot":["partial","*",9,0],"ClaudeBot":["partial","*",9,0],"Claude-User":["partial","*",9,0],"Claude-SearchBot":["partial","*",9,0],"anthropic-ai":["partial","*",9,0],"PerplexityBot":["partial","*",9,0],"Perplexity-User":["partial","*",9,0],"Google-Extended":["partial","*",9,0],"Applebot-Extended":["partial","*",9,0],"CCBot":["partial","*",9,0],"Bytespider":["partial","*",9,0],"Amazonbot":["partial","*",9,0],"meta-externalagent":["partial","*",9,0],"cohere-ai":["partial","*",9,0],"Diffbot":["partial","*",9,0],"ContextFetchBot":["partial","*",9,0]},"llms":{"status":"missing","http":404,"bytes":75208},"home":{"status":"ok","http":200,"url":"https://www.wholefoodsmarket.com/","html_tokens":41057,"markdown_tokens":243,"pruned_tokens":33}},
{"rank":3924,"domain":"oilprice.com","origin":"https://oilprice.com","robots":{"status":"ok","http":200,"url":"https://oilprice.com/robots.txt","bytes":7574,"groups":2},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["partial","*",116,0],"OAI-SearchBot":["partial","*",116,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",116,0],"Claude-SearchBot":["partial","*",116,0],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["partial","*",116,0],"Perplexity-User":["partial","*",116,0],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",116,0]},"llms":{"status":"present","http":200,"bytes":4462},"home":{"status":"ok","http":200,"url":"https://oilprice.com/","html_tokens":65620,"markdown_tokens":313,"pruned_tokens":154}},
{"rank":3925,"domain":"de.tl","origin":"https://de.tl","robots":{"status":"ok","http":200,"url":"https://www.homepage-baukasten.de/robots.txt","bytes":79,"groups":1},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"missing","http":404,"bytes":206},"home":{"status":"ok","http":200,"url":"https://www.homepage-baukasten.de/","html_tokens":8166,"markdown_tokens":325,"pruned_tokens":149}},
{"rank":3926,"domain":"blogspot.com.br","origin":"https://blogspot.com.br","robots":{"status":"missing","http":404,"url":"https://blogspot.com.br/robots.txt","bytes":1571,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":1569},"home":{"status":"http_error","http":404,"url":"https://blogspot.com.br/"}},
{"rank":3927,"domain":"px.a8.net","origin":"https://px.a8.net","robots":{"status":"missing","http":404,"url":"https://px.a8.net/robots.txt","bytes":179,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":175},"home":{"status":"error","http":200,"url":"https://px.a8.net/","error":"unparseable HTML"}},
{"rank":3928,"domain":"moj.go.jp","origin":"https://www.moj.go.jp","robots":{"status":"ok","http":200,"url":"https://www.moj.go.jp/robots.txt","bytes":129,"groups":1},"bots":{"GPTBot":["partial","*",1,0],"ChatGPT-User":["partial","*",1,0],"OAI-SearchBot":["partial","*",1,0],"ClaudeBot":["partial","*",1,0],"Claude-User":["partial","*",1,0],"Claude-SearchBot":["partial","*",1,0],"anthropic-ai":["partial","*",1,0],"PerplexityBot":["partial","*",1,0],"Perplexity-User":["partial","*",1,0],"Google-Extended":["partial","*",1,0],"Applebot-Extended":["partial","*",1,0],"CCBot":["partial","*",1,0],"Bytespider":["partial","*",1,0],"Amazonbot":["partial","*",1,0],"meta-externalagent":["partial","*",1,0],"cohere-ai":["partial","*",1,0],"Diffbot":["partial","*",1,0],"ContextFetchBot":["partial","*",1,0]},"llms":{"status":"missing","http":404,"bytes":236},"home":{"status":"ok","http":200,"url":"https://www.moj.go.jp/","html_tokens":10631,"markdown_tokens":126,"pruned_tokens":126}},
{"rank":3929,"domain":"zenwriting.net","origin":"https://zenwriting.net","robots":{"status":"missing","http":404,"url":"https://zenwriting.net/robots.txt","bytes":19,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"other_text","http":200,"bytes":3986},"home":{"status":"ok","http":200,"url":"https://zenwriting.net/","html_tokens":1944,"markdown_tokens":35,"pruned_tokens":35}},
{"rank":3930,"domain":"go.tz","origin":"https://go.tz","robots":{"status":"unreachable","http":null,"url":"https://go.tz/robots.txt","bytes":0,"groups":0,"error":"cannot resolve go.tz"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":3931,"domain":"uab.cat","origin":"https://uab.cat","robots":{"status":"ok","http":200,"url":"https://www.uab.cat/robots.txt","bytes":1166,"groups":2},"bots":{"GPTBot":["partial","*",16,0],"ChatGPT-User":["partial","*",16,0],"OAI-SearchBot":["partial","*",16,0],"ClaudeBot":["partial","*",16,0],"Claude-User":["partial","*",16,0],"Claude-SearchBot":["partial","*",16,0],"anthropic-ai":["partial","*",16,0],"PerplexityBot":["partial","*",16,0],"Perplexity-User":["partial","*",16,0],"Google-Extended":["partial","*",16,0],"Applebot-Extended":["partial","*",16,0],"CCBot":["partial","*",16,0],"Bytespider":["partial","*",16,0],"Amazonbot":["partial","*",16,0],"meta-externalagent":["partial","*",16,0],"cohere-ai":["partial","*",16,0],"Diffbot":["partial","*",16,0],"ContextFetchBot":["partial","*",16,0]},"llms":{"status":"missing","http":404,"bytes":16797},"home":{"status":"ok","http":200,"url":"https://www.uab.cat/","html_tokens":22148,"markdown_tokens":63,"pruned_tokens":63}},
{"rank":3932,"domain":"factcheck.org","origin":"https://factcheck.org","robots":{"status":"ok","http":200,"url":"https://www.factcheck.org/robots.txt","bytes":239,"groups":2},"bots":{"GPTBot":["partial","*",2,0],"ChatGPT-User":["partial","*",2,0],"OAI-SearchBot":["partial","*",2,0],"ClaudeBot":["partial","*",2,0],"Claude-User":["partial","*",2,0],"Claude-SearchBot":["partial","*",2,0],"anthropic-ai":["partial","*",2,0],"PerplexityBot":["partial","*",2,0],"Perplexity-User":["partial","*",2,0],"Google-Extended":["partial","*",2,0],"Applebot-Extended":["partial","*",2,0],"CCBot":["partial","*",2,0],"Bytespider":["partial","*",2,0],"Amazonbot":["partial","*",2,0],"meta-externalagent":["partial","*",2,0],"cohere-ai":["partial","*",2,0],"Diffbot":["partial","*",2,0],"ContextFetchBot":["partial","*",2,0]},"llms":{"status":"missing","http":404,"bytes":84102},"home":{"status":"ok","http":200,"url":"https://www.factcheck.org/","html_tokens":19261,"markdown_tokens":566,"pruned_tokens":102}},
{"rank":3933,"domain":"material.io","origin":"https://material.io","robots":{"status":"ok","http":200,"url":"https://material.io/robots.txt","bytes":55,"groups":1},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"missing","http":404,"bytes":62386},"home":{"status":"ok","http":200,"url":"https://m3.material.io/","html_tokens":15597,"markdown_tokens":9,"pruned_tokens":9}},
{"rank":3934,"domain":"umu.se","origin":"https://umu.se","robots":{"status":"ok","http":200,"url":"https://www.umu.se/robots.txt","bytes":71,"groups":1},"bots":{"GPTBot":["allowed","*",0,1],"ChatGPT-User":["allowed","*",0,1],"OAI-SearchBot":["allowed","*",0,1],"ClaudeBot":["allowed","*",0,1],"Claude-User":["allowed","*",0,1],"Claude-SearchBot":["allowed","*",0,1],"anthropic-ai":["allowed","*",0,1],"PerplexityBot":["allowed","*",0,1],"Perplexity-User":["allowed","*",0,1],"Google-Extended":["allowed","*",0,1],"Applebot-Extended":["allowed","*",0,1],"CCBot":["allowed","*",0,1],"Bytespider":["allowed","*",0,1],"Amazonbot":["allowed","*",0,1],"meta-externalagent":["allowed","*",0,1],"cohere-ai":["allowed","*",0,1],"Diffbot":["allowed","*",0,1],"ContextFetchBot":["allowed","*",0,1]},"llms":{"status":"missing","http":404,"bytes":32595},"home":{"status":"ok","http":200,"url":"https://www.umu.se/","html_tokens":30042,"markdown_tokens":129,"pruned_tokens":101}},
{"rank":3935,"domain":"dr.dk","origin":"https://dr.dk","robots":{"status":"ok","http":200,"url":"https://www.dr.dk/robots.txt","bytes":2158,"groups":37},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["blocked","named",1,0],"OAI-SearchBot":["blocked","named",1,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["blocked","named",1,0],"Claude-SearchBot":["blocked","named",1,0],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["blocked","named",1,0],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["partial","*",29,0],"ContextFetchBot":["partial","*",29,0]},"llms":{"status":"missing","http":404,"bytes":1635},"home":{"status":"ok","http":200,"url":"https://www.dr.dk/","html_tokens":274099,"markdown_tokens":84,"pruned_tokens":84}},
{"rank":3936,"domain":"motor1.com","origin":"https://motor1.com","robots":{"status":"ok","http":200,"url":"https://www.motor1.com/robots.txt","bytes":637,"groups":7},"bots":{"GPTBot":["partial","*",7,0],"ChatGPT-User":["partial","*",7,0],"OAI-SearchBot":["partial","*",7,0],"ClaudeBot":["partial","*",7,0],"Claude-User":["partial","*",7,0],"Claude-SearchBot":["partial","*",7,0],"anthropic-ai":["partial","*",7,0],"PerplexityBot":["partial","*",7,0],"Perplexity-User":["partial","*",7,0],"Google-Extended":["partial","*",7,0],"Applebot-Extended":["partial","*",7,0],"CCBot":["partial","*",7,0],"Bytespider":["partial","*",7,0],"Amazonbot":["partial","*",7,0],"meta-externalagent":["partial","*",7,0],"cohere-ai":["partial","*",7,0],"Diffbot":["partial","*",7,0],"ContextFetchBot":["partial","*",7,0]},"llms":{"status":"missing","http":404,"bytes":169},"home":{"status":"http_error","http":403,"url":"https://motor1.com/"}},
{"rank":3937,"domain":"hackerone.com","origin":"https://hackerone.com","robots":{"status":"ok","http":200,"url":"https://hackerone.com/robots.txt","bytes":43,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":1694},"home":{"status":"ok","http":200,"url":"https://www.hackerone.com/","html_tokens":100003,"markdown_tokens":327,"pruned_tokens":65}},
{"rank":3938,"domain":"eltiempo.com","origin":"https://eltiempo.com","robots":{"status":"ok","http":200,"url":"https://www.eltiempo.com/robots.txt","bytes":5092,"groups":61},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["blocked","named",1,0],"OAI-SearchBot":["blocked","named",1,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",22,0],"Claude-SearchBot":["partial","*",22,0],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["partial","*",22,0],"Google-Extended":["allowed","named",0,1],"Applebot-Extended":["partial","*",22,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["partial","*",22,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["partial","*",22,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",22,0]},"llms":{"status":"missing","http":404,"bytes":154686},"home":{"status":"ok","http":200,"url":"https://www.eltiempo.com/","html_tokens":225838,"markdown_tokens":69,"pruned_tokens":69}},
{"rank":3939,"domain":"openrouter.ai","origin":"https://openrouter.ai","robots":{"status":"ok","http":200,"url":"https://openrouter.ai/robots.txt","bytes":137,"groups":1},"bots":{"GPTBot":["partial","*",1,1],"ChatGPT-User":["partial","*",1,1],"OAI-SearchBot":["partial","*",1,1],"ClaudeBot":["partial","*",1,1],"Claude-User":["partial","*",1,1],"Claude-SearchBot":["partial","*",1,1],"anthropic-ai":["partial","*",1,1],"PerplexityBot":["partial","*",1,1],"Perplexity-User":["partial","*",1,1],"Google-Extended":["partial","*",1,1],"Applebot-Extended":["partial","*",1,1],"CCBot":["partial","*",1,1],"Bytespider":["partial","*",1,1],"Amazonbot":["partial","*",1,1],"meta-externalagent":["partial","*",1,1],"cohere-ai":["partial","*",1,1],"Diffbot":["partial","*",1,1],"ContextFetchBot":["partial","*",1,1]},"llms":{"status":"present","http":200,"bytes":8786},"home":{"status":"not_html","error":"unsupported content type text/markdown"}},
{"rank":3940,"domain":"drugabuse.gov","origin":"https://drugabuse.gov","robots":{"status":"ok","http":200,"url":"https://nida.nih.gov/robots.txt","bytes":2422,"groups":1},"bots":{"GPTBot":["partial","*",48,18],"ChatGPT-User":["partial","*",48,18],"OAI-SearchBot":["partial","*",48,18],"ClaudeBot":["partial","*",48,18],"Claude-User":["partial","*",48,18],"Claude-SearchBot":["partial","*",48,18],"anthropic-ai":["partial","*",48,18],"PerplexityBot":["partial","*",48,18],"Perplexity-User":["partial","*",48,18],"Google-Extended":["partial","*",48,18],"Applebot-Extended":["partial","*",48,18],"CCBot":["partial","*",48,18],"Bytespider":["partial","*",48,18],"Amazonbot":["partial","*",48,18],"meta-externalagent":["partial","*",48,18],"cohere-ai":["partial","*",48,18],"Diffbot":["partial","*",48,18],"ContextFetchBot":["partial","*",48,18]},"llms":{"status":"missing","http":404,"bytes":186},"home":{"status":"ok","http":200,"url":"https://nida.nih.gov/","html_tokens":18240,"markdown_tokens":191,"pruned_tokens":119}},
{"rank":3941,"domain":"humaneworld.org","origin":"https://humaneworld.org","robots":{"status":"refused","http":403,"url":"https://humaneworld.org/robots.txt","bytes":560,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":3942,"domain":"epo.org","origin":"https://epo.org","robots":{"status":"ok","http":200,"url":"https://www.epo.org/robots.txt","bytes":2417,"groups":3},"bots":{"GPTBot":["partial","*",43,18],"ChatGPT-User":["partial","*",43,18],"OAI-SearchBot":["partial","*",43,18],"ClaudeBot":["partial","*",43,18],"Claude-User":["partial","*",43,18],"Claude-SearchBot":["partial","*",43,18],"anthropic-ai":["partial","*",43,18],"PerplexityBot":["partial","*",43,18],"Perplexity-User":["partial","*",43,18],"Google-Extended":["partial","*",43,18],"Applebot-Extended":["partial","*",43,18],"CCBot":["partial","*",43,18],"Bytespider":["partial","*",43,18],"Amazonbot":["partial","*",43,18],"meta-externalagent":["partial","*",43,18],"cohere-ai":["partial","*",43,18],"Diffbot":["partial","*",43,18],"ContextFetchBot":["partial","*",43,18]},"llms":{"status":"http_error","http":403,"bytes":146},"home":{"status":"ok","http":200,"url":"https://www.epo.org/en","html_tokens":173680,"markdown_tokens":153,"pruned_tokens":153}},
{"rank":3943,"domain":"datadoghq.com","origin":"https://datadoghq.com","robots":{"status":"ok","http":200,"url":"https://www.datadoghq.com/robots.txt","bytes":525,"groups":1},"bots":{"GPTBot":["partial","*",3,0],"ChatGPT-User":["partial","*",3,0],"OAI-SearchBot":["partial","*",3,0],"ClaudeBot":["partial","*",3,0],"Claude-User":["partial","*",3,0],"Claude-SearchBot":["partial","*",3,0],"anthropic-ai":["partial","*",3,0],"PerplexityBot":["partial","*",3,0],"Perplexity-User":["partial","*",3,0],"Google-Extended":["partial","*",3,0],"Applebot-Extended":["partial","*",3,0],"CCBot":["partial","*",3,0],"Bytespider":["partial","*",3,0],"Amazonbot":["partial","*",3,0],"meta-externalagent":["partial","*",3,0],"cohere-ai":["partial","*",3,0],"Diffbot":["partial","*",3,0],"ContextFetchBot":["partial","*",3,0]},"llms":{"status":"present","http":200,"bytes":591292},"home":{"status":"ok","http":200,"url":"https://www.datadoghq.com/","html_tokens":71497,"markdown_tokens":224,"pruned_tokens":55}},
{"rank":3944,"domain":"dailysabah.com","origin":"https://dailysabah.com","robots":{"status":"html","http":200,"url":"https://dailysabah.com/robots.txt","bytes":269,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":3945,"domain":"theiet.org","origin":"https://theiet.org","robots":{"status":"ok","http":200,"url":"https://www.theiet.org/robots.txt","bytes":183,"groups":1},"bots":{"GPTBot":["partial","*",2,0],"ChatGPT-User":["partial","*",2,0],"OAI-SearchBot":["partial","*",2,0],"ClaudeBot":["partial","*",2,0],"Claude-User":["partial","*",2,0],"Claude-SearchBot":["partial","*",2,0],"anthropic-ai":["partial","*",2,0],"PerplexityBot":["partial","*",2,0],"Perplexity-User":["partial","*",2,0],"Google-Extended":["partial","*",2,0],"Applebot-Extended":["partial","*",2,0],"CCBot":["partial","*",2,0],"Bytespider":["partial","*",2,0],"Amazonbot":["partial","*",2,0],"meta-externalagent":["partial","*",2,0],"cohere-ai":["partial","*",2,0],"Diffbot":["partial","*",2,0],"ContextFetchBot":["partial","*",2,0]},"llms":{"status":"missing","http":404,"bytes":0},"home":{"status":"ok","http":200,"url":"https://www.theiet.org/","html_tokens":65943,"markdown_tokens":185,"pruned_tokens":31}},
{"rank":3946,"domain":"netvibes.com","origin":"https://netvibes.com","robots":{"status":"timeout","http":null,"url":"https://netvibes.com/robots.txt","bytes":0,"groups":0,"error":"ConnectTimeout"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":3947,"domain":"gub.uy","origin":"https://www.gub.uy","robots":{"status":"ok","http":200,"url":"https://www.gub.uy/robots.txt","bytes":1698,"groups":1},"bots":{"GPTBot":["partial","*",25,18],"ChatGPT-User":["partial","*",25,18],"OAI-SearchBot":["partial","*",25,18],"ClaudeBot":["partial","*",25,18],"Claude-User":["partial","*",25,18],"Claude-SearchBot":["partial","*",25,18],"anthropic-ai":["partial","*",25,18],"PerplexityBot":["partial","*",25,18],"Perplexity-User":["partial","*",25,18],"Google-Extended":["partial","*",25,18],"Applebot-Extended":["partial","*",25,18],"CCBot":["partial","*",25,18],"Bytespider":["partial","*",25,18],"Amazonbot":["partial","*",25,18],"meta-externalagent":["partial","*",25,18],"cohere-ai":["partial","*",25,18],"Diffbot":["partial","*",25,18],"ContextFetchBot":["partial","*",25,18]},"llms":{"status":"missing","http":404,"bytes":184},"home":{"status":"ok","http":200,"url":"https://www.gub.uy/","html_tokens":16125,"markdown_tokens":18,"pruned_tokens":18}},
{"rank":3948,"domain":"de.vu","origin":"http://de.vu","robots":{"status":"missing","http":404,"url":"http://de.vu/robots.txt","bytes":236,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":236},"home":{"status":"ok","http":200,"url":"http://de.vu/","html_tokens":49,"markdown_tokens":15,"pruned_tokens":15}},
{"rank":3949,"domain":"foxchasefarm.org","origin":"https://foxchasefarm.org","robots":{"status":"ok","http":200,"url":"https://www.foxchasefarm.org/robots.txt","bytes":145,"groups":1},"bots":{"GPTBot":["partial","*",2,1],"ChatGPT-User":["partial","*",2,1],"OAI-SearchBot":["partial","*",2,1],"ClaudeBot":["partial","*",2,1],"Claude-User":["partial","*",2,1],"Claude-SearchBot":["partial","*",2,1],"anthropic-ai":["partial","*",2,1],"PerplexityBot":["partial","*",2,1],"Perplexity-User":["partial","*",2,1],"Google-Extended":["partial","*",2,1],"Applebot-Extended":["partial","*",2,1],"CCBot":["partial","*",2,1],"Bytespider":["partial","*",2,1],"Amazonbot":["partial","*",2,1],"meta-externalagent":["partial","*",2,1],"cohere-ai":["partial","*",2,1],"Diffbot":["partial","*",2,1],"ContextFetchBot":["partial","*",2,1]},"llms":{"status":"html","http":200,"bytes":39635},"home":{"status":"ok","http":200,"url":"https://www.foxchasefarm.org/","html_tokens":7807,"markdown_tokens":2548,"pruned_tokens":145}},
{"rank":3950,"domain":"tass.com","origin":"https://tass.com","robots":{"status":"ok","http":200,"url":"https://tass.com/robots.txt","bytes":94,"groups":1},"bots":{"GPTBot":["partial","*",1,0],"ChatGPT-User":["partial","*",1,0],"OAI-SearchBot":["partial","*",1,0],"ClaudeBot":["partial","*",1,0],"Claude-User":["partial","*",1,0],"Claude-SearchBot":["partial","*",1,0],"anthropic-ai":["partial","*",1,0],"PerplexityBot":["partial","*",1,0],"Perplexity-User":["partial","*",1,0],"Google-Extended":["partial","*",1,0],"Applebot-Extended":["partial","*",1,0],"CCBot":["partial","*",1,0],"Bytespider":["partial","*",1,0],"Amazonbot":["partial","*",1,0],"meta-externalagent":["partial","*",1,0],"cohere-ai":["partial","*",1,0],"Diffbot":["partial","*",1,0],"ContextFetchBot":["partial","*",1,0]},"llms":{"status":"missing","http":404,"bytes":17749},"home":{"status":"ok","http":200,"url":"https://tass.com/","html_tokens":20223,"markdown_tokens":8,"pruned_tokens":8}},
{"rank":3951,"domain":"qut.edu.au","origin":"https://qut.edu.au","robots":{"status":"refused","http":403,"url":"https://www.qut.edu.au/robots.txt","bytes":5552,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":3952,"domain":"vocabulary.com","origin":"https://vocabulary.com","robots":{"status":"ok","http":200,"url":"https://www.vocabulary.com/robots.txt","bytes":321,"groups":4},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["partial","*",7,0],"OAI-SearchBot":["partial","*",7,0],"ClaudeBot":["partial","*",7,0],"Claude-User":["partial","*",7,0],"Claude-SearchBot":["partial","*",7,0],"anthropic-ai":["partial","*",7,0],"PerplexityBot":["partial","*",7,0],"Perplexity-User":["partial","*",7,0],"Google-Extended":["partial","*",7,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["partial","*",7,0],"Amazonbot":["partial","*",7,0],"meta-externalagent":["partial","*",7,0],"cohere-ai":["partial","*",7,0],"Diffbot":["partial","*",7,0],"ContextFetchBot":["partial","*",7,0]},"llms":{"status":"missing","http":404,"bytes":35033},"home":{"status":"ok","http":200,"url":"https://www.vocabulary.com/","html_tokens":12709,"markdown_tokens":130,"pruned_tokens":89}},
{"rank":3953,"domain":"concordia.ca","origin":"https://concordia.ca","robots":{"status":"ok","http":200,"url":"https://www.concordia.ca/robots.txt","bytes":1023,"groups":2},"bots":{"GPTBot":["partial","*",16,0],"ChatGPT-User":["partial","*",16,0],"OAI-SearchBot":["partial","*",16,0],"ClaudeBot":["partial","*",16,0],"Claude-User":["partial","*",16,0],"Claude-SearchBot":["partial","*",16,0],"anthropic-ai":["partial","*",16,0],"PerplexityBot":["partial","*",16,0],"Perplexity-User":["partial","*",16,0],"Google-Extended":["partial","*",16,0],"Applebot-Extended":["partial","*",16,0],"CCBot":["partial","*",16,0],"Bytespider":["partial","*",16,0],"Amazonbot":["partial","*",16,0],"meta-externalagent":["partial","*",16,0],"cohere-ai":["partial","*",16,0],"Diffbot":["partial","*",16,0],"ContextFetchBot":["partial","*",16,0]},"llms":{"status":"missing","http":404,"bytes":115204},"home":{"status":"ok","http":200,"url":"https://www.concordia.ca/","html_tokens":39326,"markdown_tokens":105,"pruned_tokens":105}},
{"rank":3954,"domain":"artificialintelligenceact.eu","origin":"https://artificialintelligenceact.eu","robots":{"status":"ok","http":200,"url":"https://artificialintelligenceact.eu/robots.txt","bytes":114,"groups":1},"bots":{"GPTBot":["partial","*",1,1],"ChatGPT-User":["partial","*",1,1],"OAI-SearchBot":["partial","*",1,1],"ClaudeBot":["partial","*",1,1],"Claude-User":["partial","*",1,1],"Claude-SearchBot":["partial","*",1,1],"anthropic-ai":["partial","*",1,1],"PerplexityBot":["partial","*",1,1],"Perplexity-User":["partial","*",1,1],"Google-Extended":["partial","*",1,1],"Applebot-Extended":["partial","*",1,1],"CCBot":["partial","*",1,1],"Bytespider":["partial","*",1,1],"Amazonbot":["partial","*",1,1],"meta-externalagent":["partial","*",1,1],"cohere-ai":["partial","*",1,1],"Diffbot":["partial","*",1,1],"ContextFetchBot":["partial","*",1,1]},"llms":{"status":"missing","http":404,"bytes":123336},"home":{"status":"ok","http":200,"url":"https://artificialintelligenceact.eu/","html_tokens":78594,"markdown_tokens":2292,"pruned_tokens":277}},
{"rank":3955,"domain":"capcut.com","origin":"https://capcut.com","robots":{"status":"ok","http":200,"url":"https://www.capcut.com/robots.txt","bytes":1630,"groups":2},"bots":{"GPTBot":["partial","named",18,1],"ChatGPT-User":["partial","named",18,1],"OAI-SearchBot":["partial","named",18,1],"ClaudeBot":["partial","named",18,1],"Claude-User":["partial","named",18,1],"Claude-SearchBot":["partial","named",18,1],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["partial","named",18,1],"Perplexity-User":["partial","named",18,1],"Google-Extended":["partial","named",18,1],"Applebot-Extended":["partial","named",18,1],"CCBot":["partial","named",18,1],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"present","http":200,"bytes":13897},"home":{"status":"ok","http":200,"url":"https://www.capcut.com/","html_tokens":61303,"markdown_tokens":1678,"pruned_tokens":277}},
{"rank":3956,"domain":"wpmet.com","origin":"https://wpmet.com","robots":{"status":"ok","http":200,"url":"https://wpmet.com/robots.txt","bytes":2165,"groups":2},"bots":{"GPTBot":["partial","named",4,2],"ChatGPT-User":["partial","named",4,2],"OAI-SearchBot":["partial","named",4,2],"ClaudeBot":["partial","named",4,2],"Claude-User":["partial","named",4,2],"Claude-SearchBot":["partial","named",4,2],"anthropic-ai":["partial","*",29,2],"PerplexityBot":["partial","named",4,2],"Perplexity-User":["partial","named",4,2],"Google-Extended":["partial","named",4,2],"Applebot-Extended":["partial","named",4,2],"CCBot":["partial","*",29,2],"Bytespider":["partial","named",4,2],"Amazonbot":["partial","named",4,2],"meta-externalagent":["partial","named",4,2],"cohere-ai":["partial","named",4,2],"Diffbot":["partial","*",29,2],"ContextFetchBot":["partial","*",29,2]},"llms":{"status":"html","http":200,"bytes":752420},"home":{"status":"ok","http":200,"url":"https://wpmet.com/","html_tokens":188098,"markdown_tokens":138,"pruned_tokens":138}},
{"rank":3957,"domain":"uni-kl.de","origin":"https://www.uni-kl.de","robots":{"status":"ok","http":200,"url":"https://rptu.de/robots.txt","bytes":1182,"groups":1},"bots":{"GPTBot":["partial","*",21,0],"ChatGPT-User":["partial","*",21,0],"OAI-SearchBot":["partial","*",21,0],"ClaudeBot":["partial","*",21,0],"Claude-User":["partial","*",21,0],"Claude-SearchBot":["partial","*",21,0],"anthropic-ai":["partial","*",21,0],"PerplexityBot":["partial","*",21,0],"Perplexity-User":["partial","*",21,0],"Google-Extended":["partial","*",21,0],"Applebot-Extended":["partial","*",21,0],"CCBot":["partial","*",21,0],"Bytespider":["partial","*",21,0],"Amazonbot":["partial","*",21,0],"meta-externalagent":["partial","*",21,0],"cohere-ai":["partial","*",21,0],"Diffbot":["partial","*",21,0],"ContextFetchBot":["partial","*",21,0]},"llms":{"status":"missing","http":404,"bytes":59477},"home":{"status":"ok","http":200,"url":"https://rptu.de/","html_tokens":28645,"markdown_tokens":67,"pruned_tokens":67}},
{"rank":3958,"domain":"wroclaw.pl","origin":"https://www.wroclaw.pl","robots":{"status":"ok","http":200,"url":"https://www.wroclaw.pl/robots.txt","bytes":1174,"groups":1},"bots":{"GPTBot":["partial","*",45,0],"ChatGPT-User":["partial","*",45,0],"OAI-SearchBot":["partial","*",45,0],"ClaudeBot":["partial","*",45,0],"Claude-User":["partial","*",45,0],"Claude-SearchBot":["partial","*",45,0],"anthropic-ai":["partial","*",45,0],"PerplexityBot":["partial","*",45,0],"Perplexity-User":["partial","*",45,0],"Google-Extended":["partial","*",45,0],"Applebot-Extended":["partial","*",45,0],"CCBot":["partial","*",45,0],"Bytespider":["partial","*",45,0],"Amazonbot":["partial","*",45,0],"meta-externalagent":["partial","*",45,0],"cohere-ai":["partial","*",45,0],"Diffbot":["partial","*",45,0],"ContextFetchBot":["partial","*",45,0]},"llms":{"status":"missing","http":404,"bytes":2450},"home":{"status":"ok","http":200,"url":"https://www.wroclaw.pl/","html_tokens":255189,"markdown_tokens":57,"pruned_tokens":57}},
{"rank":3959,"domain":"fpdf.org","origin":"https://fpdf.org","robots":{"status":"ok","http":200,"url":"https://fpdf.org/robots.txt","bytes":87,"groups":1},"bots":{"GPTBot":["partial","*",5,0],"ChatGPT-User":["partial","*",5,0],"OAI-SearchBot":["partial","*",5,0],"ClaudeBot":["partial","*",5,0],"Claude-User":["partial","*",5,0],"Claude-SearchBot":["partial","*",5,0],"anthropic-ai":["partial","*",5,0],"PerplexityBot":["partial","*",5,0],"Perplexity-User":["partial","*",5,0],"Google-Extended":["partial","*",5,0],"Applebot-Extended":["partial","*",5,0],"CCBot":["partial","*",5,0],"Bytespider":["partial","*",5,0],"Amazonbot":["partial","*",5,0],"meta-externalagent":["partial","*",5,0],"cohere-ai":["partial","*",5,0],"Diffbot":["partial","*",5,0],"ContextFetchBot":["partial","*",5,0]},"llms":{"status":"missing","http":404,"bytes":196},"home":{"status":"ok","http":200,"url":"https://fpdf.org/","html_tokens":169,"markdown_tokens":0,"pruned_tokens":0}},
{"rank":3960,"domain":"bmwgroup.com","origin":"https://bmwgroup.com","robots":{"status":"ok","http":200,"url":"https://www.bmwgroup.com/robots.txt","bytes":45,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":90341},"home":{"status":"ok","http":200,"url":"https://www.bmwgroup.com/en.html","html_tokens":36555,"markdown_tokens":355,"pruned_tokens":355}},
{"rank":3961,"domain":"cjr.org","origin":"https://cjr.org","robots":{"status":"ok","http":200,"url":"https://www.cjr.org/robots.txt","bytes":148,"groups":1},"bots":{"GPTBot":["partial","*",3,0],"ChatGPT-User":["partial","*",3,0],"OAI-SearchBot":["partial","*",3,0],"ClaudeBot":["partial","*",3,0],"Claude-User":["partial","*",3,0],"Claude-SearchBot":["partial","*",3,0],"anthropic-ai":["partial","*",3,0],"PerplexityBot":["partial","*",3,0],"Perplexity-User":["partial","*",3,0],"Google-Extended":["partial","*",3,0],"Applebot-Extended":["partial","*",3,0],"CCBot":["partial","*",3,0],"Bytespider":["partial","*",3,0],"Amazonbot":["partial","*",3,0],"meta-externalagent":["partial","*",3,0],"cohere-ai":["partial","*",3,0],"Diffbot":["partial","*",3,0],"ContextFetchBot":["partial","*",3,0]},"llms":{"status":"http_error","http":403,"bytes":5538},"home":{"status":"http_error","http":403,"url":"https://cjr.org/"}},
{"rank":3962,"domain":"systeme.io","origin":"https://systeme.io","robots":{"status":"ok","http":200,"url":"https://systeme.io/robots.txt","bytes":74,"groups":1},"bots":{"GPTBot":["partial","*",1,0],"ChatGPT-User":["partial","*",1,0],"OAI-SearchBot":["partial","*",1,0],"ClaudeBot":["partial","*",1,0],"Claude-User":["partial","*",1,0],"Claude-SearchBot":["partial","*",1,0],"anthropic-ai":["partial","*",1,0],"PerplexityBot":["partial","*",1,0],"Perplexity-User":["partial","*",1,0],"Google-Extended":["partial","*",1,0],"Applebot-Extended":["partial","*",1,0],"CCBot":["partial","*",1,0],"Bytespider":["partial","*",1,0],"Amazonbot":["partial","*",1,0],"meta-externalagent":["partial","*",1,0],"cohere-ai":["partial","*",1,0],"Diffbot":["partial","*",1,0],"ContextFetchBot":["partial","*",1,0]},"llms":{"status":"missing","http":404,"bytes":112238},"home":{"status":"ok","http":200,"url":"https://systeme.io/","html_tokens":69344,"markdown_tokens":206,"pruned_tokens":133}},
{"rank":3963,"domain":"hurriyet.com.tr","origin":"https://hurriyet.com.tr","robots":{"status":"ok","http":200,"url":"https://www.hurriyet.com.tr/robots.txt","bytes":736,"groups":13},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["blocked","named",1,0],"OAI-SearchBot":["blocked","named",1,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",10,0],"Claude-SearchBot":["partial","*",10,0],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["partial","*",10,0],"Google-Extended":["partial","*",10,0],"Applebot-Extended":["partial","*",10,0],"CCBot":["partial","*",10,0],"Bytespider":["partial","*",10,0],"Amazonbot":["partial","*",10,0],"meta-externalagent":["partial","*",10,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["partial","*",10,0],"ContextFetchBot":["partial","*",10,0]},"llms":{"status":"missing","http":404,"bytes":4477},"home":{"status":"ok","http":200,"url":"https://www.hurriyet.com.tr/","html_tokens":64125,"markdown_tokens":41,"pruned_tokens":41}},
{"rank":3964,"domain":"urban.org","origin":"https://urban.org","robots":{"status":"ok","http":200,"url":"https://www.urban.org/robots.txt","bytes":2107,"groups":1},"bots":{"GPTBot":["partial","*",36,18],"ChatGPT-User":["partial","*",36,18],"OAI-SearchBot":["partial","*",36,18],"ClaudeBot":["partial","*",36,18],"Claude-User":["partial","*",36,18],"Claude-SearchBot":["partial","*",36,18],"anthropic-ai":["partial","*",36,18],"PerplexityBot":["partial","*",36,18],"Perplexity-User":["partial","*",36,18],"Google-Extended":["partial","*",36,18],"Applebot-Extended":["partial","*",36,18],"CCBot":["partial","*",36,18],"Bytespider":["partial","*",36,18],"Amazonbot":["partial","*",36,18],"meta-externalagent":["partial","*",36,18],"cohere-ai":["partial","*",36,18],"Diffbot":["partial","*",36,18],"ContextFetchBot":["partial","*",36,18]},"llms":{"status":"missing","http":404,"bytes":548},"home":{"status":"ok","http":200,"url":"https://www.urban.org/","html_tokens":103478,"markdown_tokens":287,"pruned_tokens":147}},
{"rank":3965,"domain":"nbclosangeles.com","origin":"https://nbclosangeles.com","robots":{"status":"ok","http":200,"url":"https://www.nbclosangeles.com/robots.txt","bytes":11198,"groups":27},"bots":{"GPTBot":["partial","named",1,7],"ChatGPT-User":["partial","named",1,7],"OAI-SearchBot":["partial","*",13,3],"ClaudeBot":["partial","named",1,7],"Claude-User":["partial","*",13,3],"Claude-SearchBot":["partial","*",13,3],"anthropic-ai":["partial","named",1,7],"PerplexityBot":["partial","named",1,7],"Perplexity-User":["partial","*",13,3],"Google-Extended":["partial","named",1,7],"Applebot-Extended":["partial","*",13,3],"CCBot":["partial","named",1,7],"Bytespider":["partial","named",1,7],"Amazonbot":["partial","named",1,9],"meta-externalagent":["partial","*",13,3],"cohere-ai":["partial","*",13,3],"Diffbot":["partial","named",1,7],"ContextFetchBot":["partial","*",13,3]},"llms":{"status":"html","http":200,"bytes":553812},"home":{"status":"ok","http":200,"url":"https://www.nbclosangeles.com/","html_tokens":138438,"markdown_tokens":3,"pruned_tokens":3}},
{"rank":3966,"domain":"yahoo.net","origin":"https://yahoo.net","robots":{"status":"ok","http":200,"url":"https://www.yahoo.com/robots.txt","bytes":1464,"groups":3},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["blocked","named",1,0],"OAI-SearchBot":["partial","named",1,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",21,0],"Claude-SearchBot":["partial","named",1,0],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["partial","*",21,0],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["partial","*",21,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["partial","*",21,0],"meta-externalagent":["partial","*",21,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",21,0]},"llms":{"status":"http_error","http":429,"bytes":23},"home":{"status":"http_error","http":429,"url":"https://www.yahoo.com/"}},
{"rank":3967,"domain":"gap.com","origin":"https://gap.com","robots":{"status":"ok","http":200,"url":"https://www.gap.com/robots.txt","bytes":871,"groups":1},"bots":{"GPTBot":["partial","named",16,1],"ChatGPT-User":["partial","named",16,1],"OAI-SearchBot":["partial","named",16,1],"ClaudeBot":["partial","named",16,1],"Claude-User":["partial","*",16,1],"Claude-SearchBot":["partial","*",16,1],"anthropic-ai":["partial","*",16,1],"PerplexityBot":["partial","named",16,1],"Perplexity-User":["partial","*",16,1],"Google-Extended":["partial","*",16,1],"Applebot-Extended":["partial","*",16,1],"CCBot":["partial","named",16,1],"Bytespider":["partial","*",16,1],"Amazonbot":["partial","named",16,1],"meta-externalagent":["partial","named",16,1],"cohere-ai":["partial","*",16,1],"Diffbot":["partial","*",16,1],"ContextFetchBot":["partial","*",16,1]},"llms":{"status":"missing","http":404,"bytes":862747},"home":{"status":"ok","http":200,"url":"https://www.gap.com/","html_tokens":211280,"markdown_tokens":62,"pruned_tokens":39}},
{"rank":3968,"domain":"prometheus.io","origin":"https://prometheus.io","robots":{"status":"ok","http":200,"url":"https://prometheus.io/robots.txt","bytes":1248,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":29487},"home":{"status":"ok","http":200,"url":"https://prometheus.io/","html_tokens":33913,"markdown_tokens":396,"pruned_tokens":197}},
{"rank":3969,"domain":"capitalone.com","origin":"https://capitalone.com","robots":{"status":"ok","http":200,"url":"https://www.capitalone.com/robots.txt","bytes":2664,"groups":9},"bots":{"GPTBot":["partial","*",3,0],"ChatGPT-User":["partial","*",3,0],"OAI-SearchBot":["partial","*",3,0],"ClaudeBot":["partial","*",3,0],"Claude-User":["partial","*",3,0],"Claude-SearchBot":["partial","*",3,0],"anthropic-ai":["partial","*",3,0],"PerplexityBot":["partial","*",3,0],"Perplexity-User":["partial","*",3,0],"Google-Extended":["partial","*",3,0],"Applebot-Extended":["partial","*",3,0],"CCBot":["partial","*",3,0],"Bytespider":["partial","*",3,0],"Amazonbot":["partial","*",3,0],"meta-externalagent":["partial","*",3,0],"cohere-ai":["partial","*",3,0],"Diffbot":["partial","*",3,0],"ContextFetchBot":["partial","*",3,0]},"llms":{"status":"missing","http":404,"bytes":100},"home":{"status":"ok","http":200,"url":"https://www.capitalone.com/","html_tokens":198793,"markdown_tokens":185,"pruned_tokens":56}},
{"rank":3970,"domain":"nyse.com","origin":"https://nyse.com","robots":{"status":"ok","http":200,"url":"https://www.nyse.com/robots.txt","bytes":95,"groups":1},"bots":{"GPTBot":["partial","*",2,0],"ChatGPT-User":["partial","*",2,0],"OAI-SearchBot":["partial","*",2,0],"ClaudeBot":["partial","*",2,0],"Claude-User":["partial","*",2,0],"Claude-SearchBot":["partial","*",2,0],"anthropic-ai":["partial","*",2,0],"PerplexityBot":["partial","*",2,0],"Perplexity-User":["partial","*",2,0],"Google-Extended":["partial","*",2,0],"Applebot-Extended":["partial","*",2,0],"CCBot":["partial","*",2,0],"Bytespider":["partial","*",2,0],"Amazonbot":["partial","*",2,0],"meta-externalagent":["partial","*",2,0],"cohere-ai":["partial","*",2,0],"Diffbot":["partial","*",2,0],"ContextFetchBot":["partial","*",2,0]},"llms":{"status":"missing","http":404,"bytes":69105},"home":{"status":"ok","http":200,"url":"https://www.nyse.com/index","html_tokens":66190,"markdown_tokens":968,"pruned_tokens":148}},
{"rank":3971,"domain":"home.cern","origin":"https://home.cern","robots":{"status":"ok","http":200,"url":"https://home.cern/robots.txt","bytes":1654,"groups":1},"bots":{"GPTBot":["partial","*",7,53],"ChatGPT-User":["partial","*",7,53],"OAI-SearchBot":["partial","*",7,53],"ClaudeBot":["partial","*",7,53],"Claude-User":["partial","*",7,53],"Claude-SearchBot":["partial","*",7,53],"anthropic-ai":["partial","*",7,53],"PerplexityBot":["partial","*",7,53],"Perplexity-User":["partial","*",7,53],"Google-Extended":["partial","*",7,53],"Applebot-Extended":["partial","*",7,53],"CCBot":["partial","*",7,53],"Bytespider":["partial","*",7,53],"Amazonbot":["partial","*",7,53],"meta-externalagent":["partial","*",7,53],"cohere-ai":["partial","*",7,53],"Diffbot":["partial","*",7,53],"ContextFetchBot":["partial","*",7,53]},"llms":{"status":"missing","http":404,"bytes":240026},"home":{"status":"ok","http":200,"url":"https://home.cern/","html_tokens":74662,"markdown_tokens":327,"pruned_tokens":113}},
{"rank":3972,"domain":"muenchen.de","origin":"https://muenchen.de","robots":{"status":"ok","http":200,"url":"https://www.muenchen.de/robots.txt","bytes":1740,"groups":1},"bots":{"GPTBot":["partial","*",27,18],"ChatGPT-User":["partial","*",27,18],"OAI-SearchBot":["partial","*",27,18],"ClaudeBot":["partial","*",27,18],"Claude-User":["partial","*",27,18],"Claude-SearchBot":["partial","*",27,18],"anthropic-ai":["partial","*",27,18],"PerplexityBot":["partial","*",27,18],"Perplexity-User":["partial","*",27,18],"Google-Extended":["partial","*",27,18],"Applebot-Extended":["partial","*",27,18],"CCBot":["partial","*",27,18],"Bytespider":["partial","*",27,18],"Amazonbot":["partial","*",27,18],"meta-externalagent":["partial","*",27,18],"cohere-ai":["partial","*",27,18],"Diffbot":["partial","*",27,18],"ContextFetchBot":["partial","*",27,18]},"llms":{"status":"missing","http":404,"bytes":189},"home":{"status":"ok","http":200,"url":"https://www.muenchen.de/","html_tokens":72118,"markdown_tokens":9,"pruned_tokens":9}},
{"rank":3973,"domain":"informs.org","origin":"https://informs.org","robots":{"status":"ok","http":200,"url":"https://www.informs.org/robots.txt","bytes":1219,"groups":1},"bots":{"GPTBot":["partial","*",34,1],"ChatGPT-User":["partial","*",34,1],"OAI-SearchBot":["partial","*",34,1],"ClaudeBot":["partial","*",34,1],"Claude-User":["partial","*",34,1],"Claude-SearchBot":["partial","*",34,1],"anthropic-ai":["partial","*",34,1],"PerplexityBot":["partial","*",34,1],"Perplexity-User":["partial","*",34,1],"Google-Extended":["partial","*",34,1],"Applebot-Extended":["partial","*",34,1],"CCBot":["partial","*",34,1],"Bytespider":["partial","*",34,1],"Amazonbot":["partial","*",34,1],"meta-externalagent":["partial","*",34,1],"cohere-ai":["partial","*",34,1],"Diffbot":["partial","*",34,1],"ContextFetchBot":["partial","*",34,1]},"llms":{"status":"missing","http":404,"bytes":125665},"home":{"status":"ok","http":200,"url":"https://www.informs.org/","html_tokens":43892,"markdown_tokens":150,"pruned_tokens":126}},
{"rank":3974,"domain":"efe.com","origin":"https://efe.com","robots":{"status":"refused","http":403,"url":"https://efe.com/robots.txt","bytes":7355,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":3975,"domain":"wtop.com","origin":"https://wtop.com","robots":{"status":"ok","http":200,"url":"https://wtop.com/robots.txt","bytes":1320,"groups":33},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["blocked","named",1,0],"OAI-SearchBot":["blocked","named",1,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["blocked","named",1,0],"Claude-SearchBot":["blocked","named",1,0],"anthropic-ai":["partial","*",5,1],"PerplexityBot":["partial","*",5,1],"Perplexity-User":["partial","*",5,1],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["partial","*",5,1],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["partial","*",5,1],"cohere-ai":["partial","*",5,1],"Diffbot":["partial","*",5,1],"ContextFetchBot":["partial","*",5,1]},"llms":{"status":"missing","http":404,"bytes":147807},"home":{"status":"ok","http":200,"url":"https://wtop.com/","html_tokens":75126,"markdown_tokens":18,"pruned_tokens":18}},
{"rank":3976,"domain":"shef.ac.uk","origin":"https://shef.ac.uk","robots":{"status":"ok","http":200,"url":"https://sheffield.ac.uk/robots.txt","bytes":1801,"groups":1},"bots":{"GPTBot":["partial","*",28,18],"ChatGPT-User":["partial","*",28,18],"OAI-SearchBot":["partial","*",28,18],"ClaudeBot":["partial","*",28,18],"Claude-User":["partial","*",28,18],"Claude-SearchBot":["partial","*",28,18],"anthropic-ai":["partial","*",28,18],"PerplexityBot":["partial","*",28,18],"Perplexity-User":["partial","*",28,18],"Google-Extended":["partial","*",28,18],"Applebot-Extended":["partial","*",28,18],"CCBot":["partial","*",28,18],"Bytespider":["partial","*",28,18],"Amazonbot":["partial","*",28,18],"meta-externalagent":["partial","*",28,18],"cohere-ai":["partial","*",28,18],"Diffbot":["partial","*",28,18],"ContextFetchBot":["partial","*",28,18]},"llms":{"status":"missing","http":404,"bytes":24217},"home":{"status":"ok","http":200,"url":"https://sheffield.ac.uk/","html_tokens":28979,"markdown_tokens":70,"pruned_tokens":70}},
{"rank":3977,"domain":"digitalspy.com","origin":"https://digitalspy.com","robots":{"status":"ok","http":200,"url":"https://digitalspy.com/robots.txt","bytes":712,"groups":3},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["blocked","named",1,0],"OAI-SearchBot":["blocked","named",1,0],"ClaudeBot":["partial","*",19,0],"Claude-User":["partial","*",19,0],"Claude-SearchBot":["partial","*",19,0],"anthropic-ai":["partial","*",19,0],"PerplexityBot":["partial","*",19,0],"Perplexity-User":["partial","*",19,0],"Google-Extended":["partial","*",19,0],"Applebot-Extended":["partial","*",19,0],"CCBot":["partial","*",19,0],"Bytespider":["partial","*",19,0],"Amazonbot":["partial","*",19,0],"meta-externalagent":["partial","*",19,0],"cohere-ai":["partial","*",19,0],"Diffbot":["partial","*",19,0],"ContextFetchBot":["partial","*",19,0]},"llms":{"status":"missing","http":404,"bytes":84},"home":{"status":"ok","http":200,"url":"https://www.digitalspy.com/","html_tokens":142261,"markdown_tokens":71,"pruned_tokens":71}},
{"rank":3978,"domain":"jll.com","origin":"https://jll.com","robots":{"status":"ok","http":200,"url":"https://www.jll.com/robots.txt","bytes":460,"groups":1},"bots":{"GPTBot":["partial","*",8,2],"ChatGPT-User":["partial","*",8,2],"OAI-SearchBot":["partial","*",8,2],"ClaudeBot":["partial","*",8,2],"Claude-User":["partial","*",8,2],"Claude-SearchBot":["partial","*",8,2],"anthropic-ai":["partial","*",8,2],"PerplexityBot":["partial","*",8,2],"Perplexity-User":["partial","*",8,2],"Google-Extended":["partial","*",8,2],"Applebot-Extended":["partial","*",8,2],"CCBot":["partial","*",8,2],"Bytespider":["partial","*",8,2],"Amazonbot":["partial","*",8,2],"meta-externalagent":["partial","*",8,2],"cohere-ai":["partial","*",8,2],"Diffbot":["partial","*",8,2],"ContextFetchBot":["partial","*",8,2]},"llms":{"status":"html","http":200,"bytes":404762},"home":{"status":"ok","http":200,"url":"https://www.jll.com/en-us/","html_tokens":101173,"markdown_tokens":125,"pruned_tokens":125}},
{"rank":3979,"domain":"unaids.org","origin":"https://unaids.org","robots":{"status":"ok","http":200,"url":"https://unaids.org/robots.txt","bytes":2067,"groups":1},"bots":{"GPTBot":["partial","*",35,18],"ChatGPT-User":["partial","*",35,18],"OAI-SearchBot":["partial","*",35,18],"ClaudeBot":["partial","*",35,18],"Claude-User":["partial","*",35,18],"Claude-SearchBot":["partial","*",35,18],"anthropic-ai":["partial","*",35,18],"PerplexityBot":["partial","*",35,18],"Perplexity-User":["partial","*",35,18],"Google-Extended":["partial","*",35,18],"Applebot-Extended":["partial","*",35,18],"CCBot":["partial","*",35,18],"Bytespider":["partial","*",35,18],"Amazonbot":["partial","*",35,18],"meta-externalagent":["partial","*",35,18],"cohere-ai":["partial","*",35,18],"Diffbot":["partial","*",35,18],"ContextFetchBot":["partial","*",35,18]},"llms":{"status":"missing","http":404,"bytes":183},"home":{"status":"ok","http":200,"url":"https://www.unaids.org/en","html_tokens":33621,"markdown_tokens":239,"pruned_tokens":112}},
{"rank":3980,"domain":"uc.pt","origin":"https://uc.pt","robots":{"status":"ok","http":200,"url":"https://www.uc.pt/robots.txt","bytes":607,"groups":1},"bots":{"GPTBot":["partial","*",4,0],"ChatGPT-User":["partial","*",4,0],"OAI-SearchBot":["partial","*",4,0],"ClaudeBot":["partial","*",4,0],"Claude-User":["partial","*",4,0],"Claude-SearchBot":["partial","*",4,0],"anthropic-ai":["partial","*",4,0],"PerplexityBot":["partial","*",4,0],"Perplexity-User":["partial","*",4,0],"Google-Extended":["partial","*",4,0],"Applebot-Extended":["partial","*",4,0],"CCBot":["partial","*",4,0],"Bytespider":["partial","*",4,0],"Amazonbot":["partial","*",4,0],"meta-externalagent":["partial","*",4,0],"cohere-ai":["partial","*",4,0],"Diffbot":["partial","*",4,0],"ContextFetchBot":["partial","*",4,0]},"llms":{"status":"http_error","http":403,"bytes":2182},"home":{"status":"http_error","http":403,"url":"https://www.uc.pt/"}},
{"rank":3981,"domain":"aclanthology.org","origin":"https://aclanthology.org","robots":{"status":"missing","http":404,"url":"https://aclanthology.org/robots.txt","bytes":9183,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":9183},"home":{"status":"ok","http":200,"url":"https://aclanthology.org/","html_tokens":184827,"markdown_tokens":97,"pruned_tokens":97}},
{"rank":3982,"domain":"syracuse.com","origin":"https://syracuse.com","robots":{"status":"ok","http":200,"url":"https://www.syracuse.com/robots.txt","bytes":2830,"groups":4},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["partial","named",1,11],"OAI-SearchBot":["partial","named",1,11],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","named",1,11],"Claude-SearchBot":["partial","named",1,11],"anthropic-ai":["partial","*",30,3],"PerplexityBot":["partial","named",1,11],"Perplexity-User":["partial","named",1,11],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["partial","*",30,3],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",30,3]},"llms":{"status":"missing","http":404,"bytes":268656},"home":{"status":"http_error","http":403,"url":"https://www.syracuse.com/"}},
{"rank":3983,"domain":"macrotrends.net","origin":"https://macrotrends.net","robots":{"status":"ok","http":200,"url":"https://macrotrends.net/robots.txt","bytes":2676,"groups":32},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["allowed","*",0,1],"OAI-SearchBot":["allowed","*",0,1],"ClaudeBot":["blocked","named",1,0],"Claude-User":["allowed","*",0,1],"Claude-SearchBot":["allowed","*",0,1],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["allowed","*",0,1],"Perplexity-User":["allowed","*",0,1],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["allowed","*",0,1]},"llms":{"status":"http_error","http":403,"bytes":5572},"home":{"status":"http_error","http":403,"url":"https://www.macrotrends.net/"}},
{"rank":3984,"domain":"mensjournal.com","origin":"https://mensjournal.com","robots":{"status":"ok","http":200,"url":"https://www.mensjournal.com/robots.txt","bytes":301,"groups":1},"bots":{"GPTBot":["partial","*",5,5],"ChatGPT-User":["partial","*",5,5],"OAI-SearchBot":["partial","*",5,5],"ClaudeBot":["partial","*",5,5],"Claude-User":["partial","*",5,5],"Claude-SearchBot":["partial","*",5,5],"anthropic-ai":["partial","*",5,5],"PerplexityBot":["partial","*",5,5],"Perplexity-User":["partial","*",5,5],"Google-Extended":["partial","*",5,5],"Applebot-Extended":["partial","*",5,5],"CCBot":["partial","*",5,5],"Bytespider":["partial","*",5,5],"Amazonbot":["partial","*",5,5],"meta-externalagent":["partial","*",5,5],"cohere-ai":["partial","*",5,5],"Diffbot":["partial","*",5,5],"ContextFetchBot":["partial","*",5,5]},"llms":{"status":"http_error","http":403,"bytes":775},"home":{"status":"http_error","http":403,"url":"https://mensjournal.com/"}},
{"rank":3985,"domain":"unr.edu","origin":"https://unr.edu","robots":{"status":"ok","http":200,"url":"https://www.unr.edu/robots.txt","bytes":25,"groups":1},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"missing","http":404,"bytes":13087},"home":{"status":"ok","http":200,"url":"https://www.unr.edu/","html_tokens":17164,"markdown_tokens":367,"pruned_tokens":161}},
{"rank":3986,"domain":"jreast.co.jp","origin":"https://www.jreast.co.jp","robots":{"status":"refused","http":403,"url":"https://www.jreast.co.jp/robots.txt","bytes":388,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":3987,"domain":"secondlife.com","origin":"https://secondlife.com","robots":{"status":"ok","http":200,"url":"https://secondlife.com/robots.txt","bytes":445,"groups":2},"bots":{"GPTBot":["partial","*",14,0],"ChatGPT-User":["partial","*",14,0],"OAI-SearchBot":["partial","*",14,0],"ClaudeBot":["partial","*",14,0],"Claude-User":["partial","*",14,0],"Claude-SearchBot":["partial","*",14,0],"anthropic-ai":["partial","*",14,0],"PerplexityBot":["partial","*",14,0],"Perplexity-User":["partial","*",14,0],"Google-Extended":["partial","*",14,0],"Applebot-Extended":["partial","*",14,0],"CCBot":["partial","*",14,0],"Bytespider":["partial","*",14,0],"Amazonbot":["partial","*",14,0],"meta-externalagent":["partial","*",14,0],"cohere-ai":["partial","*",14,0],"Diffbot":["partial","*",14,0],"ContextFetchBot":["partial","*",14,0]},"llms":{"status":"missing","http":404,"bytes":9514},"home":{"status":"ok","http":200,"url":"https://secondlife.com/","html_tokens":10428,"markdown_tokens":237,"pruned_tokens":237}},
{"rank":3988,"domain":"greenhouse.io","origin":"https://greenhouse.io","robots":{"status":"ok","http":200,"url":"https://www.greenhouse.com/robots.txt","bytes":83,"groups":1},"bots":{"GPTBot":["partial","*",1,0],"ChatGPT-User":["partial","*",1,0],"OAI-SearchBot":["partial","*",1,0],"ClaudeBot":["partial","*",1,0],"Claude-User":["partial","*",1,0],"Claude-SearchBot":["partial","*",1,0],"anthropic-ai":["partial","*",1,0],"PerplexityBot":["partial","*",1,0],"Perplexity-User":["partial","*",1,0],"Google-Extended":["partial","*",1,0],"Applebot-Extended":["partial","*",1,0],"CCBot":["partial","*",1,0],"Bytespider":["partial","*",1,0],"Amazonbot":["partial","*",1,0],"meta-externalagent":["partial","*",1,0],"cohere-ai":["partial","*",1,0],"Diffbot":["partial","*",1,0],"ContextFetchBot":["partial","*",1,0]},"llms":{"status":"present","http":200,"bytes":21000},"home":{"status":"ok","http":200,"url":"https://www.greenhouse.com/","html_tokens":118748,"markdown_tokens":234,"pruned_tokens":83}},
{"rank":3989,"domain":"fau.de","origin":"https://fau.de","robots":{"status":"ok","http":200,"url":"https://www.fau.de/robots.txt","bytes":108,"groups":1},"bots":{"GPTBot":["partial","*",1,1],"ChatGPT-User":["partial","*",1,1],"OAI-SearchBot":["partial","*",1,1],"ClaudeBot":["partial","*",1,1],"Claude-User":["partial","*",1,1],"Claude-SearchBot":["partial","*",1,1],"anthropic-ai":["partial","*",1,1],"PerplexityBot":["partial","*",1,1],"Perplexity-User":["partial","*",1,1],"Google-Extended":["partial","*",1,1],"Applebot-Extended":["partial","*",1,1],"CCBot":["partial","*",1,1],"Bytespider":["partial","*",1,1],"Amazonbot":["partial","*",1,1],"meta-externalagent":["partial","*",1,1],"cohere-ai":["partial","*",1,1],"Diffbot":["partial","*",1,1],"ContextFetchBot":["partial","*",1,1]},"llms":{"status":"missing","http":404,"bytes":404669},"home":{"status":"ok","http":200,"url":"https://www.fau.de/","html_tokens":112699,"markdown_tokens":122,"pruned_tokens":122}},
{"rank":3990,"domain":"blogspot.ru","origin":"https://blogspot.ru","robots":{"status":"ok","http":200,"url":"https://blogspot.ru/robots.txt","bytes":6624,"groups":4},"bots":{"GPTBot":["partial","*",169,64],"ChatGPT-User":["partial","*",169,64],"OAI-SearchBot":["partial","*",169,64],"ClaudeBot":["partial","*",169,64],"Claude-User":["partial","*",169,64],"Claude-SearchBot":["partial","*",169,64],"anthropic-ai":["partial","*",169,64],"PerplexityBot":["partial","*",169,64],"Perplexity-User":["partial","*",169,64],"Google-Extended":["partial","*",169,64],"Applebot-Extended":["partial","*",169,64],"CCBot":["partial","*",169,64],"Bytespider":["partial","*",169,64],"Amazonbot":["partial","*",169,64],"meta-externalagent":["partial","*",169,64],"cohere-ai":["partial","*",169,64],"Diffbot":["partial","*",169,64],"ContextFetchBot":["partial","*",169,64]},"llms":{"status":"missing","http":404,"bytes":1569},"home":{"status":"ok","http":200,"url":"https://www.google.com/","html_tokens":21324,"markdown_tokens":30,"pruned_tokens":30}},
{"rank":3991,"domain":"berliner-zeitung.de","origin":"https://berliner-zeitung.de","robots":{"status":"ok","http":200,"url":"https://www.berliner-zeitung.de/robots.txt","bytes":1039,"groups":5},"bots":{"GPTBot":["partial","*",19,1],"ChatGPT-User":["partial","*",19,1],"OAI-SearchBot":["partial","*",19,1],"ClaudeBot":["partial","*",19,1],"Claude-User":["partial","*",19,1],"Claude-SearchBot":["partial","*",19,1],"anthropic-ai":["partial","*",19,1],"PerplexityBot":["partial","*",19,1],"Perplexity-User":["partial","*",19,1],"Google-Extended":["partial","*",19,1],"Applebot-Extended":["partial","*",19,1],"CCBot":["partial","*",19,1],"Bytespider":["partial","*",19,1],"Amazonbot":["partial","*",19,1],"meta-externalagent":["partial","*",19,1],"cohere-ai":["partial","*",19,1],"Diffbot":["partial","*",19,1],"ContextFetchBot":["partial","*",19,1]},"llms":{"status":"missing","http":404,"bytes":118691},"home":{"status":"ok","http":200,"url":"https://www.berliner-zeitung.de/","html_tokens":350794,"markdown_tokens":19,"pruned_tokens":19}},
{"rank":3992,"domain":"cardiff.ac.uk","origin":"https://cardiff.ac.uk","robots":{"status":"refused","http":403,"url":"https://www.cardiff.ac.uk/robots.txt","bytes":5555,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":3993,"domain":"snapwidget.com","origin":"https://snapwidget.com","robots":{"status":"ok","http":200,"url":"https://snapwidget.com/robots.txt","bytes":78,"groups":1},"bots":{"GPTBot":["partial","*",1,0],"ChatGPT-User":["partial","*",1,0],"OAI-SearchBot":["partial","*",1,0],"ClaudeBot":["partial","*",1,0],"Claude-User":["partial","*",1,0],"Claude-SearchBot":["partial","*",1,0],"anthropic-ai":["partial","*",1,0],"PerplexityBot":["partial","*",1,0],"Perplexity-User":["partial","*",1,0],"Google-Extended":["partial","*",1,0],"Applebot-Extended":["partial","*",1,0],"CCBot":["partial","*",1,0],"Bytespider":["partial","*",1,0],"Amazonbot":["partial","*",1,0],"meta-externalagent":["partial","*",1,0],"cohere-ai":["partial","*",1,0],"Diffbot":["partial","*",1,0],"ContextFetchBot":["partial","*",1,0]},"llms":{"status":"present","http":200,"bytes":12857},"home":{"status":"ok","http":200,"url":"https://snapwidget.com/","html_tokens":66771,"markdown_tokens":523,"pruned_tokens":118}},
{"rank":3994,"domain":"mi.us","origin":"https://mi.us","robots":{"status":"unreachable","http":null,"url":"https://mi.us/robots.txt","bytes":0,"groups":0,"error":"cannot resolve mi.us"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":3995,"domain":"snap.com","origin":"https://snap.com","robots":{"status":"ok","http":200,"url":"https://www.snap.com/robots.txt","bytes":82,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":320319},"home":{"status":"ok","http":200,"url":"https://www.snap.com/","html_tokens":70616,"markdown_tokens":27,"pruned_tokens":27}},
{"rank":3996,"domain":"lyft.com","origin":"https://lyft.com","robots":{"status":"ok","http":200,"url":"https://lyft.com/robots.txt","bytes":1046,"groups":3},"bots":{"GPTBot":["partial","*",31,0],"ChatGPT-User":["partial","*",31,0],"OAI-SearchBot":["partial","*",31,0],"ClaudeBot":["partial","*",31,0],"Claude-User":["partial","*",31,0],"Claude-SearchBot":["partial","*",31,0],"anthropic-ai":["partial","*",31,0],"PerplexityBot":["partial","*",31,0],"Perplexity-User":["partial","*",31,0],"Google-Extended":["partial","*",31,0],"Applebot-Extended":["partial","*",31,0],"CCBot":["partial","*",31,0],"Bytespider":["partial","*",31,0],"Amazonbot":["partial","*",31,0],"meta-externalagent":["partial","*",31,0],"cohere-ai":["partial","*",31,0],"Diffbot":["partial","*",31,0],"ContextFetchBot":["partial","*",31,0]},"llms":{"status":"missing","http":404,"bytes":116960},"home":{"status":"ok","http":200,"url":"https://www.lyft.com/","html_tokens":104003,"markdown_tokens":594,"pruned_tokens":247}},
{"rank":3997,"domain":"mothercare.co.id","origin":"https://mothercare.co.id","robots":{"status":"ok","http":200,"url":"https://www.mothercare.co.id/robots.txt","bytes":9017,"groups":174},"bots":{"GPTBot":["partial","*",5,2],"ChatGPT-User":["blocked","named",1,0],"OAI-SearchBot":["blocked","named",1,0],"ClaudeBot":["partial","*",5,2],"Claude-User":["partial","*",5,2],"Claude-SearchBot":["partial","*",5,2],"anthropic-ai":["partial","*",5,2],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["partial","*",5,2],"Google-Extended":["partial","*",5,2],"Applebot-Extended":["partial","*",5,2],"CCBot":["blocked","named",1,0],"Bytespider":["partial","*",5,2],"Amazonbot":["partial","*",5,2],"meta-externalagent":["partial","*",5,2],"cohere-ai":["partial","*",5,2],"Diffbot":["partial","*",5,2],"ContextFetchBot":["partial","*",5,2]},"llms":{"status":"missing","http":404,"bytes":230940},"home":{"status":"http_error","http":403,"url":"https://mothercare.co.id/"}},
{"rank":3998,"domain":"www.gov.scot","origin":"https://www.gov.scot","robots":{"status":"ok","http":200,"url":"https://www.gov.scot/robots.txt","bytes":187,"groups":1},"bots":{"GPTBot":["partial","*",5,0],"ChatGPT-User":["partial","*",5,0],"OAI-SearchBot":["partial","*",5,0],"ClaudeBot":["partial","*",5,0],"Claude-User":["partial","*",5,0],"Claude-SearchBot":["partial","*",5,0],"anthropic-ai":["partial","*",5,0],"PerplexityBot":["partial","*",5,0],"Perplexity-User":["partial","*",5,0],"Google-Extended":["partial","*",5,0],"Applebot-Extended":["partial","*",5,0],"CCBot":["partial","*",5,0],"Bytespider":["partial","*",5,0],"Amazonbot":["partial","*",5,0],"meta-externalagent":["partial","*",5,0],"cohere-ai":["partial","*",5,0],"Diffbot":["partial","*",5,0],"ContextFetchBot":["partial","*",5,0]},"llms":{"status":"missing","http":404,"bytes":21011},"home":{"status":"ok","http":200,"url":"https://www.gov.scot/","html_tokens":16996,"markdown_tokens":156,"pruned_tokens":156}},
{"rank":3999,"domain":"designrush.com","origin":"https://designrush.com","robots":{"status":"ok","http":200,"url":"https://www.designrush.com/robots.txt","bytes":1531,"groups":6},"bots":{"GPTBot":["partial","named",1,0],"ChatGPT-User":["partial","*",1,1],"OAI-SearchBot":["partial","*",1,1],"ClaudeBot":["partial","named",1,0],"Claude-User":["partial","*",1,1],"Claude-SearchBot":["partial","*",1,1],"anthropic-ai":["partial","*",1,1],"PerplexityBot":["partial","named",1,0],"Perplexity-User":["partial","*",1,1],"Google-Extended":["partial","*",1,1],"Applebot-Extended":["partial","*",1,1],"CCBot":["partial","*",1,1],"Bytespider":["blocked","named",1,0],"Amazonbot":["partial","*",1,1],"meta-externalagent":["partial","*",1,1],"cohere-ai":["partial","*",1,1],"Diffbot":["partial","*",1,1],"ContextFetchBot":["partial","*",1,1]},"llms":{"status":"present","http":200,"bytes":112121},"home":{"status":"http_error","http":403,"url":"https://designrush.com/"}},
{"rank":4000,"domain":"klm.com","origin":"https://klm.com","robots":{"status":"timeout","http":null,"url":"https://klm.com/robots.txt","bytes":0,"groups":0,"error":"timed out after 10 s"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":4001,"domain":"theprint.in","origin":"https://theprint.in","robots":{"status":"ok","http":200,"url":"https://theprint.in/robots.txt","bytes":660,"groups":5},"bots":{"GPTBot":["allowed","named",0,1],"ChatGPT-User":["partial","*",15,1],"OAI-SearchBot":["partial","*",15,1],"ClaudeBot":["partial","*",15,1],"Claude-User":["partial","*",15,1],"Claude-SearchBot":["partial","*",15,1],"anthropic-ai":["partial","*",15,1],"PerplexityBot":["partial","*",15,1],"Perplexity-User":["partial","*",15,1],"Google-Extended":["allowed","named",0,1],"Applebot-Extended":["partial","*",15,1],"CCBot":["blocked","named",1,0],"Bytespider":["partial","*",15,1],"Amazonbot":["partial","*",15,1],"meta-externalagent":["partial","*",15,1],"cohere-ai":["partial","*",15,1],"Diffbot":["partial","*",15,1],"ContextFetchBot":["partial","*",15,1]},"llms":{"status":"missing","http":404,"bytes":586689},"home":{"status":"ok","http":200,"url":"https://theprint.in/","html_tokens":251030,"markdown_tokens":245,"pruned_tokens":81}},
{"rank":4002,"domain":"coca-colacompany.com","origin":"https://coca-colacompany.com","robots":{"status":"ok","http":200,"url":"https://www.coca-colacompany.com/robots.txt","bytes":100,"groups":1},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"missing","http":404,"bytes":120608},"home":{"status":"ok","http":200,"url":"https://www.coca-colacompany.com/","html_tokens":69144,"markdown_tokens":43,"pruned_tokens":43}},
{"rank":4003,"domain":"liverpool.ac.uk","origin":"https://liverpool.ac.uk","robots":{"status":"ok","http":200,"url":"https://www.liverpool.ac.uk/robots.txt","bytes":769,"groups":3},"bots":{"GPTBot":["partial","*",10,0],"ChatGPT-User":["partial","*",10,0],"OAI-SearchBot":["partial","*",10,0],"ClaudeBot":["partial","*",10,0],"Claude-User":["partial","*",10,0],"Claude-SearchBot":["partial","*",10,0],"anthropic-ai":["partial","*",10,0],"PerplexityBot":["partial","*",10,0],"Perplexity-User":["partial","*",10,0],"Google-Extended":["partial","*",10,0],"Applebot-Extended":["partial","*",10,0],"CCBot":["partial","*",10,0],"Bytespider":["partial","*",10,0],"Amazonbot":["partial","*",10,0],"meta-externalagent":["partial","*",10,0],"cohere-ai":["partial","*",10,0],"Diffbot":["partial","*",10,0],"ContextFetchBot":["partial","*",10,0]},"llms":{"status":"missing","http":404,"bytes":28235},"home":{"status":"ok","http":200,"url":"https://www.liverpool.ac.uk/","html_tokens":25258,"markdown_tokens":128,"pruned_tokens":128}},
{"rank":4004,"domain":"cervantes.es","origin":"https://cervantes.es","robots":{"status":"ok","http":200,"url":"https://comunes.cervantes.es/mantenimiento/","bytes":1255,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"disallowed","error":"robots.txt of comunes.cervantes.es disallows /mantenimiento"},"home":{"status":"disallowed","error":"robots.txt of comunes.cervantes.es disallows /mantenimiento"}},
{"rank":4005,"domain":"progress.com","origin":"https://progress.com","robots":{"status":"ok","http":200,"url":"https://www.progress.com/robots.txt","bytes":562,"groups":1},"bots":{"GPTBot":["partial","*",7,0],"ChatGPT-User":["partial","*",7,0],"OAI-SearchBot":["partial","*",7,0],"ClaudeBot":["partial","*",7,0],"Claude-User":["partial","*",7,0],"Claude-SearchBot":["partial","*",7,0],"anthropic-ai":["partial","*",7,0],"PerplexityBot":["partial","*",7,0],"Perplexity-User":["partial","*",7,0],"Google-Extended":["partial","*",7,0],"Applebot-Extended":["partial","*",7,0],"CCBot":["partial","*",7,0],"Bytespider":["partial","*",7,0],"Amazonbot":["partial","*",7,0],"meta-externalagent":["partial","*",7,0],"cohere-ai":["partial","*",7,0],"Diffbot":["partial","*",7,0],"ContextFetchBot":["partial","*",7,0]},"llms":{"status":"present","http":200,"bytes":9848},"home":{"status":"ok","http":200,"url":"https://www.progress.com/","html_tokens":35056,"markdown_tokens":478,"pruned_tokens":172}},
{"rank":4006,"domain":"uni-trier.de","origin":"https://www.uni-trier.de","robots":{"status":"ok","http":200,"url":"https://www.uni-trier.de/robots.txt","bytes":776,"groups":13},"bots":{"GPTBot":["partial","*",3,0],"ChatGPT-User":["partial","*",3,0],"OAI-SearchBot":["partial","*",3,0],"ClaudeBot":["partial","*",3,0],"Claude-User":["partial","*",3,0],"Claude-SearchBot":["partial","*",3,0],"anthropic-ai":["partial","*",3,0],"PerplexityBot":["partial","*",3,0],"Perplexity-User":["partial","*",3,0],"Google-Extended":["partial","*",3,0],"Applebot-Extended":["partial","*",3,0],"CCBot":["partial","*",3,0],"Bytespider":["partial","*",3,0],"Amazonbot":["partial","*",3,0],"meta-externalagent":["partial","*",3,0],"cohere-ai":["partial","*",3,0],"Diffbot":["partial","*",3,0],"ContextFetchBot":["partial","*",3,0]},"llms":{"status":"missing","http":404,"bytes":431069},"home":{"status":"ok","http":200,"url":"https://www.uni-trier.de/","html_tokens":114870,"markdown_tokens":130,"pruned_tokens":130}},
{"rank":4007,"domain":"tribune.com.pk","origin":"https://tribune.com.pk","robots":{"status":"ok","http":200,"url":"https://tribune.com.pk/robots.txt","bytes":181,"groups":1},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"html","http":200,"bytes":422516},"home":{"status":"ok","http":200,"url":"https://tribune.com.pk/","html_tokens":105610,"markdown_tokens":79,"pruned_tokens":79}},
{"rank":4008,"domain":"discuz.vip","origin":"https://www.discuz.vip","robots":{"status":"ok","http":200,"url":"https://www.discuz.vip/robots.txt","bytes":0,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":146},"home":{"status":"ok","http":200,"url":"https://www.discuz.vip/","html_tokens":22168,"markdown_tokens":1015,"pruned_tokens":388}},
{"rank":4009,"domain":"rackspace.com","origin":"https://rackspace.com","robots":{"status":"ok","http":200,"url":"https://www.rackspace.com/robots.txt","bytes":1893,"groups":1},"bots":{"GPTBot":["partial","*",27,18],"ChatGPT-User":["partial","*",27,18],"OAI-SearchBot":["partial","*",27,18],"ClaudeBot":["partial","*",27,18],"Claude-User":["partial","*",27,18],"Claude-SearchBot":["partial","*",27,18],"anthropic-ai":["partial","*",27,18],"PerplexityBot":["partial","*",27,18],"Perplexity-User":["partial","*",27,18],"Google-Extended":["partial","*",27,18],"Applebot-Extended":["partial","*",27,18],"CCBot":["partial","*",27,18],"Bytespider":["partial","*",27,18],"Amazonbot":["partial","*",27,18],"meta-externalagent":["partial","*",27,18],"cohere-ai":["partial","*",27,18],"Diffbot":["partial","*",27,18],"ContextFetchBot":["partial","*",27,18]},"llms":{"status":"missing","http":404,"bytes":4167},"home":{"status":"ok","http":200,"url":"https://www.rackspace.com/","html_tokens":23934,"markdown_tokens":602,"pruned_tokens":65}},
{"rank":4010,"domain":"royalsociety.org","origin":"https://royalsociety.org","robots":{"status":"refused","http":403,"url":"https://royalsociety.org/robots.txt","bytes":1644,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":4011,"domain":"abc7news.com","origin":"https://abc7news.com","robots":{"status":"ok","http":200,"url":"https://abc7news.com/robots.txt","bytes":1127,"groups":26},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["blocked","named",1,0],"OAI-SearchBot":["partial","*",7,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",7,0],"Claude-SearchBot":["partial","*",7,0],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["partial","*",7,0],"Perplexity-User":["partial","*",7,0],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["partial","*",7,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",7,0]},"llms":{"status":"html","http":200,"bytes":73941},"home":{"status":"ok","http":200,"url":"https://abc7news.com/","html_tokens":138858,"markdown_tokens":143,"pruned_tokens":143}},
{"rank":4012,"domain":"ezinearticles.com","origin":"https://ezinearticles.com","robots":{"status":"unreachable","http":null,"url":"https://ezinearticles.com/robots.txt","bytes":0,"groups":0,"error":"cannot resolve ezinearticles.com"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":4013,"domain":"files.fm","origin":"https://files.fm","robots":{"status":"ok","http":200,"url":"https://files.fm/robots.txt","bytes":541,"groups":1},"bots":{"GPTBot":["partial","*",9,0],"ChatGPT-User":["partial","*",9,0],"OAI-SearchBot":["partial","*",9,0],"ClaudeBot":["partial","*",9,0],"Claude-User":["partial","*",9,0],"Claude-SearchBot":["partial","*",9,0],"anthropic-ai":["partial","*",9,0],"PerplexityBot":["partial","*",9,0],"Perplexity-User":["partial","*",9,0],"Google-Extended":["partial","*",9,0],"Applebot-Extended":["partial","*",9,0],"CCBot":["partial","*",9,0],"Bytespider":["partial","*",9,0],"Amazonbot":["partial","*",9,0],"meta-externalagent":["partial","*",9,0],"cohere-ai":["partial","*",9,0],"Diffbot":["partial","*",9,0],"ContextFetchBot":["partial","*",9,0]},"llms":{"status":"present","http":200,"bytes":2207},"home":{"status":"ok","http":200,"url":"https://files.fm/","html_tokens":51100,"markdown_tokens":317,"pruned_tokens":126}},
{"rank":4014,"domain":"eluniversal.com.mx","origin":"https://eluniversal.com.mx","robots":{"status":"ok","http":200,"url":"https://www.eluniversal.com.mx/robots.txt","bytes":4876,"groups":56},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["blocked","named",1,0],"OAI-SearchBot":["blocked","named",1,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["blocked","named",1,0],"Claude-SearchBot":["blocked","named",1,0],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["blocked","named",1,0],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["blocked","*",1,0]},"llms":{"status":"disallowed"},"home":{"status":"disallowed"}},
{"rank":4015,"domain":"58.com","origin":"https://58.com","robots":{"status":"ok","http":200,"url":"https://58.com/robots.txt","bytes":388,"groups":1},"bots":{"GPTBot":["partial","*",14,1],"ChatGPT-User":["partial","*",14,1],"OAI-SearchBot":["partial","*",14,1],"ClaudeBot":["partial","*",14,1],"Claude-User":["partial","*",14,1],"Claude-SearchBot":["partial","*",14,1],"anthropic-ai":["partial","*",14,1],"PerplexityBot":["partial","*",14,1],"Perplexity-User":["partial","*",14,1],"Google-Extended":["partial","*",14,1],"Applebot-Extended":["partial","*",14,1],"CCBot":["partial","*",14,1],"Bytespider":["partial","*",14,1],"Amazonbot":["partial","*",14,1],"meta-externalagent":["partial","*",14,1],"cohere-ai":["partial","*",14,1],"Diffbot":["partial","*",14,1],"ContextFetchBot":["partial","*",14,1]},"llms":{"status":"disallowed","error":"robots.txt of 404.58.com not readable (html)"},"home":{"status":"ok","http":200,"url":"https://bj.58.com/","html_tokens":28601,"markdown_tokens":686,"pruned_tokens":141}},
{"rank":4016,"domain":"openvpn.net","origin":"https://openvpn.net","robots":{"status":"ok","http":200,"url":"https://openvpn.net/robots.txt","bytes":40,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"present","http":200,"bytes":8310},"home":{"status":"ok","http":200,"url":"https://openvpn.net/","html_tokens":89017,"markdown_tokens":582,"pruned_tokens":270}},
{"rank":4017,"domain":"tasteofhome.com","origin":"https://tasteofhome.com","robots":{"status":"ok","http":200,"url":"https://www.tasteofhome.com/robots.txt","bytes":821,"groups":2},"bots":{"GPTBot":["partial","*",14,1],"ChatGPT-User":["partial","*",14,1],"OAI-SearchBot":["partial","*",14,1],"ClaudeBot":["partial","*",14,1],"Claude-User":["partial","*",14,1],"Claude-SearchBot":["partial","*",14,1],"anthropic-ai":["partial","*",14,1],"PerplexityBot":["partial","*",14,1],"Perplexity-User":["partial","*",14,1],"Google-Extended":["partial","*",14,1],"Applebot-Extended":["partial","*",14,1],"CCBot":["partial","*",14,1],"Bytespider":["partial","*",14,1],"Amazonbot":["partial","*",14,1],"meta-externalagent":["partial","*",14,1],"cohere-ai":["partial","*",14,1],"Diffbot":["partial","*",14,1],"ContextFetchBot":["partial","*",14,1]},"llms":{"status":"missing","http":404,"bytes":144286},"home":{"status":"ok","http":200,"url":"https://www.tasteofhome.com/","html_tokens":85242,"markdown_tokens":214,"pruned_tokens":82}},
{"rank":4018,"domain":"dailytelegraph.com.au","origin":"https://dailytelegraph.com.au","robots":{"status":"ok","http":200,"url":"https://www.dailytelegraph.com.au/robots.txt","bytes":4150,"groups":4},"bots":{"GPTBot":["partial","*",1,2],"ChatGPT-User":["partial","*",1,2],"OAI-SearchBot":["partial","*",1,2],"ClaudeBot":["partial","*",1,2],"Claude-User":["partial","*",1,2],"Claude-SearchBot":["partial","*",1,2],"anthropic-ai":["partial","*",1,2],"PerplexityBot":["partial","*",1,2],"Perplexity-User":["partial","*",1,2],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["partial","*",1,2],"CCBot":["partial","*",1,2],"Bytespider":["partial","*",1,2],"Amazonbot":["partial","*",1,2],"meta-externalagent":["partial","*",1,2],"cohere-ai":["partial","*",1,2],"Diffbot":["partial","*",1,2],"ContextFetchBot":["partial","*",1,2]},"llms":{"status":"disallowed"},"home":{"status":"disallowed"}},
{"rank":4019,"domain":"mediapost.com","origin":"https://mediapost.com","robots":{"status":"ok","http":200,"url":"https://www.mediapost.com/robots.txt","bytes":4083,"groups":47},"bots":{"GPTBot":["partial","*",8,0],"ChatGPT-User":["partial","*",8,0],"OAI-SearchBot":["partial","*",8,0],"ClaudeBot":["partial","*",8,0],"Claude-User":["partial","*",8,0],"Claude-SearchBot":["partial","*",8,0],"anthropic-ai":["partial","*",8,0],"PerplexityBot":["partial","*",8,0],"Perplexity-User":["partial","*",8,0],"Google-Extended":["partial","*",8,0],"Applebot-Extended":["partial","*",8,0],"CCBot":["partial","*",8,0],"Bytespider":["partial","*",8,0],"Amazonbot":["partial","*",8,0],"meta-externalagent":["partial","*",8,0],"cohere-ai":["partial","*",8,0],"Diffbot":["partial","*",8,0],"ContextFetchBot":["partial","*",8,0]},"llms":{"status":"missing","http":404,"bytes":107401},"home":{"status":"ok","http":200,"url":"https://www.mediapost.com/","html_tokens":35812,"markdown_tokens":242,"pruned_tokens":95}},
{"rank":4020,"domain":"tribuneindia.com","origin":"https://tribuneindia.com","robots":{"status":"missing","http":404,"url":"https://tribuneindia.com/robots.txt","bytes":146,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"empty","http":200,"bytes":0},"home":{"status":"ok","http":200,"url":"https://www.tribuneindia.com/","html_tokens":137028,"markdown_tokens":3934,"pruned_tokens":174}},
{"rank":4021,"domain":"psychologicalscience.org","origin":"https://psychologicalscience.org","robots":{"status":"ok","http":200,"url":"https://psychologicalscience.org/robots.txt","bytes":3376,"groups":2},"bots":{"GPTBot":["partial","*",16,0],"ChatGPT-User":["partial","*",16,0],"OAI-SearchBot":["partial","*",16,0],"ClaudeBot":["partial","*",16,0],"Claude-User":["partial","*",16,0],"Claude-SearchBot":["partial","*",16,0],"anthropic-ai":["partial","*",16,0],"PerplexityBot":["partial","*",16,0],"Perplexity-User":["partial","*",16,0],"Google-Extended":["partial","*",16,0],"Applebot-Extended":["partial","*",16,0],"CCBot":["partial","*",16,0],"Bytespider":["partial","*",16,0],"Amazonbot":["partial","*",16,0],"meta-externalagent":["partial","*",16,0],"cohere-ai":["partial","*",16,0],"Diffbot":["partial","*",16,0],"ContextFetchBot":["partial","*",16,0]},"llms":{"status":"present","http":200,"bytes":10390},"home":{"status":"ok","http":200,"url":"https://www.psychologicalscience.org/","html_tokens":24636,"markdown_tokens":629,"pruned_tokens":109}},
{"rank":4022,"domain":"grow.google","origin":"https://grow.google","robots":{"status":"ok","http":200,"url":"https://grow.google/robots.txt","bytes":578,"groups":15},"bots":{"GPTBot":["allowed","named",0,1],"ChatGPT-User":["allowed","*",0,1],"OAI-SearchBot":["allowed","named",0,1],"ClaudeBot":["allowed","named",0,1],"Claude-User":["allowed","*",0,1],"Claude-SearchBot":["allowed","named",0,1],"anthropic-ai":["allowed","named",0,1],"PerplexityBot":["allowed","named",0,1],"Perplexity-User":["allowed","*",0,1],"Google-Extended":["allowed","named",0,1],"Applebot-Extended":["allowed","named",0,1],"CCBot":["allowed","named",0,1],"Bytespider":["allowed","*",0,1],"Amazonbot":["allowed","*",0,1],"meta-externalagent":["allowed","*",0,1],"cohere-ai":["allowed","named",0,1],"Diffbot":["allowed","named",0,1],"ContextFetchBot":["allowed","*",0,1]},"llms":{"status":"html","http":200,"bytes":337865},"home":{"status":"ok","http":200,"url":"https://grow.google/","html_tokens":84418,"markdown_tokens":173,"pruned_tokens":173}},
{"rank":4023,"domain":"daad.de","origin":"https://daad.de","robots":{"status":"ok","http":200,"url":"https://www.daad.de/robots.txt","bytes":3477,"groups":1},"bots":{"GPTBot":["partial","*",99,0],"ChatGPT-User":["partial","*",99,0],"OAI-SearchBot":["partial","*",99,0],"ClaudeBot":["partial","*",99,0],"Claude-User":["partial","*",99,0],"Claude-SearchBot":["partial","*",99,0],"anthropic-ai":["partial","*",99,0],"PerplexityBot":["partial","*",99,0],"Perplexity-User":["partial","*",99,0],"Google-Extended":["partial","*",99,0],"Applebot-Extended":["partial","*",99,0],"CCBot":["partial","*",99,0],"Bytespider":["partial","*",99,0],"Amazonbot":["partial","*",99,0],"meta-externalagent":["partial","*",99,0],"cohere-ai":["partial","*",99,0],"Diffbot":["partial","*",99,0],"ContextFetchBot":["partial","*",99,0]},"llms":{"status":"missing","http":404,"bytes":139282},"home":{"status":"ok","http":200,"url":"https://www.daad.de/de/","html_tokens":62673,"markdown_tokens":385,"pruned_tokens":158}},
{"rank":4024,"domain":"suicidepreventionlifeline.org","origin":"https://suicidepreventionlifeline.org","robots":{"status":"ok","http":200,"url":"https://988lifeline.org/robots.txt","bytes":161,"groups":1},"bots":{"GPTBot":["partial","*",1,1],"ChatGPT-User":["partial","*",1,1],"OAI-SearchBot":["partial","*",1,1],"ClaudeBot":["partial","*",1,1],"Claude-User":["partial","*",1,1],"Claude-SearchBot":["partial","*",1,1],"anthropic-ai":["partial","*",1,1],"PerplexityBot":["partial","*",1,1],"Perplexity-User":["partial","*",1,1],"Google-Extended":["partial","*",1,1],"Applebot-Extended":["partial","*",1,1],"CCBot":["partial","*",1,1],"Bytespider":["partial","*",1,1],"Amazonbot":["partial","*",1,1],"meta-externalagent":["partial","*",1,1],"cohere-ai":["partial","*",1,1],"Diffbot":["partial","*",1,1],"ContextFetchBot":["partial","*",1,1]},"llms":{"status":"http_error","http":403,"bytes":5568},"home":{"status":"http_error","http":403,"url":"https://988lifeline.org/"}},
{"rank":4025,"domain":"maersk.com","origin":"https://maersk.com","robots":{"status":"ok","http":200,"url":"https://www.maersk.com/robots.txt","bytes":3916,"groups":3},"bots":{"GPTBot":["partial","*",110,23],"ChatGPT-User":["partial","*",110,23],"OAI-SearchBot":["partial","*",110,23],"ClaudeBot":["partial","*",110,23],"Claude-User":["partial","*",110,23],"Claude-SearchBot":["partial","*",110,23],"anthropic-ai":["partial","*",110,23],"PerplexityBot":["partial","*",110,23],"Perplexity-User":["partial","*",110,23],"Google-Extended":["partial","*",110,23],"Applebot-Extended":["partial","*",110,23],"CCBot":["partial","*",110,23],"Bytespider":["partial","*",110,23],"Amazonbot":["partial","*",110,23],"meta-externalagent":["partial","*",110,23],"cohere-ai":["partial","*",110,23],"Diffbot":["partial","*",110,23],"ContextFetchBot":["partial","*",110,23]},"llms":{"status":"present","http":200,"bytes":7284},"home":{"status":"ok","http":200,"url":"https://www.maersk.com/","html_tokens":19882,"markdown_tokens":394,"pruned_tokens":125}},
{"rank":4026,"domain":"detik.com","origin":"https://detik.com","robots":{"status":"ok","http":200,"url":"https://www.detik.com/robots.txt","bytes":400,"groups":3},"bots":{"GPTBot":["allowed","*",0,1],"ChatGPT-User":["allowed","*",0,1],"OAI-SearchBot":["allowed","*",0,1],"ClaudeBot":["allowed","*",0,1],"Claude-User":["allowed","*",0,1],"Claude-SearchBot":["allowed","*",0,1],"anthropic-ai":["allowed","*",0,1],"PerplexityBot":["allowed","*",0,1],"Perplexity-User":["allowed","*",0,1],"Google-Extended":["allowed","named",0,1],"Applebot-Extended":["allowed","*",0,1],"CCBot":["allowed","*",0,1],"Bytespider":["allowed","*",0,1],"Amazonbot":["allowed","*",0,1],"meta-externalagent":["allowed","*",0,1],"cohere-ai":["allowed","*",0,1],"Diffbot":["allowed","*",0,1],"ContextFetchBot":["allowed","*",0,1]},"llms":{"status":"missing","http":404,"bytes":146},"home":{"status":"ok","http":200,"url":"https://www.detik.com/","html_tokens":97893,"markdown_tokens":52,"pruned_tokens":52}},
{"rank":4027,"domain":"marieclaire.com","origin":"https://marieclaire.com","robots":{"status":"ok","http":200,"url":"https://www.marieclaire.com/robots.txt","bytes":1688,"groups":5},"bots":{"GPTBot":["partial","*",20,0],"ChatGPT-User":["partial","*",20,0],"OAI-SearchBot":["partial","*",20,0],"ClaudeBot":["partial","*",20,0],"Claude-User":["partial","*",20,0],"Claude-SearchBot":["partial","*",20,0],"anthropic-ai":["partial","*",20,0],"PerplexityBot":["partial","*",20,0],"Perplexity-User":["partial","*",20,0],"Google-Extended":["partial","*",20,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["partial","*",20,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["partial","*",20,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",20,0]},"llms":{"status":"missing","http":404,"bytes":918020},"home":{"status":"ok","http":200,"url":"https://www.marieclaire.com/","html_tokens":403129,"markdown_tokens":1938,"pruned_tokens":126}},
{"rank":4028,"domain":"zappos.com","origin":"https://zappos.com","robots":{"status":"ok","http":200,"url":"https://www.zappos.com/robots.txt","bytes":1209,"groups":1},"bots":{"GPTBot":["partial","*",27,0],"ChatGPT-User":["partial","*",27,0],"OAI-SearchBot":["partial","*",27,0],"ClaudeBot":["partial","*",27,0],"Claude-User":["partial","*",27,0],"Claude-SearchBot":["partial","*",27,0],"anthropic-ai":["partial","*",27,0],"PerplexityBot":["partial","*",27,0],"Perplexity-User":["partial","*",27,0],"Google-Extended":["partial","*",27,0],"Applebot-Extended":["partial","*",27,0],"CCBot":["partial","*",27,0],"Bytespider":["partial","*",27,0],"Amazonbot":["partial","*",27,0],"meta-externalagent":["partial","*",27,0],"cohere-ai":["partial","*",27,0],"Diffbot":["partial","*",27,0],"ContextFetchBot":["partial","*",27,0]},"llms":{"status":"missing","http":404,"bytes":239354},"home":{"status":"ok","http":200,"url":"https://www.zappos.com/","html_tokens":294971,"markdown_tokens":21,"pruned_tokens":21}},
{"rank":4029,"domain":"desmoinesregister.com","origin":"https://desmoinesregister.com","robots":{"status":"ok","http":200,"url":"https://www.desmoinesregister.com/robots.txt","bytes":12695,"groups":288},"bots":{"GPTBot":["partial","named",1,4],"ChatGPT-User":["partial","named",1,4],"OAI-SearchBot":["partial","named",1,4],"ClaudeBot":["blocked","named",1,0],"Claude-User":["blocked","named",1,0],"Claude-SearchBot":["blocked","named",1,0],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["partial","named",1,4],"Perplexity-User":["blocked","named",1,0],"Google-Extended":["partial","named",1,4],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",2,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",32,0]},"llms":{"status":"http_error","http":402,"bytes":254},"home":{"status":"http_error","http":402,"url":"https://www.desmoinesregister.com/"}},
{"rank":4030,"domain":"govtrack.us","origin":"https://govtrack.us","robots":{"status":"ok","http":200,"url":"https://govtrack.us/robots.txt","bytes":2676,"groups":32},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["allowed","*",0,1],"OAI-SearchBot":["allowed","*",0,1],"ClaudeBot":["blocked","named",1,0],"Claude-User":["allowed","*",0,1],"Claude-SearchBot":["allowed","*",0,1],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["allowed","*",0,1],"Perplexity-User":["allowed","*",0,1],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["allowed","*",0,1]},"llms":{"status":"http_error","http":403,"bytes":5083},"home":{"status":"http_error","http":403,"url":"https://govtrack.us/"}},
{"rank":4031,"domain":"kr.ua","origin":"http://kr.ua","robots":{"status":"missing","http":404,"url":"http://kr.ua/robots.txt","bytes":398,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":396},"home":{"status":"ok","http":200,"url":"http://kr.ua/","html_tokens":5265,"markdown_tokens":516,"pruned_tokens":57}},
{"rank":4032,"domain":"timesnownews.com","origin":"https://timesnownews.com","robots":{"status":"ok","http":200,"url":"https://www.timesnownews.com/robots.txt","bytes":1164,"groups":17},"bots":{"GPTBot":["allowed","named",0,1],"ChatGPT-User":["allowed","named",0,1],"OAI-SearchBot":["allowed","named",0,1],"ClaudeBot":["partial","*",6,1],"Claude-User":["partial","*",6,1],"Claude-SearchBot":["partial","*",6,1],"anthropic-ai":["partial","*",6,1],"PerplexityBot":["partial","*",6,1],"Perplexity-User":["partial","*",6,1],"Google-Extended":["partial","*",6,1],"Applebot-Extended":["partial","*",6,1],"CCBot":["partial","*",6,1],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["partial","*",6,1],"Diffbot":["partial","*",6,1],"ContextFetchBot":["partial","*",6,1]},"llms":{"status":"missing","http":404,"bytes":19289},"home":{"status":"ok","http":200,"url":"https://www.timesnownews.com/","html_tokens":210210,"markdown_tokens":138,"pruned_tokens":138}},
{"rank":4033,"domain":"mk.co.kr","origin":"https://mk.co.kr","robots":{"status":"ok","http":200,"url":"https://www.mk.co.kr/robots.txt","bytes":1217,"groups":3},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["allowed","*",0,1],"OAI-SearchBot":["allowed","*",0,1],"ClaudeBot":["blocked","named",1,0],"Claude-User":["allowed","*",0,1],"Claude-SearchBot":["allowed","*",0,1],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["allowed","*",0,1],"Perplexity-User":["allowed","*",0,1],"Google-Extended":["allowed","*",0,1],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["allowed","*",0,1]},"llms":{"status":"present","http":200,"bytes":8803},"home":{"status":"ok","http":200,"url":"https://www.mk.co.kr/","html_tokens":63718,"markdown_tokens":9,"pruned_tokens":9}},
{"rank":4034,"domain":"ultimate-guitar.com","origin":"https://ultimate-guitar.com","robots":{"status":"ok","http":200,"url":"https://www.ultimate-guitar.com/robots.txt","bytes":2184,"groups":4},"bots":{"GPTBot":["partial","*",35,0],"ChatGPT-User":["partial","*",35,0],"OAI-SearchBot":["partial","*",35,0],"ClaudeBot":["partial","*",35,0],"Claude-User":["partial","*",35,0],"Claude-SearchBot":["partial","*",35,0],"anthropic-ai":["partial","*",35,0],"PerplexityBot":["partial","*",35,0],"Perplexity-User":["partial","*",35,0],"Google-Extended":["partial","*",35,0],"Applebot-Extended":["partial","*",35,0],"CCBot":["partial","*",35,0],"Bytespider":["partial","*",35,0],"Amazonbot":["partial","*",35,0],"meta-externalagent":["partial","*",35,0],"cohere-ai":["partial","*",35,0],"Diffbot":["partial","*",35,0],"ContextFetchBot":["partial","*",35,0]},"llms":{"status":"missing","http":404,"bytes":3764},"home":{"status":"ok","http":200,"url":"https://www.ultimate-guitar.com/","html_tokens":43594,"markdown_tokens":0,"pruned_tokens":0}},
{"rank":4035,"domain":"store.shopping.yahoo.co.jp","origin":"https://store.shopping.yahoo.co.jp","robots":{"status":"ok","http":200,"url":"https://store.shopping.yahoo.co.jp/robots.txt","bytes":669,"groups":1},"bots":{"GPTBot":["partial","*",21,0],"ChatGPT-User":["partial","*",21,0],"OAI-SearchBot":["partial","*",21,0],"ClaudeBot":["partial","*",21,0],"Claude-User":["partial","*",21,0],"Claude-SearchBot":["partial","*",21,0],"anthropic-ai":["partial","*",21,0],"PerplexityBot":["partial","*",21,0],"Perplexity-User":["partial","*",21,0],"Google-Extended":["partial","*",21,0],"Applebot-Extended":["partial","*",21,0],"CCBot":["partial","*",21,0],"Bytespider":["partial","*",21,0],"Amazonbot":["partial","*",21,0],"meta-externalagent":["partial","*",21,0],"cohere-ai":["partial","*",21,0],"Diffbot":["partial","*",21,0],"ContextFetchBot":["partial","*",21,0]},"llms":{"status":"missing","http":404,"bytes":4065},"home":{"status":"ok","http":200,"url":"https://shopping.yahoo.co.jp/","html_tokens":655777,"markdown_tokens":8,"pruned_tokens":8}},
{"rank":4036,"domain":"mollie.com","origin":"https://mollie.com","robots":{"status":"ok","http":200,"url":"https://www.mollie.com/robots.txt","bytes":67,"groups":1},"bots":{"GPTBot":["allowed","*",0,1],"ChatGPT-User":["allowed","*",0,1],"OAI-SearchBot":["allowed","*",0,1],"ClaudeBot":["allowed","*",0,1],"Claude-User":["allowed","*",0,1],"Claude-SearchBot":["allowed","*",0,1],"anthropic-ai":["allowed","*",0,1],"PerplexityBot":["allowed","*",0,1],"Perplexity-User":["allowed","*",0,1],"Google-Extended":["allowed","*",0,1],"Applebot-Extended":["allowed","*",0,1],"CCBot":["allowed","*",0,1],"Bytespider":["allowed","*",0,1],"Amazonbot":["allowed","*",0,1],"meta-externalagent":["allowed","*",0,1],"cohere-ai":["allowed","*",0,1],"Diffbot":["allowed","*",0,1],"ContextFetchBot":["allowed","*",0,1]},"llms":{"status":"present","http":200,"bytes":55275},"home":{"status":"ok","http":200,"url":"https://www.mollie.com/","html_tokens":625264,"markdown_tokens":681,"pruned_tokens":102}},
{"rank":4037,"domain":"mediapart.fr","origin":"https://mediapart.fr","robots":{"status":"ok","http":200,"url":"https://www.mediapart.fr/robots.txt","bytes":14383,"groups":12},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["partial","*",318,2],"OAI-SearchBot":["partial","*",318,2],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",318,2],"Claude-SearchBot":["partial","*",318,2],"anthropic-ai":["partial","*",318,2],"PerplexityBot":["partial","*",318,2],"Perplexity-User":["partial","*",318,2],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["partial","*",318,2],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["partial","*",318,2],"Diffbot":["partial","*",318,2],"ContextFetchBot":["partial","*",318,2]},"llms":{"status":"missing","http":404,"bytes":123563},"home":{"status":"ok","http":200,"url":"https://www.mediapart.fr/","html_tokens":349796,"markdown_tokens":67,"pruned_tokens":67}},
{"rank":4038,"domain":"fox.com","origin":"https://fox.com","robots":{"status":"ok","http":200,"url":"https://www.fox.com/robots.txt","bytes":449,"groups":1},"bots":{"GPTBot":["partial","*",1,1],"ChatGPT-User":["partial","*",1,1],"OAI-SearchBot":["partial","*",1,1],"ClaudeBot":["partial","*",1,1],"Claude-User":["partial","*",1,1],"Claude-SearchBot":["partial","*",1,1],"anthropic-ai":["partial","*",1,1],"PerplexityBot":["partial","*",1,1],"Perplexity-User":["partial","*",1,1],"Google-Extended":["partial","*",1,1],"Applebot-Extended":["partial","*",1,1],"CCBot":["partial","*",1,1],"Bytespider":["partial","*",1,1],"Amazonbot":["partial","*",1,1],"meta-externalagent":["partial","*",1,1],"cohere-ai":["partial","*",1,1],"Diffbot":["partial","*",1,1],"ContextFetchBot":["partial","*",1,1]},"llms":{"status":"present","http":200,"bytes":1861},"home":{"status":"ok","http":200,"url":"https://www.fox.com/","html_tokens":75606,"markdown_tokens":584,"pruned_tokens":174}},
{"rank":4039,"domain":"zomato.com","origin":"https://zomato.com","robots":{"status":"unreachable","http":null,"url":"https://zomato.com/robots.txt","bytes":0,"groups":0,"error":"<StreamReset stream_id:1, error_code:2, remote_reset:True>"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":4040,"domain":"uni-lj.si","origin":"https://www.uni-lj.si","robots":{"status":"ok","http":200,"url":"https://www.uni-lj.si/robots.txt","bytes":31,"groups":1},"bots":{"GPTBot":["partial","*",1,0],"ChatGPT-User":["partial","*",1,0],"OAI-SearchBot":["partial","*",1,0],"ClaudeBot":["partial","*",1,0],"Claude-User":["partial","*",1,0],"Claude-SearchBot":["partial","*",1,0],"anthropic-ai":["partial","*",1,0],"PerplexityBot":["partial","*",1,0],"Perplexity-User":["partial","*",1,0],"Google-Extended":["partial","*",1,0],"Applebot-Extended":["partial","*",1,0],"CCBot":["partial","*",1,0],"Bytespider":["partial","*",1,0],"Amazonbot":["partial","*",1,0],"meta-externalagent":["partial","*",1,0],"cohere-ai":["partial","*",1,0],"Diffbot":["partial","*",1,0],"ContextFetchBot":["partial","*",1,0]},"llms":{"status":"missing","http":404,"bytes":33007},"home":{"status":"ok","http":200,"url":"https://www.uni-lj.si/","html_tokens":20073,"markdown_tokens":428,"pruned_tokens":164}},
{"rank":4041,"domain":"acpjournals.org","origin":"https://acpjournals.org","robots":{"status":"ok","http":200,"url":"https://www.acpjournals.org/robots.txt","bytes":964,"groups":6},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["partial","*",17,7],"OAI-SearchBot":["partial","*",17,7],"ClaudeBot":["partial","*",17,7],"Claude-User":["partial","*",17,7],"Claude-SearchBot":["partial","*",17,7],"anthropic-ai":["partial","*",17,7],"PerplexityBot":["partial","*",17,7],"Perplexity-User":["partial","*",17,7],"Google-Extended":["partial","*",17,7],"Applebot-Extended":["blocked","named",1,0],"CCBot":["partial","*",17,7],"Bytespider":["partial","*",17,7],"Amazonbot":["partial","*",17,7],"meta-externalagent":["partial","*",17,7],"cohere-ai":["partial","*",17,7],"Diffbot":["partial","*",17,7],"ContextFetchBot":["partial","*",17,7]},"llms":{"status":"http_error","http":403,"bytes":5568},"home":{"status":"http_error","http":403,"url":"https://acpjournals.org/"}},
{"rank":4042,"domain":"dovepress.com","origin":"https://dovepress.com","robots":{"status":"ok","http":200,"url":"https://www.dovepress.com/robots.txt","bytes":905,"groups":1},"bots":{"GPTBot":["partial","*",10,0],"ChatGPT-User":["partial","*",10,0],"OAI-SearchBot":["partial","*",10,0],"ClaudeBot":["partial","*",10,0],"Claude-User":["partial","*",10,0],"Claude-SearchBot":["partial","*",10,0],"anthropic-ai":["partial","*",10,0],"PerplexityBot":["partial","*",10,0],"Perplexity-User":["partial","*",10,0],"Google-Extended":["partial","*",10,0],"Applebot-Extended":["partial","*",10,0],"CCBot":["partial","*",10,0],"Bytespider":["partial","*",10,0],"Amazonbot":["partial","*",10,0],"meta-externalagent":["partial","*",10,0],"cohere-ai":["partial","*",10,0],"Diffbot":["partial","*",10,0],"ContextFetchBot":["partial","*",10,0]},"llms":{"status":"missing","http":404,"bytes":29772},"home":{"status":"ok","http":200,"url":"https://www.dovepress.com/","html_tokens":13928,"markdown_tokens":449,"pruned_tokens":158}},
{"rank":4043,"domain":"ucsusa.org","origin":"https://ucsusa.org","robots":{"status":"ok","http":200,"url":"https://www.ucs.org/robots.txt","bytes":2188,"groups":1},"bots":{"GPTBot":["partial","*",33,18],"ChatGPT-User":["partial","*",33,18],"OAI-SearchBot":["partial","*",33,18],"ClaudeBot":["partial","*",33,18],"Claude-User":["partial","*",33,18],"Claude-SearchBot":["partial","*",33,18],"anthropic-ai":["partial","*",33,18],"PerplexityBot":["partial","*",33,18],"Perplexity-User":["partial","*",33,18],"Google-Extended":["partial","*",33,18],"Applebot-Extended":["partial","*",33,18],"CCBot":["partial","*",33,18],"Bytespider":["partial","*",33,18],"Amazonbot":["partial","*",33,18],"meta-externalagent":["partial","*",33,18],"cohere-ai":["partial","*",33,18],"Diffbot":["partial","*",33,18],"ContextFetchBot":["partial","*",33,18]},"llms":{"status":"missing","http":404,"bytes":185},"home":{"status":"ok","http":200,"url":"https://www.ucs.org/","html_tokens":36048,"markdown_tokens":65,"pruned_tokens":65}},
{"rank":4044,"domain":"newsobserver.com","origin":"https://newsobserver.com","robots":{"status":"unreachable","http":null,"url":"https://newsobserver.com/robots.txt","bytes":0,"groups":0,"error":"<StreamReset stream_id:1, error_code:2, remote_reset:True>"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":4045,"domain":"qub.ac.uk","origin":"https://qub.ac.uk","robots":{"status":"refused","http":403,"url":"https://www.qub.ac.uk/robots.txt","bytes":919,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":4046,"domain":"isweb.co.kr","origin":"https://isweb.co.kr","robots":{"status":"ok","http":200,"url":"https://www.isweb.co.kr/robots.txt","bytes":69,"groups":1},"bots":{"GPTBot":["allowed","*",0,1],"ChatGPT-User":["allowed","*",0,1],"OAI-SearchBot":["allowed","*",0,1],"ClaudeBot":["allowed","*",0,1],"Claude-User":["allowed","*",0,1],"Claude-SearchBot":["allowed","*",0,1],"anthropic-ai":["allowed","*",0,1],"PerplexityBot":["allowed","*",0,1],"Perplexity-User":["allowed","*",0,1],"Google-Extended":["allowed","*",0,1],"Applebot-Extended":["allowed","*",0,1],"CCBot":["allowed","*",0,1],"Bytespider":["allowed","*",0,1],"Amazonbot":["allowed","*",0,1],"meta-externalagent":["allowed","*",0,1],"cohere-ai":["allowed","*",0,1],"Diffbot":["allowed","*",0,1],"ContextFetchBot":["allowed","*",0,1]},"llms":{"status":"missing","http":404,"bytes":1954},"home":{"status":"ok","http":200,"url":"https://www.isweb.co.kr/","html_tokens":38396,"markdown_tokens":228,"pruned_tokens":118}},
{"rank":4047,"domain":"rawstory.com","origin":"https://rawstory.com","robots":{"status":"refused","http":403,"url":"https://rawstory.com/robots.txt","bytes":4545,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":4048,"domain":"ebay.com.au","origin":"https://ebay.com.au","robots":{"status":"ok","http":200,"url":"https://www.ebay.com.au/robots.txt","bytes":17232,"groups":7},"bots":{"GPTBot":["partial","named",1,3],"ChatGPT-User":["partial","*",190,8],"OAI-SearchBot":["partial","*",190,8],"ClaudeBot":["partial","named",1,3],"Claude-User":["partial","*",190,8],"Claude-SearchBot":["partial","*",190,8],"anthropic-ai":["partial","named",1,3],"PerplexityBot":["partial","named",1,3],"Perplexity-User":["partial","*",190,8],"Google-Extended":["partial","*",190,8],"Applebot-Extended":["partial","named",1,3],"CCBot":["partial","named",1,3],"Bytespider":["partial","named",1,3],"Amazonbot":["partial","named",1,3],"meta-externalagent":["partial","named",1,3],"cohere-ai":["partial","*",190,8],"Diffbot":["partial","*",190,8],"ContextFetchBot":["partial","*",190,8]},"llms":{"status":"missing","http":404,"bytes":78775},"home":{"status":"http_error","http":403,"url":"https://www.ebay.com.au/"}},
{"rank":4049,"domain":"globalsecurity.org","origin":"https://globalsecurity.org","robots":{"status":"ok","http":200,"url":"https://globalsecurity.org/robots.txt","bytes":3623,"groups":26},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["partial","*",3,1],"OAI-SearchBot":["partial","*",3,1],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",3,1],"Claude-SearchBot":["partial","*",3,1],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["partial","*",3,1],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",3,1]},"llms":{"status":"missing","http":404,"bytes":648},"home":{"status":"ok","http":200,"url":"https://globalsecurity.org/","html_tokens":8203,"markdown_tokens":170,"pruned_tokens":170}},
{"rank":4050,"domain":"uoguelph.ca","origin":"https://uoguelph.ca","robots":{"status":"ok","http":200,"url":"https://www.uoguelph.ca/robots.txt","bytes":401,"groups":1},"bots":{"GPTBot":["partial","*",10,0],"ChatGPT-User":["partial","*",10,0],"OAI-SearchBot":["partial","*",10,0],"ClaudeBot":["partial","*",10,0],"Claude-User":["partial","*",10,0],"Claude-SearchBot":["partial","*",10,0],"anthropic-ai":["partial","*",10,0],"PerplexityBot":["partial","*",10,0],"Perplexity-User":["partial","*",10,0],"Google-Extended":["partial","*",10,0],"Applebot-Extended":["partial","*",10,0],"CCBot":["partial","*",10,0],"Bytespider":["partial","*",10,0],"Amazonbot":["partial","*",10,0],"meta-externalagent":["partial","*",10,0],"cohere-ai":["partial","*",10,0],"Diffbot":["partial","*",10,0],"ContextFetchBot":["partial","*",10,0]},"llms":{"status":"missing","http":404,"bytes":45777},"home":{"status":"ok","http":200,"url":"https://www.uoguelph.ca/","html_tokens":16872,"markdown_tokens":37,"pruned_tokens":37}},
{"rank":4051,"domain":"reg.ru","origin":"https://reg.ru","robots":{"status":"html","http":200,"url":"https://reg.ru/robots.txt","bytes":13602,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":4052,"domain":"wsws.org","origin":"https://wsws.org","robots":{"status":"ok","http":200,"url":"https://www.wsws.org/robots.txt","bytes":156,"groups":1},"bots":{"GPTBot":["partial","*",4,0],"ChatGPT-User":["partial","*",4,0],"OAI-SearchBot":["partial","*",4,0],"ClaudeBot":["partial","*",4,0],"Claude-User":["partial","*",4,0],"Claude-SearchBot":["partial","*",4,0],"anthropic-ai":["partial","*",4,0],"PerplexityBot":["partial","*",4,0],"Perplexity-User":["partial","*",4,0],"Google-Extended":["partial","*",4,0],"Applebot-Extended":["partial","*",4,0],"CCBot":["partial","*",4,0],"Bytespider":["partial","*",4,0],"Amazonbot":["partial","*",4,0],"meta-externalagent":["partial","*",4,0],"cohere-ai":["partial","*",4,0],"Diffbot":["partial","*",4,0],"ContextFetchBot":["partial","*",4,0]},"llms":{"status":"missing","http":404,"bytes":99942},"home":{"status":"ok","http":200,"url":"https://www.wsws.org/en?redirect=true","html_tokens":532928,"markdown_tokens":10864,"pruned_tokens":315}},
{"rank":4053,"domain":"diepresse.com","origin":"https://diepresse.com","robots":{"status":"ok","http":200,"url":"https://www.diepresse.com/robots.txt","bytes":1774,"groups":25},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["blocked","named",1,0],"OAI-SearchBot":["partial","*",14,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",14,0],"Claude-SearchBot":["partial","*",14,0],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["partial","*",14,0],"Google-Extended":["partial","*",14,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["partial","*",14,0],"meta-externalagent":["partial","*",14,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",14,0]},"llms":{"status":"missing","http":404,"bytes":187240},"home":{"status":"ok","http":200,"url":"https://www.diepresse.com/","html_tokens":179402,"markdown_tokens":725,"pruned_tokens":144}},
{"rank":4054,"domain":"gab.com","origin":"https://gab.com","robots":{"status":"ok","http":200,"url":"https://gab.com/robots.txt","bytes":866,"groups":1},"bots":{"GPTBot":["partial","*",25,0],"ChatGPT-User":["partial","*",25,0],"OAI-SearchBot":["partial","*",25,0],"ClaudeBot":["partial","*",25,0],"Claude-User":["partial","*",25,0],"Claude-SearchBot":["partial","*",25,0],"anthropic-ai":["partial","*",25,0],"PerplexityBot":["partial","*",25,0],"Perplexity-User":["partial","*",25,0],"Google-Extended":["partial","*",25,0],"Applebot-Extended":["partial","*",25,0],"CCBot":["partial","*",25,0],"Bytespider":["partial","*",25,0],"Amazonbot":["partial","*",25,0],"meta-externalagent":["partial","*",25,0],"cohere-ai":["partial","*",25,0],"Diffbot":["partial","*",25,0],"ContextFetchBot":["partial","*",25,0]},"llms":{"status":"missing","http":404,"bytes":0},"home":{"status":"ok","http":200,"url":"https://gab.com/","html_tokens":28324,"markdown_tokens":1151,"pruned_tokens":553}},
{"rank":4055,"domain":"irishexaminer.com","origin":"https://irishexaminer.com","robots":{"status":"ok","http":200,"url":"https://www.irishexaminer.com/robots.txt","bytes":2651,"groups":39},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["blocked","named",1,0],"OAI-SearchBot":["partial","*",25,0],"ClaudeBot":["partial","*",25,0],"Claude-User":["partial","*",25,0],"Claude-SearchBot":["partial","*",25,0],"anthropic-ai":["partial","*",25,0],"PerplexityBot":["partial","*",25,0],"Perplexity-User":["partial","*",25,0],"Google-Extended":["partial","*",25,0],"Applebot-Extended":["partial","*",25,0],"CCBot":["blocked","named",1,0],"Bytespider":["partial","*",25,0],"Amazonbot":["partial","*",25,0],"meta-externalagent":["partial","*",25,0],"cohere-ai":["partial","*",25,0],"Diffbot":["partial","*",25,0],"ContextFetchBot":["partial","*",25,0]},"llms":{"status":"missing","http":404,"bytes":0},"home":{"status":"ok","http":200,"url":"https://www.irishexaminer.com/","html_tokens":42335,"markdown_tokens":85,"pruned_tokens":85}},
{"rank":4056,"domain":"peraichi.com","origin":"https://peraichi.com","robots":{"status":"ok","http":200,"url":"https://peraichi.com/robots.txt","bytes":159,"groups":1},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"timeout","error":"timed out after 10 s"},"home":{"status":"ok","http":200,"url":"https://peraichi.com/","html_tokens":28387,"markdown_tokens":343,"pruned_tokens":39}},
{"rank":4057,"domain":"expressen.se","origin":"https://expressen.se","robots":{"status":"ok","http":200,"url":"https://www.expressen.se/robots.txt","bytes":4940,"groups":32},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["blocked","named",1,0],"OAI-SearchBot":["blocked","named",1,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["blocked","named",1,0],"Claude-SearchBot":["blocked","named",1,0],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["blocked","named",1,0],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["partial","*",11,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",11,0]},"llms":{"status":"http_error","http":400,"bytes":13743},"home":{"status":"ok","http":200,"url":"https://www.expressen.se/","html_tokens":78770,"markdown_tokens":26,"pruned_tokens":26}},
{"rank":4058,"domain":"xtemos.com","origin":"https://xtemos.com","robots":{"status":"ok","http":200,"url":"https://xtemos.com/robots.txt","bytes":806,"groups":2},"bots":{"GPTBot":["partial","*",16,1],"ChatGPT-User":["partial","*",16,1],"OAI-SearchBot":["partial","*",16,1],"ClaudeBot":["partial","*",16,1],"Claude-User":["partial","*",16,1],"Claude-SearchBot":["partial","*",16,1],"anthropic-ai":["partial","*",16,1],"PerplexityBot":["partial","*",16,1],"Perplexity-User":["partial","*",16,1],"Google-Extended":["partial","*",16,1],"Applebot-Extended":["partial","*",16,1],"CCBot":["partial","*",16,1],"Bytespider":["partial","*",16,1],"Amazonbot":["partial","*",16,1],"meta-externalagent":["partial","*",16,1],"cohere-ai":["partial","*",16,1],"Diffbot":["partial","*",16,1],"ContextFetchBot":["partial","*",16,1]},"llms":{"status":"other_text","http":200,"bytes":7512},"home":{"status":"ok","http":200,"url":"https://xtemos.com/","html_tokens":55860,"markdown_tokens":906,"pruned_tokens":85}},
{"rank":4059,"domain":"pipedrive.com","origin":"https://pipedrive.com","robots":{"status":"ok","http":200,"url":"https://www.pipedrive.com/robots.txt","bytes":187,"groups":1},"bots":{"GPTBot":["partial","*",5,0],"ChatGPT-User":["partial","*",5,0],"OAI-SearchBot":["partial","*",5,0],"ClaudeBot":["partial","*",5,0],"Claude-User":["partial","*",5,0],"Claude-SearchBot":["partial","*",5,0],"anthropic-ai":["partial","*",5,0],"PerplexityBot":["partial","*",5,0],"Perplexity-User":["partial","*",5,0],"Google-Extended":["partial","*",5,0],"Applebot-Extended":["partial","*",5,0],"CCBot":["partial","*",5,0],"Bytespider":["partial","*",5,0],"Amazonbot":["partial","*",5,0],"meta-externalagent":["partial","*",5,0],"cohere-ai":["partial","*",5,0],"Diffbot":["partial","*",5,0],"ContextFetchBot":["partial","*",5,0]},"llms":{"status":"present","http":200,"bytes":9143},"home":{"status":"ok","http":200,"url":"https://www.pipedrive.com/","html_tokens":104606,"markdown_tokens":1638,"pruned_tokens":442}},
{"rank":4060,"domain":"osaka-u.ac.jp","origin":"https://www.osaka-u.ac.jp","robots":{"status":"ok","http":200,"url":"https://www.osaka-u.ac.jp/robots.txt","bytes":832,"groups":1},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":83434},"home":{"status":"ok","http":200,"url":"https://www.osaka-u.ac.jp/ja","html_tokens":30776,"markdown_tokens":48,"pruned_tokens":48}},
{"rank":4061,"domain":"dejure.org","origin":"https://dejure.org","robots":{"status":"ok","http":200,"url":"https://dejure.org/robots.txt","bytes":5698,"groups":32},"bots":{"GPTBot":["partial","*",48,0],"ChatGPT-User":["partial","*",48,0],"OAI-SearchBot":["partial","*",48,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",48,0],"Claude-SearchBot":["blocked","named",1,0],"anthropic-ai":["partial","*",48,0],"PerplexityBot":["partial","*",48,0],"Perplexity-User":["partial","*",48,0],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["partial","*",48,0],"CCBot":["blocked","named",2,0],"Bytespider":["partial","*",48,0],"Amazonbot":["partial","*",48,0],"meta-externalagent":["partial","*",48,0],"cohere-ai":["partial","*",48,0],"Diffbot":["partial","*",48,0],"ContextFetchBot":["partial","*",48,0]},"llms":{"status":"missing","http":404,"bytes":15228},"home":{"status":"ok","http":200,"url":"https://dejure.org/","html_tokens":30302,"markdown_tokens":5658,"pruned_tokens":163}},
{"rank":4062,"domain":"epi.org","origin":"https://epi.org","robots":{"status":"ok","http":200,"url":"https://www.epi.org/robots.txt","bytes":127,"groups":1},"bots":{"GPTBot":["partial","*",1,1],"ChatGPT-User":["partial","*",1,1],"OAI-SearchBot":["partial","*",1,1],"ClaudeBot":["partial","*",1,1],"Claude-User":["partial","*",1,1],"Claude-SearchBot":["partial","*",1,1],"anthropic-ai":["partial","*",1,1],"PerplexityBot":["partial","*",1,1],"Perplexity-User":["partial","*",1,1],"Google-Extended":["partial","*",1,1],"Applebot-Extended":["partial","*",1,1],"CCBot":["partial","*",1,1],"Bytespider":["partial","*",1,1],"Amazonbot":["partial","*",1,1],"meta-externalagent":["partial","*",1,1],"cohere-ai":["partial","*",1,1],"Diffbot":["partial","*",1,1],"ContextFetchBot":["partial","*",1,1]},"llms":{"status":"timeout","error":"timed out after 10 s"},"home":{"status":"http_error","http":403,"url":"https://epi.org/"}},
{"rank":4063,"domain":"xunlei.com","origin":"https://xunlei.com","robots":{"status":"ok","http":200,"url":"https://www.xunlei.com/robots.txt","bytes":98,"groups":1},"bots":{"GPTBot":["partial","*",5,0],"ChatGPT-User":["partial","*",5,0],"OAI-SearchBot":["partial","*",5,0],"ClaudeBot":["partial","*",5,0],"Claude-User":["partial","*",5,0],"Claude-SearchBot":["partial","*",5,0],"anthropic-ai":["partial","*",5,0],"PerplexityBot":["partial","*",5,0],"Perplexity-User":["partial","*",5,0],"Google-Extended":["partial","*",5,0],"Applebot-Extended":["partial","*",5,0],"CCBot":["partial","*",5,0],"Bytespider":["partial","*",5,0],"Amazonbot":["partial","*",5,0],"meta-externalagent":["partial","*",5,0],"cohere-ai":["partial","*",5,0],"Diffbot":["partial","*",5,0],"ContextFetchBot":["partial","*",5,0]},"llms":{"status":"html","http":200,"bytes":43594},"home":{"status":"ok","http":200,"url":"https://www.xunlei.com/","html_tokens":10505,"markdown_tokens":18,"pruned_tokens":18}},
{"rank":4064,"domain":"citizensadvice.org.uk","origin":"https://citizensadvice.org.uk","robots":{"status":"ok","http":200,"url":"https://www.citizensadvice.org.uk/robots.txt","bytes":99,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":19158},"home":{"status":"ok","http":200,"url":"https://www.citizensadvice.org.uk/","html_tokens":6673,"markdown_tokens":134,"pruned_tokens":77}},
{"rank":4065,"domain":"sizzling-hot-deluxe-777.com","origin":"https://sizzling-hot-deluxe-777.com","robots":{"status":"ok","http":200,"url":"https://sizzling-hot-deluxe-777.com/robots.txt","bytes":1248,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":45066},"home":{"status":"ok","http":200,"url":"https://sizzling-hot-deluxe-777.com/","html_tokens":17704,"markdown_tokens":198,"pruned_tokens":115}},
{"rank":4066,"domain":"threadless.com","origin":"https://threadless.com","robots":{"status":"ok","http":200,"url":"https://www.threadless.com/robots.txt","bytes":5761,"groups":14},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["partial","*",56,1],"OAI-SearchBot":["partial","*",56,1],"ClaudeBot":["partial","*",56,1],"Claude-User":["partial","*",56,1],"Claude-SearchBot":["partial","*",56,1],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["partial","*",56,1],"Perplexity-User":["partial","*",56,1],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["partial","*",56,1],"CCBot":["blocked","named",1,0],"Bytespider":["partial","*",56,1],"Amazonbot":["partial","*",56,1],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["partial","*",56,1],"Diffbot":["partial","*",56,1],"ContextFetchBot":["partial","*",56,1]},"llms":{"status":"missing","http":404,"bytes":207900},"home":{"status":"http_error","http":403,"url":"https://www.threadless.com/"}},
{"rank":4067,"domain":"aaos.org","origin":"https://aaos.org","robots":{"status":"ok","http":200,"url":"https://www.aaos.org/robots.txt","bytes":100,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":40293},"home":{"status":"ok","http":200,"url":"https://www.aaos.org/","html_tokens":16191,"markdown_tokens":297,"pruned_tokens":161}},
{"rank":4068,"domain":"investor.gov","origin":"https://investor.gov","robots":{"status":"refused","http":403,"url":"https://investor.gov/robots.txt","bytes":376,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":4069,"domain":"nc.gov","origin":"https://nc.gov","robots":{"status":"ok","http":200,"url":"https://www.nc.gov/robots.txt","bytes":64,"groups":1},"bots":{"GPTBot":["allowed","*",0,1],"ChatGPT-User":["allowed","*",0,1],"OAI-SearchBot":["allowed","*",0,1],"ClaudeBot":["allowed","*",0,1],"Claude-User":["allowed","*",0,1],"Claude-SearchBot":["allowed","*",0,1],"anthropic-ai":["allowed","*",0,1],"PerplexityBot":["allowed","*",0,1],"Perplexity-User":["allowed","*",0,1],"Google-Extended":["allowed","*",0,1],"Applebot-Extended":["allowed","*",0,1],"CCBot":["allowed","*",0,1],"Bytespider":["allowed","*",0,1],"Amazonbot":["allowed","*",0,1],"meta-externalagent":["allowed","*",0,1],"cohere-ai":["allowed","*",0,1],"Diffbot":["allowed","*",0,1],"ContextFetchBot":["allowed","*",0,1]},"llms":{"status":"present","http":200,"bytes":1498},"home":{"status":"ok","http":200,"url":"https://www.nc.gov/","html_tokens":19481,"markdown_tokens":256,"pruned_tokens":2}},
{"rank":4070,"domain":"active.com","origin":"https://active.com","robots":{"status":"ok","http":200,"url":"https://www.active.com/robots.txt","bytes":931,"groups":1},"bots":{"GPTBot":["partial","*",25,0],"ChatGPT-User":["partial","*",25,0],"OAI-SearchBot":["partial","*",25,0],"ClaudeBot":["partial","*",25,0],"Claude-User":["partial","*",25,0],"Claude-SearchBot":["partial","*",25,0],"anthropic-ai":["partial","*",25,0],"PerplexityBot":["partial","*",25,0],"Perplexity-User":["partial","*",25,0],"Google-Extended":["partial","*",25,0],"Applebot-Extended":["partial","*",25,0],"CCBot":["partial","*",25,0],"Bytespider":["partial","*",25,0],"Amazonbot":["partial","*",25,0],"meta-externalagent":["partial","*",25,0],"cohere-ai":["partial","*",25,0],"Diffbot":["partial","*",25,0],"ContextFetchBot":["partial","*",25,0]},"llms":{"status":"present","http":200,"bytes":10975},"home":{"status":"ok","http":200,"url":"https://www.active.com/","html_tokens":60521,"markdown_tokens":94,"pruned_tokens":75}},
{"rank":4071,"domain":"uts.edu.au","origin":"https://uts.edu.au","robots":{"status":"ok","http":200,"url":"https://www.uts.edu.au/robots.txt","bytes":109,"groups":1},"bots":{"GPTBot":["partial","*",2,1],"ChatGPT-User":["partial","*",2,1],"OAI-SearchBot":["partial","*",2,1],"ClaudeBot":["partial","*",2,1],"Claude-User":["partial","*",2,1],"Claude-SearchBot":["partial","*",2,1],"anthropic-ai":["partial","*",2,1],"PerplexityBot":["partial","*",2,1],"Perplexity-User":["partial","*",2,1],"Google-Extended":["partial","*",2,1],"Applebot-Extended":["partial","*",2,1],"CCBot":["partial","*",2,1],"Bytespider":["partial","*",2,1],"Amazonbot":["partial","*",2,1],"meta-externalagent":["partial","*",2,1],"cohere-ai":["partial","*",2,1],"Diffbot":["partial","*",2,1],"ContextFetchBot":["partial","*",2,1]},"llms":{"status":"missing","http":404,"bytes":98814},"home":{"status":"ok","http":200,"url":"https://www.uts.edu.au/","html_tokens":42116,"markdown_tokens":399,"pruned_tokens":130}},
{"rank":4072,"domain":"biblehub.com","origin":"https://biblehub.com","robots":{"status":"missing","http":404,"url":"https://biblehub.com/robots.txt","bytes":991,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":991},"home":{"status":"ok","http":200,"url":"https://biblehub.com/","html_tokens":16275,"markdown_tokens":474,"pruned_tokens":157}},
{"rank":4073,"domain":"brandeis.edu","origin":"https://brandeis.edu","robots":{"status":"refused","http":403,"url":"https://brandeis.edu/robots.txt","bytes":118,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":4074,"domain":"autonews.com","origin":"https://autonews.com","robots":{"status":"refused","http":403,"url":"https://www.autonews.com/robots.txt","bytes":384,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":4075,"domain":"phoronix.com","origin":"https://phoronix.com","robots":{"status":"refused","http":403,"url":"https://phoronix.com/robots.txt","bytes":5550,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":4076,"domain":"hikvision.com","origin":"https://www.hikvision.com","robots":{"status":"ok","http":200,"url":"https://www.hikvision.com/robots.txt","bytes":2978,"groups":10},"bots":{"GPTBot":["allowed","named",0,1],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","named",0,1],"ClaudeBot":["allowed","named",0,1],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","named",0,1],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","named",0,1],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","named",0,1],"Amazonbot":["allowed","named",0,1],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"present","http":200,"bytes":4279},"home":{"status":"error","http":200,"url":"https://www.hikvision.com/","error":"unparseable HTML"}},
{"rank":4077,"domain":"visual.ly","origin":"https://visual.ly","robots":{"status":"refused","http":403,"url":"https://visual.ly/robots.txt","bytes":5508,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":4078,"domain":"online.fr","origin":"https://online.fr","robots":{"status":"missing","http":404,"url":"https://www.scaleway.com/fr/dedibox/robots.txt","bytes":429267,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":429267},"home":{"status":"ok","http":200,"url":"https://www.scaleway.com/fr/dedibox/","html_tokens":131490,"markdown_tokens":1553,"pruned_tokens":128}},
{"rank":4079,"domain":"your-server.de","origin":"https://your-server.de","robots":{"status":"missing","http":404,"url":"https://your-server.de/robots.txt","bytes":301,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":301},"home":{"status":"http_error","http":404,"url":"https://your-server.de/"}},
{"rank":4080,"domain":"nikkei.co.jp","origin":"https://www.nikkei.co.jp","robots":{"status":"ok","http":200,"url":"https://www.nikkei.co.jp/robots.txt","bytes":1167,"groups":29},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["blocked","named",1,0],"OAI-SearchBot":["blocked","named",1,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":1290},"home":{"status":"ok","http":200,"url":"https://www.nikkei.com/","html_tokens":168563,"markdown_tokens":151,"pruned_tokens":151}},
{"rank":4081,"domain":"blick.ch","origin":"https://blick.ch","robots":{"status":"ok","http":200,"url":"https://www.blick.ch/robots.txt","bytes":1422,"groups":8},"bots":{"GPTBot":["partial","*",20,0],"ChatGPT-User":["partial","*",20,0],"OAI-SearchBot":["partial","*",20,0],"ClaudeBot":["allowed","named",0,1],"Claude-User":["partial","*",20,0],"Claude-SearchBot":["partial","*",20,0],"anthropic-ai":["partial","*",20,0],"PerplexityBot":["partial","*",20,0],"Perplexity-User":["partial","*",20,0],"Google-Extended":["partial","*",20,0],"Applebot-Extended":["partial","*",20,0],"CCBot":["partial","*",20,0],"Bytespider":["partial","*",20,0],"Amazonbot":["partial","*",20,0],"meta-externalagent":["partial","*",20,0],"cohere-ai":["partial","*",20,0],"Diffbot":["partial","*",20,0],"ContextFetchBot":["partial","*",20,0]},"llms":{"status":"missing","http":404,"bytes":222679},"home":{"status":"ok","http":200,"url":"https://www.blick.ch/","html_tokens":359613,"markdown_tokens":92,"pruned_tokens":92}},
{"rank":4082,"domain":"osti.gov","origin":"https://osti.gov","robots":{"status":"ok","http":200,"url":"https://www.osti.gov/robots.txt","bytes":2548,"groups":9},"bots":{"GPTBot":["partial","*",14,0],"ChatGPT-User":["partial","*",14,0],"OAI-SearchBot":["partial","*",14,0],"ClaudeBot":["partial","*",14,0],"Claude-User":["partial","*",14,0],"Claude-SearchBot":["partial","*",14,0],"anthropic-ai":["partial","*",14,0],"PerplexityBot":["partial","*",14,0],"Perplexity-User":["partial","*",14,0],"Google-Extended":["partial","*",14,0],"Applebot-Extended":["partial","*",14,0],"CCBot":["partial","*",14,0],"Bytespider":["partial","*",14,0],"Amazonbot":["partial","*",14,0],"meta-externalagent":["partial","*",14,0],"cohere-ai":["partial","*",14,0],"Diffbot":["partial","*",14,0],"ContextFetchBot":["partial","*",14,0]},"llms":{"status":"missing","http":404,"bytes":264830},"home":{"status":"ok","http":200,"url":"https://www.osti.gov/","html_tokens":65916,"markdown_tokens":269,"pruned_tokens":141}},
{"rank":4083,"domain":"irfanview.com","origin":"https://irfanview.com","robots":{"status":"ok","http":200,"url":"https://www.irfanview.com/robots.txt","bytes":51,"groups":1},"bots":{"GPTBot":["partial","*",2,0],"ChatGPT-User":["partial","*",2,0],"OAI-SearchBot":["partial","*",2,0],"ClaudeBot":["partial","*",2,0],"Claude-User":["partial","*",2,0],"Claude-SearchBot":["partial","*",2,0],"anthropic-ai":["partial","*",2,0],"PerplexityBot":["partial","*",2,0],"Perplexity-User":["partial","*",2,0],"Google-Extended":["partial","*",2,0],"Applebot-Extended":["partial","*",2,0],"CCBot":["partial","*",2,0],"Bytespider":["partial","*",2,0],"Amazonbot":["partial","*",2,0],"meta-externalagent":["partial","*",2,0],"cohere-ai":["partial","*",2,0],"Diffbot":["partial","*",2,0],"ContextFetchBot":["partial","*",2,0]},"llms":{"status":"missing","http":404,"bytes":13},"home":{"status":"ok","http":200,"url":"https://www.irfanview.com/","html_tokens":2952,"markdown_tokens":45,"pruned_tokens":45}},
{"rank":4084,"domain":"framer.website","origin":"https://framer.website","robots":{"status":"html","http":200,"url":"https://www.framer.com","bytes":1909650,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":4085,"domain":"rkn.gov.ru","origin":"https://rkn.gov.ru","robots":{"status":"timeout","http":null,"url":"https://rkn.gov.ru/robots.txt","bytes":0,"groups":0,"error":"ConnectTimeout"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":4086,"domain":"stripes.com","origin":"https://stripes.com","robots":{"status":"ok","http":200,"url":"https://www.stripes.com/robots.txt","bytes":1307,"groups":7},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["blocked","named",1,0],"OAI-SearchBot":["partial","*",11,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",11,0],"Claude-SearchBot":["partial","*",11,0],"anthropic-ai":["partial","*",11,0],"PerplexityBot":["partial","*",11,0],"Perplexity-User":["partial","*",11,0],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["partial","*",11,0],"CCBot":["partial","*",11,0],"Bytespider":["partial","*",11,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["partial","*",11,0],"cohere-ai":["partial","*",11,0],"Diffbot":["partial","*",11,0],"ContextFetchBot":["partial","*",11,0]},"llms":{"status":"missing","http":404,"bytes":2929},"home":{"status":"ok","http":200,"url":"https://www.stripes.com/","html_tokens":44404,"markdown_tokens":212,"pruned_tokens":53}},
{"rank":4087,"domain":"dea.gov","origin":"https://dea.gov","robots":{"status":"refused","http":403,"url":"https://dea.gov/robots.txt","bytes":371,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":4088,"domain":"statesman.com","origin":"https://statesman.com","robots":{"status":"ok","http":200,"url":"https://www.statesman.com/robots.txt","bytes":5822,"groups":83},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["blocked","named",1,0],"OAI-SearchBot":["blocked","named",1,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["blocked","named",1,0],"Claude-SearchBot":["blocked","named",1,0],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["partial","*",90,0],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["partial","*",90,0],"meta-externalagent":["blocked","named",2,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",90,0]},"llms":{"status":"missing","http":404,"bytes":61567},"home":{"status":"ok","http":200,"url":"https://www.statesman.com/","html_tokens":221929,"markdown_tokens":22,"pruned_tokens":22}},
{"rank":4089,"domain":"pubhtml5.com","origin":"https://pubhtml5.com","robots":{"status":"ok","http":200,"url":"https://pubhtml5.com/robots.txt","bytes":9570,"groups":150},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["partial","*",14,10],"OAI-SearchBot":["partial","*",14,10],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",14,10],"Claude-SearchBot":["partial","*",14,10],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["partial","*",14,10],"Perplexity-User":["partial","*",14,10],"Google-Extended":["partial","*",14,10],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",2,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",14,10]},"llms":{"status":"missing","http":404,"bytes":27094},"home":{"status":"ok","http":200,"url":"https://pubhtml5.com/","html_tokens":21944,"markdown_tokens":1461,"pruned_tokens":706}},
{"rank":4090,"domain":"gob.ve","origin":"https://gob.ve","robots":{"status":"unreachable","http":null,"url":"https://gob.ve/robots.txt","bytes":0,"groups":0,"error":"cannot resolve gob.ve"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":4091,"domain":"google.bg","origin":"https://google.bg","robots":{"status":"ok","http":200,"url":"https://www.google.bg/robots.txt","bytes":6624,"groups":4},"bots":{"GPTBot":["partial","*",169,64],"ChatGPT-User":["partial","*",169,64],"OAI-SearchBot":["partial","*",169,64],"ClaudeBot":["partial","*",169,64],"Claude-User":["partial","*",169,64],"Claude-SearchBot":["partial","*",169,64],"anthropic-ai":["partial","*",169,64],"PerplexityBot":["partial","*",169,64],"Perplexity-User":["partial","*",169,64],"Google-Extended":["partial","*",169,64],"Applebot-Extended":["partial","*",169,64],"CCBot":["partial","*",169,64],"Bytespider":["partial","*",169,64],"Amazonbot":["partial","*",169,64],"meta-externalagent":["partial","*",169,64],"cohere-ai":["partial","*",169,64],"Diffbot":["partial","*",169,64],"ContextFetchBot":["partial","*",169,64]},"llms":{"status":"missing","http":404,"bytes":1569},"home":{"status":"ok","http":200,"url":"https://www.google.com/","html_tokens":21294,"markdown_tokens":30,"pruned_tokens":30}},
{"rank":4092,"domain":"k-state.edu","origin":"https://k-state.edu","robots":{"status":"ok","http":200,"url":"https://www.k-state.edu/robots.txt","bytes":4170,"groups":1},"bots":{"GPTBot":["partial","*",101,0],"ChatGPT-User":["partial","*",101,0],"OAI-SearchBot":["partial","*",101,0],"ClaudeBot":["partial","*",101,0],"Claude-User":["partial","*",101,0],"Claude-SearchBot":["partial","*",101,0],"anthropic-ai":["partial","*",101,0],"PerplexityBot":["partial","*",101,0],"Perplexity-User":["partial","*",101,0],"Google-Extended":["partial","*",101,0],"Applebot-Extended":["partial","*",101,0],"CCBot":["partial","*",101,0],"Bytespider":["partial","*",101,0],"Amazonbot":["partial","*",101,0],"meta-externalagent":["partial","*",101,0],"cohere-ai":["partial","*",101,0],"Diffbot":["partial","*",101,0],"ContextFetchBot":["partial","*",101,0]},"llms":{"status":"missing","http":404,"bytes":42112},"home":{"status":"ok","http":200,"url":"https://www.k-state.edu/","html_tokens":20048,"markdown_tokens":725,"pruned_tokens":330}},
{"rank":4093,"domain":"moodys.com","origin":"https://moodys.com","robots":{"status":"ok","http":200,"url":"https://www.moodys.com/robots.txt","bytes":860,"groups":1},"bots":{"GPTBot":["partial","*",14,0],"ChatGPT-User":["partial","*",14,0],"OAI-SearchBot":["partial","*",14,0],"ClaudeBot":["partial","*",14,0],"Claude-User":["partial","*",14,0],"Claude-SearchBot":["partial","*",14,0],"anthropic-ai":["partial","*",14,0],"PerplexityBot":["partial","*",14,0],"Perplexity-User":["partial","*",14,0],"Google-Extended":["partial","*",14,0],"Applebot-Extended":["partial","*",14,0],"CCBot":["partial","*",14,0],"Bytespider":["partial","*",14,0],"Amazonbot":["partial","*",14,0],"meta-externalagent":["partial","*",14,0],"cohere-ai":["partial","*",14,0],"Diffbot":["partial","*",14,0],"ContextFetchBot":["partial","*",14,0]},"llms":{"status":"html","http":200,"bytes":2773},"home":{"status":"ok","http":200,"url":"https://www.moodys.com/","html_tokens":34452,"markdown_tokens":495,"pruned_tokens":35}},
{"rank":4094,"domain":"webshots.com","origin":"https://webshots.com","robots":{"status":"ok","http":200,"url":"https://webshots.com/robots.txt","bytes":65,"groups":1},"bots":{"GPTBot":["allowed","*",0,1],"ChatGPT-User":["allowed","*",0,1],"OAI-SearchBot":["allowed","*",0,1],"ClaudeBot":["allowed","*",0,1],"Claude-User":["allowed","*",0,1],"Claude-SearchBot":["allowed","*",0,1],"anthropic-ai":["allowed","*",0,1],"PerplexityBot":["allowed","*",0,1],"Perplexity-User":["allowed","*",0,1],"Google-Extended":["allowed","*",0,1],"Applebot-Extended":["allowed","*",0,1],"CCBot":["allowed","*",0,1],"Bytespider":["allowed","*",0,1],"Amazonbot":["allowed","*",0,1],"meta-externalagent":["allowed","*",0,1],"cohere-ai":["allowed","*",0,1],"Diffbot":["allowed","*",0,1],"ContextFetchBot":["allowed","*",0,1]},"llms":{"status":"missing","http":404,"bytes":22261},"home":{"status":"ok","http":200,"url":"https://webshots.com/","html_tokens":14821,"markdown_tokens":729,"pruned_tokens":239}},
{"rank":4095,"domain":"unicamp.br","origin":"https://unicamp.br","robots":{"status":"ok","http":200,"url":"https://www.unicamp.br/robots.txt","bytes":5758,"groups":2},"bots":{"GPTBot":["partial","*",10,1],"ChatGPT-User":["partial","*",10,1],"OAI-SearchBot":["partial","*",10,1],"ClaudeBot":["partial","*",10,1],"Claude-User":["partial","*",10,1],"Claude-SearchBot":["partial","*",10,1],"anthropic-ai":["partial","*",10,1],"PerplexityBot":["partial","*",10,1],"Perplexity-User":["partial","*",10,1],"Google-Extended":["partial","*",10,1],"Applebot-Extended":["partial","*",10,1],"CCBot":["partial","*",10,1],"Bytespider":["partial","*",10,1],"Amazonbot":["partial","*",10,1],"meta-externalagent":["partial","*",10,1],"cohere-ai":["partial","*",10,1],"Diffbot":["partial","*",10,1],"ContextFetchBot":["partial","*",10,1]},"llms":{"status":"missing","http":404,"bytes":130819},"home":{"status":"ok","http":200,"url":"https://www.unicamp.br/","html_tokens":65360,"markdown_tokens":69,"pruned_tokens":69}},
{"rank":4096,"domain":"snowflake.com","origin":"https://snowflake.com","robots":{"status":"ok","http":200,"url":"https://www.snowflake.com/robots.txt","bytes":2805,"groups":1},"bots":{"GPTBot":["partial","*",34,4],"ChatGPT-User":["partial","*",34,4],"OAI-SearchBot":["partial","*",34,4],"ClaudeBot":["partial","*",34,4],"Claude-User":["partial","*",34,4],"Claude-SearchBot":["partial","*",34,4],"anthropic-ai":["partial","*",34,4],"PerplexityBot":["partial","*",34,4],"Perplexity-User":["partial","*",34,4],"Google-Extended":["partial","*",34,4],"Applebot-Extended":["partial","*",34,4],"CCBot":["partial","*",34,4],"Bytespider":["partial","*",34,4],"Amazonbot":["partial","*",34,4],"meta-externalagent":["partial","*",34,4],"cohere-ai":["partial","*",34,4],"Diffbot":["partial","*",34,4],"ContextFetchBot":["partial","*",34,4]},"llms":{"status":"missing","http":404,"bytes":60},"home":{"status":"ok","http":200,"url":"https://www.snowflake.com/en/","html_tokens":230827,"markdown_tokens":379,"pruned_tokens":170}},
{"rank":4097,"domain":"toronto.ca","origin":"https://toronto.ca","robots":{"status":"ok","http":200,"url":"https://www.toronto.ca/robots.txt","bytes":3572,"groups":2},"bots":{"GPTBot":["partial","*",87,0],"ChatGPT-User":["partial","*",87,0],"OAI-SearchBot":["partial","*",87,0],"ClaudeBot":["partial","*",87,0],"Claude-User":["partial","*",87,0],"Claude-SearchBot":["partial","*",87,0],"anthropic-ai":["partial","*",87,0],"PerplexityBot":["partial","*",87,0],"Perplexity-User":["partial","*",87,0],"Google-Extended":["partial","*",87,0],"Applebot-Extended":["partial","*",87,0],"CCBot":["partial","*",87,0],"Bytespider":["partial","*",87,0],"Amazonbot":["partial","*",87,0],"meta-externalagent":["partial","*",87,0],"cohere-ai":["partial","*",87,0],"Diffbot":["partial","*",87,0],"ContextFetchBot":["partial","*",87,0]},"llms":{"status":"missing","http":404,"bytes":26238},"home":{"status":"ok","http":200,"url":"https://www.toronto.ca/","html_tokens":10577,"markdown_tokens":18,"pruned_tokens":18}},
{"rank":4098,"domain":"adv.br","origin":"https://adv.br","robots":{"status":"unreachable","http":null,"url":"https://adv.br/robots.txt","bytes":0,"groups":0,"error":"cannot resolve adv.br"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":4099,"domain":"myswitzerland.com","origin":"https://myswitzerland.com","robots":{"status":"ok","http":200,"url":"https://www.myswitzerland.com/robots.txt","bytes":596,"groups":4},"bots":{"GPTBot":["partial","*",12,0],"ChatGPT-User":["partial","*",12,0],"OAI-SearchBot":["partial","*",12,0],"ClaudeBot":["partial","*",12,0],"Claude-User":["partial","*",12,0],"Claude-SearchBot":["partial","*",12,0],"anthropic-ai":["partial","*",12,0],"PerplexityBot":["partial","*",12,0],"Perplexity-User":["partial","*",12,0],"Google-Extended":["partial","*",12,0],"Applebot-Extended":["partial","*",12,0],"CCBot":["partial","*",12,0],"Bytespider":["partial","*",12,0],"Amazonbot":["partial","*",12,0],"meta-externalagent":["partial","*",12,0],"cohere-ai":["partial","*",12,0],"Diffbot":["partial","*",12,0],"ContextFetchBot":["partial","*",12,0]},"llms":{"status":"http_error","http":406,"bytes":0},"home":{"status":"http_error","http":406,"url":"https://www.myswitzerland.com/"}},
{"rank":4100,"domain":"wi.us","origin":"https://wi.us","robots":{"status":"unreachable","http":null,"url":"https://wi.us/robots.txt","bytes":0,"groups":0,"error":"cannot resolve wi.us"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":4101,"domain":"hostelworld.com","origin":"https://hostelworld.com","robots":{"status":"ok","http":200,"url":"https://www.hostelworld.com/robots.txt","bytes":1119,"groups":3},"bots":{"GPTBot":["partial","*",38,0],"ChatGPT-User":["partial","*",38,0],"OAI-SearchBot":["partial","*",38,0],"ClaudeBot":["partial","*",38,0],"Claude-User":["partial","*",38,0],"Claude-SearchBot":["partial","*",38,0],"anthropic-ai":["partial","*",38,0],"PerplexityBot":["partial","*",38,0],"Perplexity-User":["partial","*",38,0],"Google-Extended":["partial","*",38,0],"Applebot-Extended":["partial","*",38,0],"CCBot":["partial","*",38,0],"Bytespider":["partial","*",38,0],"Amazonbot":["partial","*",38,0],"meta-externalagent":["partial","*",38,0],"cohere-ai":["partial","*",38,0],"Diffbot":["partial","*",38,0],"ContextFetchBot":["partial","*",38,0]},"llms":{"status":"present","http":200,"bytes":26260},"home":{"status":"ok","http":200,"url":"https://www.hostelworld.com/","html_tokens":77159,"markdown_tokens":156,"pruned_tokens":44}},
{"rank":4102,"domain":"gatesnotes.com","origin":"https://www.gatesnotes.com","robots":{"status":"ok","http":200,"url":"https://www.gatesnotes.com/robots.txt","bytes":88,"groups":1},"bots":{"GPTBot":["partial","*",1,1],"ChatGPT-User":["partial","*",1,1],"OAI-SearchBot":["partial","*",1,1],"ClaudeBot":["partial","*",1,1],"Claude-User":["partial","*",1,1],"Claude-SearchBot":["partial","*",1,1],"anthropic-ai":["partial","*",1,1],"PerplexityBot":["partial","*",1,1],"Perplexity-User":["partial","*",1,1],"Google-Extended":["partial","*",1,1],"Applebot-Extended":["partial","*",1,1],"CCBot":["partial","*",1,1],"Bytespider":["partial","*",1,1],"Amazonbot":["partial","*",1,1],"meta-externalagent":["partial","*",1,1],"cohere-ai":["partial","*",1,1],"Diffbot":["partial","*",1,1],"ContextFetchBot":["partial","*",1,1]},"llms":{"status":"http_error","http":403,"bytes":382},"home":{"status":"http_error","http":403,"url":"https://www.gatesnotes.com/"}},
{"rank":4103,"domain":"digicert.com","origin":"https://digicert.com","robots":{"status":"ok","http":200,"url":"https://www.digicert.com/robots.txt","bytes":2657,"groups":23},"bots":{"GPTBot":["allowed","named",0,1],"ChatGPT-User":["allowed","named",0,1],"OAI-SearchBot":["allowed","named",0,1],"ClaudeBot":["allowed","named",0,1],"Claude-User":["allowed","named",0,1],"Claude-SearchBot":["allowed","named",0,1],"anthropic-ai":["partial","*",30,2],"PerplexityBot":["allowed","named",0,1],"Perplexity-User":["allowed","named",0,1],"Google-Extended":["allowed","named",0,1],"Applebot-Extended":["partial","*",30,2],"CCBot":["partial","*",30,2],"Bytespider":["partial","*",30,2],"Amazonbot":["partial","*",30,2],"meta-externalagent":["partial","*",30,2],"cohere-ai":["partial","*",30,2],"Diffbot":["partial","*",30,2],"ContextFetchBot":["partial","*",30,2]},"llms":{"status":"present","http":200,"bytes":8395},"home":{"status":"ok","http":200,"url":"https://www.digicert.com/","html_tokens":261,"markdown_tokens":21,"pruned_tokens":21}},
{"rank":4104,"domain":"casino50.com","origin":"https://casino50.com","robots":{"status":"unreachable","http":null,"url":"https://casino50.com/robots.txt","bytes":0,"groups":0,"error":"cannot resolve casino50.com"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":4105,"domain":"london.gov.uk","origin":"https://london.gov.uk","robots":{"status":"ok","http":200,"url":"https://www.london.gov.uk/robots.txt","bytes":5232,"groups":1},"bots":{"GPTBot":["partial","*",95,45],"ChatGPT-User":["partial","*",95,45],"OAI-SearchBot":["partial","*",95,45],"ClaudeBot":["partial","*",95,45],"Claude-User":["partial","*",95,45],"Claude-SearchBot":["partial","*",95,45],"anthropic-ai":["partial","*",95,45],"PerplexityBot":["partial","*",95,45],"Perplexity-User":["partial","*",95,45],"Google-Extended":["partial","*",95,45],"Applebot-Extended":["partial","*",95,45],"CCBot":["partial","*",95,45],"Bytespider":["partial","*",95,45],"Amazonbot":["partial","*",95,45],"meta-externalagent":["partial","*",95,45],"cohere-ai":["partial","*",95,45],"Diffbot":["partial","*",95,45],"ContextFetchBot":["partial","*",95,45]},"llms":{"status":"present","http":200,"bytes":5125},"home":{"status":"ok","http":200,"url":"https://www.london.gov.uk/","html_tokens":79111,"markdown_tokens":86,"pruned_tokens":63}},
{"rank":4106,"domain":"almanac.com","origin":"https://almanac.com","robots":{"status":"ok","http":200,"url":"https://www.almanac.com/robots.txt","bytes":2916,"groups":3},"bots":{"GPTBot":["allowed","named",0,1],"ChatGPT-User":["partial","*",39,19],"OAI-SearchBot":["allowed","named",0,1],"ClaudeBot":["partial","*",39,19],"Claude-User":["partial","*",39,19],"Claude-SearchBot":["partial","*",39,19],"anthropic-ai":["partial","*",39,19],"PerplexityBot":["partial","*",39,19],"Perplexity-User":["partial","*",39,19],"Google-Extended":["partial","*",39,19],"Applebot-Extended":["partial","*",39,19],"CCBot":["partial","*",39,19],"Bytespider":["partial","*",39,19],"Amazonbot":["partial","*",39,19],"meta-externalagent":["partial","*",39,19],"cohere-ai":["partial","*",39,19],"Diffbot":["partial","*",39,19],"ContextFetchBot":["partial","*",39,19]},"llms":{"status":"missing","http":404,"bytes":189},"home":{"status":"ok","http":200,"url":"https://www.almanac.com/","html_tokens":30194,"markdown_tokens":187,"pruned_tokens":115}},
{"rank":4107,"domain":"taipeitimes.com","origin":"https://taipeitimes.com","robots":{"status":"ok","http":200,"url":"https://www.taipeitimes.com/robots.txt","bytes":752,"groups":3},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["partial","*",1,0],"OAI-SearchBot":["partial","*",1,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",1,0],"Claude-SearchBot":["partial","*",1,0],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["partial","*",1,0],"Perplexity-User":["partial","*",1,0],"Google-Extended":["partial","named",1,1],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["partial","*",1,0],"meta-externalagent":["partial","named",1,1],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",1,0]},"llms":{"status":"missing","http":404,"bytes":27545},"home":{"status":"ok","http":200,"url":"https://www.taipeitimes.com/","html_tokens":20466,"markdown_tokens":3276,"pruned_tokens":75}},
{"rank":4108,"domain":"iherb.com","origin":"https://iherb.com","robots":{"status":"ok","http":200,"url":"https://www.iherb.com/robots.txt","bytes":3540,"groups":2},"bots":{"GPTBot":["partial","*",135,4],"ChatGPT-User":["partial","*",135,4],"OAI-SearchBot":["partial","*",135,4],"ClaudeBot":["partial","*",135,4],"Claude-User":["partial","*",135,4],"Claude-SearchBot":["partial","*",135,4],"anthropic-ai":["partial","*",135,4],"PerplexityBot":["partial","*",135,4],"Perplexity-User":["partial","*",135,4],"Google-Extended":["partial","*",135,4],"Applebot-Extended":["partial","*",135,4],"CCBot":["partial","*",135,4],"Bytespider":["partial","*",135,4],"Amazonbot":["partial","*",135,4],"meta-externalagent":["partial","*",135,4],"cohere-ai":["partial","*",135,4],"Diffbot":["partial","*",135,4],"ContextFetchBot":["partial","*",135,4]},"llms":{"status":"missing","http":404,"bytes":11705},"home":{"status":"http_error","http":403,"url":"https://iherb.com/"}},
{"rank":4109,"domain":"fineartamerica.com","origin":"https://fineartamerica.com","robots":{"status":"ok","http":200,"url":"https://fineartamerica.com/robots.txt","bytes":3826,"groups":1},"bots":{"GPTBot":["partial","*",81,30],"ChatGPT-User":["partial","*",81,30],"OAI-SearchBot":["partial","*",81,30],"ClaudeBot":["partial","*",81,30],"Claude-User":["partial","*",81,30],"Claude-SearchBot":["partial","*",81,30],"anthropic-ai":["partial","*",81,30],"PerplexityBot":["partial","*",81,30],"Perplexity-User":["partial","*",81,30],"Google-Extended":["partial","*",81,30],"Applebot-Extended":["partial","*",81,30],"CCBot":["partial","*",81,30],"Bytespider":["partial","*",81,30],"Amazonbot":["partial","*",81,30],"meta-externalagent":["partial","*",81,30],"cohere-ai":["partial","*",81,30],"Diffbot":["partial","*",81,30],"ContextFetchBot":["partial","*",81,30]},"llms":{"status":"missing","http":404,"bytes":128541},"home":{"status":"ok","http":200,"url":"https://fineartamerica.com/","html_tokens":90824,"markdown_tokens":936,"pruned_tokens":114}},
{"rank":4110,"domain":"ssl-images-amazon.com","origin":"https://ssl-images-amazon.com","robots":{"status":"unreachable","http":null,"url":"https://ssl-images-amazon.com/robots.txt","bytes":0,"groups":0,"error":"cannot resolve ssl-images-amazon.com"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":4111,"domain":"google.cl","origin":"https://google.cl","robots":{"status":"ok","http":200,"url":"https://www.google.cl/robots.txt","bytes":6624,"groups":4},"bots":{"GPTBot":["partial","*",169,64],"ChatGPT-User":["partial","*",169,64],"OAI-SearchBot":["partial","*",169,64],"ClaudeBot":["partial","*",169,64],"Claude-User":["partial","*",169,64],"Claude-SearchBot":["partial","*",169,64],"anthropic-ai":["partial","*",169,64],"PerplexityBot":["partial","*",169,64],"Perplexity-User":["partial","*",169,64],"Google-Extended":["partial","*",169,64],"Applebot-Extended":["partial","*",169,64],"CCBot":["partial","*",169,64],"Bytespider":["partial","*",169,64],"Amazonbot":["partial","*",169,64],"meta-externalagent":["partial","*",169,64],"cohere-ai":["partial","*",169,64],"Diffbot":["partial","*",169,64],"ContextFetchBot":["partial","*",169,64]},"llms":{"status":"missing","http":404,"bytes":1569},"home":{"status":"ok","http":200,"url":"https://www.google.com/","html_tokens":21347,"markdown_tokens":30,"pruned_tokens":30}},
{"rank":4112,"domain":"instyle.com","origin":"https://instyle.com","robots":{"status":"ok","http":200,"url":"https://www.instyle.com/robots.txt","bytes":3467,"groups":6},"bots":{"GPTBot":["partial","named",1,0],"ChatGPT-User":["partial","named",1,0],"OAI-SearchBot":["partial","named",1,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["blocked","named",1,0],"Claude-SearchBot":["partial","named",1,2],"anthropic-ai":["partial","named",1,2],"PerplexityBot":["partial","named",1,2],"Perplexity-User":["blocked","named",1,0],"Google-Extended":["partial","named",1,2],"Applebot-Extended":["blocked","named",1,0],"CCBot":["partial","named",1,2],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["partial","named",1,2],"cohere-ai":["partial","named",1,2],"Diffbot":["partial","*",2,0],"ContextFetchBot":["partial","*",2,0]},"llms":{"status":"http_error","http":402,"bytes":612},"home":{"status":"http_error","http":402,"url":"https://www.instyle.com/"}},
{"rank":4113,"domain":"giantbomb.com","origin":"https://giantbomb.com","robots":{"status":"ok","http":200,"url":"https://giantbomb.com/robots.txt","bytes":3744,"groups":33},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["partial","*",12,1],"OAI-SearchBot":["partial","*",12,1],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",12,1],"Claude-SearchBot":["partial","*",12,1],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["partial","*",12,1],"Perplexity-User":["partial","*",12,1],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",12,1]},"llms":{"status":"missing","http":404,"bytes":41656},"home":{"status":"http_error","http":403,"url":"https://giantbomb.com/"}},
{"rank":4114,"domain":"abc7ny.com","origin":"https://abc7ny.com","robots":{"status":"ok","http":200,"url":"https://abc7ny.com/robots.txt","bytes":1125,"groups":26},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["blocked","named",1,0],"OAI-SearchBot":["partial","*",7,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",7,0],"Claude-SearchBot":["partial","*",7,0],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["partial","*",7,0],"Perplexity-User":["partial","*",7,0],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["partial","*",7,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",7,0]},"llms":{"status":"html","http":200,"bytes":73668},"home":{"status":"ok","http":200,"url":"https://abc7ny.com/","html_tokens":129149,"markdown_tokens":100,"pruned_tokens":100}},
{"rank":4115,"domain":"datacamp.com","origin":"https://datacamp.com","robots":{"status":"refused","http":403,"url":"https://www.datacamp.com/robots.txt","bytes":5583,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":4116,"domain":"unblog.fr","origin":"http://unblog.fr","robots":{"status":"ok","http":200,"url":"http://unblog.fr/robots.txt","bytes":2739,"groups":55},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["blocked","named",1,0],"OAI-SearchBot":["blocked","named",1,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",12,0],"Claude-SearchBot":["partial","*",12,0],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["partial","*",12,0],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["partial","*",12,0],"Diffbot":["partial","*",12,0],"ContextFetchBot":["partial","*",12,0]},"llms":{"status":"html","http":200,"bytes":46101},"home":{"status":"ok","http":200,"url":"http://unblog.fr/","html_tokens":11482,"markdown_tokens":620,"pruned_tokens":125}},
{"rank":4117,"domain":"bham.ac.uk","origin":"https://www.bham.ac.uk","robots":{"status":"ok","http":200,"url":"https://www.birmingham.ac.uk/robots.txt","bytes":1476,"groups":2},"bots":{"GPTBot":["partial","*",33,1],"ChatGPT-User":["partial","*",33,1],"OAI-SearchBot":["partial","*",33,1],"ClaudeBot":["partial","*",33,1],"Claude-User":["partial","*",33,1],"Claude-SearchBot":["partial","*",33,1],"anthropic-ai":["partial","*",33,1],"PerplexityBot":["partial","*",33,1],"Perplexity-User":["partial","*",33,1],"Google-Extended":["partial","*",33,1],"Applebot-Extended":["partial","*",33,1],"CCBot":["partial","*",33,1],"Bytespider":["partial","*",33,1],"Amazonbot":["partial","*",33,1],"meta-externalagent":["partial","*",33,1],"cohere-ai":["partial","*",33,1],"Diffbot":["partial","*",33,1],"ContextFetchBot":["partial","*",33,1]},"llms":{"status":"missing","http":404,"bytes":14},"home":{"status":"ok","http":200,"url":"https://www.birmingham.ac.uk/","html_tokens":49687,"markdown_tokens":164,"pruned_tokens":48}},
{"rank":4118,"domain":"eastday.com","origin":"https://eastday.com","robots":{"status":"missing","http":404,"url":"https://eastday.com/robots.txt","bytes":7702,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"present","http":200,"bytes":186},"home":{"status":"ok","http":200,"url":"https://eastday.com/","html_tokens":8618,"markdown_tokens":124,"pruned_tokens":124}},
{"rank":4119,"domain":"sky.it","origin":"https://sky.it","robots":{"status":"ok","http":200,"url":"https://www.sky.it/robots.txt","bytes":656,"groups":1},"bots":{"GPTBot":["partial","*",16,3],"ChatGPT-User":["partial","*",16,3],"OAI-SearchBot":["partial","*",16,3],"ClaudeBot":["partial","*",16,3],"Claude-User":["partial","*",16,3],"Claude-SearchBot":["partial","*",16,3],"anthropic-ai":["partial","*",16,3],"PerplexityBot":["partial","*",16,3],"Perplexity-User":["partial","*",16,3],"Google-Extended":["partial","*",16,3],"Applebot-Extended":["partial","*",16,3],"CCBot":["partial","*",16,3],"Bytespider":["partial","*",16,3],"Amazonbot":["partial","*",16,3],"meta-externalagent":["partial","*",16,3],"cohere-ai":["partial","*",16,3],"Diffbot":["partial","*",16,3],"ContextFetchBot":["partial","*",16,3]},"llms":{"status":"missing","http":404,"bytes":325800},"home":{"status":"ok","http":200,"url":"https://www.sky.it/","html_tokens":136288,"markdown_tokens":1046,"pruned_tokens":121}},
{"rank":4120,"domain":"larazon.es","origin":"https://larazon.es","robots":{"status":"ok","http":200,"url":"https://www.larazon.es/robots.txt","bytes":3942,"groups":103},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["partial","*",6,2],"OAI-SearchBot":["partial","*",6,2],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",6,2],"Claude-SearchBot":["partial","*",6,2],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["partial","*",6,2],"Perplexity-User":["partial","*",6,2],"Google-Extended":["allowed","named",0,1],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["partial","*",6,2],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["partial","*",6,2],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",6,2]},"llms":{"status":"missing","http":404,"bytes":651354},"home":{"status":"ok","http":200,"url":"https://www.larazon.es/","html_tokens":274926,"markdown_tokens":159,"pruned_tokens":159}},
{"rank":4121,"domain":"sipri.org","origin":"https://sipri.org","robots":{"status":"ok","http":200,"url":"https://www.sipri.org/robots.txt","bytes":2268,"groups":2},"bots":{"GPTBot":["partial","*",36,18],"ChatGPT-User":["partial","*",36,18],"OAI-SearchBot":["partial","*",36,18],"ClaudeBot":["partial","*",36,18],"Claude-User":["partial","*",36,18],"Claude-SearchBot":["partial","*",36,18],"anthropic-ai":["partial","*",36,18],"PerplexityBot":["partial","*",36,18],"Perplexity-User":["partial","*",36,18],"Google-Extended":["partial","*",36,18],"Applebot-Extended":["partial","*",36,18],"CCBot":["partial","*",36,18],"Bytespider":["partial","*",36,18],"Amazonbot":["partial","*",36,18],"meta-externalagent":["partial","*",36,18],"cohere-ai":["partial","*",36,18],"Diffbot":["partial","*",36,18],"ContextFetchBot":["partial","*",36,18]},"llms":{"status":"missing","http":404,"bytes":187},"home":{"status":"ok","http":200,"url":"https://www.sipri.org/","html_tokens":18096,"markdown_tokens":26,"pruned_tokens":26}},
{"rank":4122,"domain":"la.gov","origin":"https://la.gov","robots":{"status":"missing","http":404,"url":"https://www.louisiana.gov/robots.txt","bytes":1606,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":1606},"home":{"status":"ok","http":200,"url":"https://www.louisiana.gov/","html_tokens":108881,"markdown_tokens":91,"pruned_tokens":91}},
{"rank":4123,"domain":"ksl.com","origin":"https://ksl.com","robots":{"status":"ok","http":200,"url":"https://www.ksl.com/robots.txt","bytes":603,"groups":4},"bots":{"GPTBot":["partial","*",12,2],"ChatGPT-User":["partial","*",12,2],"OAI-SearchBot":["partial","*",12,2],"ClaudeBot":["partial","*",12,2],"Claude-User":["partial","*",12,2],"Claude-SearchBot":["partial","*",12,2],"anthropic-ai":["partial","*",12,2],"PerplexityBot":["partial","*",12,2],"Perplexity-User":["partial","*",12,2],"Google-Extended":["partial","*",12,2],"Applebot-Extended":["partial","*",12,2],"CCBot":["partial","*",12,2],"Bytespider":["partial","*",12,2],"Amazonbot":["partial","*",12,2],"meta-externalagent":["partial","*",12,2],"cohere-ai":["partial","*",12,2],"Diffbot":["partial","*",12,2],"ContextFetchBot":["partial","*",12,2]},"llms":{"status":"html","http":200,"bytes":0},"home":{"status":"ok","http":200,"url":"https://www.ksl.com/","html_tokens":61180,"markdown_tokens":1368,"pruned_tokens":147}},
{"rank":4124,"domain":"mobygames.com","origin":"https://mobygames.com","robots":{"status":"ok","http":200,"url":"https://mobygames.com/robots.txt","bytes":2676,"groups":32},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["allowed","*",0,1],"OAI-SearchBot":["allowed","*",0,1],"ClaudeBot":["blocked","named",1,0],"Claude-User":["allowed","*",0,1],"Claude-SearchBot":["allowed","*",0,1],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["allowed","*",0,1],"Perplexity-User":["allowed","*",0,1],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["allowed","*",0,1]},"llms":{"status":"http_error","http":403,"bytes":5524},"home":{"status":"http_error","http":403,"url":"https://mobygames.com/"}},
{"rank":4125,"domain":"housebeautiful.com","origin":"https://housebeautiful.com","robots":{"status":"ok","http":200,"url":"https://housebeautiful.com/robots.txt","bytes":973,"groups":3},"bots":{"GPTBot":["partial","named",32,0],"ChatGPT-User":["partial","named",32,0],"OAI-SearchBot":["partial","named",32,0],"ClaudeBot":["partial","*",19,0],"Claude-User":["partial","*",19,0],"Claude-SearchBot":["partial","*",19,0],"anthropic-ai":["partial","*",19,0],"PerplexityBot":["partial","*",19,0],"Perplexity-User":["partial","*",19,0],"Google-Extended":["partial","*",19,0],"Applebot-Extended":["partial","*",19,0],"CCBot":["partial","*",19,0],"Bytespider":["partial","*",19,0],"Amazonbot":["partial","*",19,0],"meta-externalagent":["partial","*",19,0],"cohere-ai":["partial","*",19,0],"Diffbot":["partial","*",19,0],"ContextFetchBot":["partial","*",19,0]},"llms":{"status":"missing","http":404,"bytes":84},"home":{"status":"ok","http":200,"url":"https://www.housebeautiful.com/","html_tokens":186599,"markdown_tokens":139,"pruned_tokens":33}},
{"rank":4126,"domain":"timeweb.ru","origin":"https://timeweb.ru","robots":{"status":"html","http":200,"url":"https://timeweb.com/","bytes":504954,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":4127,"domain":"restaurantguru.com","origin":"https://restaurantguru.com","robots":{"status":"ok","http":200,"url":"https://restaurantguru.com/robots.txt","bytes":5539,"groups":10},"bots":{"GPTBot":["partial","*",29,0],"ChatGPT-User":["partial","*",29,0],"OAI-SearchBot":["partial","*",29,0],"ClaudeBot":["partial","*",29,0],"Claude-User":["partial","*",29,0],"Claude-SearchBot":["partial","*",29,0],"anthropic-ai":["partial","*",29,0],"PerplexityBot":["partial","*",29,0],"Perplexity-User":["partial","*",29,0],"Google-Extended":["partial","*",29,0],"Applebot-Extended":["partial","*",29,0],"CCBot":["partial","*",29,0],"Bytespider":["partial","*",29,0],"Amazonbot":["partial","*",29,0],"meta-externalagent":["partial","*",29,0],"cohere-ai":["partial","*",29,0],"Diffbot":["partial","*",29,0],"ContextFetchBot":["partial","*",29,0]},"llms":{"status":"present","http":200,"bytes":6132},"home":{"status":"ok","http":200,"url":"https://restaurantguru.com/","html_tokens":73625,"markdown_tokens":107,"pruned_tokens":107}},
{"rank":4128,"domain":"alabama.gov","origin":"https://alabama.gov","robots":{"status":"unreachable","http":null,"url":"https://alabama.gov/robots.txt","bytes":0,"groups":0,"error":"[SSL: CERTIFICATE_VERIFY_FAILED] certificate verify failed: unable to get local issuer certificate (_ssl.c:1028)"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":4129,"domain":"jaxa.jp","origin":"https://www.jaxa.jp","robots":{"status":"html","http":200,"url":"https://www.jaxa.jp/robots.txt","bytes":394,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":4130,"domain":"citynews.ca","origin":"https://citynews.ca","robots":{"status":"ok","http":200,"url":"https://www.citynews.ca/robots.txt","bytes":525,"groups":3},"bots":{"GPTBot":["partial","*",13,0],"ChatGPT-User":["partial","*",13,0],"OAI-SearchBot":["partial","*",13,0],"ClaudeBot":["partial","*",13,0],"Claude-User":["partial","*",13,0],"Claude-SearchBot":["partial","*",13,0],"anthropic-ai":["partial","*",13,0],"PerplexityBot":["partial","*",13,0],"Perplexity-User":["partial","*",13,0],"Google-Extended":["partial","*",13,0],"Applebot-Extended":["partial","*",13,0],"CCBot":["partial","*",13,0],"Bytespider":["partial","*",13,0],"Amazonbot":["partial","*",13,0],"meta-externalagent":["partial","*",13,0],"cohere-ai":["partial","*",13,0],"Diffbot":["partial","*",13,0],"ContextFetchBot":["partial","*",13,0]},"llms":{"status":"http_error","http":403,"bytes":146},"home":{"status":"ok","http":200,"url":"https://www.citynews.ca/","html_tokens":5575,"markdown_tokens":24,"pruned_tokens":24}},
{"rank":4131,"domain":"weddingwire.com","origin":"https://weddingwire.com","robots":{"status":"ok","http":200,"url":"https://www.weddingwire.com/robots.txt","bytes":11693,"groups":138},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["partial","*",171,0],"OAI-SearchBot":["partial","*",171,0],"ClaudeBot":["partial","*",171,0],"Claude-User":["partial","*",171,0],"Claude-SearchBot":["partial","*",171,0],"anthropic-ai":["partial","*",171,0],"PerplexityBot":["partial","*",171,0],"Perplexity-User":["partial","*",171,0],"Google-Extended":["partial","*",171,0],"Applebot-Extended":["partial","*",171,0],"CCBot":["partial","*",171,0],"Bytespider":["partial","*",171,0],"Amazonbot":["partial","*",171,0],"meta-externalagent":["partial","*",171,0],"cohere-ai":["partial","*",171,0],"Diffbot":["partial","*",171,0],"ContextFetchBot":["partial","*",171,0]},"llms":{"status":"missing","http":404,"bytes":128584},"home":{"status":"ok","http":200,"url":"https://www.weddingwire.com/","html_tokens":157972,"markdown_tokens":465,"pruned_tokens":336}},
{"rank":4132,"domain":"kbs.co.kr","origin":"https://kbs.co.kr","robots":{"status":"html","http":200,"url":"https://www.kbs.co.kr","bytes":227417,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":4133,"domain":"ni.com","origin":"https://ni.com","robots":{"status":"ok","http":200,"url":"https://www.ni.com/robots.txt","bytes":1601,"groups":1},"bots":{"GPTBot":["partial","*",45,0],"ChatGPT-User":["partial","*",45,0],"OAI-SearchBot":["partial","*",45,0],"ClaudeBot":["partial","*",45,0],"Claude-User":["partial","*",45,0],"Claude-SearchBot":["partial","*",45,0],"anthropic-ai":["partial","*",45,0],"PerplexityBot":["partial","*",45,0],"Perplexity-User":["partial","*",45,0],"Google-Extended":["partial","*",45,0],"Applebot-Extended":["partial","*",45,0],"CCBot":["partial","*",45,0],"Bytespider":["partial","*",45,0],"Amazonbot":["partial","*",45,0],"meta-externalagent":["partial","*",45,0],"cohere-ai":["partial","*",45,0],"Diffbot":["partial","*",45,0],"ContextFetchBot":["partial","*",45,0]},"llms":{"status":"present","http":200,"bytes":9772},"home":{"status":"ok","http":200,"url":"https://www.ni.com/en.html","html_tokens":32105,"markdown_tokens":384,"pruned_tokens":175}},
{"rank":4134,"domain":"bookofra-play.com","origin":"https://bookofra-play.com","robots":{"status":"ok","http":200,"url":"https://bookofra-play.com/robots.txt","bytes":451,"groups":1},"bots":{"GPTBot":["partial","*",16,0],"ChatGPT-User":["partial","*",16,0],"OAI-SearchBot":["partial","*",16,0],"ClaudeBot":["partial","*",16,0],"Claude-User":["partial","*",16,0],"Claude-SearchBot":["partial","*",16,0],"anthropic-ai":["partial","*",16,0],"PerplexityBot":["partial","*",16,0],"Perplexity-User":["partial","*",16,0],"Google-Extended":["partial","*",16,0],"Applebot-Extended":["partial","*",16,0],"CCBot":["partial","*",16,0],"Bytespider":["partial","*",16,0],"Amazonbot":["partial","*",16,0],"meta-externalagent":["partial","*",16,0],"cohere-ai":["partial","*",16,0],"Diffbot":["partial","*",16,0],"ContextFetchBot":["partial","*",16,0]},"llms":{"status":"missing","http":404,"bytes":39280},"home":{"status":"ok","http":200,"url":"https://bookofra-play.com/","html_tokens":32195,"markdown_tokens":1005,"pruned_tokens":167}},
{"rank":4135,"domain":"page-stats.de","origin":"https://page-stats.de","robots":{"status":"ok","http":200,"url":"https://page-stats.de/robots.txt","bytes":770,"groups":1},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":316},"home":{"status":"ok","http":200,"url":"https://page-stats.de/","html_tokens":45431,"markdown_tokens":45,"pruned_tokens":45}},
{"rank":4136,"domain":"cookpad.com","origin":"https://cookpad.com","robots":{"status":"ok","http":200,"url":"https://cookpad.com/robots.txt","bytes":2528,"groups":25},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["blocked","named",1,0],"OAI-SearchBot":["partial","*",10,1],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",10,1],"Claude-SearchBot":["partial","*",10,1],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["partial","*",10,1],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["partial","*",10,1],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",10,1]},"llms":{"status":"missing","http":404,"bytes":10175},"home":{"status":"ok","http":200,"url":"https://cookpad.com/us","html_tokens":29570,"markdown_tokens":27,"pruned_tokens":27}},
{"rank":4137,"domain":"nick.com","origin":"https://nick.com","robots":{"status":"ok","http":200,"url":"https://www.nick.com/robots.txt","bytes":269,"groups":2},"bots":{"GPTBot":["partial","*",2,0],"ChatGPT-User":["partial","*",2,0],"OAI-SearchBot":["partial","*",2,0],"ClaudeBot":["partial","*",2,0],"Claude-User":["partial","*",2,0],"Claude-SearchBot":["partial","*",2,0],"anthropic-ai":["partial","*",2,0],"PerplexityBot":["partial","*",2,0],"Perplexity-User":["partial","*",2,0],"Google-Extended":["partial","*",2,0],"Applebot-Extended":["partial","*",2,0],"CCBot":["partial","*",2,0],"Bytespider":["partial","*",2,0],"Amazonbot":["partial","*",2,0],"meta-externalagent":["partial","*",2,0],"cohere-ai":["partial","*",2,0],"Diffbot":["partial","*",2,0],"ContextFetchBot":["partial","*",2,0]},"llms":{"status":"missing","http":404,"bytes":682089},"home":{"status":"ok","http":200,"url":"https://www.nick.com","html_tokens":168428,"markdown_tokens":148,"pruned_tokens":148}},
{"rank":4138,"domain":"ic.ac.uk","origin":"https://www.ic.ac.uk","robots":{"status":"ok","http":200,"url":"https://www.imperial.ac.uk/robots.txt","bytes":12713,"groups":2},"bots":{"GPTBot":["partial","*",93,18],"ChatGPT-User":["partial","*",93,18],"OAI-SearchBot":["partial","*",93,18],"ClaudeBot":["partial","*",93,18],"Claude-User":["partial","*",93,18],"Claude-SearchBot":["partial","*",93,18],"anthropic-ai":["partial","*",93,18],"PerplexityBot":["partial","*",93,18],"Perplexity-User":["partial","*",93,18],"Google-Extended":["partial","*",93,18],"Applebot-Extended":["partial","*",93,18],"CCBot":["partial","*",93,18],"Bytespider":["partial","*",93,18],"Amazonbot":["partial","*",93,18],"meta-externalagent":["partial","*",93,18],"cohere-ai":["partial","*",93,18],"Diffbot":["partial","*",93,18],"ContextFetchBot":["partial","*",93,18]},"llms":{"status":"present","http":200,"bytes":4228},"home":{"status":"ok","http":200,"url":"https://www.imperial.ac.uk/","html_tokens":14244,"markdown_tokens":99,"pruned_tokens":99}},
{"rank":4139,"domain":"solo.to","origin":"https://solo.to","robots":{"status":"ok","http":200,"url":"https://solo.to/robots.txt","bytes":24,"groups":1},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"present","http":200,"bytes":4775},"home":{"status":"http_error","http":403,"url":"https://solo.to/"}},
{"rank":4140,"domain":"unibas.ch","origin":"https://unibas.ch","robots":{"status":"ok","http":200,"url":"https://www.unibas.ch/robots.txt","bytes":116,"groups":1},"bots":{"GPTBot":["allowed","*",0,1],"ChatGPT-User":["allowed","*",0,1],"OAI-SearchBot":["allowed","*",0,1],"ClaudeBot":["allowed","*",0,1],"Claude-User":["allowed","*",0,1],"Claude-SearchBot":["allowed","*",0,1],"anthropic-ai":["allowed","*",0,1],"PerplexityBot":["allowed","*",0,1],"Perplexity-User":["allowed","*",0,1],"Google-Extended":["allowed","*",0,1],"Applebot-Extended":["allowed","*",0,1],"CCBot":["allowed","*",0,1],"Bytespider":["allowed","*",0,1],"Amazonbot":["allowed","*",0,1],"meta-externalagent":["allowed","*",0,1],"cohere-ai":["allowed","*",0,1],"Diffbot":["allowed","*",0,1],"ContextFetchBot":["allowed","*",0,1]},"llms":{"status":"missing","http":404,"bytes":79999},"home":{"status":"ok","http":200,"url":"https://www.unibas.ch/de","html_tokens":199004,"markdown_tokens":57,"pruned_tokens":57}},
{"rank":4141,"domain":"charlotteobserver.com","origin":"https://charlotteobserver.com","robots":{"status":"unreachable","http":null,"url":"https://charlotteobserver.com/robots.txt","bytes":0,"groups":0,"error":"<StreamReset stream_id:1, error_code:2, remote_reset:True>"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":4142,"domain":"mentalhealth.org.uk","origin":"https://mentalhealth.org.uk","robots":{"status":"ok","http":200,"url":"https://www.mentalhealth.org.uk/robots.txt","bytes":2090,"groups":1},"bots":{"GPTBot":["partial","*",34,18],"ChatGPT-User":["partial","*",34,18],"OAI-SearchBot":["partial","*",34,18],"ClaudeBot":["partial","*",34,18],"Claude-User":["partial","*",34,18],"Claude-SearchBot":["partial","*",34,18],"anthropic-ai":["partial","*",34,18],"PerplexityBot":["partial","*",34,18],"Perplexity-User":["partial","*",34,18],"Google-Extended":["partial","*",34,18],"Applebot-Extended":["partial","*",34,18],"CCBot":["partial","*",34,18],"Bytespider":["partial","*",34,18],"Amazonbot":["partial","*",34,18],"meta-externalagent":["partial","*",34,18],"cohere-ai":["partial","*",34,18],"Diffbot":["partial","*",34,18],"ContextFetchBot":["partial","*",34,18]},"llms":{"status":"missing","http":404,"bytes":564},"home":{"status":"ok","http":200,"url":"https://www.mentalhealth.org.uk/","html_tokens":19246,"markdown_tokens":245,"pruned_tokens":148}},
{"rank":4143,"domain":"ovid.com","origin":"https://ovid.com","robots":{"status":"ok","http":200,"url":"https://www.ovid.com/ovid/robots.txt","bytes":148,"groups":3},"bots":{"GPTBot":["partial","*",1,0],"ChatGPT-User":["partial","*",1,0],"OAI-SearchBot":["partial","*",1,0],"ClaudeBot":["partial","*",1,0],"Claude-User":["partial","*",1,0],"Claude-SearchBot":["partial","*",1,0],"anthropic-ai":["partial","*",1,0],"PerplexityBot":["partial","*",1,0],"Perplexity-User":["partial","*",1,0],"Google-Extended":["partial","*",1,0],"Applebot-Extended":["partial","*",1,0],"CCBot":["partial","*",1,0],"Bytespider":["partial","*",1,0],"Amazonbot":["partial","*",1,0],"meta-externalagent":["partial","*",1,0],"cohere-ai":["partial","*",1,0],"Diffbot":["partial","*",1,0],"ContextFetchBot":["partial","*",1,0]},"llms":{"status":"missing","http":404,"bytes":8621},"home":{"status":"ok","http":200,"url":"https://www.ovid.com/","html_tokens":42961,"markdown_tokens":438,"pruned_tokens":215}},
{"rank":4144,"domain":"thenational.ae","origin":"https://thenational.ae","robots":{"status":"ok","http":200,"url":"https://www.thenationalnews.com/robots.txt","bytes":3124,"groups":2},"bots":{"GPTBot":["partial","*",26,1],"ChatGPT-User":["partial","*",26,1],"OAI-SearchBot":["partial","*",26,1],"ClaudeBot":["partial","*",26,1],"Claude-User":["partial","*",26,1],"Claude-SearchBot":["partial","*",26,1],"anthropic-ai":["partial","*",26,1],"PerplexityBot":["partial","*",26,1],"Perplexity-User":["partial","*",26,1],"Google-Extended":["partial","*",26,1],"Applebot-Extended":["partial","*",26,1],"CCBot":["partial","*",26,1],"Bytespider":["partial","*",26,1],"Amazonbot":["partial","*",26,1],"meta-externalagent":["partial","*",26,1],"cohere-ai":["partial","*",26,1],"Diffbot":["partial","*",26,1],"ContextFetchBot":["partial","*",26,1]},"llms":{"status":"missing","http":404,"bytes":473781},"home":{"status":"ok","http":200,"url":"https://www.thenationalnews.com/","html_tokens":450291,"markdown_tokens":67,"pruned_tokens":67}},
{"rank":4145,"domain":"christianitytoday.com","origin":"https://christianitytoday.com","robots":{"status":"ok","http":200,"url":"https://www.christianitytoday.com/robots.txt","bytes":2032,"groups":23},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["blocked","named",1,0],"OAI-SearchBot":["partial","*",4,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["blocked","named",1,0],"Claude-SearchBot":["blocked","named",1,0],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["partial","*",4,0],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["partial","*",4,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["partial","*",4,0],"meta-externalagent":["partial","*",4,0],"cohere-ai":["partial","*",4,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",4,0]},"llms":{"status":"missing","http":404,"bytes":325419},"home":{"status":"ok","http":200,"url":"https://www.christianitytoday.com/","html_tokens":103824,"markdown_tokens":377,"pruned_tokens":152}},
{"rank":4146,"domain":"webkit.org","origin":"https://webkit.org","robots":{"status":"ok","http":200,"url":"https://webkit.org/robots.txt","bytes":117,"groups":1},"bots":{"GPTBot":["partial","*",1,1],"ChatGPT-User":["partial","*",1,1],"OAI-SearchBot":["partial","*",1,1],"ClaudeBot":["partial","*",1,1],"Claude-User":["partial","*",1,1],"Claude-SearchBot":["partial","*",1,1],"anthropic-ai":["partial","*",1,1],"PerplexityBot":["partial","*",1,1],"Perplexity-User":["partial","*",1,1],"Google-Extended":["partial","*",1,1],"Applebot-Extended":["partial","*",1,1],"CCBot":["partial","*",1,1],"Bytespider":["partial","*",1,1],"Amazonbot":["partial","*",1,1],"meta-externalagent":["partial","*",1,1],"cohere-ai":["partial","*",1,1],"Diffbot":["partial","*",1,1],"ContextFetchBot":["partial","*",1,1]},"llms":{"status":"missing","http":404,"bytes":29779},"home":{"status":"ok","http":200,"url":"https://webkit.org/","html_tokens":16987,"markdown_tokens":79,"pruned_tokens":79}},
{"rank":4147,"domain":"openid.net","origin":"https://openid.net","robots":{"status":"ok","http":200,"url":"https://openid.net/robots.txt","bytes":153,"groups":1},"bots":{"GPTBot":["partial","*",1,1],"ChatGPT-User":["partial","*",1,1],"OAI-SearchBot":["partial","*",1,1],"ClaudeBot":["partial","*",1,1],"Claude-User":["partial","*",1,1],"Claude-SearchBot":["partial","*",1,1],"anthropic-ai":["partial","*",1,1],"PerplexityBot":["partial","*",1,1],"Perplexity-User":["partial","*",1,1],"Google-Extended":["partial","*",1,1],"Applebot-Extended":["partial","*",1,1],"CCBot":["partial","*",1,1],"Bytespider":["partial","*",1,1],"Amazonbot":["partial","*",1,1],"meta-externalagent":["partial","*",1,1],"cohere-ai":["partial","*",1,1],"Diffbot":["partial","*",1,1],"ContextFetchBot":["partial","*",1,1]},"llms":{"status":"missing","http":404,"bytes":122344},"home":{"status":"ok","http":200,"url":"https://openid.net/","html_tokens":37979,"markdown_tokens":129,"pruned_tokens":129}},
{"rank":4148,"domain":"sika.com","origin":"https://www.sika.com","robots":{"status":"ok","http":200,"url":"https://www.sika.com/robots.txt","bytes":15173,"groups":2},"bots":{"GPTBot":["allowed","named",170,0],"ChatGPT-User":["partial","*",3,3],"OAI-SearchBot":["partial","*",3,3],"ClaudeBot":["partial","*",3,3],"Claude-User":["partial","*",3,3],"Claude-SearchBot":["partial","*",3,3],"anthropic-ai":["partial","*",3,3],"PerplexityBot":["partial","*",3,3],"Perplexity-User":["partial","*",3,3],"Google-Extended":["partial","*",3,3],"Applebot-Extended":["partial","*",3,3],"CCBot":["partial","*",3,3],"Bytespider":["partial","*",3,3],"Amazonbot":["partial","*",3,3],"meta-externalagent":["partial","*",3,3],"cohere-ai":["partial","*",3,3],"Diffbot":["partial","*",3,3],"ContextFetchBot":["partial","*",3,3]},"llms":{"status":"missing","http":404,"bytes":597},"home":{"status":"ok","http":200,"url":"https://www.sika.com/","html_tokens":37457,"markdown_tokens":174,"pruned_tokens":154}},
{"rank":4149,"domain":"jupyter.org","origin":"https://jupyter.org","robots":{"status":"ok","http":200,"url":"https://jupyter.org/robots.txt","bytes":66,"groups":1},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"missing","http":404,"bytes":9379},"home":{"status":"ok","http":200,"url":"https://jupyter.org/","html_tokens":9888,"markdown_tokens":709,"pruned_tokens":217}},
{"rank":4150,"domain":"commonslibrary.parliament.uk","origin":"https://commonslibrary.parliament.uk","robots":{"status":"refused","http":403,"url":"https://commonslibrary.parliament.uk/robots.txt","bytes":5587,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":4151,"domain":"mskcc.org","origin":"https://mskcc.org","robots":{"status":"ok","http":200,"url":"https://www.mskcc.org/robots.txt","bytes":9710,"groups":2},"bots":{"GPTBot":["partial","*",146,18],"ChatGPT-User":["partial","*",146,18],"OAI-SearchBot":["partial","*",146,18],"ClaudeBot":["partial","*",146,18],"Claude-User":["partial","*",146,18],"Claude-SearchBot":["partial","*",146,18],"anthropic-ai":["partial","*",146,18],"PerplexityBot":["partial","*",146,18],"Perplexity-User":["partial","*",146,18],"Google-Extended":["partial","*",146,18],"Applebot-Extended":["partial","*",146,18],"CCBot":["partial","*",146,18],"Bytespider":["partial","*",146,18],"Amazonbot":["partial","*",146,18],"meta-externalagent":["partial","*",146,18],"cohere-ai":["partial","*",146,18],"Diffbot":["partial","*",146,18],"ContextFetchBot":["partial","*",146,18]},"llms":{"status":"missing","http":404,"bytes":187},"home":{"status":"ok","http":200,"url":"https://www.mskcc.org/","html_tokens":58359,"markdown_tokens":385,"pruned_tokens":281}},
{"rank":4152,"domain":"dazeddigital.com","origin":"https://dazeddigital.com","robots":{"status":"ok","http":200,"url":"https://www.dazeddigital.com/robots.txt","bytes":2893,"groups":2},"bots":{"GPTBot":["partial","*",26,0],"ChatGPT-User":["partial","*",26,0],"OAI-SearchBot":["partial","*",26,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",26,0],"Claude-SearchBot":["partial","*",26,0],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["partial","*",26,0],"Perplexity-User":["partial","*",26,0],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["partial","*",26,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["partial","*",26,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",26,0]},"llms":{"status":"missing","http":404,"bytes":107044},"home":{"status":"ok","http":200,"url":"https://www.dazeddigital.com/","html_tokens":60334,"markdown_tokens":141,"pruned_tokens":141}},
{"rank":4153,"domain":"jezebel.com","origin":"https://jezebel.com","robots":{"status":"ok","http":200,"url":"https://www.jezebel.com/robots.txt","bytes":580,"groups":4},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["blocked","named",1,0],"OAI-SearchBot":["partial","*",2,0],"ClaudeBot":["partial","*",2,0],"Claude-User":["partial","*",2,0],"Claude-SearchBot":["partial","*",2,0],"anthropic-ai":["partial","*",2,0],"PerplexityBot":["partial","*",2,0],"Perplexity-User":["partial","*",2,0],"Google-Extended":["partial","*",2,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["partial","*",2,0],"Bytespider":["partial","*",2,0],"Amazonbot":["partial","*",2,0],"meta-externalagent":["partial","*",2,0],"cohere-ai":["partial","*",2,0],"Diffbot":["partial","*",2,0],"ContextFetchBot":["partial","*",2,0]},"llms":{"status":"html","http":200,"bytes":238646},"home":{"status":"ok","http":200,"url":"https://www.jezebel.com/","html_tokens":59515,"markdown_tokens":3404,"pruned_tokens":174}},
{"rank":4154,"domain":"tuvsud.com","origin":"https://tuvsud.com","robots":{"status":"refused","http":403,"url":"https://tuvsud.com/robots.txt","bytes":5505,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":4155,"domain":"rustore.ru","origin":"https://rustore.ru","robots":{"status":"ok","http":200,"url":"https://www.rustore.ru/robots.txt","bytes":1799,"groups":1},"bots":{"GPTBot":["partial","*",23,7],"ChatGPT-User":["partial","*",23,7],"OAI-SearchBot":["partial","*",23,7],"ClaudeBot":["partial","*",23,7],"Claude-User":["partial","*",23,7],"Claude-SearchBot":["partial","*",23,7],"anthropic-ai":["partial","*",23,7],"PerplexityBot":["partial","*",23,7],"Perplexity-User":["partial","*",23,7],"Google-Extended":["partial","*",23,7],"Applebot-Extended":["partial","*",23,7],"CCBot":["partial","*",23,7],"Bytespider":["partial","*",23,7],"Amazonbot":["partial","*",23,7],"meta-externalagent":["partial","*",23,7],"cohere-ai":["partial","*",23,7],"Diffbot":["partial","*",23,7],"ContextFetchBot":["partial","*",23,7]},"llms":{"status":"disallowed","error":"robots.txt of www.rustore.ru disallows /challenge.html?hash429=Hdt3prbsKN71O6Xw-bOkZ-2P5VPG0VRv69MqfMEuW9P2idHL1aNMfcmGgHVMByRvbDNyW5zRP-M7"},"home":{"status":"disallowed","error":"robots.txt of www.rustore.ru disallows /challenge.html?hash429=iS8OlLqGCdEmKjL__WgFuDgzLwoV15cJ81rHxe6gdVmcctSetmcqPYgKQdHLXYD54U7-DLDI3wq-"}},
{"rank":4156,"domain":"fareharbor.com","origin":"https://fareharbor.com","robots":{"status":"ok","http":200,"url":"https://fareharbor.com/robots.txt","bytes":215,"groups":1},"bots":{"GPTBot":["partial","*",8,0],"ChatGPT-User":["partial","*",8,0],"OAI-SearchBot":["partial","*",8,0],"ClaudeBot":["partial","*",8,0],"Claude-User":["partial","*",8,0],"Claude-SearchBot":["partial","*",8,0],"anthropic-ai":["partial","*",8,0],"PerplexityBot":["partial","*",8,0],"Perplexity-User":["partial","*",8,0],"Google-Extended":["partial","*",8,0],"Applebot-Extended":["partial","*",8,0],"CCBot":["partial","*",8,0],"Bytespider":["partial","*",8,0],"Amazonbot":["partial","*",8,0],"meta-externalagent":["partial","*",8,0],"cohere-ai":["partial","*",8,0],"Diffbot":["partial","*",8,0],"ContextFetchBot":["partial","*",8,0]},"llms":{"status":"http_error","http":405,"bytes":2483},"home":{"status":"ok","http":200,"url":"https://fareharbor.com/","html_tokens":36854,"markdown_tokens":232,"pruned_tokens":164}},
{"rank":4157,"domain":"nysenate.gov","origin":"https://nysenate.gov","robots":{"status":"refused","http":403,"url":"https://nysenate.gov/robots.txt","bytes":5549,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":4158,"domain":"uea.ac.uk","origin":"https://uea.ac.uk","robots":{"status":"ok","http":200,"url":"https://www.uea.ac.uk/robots.txt","bytes":110,"groups":1},"bots":{"GPTBot":["allowed","*",0,1],"ChatGPT-User":["allowed","*",0,1],"OAI-SearchBot":["allowed","*",0,1],"ClaudeBot":["allowed","*",0,1],"Claude-User":["allowed","*",0,1],"Claude-SearchBot":["allowed","*",0,1],"anthropic-ai":["allowed","*",0,1],"PerplexityBot":["allowed","*",0,1],"Perplexity-User":["allowed","*",0,1],"Google-Extended":["allowed","*",0,1],"Applebot-Extended":["allowed","*",0,1],"CCBot":["allowed","*",0,1],"Bytespider":["allowed","*",0,1],"Amazonbot":["allowed","*",0,1],"meta-externalagent":["allowed","*",0,1],"cohere-ai":["allowed","*",0,1],"Diffbot":["allowed","*",0,1],"ContextFetchBot":["allowed","*",0,1]},"llms":{"status":"missing","http":404,"bytes":93300},"home":{"status":"ok","http":200,"url":"https://www.uea.ac.uk/","html_tokens":53207,"markdown_tokens":150,"pruned_tokens":76}},
{"rank":4159,"domain":"maven.org","origin":"https://maven.org","robots":{"status":"ok","http":200,"url":"https://maven.org/robots.txt","bytes":52,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"http_error","http":403,"bytes":3816},"home":{"status":"ok","http":200,"url":"https://maven.org/","html_tokens":954,"markdown_tokens":12,"pruned_tokens":12}},
{"rank":4160,"domain":"interia.pl","origin":"https://interia.pl","robots":{"status":"ok","http":200,"url":"https://www.interia.pl/robots.txt","bytes":62,"groups":2},"bots":{"GPTBot":["allowed","*",0,1],"ChatGPT-User":["allowed","*",0,1],"OAI-SearchBot":["allowed","*",0,1],"ClaudeBot":["allowed","*",0,1],"Claude-User":["allowed","*",0,1],"Claude-SearchBot":["allowed","*",0,1],"anthropic-ai":["allowed","*",0,1],"PerplexityBot":["allowed","*",0,1],"Perplexity-User":["allowed","*",0,1],"Google-Extended":["allowed","*",0,1],"Applebot-Extended":["allowed","*",0,1],"CCBot":["allowed","*",0,1],"Bytespider":["allowed","*",0,1],"Amazonbot":["allowed","*",0,1],"meta-externalagent":["allowed","*",0,1],"cohere-ai":["allowed","*",0,1],"Diffbot":["allowed","*",0,1],"ContextFetchBot":["allowed","*",0,1]},"llms":{"status":"missing","http":404,"bytes":146},"home":{"status":"ok","http":200,"url":"https://www.interia.pl/","html_tokens":170714,"markdown_tokens":18,"pruned_tokens":18}},
{"rank":4161,"domain":"sucuri.net","origin":"https://sucuri.net","robots":{"status":"ok","http":200,"url":"https://sucuri.net/robots.txt","bytes":24,"groups":1},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"missing","http":404,"bytes":163539},"home":{"status":"ok","http":200,"url":"https://sucuri.net/","html_tokens":98452,"markdown_tokens":1134,"pruned_tokens":24}},
{"rank":4162,"domain":"radikal.ru","origin":"https://radikal.ru","robots":{"status":"ok","http":200,"url":"https://radikal.ru/robots.txt","bytes":189,"groups":1},"bots":{"GPTBot":["partial","*",4,1],"ChatGPT-User":["partial","*",4,1],"OAI-SearchBot":["partial","*",4,1],"ClaudeBot":["partial","*",4,1],"Claude-User":["partial","*",4,1],"Claude-SearchBot":["partial","*",4,1],"anthropic-ai":["partial","*",4,1],"PerplexityBot":["partial","*",4,1],"Perplexity-User":["partial","*",4,1],"Google-Extended":["partial","*",4,1],"Applebot-Extended":["partial","*",4,1],"CCBot":["partial","*",4,1],"Bytespider":["partial","*",4,1],"Amazonbot":["partial","*",4,1],"meta-externalagent":["partial","*",4,1],"cohere-ai":["partial","*",4,1],"Diffbot":["partial","*",4,1],"ContextFetchBot":["partial","*",4,1]},"llms":{"status":"missing","http":404,"bytes":7528},"home":{"status":"ok","http":200,"url":"https://radikal.ru/","html_tokens":40629,"markdown_tokens":1085,"pruned_tokens":66}},
{"rank":4163,"domain":"toronto.edu","origin":"https://toronto.edu","robots":{"status":"refused","http":403,"url":"https://toronto.edu/robots.txt","bytes":277,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":4164,"domain":"themuse.com","origin":"https://themuse.com","robots":{"status":"server_error","http":503,"url":"https://themuse.com/robots.txt","bytes":0,"groups":0,"error":"HTTP 503"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":4165,"domain":"comscore.com","origin":"https://comscore.com","robots":{"status":"ok","http":200,"url":"https://www.comscore.com/robots.txt","bytes":989,"groups":2},"bots":{"GPTBot":["partial","*",36,0],"ChatGPT-User":["partial","*",36,0],"OAI-SearchBot":["partial","*",36,0],"ClaudeBot":["partial","*",36,0],"Claude-User":["partial","*",36,0],"Claude-SearchBot":["partial","*",36,0],"anthropic-ai":["partial","*",36,0],"PerplexityBot":["partial","*",36,0],"Perplexity-User":["partial","*",36,0],"Google-Extended":["partial","*",36,0],"Applebot-Extended":["partial","*",36,0],"CCBot":["partial","*",36,0],"Bytespider":["partial","*",36,0],"Amazonbot":["partial","*",36,0],"meta-externalagent":["partial","*",36,0],"cohere-ai":["partial","*",36,0],"Diffbot":["partial","*",36,0],"ContextFetchBot":["partial","*",36,0]},"llms":{"status":"missing","http":404,"bytes":56734},"home":{"status":"ok","http":200,"url":"https://www.comscore.com/","html_tokens":21670,"markdown_tokens":1222,"pruned_tokens":289}},
{"rank":4166,"domain":"uwm.edu","origin":"https://uwm.edu","robots":{"status":"ok","http":200,"url":"https://uwm.edu/robots.txt","bytes":5817,"groups":40},"bots":{"GPTBot":["partial","named",4,0],"ChatGPT-User":["partial","named",4,0],"OAI-SearchBot":["partial","*",19,1],"ClaudeBot":["partial","named",4,0],"Claude-User":["partial","*",19,1],"Claude-SearchBot":["partial","*",19,1],"anthropic-ai":["partial","*",19,1],"PerplexityBot":["partial","*",19,1],"Perplexity-User":["blocked","named",1,0],"Google-Extended":["partial","*",19,1],"Applebot-Extended":["partial","*",19,1],"CCBot":["partial","*",19,1],"Bytespider":["blocked","named",1,0],"Amazonbot":["partial","named",4,0],"meta-externalagent":["partial","named",4,0],"cohere-ai":["partial","*",19,1],"Diffbot":["partial","*",19,1],"ContextFetchBot":["partial","*",19,1]},"llms":{"status":"missing","http":404,"bytes":55217},"home":{"status":"ok","http":200,"url":"https://uwm.edu/","html_tokens":19621,"markdown_tokens":632,"pruned_tokens":327}},
{"rank":4167,"domain":"jdsupra.com","origin":"https://jdsupra.com","robots":{"status":"ok","http":200,"url":"https://www.jdsupra.com/robots.txt","bytes":1323,"groups":20},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["blocked","named",1,0],"OAI-SearchBot":["partial","*",16,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",16,0],"Claude-SearchBot":["partial","*",16,0],"anthropic-ai":["partial","*",16,0],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["partial","*",16,0],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["partial","*",16,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["partial","*",16,0],"ContextFetchBot":["partial","*",16,0]},"llms":{"status":"missing","http":404,"bytes":21593},"home":{"status":"ok","http":200,"url":"https://www.jdsupra.com/","html_tokens":54380,"markdown_tokens":2807,"pruned_tokens":118}},
{"rank":4168,"domain":"skyrock.com","origin":"https://skyrock.com","robots":{"status":"html","http":200,"url":"https://skyrock.fm/?orig=skyrock.com","bytes":193414,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":4169,"domain":"gazzetta.it","origin":"https://gazzetta.it","robots":{"status":"ok","http":200,"url":"https://gazzetta.it/robots.txt","bytes":4811,"groups":15},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["partial","*",96,7],"OAI-SearchBot":["partial","*",96,7],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",96,7],"Claude-SearchBot":["partial","*",96,7],"anthropic-ai":["partial","*",96,7],"PerplexityBot":["partial","*",96,7],"Perplexity-User":["partial","*",96,7],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["partial","*",96,7],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["partial","*",96,7],"Diffbot":["partial","*",96,7],"ContextFetchBot":["partial","*",96,7]},"llms":{"status":"disallowed"},"home":{"status":"ok","http":200,"url":"https://www.gazzetta.it/","html_tokens":117056,"markdown_tokens":71,"pruned_tokens":18}},
{"rank":4170,"domain":"albany.edu","origin":"https://albany.edu","robots":{"status":"ok","http":200,"url":"https://www.albany.edu/robots.txt","bytes":2259,"groups":1},"bots":{"GPTBot":["partial","*",36,18],"ChatGPT-User":["partial","*",36,18],"OAI-SearchBot":["partial","*",36,18],"ClaudeBot":["partial","*",36,18],"Claude-User":["partial","*",36,18],"Claude-SearchBot":["partial","*",36,18],"anthropic-ai":["partial","*",36,18],"PerplexityBot":["partial","*",36,18],"Perplexity-User":["partial","*",36,18],"Google-Extended":["partial","*",36,18],"Applebot-Extended":["partial","*",36,18],"CCBot":["partial","*",36,18],"Bytespider":["partial","*",36,18],"Amazonbot":["partial","*",36,18],"meta-externalagent":["partial","*",36,18],"cohere-ai":["partial","*",36,18],"Diffbot":["partial","*",36,18],"ContextFetchBot":["partial","*",36,18]},"llms":{"status":"missing","http":404,"bytes":15388},"home":{"status":"ok","http":200,"url":"https://www.albany.edu/","html_tokens":77337,"markdown_tokens":119,"pruned_tokens":86}},
{"rank":4171,"domain":"journaldunet.com","origin":"https://journaldunet.com","robots":{"status":"ok","http":200,"url":"https://journaldunet.com/robots.txt","bytes":1579,"groups":8},"bots":{"GPTBot":["partial","*",32,0],"ChatGPT-User":["partial","*",32,0],"OAI-SearchBot":["partial","*",32,0],"ClaudeBot":["partial","*",32,0],"Claude-User":["partial","*",32,0],"Claude-SearchBot":["partial","*",32,0],"anthropic-ai":["partial","*",32,0],"PerplexityBot":["partial","*",32,0],"Perplexity-User":["partial","*",32,0],"Google-Extended":["partial","*",32,0],"Applebot-Extended":["partial","*",32,0],"CCBot":["partial","*",32,0],"Bytespider":["partial","*",32,0],"Amazonbot":["partial","*",32,0],"meta-externalagent":["partial","*",32,0],"cohere-ai":["partial","*",32,0],"Diffbot":["partial","*",32,0],"ContextFetchBot":["partial","*",32,0]},"llms":{"status":"missing","http":404,"bytes":14452},"home":{"status":"ok","http":200,"url":"https://www.journaldunet.com/","html_tokens":32164,"markdown_tokens":267,"pruned_tokens":174}},
{"rank":4172,"domain":"waterstones.com","origin":"https://waterstones.com","robots":{"status":"refused","http":403,"url":"https://www.waterstones.com/robots.txt","bytes":75712,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":4173,"domain":"kent.ac.uk","origin":"https://kent.ac.uk","robots":{"status":"ok","http":200,"url":"https://www.kent.ac.uk/robots.txt","bytes":1163,"groups":2},"bots":{"GPTBot":["allowed","named",0,10],"ChatGPT-User":["allowed","named",0,10],"OAI-SearchBot":["partial","*",3,20],"ClaudeBot":["partial","*",3,20],"Claude-User":["partial","*",3,20],"Claude-SearchBot":["partial","*",3,20],"anthropic-ai":["allowed","named",0,10],"PerplexityBot":["allowed","named",0,10],"Perplexity-User":["partial","*",3,20],"Google-Extended":["allowed","named",0,10],"Applebot-Extended":["partial","*",3,20],"CCBot":["allowed","named",0,10],"Bytespider":["partial","*",3,20],"Amazonbot":["partial","*",3,20],"meta-externalagent":["partial","*",3,20],"cohere-ai":["partial","*",3,20],"Diffbot":["partial","*",3,20],"ContextFetchBot":["partial","*",3,20]},"llms":{"status":"missing","http":404,"bytes":456885},"home":{"status":"ok","http":200,"url":"https://www.kent.ac.uk/","html_tokens":238382,"markdown_tokens":95,"pruned_tokens":95}},
{"rank":4174,"domain":"healthaffairs.org","origin":"https://healthaffairs.org","robots":{"status":"ok","http":200,"url":"https://www.healthaffairs.org/robots.txt","bytes":934,"groups":5},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["allowed","named",0,1],"OAI-SearchBot":["allowed","named",0,1],"ClaudeBot":["partial","*",14,6],"Claude-User":["allowed","named",0,1],"Claude-SearchBot":["allowed","named",0,1],"anthropic-ai":["partial","*",14,6],"PerplexityBot":["partial","*",14,6],"Perplexity-User":["partial","*",14,6],"Google-Extended":["partial","named",1,2],"Applebot-Extended":["partial","*",14,6],"CCBot":["partial","*",14,6],"Bytespider":["partial","*",14,6],"Amazonbot":["partial","*",14,6],"meta-externalagent":["partial","*",14,6],"cohere-ai":["partial","*",14,6],"Diffbot":["partial","*",14,6],"ContextFetchBot":["partial","*",14,6]},"llms":{"status":"http_error","http":403,"bytes":5570},"home":{"status":"http_error","http":403,"url":"https://healthaffairs.org/"}},
{"rank":4175,"domain":"kent.edu","origin":"https://www.kent.edu","robots":{"status":"ok","http":200,"url":"https://www.kent.edu/robots.txt","bytes":2069,"groups":1},"bots":{"GPTBot":["partial","*",34,18],"ChatGPT-User":["partial","*",34,18],"OAI-SearchBot":["partial","*",34,18],"ClaudeBot":["partial","*",34,18],"Claude-User":["partial","*",34,18],"Claude-SearchBot":["partial","*",34,18],"anthropic-ai":["partial","*",34,18],"PerplexityBot":["partial","*",34,18],"Perplexity-User":["partial","*",34,18],"Google-Extended":["partial","*",34,18],"Applebot-Extended":["partial","*",34,18],"CCBot":["partial","*",34,18],"Bytespider":["partial","*",34,18],"Amazonbot":["partial","*",34,18],"meta-externalagent":["partial","*",34,18],"cohere-ai":["partial","*",34,18],"Diffbot":["partial","*",34,18],"ContextFetchBot":["partial","*",34,18]},"llms":{"status":"missing","http":404,"bytes":186},"home":{"status":"ok","http":200,"url":"https://www.kent.edu/","html_tokens":101541,"markdown_tokens":824,"pruned_tokens":240}},
{"rank":4176,"domain":"elte.hu","origin":"https://elte.hu","robots":{"status":"ok","http":200,"url":"https://elte.hu/robots.txt","bytes":3307,"groups":8},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["partial","named",3,0],"OAI-SearchBot":["partial","*",37,22],"ClaudeBot":["partial","*",37,22],"Claude-User":["partial","*",37,22],"Claude-SearchBot":["partial","*",37,22],"anthropic-ai":["partial","*",37,22],"PerplexityBot":["partial","*",37,22],"Perplexity-User":["partial","*",37,22],"Google-Extended":["partial","*",37,22],"Applebot-Extended":["partial","*",37,22],"CCBot":["blocked","named",1,0],"Bytespider":["partial","*",37,22],"Amazonbot":["partial","*",37,22],"meta-externalagent":["partial","*",37,22],"cohere-ai":["partial","*",37,22],"Diffbot":["partial","*",37,22],"ContextFetchBot":["partial","*",37,22]},"llms":{"status":"missing","http":404,"bytes":181},"home":{"status":"ok","http":200,"url":"https://elte.hu/","html_tokens":108151,"markdown_tokens":481,"pruned_tokens":148}},
{"rank":4177,"domain":"ladepeche.fr","origin":"https://ladepeche.fr","robots":{"status":"ok","http":200,"url":"https://www.ladepeche.fr/robots.txt","bytes":8903,"groups":240},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["allowed","named",0,0],"OAI-SearchBot":["allowed","named",0,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",5,0],"Claude-SearchBot":["partial","*",5,0],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["partial","*",5,0],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["partial","*",5,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["partial","*",5,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",5,0]},"llms":{"status":"missing","http":404,"bytes":41948},"home":{"status":"ok","http":200,"url":"https://www.ladepeche.fr/","html_tokens":120035,"markdown_tokens":410,"pruned_tokens":140}},
{"rank":4178,"domain":"tripadvisor.es","origin":"https://tripadvisor.es","robots":{"status":"ok","http":200,"url":"https://www.tripadvisor.es/robots.txt","bytes":22871,"groups":9},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["partial","named",623,8],"OAI-SearchBot":["partial","named",623,8],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",621,8],"Claude-SearchBot":["partial","*",621,8],"anthropic-ai":["partial","*",621,8],"PerplexityBot":["partial","named",625,8],"Perplexity-User":["partial","*",621,8],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["partial","*",621,8],"ContextFetchBot":["partial","*",621,8]},"llms":{"status":"http_error","http":403,"bytes":774},"home":{"status":"http_error","http":403,"url":"https://www.tripadvisor.es/"}},
{"rank":4179,"domain":"icj-cij.org","origin":"https://icj-cij.org","robots":{"status":"ok","http":200,"url":"https://icj-cij.org/robots.txt","bytes":2027,"groups":1},"bots":{"GPTBot":["partial","*",34,18],"ChatGPT-User":["partial","*",34,18],"OAI-SearchBot":["partial","*",34,18],"ClaudeBot":["partial","*",34,18],"Claude-User":["partial","*",34,18],"Claude-SearchBot":["partial","*",34,18],"anthropic-ai":["partial","*",34,18],"PerplexityBot":["partial","*",34,18],"Perplexity-User":["partial","*",34,18],"Google-Extended":["partial","*",34,18],"Applebot-Extended":["partial","*",34,18],"CCBot":["partial","*",34,18],"Bytespider":["partial","*",34,18],"Amazonbot":["partial","*",34,18],"meta-externalagent":["partial","*",34,18],"cohere-ai":["partial","*",34,18],"Diffbot":["partial","*",34,18],"ContextFetchBot":["partial","*",34,18]},"llms":{"status":"missing","http":404,"bytes":546},"home":{"status":"ok","http":200,"url":"https://icj-cij.org/","html_tokens":1536,"markdown_tokens":5,"pruned_tokens":5}},
{"rank":4180,"domain":"anotepad.com","origin":"https://anotepad.com","robots":{"status":"ok","http":200,"url":"https://anotepad.com/robots.txt","bytes":13,"groups":1},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"present","http":200,"bytes":3037},"home":{"status":"ok","http":200,"url":"https://anotepad.com/","html_tokens":14046,"markdown_tokens":739,"pruned_tokens":124}},
{"rank":4181,"domain":"redalyc.org","origin":"https://www.redalyc.org","robots":{"status":"server_error","http":503,"url":"https://www.redalyc.org/robots.txt","bytes":391,"groups":0,"error":"HTTP 503"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":4182,"domain":"pbc.gov.cn","origin":"https://www.pbc.gov.cn","robots":{"status":"ok","http":200,"url":"https://www.pbc.gov.cn/robots.txt","bytes":64,"groups":2},"bots":{"GPTBot":["blocked","*",1,0],"ChatGPT-User":["blocked","*",1,0],"OAI-SearchBot":["blocked","*",1,0],"ClaudeBot":["blocked","*",1,0],"Claude-User":["blocked","*",1,0],"Claude-SearchBot":["blocked","*",1,0],"anthropic-ai":["blocked","*",1,0],"PerplexityBot":["blocked","*",1,0],"Perplexity-User":["blocked","*",1,0],"Google-Extended":["blocked","*",1,0],"Applebot-Extended":["blocked","*",1,0],"CCBot":["blocked","*",1,0],"Bytespider":["blocked","*",1,0],"Amazonbot":["blocked","*",1,0],"meta-externalagent":["blocked","*",1,0],"cohere-ai":["blocked","*",1,0],"Diffbot":["blocked","*",1,0],"ContextFetchBot":["blocked","*",1,0]},"llms":{"status":"disallowed"},"home":{"status":"disallowed"}},
{"rank":4183,"domain":"chaturbate.com","origin":"https://chaturbate.com","robots":{"status":"html","http":200,"url":"https://chaturbate.com/?next=/robots.txt","bytes":203991,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":4184,"domain":"nbcchicago.com","origin":"https://nbcchicago.com","robots":{"status":"ok","http":200,"url":"https://www.nbcchicago.com/robots.txt","bytes":9890,"groups":27},"bots":{"GPTBot":["partial","named",1,6],"ChatGPT-User":["partial","named",1,6],"OAI-SearchBot":["partial","*",13,3],"ClaudeBot":["partial","named",1,6],"Claude-User":["partial","*",13,3],"Claude-SearchBot":["partial","*",13,3],"anthropic-ai":["partial","named",1,6],"PerplexityBot":["partial","named",1,6],"Perplexity-User":["partial","*",13,3],"Google-Extended":["partial","named",1,6],"Applebot-Extended":["partial","*",13,3],"CCBot":["partial","named",1,6],"Bytespider":["partial","named",1,6],"Amazonbot":["partial","named",1,8],"meta-externalagent":["partial","*",13,3],"cohere-ai":["partial","*",13,3],"Diffbot":["partial","named",1,6],"ContextFetchBot":["partial","*",13,3]},"llms":{"status":"html","http":200,"bytes":459365},"home":{"status":"ok","http":200,"url":"https://www.nbcchicago.com/","html_tokens":114832,"markdown_tokens":3,"pruned_tokens":3}},
{"rank":4185,"domain":"nmsu.edu","origin":"https://nmsu.edu","robots":{"status":"missing","http":404,"url":"https://nmsu.edu/robots.txt","bytes":196,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":196},"home":{"status":"ok","http":200,"url":"https://nmsu.edu/","html_tokens":23512,"markdown_tokens":101,"pruned_tokens":85}},
{"rank":4186,"domain":"wm.edu","origin":"https://wm.edu","robots":{"status":"ok","http":200,"url":"https://www.wm.edu/robots.txt","bytes":1338,"groups":1},"bots":{"GPTBot":["allowed","*",0,1],"ChatGPT-User":["allowed","*",0,1],"OAI-SearchBot":["allowed","*",0,1],"ClaudeBot":["allowed","*",0,1],"Claude-User":["allowed","*",0,1],"Claude-SearchBot":["allowed","*",0,1],"anthropic-ai":["allowed","*",0,1],"PerplexityBot":["allowed","*",0,1],"Perplexity-User":["allowed","*",0,1],"Google-Extended":["allowed","*",0,1],"Applebot-Extended":["allowed","*",0,1],"CCBot":["allowed","*",0,1],"Bytespider":["allowed","*",0,1],"Amazonbot":["allowed","*",0,1],"meta-externalagent":["allowed","*",0,1],"cohere-ai":["allowed","*",0,1],"Diffbot":["allowed","*",0,1],"ContextFetchBot":["allowed","*",0,1]},"llms":{"status":"missing","http":404,"bytes":44709},"home":{"status":"ok","http":200,"url":"https://www.wm.edu/","html_tokens":41875,"markdown_tokens":405,"pruned_tokens":268}},
{"rank":4187,"domain":"fujitv.co.jp","origin":"https://fujitv.co.jp","robots":{"status":"missing","http":404,"url":"https://fujitv.co.jp/robots.txt","bytes":12,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":12},"home":{"status":"not_html","error":"unsupported content type text/plain"}},
{"rank":4188,"domain":"opm.gov","origin":"https://opm.gov","robots":{"status":"ok","http":200,"url":"https://www.opm.gov/robots.txt","bytes":339,"groups":2},"bots":{"GPTBot":["partial","*",8,0],"ChatGPT-User":["partial","*",8,0],"OAI-SearchBot":["partial","*",8,0],"ClaudeBot":["partial","*",8,0],"Claude-User":["partial","*",8,0],"Claude-SearchBot":["partial","*",8,0],"anthropic-ai":["partial","*",8,0],"PerplexityBot":["partial","*",8,0],"Perplexity-User":["partial","*",8,0],"Google-Extended":["partial","*",8,0],"Applebot-Extended":["partial","*",8,0],"CCBot":["partial","*",8,0],"Bytespider":["partial","*",8,0],"Amazonbot":["partial","*",8,0],"meta-externalagent":["partial","*",8,0],"cohere-ai":["partial","*",8,0],"Diffbot":["partial","*",8,0],"ContextFetchBot":["partial","*",8,0]},"llms":{"status":"missing","http":404,"bytes":56276},"home":{"status":"ok","http":200,"url":"https://www.opm.gov/","html_tokens":16181,"markdown_tokens":302,"pruned_tokens":169}},
{"rank":4189,"domain":"ucd.ie","origin":"https://ucd.ie","robots":{"status":"refused","http":403,"url":"https://www.ucd.ie/robots.txt","bytes":919,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":4190,"domain":"monash.edu.au","origin":"https://monash.edu.au","robots":{"status":"refused","http":403,"url":"https://monash.edu.au/robots.txt","bytes":5530,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":4191,"domain":"mrbetgames.com","origin":"https://mrbetgames.com","robots":{"status":"ok","http":200,"url":"https://mrbetgames.com/robots.txt","bytes":532,"groups":1},"bots":{"GPTBot":["partial","*",19,0],"ChatGPT-User":["partial","*",19,0],"OAI-SearchBot":["partial","*",19,0],"ClaudeBot":["partial","*",19,0],"Claude-User":["partial","*",19,0],"Claude-SearchBot":["partial","*",19,0],"anthropic-ai":["partial","*",19,0],"PerplexityBot":["partial","*",19,0],"Perplexity-User":["partial","*",19,0],"Google-Extended":["partial","*",19,0],"Applebot-Extended":["partial","*",19,0],"CCBot":["partial","*",19,0],"Bytespider":["partial","*",19,0],"Amazonbot":["partial","*",19,0],"meta-externalagent":["partial","*",19,0],"cohere-ai":["partial","*",19,0],"Diffbot":["partial","*",19,0],"ContextFetchBot":["partial","*",19,0]},"llms":{"status":"missing","http":404,"bytes":46408},"home":{"status":"ok","http":200,"url":"https://mrbetgames.com/","html_tokens":23405,"markdown_tokens":1587,"pruned_tokens":115}},
{"rank":4192,"domain":"audacy.com","origin":"https://audacy.com","robots":{"status":"ok","http":200,"url":"https://www.audacy.com/robots.txt","bytes":278,"groups":5},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["partial","*",5,1],"OAI-SearchBot":["partial","*",5,1],"ClaudeBot":["partial","*",5,1],"Claude-User":["partial","*",5,1],"Claude-SearchBot":["partial","*",5,1],"anthropic-ai":["partial","*",5,1],"PerplexityBot":["partial","*",5,1],"Perplexity-User":["partial","*",5,1],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["partial","*",5,1],"CCBot":["blocked","named",1,0],"Bytespider":["partial","*",5,1],"Amazonbot":["partial","*",5,1],"meta-externalagent":["partial","*",5,1],"cohere-ai":["partial","*",5,1],"Diffbot":["partial","*",5,1],"ContextFetchBot":["partial","*",5,1]},"llms":{"status":"missing","http":404,"bytes":383881},"home":{"status":"ok","http":200,"url":"https://www.audacy.com/","html_tokens":2198,"markdown_tokens":12,"pruned_tokens":12}},
{"rank":4193,"domain":"securityweek.com","origin":"https://securityweek.com","robots":{"status":"ok","http":200,"url":"https://securityweek.com/robots.txt","bytes":455,"groups":1},"bots":{"GPTBot":["partial","*",18,2],"ChatGPT-User":["partial","*",18,2],"OAI-SearchBot":["partial","*",18,2],"ClaudeBot":["partial","*",18,2],"Claude-User":["partial","*",18,2],"Claude-SearchBot":["partial","*",18,2],"anthropic-ai":["partial","*",18,2],"PerplexityBot":["partial","*",18,2],"Perplexity-User":["partial","*",18,2],"Google-Extended":["partial","*",18,2],"Applebot-Extended":["partial","*",18,2],"CCBot":["partial","*",18,2],"Bytespider":["partial","*",18,2],"Amazonbot":["partial","*",18,2],"meta-externalagent":["partial","*",18,2],"cohere-ai":["partial","*",18,2],"Diffbot":["partial","*",18,2],"ContextFetchBot":["partial","*",18,2]},"llms":{"status":"http_error","http":403,"bytes":6364},"home":{"status":"http_error","http":403,"url":"https://securityweek.com/"}},
{"rank":4194,"domain":"southwest.com","origin":"https://southwest.com","robots":{"status":"ok","http":200,"url":"https://www.southwest.com/robots.txt","bytes":531,"groups":3},"bots":{"GPTBot":["partial","*",2,0],"ChatGPT-User":["partial","*",2,0],"OAI-SearchBot":["partial","*",2,0],"ClaudeBot":["partial","*",2,0],"Claude-User":["partial","*",2,0],"Claude-SearchBot":["partial","*",2,0],"anthropic-ai":["partial","*",2,0],"PerplexityBot":["partial","*",2,0],"Perplexity-User":["partial","*",2,0],"Google-Extended":["partial","*",2,0],"Applebot-Extended":["partial","*",2,0],"CCBot":["partial","*",2,0],"Bytespider":["partial","*",2,0],"Amazonbot":["partial","*",2,0],"meta-externalagent":["partial","*",2,0],"cohere-ai":["partial","*",2,0],"Diffbot":["partial","*",2,0],"ContextFetchBot":["partial","*",2,0]},"llms":{"status":"missing","http":404,"bytes":9280},"home":{"status":"ok","http":200,"url":"https://www.southwest.com/","html_tokens":1917,"markdown_tokens":0,"pruned_tokens":0}},
{"rank":4195,"domain":"bplaced.net","origin":"https://bplaced.net","robots":{"status":"ok","http":200,"url":"https://bplaced.net/robots.txt","bytes":45,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"html","http":200,"bytes":54935},"home":{"status":"ok","http":200,"url":"https://www.bplaced.net","html_tokens":13721,"markdown_tokens":759,"pruned_tokens":50}},
{"rank":4196,"domain":"cert.org","origin":"https://cert.org","robots":{"status":"ok","http":200,"url":"https://www.sei.cmu.edu/robots.txt","bytes":211,"groups":1},"bots":{"GPTBot":["partial","*",7,0],"ChatGPT-User":["partial","*",7,0],"OAI-SearchBot":["partial","*",7,0],"ClaudeBot":["partial","*",7,0],"Claude-User":["partial","*",7,0],"Claude-SearchBot":["partial","*",7,0],"anthropic-ai":["partial","*",7,0],"PerplexityBot":["partial","*",7,0],"Perplexity-User":["partial","*",7,0],"Google-Extended":["partial","*",7,0],"Applebot-Extended":["partial","*",7,0],"CCBot":["partial","*",7,0],"Bytespider":["partial","*",7,0],"Amazonbot":["partial","*",7,0],"meta-externalagent":["partial","*",7,0],"cohere-ai":["partial","*",7,0],"Diffbot":["partial","*",7,0],"ContextFetchBot":["partial","*",7,0]},"llms":{"status":"missing","http":404,"bytes":46870},"home":{"status":"ok","http":200,"url":"https://www.sei.cmu.edu/divisions/cert/","html_tokens":40885,"markdown_tokens":954,"pruned_tokens":338}},
{"rank":4197,"domain":"internic.net","origin":"https://internic.net","robots":{"status":"ok","http":200,"url":"https://www.internic.net/robots.txt","bytes":385,"groups":2},"bots":{"GPTBot":["partial","*",1,0],"ChatGPT-User":["partial","*",1,0],"OAI-SearchBot":["partial","*",1,0],"ClaudeBot":["partial","*",1,0],"Claude-User":["partial","*",1,0],"Claude-SearchBot":["partial","*",1,0],"anthropic-ai":["partial","*",1,0],"PerplexityBot":["partial","*",1,0],"Perplexity-User":["partial","*",1,0],"Google-Extended":["partial","*",1,0],"Applebot-Extended":["partial","*",1,0],"CCBot":["partial","*",1,0],"Bytespider":["partial","*",1,0],"Amazonbot":["partial","*",1,0],"meta-externalagent":["partial","*",1,0],"cohere-ai":["partial","*",1,0],"Diffbot":["partial","*",1,0],"ContextFetchBot":["partial","*",1,0]},"llms":{"status":"missing","http":404,"bytes":1403},"home":{"status":"ok","http":200,"url":"https://www.internic.net/","html_tokens":2267,"markdown_tokens":90,"pruned_tokens":90}},
{"rank":4198,"domain":"trenitalia.com","origin":"https://trenitalia.com","robots":{"status":"html","http":200,"url":"https://www.trenitalia.com/it.html","bytes":560377,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":4199,"domain":"llvm.org","origin":"https://llvm.org","robots":{"status":"ok","http":200,"url":"https://llvm.org/robots.txt","bytes":310,"groups":1},"bots":{"GPTBot":["partial","*",14,0],"ChatGPT-User":["partial","*",14,0],"OAI-SearchBot":["partial","*",14,0],"ClaudeBot":["partial","*",14,0],"Claude-User":["partial","*",14,0],"Claude-SearchBot":["partial","*",14,0],"anthropic-ai":["partial","*",14,0],"PerplexityBot":["partial","*",14,0],"Perplexity-User":["partial","*",14,0],"Google-Extended":["partial","*",14,0],"Applebot-Extended":["partial","*",14,0],"CCBot":["partial","*",14,0],"Bytespider":["partial","*",14,0],"Amazonbot":["partial","*",14,0],"meta-externalagent":["partial","*",14,0],"cohere-ai":["partial","*",14,0],"Diffbot":["partial","*",14,0],"ContextFetchBot":["partial","*",14,0]},"llms":{"status":"missing","http":404,"bytes":280},"home":{"status":"ok","http":200,"url":"https://llvm.org/","html_tokens":4480,"markdown_tokens":1926,"pruned_tokens":303}},
{"rank":4200,"domain":"etsi.org","origin":"https://etsi.org","robots":{"status":"ok","http":200,"url":"https://etsi.org/robots.txt","bytes":49,"groups":1},"bots":{"GPTBot":["partial","*",2,0],"ChatGPT-User":["partial","*",2,0],"OAI-SearchBot":["partial","*",2,0],"ClaudeBot":["partial","*",2,0],"Claude-User":["partial","*",2,0],"Claude-SearchBot":["partial","*",2,0],"anthropic-ai":["partial","*",2,0],"PerplexityBot":["partial","*",2,0],"Perplexity-User":["partial","*",2,0],"Google-Extended":["partial","*",2,0],"Applebot-Extended":["partial","*",2,0],"CCBot":["partial","*",2,0],"Bytespider":["partial","*",2,0],"Amazonbot":["partial","*",2,0],"meta-externalagent":["partial","*",2,0],"cohere-ai":["partial","*",2,0],"Diffbot":["partial","*",2,0],"ContextFetchBot":["partial","*",2,0]},"llms":{"status":"missing","http":404,"bytes":130220},"home":{"status":"ok","http":200,"url":"https://www.etsi.org/","html_tokens":47660,"markdown_tokens":67,"pruned_tokens":61}},
{"rank":4201,"domain":"idealist.org","origin":"https://idealist.org","robots":{"status":"ok","http":200,"url":"https://www.idealist.org/robots.txt","bytes":153,"groups":2},"bots":{"GPTBot":["partial","*",1,0],"ChatGPT-User":["partial","*",1,0],"OAI-SearchBot":["partial","*",1,0],"ClaudeBot":["partial","*",1,0],"Claude-User":["partial","*",1,0],"Claude-SearchBot":["partial","*",1,0],"anthropic-ai":["partial","*",1,0],"PerplexityBot":["partial","*",1,0],"Perplexity-User":["partial","*",1,0],"Google-Extended":["partial","named",1,0],"Applebot-Extended":["partial","*",1,0],"CCBot":["partial","*",1,0],"Bytespider":["partial","*",1,0],"Amazonbot":["partial","*",1,0],"meta-externalagent":["partial","*",1,0],"cohere-ai":["partial","*",1,0],"Diffbot":["partial","*",1,0],"ContextFetchBot":["partial","*",1,0]},"llms":{"status":"missing","http":404,"bytes":9},"home":{"status":"ok","http":200,"url":"https://www.idealist.org/en","html_tokens":47151,"markdown_tokens":106,"pruned_tokens":106}},
{"rank":4202,"domain":"duma.gov.ru","origin":"http://duma.gov.ru","robots":{"status":"ok","http":200,"url":"http://duma.gov.ru/robots.txt","bytes":105,"groups":1},"bots":{"GPTBot":["partial","*",3,0],"ChatGPT-User":["partial","*",3,0],"OAI-SearchBot":["partial","*",3,0],"ClaudeBot":["partial","*",3,0],"Claude-User":["partial","*",3,0],"Claude-SearchBot":["partial","*",3,0],"anthropic-ai":["partial","*",3,0],"PerplexityBot":["partial","*",3,0],"Perplexity-User":["partial","*",3,0],"Google-Extended":["partial","*",3,0],"Applebot-Extended":["partial","*",3,0],"CCBot":["partial","*",3,0],"Bytespider":["partial","*",3,0],"Amazonbot":["partial","*",3,0],"meta-externalagent":["partial","*",3,0],"cohere-ai":["partial","*",3,0],"Diffbot":["partial","*",3,0],"ContextFetchBot":["partial","*",3,0]},"llms":{"status":"missing","http":404,"bytes":229973},"home":{"status":"ok","http":200,"url":"http://duma.gov.ru/","html_tokens":73124,"markdown_tokens":28,"pruned_tokens":28}},
{"rank":4203,"domain":"arsys.es","origin":"https://arsys.es","robots":{"status":"ok","http":200,"url":"https://www.arsys.es/robots.txt","bytes":2610,"groups":1},"bots":{"GPTBot":["partial","*",94,0],"ChatGPT-User":["partial","*",94,0],"OAI-SearchBot":["partial","*",94,0],"ClaudeBot":["partial","*",94,0],"Claude-User":["partial","*",94,0],"Claude-SearchBot":["partial","*",94,0],"anthropic-ai":["partial","*",94,0],"PerplexityBot":["partial","*",94,0],"Perplexity-User":["partial","*",94,0],"Google-Extended":["partial","*",94,0],"Applebot-Extended":["partial","*",94,0],"CCBot":["partial","*",94,0],"Bytespider":["partial","*",94,0],"Amazonbot":["partial","*",94,0],"meta-externalagent":["partial","*",94,0],"cohere-ai":["partial","*",94,0],"Diffbot":["partial","*",94,0],"ContextFetchBot":["partial","*",94,0]},"llms":{"status":"missing","http":404,"bytes":594999},"home":{"status":"ok","http":200,"url":"https://www.arsys.es/","html_tokens":150176,"markdown_tokens":538,"pruned_tokens":165}},
{"rank":4204,"domain":"duckdns.org","origin":"https://duckdns.org","robots":{"status":"missing","http":404,"url":"https://www.duckdns.org/robots.txt","bytes":7834,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":7834},"home":{"status":"ok","http":200,"url":"https://www.duckdns.org/","html_tokens":2103,"markdown_tokens":41,"pruned_tokens":41}},
{"rank":4205,"domain":"bitcointalk.org","origin":"https://bitcointalk.org","robots":{"status":"ok","http":200,"url":"https://bitcointalk.org/robots.txt","bytes":45,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":563},"home":{"status":"ok","http":200,"url":"https://bitcointalk.org/","html_tokens":23210,"markdown_tokens":1513,"pruned_tokens":165}},
{"rank":4206,"domain":"xiph.org","origin":"https://xiph.org","robots":{"status":"ok","http":200,"url":"https://xiph.org/robots.txt","bytes":2674,"groups":30},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":146},"home":{"status":"ok","http":200,"url":"https://xiph.org/","html_tokens":2182,"markdown_tokens":424,"pruned_tokens":355}},
{"rank":4207,"domain":"tohoku.ac.jp","origin":"https://www.tohoku.ac.jp","robots":{"status":"ok","http":200,"url":"https://www.tohoku.ac.jp/robots.txt","bytes":166,"groups":1},"bots":{"GPTBot":["partial","*",4,0],"ChatGPT-User":["partial","*",4,0],"OAI-SearchBot":["partial","*",4,0],"ClaudeBot":["partial","*",4,0],"Claude-User":["partial","*",4,0],"Claude-SearchBot":["partial","*",4,0],"anthropic-ai":["partial","*",4,0],"PerplexityBot":["partial","*",4,0],"Perplexity-User":["partial","*",4,0],"Google-Extended":["partial","*",4,0],"Applebot-Extended":["partial","*",4,0],"CCBot":["partial","*",4,0],"Bytespider":["partial","*",4,0],"Amazonbot":["partial","*",4,0],"meta-externalagent":["partial","*",4,0],"cohere-ai":["partial","*",4,0],"Diffbot":["partial","*",4,0],"ContextFetchBot":["partial","*",4,0]},"llms":{"status":"missing","http":404,"bytes":19749},"home":{"status":"ok","http":200,"url":"https://www.tohoku.ac.jp/en/","html_tokens":14054,"markdown_tokens":894,"pruned_tokens":138}},
{"rank":4208,"domain":"ga.gov","origin":"https://ga.gov","robots":{"status":"ok","http":200,"url":"https://georgia.gov/robots.txt","bytes":1677,"groups":1},"bots":{"GPTBot":["partial","*",21,18],"ChatGPT-User":["partial","*",21,18],"OAI-SearchBot":["partial","*",21,18],"ClaudeBot":["partial","*",21,18],"Claude-User":["partial","*",21,18],"Claude-SearchBot":["partial","*",21,18],"anthropic-ai":["partial","*",21,18],"PerplexityBot":["partial","*",21,18],"Perplexity-User":["partial","*",21,18],"Google-Extended":["partial","*",21,18],"Applebot-Extended":["partial","*",21,18],"CCBot":["partial","*",21,18],"Bytespider":["partial","*",21,18],"Amazonbot":["partial","*",21,18],"meta-externalagent":["partial","*",21,18],"cohere-ai":["partial","*",21,18],"Diffbot":["partial","*",21,18],"ContextFetchBot":["partial","*",21,18]},"llms":{"status":"missing","http":404,"bytes":185},"home":{"status":"ok","http":200,"url":"https://georgia.gov/","html_tokens":25461,"markdown_tokens":335,"pruned_tokens":73}},
{"rank":4209,"domain":"cityam.com","origin":"https://cityam.com","robots":{"status":"ok","http":200,"url":"https://www.cityam.com/robots.txt","bytes":75,"groups":1},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"missing","http":404,"bytes":77595},"home":{"status":"ok","http":200,"url":"https://www.cityam.com/","html_tokens":53544,"markdown_tokens":38,"pruned_tokens":38}},
{"rank":4210,"domain":"abc7chicago.com","origin":"https://abc7chicago.com","robots":{"status":"ok","http":200,"url":"https://abc7chicago.com/robots.txt","bytes":1130,"groups":26},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["blocked","named",1,0],"OAI-SearchBot":["partial","*",7,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",7,0],"Claude-SearchBot":["partial","*",7,0],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["partial","*",7,0],"Perplexity-User":["partial","*",7,0],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["partial","*",7,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",7,0]},"llms":{"status":"html","http":200,"bytes":73747},"home":{"status":"ok","http":200,"url":"https://abc7chicago.com/","html_tokens":119999,"markdown_tokens":91,"pruned_tokens":91}},
{"rank":4211,"domain":"governo.it","origin":"https://governo.it","robots":{"status":"ok","http":200,"url":"https://www.governo.it/robots.txt","bytes":2189,"groups":1},"bots":{"GPTBot":["partial","*",36,32],"ChatGPT-User":["partial","*",36,32],"OAI-SearchBot":["partial","*",36,32],"ClaudeBot":["partial","*",36,32],"Claude-User":["partial","*",36,32],"Claude-SearchBot":["partial","*",36,32],"anthropic-ai":["partial","*",36,32],"PerplexityBot":["partial","*",36,32],"Perplexity-User":["partial","*",36,32],"Google-Extended":["partial","*",36,32],"Applebot-Extended":["partial","*",36,32],"CCBot":["partial","*",36,32],"Bytespider":["partial","*",36,32],"Amazonbot":["partial","*",36,32],"meta-externalagent":["partial","*",36,32],"cohere-ai":["partial","*",36,32],"Diffbot":["partial","*",36,32],"ContextFetchBot":["partial","*",36,32]},"llms":{"status":"missing","http":404,"bytes":294},"home":{"status":"ok","http":200,"url":"https://www.governo.it/it","html_tokens":15900,"markdown_tokens":276,"pruned_tokens":131}},
{"rank":4212,"domain":"du.edu","origin":"https://du.edu","robots":{"status":"refused","http":403,"url":"https://du.edu/robots.txt","bytes":560,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":4213,"domain":"privacypolicies.com","origin":"https://privacypolicies.com","robots":{"status":"ok","http":200,"url":"https://www.privacypolicies.com/robots.txt","bytes":2134,"groups":1},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"http_error","http":403,"bytes":5558},"home":{"status":"http_error","http":403,"url":"https://privacypolicies.com/"}},
{"rank":4214,"domain":"the-scientist.com","origin":"https://the-scientist.com","robots":{"status":"refused","http":403,"url":"https://www.the-scientist.com/robots.txt","bytes":919,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":4215,"domain":"idv.tw","origin":"https://idv.tw","robots":{"status":"unreachable","http":null,"url":"https://idv.tw/robots.txt","bytes":0,"groups":0,"error":"cannot resolve idv.tw"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":4216,"domain":"cnnbrasil.com.br","origin":"https://cnnbrasil.com.br","robots":{"status":"ok","http":200,"url":"https://www.cnnbrasil.com.br/robots.txt","bytes":871,"groups":2},"bots":{"GPTBot":["partial","*",17,1],"ChatGPT-User":["partial","*",17,1],"OAI-SearchBot":["partial","*",17,1],"ClaudeBot":["partial","*",17,1],"Claude-User":["partial","*",17,1],"Claude-SearchBot":["partial","*",17,1],"anthropic-ai":["partial","*",17,1],"PerplexityBot":["partial","*",17,1],"Perplexity-User":["partial","*",17,1],"Google-Extended":["partial","*",17,1],"Applebot-Extended":["partial","*",17,1],"CCBot":["partial","*",17,1],"Bytespider":["partial","*",17,1],"Amazonbot":["partial","*",17,1],"meta-externalagent":["partial","*",17,1],"cohere-ai":["partial","*",17,1],"Diffbot":["partial","*",17,1],"ContextFetchBot":["partial","*",17,1]},"llms":{"status":"missing","http":404,"bytes":187515},"home":{"status":"ok","http":200,"url":"https://www.cnnbrasil.com.br","html_tokens":265977,"markdown_tokens":8,"pruned_tokens":8}},
{"rank":4217,"domain":"ellenmacarthurfoundation.org","origin":"https://ellenmacarthurfoundation.org","robots":{"status":"ok","http":200,"url":"https://www.ellenmacarthurfoundation.org/robots.txt","bytes":91,"groups":1},"bots":{"GPTBot":["allowed","*",0,1],"ChatGPT-User":["allowed","*",0,1],"OAI-SearchBot":["allowed","*",0,1],"ClaudeBot":["allowed","*",0,1],"Claude-User":["allowed","*",0,1],"Claude-SearchBot":["allowed","*",0,1],"anthropic-ai":["allowed","*",0,1],"PerplexityBot":["allowed","*",0,1],"Perplexity-User":["allowed","*",0,1],"Google-Extended":["allowed","*",0,1],"Applebot-Extended":["allowed","*",0,1],"CCBot":["allowed","*",0,1],"Bytespider":["allowed","*",0,1],"Amazonbot":["allowed","*",0,1],"meta-externalagent":["allowed","*",0,1],"cohere-ai":["allowed","*",0,1],"Diffbot":["allowed","*",0,1],"ContextFetchBot":["allowed","*",0,1]},"llms":{"status":"missing","http":404,"bytes":611856},"home":{"status":"ok","http":200,"url":"https://www.ellenmacarthurfoundation.org/","html_tokens":245134,"markdown_tokens":359,"pruned_tokens":111}},
{"rank":4218,"domain":"www.gov.il","origin":"https://www.gov.il","robots":{"status":"html","http":200,"url":"https://www.gov.il/robots.txt","bytes":8605,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":4219,"domain":"gva.es","origin":"https://gva.es","robots":{"status":"unreachable","http":null,"url":"https://gva.es/robots.txt","bytes":0,"groups":0,"error":"cannot resolve gva.es"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":4220,"domain":"wccftech.com","origin":"https://wccftech.com","robots":{"status":"ok","http":200,"url":"https://wccftech.com/robots.txt","bytes":1740,"groups":3},"bots":{"GPTBot":["partial","*",6,0],"ChatGPT-User":["partial","*",6,0],"OAI-SearchBot":["partial","*",6,0],"ClaudeBot":["partial","*",6,0],"Claude-User":["partial","*",6,0],"Claude-SearchBot":["partial","*",6,0],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["partial","*",6,0],"Perplexity-User":["partial","*",6,0],"Google-Extended":["partial","*",6,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["partial","*",6,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",6,0]},"llms":{"status":"missing","http":404,"bytes":100160},"home":{"status":"ok","http":200,"url":"https://wccftech.com/","html_tokens":50643,"markdown_tokens":4,"pruned_tokens":4}},
{"rank":4221,"domain":"thespruceeats.com","origin":"https://thespruceeats.com","robots":{"status":"ok","http":200,"url":"https://www.thespruceeats.com/robots.txt","bytes":3479,"groups":6},"bots":{"GPTBot":["partial","named",1,0],"ChatGPT-User":["partial","named",1,0],"OAI-SearchBot":["partial","named",1,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["blocked","named",1,0],"Claude-SearchBot":["partial","named",1,2],"anthropic-ai":["partial","named",1,2],"PerplexityBot":["partial","named",1,2],"Perplexity-User":["blocked","named",1,0],"Google-Extended":["partial","named",1,2],"Applebot-Extended":["blocked","named",1,0],"CCBot":["partial","named",1,2],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["partial","named",1,2],"cohere-ai":["partial","named",1,2],"Diffbot":["partial","*",2,0],"ContextFetchBot":["partial","*",2,0]},"llms":{"status":"http_error","http":402,"bytes":612},"home":{"status":"http_error","http":402,"url":"https://www.thespruceeats.com/"}},
{"rank":4222,"domain":"blogfreely.net","origin":"https://blogfreely.net","robots":{"status":"missing","http":404,"url":"https://blogfreely.net/robots.txt","bytes":19,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"html","http":200,"bytes":6549},"home":{"status":"ok","http":200,"url":"https://blogfreely.net/","html_tokens":1638,"markdown_tokens":35,"pruned_tokens":35}},
{"rank":4223,"domain":"squidoo.com","origin":"https://squidoo.com","robots":{"status":"unreachable","http":null,"url":"https://squidoo.com/robots.txt","bytes":0,"groups":0,"error":"[SSL: CERTIFICATE_VERIFY_FAILED] certificate verify failed: Hostname mismatch, certificate is not valid for 'squidoo.com'. (_ssl.c:1028)"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":4224,"domain":"undocs.org","origin":"https://undocs.org","robots":{"status":"html","http":200,"url":"https://docs.un.org/robots.txt","bytes":3958,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":4225,"domain":"cbs.nl","origin":"https://cbs.nl","robots":{"status":"ok","http":200,"url":"https://www.cbs.nl/robots.txt","bytes":39035,"groups":1},"bots":{"GPTBot":["partial","*",488,0],"ChatGPT-User":["partial","*",488,0],"OAI-SearchBot":["partial","*",488,0],"ClaudeBot":["partial","*",488,0],"Claude-User":["partial","*",488,0],"Claude-SearchBot":["partial","*",488,0],"anthropic-ai":["partial","*",488,0],"PerplexityBot":["partial","*",488,0],"Perplexity-User":["partial","*",488,0],"Google-Extended":["partial","*",488,0],"Applebot-Extended":["partial","*",488,0],"CCBot":["partial","*",488,0],"Bytespider":["partial","*",488,0],"Amazonbot":["partial","*",488,0],"meta-externalagent":["partial","*",488,0],"cohere-ai":["partial","*",488,0],"Diffbot":["partial","*",488,0],"ContextFetchBot":["partial","*",488,0]},"llms":{"status":"missing","http":404,"bytes":37132},"home":{"status":"ok","http":200,"url":"https://www.cbs.nl/","html_tokens":16291,"markdown_tokens":12,"pruned_tokens":12}},
{"rank":4226,"domain":"ithome.com","origin":"https://ithome.com","robots":{"status":"ok","http":200,"url":"https://www.ithome.com/robots.txt","bytes":379,"groups":2},"bots":{"GPTBot":["partial","*",7,0],"ChatGPT-User":["partial","*",7,0],"OAI-SearchBot":["partial","*",7,0],"ClaudeBot":["partial","*",7,0],"Claude-User":["partial","*",7,0],"Claude-SearchBot":["partial","*",7,0],"anthropic-ai":["partial","*",7,0],"PerplexityBot":["partial","*",7,0],"Perplexity-User":["partial","*",7,0],"Google-Extended":["partial","*",7,0],"Applebot-Extended":["partial","*",7,0],"CCBot":["partial","*",7,0],"Bytespider":["partial","*",7,0],"Amazonbot":["partial","*",7,0],"meta-externalagent":["partial","*",7,0],"cohere-ai":["partial","*",7,0],"Diffbot":["partial","*",7,0],"ContextFetchBot":["partial","*",7,0]},"llms":{"status":"timeout","error":"ConnectTimeout"},"home":{"status":"timeout","error":"ConnectTimeout"}},
{"rank":4227,"domain":"masslive.com","origin":"https://masslive.com","robots":{"status":"ok","http":200,"url":"https://www.masslive.com/robots.txt","bytes":2830,"groups":4},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["partial","named",1,11],"OAI-SearchBot":["partial","named",1,11],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","named",1,11],"Claude-SearchBot":["partial","named",1,11],"anthropic-ai":["partial","*",30,3],"PerplexityBot":["partial","named",1,11],"Perplexity-User":["partial","named",1,11],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["partial","*",30,3],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",30,3]},"llms":{"status":"missing","http":404,"bytes":242981},"home":{"status":"http_error","http":403,"url":"https://www.masslive.com/"}},
{"rank":4228,"domain":"powells.com","origin":"https://powells.com","robots":{"status":"ok","http":200,"url":"https://www.powells.com/robots.txt","bytes":331,"groups":1},"bots":{"GPTBot":["partial","*",11,0],"ChatGPT-User":["partial","*",11,0],"OAI-SearchBot":["partial","*",11,0],"ClaudeBot":["partial","*",11,0],"Claude-User":["partial","*",11,0],"Claude-SearchBot":["partial","*",11,0],"anthropic-ai":["partial","*",11,0],"PerplexityBot":["partial","*",11,0],"Perplexity-User":["partial","*",11,0],"Google-Extended":["partial","*",11,0],"Applebot-Extended":["partial","*",11,0],"CCBot":["partial","*",11,0],"Bytespider":["partial","*",11,0],"Amazonbot":["partial","*",11,0],"meta-externalagent":["partial","*",11,0],"cohere-ai":["partial","*",11,0],"Diffbot":["partial","*",11,0],"ContextFetchBot":["partial","*",11,0]},"llms":{"status":"missing","http":404,"bytes":897453},"home":{"status":"http_error","http":403,"url":"https://www.powells.com/"}},
{"rank":4229,"domain":"abdn.ac.uk","origin":"https://abdn.ac.uk","robots":{"status":"ok","http":200,"url":"https://www.abdn.ac.uk/robots.txt","bytes":169,"groups":1},"bots":{"GPTBot":["partial","*",2,0],"ChatGPT-User":["partial","*",2,0],"OAI-SearchBot":["partial","*",2,0],"ClaudeBot":["partial","*",2,0],"Claude-User":["partial","*",2,0],"Claude-SearchBot":["partial","*",2,0],"anthropic-ai":["partial","*",2,0],"PerplexityBot":["partial","*",2,0],"Perplexity-User":["partial","*",2,0],"Google-Extended":["partial","*",2,0],"Applebot-Extended":["partial","*",2,0],"CCBot":["partial","*",2,0],"Bytespider":["partial","*",2,0],"Amazonbot":["partial","*",2,0],"meta-externalagent":["partial","*",2,0],"cohere-ai":["partial","*",2,0],"Diffbot":["partial","*",2,0],"ContextFetchBot":["partial","*",2,0]},"llms":{"status":"unreachable","error":"ReadError"},"home":{"status":"ok","http":200,"url":"https://www.abdn.ac.uk/","html_tokens":17470,"markdown_tokens":259,"pruned_tokens":85}},
{"rank":4230,"domain":"prevention.com","origin":"https://prevention.com","robots":{"status":"ok","http":200,"url":"https://prevention.com/robots.txt","bytes":965,"groups":3},"bots":{"GPTBot":["partial","named",32,0],"ChatGPT-User":["partial","named",32,0],"OAI-SearchBot":["partial","named",32,0],"ClaudeBot":["partial","*",19,0],"Claude-User":["partial","*",19,0],"Claude-SearchBot":["partial","*",19,0],"anthropic-ai":["partial","*",19,0],"PerplexityBot":["partial","*",19,0],"Perplexity-User":["partial","*",19,0],"Google-Extended":["partial","*",19,0],"Applebot-Extended":["partial","*",19,0],"CCBot":["partial","*",19,0],"Bytespider":["partial","*",19,0],"Amazonbot":["partial","*",19,0],"meta-externalagent":["partial","*",19,0],"cohere-ai":["partial","*",19,0],"Diffbot":["partial","*",19,0],"ContextFetchBot":["partial","*",19,0]},"llms":{"status":"missing","http":404,"bytes":84},"home":{"status":"ok","http":200,"url":"https://www.prevention.com/","html_tokens":173937,"markdown_tokens":20,"pruned_tokens":20}},
{"rank":4231,"domain":"simonwillison.net","origin":"https://simonwillison.net","robots":{"status":"ok","http":200,"url":"https://simonwillison.net/robots.txt","bytes":136,"groups":2},"bots":{"GPTBot":["partial","*",2,0],"ChatGPT-User":["allowed","named",0,0],"OAI-SearchBot":["partial","*",2,0],"ClaudeBot":["partial","*",2,0],"Claude-User":["partial","*",2,0],"Claude-SearchBot":["partial","*",2,0],"anthropic-ai":["partial","*",2,0],"PerplexityBot":["partial","*",2,0],"Perplexity-User":["partial","*",2,0],"Google-Extended":["partial","*",2,0],"Applebot-Extended":["partial","*",2,0],"CCBot":["partial","*",2,0],"Bytespider":["partial","*",2,0],"Amazonbot":["partial","*",2,0],"meta-externalagent":["partial","*",2,0],"cohere-ai":["partial","*",2,0],"Diffbot":["partial","*",2,0],"ContextFetchBot":["partial","*",2,0]},"llms":{"status":"missing","http":404,"bytes":11030},"home":{"status":"ok","http":200,"url":"https://simonwillison.net/","html_tokens":24056,"markdown_tokens":6127,"pruned_tokens":101}},
{"rank":4232,"domain":"podomatic.com","origin":"https://podomatic.com","robots":{"status":"ok","http":200,"url":"https://podomatic.com/robots.txt","bytes":920,"groups":2},"bots":{"GPTBot":["partial","*",35,0],"ChatGPT-User":["partial","*",35,0],"OAI-SearchBot":["partial","*",35,0],"ClaudeBot":["partial","*",35,0],"Claude-User":["partial","*",35,0],"Claude-SearchBot":["partial","*",35,0],"anthropic-ai":["partial","*",35,0],"PerplexityBot":["partial","*",35,0],"Perplexity-User":["partial","*",35,0],"Google-Extended":["partial","*",35,0],"Applebot-Extended":["partial","*",35,0],"CCBot":["partial","*",35,0],"Bytespider":["partial","*",35,0],"Amazonbot":["partial","*",35,0],"meta-externalagent":["partial","*",35,0],"cohere-ai":["partial","*",35,0],"Diffbot":["partial","*",35,0],"ContextFetchBot":["partial","*",35,0]},"llms":{"status":"missing","http":404,"bytes":12947},"home":{"status":"ok","http":200,"url":"https://podomatic.com/","html_tokens":12440,"markdown_tokens":166,"pruned_tokens":51}},
{"rank":4233,"domain":"google.com.ph","origin":"https://google.com.ph","robots":{"status":"ok","http":200,"url":"https://www.google.com.ph/robots.txt","bytes":6624,"groups":4},"bots":{"GPTBot":["partial","*",169,64],"ChatGPT-User":["partial","*",169,64],"OAI-SearchBot":["partial","*",169,64],"ClaudeBot":["partial","*",169,64],"Claude-User":["partial","*",169,64],"Claude-SearchBot":["partial","*",169,64],"anthropic-ai":["partial","*",169,64],"PerplexityBot":["partial","*",169,64],"Perplexity-User":["partial","*",169,64],"Google-Extended":["partial","*",169,64],"Applebot-Extended":["partial","*",169,64],"CCBot":["partial","*",169,64],"Bytespider":["partial","*",169,64],"Amazonbot":["partial","*",169,64],"meta-externalagent":["partial","*",169,64],"cohere-ai":["partial","*",169,64],"Diffbot":["partial","*",169,64],"ContextFetchBot":["partial","*",169,64]},"llms":{"status":"missing","http":404,"bytes":1569},"home":{"status":"ok","http":200,"url":"https://www.google.com/","html_tokens":21304,"markdown_tokens":30,"pruned_tokens":30}},
{"rank":4234,"domain":"cnews.ru","origin":"https://cnews.ru","robots":{"status":"ok","http":200,"url":"https://www.cnews.ru/robots.txt","bytes":439,"groups":1},"bots":{"GPTBot":["partial","*",11,0],"ChatGPT-User":["partial","*",11,0],"OAI-SearchBot":["partial","*",11,0],"ClaudeBot":["partial","*",11,0],"Claude-User":["partial","*",11,0],"Claude-SearchBot":["partial","*",11,0],"anthropic-ai":["partial","*",11,0],"PerplexityBot":["partial","*",11,0],"Perplexity-User":["partial","*",11,0],"Google-Extended":["partial","*",11,0],"Applebot-Extended":["partial","*",11,0],"CCBot":["partial","*",11,0],"Bytespider":["partial","*",11,0],"Amazonbot":["partial","*",11,0],"meta-externalagent":["partial","*",11,0],"cohere-ai":["partial","*",11,0],"Diffbot":["partial","*",11,0],"ContextFetchBot":["partial","*",11,0]},"llms":{"status":"missing","http":404,"bytes":12967},"home":{"status":"ok","http":200,"url":"https://www.cnews.ru/","html_tokens":60267,"markdown_tokens":241,"pruned_tokens":52}},
{"rank":4235,"domain":"mee.gov.cn","origin":"https://www.mee.gov.cn","robots":{"status":"html","http":200,"url":"https://www.mee.gov.cn/404/index.shtml","bytes":33583,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":4236,"domain":"scikit-learn.org","origin":"https://scikit-learn.org","robots":{"status":"missing","http":404,"url":"https://scikit-learn.org/robots.txt","bytes":9379,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":9379},"home":{"status":"ok","http":200,"url":"https://scikit-learn.org/","html_tokens":515,"markdown_tokens":0,"pruned_tokens":0}},
{"rank":4237,"domain":"juejin.cn","origin":"https://juejin.cn","robots":{"status":"ok","http":200,"url":"https://juejin.cn/robots.txt","bytes":844,"groups":1},"bots":{"GPTBot":["partial","*",17,0],"ChatGPT-User":["partial","*",17,0],"OAI-SearchBot":["partial","*",17,0],"ClaudeBot":["partial","*",17,0],"Claude-User":["partial","*",17,0],"Claude-SearchBot":["partial","*",17,0],"anthropic-ai":["partial","*",17,0],"PerplexityBot":["partial","*",17,0],"Perplexity-User":["partial","*",17,0],"Google-Extended":["partial","*",17,0],"Applebot-Extended":["partial","*",17,0],"CCBot":["partial","*",17,0],"Bytespider":["partial","*",17,0],"Amazonbot":["partial","*",17,0],"meta-externalagent":["partial","*",17,0],"cohere-ai":["partial","*",17,0],"Diffbot":["partial","*",17,0],"ContextFetchBot":["partial","*",17,0]},"llms":{"status":"html","http":200,"bytes":348537},"home":{"status":"ok","http":200,"url":"https://juejin.cn/","html_tokens":83181,"markdown_tokens":25,"pruned_tokens":25}},
{"rank":4238,"domain":"helpnetsecurity.com","origin":"https://helpnetsecurity.com","robots":{"status":"ok","http":200,"url":"https://www.helpnetsecurity.com/robots.txt","bytes":181,"groups":1},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"missing","http":404,"bytes":63140},"home":{"status":"ok","http":200,"url":"https://www.helpnetsecurity.com/","html_tokens":26537,"markdown_tokens":17,"pruned_tokens":17}},
{"rank":4239,"domain":"cracked.com","origin":"https://cracked.com","robots":{"status":"ok","http":200,"url":"https://www.cracked.com/robots.txt","bytes":325,"groups":3},"bots":{"GPTBot":["partial","*",8,0],"ChatGPT-User":["partial","*",8,0],"OAI-SearchBot":["partial","*",8,0],"ClaudeBot":["partial","*",8,0],"Claude-User":["partial","*",8,0],"Claude-SearchBot":["partial","*",8,0],"anthropic-ai":["partial","*",8,0],"PerplexityBot":["partial","*",8,0],"Perplexity-User":["partial","*",8,0],"Google-Extended":["partial","*",8,0],"Applebot-Extended":["partial","*",8,0],"CCBot":["partial","*",8,0],"Bytespider":["partial","*",8,0],"Amazonbot":["partial","*",8,0],"meta-externalagent":["partial","*",8,0],"cohere-ai":["partial","*",8,0],"Diffbot":["partial","*",8,0],"ContextFetchBot":["partial","*",8,0]},"llms":{"status":"http_error","http":403,"bytes":919},"home":{"status":"http_error","http":403,"url":"https://cracked.com/"}},
{"rank":4240,"domain":"backblaze.com","origin":"https://backblaze.com","robots":{"status":"ok","http":200,"url":"https://www.backblaze.com/robots.txt","bytes":777,"groups":12},"bots":{"GPTBot":["allowed","named",0,1],"ChatGPT-User":["allowed","named",0,1],"OAI-SearchBot":["allowed","named",0,1],"ClaudeBot":["allowed","named",0,1],"Claude-User":["allowed","named",0,1],"Claude-SearchBot":["allowed","named",0,1],"anthropic-ai":["partial","*",11,0],"PerplexityBot":["allowed","named",0,1],"Perplexity-User":["allowed","named",0,1],"Google-Extended":["allowed","named",0,1],"Applebot-Extended":["allowed","named",0,1],"CCBot":["allowed","named",0,1],"Bytespider":["partial","*",11,0],"Amazonbot":["partial","*",11,0],"meta-externalagent":["partial","*",11,0],"cohere-ai":["partial","*",11,0],"Diffbot":["partial","*",11,0],"ContextFetchBot":["partial","*",11,0]},"llms":{"status":"present","http":200,"bytes":15824},"home":{"status":"ok","http":200,"url":"https://www.backblaze.com/","html_tokens":47393,"markdown_tokens":425,"pruned_tokens":56}},
{"rank":4241,"domain":"uct.ac.za","origin":"https://uct.ac.za","robots":{"status":"ok","http":200,"url":"https://uct.ac.za/robots.txt","bytes":4465,"groups":18},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["partial","*",32,18],"OAI-SearchBot":["partial","named",12,0],"ClaudeBot":["partial","*",32,18],"Claude-User":["partial","*",32,18],"Claude-SearchBot":["partial","*",32,18],"anthropic-ai":["partial","*",32,18],"PerplexityBot":["partial","*",32,18],"Perplexity-User":["partial","*",32,18],"Google-Extended":["partial","*",32,18],"Applebot-Extended":["partial","*",32,18],"CCBot":["partial","*",32,18],"Bytespider":["blocked","named",1,0],"Amazonbot":["partial","*",32,18],"meta-externalagent":["partial","*",32,18],"cohere-ai":["partial","*",32,18],"Diffbot":["partial","*",32,18],"ContextFetchBot":["partial","*",32,18]},"llms":{"status":"timeout","error":"timed out after 10 s"},"home":{"status":"ok","http":200,"url":"https://uct.ac.za/","html_tokens":59458,"markdown_tokens":210,"pruned_tokens":106}},
{"rank":4242,"domain":"indymedia.org","origin":"https://indymedia.org","robots":{"status":"missing","http":404,"url":"https://indymedia.org/robots.txt","bytes":236,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":236},"home":{"status":"ok","http":200,"url":"https://indymedia.org/","html_tokens":3669,"markdown_tokens":808,"pruned_tokens":78}},
{"rank":4243,"domain":"overdrive.com","origin":"https://overdrive.com","robots":{"status":"ok","http":200,"url":"https://www.overdrive.com/robots.txt","bytes":906,"groups":2},"bots":{"GPTBot":["partial","*",26,1],"ChatGPT-User":["partial","*",26,1],"OAI-SearchBot":["partial","*",26,1],"ClaudeBot":["partial","*",26,1],"Claude-User":["partial","*",26,1],"Claude-SearchBot":["partial","*",26,1],"anthropic-ai":["partial","*",26,1],"PerplexityBot":["partial","*",26,1],"Perplexity-User":["partial","*",26,1],"Google-Extended":["partial","*",26,1],"Applebot-Extended":["partial","*",26,1],"CCBot":["partial","*",26,1],"Bytespider":["blocked","named",1,0],"Amazonbot":["partial","*",26,1],"meta-externalagent":["partial","*",26,1],"cohere-ai":["partial","*",26,1],"Diffbot":["partial","*",26,1],"ContextFetchBot":["partial","*",26,1]},"llms":{"status":"missing","http":404,"bytes":0},"home":{"status":"ok","http":200,"url":"https://www.overdrive.com/","html_tokens":7516,"markdown_tokens":34,"pruned_tokens":34}},
{"rank":4244,"domain":"zebra.com","origin":"https://zebra.com","robots":{"status":"ok","http":200,"url":"https://www.zebra.com/robots.txt","bytes":9463,"groups":1},"bots":{"GPTBot":["partial","*",215,1],"ChatGPT-User":["partial","*",215,1],"OAI-SearchBot":["partial","*",215,1],"ClaudeBot":["partial","*",215,1],"Claude-User":["partial","*",215,1],"Claude-SearchBot":["partial","*",215,1],"anthropic-ai":["partial","*",215,1],"PerplexityBot":["partial","*",215,1],"Perplexity-User":["partial","*",215,1],"Google-Extended":["partial","*",215,1],"Applebot-Extended":["partial","*",215,1],"CCBot":["partial","*",215,1],"Bytespider":["partial","*",215,1],"Amazonbot":["partial","*",215,1],"meta-externalagent":["partial","*",215,1],"cohere-ai":["partial","*",215,1],"Diffbot":["partial","*",215,1],"ContextFetchBot":["partial","*",215,1]},"llms":{"status":"too_large","error":"response larger than 5 MB"},"home":{"status":"ok","http":200,"url":"https://www.zebra.com/us/en.html","html_tokens":102331,"markdown_tokens":2485,"pruned_tokens":1382}},
{"rank":4245,"domain":"uni-wuerzburg.de","origin":"https://uni-wuerzburg.de","robots":{"status":"ok","http":200,"url":"https://www.uni-wuerzburg.de/robots.txt","bytes":83,"groups":1},"bots":{"GPTBot":["partial","*",2,0],"ChatGPT-User":["partial","*",2,0],"OAI-SearchBot":["partial","*",2,0],"ClaudeBot":["partial","*",2,0],"Claude-User":["partial","*",2,0],"Claude-SearchBot":["partial","*",2,0],"anthropic-ai":["partial","*",2,0],"PerplexityBot":["partial","*",2,0],"Perplexity-User":["partial","*",2,0],"Google-Extended":["partial","*",2,0],"Applebot-Extended":["partial","*",2,0],"CCBot":["partial","*",2,0],"Bytespider":["partial","*",2,0],"Amazonbot":["partial","*",2,0],"meta-externalagent":["partial","*",2,0],"cohere-ai":["partial","*",2,0],"Diffbot":["partial","*",2,0],"ContextFetchBot":["partial","*",2,0]},"llms":{"status":"missing","http":404,"bytes":69523},"home":{"status":"ok","http":200,"url":"https://www.uni-wuerzburg.de/","html_tokens":25132,"markdown_tokens":546,"pruned_tokens":104}},
{"rank":4246,"domain":"sfsu.edu","origin":"https://sfsu.edu","robots":{"status":"refused","http":403,"url":"https://sfsu.edu/robots.txt","bytes":560,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":4247,"domain":"sandisk.com","origin":"https://sandisk.com","robots":{"status":"ok","http":200,"url":"https://www.sandisk.com/robots.txt","bytes":1313,"groups":3},"bots":{"GPTBot":["partial","*",14,5],"ChatGPT-User":["partial","*",14,5],"OAI-SearchBot":["partial","*",14,5],"ClaudeBot":["partial","*",14,5],"Claude-User":["partial","*",14,5],"Claude-SearchBot":["partial","*",14,5],"anthropic-ai":["partial","*",14,5],"PerplexityBot":["partial","*",14,5],"Perplexity-User":["partial","*",14,5],"Google-Extended":["partial","*",14,5],"Applebot-Extended":["partial","*",14,5],"CCBot":["partial","*",14,5],"Bytespider":["partial","*",14,5],"Amazonbot":["partial","*",14,5],"meta-externalagent":["partial","*",14,5],"cohere-ai":["partial","*",14,5],"Diffbot":["partial","*",14,5],"ContextFetchBot":["partial","*",14,5]},"llms":{"status":"present","http":200,"bytes":40447},"home":{"status":"ok","http":200,"url":"https://www.sandisk.com/","html_tokens":354196,"markdown_tokens":468,"pruned_tokens":304}},
{"rank":4248,"domain":"pennlive.com","origin":"https://pennlive.com","robots":{"status":"ok","http":200,"url":"https://www.pennlive.com/robots.txt","bytes":2858,"groups":4},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["partial","named",1,12],"OAI-SearchBot":["partial","named",1,12],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","named",1,12],"Claude-SearchBot":["partial","named",1,12],"anthropic-ai":["partial","*",30,3],"PerplexityBot":["partial","named",1,12],"Perplexity-User":["partial","named",1,12],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["partial","*",30,3],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",30,3]},"llms":{"status":"http_error","http":403,"bytes":772},"home":{"status":"http_error","http":403,"url":"https://www.pennlive.com/"}},
{"rank":4249,"domain":"iconfinder.com","origin":"https://iconfinder.com","robots":{"status":"refused","http":403,"url":"https://www.freepik.com/icons","bytes":7530,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":4250,"domain":"pastelink.net","origin":"https://pastelink.net","robots":{"status":"ok","http":200,"url":"https://pastelink.net/robots.txt","bytes":92,"groups":1},"bots":{"GPTBot":["partial","*",2,0],"ChatGPT-User":["partial","*",2,0],"OAI-SearchBot":["partial","*",2,0],"ClaudeBot":["partial","*",2,0],"Claude-User":["partial","*",2,0],"Claude-SearchBot":["partial","*",2,0],"anthropic-ai":["partial","*",2,0],"PerplexityBot":["partial","*",2,0],"Perplexity-User":["partial","*",2,0],"Google-Extended":["partial","*",2,0],"Applebot-Extended":["partial","*",2,0],"CCBot":["partial","*",2,0],"Bytespider":["partial","*",2,0],"Amazonbot":["partial","*",2,0],"meta-externalagent":["partial","*",2,0],"cohere-ai":["partial","*",2,0],"Diffbot":["partial","*",2,0],"ContextFetchBot":["partial","*",2,0]},"llms":{"status":"html","http":200,"bytes":13310},"home":{"status":"ok","http":200,"url":"https://pastelink.net/","html_tokens":5673,"markdown_tokens":701,"pruned_tokens":145}},
{"rank":4251,"domain":"borlabs.io","origin":"https://borlabs.io","robots":{"status":"ok","http":200,"url":"https://borlabs.io/robots.txt","bytes":267,"groups":2},"bots":{"GPTBot":["partial","*",2,1],"ChatGPT-User":["partial","*",2,1],"OAI-SearchBot":["partial","*",2,1],"ClaudeBot":["partial","*",2,1],"Claude-User":["partial","*",2,1],"Claude-SearchBot":["partial","*",2,1],"anthropic-ai":["partial","*",2,1],"PerplexityBot":["partial","*",2,1],"Perplexity-User":["partial","*",2,1],"Google-Extended":["partial","*",2,1],"Applebot-Extended":["partial","*",2,1],"CCBot":["partial","*",2,1],"Bytespider":["partial","*",2,1],"Amazonbot":["partial","*",2,1],"meta-externalagent":["partial","*",2,1],"cohere-ai":["partial","*",2,1],"Diffbot":["partial","*",2,1],"ContextFetchBot":["partial","*",2,1]},"llms":{"status":"missing","http":404,"bytes":252746},"home":{"status":"ok","http":200,"url":"https://borlabs.io/","html_tokens":70250,"markdown_tokens":1749,"pruned_tokens":655}},
{"rank":4252,"domain":"plurk.com","origin":"https://plurk.com","robots":{"status":"ok","http":200,"url":"https://www.plurk.com/robots.txt","bytes":304,"groups":1},"bots":{"GPTBot":["partial","*",11,1],"ChatGPT-User":["partial","*",11,1],"OAI-SearchBot":["partial","*",11,1],"ClaudeBot":["partial","*",11,1],"Claude-User":["partial","*",11,1],"Claude-SearchBot":["partial","*",11,1],"anthropic-ai":["partial","*",11,1],"PerplexityBot":["partial","*",11,1],"Perplexity-User":["partial","*",11,1],"Google-Extended":["partial","*",11,1],"Applebot-Extended":["partial","*",11,1],"CCBot":["partial","*",11,1],"Bytespider":["partial","*",11,1],"Amazonbot":["partial","*",11,1],"meta-externalagent":["partial","*",11,1],"cohere-ai":["partial","*",11,1],"Diffbot":["partial","*",11,1],"ContextFetchBot":["partial","*",11,1]},"llms":{"status":"html","http":200,"bytes":9607},"home":{"status":"ok","http":200,"url":"https://www.plurk.com/portal/","html_tokens":19444,"markdown_tokens":27,"pruned_tokens":26}},
{"rank":4253,"domain":"excite.co.jp","origin":"https://excite.co.jp","robots":{"status":"ok","http":200,"url":"https://www.excite.co.jp/robots.txt","bytes":508,"groups":1},"bots":{"GPTBot":["partial","*",15,0],"ChatGPT-User":["partial","*",15,0],"OAI-SearchBot":["partial","*",15,0],"ClaudeBot":["partial","*",15,0],"Claude-User":["partial","*",15,0],"Claude-SearchBot":["partial","*",15,0],"anthropic-ai":["partial","*",15,0],"PerplexityBot":["partial","*",15,0],"Perplexity-User":["partial","*",15,0],"Google-Extended":["partial","*",15,0],"Applebot-Extended":["partial","*",15,0],"CCBot":["partial","*",15,0],"Bytespider":["partial","*",15,0],"Amazonbot":["partial","*",15,0],"meta-externalagent":["partial","*",15,0],"cohere-ai":["partial","*",15,0],"Diffbot":["partial","*",15,0],"ContextFetchBot":["partial","*",15,0]},"llms":{"status":"missing","http":404,"bytes":7723},"home":{"status":"ok","http":200,"url":"https://www.excite.co.jp/","html_tokens":28716,"markdown_tokens":11,"pruned_tokens":11}},
{"rank":4254,"domain":"eatbu.com","origin":"https://eatbu.com","robots":{"status":"ok","http":200,"url":"https://eatbu.com/robots.txt","bytes":127,"groups":1},"bots":{"GPTBot":["partial","*",2,0],"ChatGPT-User":["partial","*",2,0],"OAI-SearchBot":["partial","*",2,0],"ClaudeBot":["partial","*",2,0],"Claude-User":["partial","*",2,0],"Claude-SearchBot":["partial","*",2,0],"anthropic-ai":["partial","*",2,0],"PerplexityBot":["partial","*",2,0],"Perplexity-User":["partial","*",2,0],"Google-Extended":["partial","*",2,0],"Applebot-Extended":["partial","*",2,0],"CCBot":["partial","*",2,0],"Bytespider":["partial","*",2,0],"Amazonbot":["partial","*",2,0],"meta-externalagent":["partial","*",2,0],"cohere-ai":["partial","*",2,0],"Diffbot":["partial","*",2,0],"ContextFetchBot":["partial","*",2,0]},"llms":{"status":"missing","http":404,"bytes":19248},"home":{"status":"ok","http":200,"url":"https://website.dish.co","html_tokens":4788,"markdown_tokens":51,"pruned_tokens":51}},
{"rank":4255,"domain":"iwm.org.uk","origin":"https://iwm.org.uk","robots":{"status":"ok","http":200,"url":"https://www.iwm.org.uk/robots.txt","bytes":2122,"groups":1},"bots":{"GPTBot":["partial","*",35,19],"ChatGPT-User":["partial","*",35,19],"OAI-SearchBot":["partial","*",35,19],"ClaudeBot":["partial","*",35,19],"Claude-User":["partial","*",35,19],"Claude-SearchBot":["partial","*",35,19],"anthropic-ai":["partial","*",35,19],"PerplexityBot":["partial","*",35,19],"Perplexity-User":["partial","*",35,19],"Google-Extended":["partial","*",35,19],"Applebot-Extended":["partial","*",35,19],"CCBot":["partial","*",35,19],"Bytespider":["partial","*",35,19],"Amazonbot":["partial","*",35,19],"meta-externalagent":["partial","*",35,19],"cohere-ai":["partial","*",35,19],"Diffbot":["partial","*",35,19],"ContextFetchBot":["partial","*",35,19]},"llms":{"status":"present","http":200,"bytes":18024},"home":{"status":"http_error","http":403,"url":"https://iwm.org.uk/"}},
{"rank":4256,"domain":"nestle.com","origin":"https://nestle.com","robots":{"status":"ok","http":200,"url":"https://www.nestle.com/robots.txt","bytes":2196,"groups":1},"bots":{"GPTBot":["partial","*",29,18],"ChatGPT-User":["partial","*",29,18],"OAI-SearchBot":["partial","*",29,18],"ClaudeBot":["partial","*",29,18],"Claude-User":["partial","*",29,18],"Claude-SearchBot":["partial","*",29,18],"anthropic-ai":["partial","*",29,18],"PerplexityBot":["partial","*",29,18],"Perplexity-User":["partial","*",29,18],"Google-Extended":["partial","*",29,18],"Applebot-Extended":["partial","*",29,18],"CCBot":["partial","*",29,18],"Bytespider":["partial","*",29,18],"Amazonbot":["partial","*",29,18],"meta-externalagent":["partial","*",29,18],"cohere-ai":["partial","*",29,18],"Diffbot":["partial","*",29,18],"ContextFetchBot":["partial","*",29,18]},"llms":{"status":"http_error","http":403,"bytes":40686},"home":{"status":"http_error","http":403,"url":"https://www.nestle.com/"}},
{"rank":4257,"domain":"guggenheim.org","origin":"https://guggenheim.org","robots":{"status":"refused","http":403,"url":"https://guggenheim.org/robots.txt","bytes":560,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":4258,"domain":"fudan.edu.cn","origin":"https://fudan.edu.cn","robots":{"status":"missing","http":404,"url":"https://fudan.edu.cn/robots.txt","bytes":208,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":206},"home":{"status":"ok","http":200,"url":"https://fudan.edu.cn/","html_tokens":40866,"markdown_tokens":2711,"pruned_tokens":143}},
{"rank":4259,"domain":"einpresswire.com","origin":"https://einpresswire.com","robots":{"status":"ok","http":200,"url":"https://www.einpresswire.com/robots.txt","bytes":695,"groups":3},"bots":{"GPTBot":["partial","*",6,0],"ChatGPT-User":["partial","*",6,0],"OAI-SearchBot":["partial","*",6,0],"ClaudeBot":["partial","*",6,0],"Claude-User":["partial","*",6,0],"Claude-SearchBot":["partial","*",6,0],"anthropic-ai":["partial","*",6,0],"PerplexityBot":["partial","*",6,0],"Perplexity-User":["partial","*",6,0],"Google-Extended":["partial","*",6,0],"Applebot-Extended":["partial","*",6,0],"CCBot":["partial","*",6,0],"Bytespider":["partial","*",6,0],"Amazonbot":["partial","*",6,0],"meta-externalagent":["partial","*",6,0],"cohere-ai":["partial","*",6,0],"Diffbot":["partial","*",6,0],"ContextFetchBot":["partial","*",6,0]},"llms":{"status":"missing","http":404,"bytes":98143},"home":{"status":"ok","http":200,"url":"https://www.einpresswire.com/","html_tokens":33768,"markdown_tokens":64,"pruned_tokens":64}},
{"rank":4260,"domain":"nebraska.gov","origin":"https://nebraska.gov","robots":{"status":"ok","http":200,"url":"https://www.nebraska.gov/robots.txt","bytes":97,"groups":1},"bots":{"GPTBot":["partial","*",2,2],"ChatGPT-User":["partial","*",2,2],"OAI-SearchBot":["partial","*",2,2],"ClaudeBot":["partial","*",2,2],"Claude-User":["partial","*",2,2],"Claude-SearchBot":["partial","*",2,2],"anthropic-ai":["partial","*",2,2],"PerplexityBot":["partial","*",2,2],"Perplexity-User":["partial","*",2,2],"Google-Extended":["partial","*",2,2],"Applebot-Extended":["partial","*",2,2],"CCBot":["partial","*",2,2],"Bytespider":["partial","*",2,2],"Amazonbot":["partial","*",2,2],"meta-externalagent":["partial","*",2,2],"cohere-ai":["partial","*",2,2],"Diffbot":["partial","*",2,2],"ContextFetchBot":["partial","*",2,2]},"llms":{"status":"missing","http":404,"bytes":10077},"home":{"status":"ok","http":200,"url":"https://www.nebraska.gov/","html_tokens":8076,"markdown_tokens":35,"pruned_tokens":35}},
{"rank":4261,"domain":"tf1.fr","origin":"https://tf1.fr","robots":{"status":"ok","http":200,"url":"https://www.tf1.fr/robots.txt","bytes":3653,"groups":4},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["blocked","named",1,0],"OAI-SearchBot":["partial","*",45,1],"ClaudeBot":["blocked","named",1,0],"Claude-User":["blocked","named",1,0],"Claude-SearchBot":["partial","*",45,1],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["partial","*",45,1],"Perplexity-User":["blocked","named",1,0],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",45,1]},"llms":{"status":"missing","http":404,"bytes":565580},"home":{"status":"too_large","error":"response larger than 5 MB"}},
{"rank":4262,"domain":"jisc.ac.uk","origin":"https://jisc.ac.uk","robots":{"status":"unreachable","http":null,"url":"https://jisc.ac.uk/robots.txt","bytes":0,"groups":0,"error":"[SSL: CERTIFICATE_VERIFY_FAILED] certificate verify failed: unable to get local issuer certificate (_ssl.c:1028)"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":4263,"domain":"artic.edu","origin":"https://artic.edu","robots":{"status":"ok","http":200,"url":"https://www.artic.edu/robots.txt","bytes":249,"groups":1},"bots":{"GPTBot":["partial","*",7,0],"ChatGPT-User":["partial","*",7,0],"OAI-SearchBot":["partial","*",7,0],"ClaudeBot":["partial","*",7,0],"Claude-User":["partial","*",7,0],"Claude-SearchBot":["partial","*",7,0],"anthropic-ai":["partial","*",7,0],"PerplexityBot":["partial","*",7,0],"Perplexity-User":["partial","*",7,0],"Google-Extended":["partial","*",7,0],"Applebot-Extended":["partial","*",7,0],"CCBot":["partial","*",7,0],"Bytespider":["partial","*",7,0],"Amazonbot":["partial","*",7,0],"meta-externalagent":["partial","*",7,0],"cohere-ai":["partial","*",7,0],"Diffbot":["partial","*",7,0],"ContextFetchBot":["partial","*",7,0]},"llms":{"status":"missing","http":404,"bytes":167981},"home":{"status":"ok","http":200,"url":"https://www.artic.edu/","html_tokens":65483,"markdown_tokens":218,"pruned_tokens":169}},
{"rank":4264,"domain":"aenetworks.tv","origin":"https://aenetworks.tv","robots":{"status":"html","http":200,"url":"https://www.hearstnetworks.com/","bytes":42660,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":4265,"domain":"iacr.org","origin":"https://iacr.org","robots":{"status":"ok","http":200,"url":"https://iacr.org/robots.txt","bytes":461,"groups":6},"bots":{"GPTBot":["partial","*",12,1],"ChatGPT-User":["partial","*",12,1],"OAI-SearchBot":["partial","*",12,1],"ClaudeBot":["blocked","named",2,0],"Claude-User":["partial","*",12,1],"Claude-SearchBot":["partial","*",12,1],"anthropic-ai":["partial","*",12,1],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["partial","*",12,1],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["partial","*",12,1],"CCBot":["blocked","named",1,0],"Bytespider":["partial","*",12,1],"Amazonbot":["partial","*",12,1],"meta-externalagent":["partial","*",12,1],"cohere-ai":["partial","*",12,1],"Diffbot":["partial","*",12,1],"ContextFetchBot":["partial","*",12,1]},"llms":{"status":"missing","http":404,"bytes":311},"home":{"status":"ok","http":200,"url":"https://iacr.org/","html_tokens":7975,"markdown_tokens":298,"pruned_tokens":78}},
{"rank":4266,"domain":"setmore.com","origin":"https://setmore.com","robots":{"status":"ok","http":200,"url":"https://www.setmore.com/robots.txt","bytes":426,"groups":2},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"present","http":200,"bytes":6922},"home":{"status":"ok","http":200,"url":"https://www.setmore.com/","html_tokens":155220,"markdown_tokens":1035,"pruned_tokens":262}},
{"rank":4267,"domain":"avira.com","origin":"https://avira.com","robots":{"status":"refused","http":403,"url":"https://www.avira.com/robots.txt","bytes":379,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":4268,"domain":"videojs.org","origin":"https://videojs.org","robots":{"status":"ok","http":200,"url":"https://videojs.org/robots.txt","bytes":62,"groups":1},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"present","http":200,"bytes":2814},"home":{"status":"ok","http":200,"url":"https://videojs.org/","html_tokens":38858,"markdown_tokens":216,"pruned_tokens":95}},
{"rank":4269,"domain":"cnipa.gov.cn","origin":"https://www.cnipa.gov.cn","robots":{"status":"missing","http":404,"url":"https://www.cnipa.gov.cn/robots.txt","bytes":146,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":146},"home":{"status":"ok","http":200,"url":"https://www.cnipa.gov.cn/","html_tokens":11186,"markdown_tokens":1,"pruned_tokens":1}},
{"rank":4270,"domain":"angi.com","origin":"https://angi.com","robots":{"status":"ok","http":200,"url":"https://www.angi.com/robots.txt","bytes":3435,"groups":5},"bots":{"GPTBot":["partial","*",13,1],"ChatGPT-User":["partial","*",13,1],"OAI-SearchBot":["partial","*",13,1],"ClaudeBot":["partial","*",13,1],"Claude-User":["partial","*",13,1],"Claude-SearchBot":["partial","*",13,1],"anthropic-ai":["partial","*",13,1],"PerplexityBot":["partial","*",13,1],"Perplexity-User":["partial","*",13,1],"Google-Extended":["partial","*",13,1],"Applebot-Extended":["partial","*",13,1],"CCBot":["partial","*",13,1],"Bytespider":["partial","*",13,1],"Amazonbot":["partial","*",13,1],"meta-externalagent":["partial","*",13,1],"cohere-ai":["partial","*",13,1],"Diffbot":["partial","*",13,1],"ContextFetchBot":["partial","*",13,1]},"llms":{"status":"missing","http":404,"bytes":40315},"home":{"status":"http_error","http":403,"url":"https://www.angi.com/"}},
{"rank":4271,"domain":"seattle.gov","origin":"https://seattle.gov","robots":{"status":"ok","http":200,"url":"https://seattle.gov/robots.txt","bytes":135,"groups":2},"bots":{"GPTBot":["allowed","*",0,1],"ChatGPT-User":["allowed","*",0,1],"OAI-SearchBot":["allowed","*",0,1],"ClaudeBot":["allowed","*",0,1],"Claude-User":["allowed","*",0,1],"Claude-SearchBot":["allowed","*",0,1],"anthropic-ai":["allowed","*",0,1],"PerplexityBot":["allowed","*",0,1],"Perplexity-User":["allowed","*",0,1],"Google-Extended":["allowed","*",0,1],"Applebot-Extended":["allowed","*",0,1],"CCBot":["allowed","*",0,1],"Bytespider":["allowed","*",0,1],"Amazonbot":["allowed","*",0,1],"meta-externalagent":["allowed","*",0,1],"cohere-ai":["allowed","*",0,1],"Diffbot":["allowed","*",0,1],"ContextFetchBot":["allowed","*",0,1]},"llms":{"status":"missing","http":404,"bytes":24248},"home":{"status":"ok","http":200,"url":"https://seattle.gov/","html_tokens":11288,"markdown_tokens":105,"pruned_tokens":67}},
{"rank":4272,"domain":"nalog.ru","origin":"https://nalog.ru","robots":{"status":"ok","http":200,"url":"https://www.nalog.gov.ru/robots.txt","bytes":766,"groups":2},"bots":{"GPTBot":["partial","*",6,0],"ChatGPT-User":["partial","*",6,0],"OAI-SearchBot":["partial","*",6,0],"ClaudeBot":["partial","*",6,0],"Claude-User":["partial","*",6,0],"Claude-SearchBot":["partial","*",6,0],"anthropic-ai":["partial","*",6,0],"PerplexityBot":["partial","*",6,0],"Perplexity-User":["partial","*",6,0],"Google-Extended":["partial","*",6,0],"Applebot-Extended":["partial","*",6,0],"CCBot":["partial","*",6,0],"Bytespider":["partial","*",6,0],"Amazonbot":["partial","*",6,0],"meta-externalagent":["partial","*",6,0],"cohere-ai":["partial","*",6,0],"Diffbot":["partial","*",6,0],"ContextFetchBot":["partial","*",6,0]},"llms":{"status":"missing","http":404,"bytes":467952},"home":{"status":"ok","http":200,"url":"https://www.nalog.gov.ru/rn77/","html_tokens":35291,"markdown_tokens":89,"pruned_tokens":89}},
{"rank":4273,"domain":"singaporeair.com","origin":"https://singaporeair.com","robots":{"status":"missing","http":404,"url":"https://singaporeair.com/robots.txt","bytes":11911,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":11911},"home":{"status":"http_error","http":404,"url":"https://singaporeair.com/"}},
{"rank":4274,"domain":"bts.gov","origin":"https://www.bts.gov","robots":{"status":"refused","http":403,"url":"https://www.bts.gov/robots.txt","bytes":379,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":4275,"domain":"runsignup.com","origin":"https://runsignup.com","robots":{"status":"ok","http":200,"url":"https://runsignup.com/robots.txt","bytes":1450,"groups":15},"bots":{"GPTBot":["partial","*",27,0],"ChatGPT-User":["partial","*",27,0],"OAI-SearchBot":["partial","*",27,0],"ClaudeBot":["partial","*",27,0],"Claude-User":["partial","*",27,0],"Claude-SearchBot":["partial","*",27,0],"anthropic-ai":["partial","*",27,0],"PerplexityBot":["partial","*",27,0],"Perplexity-User":["partial","*",27,0],"Google-Extended":["partial","*",27,0],"Applebot-Extended":["partial","*",27,0],"CCBot":["partial","*",27,0],"Bytespider":["partial","*",27,0],"Amazonbot":["partial","*",27,0],"meta-externalagent":["partial","*",27,0],"cohere-ai":["partial","*",27,0],"Diffbot":["partial","*",27,0],"ContextFetchBot":["partial","*",27,0]},"llms":{"status":"missing","http":404,"bytes":21861},"home":{"status":"ok","http":200,"url":"https://runsignup.com/","html_tokens":30699,"markdown_tokens":2475,"pruned_tokens":281}},
{"rank":4276,"domain":"courant.com","origin":"https://courant.com","robots":{"status":"ok","http":200,"url":"https://www.courant.com/robots.txt","bytes":4472,"groups":68},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["blocked","named",1,0],"OAI-SearchBot":["blocked","named",1,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["blocked","named",1,0],"Claude-SearchBot":["blocked","named",1,0],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["blocked","named",1,0],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["partial","*",8,1],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",8,1]},"llms":{"status":"missing","http":404,"bytes":76844},"home":{"status":"ok","http":200,"url":"https://www.courant.com/","html_tokens":59271,"markdown_tokens":248,"pruned_tokens":85}},
{"rank":4277,"domain":"heraldscotland.com","origin":"https://heraldscotland.com","robots":{"status":"ok","http":200,"url":"https://www.heraldscotland.com/robots.txt","bytes":3291,"groups":3},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["blocked","named",1,0],"OAI-SearchBot":["blocked","named",1,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["blocked","named",1,0],"Claude-SearchBot":["blocked","named",1,0],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["partial","*",7,0],"Google-Extended":["partial","*",7,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",7,0]},"llms":{"status":"http_error","http":503,"bytes":209},"home":{"status":"ok","http":200,"url":"https://www.heraldscotland.com","html_tokens":105604,"markdown_tokens":39,"pruned_tokens":39}},
{"rank":4278,"domain":"exlibrisgroup.com","origin":"https://exlibrisgroup.com","robots":{"status":"refused","http":403,"url":"https://exlibrisgroup.com/robots.txt","bytes":560,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":4279,"domain":"google.rs","origin":"https://google.rs","robots":{"status":"ok","http":200,"url":"https://www.google.rs/robots.txt","bytes":6624,"groups":4},"bots":{"GPTBot":["partial","*",169,64],"ChatGPT-User":["partial","*",169,64],"OAI-SearchBot":["partial","*",169,64],"ClaudeBot":["partial","*",169,64],"Claude-User":["partial","*",169,64],"Claude-SearchBot":["partial","*",169,64],"anthropic-ai":["partial","*",169,64],"PerplexityBot":["partial","*",169,64],"Perplexity-User":["partial","*",169,64],"Google-Extended":["partial","*",169,64],"Applebot-Extended":["partial","*",169,64],"CCBot":["partial","*",169,64],"Bytespider":["partial","*",169,64],"Amazonbot":["partial","*",169,64],"meta-externalagent":["partial","*",169,64],"cohere-ai":["partial","*",169,64],"Diffbot":["partial","*",169,64],"ContextFetchBot":["partial","*",169,64]},"llms":{"status":"missing","http":404,"bytes":1569},"home":{"status":"ok","http":200,"url":"https://www.google.com/","html_tokens":21341,"markdown_tokens":30,"pruned_tokens":30}},
{"rank":4280,"domain":"rss.com","origin":"https://rss.com","robots":{"status":"ok","http":200,"url":"https://rss.com/robots.txt","bytes":127,"groups":2},"bots":{"GPTBot":["allowed","*",0,1],"ChatGPT-User":["allowed","*",0,1],"OAI-SearchBot":["allowed","*",0,1],"ClaudeBot":["allowed","*",0,1],"Claude-User":["allowed","*",0,1],"Claude-SearchBot":["allowed","*",0,1],"anthropic-ai":["allowed","*",0,1],"PerplexityBot":["allowed","*",0,1],"Perplexity-User":["allowed","*",0,1],"Google-Extended":["allowed","*",0,1],"Applebot-Extended":["allowed","*",0,1],"CCBot":["allowed","*",0,1],"Bytespider":["blocked","named",1,0],"Amazonbot":["allowed","*",0,1],"meta-externalagent":["allowed","*",0,1],"cohere-ai":["allowed","*",0,1],"Diffbot":["allowed","*",0,1],"ContextFetchBot":["allowed","*",0,1]},"llms":{"status":"missing","http":404,"bytes":99994},"home":{"status":"ok","http":200,"url":"https://rss.com/","html_tokens":37694,"markdown_tokens":861,"pruned_tokens":217}},
{"rank":4281,"domain":"mex.com","origin":"https://mex.com","robots":{"status":"ok","http":200,"url":"https://mex.com/robots.txt","bytes":0,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":7624},"home":{"status":"ok","http":200,"url":"https://mex.com/","html_tokens":9064,"markdown_tokens":868,"pruned_tokens":854}},
{"rank":4282,"domain":"sudouest.fr","origin":"https://sudouest.fr","robots":{"status":"ok","http":200,"url":"https://www.sudouest.fr/robots.txt","bytes":4912,"groups":105},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["blocked","named",1,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["blocked","named",1,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":111843},"home":{"status":"ok","http":200,"url":"https://www.sudouest.fr/","html_tokens":155275,"markdown_tokens":904,"pruned_tokens":121}},
{"rank":4283,"domain":"docdroid.net","origin":"https://docdroid.net","robots":{"status":"ok","http":200,"url":"https://docdroid.net/robots.txt","bytes":24,"groups":1},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"missing","http":404,"bytes":11224},"home":{"status":"ok","http":200,"url":"https://www.docdroid.com","html_tokens":6834,"markdown_tokens":172,"pruned_tokens":172}},
{"rank":4284,"domain":"pokemon.com","origin":"https://pokemon.com","robots":{"status":"missing","http":404,"url":"https://www.pokemon.com/robots.txt","bytes":15230,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":15230},"home":{"status":"ok","http":200,"url":"http://www.pokemon.com/","html_tokens":259,"markdown_tokens":21,"pruned_tokens":21}},
{"rank":4285,"domain":"doubao.com","origin":"https://doubao.com","robots":{"status":"ok","http":200,"url":"https://www.doubao.com/robots.txt","bytes":115,"groups":1},"bots":{"GPTBot":["partial","*",5,1],"ChatGPT-User":["partial","*",5,1],"OAI-SearchBot":["partial","*",5,1],"ClaudeBot":["partial","*",5,1],"Claude-User":["partial","*",5,1],"Claude-SearchBot":["partial","*",5,1],"anthropic-ai":["partial","*",5,1],"PerplexityBot":["partial","*",5,1],"Perplexity-User":["partial","*",5,1],"Google-Extended":["partial","*",5,1],"Applebot-Extended":["partial","*",5,1],"CCBot":["partial","*",5,1],"Bytespider":["partial","*",5,1],"Amazonbot":["partial","*",5,1],"meta-externalagent":["partial","*",5,1],"cohere-ai":["partial","*",5,1],"Diffbot":["partial","*",5,1],"ContextFetchBot":["partial","*",5,1]},"llms":{"status":"html","http":200,"bytes":192194},"home":{"status":"ok","http":200,"url":"https://www.doubao.com/chat/","html_tokens":68586,"markdown_tokens":0,"pruned_tokens":0}},
{"rank":4286,"domain":"newadvent.org","origin":"https://newadvent.org","robots":{"status":"ok","http":200,"url":"https://www.newadvent.org/robots.txt","bytes":93,"groups":2},"bots":{"GPTBot":["partial","*",2,0],"ChatGPT-User":["partial","*",2,0],"OAI-SearchBot":["partial","*",2,0],"ClaudeBot":["partial","*",2,0],"Claude-User":["partial","*",2,0],"Claude-SearchBot":["partial","*",2,0],"anthropic-ai":["partial","*",2,0],"PerplexityBot":["partial","*",2,0],"Perplexity-User":["partial","*",2,0],"Google-Extended":["partial","*",2,0],"Applebot-Extended":["partial","*",2,0],"CCBot":["partial","*",2,0],"Bytespider":["partial","*",2,0],"Amazonbot":["partial","*",2,0],"meta-externalagent":["partial","*",2,0],"cohere-ai":["partial","*",2,0],"Diffbot":["partial","*",2,0],"ContextFetchBot":["partial","*",2,0]},"llms":{"status":"missing","http":404,"bytes":563},"home":{"status":"ok","http":200,"url":"https://www.newadvent.org/","html_tokens":10249,"markdown_tokens":3264,"pruned_tokens":172}},
{"rank":4287,"domain":"sportskeeda.com","origin":"https://sportskeeda.com","robots":{"status":"refused","http":405,"url":"https://www.sportskeeda.com/robots.txt","bytes":2213,"groups":0,"error":"HTTP 405"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":4288,"domain":"ibge.gov.br","origin":"https://www.ibge.gov.br","robots":{"status":"ok","http":200,"url":"https://www.ibge.gov.br/robots.txt","bytes":1882,"groups":1},"bots":{"GPTBot":["partial","*",17,40],"ChatGPT-User":["partial","*",17,40],"OAI-SearchBot":["partial","*",17,40],"ClaudeBot":["partial","*",17,40],"Claude-User":["partial","*",17,40],"Claude-SearchBot":["partial","*",17,40],"anthropic-ai":["partial","*",17,40],"PerplexityBot":["partial","*",17,40],"Perplexity-User":["partial","*",17,40],"Google-Extended":["partial","*",17,40],"Applebot-Extended":["partial","*",17,40],"CCBot":["partial","*",17,40],"Bytespider":["partial","*",17,40],"Amazonbot":["partial","*",17,40],"meta-externalagent":["partial","*",17,40],"cohere-ai":["partial","*",17,40],"Diffbot":["partial","*",17,40],"ContextFetchBot":["partial","*",17,40]},"llms":{"status":"http_error","http":403,"bytes":5547},"home":{"status":"http_error","http":403,"url":"https://www.ibge.gov.br/"}},
{"rank":4289,"domain":"belastingdienst.nl","origin":"https://belastingdienst.nl","robots":{"status":"ok","http":200,"url":"https://www.belastingdienst.nl/robots.txt","bytes":3037,"groups":1},"bots":{"GPTBot":["partial","*",41,0],"ChatGPT-User":["partial","*",41,0],"OAI-SearchBot":["partial","*",41,0],"ClaudeBot":["partial","*",41,0],"Claude-User":["partial","*",41,0],"Claude-SearchBot":["partial","*",41,0],"anthropic-ai":["partial","*",41,0],"PerplexityBot":["partial","*",41,0],"Perplexity-User":["partial","*",41,0],"Google-Extended":["partial","*",41,0],"Applebot-Extended":["partial","*",41,0],"CCBot":["partial","*",41,0],"Bytespider":["partial","*",41,0],"Amazonbot":["partial","*",41,0],"meta-externalagent":["partial","*",41,0],"cohere-ai":["partial","*",41,0],"Diffbot":["partial","*",41,0],"ContextFetchBot":["partial","*",41,0]},"llms":{"status":"missing","http":404,"bytes":11323},"home":{"status":"ok","http":200,"url":"https://www.belastingdienst.nl/wps/wcm/connect/nl/home/home","html_tokens":6394,"markdown_tokens":88,"pruned_tokens":26}},
{"rank":4290,"domain":"leagueoflegends.com","origin":"https://leagueoflegends.com","robots":{"status":"ok","http":200,"url":"https://www.leagueoflegends.com/robots.txt","bytes":84,"groups":1},"bots":{"GPTBot":["allowed","*",0,1],"ChatGPT-User":["allowed","*",0,1],"OAI-SearchBot":["allowed","*",0,1],"ClaudeBot":["allowed","*",0,1],"Claude-User":["allowed","*",0,1],"Claude-SearchBot":["allowed","*",0,1],"anthropic-ai":["allowed","*",0,1],"PerplexityBot":["allowed","*",0,1],"Perplexity-User":["allowed","*",0,1],"Google-Extended":["allowed","*",0,1],"Applebot-Extended":["allowed","*",0,1],"CCBot":["allowed","*",0,1],"Bytespider":["allowed","*",0,1],"Amazonbot":["allowed","*",0,1],"meta-externalagent":["allowed","*",0,1],"cohere-ai":["allowed","*",0,1],"Diffbot":["allowed","*",0,1],"ContextFetchBot":["allowed","*",0,1]},"llms":{"status":"missing","http":404,"bytes":2482},"home":{"status":"ok","http":200,"url":"https://www.leagueoflegends.com/en-us/","html_tokens":64414,"markdown_tokens":138,"pruned_tokens":138}},
{"rank":4291,"domain":"nuget.org","origin":"https://nuget.org","robots":{"status":"ok","http":200,"url":"https://www.nuget.org/public/robots.txt","bytes":149,"groups":1},"bots":{"GPTBot":["partial","*",4,0],"ChatGPT-User":["partial","*",4,0],"OAI-SearchBot":["partial","*",4,0],"ClaudeBot":["partial","*",4,0],"Claude-User":["partial","*",4,0],"Claude-SearchBot":["partial","*",4,0],"anthropic-ai":["partial","*",4,0],"PerplexityBot":["partial","*",4,0],"Perplexity-User":["partial","*",4,0],"Google-Extended":["partial","*",4,0],"Applebot-Extended":["partial","*",4,0],"CCBot":["partial","*",4,0],"Bytespider":["partial","*",4,0],"Amazonbot":["partial","*",4,0],"meta-externalagent":["partial","*",4,0],"cohere-ai":["partial","*",4,0],"Diffbot":["partial","*",4,0],"ContextFetchBot":["partial","*",4,0]},"llms":{"status":"missing","http":404,"bytes":13665},"home":{"status":"ok","http":200,"url":"https://www.nuget.org/","html_tokens":4212,"markdown_tokens":200,"pruned_tokens":115}},
{"rank":4292,"domain":"doi.gov","origin":"https://doi.gov","robots":{"status":"ok","http":200,"url":"https://www.doi.gov/robots.txt","bytes":2403,"groups":1},"bots":{"GPTBot":["partial","*",35,22],"ChatGPT-User":["partial","*",35,22],"OAI-SearchBot":["partial","*",35,22],"ClaudeBot":["partial","*",35,22],"Claude-User":["partial","*",35,22],"Claude-SearchBot":["partial","*",35,22],"anthropic-ai":["partial","*",35,22],"PerplexityBot":["partial","*",35,22],"Perplexity-User":["partial","*",35,22],"Google-Extended":["partial","*",35,22],"Applebot-Extended":["partial","*",35,22],"CCBot":["partial","*",35,22],"Bytespider":["partial","*",35,22],"Amazonbot":["partial","*",35,22],"meta-externalagent":["partial","*",35,22],"cohere-ai":["partial","*",35,22],"Diffbot":["partial","*",35,22],"ContextFetchBot":["partial","*",35,22]},"llms":{"status":"missing","http":404,"bytes":185},"home":{"status":"ok","http":200,"url":"https://www.doi.gov/","html_tokens":14494,"markdown_tokens":99,"pruned_tokens":99}},
{"rank":4293,"domain":"moneysavingexpert.com","origin":"https://moneysavingexpert.com","robots":{"status":"ok","http":200,"url":"https://www.moneysavingexpert.com/robots.txt","bytes":2420,"groups":3},"bots":{"GPTBot":["allowed","named",0,0],"ChatGPT-User":["allowed","named",0,0],"OAI-SearchBot":["allowed","named",0,0],"ClaudeBot":["allowed","named",0,0],"Claude-User":["allowed","named",0,0],"Claude-SearchBot":["allowed","named",0,0],"anthropic-ai":["partial","*",16,2],"PerplexityBot":["allowed","named",0,0],"Perplexity-User":["allowed","named",0,0],"Google-Extended":["partial","*",16,2],"Applebot-Extended":["partial","*",16,2],"CCBot":["allowed","named",0,0],"Bytespider":["partial","*",16,2],"Amazonbot":["partial","*",16,2],"meta-externalagent":["partial","*",16,2],"cohere-ai":["partial","*",16,2],"Diffbot":["partial","*",16,2],"ContextFetchBot":["partial","*",16,2]},"llms":{"status":"missing","http":404,"bytes":424128},"home":{"status":"http_error","http":403,"url":"https://moneysavingexpert.com/"}},
{"rank":4294,"domain":"unilever.com","origin":"https://unilever.com","robots":{"status":"refused","http":403,"url":"https://www.unilever.com/robots.txt","bytes":384,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":4295,"domain":"prnt.sc","origin":"https://prnt.sc","robots":{"status":"ok","http":200,"url":"https://prnt.sc/robots.txt","bytes":40,"groups":1},"bots":{"GPTBot":["allowed","*",0,1],"ChatGPT-User":["allowed","*",0,1],"OAI-SearchBot":["allowed","*",0,1],"ClaudeBot":["allowed","*",0,1],"Claude-User":["allowed","*",0,1],"Claude-SearchBot":["allowed","*",0,1],"anthropic-ai":["allowed","*",0,1],"PerplexityBot":["allowed","*",0,1],"Perplexity-User":["allowed","*",0,1],"Google-Extended":["allowed","*",0,1],"Applebot-Extended":["allowed","*",0,1],"CCBot":["allowed","*",0,1],"Bytespider":["allowed","*",0,1],"Amazonbot":["allowed","*",0,1],"meta-externalagent":["allowed","*",0,1],"cohere-ai":["allowed","*",0,1],"Diffbot":["allowed","*",0,1],"ContextFetchBot":["allowed","*",0,1]},"llms":{"status":"html","http":200,"bytes":16424},"home":{"status":"ok","http":200,"url":"https://prnt.sc/","html_tokens":3279,"markdown_tokens":13,"pruned_tokens":13}},
{"rank":4296,"domain":"klook.com","origin":"https://klook.com","robots":{"status":"ok","http":200,"url":"https://www.klook.com/robots.txt","bytes":3129,"groups":5},"bots":{"GPTBot":["partial","*",16,3],"ChatGPT-User":["partial","*",16,3],"OAI-SearchBot":["partial","*",16,3],"ClaudeBot":["partial","*",16,3],"Claude-User":["partial","*",16,3],"Claude-SearchBot":["partial","*",16,3],"anthropic-ai":["partial","*",16,3],"PerplexityBot":["partial","*",16,3],"Perplexity-User":["partial","*",16,3],"Google-Extended":["partial","*",16,3],"Applebot-Extended":["partial","*",16,3],"CCBot":["partial","*",16,3],"Bytespider":["partial","*",16,3],"Amazonbot":["partial","*",16,3],"meta-externalagent":["partial","*",16,3],"cohere-ai":["partial","*",16,3],"Diffbot":["partial","*",16,3],"ContextFetchBot":["partial","*",16,3]},"llms":{"status":"present","http":200,"bytes":7782},"home":{"status":"http_error","http":403,"url":"https://www.klook.com/"}},
{"rank":4297,"domain":"merck.com","origin":"https://merck.com","robots":{"status":"ok","http":200,"url":"https://www.merck.com/robots.txt","bytes":115,"groups":1},"bots":{"GPTBot":["partial","*",1,1],"ChatGPT-User":["partial","*",1,1],"OAI-SearchBot":["partial","*",1,1],"ClaudeBot":["partial","*",1,1],"Claude-User":["partial","*",1,1],"Claude-SearchBot":["partial","*",1,1],"anthropic-ai":["partial","*",1,1],"PerplexityBot":["partial","*",1,1],"Perplexity-User":["partial","*",1,1],"Google-Extended":["partial","*",1,1],"Applebot-Extended":["partial","*",1,1],"CCBot":["partial","*",1,1],"Bytespider":["partial","*",1,1],"Amazonbot":["partial","*",1,1],"meta-externalagent":["partial","*",1,1],"cohere-ai":["partial","*",1,1],"Diffbot":["partial","*",1,1],"ContextFetchBot":["partial","*",1,1]},"llms":{"status":"missing","http":404,"bytes":59493},"home":{"status":"ok","http":200,"url":"https://www.merck.com/","html_tokens":33525,"markdown_tokens":211,"pruned_tokens":126}},
{"rank":4298,"domain":"kew.org","origin":"https://kew.org","robots":{"status":"refused","http":403,"url":"https://www.kew.org/robots.txt","bytes":5548,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":4299,"domain":"plusone.google.com","origin":"https://plusone.google.com","robots":{"status":"missing","http":404,"url":"https://plusone.google.com/robots.txt","bytes":1571,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":1569},"home":{"status":"disallowed","error":"robots.txt of plus.google.com not readable (html)"}},
{"rank":4300,"domain":"kharkov.ua","origin":"https://www.kharkov.ua","robots":{"status":"ok","http":200,"url":"https://www.kharkov.ua/robots.txt","bytes":243,"groups":2},"bots":{"GPTBot":["partial","*",3,0],"ChatGPT-User":["partial","*",3,0],"OAI-SearchBot":["partial","*",3,0],"ClaudeBot":["partial","*",3,0],"Claude-User":["partial","*",3,0],"Claude-SearchBot":["partial","*",3,0],"anthropic-ai":["partial","*",3,0],"PerplexityBot":["partial","*",3,0],"Perplexity-User":["partial","*",3,0],"Google-Extended":["partial","*",3,0],"Applebot-Extended":["partial","*",3,0],"CCBot":["partial","*",3,0],"Bytespider":["partial","*",3,0],"Amazonbot":["partial","*",3,0],"meta-externalagent":["partial","*",3,0],"cohere-ai":["partial","*",3,0],"Diffbot":["partial","*",3,0],"ContextFetchBot":["partial","*",3,0]},"llms":{"status":"missing","http":404,"bytes":236},"home":{"status":"ok","http":200,"url":"https://www.kharkov.ua/","html_tokens":9533,"markdown_tokens":179,"pruned_tokens":21}},
{"rank":4301,"domain":"autohome.com.cn","origin":"https://autohome.com.cn","robots":{"status":"timeout","http":null,"url":"https://autohome.com.cn/robots.txt","bytes":0,"groups":0,"error":"ConnectTimeout"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":4302,"domain":"company.site","origin":"https://company.site","robots":{"status":"refused","http":403,"url":"https://company.site/robots.txt","bytes":263,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":4303,"domain":"ranker.com","origin":"https://ranker.com","robots":{"status":"ok","http":200,"url":"https://ranker.com/robots.txt","bytes":3406,"groups":4},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["partial","named",11,1],"OAI-SearchBot":["partial","named",11,1],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","named",11,1],"Claude-SearchBot":["partial","named",11,1],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["partial","named",11,1],"Perplexity-User":["partial","named",11,1],"Google-Extended":["partial","*",11,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",11,0]},"llms":{"status":"missing","http":404,"bytes":15774},"home":{"status":"ok","http":200,"url":"https://www.ranker.com/","html_tokens":117328,"markdown_tokens":497,"pruned_tokens":88}},
{"rank":4304,"domain":"vrt.be","origin":"https://vrt.be","robots":{"status":"ok","http":200,"url":"https://www.vrt.be/robots.txt","bytes":1099,"groups":3},"bots":{"GPTBot":["partial","named",1,0],"ChatGPT-User":["partial","named",1,0],"OAI-SearchBot":["partial","named",1,0],"ClaudeBot":["partial","named",1,0],"Claude-User":["partial","*",6,0],"Claude-SearchBot":["partial","*",6,0],"anthropic-ai":["partial","named",1,0],"PerplexityBot":["partial","named",1,0],"Perplexity-User":["partial","*",6,0],"Google-Extended":["partial","named",1,0],"Applebot-Extended":["partial","named",1,0],"CCBot":["partial","named",1,0],"Bytespider":["partial","named",1,0],"Amazonbot":["partial","named",1,0],"meta-externalagent":["partial","named",1,0],"cohere-ai":["partial","named",1,0],"Diffbot":["partial","named",1,0],"ContextFetchBot":["partial","*",6,0]},"llms":{"status":"missing","http":404,"bytes":236},"home":{"status":"ok","http":200,"url":"https://www.vrt.be/nl","html_tokens":102873,"markdown_tokens":165,"pruned_tokens":165}},
{"rank":4305,"domain":"sd.gov","origin":"https://sd.gov","robots":{"status":"ok","http":200,"url":"https://www.sd.gov/robots.txt","bytes":0,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":196},"home":{"status":"ok","http":200,"url":"https://www.sd.gov/cs","html_tokens":243808,"markdown_tokens":20,"pruned_tokens":20}},
{"rank":4306,"domain":"educause.edu","origin":"https://educause.edu","robots":{"status":"ok","http":200,"url":"https://www.educause.edu/robots.txt","bytes":902,"groups":10},"bots":{"GPTBot":["partial","*",9,0],"ChatGPT-User":["partial","*",9,0],"OAI-SearchBot":["partial","*",9,0],"ClaudeBot":["partial","*",9,0],"Claude-User":["partial","*",9,0],"Claude-SearchBot":["partial","*",9,0],"anthropic-ai":["partial","*",9,0],"PerplexityBot":["partial","*",9,0],"Perplexity-User":["partial","*",9,0],"Google-Extended":["partial","*",9,0],"Applebot-Extended":["partial","*",9,0],"CCBot":["partial","*",9,0],"Bytespider":["partial","*",9,0],"Amazonbot":["partial","*",9,0],"meta-externalagent":["partial","*",9,0],"cohere-ai":["partial","*",9,0],"Diffbot":["partial","*",9,0],"ContextFetchBot":["partial","*",9,0]},"llms":{"status":"missing","http":404,"bytes":56171},"home":{"status":"http_error","http":403,"url":"https://educause.edu/"}},
{"rank":4307,"domain":"register.it","origin":"https://register.it","robots":{"status":"ok","http":200,"url":"https://www.register.it/robots.txt","bytes":282,"groups":1},"bots":{"GPTBot":["partial","*",13,0],"ChatGPT-User":["partial","*",13,0],"OAI-SearchBot":["partial","*",13,0],"ClaudeBot":["partial","*",13,0],"Claude-User":["partial","*",13,0],"Claude-SearchBot":["partial","*",13,0],"anthropic-ai":["partial","*",13,0],"PerplexityBot":["partial","*",13,0],"Perplexity-User":["partial","*",13,0],"Google-Extended":["partial","*",13,0],"Applebot-Extended":["partial","*",13,0],"CCBot":["partial","*",13,0],"Bytespider":["partial","*",13,0],"Amazonbot":["partial","*",13,0],"meta-externalagent":["partial","*",13,0],"cohere-ai":["partial","*",13,0],"Diffbot":["partial","*",13,0],"ContextFetchBot":["partial","*",13,0]},"llms":{"status":"missing","http":404,"bytes":19},"home":{"status":"ok","http":200,"url":"https://www.register.it/","html_tokens":33562,"markdown_tokens":258,"pruned_tokens":34}},
{"rank":4308,"domain":"thisoldhouse.com","origin":"https://thisoldhouse.com","robots":{"status":"ok","http":200,"url":"https://thisoldhouse.com/robots.txt","bytes":1091,"groups":20},"bots":{"GPTBot":["allowed","named",0,1],"ChatGPT-User":["partial","*",8,0],"OAI-SearchBot":["partial","*",8,0],"ClaudeBot":["allowed","named",0,1],"Claude-User":["allowed","named",0,1],"Claude-SearchBot":["allowed","named",0,1],"anthropic-ai":["partial","*",8,0],"PerplexityBot":["allowed","named",0,1],"Perplexity-User":["partial","*",8,0],"Google-Extended":["allowed","named",0,1],"Applebot-Extended":["partial","*",8,0],"CCBot":["allowed","named",0,1],"Bytespider":["partial","*",8,0],"Amazonbot":["partial","*",8,0],"meta-externalagent":["partial","*",8,0],"cohere-ai":["partial","*",8,0],"Diffbot":["partial","*",8,0],"ContextFetchBot":["partial","*",8,0]},"llms":{"status":"missing","http":404,"bytes":197896},"home":{"status":"ok","http":200,"url":"https://www.thisoldhouse.com/","html_tokens":117965,"markdown_tokens":205,"pruned_tokens":86}},
{"rank":4309,"domain":"government.nl","origin":"https://government.nl","robots":{"status":"ok","http":200,"url":"https://www.government.nl/robots.txt","bytes":62,"groups":1},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"missing","http":404,"bytes":148559},"home":{"status":"ok","http":200,"url":"https://www.government.nl/","html_tokens":77289,"markdown_tokens":457,"pruned_tokens":169}},
{"rank":4310,"domain":"bhf.org.uk","origin":"https://bhf.org.uk","robots":{"status":"ok","http":200,"url":"https://www.bhf.org.uk/robots.txt","bytes":20481,"groups":557},"bots":{"GPTBot":["partial","*",5,2],"ChatGPT-User":["partial","*",5,2],"OAI-SearchBot":["partial","*",5,2],"ClaudeBot":["partial","*",5,2],"Claude-User":["partial","*",5,2],"Claude-SearchBot":["partial","*",5,2],"anthropic-ai":["partial","*",5,2],"PerplexityBot":["partial","*",5,2],"Perplexity-User":["partial","*",5,2],"Google-Extended":["partial","*",5,2],"Applebot-Extended":["partial","*",5,2],"CCBot":["blocked","named",1,0],"Bytespider":["partial","*",5,2],"Amazonbot":["partial","*",5,2],"meta-externalagent":["partial","*",5,2],"cohere-ai":["partial","*",5,2],"Diffbot":["partial","*",5,2],"ContextFetchBot":["partial","*",5,2]},"llms":{"status":"missing","http":404,"bytes":85506},"home":{"status":"ok","http":200,"url":"https://www.bhf.org.uk/","html_tokens":30737,"markdown_tokens":304,"pruned_tokens":304}},
{"rank":4311,"domain":"equifax.com","origin":"https://equifax.com","robots":{"status":"ok","http":200,"url":"https://www.equifax.com/robots.txt","bytes":575,"groups":3},"bots":{"GPTBot":["partial","*",13,0],"ChatGPT-User":["partial","*",13,0],"OAI-SearchBot":["partial","*",13,0],"ClaudeBot":["partial","*",13,0],"Claude-User":["partial","*",13,0],"Claude-SearchBot":["partial","*",13,0],"anthropic-ai":["partial","*",13,0],"PerplexityBot":["partial","*",13,0],"Perplexity-User":["partial","*",13,0],"Google-Extended":["partial","*",13,0],"Applebot-Extended":["partial","*",13,0],"CCBot":["partial","*",13,0],"Bytespider":["partial","*",13,0],"Amazonbot":["partial","*",13,0],"meta-externalagent":["partial","*",13,0],"cohere-ai":["partial","*",13,0],"Diffbot":["partial","*",13,0],"ContextFetchBot":["partial","*",13,0]},"llms":{"status":"other_text","http":200,"bytes":13665},"home":{"status":"ok","http":200,"url":"https://www.equifax.com/","html_tokens":56540,"markdown_tokens":566,"pruned_tokens":227}},
{"rank":4312,"domain":"radio.fm","origin":"https://radio.fm","robots":{"status":"missing","http":404,"url":"https://radio.fm/robots.txt","bytes":1245,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":1245},"home":{"status":"ok","http":200,"url":"https://radio.fm/","html_tokens":7814,"markdown_tokens":144,"pruned_tokens":96}},
{"rank":4313,"domain":"businessofapps.com","origin":"https://businessofapps.com","robots":{"status":"refused","http":403,"url":"https://businessofapps.com/robots.txt","bytes":5560,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":4314,"domain":"europe1.fr","origin":"https://europe1.fr","robots":{"status":"ok","http":200,"url":"https://www.europe1.fr/robots.txt","bytes":1165,"groups":3},"bots":{"GPTBot":["partial","*",40,0],"ChatGPT-User":["partial","*",40,0],"OAI-SearchBot":["partial","*",40,0],"ClaudeBot":["partial","*",40,0],"Claude-User":["partial","*",40,0],"Claude-SearchBot":["partial","*",40,0],"anthropic-ai":["partial","*",40,0],"PerplexityBot":["partial","*",40,0],"Perplexity-User":["partial","*",40,0],"Google-Extended":["partial","*",40,0],"Applebot-Extended":["partial","*",40,0],"CCBot":["blocked","named",1,0],"Bytespider":["partial","*",40,0],"Amazonbot":["partial","*",40,0],"meta-externalagent":["partial","*",40,0],"cohere-ai":["partial","*",40,0],"Diffbot":["partial","*",40,0],"ContextFetchBot":["partial","*",40,0]},"llms":{"status":"missing","http":404,"bytes":188},"home":{"status":"http_error","http":403,"url":"https://www.europe1.fr/"}},
{"rank":4315,"domain":"smartrecruiters.com","origin":"https://smartrecruiters.com","robots":{"status":"ok","http":200,"url":"https://www.smartrecruiters.com/robots.txt","bytes":2153,"groups":1},"bots":{"GPTBot":["partial","*",73,0],"ChatGPT-User":["partial","*",73,0],"OAI-SearchBot":["partial","*",73,0],"ClaudeBot":["partial","*",73,0],"Claude-User":["partial","*",73,0],"Claude-SearchBot":["partial","*",73,0],"anthropic-ai":["partial","*",73,0],"PerplexityBot":["partial","*",73,0],"Perplexity-User":["partial","*",73,0],"Google-Extended":["partial","*",73,0],"Applebot-Extended":["partial","*",73,0],"CCBot":["partial","*",73,0],"Bytespider":["partial","*",73,0],"Amazonbot":["partial","*",73,0],"meta-externalagent":["partial","*",73,0],"cohere-ai":["partial","*",73,0],"Diffbot":["partial","*",73,0],"ContextFetchBot":["partial","*",73,0]},"llms":{"status":"other_text","http":200,"bytes":643594},"home":{"status":"ok","http":200,"url":"https://www.smartrecruiters.com/","html_tokens":54451,"markdown_tokens":155,"pruned_tokens":62}},
{"rank":4316,"domain":"cars.com","origin":"https://cars.com","robots":{"status":"refused","http":403,"url":"https://www.cars.com/robots.txt","bytes":4558,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":4317,"domain":"giz.de","origin":"https://giz.de","robots":{"status":"ok","http":200,"url":"https://www.giz.de/robots.txt","bytes":2027,"groups":1},"bots":{"GPTBot":["partial","*",34,18],"ChatGPT-User":["partial","*",34,18],"OAI-SearchBot":["partial","*",34,18],"ClaudeBot":["partial","*",34,18],"Claude-User":["partial","*",34,18],"Claude-SearchBot":["partial","*",34,18],"anthropic-ai":["partial","*",34,18],"PerplexityBot":["partial","*",34,18],"Perplexity-User":["partial","*",34,18],"Google-Extended":["partial","*",34,18],"Applebot-Extended":["partial","*",34,18],"CCBot":["partial","*",34,18],"Bytespider":["partial","*",34,18],"Amazonbot":["partial","*",34,18],"meta-externalagent":["partial","*",34,18],"cohere-ai":["partial","*",34,18],"Diffbot":["partial","*",34,18],"ContextFetchBot":["partial","*",34,18]},"llms":{"status":"missing","http":404,"bytes":63000},"home":{"status":"ok","http":200,"url":"https://www.giz.de/de","html_tokens":30253,"markdown_tokens":133,"pruned_tokens":87}},
{"rank":4318,"domain":"plex.tv","origin":"https://plex.tv","robots":{"status":"ok","http":200,"url":"https://plex.tv/robots.txt","bytes":120,"groups":1},"bots":{"GPTBot":["partial","*",6,0],"ChatGPT-User":["partial","*",6,0],"OAI-SearchBot":["partial","*",6,0],"ClaudeBot":["partial","*",6,0],"Claude-User":["partial","*",6,0],"Claude-SearchBot":["partial","*",6,0],"anthropic-ai":["partial","*",6,0],"PerplexityBot":["partial","*",6,0],"Perplexity-User":["partial","*",6,0],"Google-Extended":["partial","*",6,0],"Applebot-Extended":["partial","*",6,0],"CCBot":["partial","*",6,0],"Bytespider":["partial","*",6,0],"Amazonbot":["partial","*",6,0],"meta-externalagent":["partial","*",6,0],"cohere-ai":["partial","*",6,0],"Diffbot":["partial","*",6,0],"ContextFetchBot":["partial","*",6,0]},"llms":{"status":"missing","http":404,"bytes":1118},"home":{"status":"ok","http":200,"url":"https://watch.plex.tv","html_tokens":827121,"markdown_tokens":479,"pruned_tokens":414}},
{"rank":4319,"domain":"rbi.org.in","origin":"https://rbi.org.in","robots":{"status":"refused","http":418,"url":"https://rbi.org.in/robots.txt","bytes":626,"groups":0,"error":"HTTP 418"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":4320,"domain":"lehigh.edu","origin":"https://lehigh.edu","robots":{"status":"ok","http":200,"url":"https://www.lehigh.edu/robots.txt","bytes":113,"groups":1},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"missing","http":404,"bytes":635},"home":{"status":"ok","http":200,"url":"https://www2.lehigh.edu/","html_tokens":51024,"markdown_tokens":249,"pruned_tokens":184}},
{"rank":4321,"domain":"uscg.mil","origin":"https://www.uscg.mil","robots":{"status":"refused","http":403,"url":"https://www.uscg.mil/robots.txt","bytes":378,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":4322,"domain":"kinopoisk.ru","origin":"https://kinopoisk.ru","robots":{"status":"ok","http":200,"url":"https://www.kinopoisk.ru/robots.txt","bytes":2629,"groups":2},"bots":{"GPTBot":["partial","*",21,0],"ChatGPT-User":["partial","*",21,0],"OAI-SearchBot":["partial","*",21,0],"ClaudeBot":["partial","*",21,0],"Claude-User":["partial","*",21,0],"Claude-SearchBot":["partial","*",21,0],"anthropic-ai":["partial","*",21,0],"PerplexityBot":["partial","*",21,0],"Perplexity-User":["partial","*",21,0],"Google-Extended":["partial","*",21,0],"Applebot-Extended":["partial","*",21,0],"CCBot":["partial","*",21,0],"Bytespider":["partial","*",21,0],"Amazonbot":["partial","*",21,0],"meta-externalagent":["partial","*",21,0],"cohere-ai":["partial","*",21,0],"Diffbot":["partial","*",21,0],"ContextFetchBot":["partial","*",21,0]},"llms":{"status":"missing","http":404,"bytes":10241},"home":{"status":"ok","http":200,"url":"https://www.kinopoisk.ru/","html_tokens":88520,"markdown_tokens":20,"pruned_tokens":20}},
{"rank":4323,"domain":"financialcontent.com","origin":"https://financialcontent.com","robots":{"status":"html","http":200,"url":"https://www.financialcontent.com/stocks","bytes":112467,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":4324,"domain":"expansion.com","origin":"https://expansion.com","robots":{"status":"ok","http":200,"url":"https://www.expansion.com/robots.txt","bytes":5126,"groups":117},"bots":{"GPTBot":["partial","named",1,3],"ChatGPT-User":["partial","named",1,3],"OAI-SearchBot":["partial","named",1,3],"ClaudeBot":["partial","*",13,0],"Claude-User":["partial","*",13,0],"Claude-SearchBot":["partial","*",13,0],"anthropic-ai":["partial","named",1,3],"PerplexityBot":["partial","named",1,3],"Perplexity-User":["partial","*",13,0],"Google-Extended":["partial","named",1,3],"Applebot-Extended":["partial","*",13,0],"CCBot":["partial","named",1,3],"Bytespider":["partial","*",13,0],"Amazonbot":["partial","*",13,0],"meta-externalagent":["partial","*",13,0],"cohere-ai":["partial","*",13,0],"Diffbot":["partial","*",13,0],"ContextFetchBot":["partial","*",13,0]},"llms":{"status":"missing","http":404,"bytes":162},"home":{"status":"ok","http":200,"url":"https://www.expansion.com/","html_tokens":130658,"markdown_tokens":454,"pruned_tokens":150}},
{"rank":4325,"domain":"square-enix.com","origin":"https://square-enix.com","robots":{"status":"missing","http":404,"url":"https://www.square-enix.com/robots.txt","bytes":5651,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":5651},"home":{"status":"ok","http":200,"url":"https://www.square-enix.com/","html_tokens":1795,"markdown_tokens":90,"pruned_tokens":78}},
{"rank":4326,"domain":"clickn.co.kr","origin":"https://clickn.co.kr","robots":{"status":"ok","http":200,"url":"https://www.clickn.co.kr/robots.txt","bytes":139,"groups":1},"bots":{"GPTBot":["partial","*",3,0],"ChatGPT-User":["partial","*",3,0],"OAI-SearchBot":["partial","*",3,0],"ClaudeBot":["partial","*",3,0],"Claude-User":["partial","*",3,0],"Claude-SearchBot":["partial","*",3,0],"anthropic-ai":["partial","*",3,0],"PerplexityBot":["partial","*",3,0],"Perplexity-User":["partial","*",3,0],"Google-Extended":["partial","*",3,0],"Applebot-Extended":["partial","*",3,0],"CCBot":["partial","*",3,0],"Bytespider":["partial","*",3,0],"Amazonbot":["partial","*",3,0],"meta-externalagent":["partial","*",3,0],"cohere-ai":["partial","*",3,0],"Diffbot":["partial","*",3,0],"ContextFetchBot":["partial","*",3,0]},"llms":{"status":"html","http":200,"bytes":2375},"home":{"status":"ok","http":200,"url":"https://www.clickn.co.kr/","html_tokens":14869,"markdown_tokens":51,"pruned_tokens":51}},
{"rank":4327,"domain":"wordpressfoundation.org","origin":"https://wordpressfoundation.org","robots":{"status":"ok","http":200,"url":"https://wordpressfoundation.org/robots.txt","bytes":184,"groups":1},"bots":{"GPTBot":["partial","*",1,1],"ChatGPT-User":["partial","*",1,1],"OAI-SearchBot":["partial","*",1,1],"ClaudeBot":["partial","*",1,1],"Claude-User":["partial","*",1,1],"Claude-SearchBot":["partial","*",1,1],"anthropic-ai":["partial","*",1,1],"PerplexityBot":["partial","*",1,1],"Perplexity-User":["partial","*",1,1],"Google-Extended":["partial","*",1,1],"Applebot-Extended":["partial","*",1,1],"CCBot":["partial","*",1,1],"Bytespider":["partial","*",1,1],"Amazonbot":["partial","*",1,1],"meta-externalagent":["partial","*",1,1],"cohere-ai":["partial","*",1,1],"Diffbot":["partial","*",1,1],"ContextFetchBot":["partial","*",1,1]},"llms":{"status":"missing","http":404,"bytes":76145},"home":{"status":"ok","http":200,"url":"https://wordpressfoundation.org/","html_tokens":18238,"markdown_tokens":319,"pruned_tokens":169}},
{"rank":4328,"domain":"techinasia.com","origin":"https://techinasia.com","robots":{"status":"refused","http":403,"url":"https://techinasia.com/robots.txt","bytes":118,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":4329,"domain":"ms.gov","origin":"https://ms.gov","robots":{"status":"ok","http":200,"url":"https://www.ms.gov/robots.txt","bytes":2027,"groups":1},"bots":{"GPTBot":["partial","*",34,18],"ChatGPT-User":["partial","*",34,18],"OAI-SearchBot":["partial","*",34,18],"ClaudeBot":["partial","*",34,18],"Claude-User":["partial","*",34,18],"Claude-SearchBot":["partial","*",34,18],"anthropic-ai":["partial","*",34,18],"PerplexityBot":["partial","*",34,18],"Perplexity-User":["partial","*",34,18],"Google-Extended":["partial","*",34,18],"Applebot-Extended":["partial","*",34,18],"CCBot":["partial","*",34,18],"Bytespider":["partial","*",34,18],"Amazonbot":["partial","*",34,18],"meta-externalagent":["partial","*",34,18],"cohere-ai":["partial","*",34,18],"Diffbot":["partial","*",34,18],"ContextFetchBot":["partial","*",34,18]},"llms":{"status":"missing","http":404,"bytes":291},"home":{"status":"ok","http":200,"url":"https://www.ms.gov/","html_tokens":11072,"markdown_tokens":64,"pruned_tokens":64}},
{"rank":4330,"domain":"csun.edu","origin":"https://csun.edu","robots":{"status":"ok","http":200,"url":"https://www.csun.edu/robots.txt","bytes":2231,"groups":1},"bots":{"GPTBot":["partial","*",37,32],"ChatGPT-User":["partial","*",37,32],"OAI-SearchBot":["partial","*",37,32],"ClaudeBot":["partial","*",37,32],"Claude-User":["partial","*",37,32],"Claude-SearchBot":["partial","*",37,32],"anthropic-ai":["partial","*",37,32],"PerplexityBot":["partial","*",37,32],"Perplexity-User":["partial","*",37,32],"Google-Extended":["partial","*",37,32],"Applebot-Extended":["partial","*",37,32],"CCBot":["partial","*",37,32],"Bytespider":["partial","*",37,32],"Amazonbot":["partial","*",37,32],"meta-externalagent":["partial","*",37,32],"cohere-ai":["partial","*",37,32],"Diffbot":["partial","*",37,32],"ContextFetchBot":["partial","*",37,32]},"llms":{"status":"unreachable","error":"ReadError"},"home":{"status":"ok","http":200,"url":"https://www.csun.edu/","html_tokens":22913,"markdown_tokens":130,"pruned_tokens":72}},
{"rank":4331,"domain":"lush.com","origin":"https://lush.com","robots":{"status":"refused","http":403,"url":"https://www.lush.com/robots.txt","bytes":5554,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":4332,"domain":"wechat.com","origin":"https://wechat.com","robots":{"status":"missing","http":404,"url":"https://www.wechat.com/robots.txt","bytes":0,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":0},"home":{"status":"ok","http":200,"url":"https://www.wechat.com/en","html_tokens":21619,"markdown_tokens":49,"pruned_tokens":49}},
{"rank":4333,"domain":"clickcease.com","origin":"https://clickcease.com","robots":{"status":"ok","http":200,"url":"https://www.clickcease.com/robots.txt","bytes":877,"groups":1},"bots":{"GPTBot":["partial","*",18,0],"ChatGPT-User":["partial","*",18,0],"OAI-SearchBot":["partial","*",18,0],"ClaudeBot":["partial","*",18,0],"Claude-User":["partial","*",18,0],"Claude-SearchBot":["partial","*",18,0],"anthropic-ai":["partial","*",18,0],"PerplexityBot":["partial","*",18,0],"Perplexity-User":["partial","*",18,0],"Google-Extended":["partial","*",18,0],"Applebot-Extended":["partial","*",18,0],"CCBot":["partial","*",18,0],"Bytespider":["partial","*",18,0],"Amazonbot":["partial","*",18,0],"meta-externalagent":["partial","*",18,0],"cohere-ai":["partial","*",18,0],"Diffbot":["partial","*",18,0],"ContextFetchBot":["partial","*",18,0]},"llms":{"status":"missing","http":404,"bytes":59326},"home":{"status":"ok","http":200,"url":"https://www.clickcease.com/","html_tokens":40952,"markdown_tokens":111,"pruned_tokens":65}},
{"rank":4334,"domain":"sonarsource.com","origin":"https://sonarsource.com","robots":{"status":"ok","http":200,"url":"https://www.sonarsource.com/robots.txt","bytes":1101,"groups":2},"bots":{"GPTBot":["partial","named",2,1],"ChatGPT-User":["partial","named",2,1],"OAI-SearchBot":["partial","named",2,1],"ClaudeBot":["partial","named",2,1],"Claude-User":["partial","named",2,1],"Claude-SearchBot":["partial","named",2,1],"anthropic-ai":["partial","named",2,1],"PerplexityBot":["partial","named",2,1],"Perplexity-User":["partial","named",2,1],"Google-Extended":["partial","named",2,1],"Applebot-Extended":["partial","named",2,1],"CCBot":["partial","named",2,1],"Bytespider":["partial","named",2,1],"Amazonbot":["partial","*",2,1],"meta-externalagent":["partial","named",2,1],"cohere-ai":["partial","named",2,1],"Diffbot":["partial","named",2,1],"ContextFetchBot":["partial","*",2,1]},"llms":{"status":"present","http":200,"bytes":21159},"home":{"status":"ok","http":200,"url":"https://www.sonarsource.com/","html_tokens":260309,"markdown_tokens":799,"pruned_tokens":88}},
{"rank":4335,"domain":"usyd.edu.au","origin":"https://usyd.edu.au","robots":{"status":"ok","http":200,"url":"https://www.sydney.edu.au/robots.txt","bytes":2137,"groups":2},"bots":{"GPTBot":["partial","*",34,4],"ChatGPT-User":["partial","*",34,4],"OAI-SearchBot":["partial","*",34,4],"ClaudeBot":["partial","*",34,4],"Claude-User":["partial","*",34,4],"Claude-SearchBot":["partial","*",34,4],"anthropic-ai":["partial","*",34,4],"PerplexityBot":["partial","*",34,4],"Perplexity-User":["partial","*",34,4],"Google-Extended":["partial","*",34,4],"Applebot-Extended":["partial","*",34,4],"CCBot":["partial","*",34,4],"Bytespider":["partial","*",34,4],"Amazonbot":["partial","*",34,4],"meta-externalagent":["partial","*",34,4],"cohere-ai":["partial","*",34,4],"Diffbot":["partial","*",34,4],"ContextFetchBot":["partial","*",34,4]},"llms":{"status":"html","http":200,"bytes":18537},"home":{"status":"ok","http":200,"url":"https://www.sydney.edu.au/","html_tokens":16972,"markdown_tokens":155,"pruned_tokens":80}},
{"rank":4336,"domain":"jpmorganchase.com","origin":"https://jpmorganchase.com","robots":{"status":"ok","http":200,"url":"https://www.jpmorganchase.com/robots.txt","bytes":437,"groups":1},"bots":{"GPTBot":["partial","*",6,1],"ChatGPT-User":["partial","*",6,1],"OAI-SearchBot":["partial","*",6,1],"ClaudeBot":["partial","*",6,1],"Claude-User":["partial","*",6,1],"Claude-SearchBot":["partial","*",6,1],"anthropic-ai":["partial","*",6,1],"PerplexityBot":["partial","*",6,1],"Perplexity-User":["partial","*",6,1],"Google-Extended":["partial","*",6,1],"Applebot-Extended":["partial","*",6,1],"CCBot":["partial","*",6,1],"Bytespider":["partial","*",6,1],"Amazonbot":["partial","*",6,1],"meta-externalagent":["partial","*",6,1],"cohere-ai":["partial","*",6,1],"Diffbot":["partial","*",6,1],"ContextFetchBot":["partial","*",6,1]},"llms":{"status":"missing","http":404,"bytes":6957},"home":{"status":"ok","http":200,"url":"https://www.jpmorganchase.com/","html_tokens":28683,"markdown_tokens":140,"pruned_tokens":64}},
{"rank":4337,"domain":"positivepsychology.com","origin":"https://positivepsychology.com","robots":{"status":"ok","http":200,"url":"https://positivepsychology.com/robots.txt","bytes":2509,"groups":40},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["allowed","named",0,1],"OAI-SearchBot":["allowed","named",0,1],"ClaudeBot":["blocked","named",1,0],"Claude-User":["allowed","named",0,1],"Claude-SearchBot":["allowed","named",0,1],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["allowed","named",0,1],"Perplexity-User":["allowed","named",0,1],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["partial","*",11,1],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["partial","*",11,1],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",11,1]},"llms":{"status":"http_error","http":403,"bytes":5810},"home":{"status":"http_error","http":403,"url":"https://positivepsychology.com/"}},
{"rank":4338,"domain":"square.link","origin":"https://square.link","robots":{"status":"refused","http":405,"url":"https://square.link/robots.txt","bytes":177,"groups":0,"error":"HTTP 405"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":4339,"domain":"biz.ua","origin":"https://biz.ua","robots":{"status":"unreachable","http":null,"url":"https://biz.ua/robots.txt","bytes":0,"groups":0,"error":"cannot resolve biz.ua"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":4340,"domain":"fullerton.edu","origin":"https://www.fullerton.edu","robots":{"status":"ok","http":200,"url":"https://www.fullerton.edu/robots.txt","bytes":46,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":1606},"home":{"status":"ok","http":200,"url":"https://www.fullerton.edu/","html_tokens":30729,"markdown_tokens":294,"pruned_tokens":102}},
{"rank":4341,"domain":"newsbreak.com","origin":"https://newsbreak.com","robots":{"status":"ok","http":200,"url":"https://www.newsbreak.com/robots.txt","bytes":5029,"groups":37},"bots":{"GPTBot":["partial","named",1,6],"ChatGPT-User":["partial","named",1,6],"OAI-SearchBot":["partial","named",1,6],"ClaudeBot":["partial","named",1,6],"Claude-User":["partial","*",12,0],"Claude-SearchBot":["partial","*",12,0],"anthropic-ai":["partial","named",1,6],"PerplexityBot":["partial","named",1,6],"Perplexity-User":["partial","named",1,6],"Google-Extended":["partial","named",1,6],"Applebot-Extended":["partial","named",1,6],"CCBot":["partial","named",1,6],"Bytespider":["blocked","named",1,0],"Amazonbot":["partial","named",1,6],"meta-externalagent":["partial","named",2,12],"cohere-ai":["partial","named",1,6],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",12,0]},"llms":{"status":"present","http":200,"bytes":1887},"home":{"status":"ok","http":200,"url":"https://www.newsbreak.com/","html_tokens":140140,"markdown_tokens":138,"pruned_tokens":138}},
{"rank":4342,"domain":"igi-global.com","origin":"https://igi-global.com","robots":{"status":"refused","http":429,"url":"https://igi-global.com/robots.txt","bytes":1484,"groups":0,"error":"HTTP 429"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":4343,"domain":"wikipedia.com","origin":"https://wikipedia.com","robots":{"status":"ok","http":200,"url":"https://en.wikipedia.org/robots.txt","bytes":28275,"groups":33},"bots":{"GPTBot":["partial","*",428,4],"ChatGPT-User":["partial","*",428,4],"OAI-SearchBot":["partial","*",428,4],"ClaudeBot":["partial","*",428,4],"Claude-User":["partial","*",428,4],"Claude-SearchBot":["partial","*",428,4],"anthropic-ai":["partial","*",428,4],"PerplexityBot":["partial","*",428,4],"Perplexity-User":["partial","*",428,4],"Google-Extended":["partial","*",428,4],"Applebot-Extended":["partial","*",428,4],"CCBot":["partial","*",428,4],"Bytespider":["partial","*",428,4],"Amazonbot":["partial","*",428,4],"meta-externalagent":["partial","*",428,4],"cohere-ai":["partial","*",428,4],"Diffbot":["partial","*",428,4],"ContextFetchBot":["partial","*",428,4]},"llms":{"status":"missing","http":404,"bytes":1868},"home":{"status":"ok","http":200,"url":"https://www.wikipedia.org/","html_tokens":30020,"markdown_tokens":665,"pruned_tokens":298}},
{"rank":4344,"domain":"joelonsoftware.com","origin":"https://joelonsoftware.com","robots":{"status":"ok","http":200,"url":"https://www.joelonsoftware.com/robots.txt","bytes":175,"groups":1},"bots":{"GPTBot":["partial","*",1,1],"ChatGPT-User":["partial","*",1,1],"OAI-SearchBot":["partial","*",1,1],"ClaudeBot":["partial","*",1,1],"Claude-User":["partial","*",1,1],"Claude-SearchBot":["partial","*",1,1],"anthropic-ai":["partial","*",1,1],"PerplexityBot":["partial","*",1,1],"Perplexity-User":["partial","*",1,1],"Google-Extended":["partial","*",1,1],"Applebot-Extended":["partial","*",1,1],"CCBot":["partial","*",1,1],"Bytespider":["partial","*",1,1],"Amazonbot":["partial","*",1,1],"meta-externalagent":["partial","*",1,1],"cohere-ai":["partial","*",1,1],"Diffbot":["partial","*",1,1],"ContextFetchBot":["partial","*",1,1]},"llms":{"status":"missing","http":404,"bytes":49533},"home":{"status":"ok","http":200,"url":"https://www.joelonsoftware.com/","html_tokens":21705,"markdown_tokens":252,"pruned_tokens":93}},
{"rank":4345,"domain":"free-daily-spins.com","origin":"https://free-daily-spins.com","robots":{"status":"ok","http":200,"url":"https://free-daily-spins.com/robots.txt","bytes":734,"groups":1},"bots":{"GPTBot":["partial","*",15,13],"ChatGPT-User":["partial","*",15,13],"OAI-SearchBot":["partial","*",15,13],"ClaudeBot":["partial","*",15,13],"Claude-User":["partial","*",15,13],"Claude-SearchBot":["partial","*",15,13],"anthropic-ai":["partial","*",15,13],"PerplexityBot":["partial","*",15,13],"Perplexity-User":["partial","*",15,13],"Google-Extended":["partial","*",15,13],"Applebot-Extended":["partial","*",15,13],"CCBot":["partial","*",15,13],"Bytespider":["partial","*",15,13],"Amazonbot":["partial","*",15,13],"meta-externalagent":["partial","*",15,13],"cohere-ai":["partial","*",15,13],"Diffbot":["partial","*",15,13],"ContextFetchBot":["partial","*",15,13]},"llms":{"status":"missing","http":404,"bytes":52441},"home":{"status":"ok","http":200,"url":"https://free-daily-spins.com/","html_tokens":27216,"markdown_tokens":1461,"pruned_tokens":419}},
{"rank":4346,"domain":"podigee.io","origin":"https://www.podigee.io","robots":{"status":"ok","http":200,"url":"https://www.podigee.com/robots.txt","bytes":68,"groups":1},"bots":{"GPTBot":["allowed","*",0,1],"ChatGPT-User":["allowed","*",0,1],"OAI-SearchBot":["allowed","*",0,1],"ClaudeBot":["allowed","*",0,1],"Claude-User":["allowed","*",0,1],"Claude-SearchBot":["allowed","*",0,1],"anthropic-ai":["allowed","*",0,1],"PerplexityBot":["allowed","*",0,1],"Perplexity-User":["allowed","*",0,1],"Google-Extended":["allowed","*",0,1],"Applebot-Extended":["allowed","*",0,1],"CCBot":["allowed","*",0,1],"Bytespider":["allowed","*",0,1],"Amazonbot":["allowed","*",0,1],"meta-externalagent":["allowed","*",0,1],"cohere-ai":["allowed","*",0,1],"Diffbot":["allowed","*",0,1],"ContextFetchBot":["allowed","*",0,1]},"llms":{"status":"missing","http":404,"bytes":70873},"home":{"status":"ok","http":200,"url":"https://www.podigee.com","html_tokens":33027,"markdown_tokens":1759,"pruned_tokens":249}},
{"rank":4347,"domain":"j.mp","origin":"https://j.mp","robots":{"status":"ok","http":200,"url":"https://j.mp/robots.txt","bytes":123,"groups":1},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"missing","http":404,"bytes":6075},"home":{"status":"http_error","http":404,"url":"https://j.mp/"}},
{"rank":4348,"domain":"smartinsights.com","origin":"https://smartinsights.com","robots":{"status":"ok","http":200,"url":"https://www.smartinsights.com/robots.txt","bytes":179,"groups":1},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"missing","http":404,"bytes":0},"home":{"status":"http_error","http":403,"url":"https://smartinsights.com/"}},
{"rank":4349,"domain":"zp.ua","origin":"http://zp.ua","robots":{"status":"missing","http":404,"url":"http://zp.ua/robots.txt","bytes":236,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":236},"home":{"status":"ok","http":200,"url":"http://zp.ua/","html_tokens":144,"markdown_tokens":21,"pruned_tokens":21}},
{"rank":4350,"domain":"nc.us","origin":"https://nc.us","robots":{"status":"unreachable","http":null,"url":"https://nc.us/robots.txt","bytes":0,"groups":0,"error":"cannot resolve nc.us"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":4351,"domain":"oed.com","origin":"https://oed.com","robots":{"status":"ok","http":200,"url":"https://www.oed.com/robots.txt?tl=true","bytes":309,"groups":3},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["partial","*",4,1],"OAI-SearchBot":["partial","*",4,1],"ClaudeBot":["partial","*",4,1],"Claude-User":["partial","*",4,1],"Claude-SearchBot":["partial","*",4,1],"anthropic-ai":["partial","*",4,1],"PerplexityBot":["partial","*",4,1],"Perplexity-User":["partial","*",4,1],"Google-Extended":["partial","*",4,1],"Applebot-Extended":["partial","*",4,1],"CCBot":["partial","*",4,1],"Bytespider":["partial","*",4,1],"Amazonbot":["partial","*",4,1],"meta-externalagent":["partial","*",4,1],"cohere-ai":["partial","*",4,1],"Diffbot":["partial","*",4,1],"ContextFetchBot":["partial","*",4,1]},"llms":{"status":"disallowed","error":"robots.txt of www.oed.com disallows /oauth2/authorization/transparent?p=eyJlbmMiOiJBMjU2Q0JDLUhTNTEyIiwiYWxnIjoiZGlyIn0..8r649irzM2H7LtT"},"home":{"status":"disallowed","error":"robots.txt of www.oed.com disallows /oauth2/authorization/transparent?p=eyJlbmMiOiJBMjU2Q0JDLUhTNTEyIiwiYWxnIjoiZGlyIn0..MsdtzojUfYYdx02"}},
{"rank":4352,"domain":"marksandspencer.com","origin":"https://marksandspencer.com","robots":{"status":"ok","http":200,"url":"https://www.marksandspencer.com/robots.txt","bytes":4293,"groups":1},"bots":{"GPTBot":["partial","*",139,3],"ChatGPT-User":["partial","*",139,3],"OAI-SearchBot":["partial","*",139,3],"ClaudeBot":["partial","*",139,3],"Claude-User":["partial","*",139,3],"Claude-SearchBot":["partial","*",139,3],"anthropic-ai":["partial","*",139,3],"PerplexityBot":["partial","*",139,3],"Perplexity-User":["partial","*",139,3],"Google-Extended":["partial","*",139,3],"Applebot-Extended":["partial","*",139,3],"CCBot":["partial","*",139,3],"Bytespider":["partial","*",139,3],"Amazonbot":["partial","*",139,3],"meta-externalagent":["partial","*",139,3],"cohere-ai":["partial","*",139,3],"Diffbot":["partial","*",139,3],"ContextFetchBot":["partial","*",139,3]},"llms":{"status":"missing","http":404,"bytes":108927},"home":{"status":"ok","http":200,"url":"https://www.marksandspencer.com/","html_tokens":143625,"markdown_tokens":704,"pruned_tokens":158}},
{"rank":4353,"domain":"uservoice.com","origin":"https://uservoice.com","robots":{"status":"ok","http":200,"url":"https://uservoice.com/robots.txt","bytes":198,"groups":2},"bots":{"GPTBot":["allowed","*",0,1],"ChatGPT-User":["allowed","*",0,1],"OAI-SearchBot":["allowed","*",0,1],"ClaudeBot":["allowed","*",0,1],"Claude-User":["allowed","*",0,1],"Claude-SearchBot":["allowed","*",0,1],"anthropic-ai":["allowed","*",0,1],"PerplexityBot":["allowed","*",0,1],"Perplexity-User":["allowed","*",0,1],"Google-Extended":["allowed","*",0,1],"Applebot-Extended":["allowed","*",0,1],"CCBot":["allowed","*",0,1],"Bytespider":["allowed","*",0,1],"Amazonbot":["allowed","*",0,1],"meta-externalagent":["allowed","*",0,1],"cohere-ai":["allowed","*",0,1],"Diffbot":["allowed","*",0,1],"ContextFetchBot":["allowed","*",0,1]},"llms":{"status":"present","http":200,"bytes":6506},"home":{"status":"ok","http":200,"url":"https://uservoice.com/","html_tokens":13983,"markdown_tokens":602,"pruned_tokens":343}},
{"rank":4354,"domain":"opensource.com","origin":"https://opensource.com","robots":{"status":"ok","http":200,"url":"https://opensource.com/robots.txt","bytes":2027,"groups":1},"bots":{"GPTBot":["partial","*",34,18],"ChatGPT-User":["partial","*",34,18],"OAI-SearchBot":["partial","*",34,18],"ClaudeBot":["partial","*",34,18],"Claude-User":["partial","*",34,18],"Claude-SearchBot":["partial","*",34,18],"anthropic-ai":["partial","*",34,18],"PerplexityBot":["partial","*",34,18],"Perplexity-User":["partial","*",34,18],"Google-Extended":["partial","*",34,18],"Applebot-Extended":["partial","*",34,18],"CCBot":["partial","*",34,18],"Bytespider":["partial","*",34,18],"Amazonbot":["partial","*",34,18],"meta-externalagent":["partial","*",34,18],"cohere-ai":["partial","*",34,18],"Diffbot":["partial","*",34,18],"ContextFetchBot":["partial","*",34,18]},"llms":{"status":"missing","http":404,"bytes":15429},"home":{"status":"ok","http":200,"url":"https://opensource.com/","html_tokens":11419,"markdown_tokens":33,"pruned_tokens":33}},
{"rank":4355,"domain":"gq-magazine.co.uk","origin":"https://gq-magazine.co.uk","robots":{"status":"ok","http":200,"url":"https://www.gq-magazine.co.uk/robots.txt","bytes":2179,"groups":3},"bots":{"GPTBot":["partial","*",14,12],"ChatGPT-User":["partial","*",14,12],"OAI-SearchBot":["partial","*",14,12],"ClaudeBot":["blocked","named",1,0],"Claude-User":["blocked","named",1,0],"Claude-SearchBot":["blocked","named",1,0],"anthropic-ai":["partial","*",14,12],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["blocked","named",1,0],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",14,12]},"llms":{"status":"missing","http":404,"bytes":686381},"home":{"status":"ok","http":200,"url":"https://www.gq-magazine.co.uk/","html_tokens":447215,"markdown_tokens":1997,"pruned_tokens":125}},
{"rank":4356,"domain":"virgin.com","origin":"https://virgin.com","robots":{"status":"refused","http":403,"url":"https://www.virgin.com/robots.txt","bytes":919,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":4357,"domain":"casino.org","origin":"https://casino.org","robots":{"status":"ok","http":200,"url":"https://www.casino.org/robots.txt","bytes":2585,"groups":2},"bots":{"GPTBot":["partial","*",49,2],"ChatGPT-User":["partial","*",49,2],"OAI-SearchBot":["partial","*",49,2],"ClaudeBot":["partial","*",49,2],"Claude-User":["partial","*",49,2],"Claude-SearchBot":["partial","*",49,2],"anthropic-ai":["partial","*",49,2],"PerplexityBot":["partial","*",49,2],"Perplexity-User":["partial","*",49,2],"Google-Extended":["partial","*",49,2],"Applebot-Extended":["partial","*",49,2],"CCBot":["partial","*",49,2],"Bytespider":["partial","*",49,2],"Amazonbot":["partial","*",49,2],"meta-externalagent":["partial","*",49,2],"cohere-ai":["partial","*",49,2],"Diffbot":["partial","*",49,2],"ContextFetchBot":["partial","*",49,2]},"llms":{"status":"missing","http":404,"bytes":7362},"home":{"status":"ok","http":200,"url":"https://www.casino.org/","html_tokens":217293,"markdown_tokens":1100,"pruned_tokens":259}},
{"rank":4358,"domain":"lesoir.be","origin":"https://lesoir.be","robots":{"status":"ok","http":200,"url":"https://www.lesoir.be/robots.txt","bytes":6179,"groups":3},"bots":{"GPTBot":["partial","named",1,7],"ChatGPT-User":["partial","named",1,7],"OAI-SearchBot":["partial","named",1,7],"ClaudeBot":["partial","named",1,7],"Claude-User":["partial","named",1,7],"Claude-SearchBot":["partial","named",1,7],"anthropic-ai":["partial","named",1,7],"PerplexityBot":["partial","named",1,7],"Perplexity-User":["partial","named",1,7],"Google-Extended":["partial","named",1,7],"Applebot-Extended":["partial","named",1,7],"CCBot":["partial","named",1,7],"Bytespider":["partial","named",1,7],"Amazonbot":["partial","named",1,7],"meta-externalagent":["partial","named",1,7],"cohere-ai":["partial","named",1,7],"Diffbot":["partial","named",1,7],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"present","http":200,"bytes":1664},"home":{"status":"ok","http":200,"url":"https://www.lesoir.be/","html_tokens":155853,"markdown_tokens":988,"pruned_tokens":138}},
{"rank":4359,"domain":"ixbt.com","origin":"https://ixbt.com","robots":{"status":"ok","http":200,"url":"https://ixbt.com/robots.txt","bytes":2987,"groups":2},"bots":{"GPTBot":["partial","*",95,10],"ChatGPT-User":["partial","*",95,10],"OAI-SearchBot":["partial","*",95,10],"ClaudeBot":["partial","*",95,10],"Claude-User":["partial","*",95,10],"Claude-SearchBot":["partial","*",95,10],"anthropic-ai":["partial","*",95,10],"PerplexityBot":["partial","*",95,10],"Perplexity-User":["partial","*",95,10],"Google-Extended":["partial","*",95,10],"Applebot-Extended":["partial","*",95,10],"CCBot":["partial","*",95,10],"Bytespider":["partial","*",95,10],"Amazonbot":["partial","*",95,10],"meta-externalagent":["partial","*",95,10],"cohere-ai":["partial","*",95,10],"Diffbot":["partial","*",95,10],"ContextFetchBot":["partial","*",95,10]},"llms":{"status":"missing","http":404,"bytes":153564},"home":{"status":"ok","http":200,"url":"https://www.ixbt.com/","html_tokens":171604,"markdown_tokens":113,"pruned_tokens":113}},
{"rank":4360,"domain":"ind.br","origin":"https://ind.br","robots":{"status":"unreachable","http":null,"url":"https://ind.br/robots.txt","bytes":0,"groups":0,"error":"cannot resolve ind.br"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":4361,"domain":"medpagetoday.com","origin":"https://medpagetoday.com","robots":{"status":"ok","http":200,"url":"https://www.medpagetoday.com/robots.txt","bytes":5644,"groups":2},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["blocked","named",1,0],"OAI-SearchBot":["blocked","named",1,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["blocked","named",1,0],"Claude-SearchBot":["blocked","named",1,0],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["blocked","named",1,0],"Google-Extended":["partial","*",4,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",4,0]},"llms":{"status":"missing","http":404,"bytes":141227},"home":{"status":"ok","http":200,"url":"https://www.medpagetoday.com/","html_tokens":48664,"markdown_tokens":313,"pruned_tokens":58}},
{"rank":4362,"domain":"crn.com","origin":"https://crn.com","robots":{"status":"ok","http":200,"url":"https://www.crn.com/robots.txt","bytes":1057,"groups":3},"bots":{"GPTBot":["partial","*",31,1],"ChatGPT-User":["partial","*",31,1],"OAI-SearchBot":["partial","*",31,1],"ClaudeBot":["partial","*",31,1],"Claude-User":["partial","*",31,1],"Claude-SearchBot":["partial","*",31,1],"anthropic-ai":["partial","*",31,1],"PerplexityBot":["partial","*",31,1],"Perplexity-User":["partial","*",31,1],"Google-Extended":["partial","*",31,1],"Applebot-Extended":["partial","*",31,1],"CCBot":["partial","*",31,1],"Bytespider":["partial","*",31,1],"Amazonbot":["partial","*",31,1],"meta-externalagent":["partial","*",31,1],"cohere-ai":["partial","*",31,1],"Diffbot":["partial","*",31,1],"ContextFetchBot":["partial","*",31,1]},"llms":{"status":"present","http":200,"bytes":1606},"home":{"status":"ok","http":200,"url":"https://www.crn.com/","html_tokens":1157,"markdown_tokens":195,"pruned_tokens":195}},
{"rank":4363,"domain":"uni-bielefeld.de","origin":"https://uni-bielefeld.de","robots":{"status":"ok","http":200,"url":"https://www.uni-bielefeld.de/robots.txt","bytes":571,"groups":4},"bots":{"GPTBot":["partial","*",18,1],"ChatGPT-User":["partial","*",18,1],"OAI-SearchBot":["partial","*",18,1],"ClaudeBot":["partial","*",18,1],"Claude-User":["partial","*",18,1],"Claude-SearchBot":["partial","*",18,1],"anthropic-ai":["partial","*",18,1],"PerplexityBot":["partial","*",18,1],"Perplexity-User":["partial","*",18,1],"Google-Extended":["partial","*",18,1],"Applebot-Extended":["partial","*",18,1],"CCBot":["partial","*",18,1],"Bytespider":["partial","*",18,1],"Amazonbot":["partial","*",18,1],"meta-externalagent":["partial","*",18,1],"cohere-ai":["partial","*",18,1],"Diffbot":["partial","*",18,1],"ContextFetchBot":["partial","*",18,1]},"llms":{"status":"missing","http":404,"bytes":25951},"home":{"status":"ok","http":200,"url":"https://www.uni-bielefeld.de/","html_tokens":38450,"markdown_tokens":83,"pruned_tokens":83}},
{"rank":4364,"domain":"joyn.de","origin":"https://joyn.de","robots":{"status":"ok","http":200,"url":"https://www.joyn.de/robots.txt","bytes":548,"groups":2},"bots":{"GPTBot":["partial","*",2,4],"ChatGPT-User":["partial","*",2,4],"OAI-SearchBot":["partial","*",2,4],"ClaudeBot":["partial","*",2,4],"Claude-User":["partial","*",2,4],"Claude-SearchBot":["partial","*",2,4],"anthropic-ai":["partial","*",2,4],"PerplexityBot":["partial","*",2,4],"Perplexity-User":["partial","*",2,4],"Google-Extended":["partial","*",2,4],"Applebot-Extended":["partial","*",2,4],"CCBot":["partial","*",2,4],"Bytespider":["partial","*",2,4],"Amazonbot":["partial","*",2,4],"meta-externalagent":["partial","*",2,4],"cohere-ai":["partial","*",2,4],"Diffbot":["partial","*",2,4],"ContextFetchBot":["partial","*",2,4]},"llms":{"status":"missing","http":404,"bytes":63633},"home":{"status":"ok","http":200,"url":"https://www.joyn.de/","html_tokens":36625,"markdown_tokens":1247,"pruned_tokens":148}},
{"rank":4365,"domain":"sverigesradio.se","origin":"https://sverigesradio.se","robots":{"status":"refused","http":403,"url":"https://www.sverigesradio.se/robots.txt","bytes":388,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":4366,"domain":"boehringer-ingelheim.com","origin":"https://boehringer-ingelheim.com","robots":{"status":"ok","http":200,"url":"https://www.boehringer-ingelheim.com/robots.txt","bytes":3055,"groups":1},"bots":{"GPTBot":["partial","*",30,18],"ChatGPT-User":["partial","*",30,18],"OAI-SearchBot":["partial","*",30,18],"ClaudeBot":["partial","*",30,18],"Claude-User":["partial","*",30,18],"Claude-SearchBot":["partial","*",30,18],"anthropic-ai":["partial","*",30,18],"PerplexityBot":["partial","*",30,18],"Perplexity-User":["partial","*",30,18],"Google-Extended":["partial","*",30,18],"Applebot-Extended":["partial","*",30,18],"CCBot":["partial","*",30,18],"Bytespider":["partial","*",30,18],"Amazonbot":["partial","*",30,18],"meta-externalagent":["partial","*",30,18],"cohere-ai":["partial","*",30,18],"Diffbot":["partial","*",30,18],"ContextFetchBot":["partial","*",30,18]},"llms":{"status":"http_error","http":403,"bytes":363},"home":{"status":"ok","http":200,"url":"https://www.boehringer-ingelheim.com/","html_tokens":53,"markdown_tokens":0,"pruned_tokens":0}},
{"rank":4367,"domain":"stonybrook.edu","origin":"https://stonybrook.edu","robots":{"status":"ok","http":200,"url":"https://www.stonybrook.edu/robots.txt","bytes":1456,"groups":1},"bots":{"GPTBot":["partial","*",46,0],"ChatGPT-User":["partial","*",46,0],"OAI-SearchBot":["partial","*",46,0],"ClaudeBot":["partial","*",46,0],"Claude-User":["partial","*",46,0],"Claude-SearchBot":["partial","*",46,0],"anthropic-ai":["partial","*",46,0],"PerplexityBot":["partial","*",46,0],"Perplexity-User":["partial","*",46,0],"Google-Extended":["partial","*",46,0],"Applebot-Extended":["partial","*",46,0],"CCBot":["partial","*",46,0],"Bytespider":["partial","*",46,0],"Amazonbot":["partial","*",46,0],"meta-externalagent":["partial","*",46,0],"cohere-ai":["partial","*",46,0],"Diffbot":["partial","*",46,0],"ContextFetchBot":["partial","*",46,0]},"llms":{"status":"missing","http":404,"bytes":90395},"home":{"status":"ok","http":200,"url":"https://www.stonybrook.edu/","html_tokens":36945,"markdown_tokens":258,"pruned_tokens":70}},
{"rank":4368,"domain":"jneurosci.org","origin":"https://www.jneurosci.org","robots":{"status":"refused","http":403,"url":"https://www.jneurosci.org/robots.txt","bytes":5534,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":4369,"domain":"fincen.gov","origin":"https://www.fincen.gov","robots":{"status":"ok","http":200,"url":"https://www.fincen.gov/robots.txt","bytes":2027,"groups":1},"bots":{"GPTBot":["partial","*",34,18],"ChatGPT-User":["partial","*",34,18],"OAI-SearchBot":["partial","*",34,18],"ClaudeBot":["partial","*",34,18],"Claude-User":["partial","*",34,18],"Claude-SearchBot":["partial","*",34,18],"anthropic-ai":["partial","*",34,18],"PerplexityBot":["partial","*",34,18],"Perplexity-User":["partial","*",34,18],"Google-Extended":["partial","*",34,18],"Applebot-Extended":["partial","*",34,18],"CCBot":["partial","*",34,18],"Bytespider":["partial","*",34,18],"Amazonbot":["partial","*",34,18],"meta-externalagent":["partial","*",34,18],"cohere-ai":["partial","*",34,18],"Diffbot":["partial","*",34,18],"ContextFetchBot":["partial","*",34,18]},"llms":{"status":"missing","http":404,"bytes":188},"home":{"status":"ok","http":200,"url":"https://www.fincen.gov/","html_tokens":9912,"markdown_tokens":177,"pruned_tokens":23}},
{"rank":4370,"domain":"persee.fr","origin":"https://www.persee.fr","robots":{"status":"ok","http":200,"url":"https://www.persee.fr/robots.txt","bytes":428,"groups":4},"bots":{"GPTBot":["blocked","*",1,0],"ChatGPT-User":["blocked","*",1,0],"OAI-SearchBot":["blocked","*",1,0],"ClaudeBot":["blocked","*",1,0],"Claude-User":["blocked","*",1,0],"Claude-SearchBot":["blocked","*",1,0],"anthropic-ai":["blocked","*",1,0],"PerplexityBot":["blocked","*",1,0],"Perplexity-User":["blocked","*",1,0],"Google-Extended":["blocked","*",1,0],"Applebot-Extended":["blocked","*",1,0],"CCBot":["blocked","*",1,0],"Bytespider":["blocked","*",1,0],"Amazonbot":["blocked","*",1,0],"meta-externalagent":["blocked","*",1,0],"cohere-ai":["blocked","*",1,0],"Diffbot":["blocked","*",1,0],"ContextFetchBot":["blocked","*",1,0]},"llms":{"status":"disallowed"},"home":{"status":"disallowed"}},
{"rank":4371,"domain":"lu.ma","origin":"https://lu.ma","robots":{"status":"ok","http":200,"url":"https://luma.com/robots.txt","bytes":150,"groups":1},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":27821},"home":{"status":"ok","http":200,"url":"https://luma.com/","html_tokens":11280,"markdown_tokens":91,"pruned_tokens":91}},
{"rank":4372,"domain":"ptc.com","origin":"https://ptc.com","robots":{"status":"refused","http":403,"url":"https://www.ptc.com/robots.txt","bytes":379,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":4373,"domain":"gate.com","origin":"https://gate.com","robots":{"status":"ok","http":200,"url":"https://www.gate.com/robots.txt","bytes":5595,"groups":2},"bots":{"GPTBot":["partial","*",109,17],"ChatGPT-User":["partial","*",109,17],"OAI-SearchBot":["partial","*",109,17],"ClaudeBot":["partial","*",109,17],"Claude-User":["partial","*",109,17],"Claude-SearchBot":["partial","*",109,17],"anthropic-ai":["partial","*",109,17],"PerplexityBot":["partial","*",109,17],"Perplexity-User":["partial","*",109,17],"Google-Extended":["partial","*",109,17],"Applebot-Extended":["partial","*",109,17],"CCBot":["partial","*",109,17],"Bytespider":["partial","*",109,17],"Amazonbot":["partial","*",109,17],"meta-externalagent":["partial","*",109,17],"cohere-ai":["partial","*",109,17],"Diffbot":["partial","*",109,17],"ContextFetchBot":["partial","*",109,17]},"llms":{"status":"other_text","http":200,"bytes":11765},"home":{"status":"ok","http":200,"url":"https://www.gate.com/","html_tokens":51538,"markdown_tokens":6,"pruned_tokens":6}},
{"rank":4374,"domain":"lib.ru","origin":"https://lib.ru","robots":{"status":"ok","http":200,"url":"https://lib.ru/robots.txt","bytes":263,"groups":1},"bots":{"GPTBot":["partial","*",13,0],"ChatGPT-User":["partial","*",13,0],"OAI-SearchBot":["partial","*",13,0],"ClaudeBot":["partial","*",13,0],"Claude-User":["partial","*",13,0],"Claude-SearchBot":["partial","*",13,0],"anthropic-ai":["partial","*",13,0],"PerplexityBot":["partial","*",13,0],"Perplexity-User":["partial","*",13,0],"Google-Extended":["partial","*",13,0],"Applebot-Extended":["partial","*",13,0],"CCBot":["partial","*",13,0],"Bytespider":["partial","*",13,0],"Amazonbot":["partial","*",13,0],"meta-externalagent":["partial","*",13,0],"cohere-ai":["partial","*",13,0],"Diffbot":["partial","*",13,0],"ContextFetchBot":["partial","*",13,0]},"llms":{"status":"missing","http":404,"bytes":625},"home":{"status":"ok","http":200,"url":"https://lib.ru/","html_tokens":5584,"markdown_tokens":1610,"pruned_tokens":148}},
{"rank":4375,"domain":"utdallas.edu","origin":"https://utdallas.edu","robots":{"status":"ok","http":200,"url":"https://utdallas.edu/prod/robots.txt?robots=1","bytes":1341,"groups":26},"bots":{"GPTBot":["partial","*",3,2],"ChatGPT-User":["partial","*",3,2],"OAI-SearchBot":["partial","*",3,2],"ClaudeBot":["partial","*",3,2],"Claude-User":["partial","*",3,2],"Claude-SearchBot":["partial","*",3,2],"anthropic-ai":["partial","*",3,2],"PerplexityBot":["partial","*",3,2],"Perplexity-User":["partial","*",3,2],"Google-Extended":["partial","*",3,2],"Applebot-Extended":["partial","*",3,2],"CCBot":["partial","*",3,2],"Bytespider":["partial","*",3,2],"Amazonbot":["partial","*",3,2],"meta-externalagent":["partial","*",3,2],"cohere-ai":["partial","*",3,2],"Diffbot":["partial","*",3,2],"ContextFetchBot":["partial","*",3,2]},"llms":{"status":"missing","http":404,"bytes":161141},"home":{"status":"ok","http":200,"url":"https://utdallas.edu/","html_tokens":48402,"markdown_tokens":274,"pruned_tokens":162}},
{"rank":4376,"domain":"morgenpost.de","origin":"https://morgenpost.de","robots":{"status":"ok","http":200,"url":"https://www.morgenpost.de/robots.txt","bytes":6366,"groups":7},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["partial","named",27,0],"OAI-SearchBot":["partial","*",26,3],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",26,3],"Claude-SearchBot":["partial","named",27,0],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["partial","named",27,0],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",26,3]},"llms":{"status":"missing","http":404,"bytes":295566},"home":{"status":"ok","http":200,"url":"https://www.morgenpost.de/","html_tokens":287145,"markdown_tokens":3341,"pruned_tokens":119}},
{"rank":4377,"domain":"mymodernmet.com","origin":"https://mymodernmet.com","robots":{"status":"ok","http":200,"url":"https://mymodernmet.com/robots.txt","bytes":146,"groups":2},"bots":{"GPTBot":["partial","*",1,0],"ChatGPT-User":["partial","*",1,0],"OAI-SearchBot":["partial","*",1,0],"ClaudeBot":["partial","*",1,0],"Claude-User":["partial","*",1,0],"Claude-SearchBot":["partial","*",1,0],"anthropic-ai":["partial","*",1,0],"PerplexityBot":["partial","*",1,0],"Perplexity-User":["partial","*",1,0],"Google-Extended":["partial","*",1,0],"Applebot-Extended":["partial","*",1,0],"CCBot":["partial","*",1,0],"Bytespider":["partial","*",1,0],"Amazonbot":["partial","*",1,0],"meta-externalagent":["partial","*",1,0],"cohere-ai":["partial","*",1,0],"Diffbot":["partial","*",1,0],"ContextFetchBot":["partial","*",1,0]},"llms":{"status":"missing","http":404,"bytes":261253},"home":{"status":"ok","http":200,"url":"https://mymodernmet.com/","html_tokens":90265,"markdown_tokens":569,"pruned_tokens":147}},
{"rank":4378,"domain":"emeraldinsight.com","origin":"https://emeraldinsight.com","robots":{"status":"ok","http":200,"url":"https://www.emerald.com/robots.txt","bytes":1952,"groups":7},"bots":{"GPTBot":["partial","*",52,3],"ChatGPT-User":["allowed","named",0,1],"OAI-SearchBot":["allowed","named",0,1],"ClaudeBot":["partial","*",52,3],"Claude-User":["allowed","named",0,1],"Claude-SearchBot":["partial","*",52,3],"anthropic-ai":["partial","*",52,3],"PerplexityBot":["partial","*",52,3],"Perplexity-User":["allowed","named",0,1],"Google-Extended":["partial","*",52,3],"Applebot-Extended":["partial","*",52,3],"CCBot":["partial","*",52,3],"Bytespider":["partial","*",52,3],"Amazonbot":["partial","*",52,3],"meta-externalagent":["partial","*",52,3],"cohere-ai":["partial","*",52,3],"Diffbot":["partial","*",52,3],"ContextFetchBot":["partial","*",52,3]},"llms":{"status":"http_error","http":403,"bytes":5526},"home":{"status":"http_error","http":403,"url":"https://www.emerald.com/"}},
{"rank":4379,"domain":"metacpan.org","origin":"https://metacpan.org","robots":{"status":"ok","http":200,"url":"https://metacpan.org/robots.txt","bytes":427,"groups":1},"bots":{"GPTBot":["partial","*",8,0],"ChatGPT-User":["partial","*",8,0],"OAI-SearchBot":["partial","*",8,0],"ClaudeBot":["partial","*",8,0],"Claude-User":["partial","*",8,0],"Claude-SearchBot":["partial","*",8,0],"anthropic-ai":["partial","*",8,0],"PerplexityBot":["partial","*",8,0],"Perplexity-User":["partial","*",8,0],"Google-Extended":["partial","*",8,0],"Applebot-Extended":["partial","*",8,0],"CCBot":["partial","*",8,0],"Bytespider":["partial","*",8,0],"Amazonbot":["partial","*",8,0],"meta-externalagent":["partial","*",8,0],"cohere-ai":["partial","*",8,0],"Diffbot":["partial","*",8,0],"ContextFetchBot":["partial","*",8,0]},"llms":{"status":"html","http":200,"bytes":3038},"home":{"status":"ok","http":200,"url":"https://metacpan.org/","html_tokens":4040,"markdown_tokens":168,"pruned_tokens":168}},
{"rank":4380,"domain":"monsterinsights.com","origin":"https://monsterinsights.com","robots":{"status":"ok","http":200,"url":"https://www.monsterinsights.com/robots.txt","bytes":279,"groups":1},"bots":{"GPTBot":["partial","*",2,1],"ChatGPT-User":["partial","*",2,1],"OAI-SearchBot":["partial","*",2,1],"ClaudeBot":["partial","*",2,1],"Claude-User":["partial","*",2,1],"Claude-SearchBot":["partial","*",2,1],"anthropic-ai":["partial","*",2,1],"PerplexityBot":["partial","*",2,1],"Perplexity-User":["partial","*",2,1],"Google-Extended":["partial","*",2,1],"Applebot-Extended":["partial","*",2,1],"CCBot":["partial","*",2,1],"Bytespider":["partial","*",2,1],"Amazonbot":["partial","*",2,1],"meta-externalagent":["partial","*",2,1],"cohere-ai":["partial","*",2,1],"Diffbot":["partial","*",2,1],"ContextFetchBot":["partial","*",2,1]},"llms":{"status":"other_text","http":200,"bytes":298090},"home":{"status":"ok","http":200,"url":"https://www.monsterinsights.com/","html_tokens":56087,"markdown_tokens":1710,"pruned_tokens":976}},
{"rank":4381,"domain":"wikiloc.com","origin":"https://wikiloc.com","robots":{"status":"refused","http":403,"url":"https://wikiloc.com/robots.txt","bytes":5527,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":4382,"domain":"transunion.com","origin":"https://transunion.com","robots":{"status":"refused","http":403,"url":"https://transunion.com/robots.txt","bytes":4547,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":4383,"domain":"forward.com","origin":"https://forward.com","robots":{"status":"ok","http":200,"url":"https://forward.com/robots.txt","bytes":331,"groups":1},"bots":{"GPTBot":["partial","*",11,1],"ChatGPT-User":["partial","*",11,1],"OAI-SearchBot":["partial","*",11,1],"ClaudeBot":["partial","*",11,1],"Claude-User":["partial","*",11,1],"Claude-SearchBot":["partial","*",11,1],"anthropic-ai":["partial","*",11,1],"PerplexityBot":["partial","*",11,1],"Perplexity-User":["partial","*",11,1],"Google-Extended":["partial","*",11,1],"Applebot-Extended":["partial","*",11,1],"CCBot":["partial","*",11,1],"Bytespider":["partial","*",11,1],"Amazonbot":["partial","*",11,1],"meta-externalagent":["partial","*",11,1],"cohere-ai":["partial","*",11,1],"Diffbot":["partial","*",11,1],"ContextFetchBot":["partial","*",11,1]},"llms":{"status":"other_text","http":200,"bytes":3655},"home":{"status":"http_error","http":403,"url":"https://forward.com/"}},
{"rank":4384,"domain":"ucc.ie","origin":"https://ucc.ie","robots":{"status":"ok","http":200,"url":"https://www.ucc.ie/robots.txt","bytes":747,"groups":1},"bots":{"GPTBot":["partial","*",23,1],"ChatGPT-User":["partial","*",23,1],"OAI-SearchBot":["partial","*",23,1],"ClaudeBot":["partial","*",23,1],"Claude-User":["partial","*",23,1],"Claude-SearchBot":["partial","*",23,1],"anthropic-ai":["partial","*",23,1],"PerplexityBot":["partial","*",23,1],"Perplexity-User":["partial","*",23,1],"Google-Extended":["partial","*",23,1],"Applebot-Extended":["partial","*",23,1],"CCBot":["partial","*",23,1],"Bytespider":["partial","*",23,1],"Amazonbot":["partial","*",23,1],"meta-externalagent":["partial","*",23,1],"cohere-ai":["partial","*",23,1],"Diffbot":["partial","*",23,1],"ContextFetchBot":["partial","*",23,1]},"llms":{"status":"missing","http":404,"bytes":847},"home":{"status":"ok","http":200,"url":"https://www.ucc.ie/en/","html_tokens":13371,"markdown_tokens":476,"pruned_tokens":15}},
{"rank":4385,"domain":"cultura.gov.it","origin":"https://cultura.gov.it","robots":{"status":"timeout","http":null,"url":"https://cultura.gov.it/robots.txt","bytes":0,"groups":0,"error":"ConnectTimeout"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":4386,"domain":"mozdev.org","origin":"https://mozdev.org","robots":{"status":"ok","http":200,"url":"https://www.mozilla.org/robots.txt","bytes":66,"groups":1},"bots":{"GPTBot":["partial","*",2,0],"ChatGPT-User":["partial","*",2,0],"OAI-SearchBot":["partial","*",2,0],"ClaudeBot":["partial","*",2,0],"Claude-User":["partial","*",2,0],"Claude-SearchBot":["partial","*",2,0],"anthropic-ai":["partial","*",2,0],"PerplexityBot":["partial","*",2,0],"Perplexity-User":["partial","*",2,0],"Google-Extended":["partial","*",2,0],"Applebot-Extended":["partial","*",2,0],"CCBot":["partial","*",2,0],"Bytespider":["partial","*",2,0],"Amazonbot":["partial","*",2,0],"meta-externalagent":["partial","*",2,0],"cohere-ai":["partial","*",2,0],"Diffbot":["partial","*",2,0],"ContextFetchBot":["partial","*",2,0]},"llms":{"status":"missing","http":404,"bytes":8955},"home":{"status":"ok","http":200,"url":"https://www.mozilla.org/en-US/","html_tokens":28571,"markdown_tokens":229,"pruned_tokens":162}},
{"rank":4387,"domain":"longisland.com","origin":"https://longisland.com","robots":{"status":"ok","http":200,"url":"https://longisland.com/robots.txt","bytes":216,"groups":2},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["partial","*",6,0],"OAI-SearchBot":["partial","*",6,0],"ClaudeBot":["partial","*",6,0],"Claude-User":["partial","*",6,0],"Claude-SearchBot":["partial","*",6,0],"anthropic-ai":["partial","*",6,0],"PerplexityBot":["partial","*",6,0],"Perplexity-User":["partial","*",6,0],"Google-Extended":["partial","*",6,0],"Applebot-Extended":["partial","*",6,0],"CCBot":["partial","*",6,0],"Bytespider":["partial","*",6,0],"Amazonbot":["partial","*",6,0],"meta-externalagent":["partial","*",6,0],"cohere-ai":["partial","*",6,0],"Diffbot":["partial","*",6,0],"ContextFetchBot":["partial","*",6,0]},"llms":{"status":"timeout","error":"timed out after 10 s"},"home":{"status":"ok","http":200,"url":"https://longisland.com/","html_tokens":25405,"markdown_tokens":542,"pruned_tokens":462}},
{"rank":4388,"domain":"hsbc.com","origin":"https://hsbc.com","robots":{"status":"ok","http":200,"url":"https://www.hsbc.com/robots.txt","bytes":24,"groups":1},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"timeout","error":"ConnectTimeout"},"home":{"status":"timeout","error":"ConnectTimeout"}},
{"rank":4389,"domain":"iht.com","origin":"http://iht.com","robots":{"status":"refused","http":403,"url":"https://www.nytimes.com","bytes":771,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":4390,"domain":"petitiononline.com","origin":"https://petitiononline.com","robots":{"status":"html","http":200,"url":"https://www.petitiononline.com/","bytes":1998,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":4391,"domain":"inhabitat.com","origin":"https://inhabitat.com","robots":{"status":"ok","http":200,"url":"https://inhabitat.com/robots.txt","bytes":668,"groups":2},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["partial","*",13,1],"OAI-SearchBot":["partial","*",13,1],"ClaudeBot":["partial","*",13,1],"Claude-User":["partial","*",13,1],"Claude-SearchBot":["partial","*",13,1],"anthropic-ai":["partial","*",13,1],"PerplexityBot":["partial","*",13,1],"Perplexity-User":["partial","*",13,1],"Google-Extended":["partial","*",13,1],"Applebot-Extended":["partial","*",13,1],"CCBot":["partial","*",13,1],"Bytespider":["partial","*",13,1],"Amazonbot":["partial","*",13,1],"meta-externalagent":["partial","*",13,1],"cohere-ai":["partial","*",13,1],"Diffbot":["partial","*",13,1],"ContextFetchBot":["partial","*",13,1]},"llms":{"status":"http_error","http":500,"bytes":3088},"home":{"status":"ok","http":200,"url":"https://inhabitat.com/","html_tokens":68667,"markdown_tokens":117,"pruned_tokens":117}},
{"rank":4392,"domain":"authorize.net","origin":"https://authorize.net","robots":{"status":"ok","http":200,"url":"https://www.authorize.net/robots.txt","bytes":367,"groups":1},"bots":{"GPTBot":["partial","*",13,0],"ChatGPT-User":["partial","*",13,0],"OAI-SearchBot":["partial","*",13,0],"ClaudeBot":["partial","*",13,0],"Claude-User":["partial","*",13,0],"Claude-SearchBot":["partial","*",13,0],"anthropic-ai":["partial","*",13,0],"PerplexityBot":["partial","*",13,0],"Perplexity-User":["partial","*",13,0],"Google-Extended":["partial","*",13,0],"Applebot-Extended":["partial","*",13,0],"CCBot":["partial","*",13,0],"Bytespider":["partial","*",13,0],"Amazonbot":["partial","*",13,0],"meta-externalagent":["partial","*",13,0],"cohere-ai":["partial","*",13,0],"Diffbot":["partial","*",13,0],"ContextFetchBot":["partial","*",13,0]},"llms":{"status":"missing","http":404,"bytes":32069},"home":{"status":"ok","http":200,"url":"https://www.authorize.net/","html_tokens":48267,"markdown_tokens":181,"pruned_tokens":131}},
{"rank":4393,"domain":"feedproxy.google.com","origin":"https://feedproxy.google.com","robots":{"status":"missing","http":404,"url":"https://feedproxy.google.com/robots.txt","bytes":1652,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":1652},"home":{"status":"http_error","http":404,"url":"https://feedproxy.google.com/"}},
{"rank":4394,"domain":"wpbeginner.com","origin":"https://wpbeginner.com","robots":{"status":"ok","http":200,"url":"https://wpbeginner.com/robots.txt","bytes":601,"groups":1},"bots":{"GPTBot":["partial","*",21,3],"ChatGPT-User":["partial","*",21,3],"OAI-SearchBot":["partial","*",21,3],"ClaudeBot":["partial","*",21,3],"Claude-User":["partial","*",21,3],"Claude-SearchBot":["partial","*",21,3],"anthropic-ai":["partial","*",21,3],"PerplexityBot":["partial","*",21,3],"Perplexity-User":["partial","*",21,3],"Google-Extended":["partial","*",21,3],"Applebot-Extended":["partial","*",21,3],"CCBot":["partial","*",21,3],"Bytespider":["partial","*",21,3],"Amazonbot":["partial","*",21,3],"meta-externalagent":["partial","*",21,3],"cohere-ai":["partial","*",21,3],"Diffbot":["partial","*",21,3],"ContextFetchBot":["partial","*",21,3]},"llms":{"status":"other_text","http":200,"bytes":508165},"home":{"status":"ok","http":200,"url":"https://www.wpbeginner.com/","html_tokens":79926,"markdown_tokens":244,"pruned_tokens":134}},
{"rank":4395,"domain":"strath.ac.uk","origin":"https://www.strath.ac.uk","robots":{"status":"ok","http":200,"url":"https://www.strath.ac.uk/robots.txt","bytes":485,"groups":1},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":38688},"home":{"status":"ok","http":200,"url":"https://www.strath.ac.uk/","html_tokens":12153,"markdown_tokens":376,"pruned_tokens":164}},
{"rank":4396,"domain":"studio.site","origin":"https://www.studio.site","robots":{"status":"ok","http":200,"url":"https://www.studio.site/robots.txt","bytes":44,"groups":1},"bots":{"GPTBot":["allowed","*",0,1],"ChatGPT-User":["allowed","*",0,1],"OAI-SearchBot":["allowed","*",0,1],"ClaudeBot":["allowed","*",0,1],"Claude-User":["allowed","*",0,1],"Claude-SearchBot":["allowed","*",0,1],"anthropic-ai":["allowed","*",0,1],"PerplexityBot":["allowed","*",0,1],"Perplexity-User":["allowed","*",0,1],"Google-Extended":["allowed","*",0,1],"Applebot-Extended":["allowed","*",0,1],"CCBot":["allowed","*",0,1],"Bytespider":["allowed","*",0,1],"Amazonbot":["allowed","*",0,1],"meta-externalagent":["allowed","*",0,1],"cohere-ai":["allowed","*",0,1],"Diffbot":["allowed","*",0,1],"ContextFetchBot":["allowed","*",0,1]},"llms":{"status":"missing","http":404,"bytes":7977},"home":{"status":"http_error","http":404,"url":"https://www.studio.site/"}},
{"rank":4397,"domain":"prospect.org","origin":"https://prospect.org","robots":{"status":"ok","http":200,"url":"https://prospect.org/robots.txt","bytes":529,"groups":2},"bots":{"GPTBot":["partial","*",6,1],"ChatGPT-User":["partial","*",6,1],"OAI-SearchBot":["partial","*",6,1],"ClaudeBot":["partial","*",6,1],"Claude-User":["partial","*",6,1],"Claude-SearchBot":["partial","*",6,1],"anthropic-ai":["partial","*",6,1],"PerplexityBot":["partial","*",6,1],"Perplexity-User":["partial","*",6,1],"Google-Extended":["partial","*",6,1],"Applebot-Extended":["partial","*",6,1],"CCBot":["partial","*",6,1],"Bytespider":["partial","*",6,1],"Amazonbot":["partial","*",6,1],"meta-externalagent":["partial","*",6,1],"cohere-ai":["partial","*",6,1],"Diffbot":["partial","*",6,1],"ContextFetchBot":["partial","*",6,1]},"llms":{"status":"missing","http":404,"bytes":266291},"home":{"status":"ok","http":200,"url":"https://prospect.org/","html_tokens":118572,"markdown_tokens":117,"pruned_tokens":117}},
{"rank":4398,"domain":"angel.co","origin":"https://angel.co","robots":{"status":"ok","http":200,"url":"https://wellfound.com/robots.txt","bytes":964,"groups":1},"bots":{"GPTBot":["partial","*",31,0],"ChatGPT-User":["partial","*",31,0],"OAI-SearchBot":["partial","*",31,0],"ClaudeBot":["partial","*",31,0],"Claude-User":["partial","*",31,0],"Claude-SearchBot":["partial","*",31,0],"anthropic-ai":["partial","*",31,0],"PerplexityBot":["partial","*",31,0],"Perplexity-User":["partial","*",31,0],"Google-Extended":["partial","*",31,0],"Applebot-Extended":["partial","*",31,0],"CCBot":["partial","*",31,0],"Bytespider":["partial","*",31,0],"Amazonbot":["partial","*",31,0],"meta-externalagent":["partial","*",31,0],"cohere-ai":["partial","*",31,0],"Diffbot":["partial","*",31,0],"ContextFetchBot":["partial","*",31,0]},"llms":{"status":"missing","http":404,"bytes":0},"home":{"status":"ok","http":200,"url":"https://wellfound.com/","html_tokens":31886,"markdown_tokens":692,"pruned_tokens":43}},
{"rank":4399,"domain":"niu.edu","origin":"https://niu.edu","robots":{"status":"ok","http":200,"url":"https://www.niu.edu/robots.txt","bytes":557,"groups":6},"bots":{"GPTBot":["partial","*",5,0],"ChatGPT-User":["partial","*",5,0],"OAI-SearchBot":["partial","*",5,0],"ClaudeBot":["partial","*",5,0],"Claude-User":["partial","*",5,0],"Claude-SearchBot":["partial","*",5,0],"anthropic-ai":["partial","*",5,0],"PerplexityBot":["partial","*",5,0],"Perplexity-User":["partial","*",5,0],"Google-Extended":["partial","*",5,0],"Applebot-Extended":["partial","*",5,0],"CCBot":["partial","*",5,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["partial","*",5,0],"meta-externalagent":["partial","*",5,0],"cohere-ai":["partial","*",5,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",5,0]},"llms":{"status":"present","http":200,"bytes":15840},"home":{"status":"ok","http":200,"url":"https://www.niu.edu/index.shtml","html_tokens":31892,"markdown_tokens":70,"pruned_tokens":65}},
{"rank":4400,"domain":"verbraucherzentrale.de","origin":"https://verbraucherzentrale.de","robots":{"status":"ok","http":200,"url":"https://www.verbraucherzentrale.de/robots.txt","bytes":2005,"groups":1},"bots":{"GPTBot":["partial","*",33,18],"ChatGPT-User":["partial","*",33,18],"OAI-SearchBot":["partial","*",33,18],"ClaudeBot":["partial","*",33,18],"Claude-User":["partial","*",33,18],"Claude-SearchBot":["partial","*",33,18],"anthropic-ai":["partial","*",33,18],"PerplexityBot":["partial","*",33,18],"Perplexity-User":["partial","*",33,18],"Google-Extended":["partial","*",33,18],"Applebot-Extended":["partial","*",33,18],"CCBot":["partial","*",33,18],"Bytespider":["partial","*",33,18],"Amazonbot":["partial","*",33,18],"meta-externalagent":["partial","*",33,18],"cohere-ai":["partial","*",33,18],"Diffbot":["partial","*",33,18],"ContextFetchBot":["partial","*",33,18]},"llms":{"status":"http_error","http":403,"bytes":162},"home":{"status":"ok","http":200,"url":"https://www.verbraucherzentrale.de","html_tokens":263531,"markdown_tokens":138,"pruned_tokens":60}},
{"rank":4401,"domain":"healthdirect.gov.au","origin":"https://healthdirect.gov.au","robots":{"status":"ok","http":200,"url":"https://www.healthdirect.gov.au/robots.txt","bytes":371,"groups":1},"bots":{"GPTBot":["partial","*",4,0],"ChatGPT-User":["partial","*",4,0],"OAI-SearchBot":["partial","*",4,0],"ClaudeBot":["partial","*",4,0],"Claude-User":["partial","*",4,0],"Claude-SearchBot":["partial","*",4,0],"anthropic-ai":["partial","*",4,0],"PerplexityBot":["partial","*",4,0],"Perplexity-User":["partial","*",4,0],"Google-Extended":["partial","*",4,0],"Applebot-Extended":["partial","*",4,0],"CCBot":["partial","*",4,0],"Bytespider":["partial","*",4,0],"Amazonbot":["partial","*",4,0],"meta-externalagent":["partial","*",4,0],"cohere-ai":["partial","*",4,0],"Diffbot":["partial","*",4,0],"ContextFetchBot":["partial","*",4,0]},"llms":{"status":"missing","http":404,"bytes":84513},"home":{"status":"ok","http":200,"url":"https://www.healthdirect.gov.au/","html_tokens":13431,"markdown_tokens":0,"pruned_tokens":0}},
{"rank":4402,"domain":"ametsoc.org","origin":"https://ametsoc.org","robots":{"status":"ok","http":200,"url":"https://www.ametsoc.org/robots.txt","bytes":148,"groups":1},"bots":{"GPTBot":["partial","*",4,0],"ChatGPT-User":["partial","*",4,0],"OAI-SearchBot":["partial","*",4,0],"ClaudeBot":["partial","*",4,0],"Claude-User":["partial","*",4,0],"Claude-SearchBot":["partial","*",4,0],"anthropic-ai":["partial","*",4,0],"PerplexityBot":["partial","*",4,0],"Perplexity-User":["partial","*",4,0],"Google-Extended":["partial","*",4,0],"Applebot-Extended":["partial","*",4,0],"CCBot":["partial","*",4,0],"Bytespider":["partial","*",4,0],"Amazonbot":["partial","*",4,0],"meta-externalagent":["partial","*",4,0],"cohere-ai":["partial","*",4,0],"Diffbot":["partial","*",4,0],"ContextFetchBot":["partial","*",4,0]},"llms":{"status":"present","http":200,"bytes":3539},"home":{"status":"http_error","http":403,"url":"https://www.ametsoc.org/"}},
{"rank":4403,"domain":"datacenterdynamics.com","origin":"https://datacenterdynamics.com","robots":{"status":"missing","http":404,"url":"https://www.datacenterdynamics.com/en/robots.txt","bytes":73280,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":73272},"home":{"status":"http_error","http":403,"url":"https://datacenterdynamics.com/"}},
{"rank":4404,"domain":"bsky.social","origin":"https://bsky.social","robots":{"status":"ok","http":200,"url":"https://bsky.social/robots.txt","bytes":69,"groups":1},"bots":{"GPTBot":["allowed","*",0,1],"ChatGPT-User":["allowed","*",0,1],"OAI-SearchBot":["allowed","*",0,1],"ClaudeBot":["allowed","*",0,1],"Claude-User":["allowed","*",0,1],"Claude-SearchBot":["allowed","*",0,1],"anthropic-ai":["allowed","*",0,1],"PerplexityBot":["allowed","*",0,1],"Perplexity-User":["allowed","*",0,1],"Google-Extended":["allowed","*",0,1],"Applebot-Extended":["allowed","*",0,1],"CCBot":["allowed","*",0,1],"Bytespider":["allowed","*",0,1],"Amazonbot":["allowed","*",0,1],"meta-externalagent":["allowed","*",0,1],"cohere-ai":["allowed","*",0,1],"Diffbot":["allowed","*",0,1],"ContextFetchBot":["allowed","*",0,1]},"llms":{"status":"missing","http":404,"bytes":147},"home":{"status":"ok","http":200,"url":"https://bsky.social/about","html_tokens":15847,"markdown_tokens":787,"pruned_tokens":41}},
{"rank":4405,"domain":"francetravail.fr","origin":"https://francetravail.fr","robots":{"status":"ok","http":200,"url":"https://www.francetravail.fr/robots.txt","bytes":396,"groups":1},"bots":{"GPTBot":["partial","*",13,0],"ChatGPT-User":["partial","*",13,0],"OAI-SearchBot":["partial","*",13,0],"ClaudeBot":["partial","*",13,0],"Claude-User":["partial","*",13,0],"Claude-SearchBot":["partial","*",13,0],"anthropic-ai":["partial","*",13,0],"PerplexityBot":["partial","*",13,0],"Perplexity-User":["partial","*",13,0],"Google-Extended":["partial","*",13,0],"Applebot-Extended":["partial","*",13,0],"CCBot":["partial","*",13,0],"Bytespider":["partial","*",13,0],"Amazonbot":["partial","*",13,0],"meta-externalagent":["partial","*",13,0],"cohere-ai":["partial","*",13,0],"Diffbot":["partial","*",13,0],"ContextFetchBot":["partial","*",13,0]},"llms":{"status":"missing","http":404,"bytes":2870},"home":{"status":"ok","http":200,"url":"https://www.francetravail.fr/accueil/","html_tokens":5816,"markdown_tokens":0,"pruned_tokens":0}},
{"rank":4406,"domain":"juniper.net","origin":"https://juniper.net","robots":{"status":"ok","http":200,"url":"https://www.juniper.net/robots.txt","bytes":16071,"groups":1},"bots":{"GPTBot":["partial","*",297,0],"ChatGPT-User":["partial","*",297,0],"OAI-SearchBot":["partial","*",297,0],"ClaudeBot":["partial","*",297,0],"Claude-User":["partial","*",297,0],"Claude-SearchBot":["partial","*",297,0],"anthropic-ai":["partial","*",297,0],"PerplexityBot":["partial","*",297,0],"Perplexity-User":["partial","*",297,0],"Google-Extended":["partial","*",297,0],"Applebot-Extended":["partial","*",297,0],"CCBot":["partial","*",297,0],"Bytespider":["partial","*",297,0],"Amazonbot":["partial","*",297,0],"meta-externalagent":["partial","*",297,0],"cohere-ai":["partial","*",297,0],"Diffbot":["partial","*",297,0],"ContextFetchBot":["partial","*",297,0]},"llms":{"status":"present","http":200,"bytes":45700},"home":{"status":"ok","http":200,"url":"https://www.juniper.net/","html_tokens":190064,"markdown_tokens":787,"pruned_tokens":146}},
{"rank":4407,"domain":"linuxjournal.com","origin":"https://linuxjournal.com","robots":{"status":"ok","http":200,"url":"https://www.linuxjournal.com/robots.txt","bytes":1706,"groups":1},"bots":{"GPTBot":["partial","*",26,18],"ChatGPT-User":["partial","*",26,18],"OAI-SearchBot":["partial","*",26,18],"ClaudeBot":["partial","*",26,18],"Claude-User":["partial","*",26,18],"Claude-SearchBot":["partial","*",26,18],"anthropic-ai":["partial","*",26,18],"PerplexityBot":["partial","*",26,18],"Perplexity-User":["partial","*",26,18],"Google-Extended":["partial","*",26,18],"Applebot-Extended":["partial","*",26,18],"CCBot":["partial","*",26,18],"Bytespider":["partial","*",26,18],"Amazonbot":["partial","*",26,18],"meta-externalagent":["partial","*",26,18],"cohere-ai":["partial","*",26,18],"Diffbot":["partial","*",26,18],"ContextFetchBot":["partial","*",26,18]},"llms":{"status":"http_error","http":403,"bytes":146},"home":{"status":"ok","http":200,"url":"https://www.linuxjournal.com/","html_tokens":13740,"markdown_tokens":251,"pruned_tokens":78}},
{"rank":4408,"domain":"screamingfrog.co.uk","origin":"https://screamingfrog.co.uk","robots":{"status":"ok","http":200,"url":"https://screamingfrog.co.uk/robots.txt","bytes":1883,"groups":6},"bots":{"GPTBot":["partial","*",6,0],"ChatGPT-User":["partial","*",6,0],"OAI-SearchBot":["partial","*",6,0],"ClaudeBot":["partial","*",6,0],"Claude-User":["partial","*",6,0],"Claude-SearchBot":["partial","*",6,0],"anthropic-ai":["partial","*",6,0],"PerplexityBot":["partial","*",6,0],"Perplexity-User":["partial","*",6,0],"Google-Extended":["partial","*",6,0],"Applebot-Extended":["partial","*",6,0],"CCBot":["partial","*",6,0],"Bytespider":["partial","*",6,0],"Amazonbot":["partial","*",6,0],"meta-externalagent":["partial","*",6,0],"cohere-ai":["partial","*",6,0],"Diffbot":["partial","*",6,0],"ContextFetchBot":["partial","*",6,0]},"llms":{"status":"missing","http":404,"bytes":167040},"home":{"status":"ok","http":200,"url":"https://www.screamingfrog.co.uk/","html_tokens":50110,"markdown_tokens":187,"pruned_tokens":69}},
{"rank":4409,"domain":"brightlocal.com","origin":"https://brightlocal.com","robots":{"status":"ok","http":200,"url":"https://www.brightlocal.com/robots.txt","bytes":1148,"groups":19},"bots":{"GPTBot":["partial","*",1,1],"ChatGPT-User":["allowed","named",0,1],"OAI-SearchBot":["allowed","named",0,1],"ClaudeBot":["partial","*",1,1],"Claude-User":["allowed","named",0,1],"Claude-SearchBot":["allowed","named",0,1],"anthropic-ai":["partial","*",1,1],"PerplexityBot":["allowed","named",0,1],"Perplexity-User":["partial","*",1,1],"Google-Extended":["partial","*",1,1],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",1,1]},"llms":{"status":"present","http":200,"bytes":11375},"home":{"status":"ok","http":200,"url":"https://www.brightlocal.com/","html_tokens":154515,"markdown_tokens":756,"pruned_tokens":89}},
{"rank":4410,"domain":"xdebug.org","origin":"https://xdebug.org","robots":{"status":"missing","http":404,"url":"https://xdebug.org/robots.txt","bytes":4296,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":4296},"home":{"status":"ok","http":200,"url":"https://xdebug.org/","html_tokens":2179,"markdown_tokens":300,"pruned_tokens":142}},
{"rank":4411,"domain":"ig.com.br","origin":"https://ig.com.br","robots":{"status":"ok","http":200,"url":"https://www.ig.com.br/robots.txt","bytes":339,"groups":4},"bots":{"GPTBot":["partial","*",4,1],"ChatGPT-User":["partial","*",4,1],"OAI-SearchBot":["partial","*",4,1],"ClaudeBot":["partial","*",4,1],"Claude-User":["partial","*",4,1],"Claude-SearchBot":["partial","*",4,1],"anthropic-ai":["partial","*",4,1],"PerplexityBot":["partial","*",4,1],"Perplexity-User":["partial","*",4,1],"Google-Extended":["partial","*",4,1],"Applebot-Extended":["partial","*",4,1],"CCBot":["partial","*",4,1],"Bytespider":["partial","*",4,1],"Amazonbot":["partial","*",4,1],"meta-externalagent":["partial","*",4,1],"cohere-ai":["partial","*",4,1],"Diffbot":["partial","*",4,1],"ContextFetchBot":["partial","*",4,1]},"llms":{"status":"missing","http":404,"bytes":70569},"home":{"status":"ok","http":200,"url":"https://www.ig.com.br/","html_tokens":54668,"markdown_tokens":25,"pruned_tokens":25}},
{"rank":4412,"domain":"heraldsun.com.au","origin":"https://heraldsun.com.au","robots":{"status":"ok","http":200,"url":"https://www.heraldsun.com.au/robots.txt","bytes":5127,"groups":4},"bots":{"GPTBot":["partial","*",1,2],"ChatGPT-User":["partial","*",1,2],"OAI-SearchBot":["partial","*",1,2],"ClaudeBot":["partial","*",1,2],"Claude-User":["partial","*",1,2],"Claude-SearchBot":["partial","*",1,2],"anthropic-ai":["partial","*",1,2],"PerplexityBot":["partial","*",1,2],"Perplexity-User":["partial","*",1,2],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["partial","*",1,2],"CCBot":["partial","*",1,2],"Bytespider":["partial","*",1,2],"Amazonbot":["partial","*",1,2],"meta-externalagent":["partial","*",1,2],"cohere-ai":["partial","*",1,2],"Diffbot":["partial","*",1,2],"ContextFetchBot":["partial","*",1,2]},"llms":{"status":"disallowed"},"home":{"status":"disallowed"}},
{"rank":4413,"domain":"atom.com","origin":"https://atom.com","robots":{"status":"ok","http":200,"url":"https://www.atom.com/robots.txt","bytes":4006,"groups":4},"bots":{"GPTBot":["allowed","named",0,1],"ChatGPT-User":["partial","*",103,0],"OAI-SearchBot":["partial","named",6,1],"ClaudeBot":["partial","*",103,0],"Claude-User":["partial","*",103,0],"Claude-SearchBot":["partial","*",103,0],"anthropic-ai":["partial","*",103,0],"PerplexityBot":["partial","*",103,0],"Perplexity-User":["partial","*",103,0],"Google-Extended":["partial","*",103,0],"Applebot-Extended":["partial","*",103,0],"CCBot":["partial","*",103,0],"Bytespider":["partial","*",103,0],"Amazonbot":["partial","*",103,0],"meta-externalagent":["partial","*",103,0],"cohere-ai":["partial","*",103,0],"Diffbot":["partial","*",103,0],"ContextFetchBot":["partial","*",103,0]},"llms":{"status":"present","http":200,"bytes":14803},"home":{"status":"ok","http":200,"url":"https://www.atom.com/","html_tokens":248092,"markdown_tokens":185,"pruned_tokens":81}},
{"rank":4414,"domain":"dataliberation.org","origin":"https://dataliberation.org","robots":{"status":"html","http":200,"url":"https://support.google.com/accounts/answer/3024190","bytes":1661192,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":4415,"domain":"mailenable.com","origin":"https://mailenable.com","robots":{"status":"ok","http":200,"url":"https://mailenable.com/robots.txt","bytes":687,"groups":8},"bots":{"GPTBot":["partial","*",11,1],"ChatGPT-User":["partial","*",11,1],"OAI-SearchBot":["partial","*",11,1],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",11,1],"Claude-SearchBot":["partial","*",11,1],"anthropic-ai":["partial","*",11,1],"PerplexityBot":["partial","*",11,1],"Perplexity-User":["partial","*",11,1],"Google-Extended":["partial","*",11,1],"Applebot-Extended":["partial","*",11,1],"CCBot":["partial","*",11,1],"Bytespider":["partial","*",11,1],"Amazonbot":["partial","*",11,1],"meta-externalagent":["partial","named",1,0],"cohere-ai":["partial","*",11,1],"Diffbot":["partial","*",11,1],"ContextFetchBot":["partial","*",11,1]},"llms":{"status":"html","http":200,"bytes":18556},"home":{"status":"ok","http":200,"url":"https://mailenable.com/","html_tokens":8050,"markdown_tokens":516,"pruned_tokens":397}},
{"rank":4416,"domain":"io9.com","origin":"https://io9.com","robots":{"status":"ok","http":200,"url":"https://gizmodo.com/robots.txt","bytes":1223,"groups":18},"bots":{"GPTBot":["partial","*",16,1],"ChatGPT-User":["partial","*",16,1],"OAI-SearchBot":["partial","*",16,1],"ClaudeBot":["partial","*",16,1],"Claude-User":["partial","*",16,1],"Claude-SearchBot":["partial","*",16,1],"anthropic-ai":["partial","*",16,1],"PerplexityBot":["partial","*",16,1],"Perplexity-User":["partial","*",16,1],"Google-Extended":["partial","*",16,1],"Applebot-Extended":["partial","*",16,1],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["partial","*",16,1],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",16,1]},"llms":{"status":"missing","http":404,"bytes":146541},"home":{"status":"ok","http":200,"url":"https://gizmodo.com/io9","html_tokens":94427,"markdown_tokens":57,"pruned_tokens":55}},
{"rank":4417,"domain":"sbb.ch","origin":"https://sbb.ch","robots":{"status":"refused","http":403,"url":"https://www.sbb.ch/de/robots.txt","bytes":4926,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":4418,"domain":"flutter.dev","origin":"https://flutter.dev","robots":{"status":"missing","http":404,"url":"https://flutter.dev/robots.txt","bytes":46331,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":46331},"home":{"status":"ok","http":200,"url":"https://flutter.dev/","html_tokens":18421,"markdown_tokens":171,"pruned_tokens":171}},
{"rank":4419,"domain":"hellomagazine.com","origin":"https://hellomagazine.com","robots":{"status":"ok","http":200,"url":"https://www.hellomagazine.com/robots.txt","bytes":820,"groups":12},"bots":{"GPTBot":["partial","*",9,0],"ChatGPT-User":["partial","*",9,0],"OAI-SearchBot":["blocked","named",1,0],"ClaudeBot":["partial","*",9,0],"Claude-User":["partial","*",9,0],"Claude-SearchBot":["partial","*",9,0],"anthropic-ai":["partial","*",9,0],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["blocked","named",1,0],"Google-Extended":["partial","*",9,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["partial","*",9,0],"meta-externalagent":["partial","*",9,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["partial","*",9,0],"ContextFetchBot":["partial","*",9,0]},"llms":{"status":"missing","http":404,"bytes":75789},"home":{"status":"ok","http":200,"url":"https://www.hellomagazine.com/us/","html_tokens":131669,"markdown_tokens":422,"pruned_tokens":171}},
{"rank":4420,"domain":"kurier.at","origin":"https://kurier.at","robots":{"status":"ok","http":200,"url":"https://kurier.at/robots.txt","bytes":320,"groups":1},"bots":{"GPTBot":["partial","*",3,0],"ChatGPT-User":["partial","*",3,0],"OAI-SearchBot":["partial","*",3,0],"ClaudeBot":["partial","*",3,0],"Claude-User":["partial","*",3,0],"Claude-SearchBot":["partial","*",3,0],"anthropic-ai":["partial","*",3,0],"PerplexityBot":["partial","*",3,0],"Perplexity-User":["partial","*",3,0],"Google-Extended":["partial","*",3,0],"Applebot-Extended":["partial","*",3,0],"CCBot":["partial","*",3,0],"Bytespider":["partial","*",3,0],"Amazonbot":["partial","*",3,0],"meta-externalagent":["partial","*",3,0],"cohere-ai":["partial","*",3,0],"Diffbot":["partial","*",3,0],"ContextFetchBot":["partial","*",3,0]},"llms":{"status":"missing","http":404,"bytes":214357},"home":{"status":"ok","http":200,"url":"https://kurier.at/","html_tokens":339653,"markdown_tokens":23,"pruned_tokens":23}},
{"rank":4421,"domain":"ifc.org","origin":"https://ifc.org","robots":{"status":"ok","http":200,"url":"https://www.ifc.org/robots.txt","bytes":231,"groups":1},"bots":{"GPTBot":["partial","*",8,1],"ChatGPT-User":["partial","*",8,1],"OAI-SearchBot":["partial","*",8,1],"ClaudeBot":["partial","*",8,1],"Claude-User":["partial","*",8,1],"Claude-SearchBot":["partial","*",8,1],"anthropic-ai":["partial","*",8,1],"PerplexityBot":["partial","*",8,1],"Perplexity-User":["partial","*",8,1],"Google-Extended":["partial","*",8,1],"Applebot-Extended":["partial","*",8,1],"CCBot":["partial","*",8,1],"Bytespider":["partial","*",8,1],"Amazonbot":["partial","*",8,1],"meta-externalagent":["partial","*",8,1],"cohere-ai":["partial","*",8,1],"Diffbot":["partial","*",8,1],"ContextFetchBot":["partial","*",8,1]},"llms":{"status":"missing","http":404,"bytes":100826},"home":{"status":"ok","http":200,"url":"https://www.ifc.org/en/home","html_tokens":23207,"markdown_tokens":540,"pruned_tokens":335}},
{"rank":4422,"domain":"evolution.com","origin":"https://evolution.com","robots":{"status":"ok","http":200,"url":"https://www.evolution.com/robots.txt","bytes":23,"groups":1},"bots":{"GPTBot":["allowed","*",0,1],"ChatGPT-User":["allowed","*",0,1],"OAI-SearchBot":["allowed","*",0,1],"ClaudeBot":["allowed","*",0,1],"Claude-User":["allowed","*",0,1],"Claude-SearchBot":["allowed","*",0,1],"anthropic-ai":["allowed","*",0,1],"PerplexityBot":["allowed","*",0,1],"Perplexity-User":["allowed","*",0,1],"Google-Extended":["allowed","*",0,1],"Applebot-Extended":["allowed","*",0,1],"CCBot":["allowed","*",0,1],"Bytespider":["allowed","*",0,1],"Amazonbot":["allowed","*",0,1],"meta-externalagent":["allowed","*",0,1],"cohere-ai":["allowed","*",0,1],"Diffbot":["allowed","*",0,1],"ContextFetchBot":["allowed","*",0,1]},"llms":{"status":"missing","http":404,"bytes":54423},"home":{"status":"ok","http":200,"url":"https://www.evolution.com/","html_tokens":44833,"markdown_tokens":82,"pruned_tokens":82}},
{"rank":4423,"domain":"motorsport.com","origin":"https://motorsport.com","robots":{"status":"ok","http":200,"url":"https://motorsport.com/robots.txt","bytes":1635,"groups":7},"bots":{"GPTBot":["partial","*",10,0],"ChatGPT-User":["partial","*",10,0],"OAI-SearchBot":["partial","*",10,0],"ClaudeBot":["partial","*",10,0],"Claude-User":["partial","*",10,0],"Claude-SearchBot":["partial","*",10,0],"anthropic-ai":["partial","*",10,0],"PerplexityBot":["partial","*",10,0],"Perplexity-User":["partial","*",10,0],"Google-Extended":["partial","*",10,0],"Applebot-Extended":["partial","*",10,0],"CCBot":["partial","*",10,0],"Bytespider":["partial","*",10,0],"Amazonbot":["partial","*",10,0],"meta-externalagent":["partial","*",10,0],"cohere-ai":["partial","*",10,0],"Diffbot":["partial","*",10,0],"ContextFetchBot":["partial","*",10,0]},"llms":{"status":"missing","http":404,"bytes":178484},"home":{"status":"http_error","http":403,"url":"https://motorsport.com/"}},
{"rank":4424,"domain":"escapistmagazine.com","origin":"https://escapistmagazine.com","robots":{"status":"refused","http":403,"url":"https://escapistmagazine.com/robots.txt","bytes":5579,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":4425,"domain":"givebutter.com","origin":"https://givebutter.com","robots":{"status":"ok","http":200,"url":"https://givebutter.com/robots.txt","bytes":67,"groups":1},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"present","http":200,"bytes":16088},"home":{"status":"ok","http":200,"url":"https://givebutter.com/","html_tokens":68893,"markdown_tokens":490,"pruned_tokens":180}},
{"rank":4426,"domain":"lc.chat","origin":"https://lc.chat","robots":{"status":"ok","http":200,"url":"https://lc.chat/robots.txt","bytes":24,"groups":1},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"missing","http":404,"bytes":333},"home":{"status":"ok","http":200,"url":"https://direct.lc.chat/","html_tokens":23894,"markdown_tokens":694,"pruned_tokens":33}},
{"rank":4427,"domain":"nintendo.co.jp","origin":"https://nintendo.co.jp","robots":{"status":"ok","http":200,"url":"https://www.nintendo.co.jp/robots.txt","bytes":1046,"groups":25},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["partial","*",4,0],"OAI-SearchBot":["partial","*",4,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",4,0],"Claude-SearchBot":["partial","*",4,0],"anthropic-ai":["partial","*",4,0],"PerplexityBot":["partial","*",4,0],"Perplexity-User":["partial","*",4,0],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["partial","*",4,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",4,0]},"llms":{"status":"missing","http":404,"bytes":4509},"home":{"status":"ok","http":200,"url":"https://www.nintendo.com/jp/index.html","html_tokens":7105,"markdown_tokens":12,"pruned_tokens":12}},
{"rank":4428,"domain":"tas.gov.au","origin":"https://www.tas.gov.au","robots":{"status":"refused","http":403,"url":"https://www.tas.gov.au/robots.txt","bytes":5552,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":4429,"domain":"az.gov","origin":"https://az.gov","robots":{"status":"refused","http":403,"url":"https://az.gov/robots.txt","bytes":5501,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":4430,"domain":"la-croix.com","origin":"https://la-croix.com","robots":{"status":"ok","http":200,"url":"https://www.la-croix.com/robots.txt","bytes":1444,"groups":1},"bots":{"GPTBot":["partial","*",30,0],"ChatGPT-User":["partial","*",30,0],"OAI-SearchBot":["partial","*",30,0],"ClaudeBot":["partial","*",30,0],"Claude-User":["partial","*",30,0],"Claude-SearchBot":["partial","*",30,0],"anthropic-ai":["partial","*",30,0],"PerplexityBot":["partial","*",30,0],"Perplexity-User":["partial","*",30,0],"Google-Extended":["partial","*",30,0],"Applebot-Extended":["partial","*",30,0],"CCBot":["partial","*",30,0],"Bytespider":["partial","*",30,0],"Amazonbot":["partial","*",30,0],"meta-externalagent":["partial","*",30,0],"cohere-ai":["partial","*",30,0],"Diffbot":["partial","*",30,0],"ContextFetchBot":["partial","*",30,0]},"llms":{"status":"missing","http":404,"bytes":199156},"home":{"status":"ok","http":200,"url":"https://www.la-croix.com/","html_tokens":118988,"markdown_tokens":100,"pruned_tokens":100}},
{"rank":4431,"domain":"dion.ne.jp","origin":"https://dion.ne.jp","robots":{"status":"unreachable","http":null,"url":"https://dion.ne.jp/robots.txt","bytes":0,"groups":0,"error":"cannot resolve dion.ne.jp"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":4432,"domain":"nbcbayarea.com","origin":"https://nbcbayarea.com","robots":{"status":"ok","http":200,"url":"https://www.nbcbayarea.com/robots.txt","bytes":10538,"groups":27},"bots":{"GPTBot":["partial","named",1,6],"ChatGPT-User":["partial","named",1,6],"OAI-SearchBot":["partial","*",13,3],"ClaudeBot":["partial","named",1,6],"Claude-User":["partial","*",13,3],"Claude-SearchBot":["partial","*",13,3],"anthropic-ai":["partial","named",1,6],"PerplexityBot":["partial","named",1,6],"Perplexity-User":["partial","*",13,3],"Google-Extended":["partial","named",1,6],"Applebot-Extended":["partial","*",13,3],"CCBot":["partial","named",1,6],"Bytespider":["partial","named",1,6],"Amazonbot":["partial","named",1,8],"meta-externalagent":["partial","*",13,3],"cohere-ai":["partial","*",13,3],"Diffbot":["partial","named",1,6],"ContextFetchBot":["partial","*",13,3]},"llms":{"status":"timeout","error":"timed out after 10 s"},"home":{"status":"ok","http":200,"url":"https://www.nbcbayarea.com/","html_tokens":143553,"markdown_tokens":3,"pruned_tokens":3}},
{"rank":4433,"domain":"sciencefocus.com","origin":"https://sciencefocus.com","robots":{"status":"ok","http":200,"url":"https://www.sciencefocus.com/robots.txt","bytes":255,"groups":2},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["partial","*",4,0],"OAI-SearchBot":["partial","*",4,0],"ClaudeBot":["partial","*",4,0],"Claude-User":["partial","*",4,0],"Claude-SearchBot":["partial","*",4,0],"anthropic-ai":["partial","*",4,0],"PerplexityBot":["partial","*",4,0],"Perplexity-User":["partial","*",4,0],"Google-Extended":["partial","*",4,0],"Applebot-Extended":["partial","*",4,0],"CCBot":["partial","*",4,0],"Bytespider":["partial","*",4,0],"Amazonbot":["partial","*",4,0],"meta-externalagent":["partial","*",4,0],"cohere-ai":["partial","*",4,0],"Diffbot":["partial","*",4,0],"ContextFetchBot":["partial","*",4,0]},"llms":{"status":"missing","http":404,"bytes":969604},"home":{"status":"ok","http":200,"url":"https://www.sciencefocus.com/","html_tokens":367885,"markdown_tokens":847,"pruned_tokens":36}},
{"rank":4434,"domain":"technet.com","origin":"https://www.technet.com","robots":{"status":"html","http":200,"url":"https://learn.microsoft.com/en-us/","bytes":26788,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":4435,"domain":"google.com.co","origin":"https://google.com.co","robots":{"status":"ok","http":200,"url":"https://www.google.com.co/robots.txt","bytes":6624,"groups":4},"bots":{"GPTBot":["partial","*",169,64],"ChatGPT-User":["partial","*",169,64],"OAI-SearchBot":["partial","*",169,64],"ClaudeBot":["partial","*",169,64],"Claude-User":["partial","*",169,64],"Claude-SearchBot":["partial","*",169,64],"anthropic-ai":["partial","*",169,64],"PerplexityBot":["partial","*",169,64],"Perplexity-User":["partial","*",169,64],"Google-Extended":["partial","*",169,64],"Applebot-Extended":["partial","*",169,64],"CCBot":["partial","*",169,64],"Bytespider":["partial","*",169,64],"Amazonbot":["partial","*",169,64],"meta-externalagent":["partial","*",169,64],"cohere-ai":["partial","*",169,64],"Diffbot":["partial","*",169,64],"ContextFetchBot":["partial","*",169,64]},"llms":{"status":"missing","http":404,"bytes":1569},"home":{"status":"ok","http":200,"url":"https://www.google.com/","html_tokens":21347,"markdown_tokens":30,"pruned_tokens":30}},
{"rank":4436,"domain":"eur.nl","origin":"https://eur.nl","robots":{"status":"ok","http":200,"url":"https://www.eur.nl/robots.txt","bytes":2318,"groups":1},"bots":{"GPTBot":["partial","*",46,18],"ChatGPT-User":["partial","*",46,18],"OAI-SearchBot":["partial","*",46,18],"ClaudeBot":["partial","*",46,18],"Claude-User":["partial","*",46,18],"Claude-SearchBot":["partial","*",46,18],"anthropic-ai":["partial","*",46,18],"PerplexityBot":["partial","*",46,18],"Perplexity-User":["partial","*",46,18],"Google-Extended":["partial","*",46,18],"Applebot-Extended":["partial","*",46,18],"CCBot":["partial","*",46,18],"Bytespider":["partial","*",46,18],"Amazonbot":["partial","*",46,18],"meta-externalagent":["partial","*",46,18],"cohere-ai":["partial","*",46,18],"Diffbot":["partial","*",46,18],"ContextFetchBot":["partial","*",46,18]},"llms":{"status":"missing","http":404,"bytes":144809},"home":{"status":"ok","http":200,"url":"https://www.eur.nl/","html_tokens":42080,"markdown_tokens":69,"pruned_tokens":69}},
{"rank":4437,"domain":"asahi-net.or.jp","origin":"https://www.asahi-net.or.jp","robots":{"status":"missing","http":404,"url":"https://www.asahi-net.or.jp/robots.txt","bytes":236,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":236},"home":{"status":"ok","http":200,"url":"https://asahi-net.jp/","html_tokens":10568,"markdown_tokens":46,"pruned_tokens":46}},
{"rank":4438,"domain":"wilsoncenter.org","origin":"https://wilsoncenter.org","robots":{"status":"ok","http":200,"url":"https://www.wilsoncenter.org/robots.txt","bytes":2374,"groups":3},"bots":{"GPTBot":["partial","*",35,18],"ChatGPT-User":["partial","*",35,18],"OAI-SearchBot":["partial","*",35,18],"ClaudeBot":["partial","*",35,18],"Claude-User":["partial","*",35,18],"Claude-SearchBot":["partial","*",35,18],"anthropic-ai":["partial","*",35,18],"PerplexityBot":["partial","*",35,18],"Perplexity-User":["partial","*",35,18],"Google-Extended":["partial","*",35,18],"Applebot-Extended":["partial","*",35,18],"CCBot":["partial","*",35,18],"Bytespider":["partial","*",35,18],"Amazonbot":["partial","*",35,18],"meta-externalagent":["partial","*",35,18],"cohere-ai":["partial","*",35,18],"Diffbot":["partial","*",35,18],"ContextFetchBot":["partial","*",35,18]},"llms":{"status":"missing","http":404,"bytes":198},"home":{"status":"ok","http":200,"url":"https://www.wilsoncenter.org/","html_tokens":19170,"markdown_tokens":87,"pruned_tokens":87}},
{"rank":4439,"domain":"elysee.fr","origin":"https://elysee.fr","robots":{"status":"html","http":200,"url":"https://www.elysee.fr/","bytes":228429,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":4440,"domain":"aph.gov.au","origin":"https://aph.gov.au","robots":{"status":"refused","http":403,"url":"https://aph.gov.au/robots.txt","bytes":411,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":4441,"domain":"eetimes.com","origin":"https://eetimes.com","robots":{"status":"ok","http":200,"url":"https://www.eetimes.com/robots.txt","bytes":941,"groups":13},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["partial","*",12,3],"OAI-SearchBot":["partial","*",12,3],"ClaudeBot":["partial","*",12,3],"Claude-User":["partial","*",12,3],"Claude-SearchBot":["partial","*",12,3],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["partial","*",12,3],"Perplexity-User":["partial","*",12,3],"Google-Extended":["partial","*",12,3],"Applebot-Extended":["partial","*",12,3],"CCBot":["blocked","named",1,0],"Bytespider":["partial","*",12,3],"Amazonbot":["partial","*",12,3],"meta-externalagent":["partial","*",12,3],"cohere-ai":["blocked","named",1,0],"Diffbot":["partial","*",12,3],"ContextFetchBot":["partial","*",12,3]},"llms":{"status":"timeout","error":"timed out after 10 s"},"home":{"status":"ok","http":200,"url":"https://www.eetimes.com/","html_tokens":83368,"markdown_tokens":59,"pruned_tokens":59}},
{"rank":4442,"domain":"xoom.com","origin":"https://xoom.com","robots":{"status":"ok","http":200,"url":"https://www.xoom.com/robots.txt","bytes":417,"groups":1},"bots":{"GPTBot":["partial","*",11,0],"ChatGPT-User":["partial","*",11,0],"OAI-SearchBot":["partial","*",11,0],"ClaudeBot":["partial","*",11,0],"Claude-User":["partial","*",11,0],"Claude-SearchBot":["partial","*",11,0],"anthropic-ai":["partial","*",11,0],"PerplexityBot":["partial","*",11,0],"Perplexity-User":["partial","*",11,0],"Google-Extended":["partial","*",11,0],"Applebot-Extended":["partial","*",11,0],"CCBot":["partial","*",11,0],"Bytespider":["partial","*",11,0],"Amazonbot":["partial","*",11,0],"meta-externalagent":["partial","*",11,0],"cohere-ai":["partial","*",11,0],"Diffbot":["partial","*",11,0],"ContextFetchBot":["partial","*",11,0]},"llms":{"status":"missing","http":404,"bytes":744},"home":{"status":"ok","http":200,"url":"https://www.xoom.com/","html_tokens":55491,"markdown_tokens":116,"pruned_tokens":28}},
{"rank":4443,"domain":"exxonmobil.com","origin":"https://exxonmobil.com","robots":{"status":"ok","http":200,"url":"https://www.exxonmobil.com/robots.txt","bytes":1228,"groups":1},"bots":{"GPTBot":["partial","*",11,0],"ChatGPT-User":["partial","*",11,0],"OAI-SearchBot":["partial","*",11,0],"ClaudeBot":["partial","*",11,0],"Claude-User":["partial","*",11,0],"Claude-SearchBot":["partial","*",11,0],"anthropic-ai":["partial","*",11,0],"PerplexityBot":["partial","*",11,0],"Perplexity-User":["partial","*",11,0],"Google-Extended":["partial","*",11,0],"Applebot-Extended":["partial","*",11,0],"CCBot":["partial","*",11,0],"Bytespider":["partial","*",11,0],"Amazonbot":["partial","*",11,0],"meta-externalagent":["partial","*",11,0],"cohere-ai":["partial","*",11,0],"Diffbot":["partial","*",11,0],"ContextFetchBot":["partial","*",11,0]},"llms":{"status":"missing","http":404,"bytes":1245},"home":{"status":"ok","http":200,"url":"https://corporate.exxonmobil.com","html_tokens":30078,"markdown_tokens":333,"pruned_tokens":225}},
{"rank":4444,"domain":"google.ae","origin":"https://google.ae","robots":{"status":"ok","http":200,"url":"https://www.google.ae/robots.txt","bytes":6624,"groups":4},"bots":{"GPTBot":["partial","*",169,64],"ChatGPT-User":["partial","*",169,64],"OAI-SearchBot":["partial","*",169,64],"ClaudeBot":["partial","*",169,64],"Claude-User":["partial","*",169,64],"Claude-SearchBot":["partial","*",169,64],"anthropic-ai":["partial","*",169,64],"PerplexityBot":["partial","*",169,64],"Perplexity-User":["partial","*",169,64],"Google-Extended":["partial","*",169,64],"Applebot-Extended":["partial","*",169,64],"CCBot":["partial","*",169,64],"Bytespider":["partial","*",169,64],"Amazonbot":["partial","*",169,64],"meta-externalagent":["partial","*",169,64],"cohere-ai":["partial","*",169,64],"Diffbot":["partial","*",169,64],"ContextFetchBot":["partial","*",169,64]},"llms":{"status":"missing","http":404,"bytes":1569},"home":{"status":"ok","http":200,"url":"https://www.google.com/","html_tokens":21287,"markdown_tokens":30,"pruned_tokens":30}},
{"rank":4445,"domain":"gamasutra.com","origin":"https://gamasutra.com","robots":{"status":"ok","http":200,"url":"https://www.gamedeveloper.com/robots.txt","bytes":1559,"groups":2},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["partial","*",6,0],"OAI-SearchBot":["partial","*",6,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",6,0],"Claude-SearchBot":["partial","*",6,0],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["partial","*",6,0],"Perplexity-User":["partial","*",6,0],"Google-Extended":["partial","*",6,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",6,0]},"llms":{"status":"missing","http":404,"bytes":332195},"home":{"status":"ok","http":200,"url":"https://www.gamedeveloper.com/","html_tokens":128264,"markdown_tokens":92,"pruned_tokens":92}},
{"rank":4446,"domain":"almalinux.org","origin":"https://almalinux.org","robots":{"status":"missing","http":404,"url":"https://almalinux.org/robots.txt","bytes":18320,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":18320},"home":{"status":"ok","http":200,"url":"https://almalinux.org/","html_tokens":12252,"markdown_tokens":513,"pruned_tokens":447}},
{"rank":4447,"domain":"actionnetwork.org","origin":"https://actionnetwork.org","robots":{"status":"ok","http":200,"url":"https://actionnetwork.org/robots.txt","bytes":199,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":43293},"home":{"status":"ok","http":200,"url":"https://actionnetwork.org/","html_tokens":28136,"markdown_tokens":135,"pruned_tokens":47}},
{"rank":4448,"domain":"uni-kiel.de","origin":"https://www.uni-kiel.de","robots":{"status":"ok","http":200,"url":"https://www.uni-kiel.de/robots.txt","bytes":783,"groups":2},"bots":{"GPTBot":["blocked","*",1,0],"ChatGPT-User":["blocked","*",1,0],"OAI-SearchBot":["blocked","*",1,0],"ClaudeBot":["blocked","*",1,0],"Claude-User":["blocked","*",1,0],"Claude-SearchBot":["blocked","*",1,0],"anthropic-ai":["blocked","*",1,0],"PerplexityBot":["blocked","*",1,0],"Perplexity-User":["blocked","*",1,0],"Google-Extended":["blocked","*",1,0],"Applebot-Extended":["blocked","*",1,0],"CCBot":["blocked","*",1,0],"Bytespider":["blocked","*",1,0],"Amazonbot":["blocked","*",1,0],"meta-externalagent":["blocked","*",1,0],"cohere-ai":["blocked","*",1,0],"Diffbot":["blocked","*",1,0],"ContextFetchBot":["blocked","*",1,0]},"llms":{"status":"disallowed"},"home":{"status":"disallowed"}},
{"rank":4449,"domain":"freeserve.co.uk","origin":"https://freeserve.co.uk","robots":{"status":"unreachable","http":null,"url":"https://freeserve.co.uk/robots.txt","bytes":0,"groups":0,"error":"cannot resolve freeserve.co.uk"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":4450,"domain":"uri.edu","origin":"https://uri.edu","robots":{"status":"refused","http":403,"url":"https://uri.edu/robots.txt","bytes":560,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":4451,"domain":"law360.com","origin":"https://law360.com","robots":{"status":"ok","http":200,"url":"https://www.law360.com/robots.txt","bytes":594,"groups":4},"bots":{"GPTBot":["partial","*",4,0],"ChatGPT-User":["partial","*",4,0],"OAI-SearchBot":["partial","*",4,0],"ClaudeBot":["partial","*",4,0],"Claude-User":["partial","*",4,0],"Claude-SearchBot":["partial","*",4,0],"anthropic-ai":["partial","*",4,0],"PerplexityBot":["partial","*",4,0],"Perplexity-User":["partial","*",4,0],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["partial","*",4,0],"CCBot":["partial","*",4,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["partial","*",4,0],"meta-externalagent":["partial","*",4,0],"cohere-ai":["partial","*",4,0],"Diffbot":["partial","*",4,0],"ContextFetchBot":["partial","*",4,0]},"llms":{"status":"missing","http":404,"bytes":52657},"home":{"status":"ok","http":200,"url":"https://www.law360.com/","html_tokens":41461,"markdown_tokens":301,"pruned_tokens":120}},
{"rank":4452,"domain":"annualcreditreport.com","origin":"https://annualcreditreport.com","robots":{"status":"refused","http":403,"url":"https://www.annualcreditreport.com/robots.txt","bytes":394,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":4453,"domain":"wnyc.org","origin":"https://wnyc.org","robots":{"status":"ok","http":200,"url":"https://wnyc.org/robots.txt","bytes":333,"groups":1},"bots":{"GPTBot":["partial","*",13,0],"ChatGPT-User":["partial","*",13,0],"OAI-SearchBot":["partial","*",13,0],"ClaudeBot":["partial","*",13,0],"Claude-User":["partial","*",13,0],"Claude-SearchBot":["partial","*",13,0],"anthropic-ai":["partial","*",13,0],"PerplexityBot":["partial","*",13,0],"Perplexity-User":["partial","*",13,0],"Google-Extended":["partial","*",13,0],"Applebot-Extended":["partial","*",13,0],"CCBot":["partial","*",13,0],"Bytespider":["partial","*",13,0],"Amazonbot":["partial","*",13,0],"meta-externalagent":["partial","*",13,0],"cohere-ai":["partial","*",13,0],"Diffbot":["partial","*",13,0],"ContextFetchBot":["partial","*",13,0]},"llms":{"status":"missing","http":404,"bytes":741366},"home":{"status":"ok","http":200,"url":"https://wnyc.org/home","html_tokens":226499,"markdown_tokens":753,"pruned_tokens":238}},
{"rank":4454,"domain":"arnebrachhold.de","origin":"https://arnebrachhold.de","robots":{"status":"ok","http":200,"url":"https://www.arnebrachhold.de/robots.txt","bytes":155,"groups":2},"bots":{"GPTBot":["partial","*",2,0],"ChatGPT-User":["partial","*",2,0],"OAI-SearchBot":["partial","*",2,0],"ClaudeBot":["partial","*",2,0],"Claude-User":["partial","*",2,0],"Claude-SearchBot":["partial","*",2,0],"anthropic-ai":["partial","*",2,0],"PerplexityBot":["partial","*",2,0],"Perplexity-User":["partial","*",2,0],"Google-Extended":["partial","*",2,0],"Applebot-Extended":["partial","*",2,0],"CCBot":["partial","*",2,0],"Bytespider":["partial","*",2,0],"Amazonbot":["partial","*",2,0],"meta-externalagent":["partial","*",2,0],"cohere-ai":["partial","*",2,0],"Diffbot":["partial","*",2,0],"ContextFetchBot":["partial","*",2,0]},"llms":{"status":"missing","http":404,"bytes":3162},"home":{"status":"ok","http":200,"url":"https://www.arnebrachhold.de/","html_tokens":1570,"markdown_tokens":169,"pruned_tokens":169}},
{"rank":4455,"domain":"heavy.com","origin":"https://heavy.com","robots":{"status":"ok","http":200,"url":"https://heavy.com/robots.txt","bytes":639,"groups":4},"bots":{"GPTBot":["partial","*",15,0],"ChatGPT-User":["partial","*",15,0],"OAI-SearchBot":["partial","*",15,0],"ClaudeBot":["partial","*",15,0],"Claude-User":["partial","*",15,0],"Claude-SearchBot":["partial","*",15,0],"anthropic-ai":["partial","*",15,0],"PerplexityBot":["partial","*",15,0],"Perplexity-User":["partial","*",15,0],"Google-Extended":["partial","*",15,0],"Applebot-Extended":["partial","*",15,0],"CCBot":["partial","*",15,0],"Bytespider":["partial","*",15,0],"Amazonbot":["partial","*",15,0],"meta-externalagent":["partial","*",15,0],"cohere-ai":["partial","*",15,0],"Diffbot":["partial","*",15,0],"ContextFetchBot":["partial","*",15,0]},"llms":{"status":"missing","http":404,"bytes":52424},"home":{"status":"ok","http":200,"url":"https://heavy.com/","html_tokens":46608,"markdown_tokens":5,"pruned_tokens":5}},
{"rank":4456,"domain":"pastemagazine.com","origin":"https://pastemagazine.com","robots":{"status":"ok","http":200,"url":"https://www.pastemagazine.com/robots.txt","bytes":367,"groups":2},"bots":{"GPTBot":["partial","*",3,0],"ChatGPT-User":["partial","*",3,0],"OAI-SearchBot":["partial","*",3,0],"ClaudeBot":["partial","*",3,0],"Claude-User":["partial","*",3,0],"Claude-SearchBot":["partial","*",3,0],"anthropic-ai":["partial","*",3,0],"PerplexityBot":["partial","*",3,0],"Perplexity-User":["partial","*",3,0],"Google-Extended":["partial","*",3,0],"Applebot-Extended":["partial","*",3,0],"CCBot":["partial","*",3,0],"Bytespider":["partial","*",3,0],"Amazonbot":["partial","*",3,0],"meta-externalagent":["partial","*",3,0],"cohere-ai":["partial","*",3,0],"Diffbot":["partial","*",3,0],"ContextFetchBot":["partial","*",3,0]},"llms":{"status":"missing","http":404,"bytes":98585},"home":{"status":"ok","http":200,"url":"https://www.pastemagazine.com/","html_tokens":43750,"markdown_tokens":966,"pruned_tokens":175}},
{"rank":4457,"domain":"bravesites.com","origin":"https://bravesites.com","robots":{"status":"missing","http":404,"url":"https://bravesites.com/robots.txt","bytes":8839,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":8839},"home":{"status":"ok","http":200,"url":"https://bravenet.com/","html_tokens":11702,"markdown_tokens":160,"pruned_tokens":20}},
{"rank":4458,"domain":"ens.fr","origin":"https://ens.fr","robots":{"status":"ok","http":200,"url":"https://www.ens.psl.eu/robots.txt","bytes":2141,"groups":1},"bots":{"GPTBot":["partial","*",35,22],"ChatGPT-User":["partial","*",35,22],"OAI-SearchBot":["partial","*",35,22],"ClaudeBot":["partial","*",35,22],"Claude-User":["partial","*",35,22],"Claude-SearchBot":["partial","*",35,22],"anthropic-ai":["partial","*",35,22],"PerplexityBot":["partial","*",35,22],"Perplexity-User":["partial","*",35,22],"Google-Extended":["partial","*",35,22],"Applebot-Extended":["partial","*",35,22],"CCBot":["partial","*",35,22],"Bytespider":["partial","*",35,22],"Amazonbot":["partial","*",35,22],"meta-externalagent":["partial","*",35,22],"cohere-ai":["partial","*",35,22],"Diffbot":["partial","*",35,22],"ContextFetchBot":["partial","*",35,22]},"llms":{"status":"missing","http":404,"bytes":188},"home":{"status":"ok","http":200,"url":"https://www.ens.psl.eu/","html_tokens":35358,"markdown_tokens":49,"pruned_tokens":49}},
{"rank":4459,"domain":"telekom.de","origin":"https://telekom.de","robots":{"status":"html","http":202,"url":"https://www.telekom.de/robots.txt","bytes":1995,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":4460,"domain":"diagrams.net","origin":"https://diagrams.net","robots":{"status":"ok","http":200,"url":"https://www.drawio.com/robots.txt","bytes":68,"groups":1},"bots":{"GPTBot":["allowed","*",0,1],"ChatGPT-User":["allowed","*",0,1],"OAI-SearchBot":["allowed","*",0,1],"ClaudeBot":["allowed","*",0,1],"Claude-User":["allowed","*",0,1],"Claude-SearchBot":["allowed","*",0,1],"anthropic-ai":["allowed","*",0,1],"PerplexityBot":["allowed","*",0,1],"Perplexity-User":["allowed","*",0,1],"Google-Extended":["allowed","*",0,1],"Applebot-Extended":["allowed","*",0,1],"CCBot":["allowed","*",0,1],"Bytespider":["allowed","*",0,1],"Amazonbot":["allowed","*",0,1],"meta-externalagent":["allowed","*",0,1],"cohere-ai":["allowed","*",0,1],"Diffbot":["allowed","*",0,1],"ContextFetchBot":["allowed","*",0,1]},"llms":{"status":"present","http":200,"bytes":6486},"home":{"status":"ok","http":200,"url":"https://www.drawio.com/","html_tokens":5495,"markdown_tokens":314,"pruned_tokens":50}},
{"rank":4461,"domain":"mydrivers.com","origin":"https://mydrivers.com","robots":{"status":"ok","http":200,"url":"https://www.mydrivers.com/robots.txt","bytes":119,"groups":1},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"missing","http":404,"bytes":1231},"home":{"status":"ok","http":200,"url":"https://www.mydrivers.com/","html_tokens":81305,"markdown_tokens":135,"pruned_tokens":135}},
{"rank":4462,"domain":"tripadvisor.it","origin":"https://tripadvisor.it","robots":{"status":"ok","http":200,"url":"https://www.tripadvisor.it/robots.txt","bytes":22865,"groups":9},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["partial","named",623,8],"OAI-SearchBot":["partial","named",623,8],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",621,8],"Claude-SearchBot":["partial","*",621,8],"anthropic-ai":["partial","*",621,8],"PerplexityBot":["partial","named",625,8],"Perplexity-User":["partial","*",621,8],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["partial","*",621,8],"ContextFetchBot":["partial","*",621,8]},"llms":{"status":"http_error","http":403,"bytes":774},"home":{"status":"http_error","http":403,"url":"https://www.tripadvisor.it/"}},
{"rank":4463,"domain":"mindspring.com","origin":"https://mindspring.com","robots":{"status":"refused","http":403,"url":"https://www.earthlink.net/","bytes":26199,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":4464,"domain":"fitchratings.com","origin":"https://fitchratings.com","robots":{"status":"ok","http":200,"url":"https://www.fitchratings.com/robots.txt","bytes":192,"groups":1},"bots":{"GPTBot":["partial","*",6,0],"ChatGPT-User":["partial","*",6,0],"OAI-SearchBot":["partial","*",6,0],"ClaudeBot":["partial","*",6,0],"Claude-User":["partial","*",6,0],"Claude-SearchBot":["partial","*",6,0],"anthropic-ai":["partial","*",6,0],"PerplexityBot":["partial","*",6,0],"Perplexity-User":["partial","*",6,0],"Google-Extended":["partial","*",6,0],"Applebot-Extended":["partial","*",6,0],"CCBot":["partial","*",6,0],"Bytespider":["partial","*",6,0],"Amazonbot":["partial","*",6,0],"meta-externalagent":["partial","*",6,0],"cohere-ai":["partial","*",6,0],"Diffbot":["partial","*",6,0],"ContextFetchBot":["partial","*",6,0]},"llms":{"status":"present","http":200,"bytes":23334},"home":{"status":"ok","http":200,"url":"https://www.fitchratings.com/","html_tokens":451153,"markdown_tokens":74,"pruned_tokens":74}},
{"rank":4465,"domain":"donga.com","origin":"https://donga.com","robots":{"status":"ok","http":200,"url":"https://www.donga.com/robots.txt","bytes":665,"groups":1},"bots":{"GPTBot":["partial","*",15,1],"ChatGPT-User":["partial","*",15,1],"OAI-SearchBot":["partial","*",15,1],"ClaudeBot":["partial","*",15,1],"Claude-User":["partial","*",15,1],"Claude-SearchBot":["partial","*",15,1],"anthropic-ai":["partial","*",15,1],"PerplexityBot":["partial","*",15,1],"Perplexity-User":["partial","*",15,1],"Google-Extended":["partial","*",15,1],"Applebot-Extended":["partial","*",15,1],"CCBot":["partial","*",15,1],"Bytespider":["partial","*",15,1],"Amazonbot":["partial","*",15,1],"meta-externalagent":["partial","*",15,1],"cohere-ai":["partial","*",15,1],"Diffbot":["partial","*",15,1],"ContextFetchBot":["partial","*",15,1]},"llms":{"status":"missing","http":404,"bytes":91492},"home":{"status":"ok","http":200,"url":"https://www.donga.com/","html_tokens":53193,"markdown_tokens":125,"pruned_tokens":125}},
{"rank":4466,"domain":"bom.gov.au","origin":"https://bom.gov.au","robots":{"status":"refused","http":403,"url":"https://www.bom.gov.au","bytes":1384,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":4467,"domain":"activestate.com","origin":"https://activestate.com","robots":{"status":"ok","http":200,"url":"https://www.activestate.com/robots.txt","bytes":52,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"present","http":200,"bytes":13066},"home":{"status":"ok","http":200,"url":"https://www.activestate.com/","html_tokens":130149,"markdown_tokens":898,"pruned_tokens":291}},
{"rank":4468,"domain":"freshmeat.net","origin":"https://freshmeat.net","robots":{"status":"html","http":200,"url":"https://sourceforge.net/directory/software-development/","bytes":390661,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":4469,"domain":"vanguard.com","origin":"https://vanguard.com","robots":{"status":"ok","http":200,"url":"https://www.vanguard.com/robots.txt","bytes":940,"groups":1},"bots":{"GPTBot":["partial","*",21,0],"ChatGPT-User":["partial","*",21,0],"OAI-SearchBot":["partial","*",21,0],"ClaudeBot":["partial","*",21,0],"Claude-User":["partial","*",21,0],"Claude-SearchBot":["partial","*",21,0],"anthropic-ai":["partial","*",21,0],"PerplexityBot":["partial","*",21,0],"Perplexity-User":["partial","*",21,0],"Google-Extended":["partial","*",21,0],"Applebot-Extended":["partial","*",21,0],"CCBot":["partial","*",21,0],"Bytespider":["partial","*",21,0],"Amazonbot":["partial","*",21,0],"meta-externalagent":["partial","*",21,0],"cohere-ai":["partial","*",21,0],"Diffbot":["partial","*",21,0],"ContextFetchBot":["partial","*",21,0]},"llms":{"status":"missing","http":404,"bytes":66901},"home":{"status":"ok","http":200,"url":"https://investor.vanguard.com/corporate-portal","html_tokens":6990,"markdown_tokens":86,"pruned_tokens":86}},
{"rank":4470,"domain":"icc-cpi.int","origin":"https://icc-cpi.int","robots":{"status":"ok","http":200,"url":"https://www.icc-cpi.int/robots.txt","bytes":2141,"groups":1},"bots":{"GPTBot":["partial","*",35,22],"ChatGPT-User":["partial","*",35,22],"OAI-SearchBot":["partial","*",35,22],"ClaudeBot":["partial","*",35,22],"Claude-User":["partial","*",35,22],"Claude-SearchBot":["partial","*",35,22],"anthropic-ai":["partial","*",35,22],"PerplexityBot":["partial","*",35,22],"Perplexity-User":["partial","*",35,22],"Google-Extended":["partial","*",35,22],"Applebot-Extended":["partial","*",35,22],"CCBot":["partial","*",35,22],"Bytespider":["partial","*",35,22],"Amazonbot":["partial","*",35,22],"meta-externalagent":["partial","*",35,22],"cohere-ai":["partial","*",35,22],"Diffbot":["partial","*",35,22],"ContextFetchBot":["partial","*",35,22]},"llms":{"status":"missing","http":404,"bytes":189},"home":{"status":"ok","http":200,"url":"https://www.icc-cpi.int/","html_tokens":21459,"markdown_tokens":212,"pruned_tokens":148}},
{"rank":4471,"domain":"healio.com","origin":"https://healio.com","robots":{"status":"ok","http":200,"url":"https://www.healio.com/robots.txt","bytes":1565,"groups":30},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["blocked","named",1,0],"OAI-SearchBot":["blocked","named",1,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",20,1],"Claude-SearchBot":["partial","*",20,1],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["blocked","named",1,0],"Google-Extended":["partial","*",20,1],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",20,1]},"llms":{"status":"missing","http":404,"bytes":4994},"home":{"status":"ok","http":200,"url":"https://www.healio.com/","html_tokens":104015,"markdown_tokens":107,"pruned_tokens":107}},
{"rank":4472,"domain":"confex.com","origin":"https://confex.com","robots":{"status":"missing","http":404,"url":"https://confex.com/robots.txt","bytes":146,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":146},"home":{"status":"ok","http":200,"url":"https://confex.com/","html_tokens":4661,"markdown_tokens":466,"pruned_tokens":267}},
{"rank":4473,"domain":"mountsinai.org","origin":"https://mountsinai.org","robots":{"status":"html","http":200,"url":"https://www.mountsinai.org/","bytes":172865,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":4474,"domain":"mikrotik.com","origin":"https://mikrotik.com","robots":{"status":"ok","http":200,"url":"https://mikrotik.com/robots.txt","bytes":66,"groups":1},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"missing","http":404,"bytes":14},"home":{"status":"ok","http":200,"url":"https://mikrotik.com/","html_tokens":79660,"markdown_tokens":649,"pruned_tokens":379}},
{"rank":4475,"domain":"stability.ai","origin":"https://stability.ai","robots":{"status":"ok","http":200,"url":"https://stability.ai/robots.txt","bytes":1503,"groups":1},"bots":{"GPTBot":["partial","named",27,1],"ChatGPT-User":["partial","*",27,1],"OAI-SearchBot":["partial","*",27,1],"ClaudeBot":["partial","named",27,1],"Claude-User":["partial","*",27,1],"Claude-SearchBot":["partial","*",27,1],"anthropic-ai":["partial","named",27,1],"PerplexityBot":["partial","*",27,1],"Perplexity-User":["partial","*",27,1],"Google-Extended":["partial","named",27,1],"Applebot-Extended":["partial","named",27,1],"CCBot":["partial","named",27,1],"Bytespider":["partial","named",27,1],"Amazonbot":["partial","named",27,1],"meta-externalagent":["partial","named",27,1],"cohere-ai":["partial","named",27,1],"Diffbot":["partial","*",27,1],"ContextFetchBot":["partial","*",27,1]},"llms":{"status":"missing","http":404,"bytes":0},"home":{"status":"ok","http":200,"url":"https://stability.ai/","html_tokens":126191,"markdown_tokens":272,"pruned_tokens":106}},
{"rank":4476,"domain":"public.lu","origin":"https://public.lu","robots":{"status":"html","http":200,"url":"https://etat.public.lu/fr.html","bytes":93649,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":4477,"domain":"tidd.ly","origin":"https://tidd.ly","robots":{"status":"ok","http":200,"url":"https://tidd.ly/robots.txt","bytes":123,"groups":1},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"missing","http":404,"bytes":6075},"home":{"status":"ok","http":200,"url":"https://www.awin.com","html_tokens":74125,"markdown_tokens":946,"pruned_tokens":102}},
{"rank":4478,"domain":"cigna.com","origin":"https://cigna.com","robots":{"status":"ok","http":200,"url":"https://www.cigna.com/robots.txt","bytes":4041,"groups":1},"bots":{"GPTBot":["partial","*",79,23],"ChatGPT-User":["partial","*",79,23],"OAI-SearchBot":["partial","*",79,23],"ClaudeBot":["partial","*",79,23],"Claude-User":["partial","*",79,23],"Claude-SearchBot":["partial","*",79,23],"anthropic-ai":["partial","*",79,23],"PerplexityBot":["partial","*",79,23],"Perplexity-User":["partial","*",79,23],"Google-Extended":["partial","*",79,23],"Applebot-Extended":["partial","*",79,23],"CCBot":["partial","*",79,23],"Bytespider":["partial","*",79,23],"Amazonbot":["partial","*",79,23],"meta-externalagent":["partial","*",79,23],"cohere-ai":["partial","*",79,23],"Diffbot":["partial","*",79,23],"ContextFetchBot":["partial","*",79,23]},"llms":{"status":"other_text","http":200,"bytes":9004},"home":{"status":"ok","http":200,"url":"https://www.cigna.com/","html_tokens":43475,"markdown_tokens":1699,"pruned_tokens":655}},
{"rank":4479,"domain":"unric.org","origin":"https://unric.org","robots":{"status":"ok","http":200,"url":"https://unric.org/robots.txt","bytes":4802,"groups":16},"bots":{"GPTBot":["allowed","named",0,1],"ChatGPT-User":["allowed","named",0,1],"OAI-SearchBot":["allowed","named",0,1],"ClaudeBot":["allowed","named",0,1],"Claude-User":["allowed","named",0,1],"Claude-SearchBot":["allowed","named",0,1],"anthropic-ai":["partial","*",33,28],"PerplexityBot":["allowed","named",0,1],"Perplexity-User":["partial","*",33,28],"Google-Extended":["allowed","named",0,1],"Applebot-Extended":["allowed","named",0,1],"CCBot":["partial","*",33,28],"Bytespider":["allowed","named",0,1],"Amazonbot":["partial","*",33,28],"meta-externalagent":["partial","*",33,28],"cohere-ai":["partial","*",33,28],"Diffbot":["allowed","named",0,1],"ContextFetchBot":["partial","*",33,28]},"llms":{"status":"present","http":200,"bytes":17413},"home":{"status":"http_error","http":403,"url":"https://unric.org/"}},
{"rank":4480,"domain":"ubi.com","origin":"https://ubi.com","robots":{"status":"html","http":200,"url":"https://www.ubisoft.com/en-us","bytes":548368,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":4481,"domain":"prestashop-project.org","origin":"https://prestashop-project.org","robots":{"status":"ok","http":200,"url":"https://www.prestashop-project.org/robots.txt","bytes":13,"groups":1},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"missing","http":404,"bytes":10627},"home":{"status":"ok","http":200,"url":"https://www.prestashop-project.org/","html_tokens":48637,"markdown_tokens":66,"pruned_tokens":66}},
{"rank":4482,"domain":"asha.org","origin":"https://asha.org","robots":{"status":"ok","http":200,"url":"https://www.asha.org/robots.txt","bytes":1071,"groups":19},"bots":{"GPTBot":["partial","*",9,0],"ChatGPT-User":["partial","*",9,0],"OAI-SearchBot":["partial","*",9,0],"ClaudeBot":["partial","*",9,0],"Claude-User":["partial","*",9,0],"Claude-SearchBot":["partial","*",9,0],"anthropic-ai":["partial","*",9,0],"PerplexityBot":["partial","*",9,0],"Perplexity-User":["partial","*",9,0],"Google-Extended":["partial","*",9,0],"Applebot-Extended":["partial","*",9,0],"CCBot":["partial","*",9,0],"Bytespider":["partial","*",9,0],"Amazonbot":["partial","*",9,0],"meta-externalagent":["partial","*",9,0],"cohere-ai":["partial","*",9,0],"Diffbot":["partial","*",9,0],"ContextFetchBot":["partial","*",9,0]},"llms":{"status":"html","http":200,"bytes":53645},"home":{"status":"ok","http":200,"url":"https://www.asha.org/","html_tokens":19369,"markdown_tokens":28,"pruned_tokens":28}},
{"rank":4483,"domain":"nespresso.com","origin":"https://nespresso.com","robots":{"status":"ok","http":200,"url":"https://www.nespresso.com/robots.txt","bytes":6143,"groups":1},"bots":{"GPTBot":["partial","*",127,86],"ChatGPT-User":["partial","*",127,86],"OAI-SearchBot":["partial","*",127,86],"ClaudeBot":["partial","*",127,86],"Claude-User":["partial","*",127,86],"Claude-SearchBot":["partial","*",127,86],"anthropic-ai":["partial","*",127,86],"PerplexityBot":["partial","*",127,86],"Perplexity-User":["partial","*",127,86],"Google-Extended":["partial","*",127,86],"Applebot-Extended":["partial","*",127,86],"CCBot":["partial","*",127,86],"Bytespider":["partial","*",127,86],"Amazonbot":["partial","*",127,86],"meta-externalagent":["partial","*",127,86],"cohere-ai":["partial","*",127,86],"Diffbot":["partial","*",127,86],"ContextFetchBot":["partial","*",127,86]},"llms":{"status":"http_error","http":403,"bytes":13},"home":{"status":"ok","http":200,"url":"https://www.nespresso.com/us/en","html_tokens":60010,"markdown_tokens":35,"pruned_tokens":29}},
{"rank":4484,"domain":"lrb.co.uk","origin":"https://lrb.co.uk","robots":{"status":"ok","http":200,"url":"https://www.lrb.co.uk/robots.txt","bytes":5633,"groups":36},"bots":{"GPTBot":["partial","named",5,0],"ChatGPT-User":["partial","named",5,0],"OAI-SearchBot":["partial","named",5,0],"ClaudeBot":["partial","named",5,0],"Claude-User":["partial","named",5,0],"Claude-SearchBot":["partial","named",5,0],"anthropic-ai":["partial","named",10,0],"PerplexityBot":["partial","named",5,0],"Perplexity-User":["partial","*",2,0],"Google-Extended":["partial","named",5,0],"Applebot-Extended":["partial","named",5,0],"CCBot":["blocked","named",1,0],"Bytespider":["partial","named",5,0],"Amazonbot":["partial","named",5,0],"meta-externalagent":["partial","*",2,0],"cohere-ai":["partial","*",2,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",2,0]},"llms":{"status":"missing","http":404,"bytes":210128},"home":{"status":"ok","http":200,"url":"https://www.lrb.co.uk/","html_tokens":114905,"markdown_tokens":168,"pruned_tokens":78}},
{"rank":4485,"domain":"businessinsider.in","origin":"https://businessinsider.in","robots":{"status":"unreachable","http":null,"url":"https://businessinsider.in/robots.txt","bytes":0,"groups":0,"error":"cannot resolve businessinsider.in"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":4486,"domain":"scu.edu","origin":"https://scu.edu","robots":{"status":"ok","http":200,"url":"https://www.scu.edu/robots.txt","bytes":400,"groups":2},"bots":{"GPTBot":["partial","*",11,1],"ChatGPT-User":["partial","*",11,1],"OAI-SearchBot":["partial","*",11,1],"ClaudeBot":["partial","*",11,1],"Claude-User":["partial","*",11,1],"Claude-SearchBot":["partial","*",11,1],"anthropic-ai":["partial","*",11,1],"PerplexityBot":["partial","*",11,1],"Perplexity-User":["partial","*",11,1],"Google-Extended":["partial","*",11,1],"Applebot-Extended":["partial","*",11,1],"CCBot":["partial","*",11,1],"Bytespider":["partial","*",11,1],"Amazonbot":["partial","*",11,1],"meta-externalagent":["partial","*",11,1],"cohere-ai":["partial","*",11,1],"Diffbot":["partial","*",11,1],"ContextFetchBot":["partial","*",11,1]},"llms":{"status":"missing","http":404,"bytes":35871},"home":{"status":"ok","http":200,"url":"https://www.scu.edu/","html_tokens":19546,"markdown_tokens":206,"pruned_tokens":102}},
{"rank":4487,"domain":"kh.ua","origin":"https://www.kh.ua","robots":{"status":"ok","http":200,"url":"https://www.kh.ua/robots.txt","bytes":243,"groups":2},"bots":{"GPTBot":["partial","*",3,0],"ChatGPT-User":["partial","*",3,0],"OAI-SearchBot":["partial","*",3,0],"ClaudeBot":["partial","*",3,0],"Claude-User":["partial","*",3,0],"Claude-SearchBot":["partial","*",3,0],"anthropic-ai":["partial","*",3,0],"PerplexityBot":["partial","*",3,0],"Perplexity-User":["partial","*",3,0],"Google-Extended":["partial","*",3,0],"Applebot-Extended":["partial","*",3,0],"CCBot":["partial","*",3,0],"Bytespider":["partial","*",3,0],"Amazonbot":["partial","*",3,0],"meta-externalagent":["partial","*",3,0],"cohere-ai":["partial","*",3,0],"Diffbot":["partial","*",3,0],"ContextFetchBot":["partial","*",3,0]},"llms":{"status":"missing","http":404,"bytes":236},"home":{"status":"ok","http":200,"url":"https://www.kh.ua/","html_tokens":9491,"markdown_tokens":179,"pruned_tokens":21}},
{"rank":4488,"domain":"iisd.org","origin":"https://iisd.org","robots":{"status":"ok","http":200,"url":"https://www.iisd.org/robots.txt","bytes":4312,"groups":33},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["partial","*",22,19],"OAI-SearchBot":["partial","*",22,19],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",22,19],"Claude-SearchBot":["partial","*",22,19],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["partial","*",22,19],"Perplexity-User":["partial","*",22,19],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",22,19]},"llms":{"status":"missing","http":404,"bytes":547},"home":{"status":"ok","http":200,"url":"https://www.iisd.org/","html_tokens":30026,"markdown_tokens":1286,"pruned_tokens":380}},
{"rank":4489,"domain":"cash.app","origin":"https://cash.app","robots":{"status":"ok","http":200,"url":"https://cash.app/robots.txt","bytes":499,"groups":1},"bots":{"GPTBot":["partial","*",16,1],"ChatGPT-User":["partial","*",16,1],"OAI-SearchBot":["partial","*",16,1],"ClaudeBot":["partial","*",16,1],"Claude-User":["partial","*",16,1],"Claude-SearchBot":["partial","*",16,1],"anthropic-ai":["partial","*",16,1],"PerplexityBot":["partial","*",16,1],"Perplexity-User":["partial","*",16,1],"Google-Extended":["partial","*",16,1],"Applebot-Extended":["partial","*",16,1],"CCBot":["partial","*",16,1],"Bytespider":["partial","*",16,1],"Amazonbot":["partial","*",16,1],"meta-externalagent":["partial","*",16,1],"cohere-ai":["partial","*",16,1],"Diffbot":["partial","*",16,1],"ContextFetchBot":["partial","*",16,1]},"llms":{"status":"missing","http":404,"bytes":20077},"home":{"status":"ok","http":200,"url":"https://cash.app/","html_tokens":122033,"markdown_tokens":127,"pruned_tokens":127}},
{"rank":4490,"domain":"mimecastprotect.com","origin":"https://mimecastprotect.com","robots":{"status":"unreachable","http":null,"url":"https://mimecastprotect.com/robots.txt","bytes":0,"groups":0,"error":"cannot resolve mimecastprotect.com"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":4491,"domain":"test.de","origin":"https://test.de","robots":{"status":"ok","http":200,"url":"https://www.test.de/robots.txt","bytes":1431,"groups":19},"bots":{"GPTBot":["partial","named",1,1],"ChatGPT-User":["blocked","named",1,0],"OAI-SearchBot":["partial","*",6,1],"ClaudeBot":["partial","named",1,1],"Claude-User":["partial","*",6,1],"Claude-SearchBot":["partial","*",6,1],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["partial","*",6,1],"Perplexity-User":["partial","*",6,1],"Google-Extended":["partial","named",1,1],"Applebot-Extended":["partial","*",6,1],"CCBot":["blocked","named",1,0],"Bytespider":["partial","*",6,1],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["partial","*",6,1],"cohere-ai":["partial","*",6,1],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",6,1]},"llms":{"status":"missing","http":404,"bytes":527},"home":{"status":"ok","http":200,"url":"https://www.test.de/","html_tokens":203528,"markdown_tokens":1254,"pruned_tokens":67}},
{"rank":4492,"domain":"universityofcalifornia.edu","origin":"https://universityofcalifornia.edu","robots":{"status":"refused","http":403,"url":"https://universityofcalifornia.edu/robots.txt","bytes":560,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":4493,"domain":"www.ne.jp","origin":"https://www.ne.jp","robots":{"status":"missing","http":404,"url":"https://www.ne.jp/robots.txt","bytes":236,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":236},"home":{"status":"ok","http":200,"url":"https://asahi-net.jp/","html_tokens":10568,"markdown_tokens":46,"pruned_tokens":46}},
{"rank":4494,"domain":"docusign.com","origin":"https://docusign.com","robots":{"status":"ok","http":200,"url":"https://www.docusign.com/robots.txt","bytes":1895,"groups":2},"bots":{"GPTBot":["partial","*",23,18],"ChatGPT-User":["partial","*",23,18],"OAI-SearchBot":["partial","*",23,18],"ClaudeBot":["partial","*",23,18],"Claude-User":["partial","*",23,18],"Claude-SearchBot":["partial","*",23,18],"anthropic-ai":["partial","*",23,18],"PerplexityBot":["partial","*",23,18],"Perplexity-User":["partial","*",23,18],"Google-Extended":["partial","*",23,18],"Applebot-Extended":["partial","*",23,18],"CCBot":["partial","*",23,18],"Bytespider":["partial","*",23,18],"Amazonbot":["partial","*",23,18],"meta-externalagent":["partial","*",23,18],"cohere-ai":["partial","*",23,18],"Diffbot":["partial","*",23,18],"ContextFetchBot":["partial","*",23,18]},"llms":{"status":"missing","http":404,"bytes":515760},"home":{"status":"ok","http":200,"url":"https://www.docusign.com/","html_tokens":241064,"markdown_tokens":779,"pruned_tokens":48}},
{"rank":4495,"domain":"netfirms.com","origin":"https://netfirms.com","robots":{"status":"refused","http":403,"url":"https://secure.netfirms.com/secure/login.html","bytes":5582,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":4496,"domain":"postimage.org","origin":"https://postimage.org","robots":{"status":"ok","http":200,"url":"https://postimage.org/robots.txt","bytes":1248,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":0},"home":{"status":"ok","http":200,"url":"https://postimages.org/","html_tokens":10448,"markdown_tokens":198,"pruned_tokens":25}},
{"rank":4497,"domain":"preply.com","origin":"https://preply.com","robots":{"status":"ok","http":200,"url":"https://preply.com/robots.txt","bytes":3641,"groups":9},"bots":{"GPTBot":["allowed","named",0,1],"ChatGPT-User":["partial","*",41,1],"OAI-SearchBot":["partial","*",41,1],"ClaudeBot":["partial","*",41,1],"Claude-User":["partial","*",41,1],"Claude-SearchBot":["partial","*",41,1],"anthropic-ai":["partial","*",41,1],"PerplexityBot":["allowed","named",0,1],"Perplexity-User":["partial","*",41,1],"Google-Extended":["allowed","named",0,1],"Applebot-Extended":["partial","*",41,1],"CCBot":["partial","*",41,1],"Bytespider":["partial","*",41,1],"Amazonbot":["partial","*",41,1],"meta-externalagent":["allowed","named",0,1],"cohere-ai":["partial","*",41,1],"Diffbot":["partial","*",41,1],"ContextFetchBot":["partial","*",41,1]},"llms":{"status":"present","http":200,"bytes":3256},"home":{"status":"ok","http":200,"url":"https://preply.com/","html_tokens":332351,"markdown_tokens":255,"pruned_tokens":53}},
{"rank":4498,"domain":"massgeneralbrigham.org","origin":"https://massgeneralbrigham.org","robots":{"status":"ok","http":200,"url":"https://www.massgeneralbrigham.org/robots.txt","bytes":1097,"groups":3},"bots":{"GPTBot":["partial","*",31,0],"ChatGPT-User":["partial","*",31,0],"OAI-SearchBot":["partial","*",31,0],"ClaudeBot":["partial","*",31,0],"Claude-User":["partial","*",31,0],"Claude-SearchBot":["partial","*",31,0],"anthropic-ai":["partial","*",31,0],"PerplexityBot":["partial","*",31,0],"Perplexity-User":["partial","*",31,0],"Google-Extended":["partial","*",31,0],"Applebot-Extended":["partial","*",31,0],"CCBot":["partial","*",31,0],"Bytespider":["partial","*",31,0],"Amazonbot":["partial","*",31,0],"meta-externalagent":["partial","*",31,0],"cohere-ai":["partial","*",31,0],"Diffbot":["partial","*",31,0],"ContextFetchBot":["partial","*",31,0]},"llms":{"status":"missing","http":404,"bytes":398},"home":{"status":"ok","http":200,"url":"https://www.massgeneralbrigham.org/","html_tokens":4288,"markdown_tokens":268,"pruned_tokens":170}},
{"rank":4499,"domain":"thebulletin.org","origin":"https://thebulletin.org","robots":{"status":"ok","http":200,"url":"https://thebulletin.org/robots.txt","bytes":996,"groups":21},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["partial","*",3,0],"OAI-SearchBot":["partial","*",3,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",3,0],"Claude-SearchBot":["partial","*",3,0],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["partial","*",3,0],"Perplexity-User":["partial","*",3,0],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",2,0],"cohere-ai":["partial","*",3,0],"Diffbot":["partial","*",3,0],"ContextFetchBot":["partial","*",3,0]},"llms":{"status":"missing","http":404,"bytes":361245},"home":{"status":"ok","http":200,"url":"https://thebulletin.org/","html_tokens":100011,"markdown_tokens":189,"pruned_tokens":170}},
{"rank":4500,"domain":"naver.me","origin":"https://naver.me","robots":{"status":"ok","http":200,"url":"https://naver.me/robots.txt","bytes":24,"groups":1},"bots":{"GPTBot":["allowed","*",0,1],"ChatGPT-User":["allowed","*",0,1],"OAI-SearchBot":["allowed","*",0,1],"ClaudeBot":["allowed","*",0,1],"Claude-User":["allowed","*",0,1],"Claude-SearchBot":["allowed","*",0,1],"anthropic-ai":["allowed","*",0,1],"PerplexityBot":["allowed","*",0,1],"Perplexity-User":["allowed","*",0,1],"Google-Extended":["allowed","*",0,1],"Applebot-Extended":["allowed","*",0,1],"CCBot":["allowed","*",0,1],"Bytespider":["allowed","*",0,1],"Amazonbot":["allowed","*",0,1],"meta-externalagent":["allowed","*",0,1],"cohere-ai":["allowed","*",0,1],"Diffbot":["allowed","*",0,1],"ContextFetchBot":["allowed","*",0,1]},"llms":{"status":"missing","http":404,"bytes":2766},"home":{"status":"http_error","http":404,"url":"https://naver.me/"}},
{"rank":4501,"domain":"melia.com","origin":"https://melia.com","robots":{"status":"refused","http":403,"url":"https://www.melia.com/robots.txt","bytes":381,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":4502,"domain":"gob.do","origin":"https://www.gob.do","robots":{"status":"ok","http":200,"url":"https://www.gob.do/robots.txt","bytes":22,"groups":1},"bots":{"GPTBot":["allowed","*",0,1],"ChatGPT-User":["allowed","*",0,1],"OAI-SearchBot":["allowed","*",0,1],"ClaudeBot":["allowed","*",0,1],"Claude-User":["allowed","*",0,1],"Claude-SearchBot":["allowed","*",0,1],"anthropic-ai":["allowed","*",0,1],"PerplexityBot":["allowed","*",0,1],"Perplexity-User":["allowed","*",0,1],"Google-Extended":["allowed","*",0,1],"Applebot-Extended":["allowed","*",0,1],"CCBot":["allowed","*",0,1],"Bytespider":["allowed","*",0,1],"Amazonbot":["allowed","*",0,1],"meta-externalagent":["allowed","*",0,1],"cohere-ai":["allowed","*",0,1],"Diffbot":["allowed","*",0,1],"ContextFetchBot":["allowed","*",0,1]},"llms":{"status":"missing","http":404,"bytes":30852},"home":{"status":"ok","http":200,"url":"https://www.gob.do/","html_tokens":32840,"markdown_tokens":66,"pruned_tokens":65}},
{"rank":4503,"domain":"ahram.org.eg","origin":"https://ahram.org.eg","robots":{"status":"ok","http":200,"url":"https://ahram.org.eg/robots.txt","bytes":0,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"html","http":200,"bytes":115274},"home":{"status":"http_error","http":403,"url":"https://ahram.org.eg/"}},
{"rank":4504,"domain":"aaas.org","origin":"https://aaas.org","robots":{"status":"refused","http":403,"url":"https://aaas.org/robots.txt","bytes":560,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":4505,"domain":"draftkings.com","origin":"https://draftkings.com","robots":{"status":"refused","http":403,"url":"https://www.draftkings.com/robots.txt","bytes":386,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":4506,"domain":"nationalgallery.org.uk","origin":"https://nationalgallery.org.uk","robots":{"status":"ok","http":200,"url":"https://www.nationalgallery.org.uk/robots.txt","bytes":492,"groups":2},"bots":{"GPTBot":["partial","*",2,0],"ChatGPT-User":["partial","*",2,0],"OAI-SearchBot":["partial","*",2,0],"ClaudeBot":["partial","*",2,0],"Claude-User":["partial","*",2,0],"Claude-SearchBot":["partial","*",2,0],"anthropic-ai":["partial","*",2,0],"PerplexityBot":["partial","*",2,0],"Perplexity-User":["partial","*",2,0],"Google-Extended":["partial","*",2,0],"Applebot-Extended":["partial","*",2,0],"CCBot":["partial","*",2,0],"Bytespider":["partial","*",2,0],"Amazonbot":["partial","*",2,0],"meta-externalagent":["partial","*",2,0],"cohere-ai":["partial","*",2,0],"Diffbot":["partial","*",2,0],"ContextFetchBot":["partial","*",2,0]},"llms":{"status":"missing","http":404,"bytes":0},"home":{"status":"ok","http":200,"url":"https://www.nationalgallery.org.uk/","html_tokens":28884,"markdown_tokens":642,"pruned_tokens":25}},
{"rank":4507,"domain":"hitachi.com","origin":"https://www.hitachi.com","robots":{"status":"ok","http":200,"url":"https://www.hitachi.com/robots.txt","bytes":1073,"groups":25},"bots":{"GPTBot":["allowed","*",0,1],"ChatGPT-User":["allowed","*",0,1],"OAI-SearchBot":["allowed","*",0,1],"ClaudeBot":["allowed","*",0,1],"Claude-User":["allowed","*",0,1],"Claude-SearchBot":["allowed","*",0,1],"anthropic-ai":["allowed","*",0,1],"PerplexityBot":["allowed","*",0,1],"Perplexity-User":["allowed","*",0,1],"Google-Extended":["allowed","*",0,1],"Applebot-Extended":["allowed","*",0,1],"CCBot":["allowed","*",0,1],"Bytespider":["allowed","*",0,1],"Amazonbot":["allowed","*",0,1],"meta-externalagent":["allowed","*",0,1],"cohere-ai":["allowed","*",0,1],"Diffbot":["allowed","*",0,1],"ContextFetchBot":["allowed","*",0,1]},"llms":{"status":"missing","http":404,"bytes":8521},"home":{"status":"ok","http":200,"url":"https://www.hitachi.com/en/","html_tokens":39562,"markdown_tokens":396,"pruned_tokens":355}},
{"rank":4508,"domain":"roche.com","origin":"https://roche.com","robots":{"status":"ok","http":200,"url":"https://www.roche.com/robots.txt","bytes":220,"groups":1},"bots":{"GPTBot":["partial","*",3,1],"ChatGPT-User":["partial","*",3,1],"OAI-SearchBot":["partial","*",3,1],"ClaudeBot":["partial","*",3,1],"Claude-User":["partial","*",3,1],"Claude-SearchBot":["partial","*",3,1],"anthropic-ai":["partial","*",3,1],"PerplexityBot":["partial","*",3,1],"Perplexity-User":["partial","*",3,1],"Google-Extended":["partial","*",3,1],"Applebot-Extended":["partial","*",3,1],"CCBot":["partial","*",3,1],"Bytespider":["partial","*",3,1],"Amazonbot":["partial","*",3,1],"meta-externalagent":["partial","*",3,1],"cohere-ai":["partial","*",3,1],"Diffbot":["partial","*",3,1],"ContextFetchBot":["partial","*",3,1]},"llms":{"status":"missing","http":404,"bytes":405558},"home":{"status":"ok","http":200,"url":"https://www.roche.com/","html_tokens":108254,"markdown_tokens":258,"pruned_tokens":128}},
{"rank":4509,"domain":"nalog.gov.ru","origin":"https://nalog.gov.ru","robots":{"status":"ok","http":200,"url":"https://www.nalog.gov.ru/robots.txt","bytes":766,"groups":2},"bots":{"GPTBot":["partial","*",6,0],"ChatGPT-User":["partial","*",6,0],"OAI-SearchBot":["partial","*",6,0],"ClaudeBot":["partial","*",6,0],"Claude-User":["partial","*",6,0],"Claude-SearchBot":["partial","*",6,0],"anthropic-ai":["partial","*",6,0],"PerplexityBot":["partial","*",6,0],"Perplexity-User":["partial","*",6,0],"Google-Extended":["partial","*",6,0],"Applebot-Extended":["partial","*",6,0],"CCBot":["partial","*",6,0],"Bytespider":["partial","*",6,0],"Amazonbot":["partial","*",6,0],"meta-externalagent":["partial","*",6,0],"cohere-ai":["partial","*",6,0],"Diffbot":["partial","*",6,0],"ContextFetchBot":["partial","*",6,0]},"llms":{"status":"missing","http":404,"bytes":467952},"home":{"status":"ok","http":200,"url":"https://www.nalog.gov.ru/rn77/","html_tokens":35291,"markdown_tokens":89,"pruned_tokens":89}},
{"rank":4510,"domain":"pandasecurity.com","origin":"https://pandasecurity.com","robots":{"status":"ok","http":200,"url":"https://www.pandasecurity.com/robots.txt","bytes":908,"groups":1},"bots":{"GPTBot":["partial","*",16,1],"ChatGPT-User":["partial","*",16,1],"OAI-SearchBot":["partial","*",16,1],"ClaudeBot":["partial","*",16,1],"Claude-User":["partial","*",16,1],"Claude-SearchBot":["partial","*",16,1],"anthropic-ai":["partial","*",16,1],"PerplexityBot":["partial","*",16,1],"Perplexity-User":["partial","*",16,1],"Google-Extended":["partial","*",16,1],"Applebot-Extended":["partial","*",16,1],"CCBot":["partial","*",16,1],"Bytespider":["partial","*",16,1],"Amazonbot":["partial","*",16,1],"meta-externalagent":["partial","*",16,1],"cohere-ai":["partial","*",16,1],"Diffbot":["partial","*",16,1],"ContextFetchBot":["partial","*",16,1]},"llms":{"status":"missing","http":404,"bytes":47529},"home":{"status":"ok","http":200,"url":"https://www.pandasecurity.com/","html_tokens":5526,"markdown_tokens":10,"pruned_tokens":10}},
{"rank":4511,"domain":"freesound.org","origin":"https://freesound.org","robots":{"status":"ok","http":200,"url":"https://freesound.org/robots.txt","bytes":509,"groups":5},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["partial","*",15,0],"OAI-SearchBot":["partial","*",15,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",15,0],"Claude-SearchBot":["partial","*",15,0],"anthropic-ai":["partial","*",15,0],"PerplexityBot":["partial","*",15,0],"Perplexity-User":["partial","*",15,0],"Google-Extended":["partial","*",15,0],"Applebot-Extended":["partial","*",15,0],"CCBot":["partial","*",15,0],"Bytespider":["partial","*",15,0],"Amazonbot":["partial","*",15,0],"meta-externalagent":["partial","*",15,0],"cohere-ai":["partial","*",15,0],"Diffbot":["partial","*",15,0],"ContextFetchBot":["partial","*",15,0]},"llms":{"status":"missing","http":404,"bytes":16136},"home":{"status":"ok","http":200,"url":"https://freesound.org/","html_tokens":105766,"markdown_tokens":491,"pruned_tokens":302}},
{"rank":4512,"domain":"essex.ac.uk","origin":"https://essex.ac.uk","robots":{"status":"refused","http":403,"url":"https://essex.ac.uk/robots.txt","bytes":3659,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":4513,"domain":"mez.ink","origin":"https://mez.ink","robots":{"status":"html","http":200,"url":"https://mez.ink/robots.txt","bytes":15486,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":4514,"domain":"pewtrusts.org","origin":"https://pewtrusts.org","robots":{"status":"ok","http":200,"url":"https://www.pew.org/robots.txt","bytes":610,"groups":2},"bots":{"GPTBot":["partial","*",10,0],"ChatGPT-User":["partial","*",10,0],"OAI-SearchBot":["partial","*",10,0],"ClaudeBot":["partial","*",10,0],"Claude-User":["partial","*",10,0],"Claude-SearchBot":["partial","*",10,0],"anthropic-ai":["partial","*",10,0],"PerplexityBot":["partial","*",10,0],"Perplexity-User":["partial","*",10,0],"Google-Extended":["partial","*",10,0],"Applebot-Extended":["partial","*",10,0],"CCBot":["partial","*",10,0],"Bytespider":["partial","*",10,0],"Amazonbot":["partial","*",10,0],"meta-externalagent":["partial","*",10,0],"cohere-ai":["partial","*",10,0],"Diffbot":["partial","*",10,0],"ContextFetchBot":["partial","*",10,0]},"llms":{"status":"missing","http":404,"bytes":175167},"home":{"status":"http_error","http":403,"url":"https://www.pew.org/"}},
{"rank":4515,"domain":"carousell.sg","origin":"https://carousell.sg","robots":{"status":"ok","http":200,"url":"https://www.carousell.sg/robots.txt","bytes":882,"groups":4},"bots":{"GPTBot":["allowed","named",0,1],"ChatGPT-User":["allowed","named",0,1],"OAI-SearchBot":["allowed","named",0,1],"ClaudeBot":["allowed","named",0,1],"Claude-User":["allowed","named",0,1],"Claude-SearchBot":["allowed","named",0,1],"anthropic-ai":["partial","*",9,2],"PerplexityBot":["allowed","named",0,1],"Perplexity-User":["allowed","named",0,1],"Google-Extended":["allowed","named",0,1],"Applebot-Extended":["allowed","named",0,1],"CCBot":["partial","*",9,2],"Bytespider":["partial","*",9,2],"Amazonbot":["partial","*",9,2],"meta-externalagent":["allowed","named",0,1],"cohere-ai":["partial","*",9,2],"Diffbot":["partial","*",9,2],"ContextFetchBot":["partial","*",9,2]},"llms":{"status":"http_error","http":403,"bytes":5544},"home":{"status":"http_error","http":403,"url":"https://carousell.sg/"}},
{"rank":4516,"domain":"townhall.com","origin":"https://townhall.com","robots":{"status":"ok","http":200,"url":"https://townhall.com/robots.txt","bytes":1104,"groups":2},"bots":{"GPTBot":["partial","*",7,1],"ChatGPT-User":["partial","*",7,1],"OAI-SearchBot":["partial","*",7,1],"ClaudeBot":["partial","*",7,1],"Claude-User":["partial","*",7,1],"Claude-SearchBot":["partial","*",7,1],"anthropic-ai":["partial","*",7,1],"PerplexityBot":["partial","*",7,1],"Perplexity-User":["partial","*",7,1],"Google-Extended":["partial","*",7,1],"Applebot-Extended":["partial","*",7,1],"CCBot":["partial","*",7,1],"Bytespider":["partial","*",7,1],"Amazonbot":["partial","*",7,1],"meta-externalagent":["partial","*",7,1],"cohere-ai":["partial","*",7,1],"Diffbot":["partial","*",7,1],"ContextFetchBot":["partial","*",7,1]},"llms":{"status":"present","http":200,"bytes":1535},"home":{"status":"ok","http":200,"url":"https://townhall.com/","html_tokens":49061,"markdown_tokens":26,"pruned_tokens":26}},
{"rank":4517,"domain":"ipa.go.jp","origin":"https://www.ipa.go.jp","robots":{"status":"missing","http":404,"url":"https://www.ipa.go.jp/robots.txt","bytes":40318,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":40318},"home":{"status":"ok","http":200,"url":"https://www.ipa.go.jp/","html_tokens":23083,"markdown_tokens":339,"pruned_tokens":145}},
{"rank":4518,"domain":"nabble.com","origin":"https://nabble.com","robots":{"status":"missing","http":404,"url":"https://nabble.com/robots.txt","bytes":1277,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":1275},"home":{"status":"ok","http":200,"url":"https://nabble.com/","html_tokens":2571,"markdown_tokens":54,"pruned_tokens":54}},
{"rank":4519,"domain":"mhanational.org","origin":"https://mhanational.org","robots":{"status":"ok","http":200,"url":"https://mhanational.org/robots.txt","bytes":2512,"groups":44},"bots":{"GPTBot":["allowed","named",0,0],"ChatGPT-User":["partial","*",3,1],"OAI-SearchBot":["partial","*",3,1],"ClaudeBot":["partial","*",3,1],"Claude-User":["partial","*",3,1],"Claude-SearchBot":["partial","*",3,1],"anthropic-ai":["partial","*",3,1],"PerplexityBot":["partial","*",3,1],"Perplexity-User":["partial","*",3,1],"Google-Extended":["partial","*",3,1],"Applebot-Extended":["partial","*",3,1],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["partial","*",3,1],"meta-externalagent":["partial","*",3,1],"cohere-ai":["partial","*",3,1],"Diffbot":["partial","*",3,1],"ContextFetchBot":["partial","*",3,1]},"llms":{"status":"missing","http":404,"bytes":119570},"home":{"status":"ok","http":200,"url":"https://mhanational.org/","html_tokens":35920,"markdown_tokens":199,"pruned_tokens":172}},
{"rank":4520,"domain":"newrelic.com","origin":"https://newrelic.com","robots":{"status":"ok","http":200,"url":"https://newrelic.com/robots.txt","bytes":2556,"groups":5},"bots":{"GPTBot":["partial","*",39,18],"ChatGPT-User":["partial","*",39,18],"OAI-SearchBot":["partial","*",39,18],"ClaudeBot":["partial","*",39,18],"Claude-User":["partial","*",39,18],"Claude-SearchBot":["partial","*",39,18],"anthropic-ai":["partial","*",39,18],"PerplexityBot":["partial","*",39,18],"Perplexity-User":["partial","*",39,18],"Google-Extended":["partial","*",39,18],"Applebot-Extended":["partial","*",39,18],"CCBot":["partial","*",39,18],"Bytespider":["partial","*",39,18],"Amazonbot":["partial","*",39,18],"meta-externalagent":["partial","*",39,18],"cohere-ai":["partial","*",39,18],"Diffbot":["partial","*",39,18],"ContextFetchBot":["partial","*",39,18]},"llms":{"status":"present","http":200,"bytes":43622},"home":{"status":"ok","http":200,"url":"https://newrelic.com/","html_tokens":167975,"markdown_tokens":202,"pruned_tokens":90}},
{"rank":4521,"domain":"microfocus.com","origin":"https://microfocus.com","robots":{"status":"ok","http":200,"url":"https://www.microfocus.com/robots.txt","bytes":1727,"groups":4},"bots":{"GPTBot":["partial","*",44,0],"ChatGPT-User":["partial","*",44,0],"OAI-SearchBot":["partial","*",44,0],"ClaudeBot":["partial","*",44,0],"Claude-User":["partial","*",44,0],"Claude-SearchBot":["partial","*",44,0],"anthropic-ai":["partial","*",44,0],"PerplexityBot":["partial","*",44,0],"Perplexity-User":["partial","*",44,0],"Google-Extended":["partial","*",44,0],"Applebot-Extended":["partial","*",44,0],"CCBot":["partial","*",44,0],"Bytespider":["partial","*",44,0],"Amazonbot":["partial","*",44,0],"meta-externalagent":["partial","*",44,0],"cohere-ai":["partial","*",44,0],"Diffbot":["partial","*",44,0],"ContextFetchBot":["partial","*",44,0]},"llms":{"status":"missing","http":404,"bytes":226836},"home":{"status":"ok","http":200,"url":"https://www.opentext.com/about/brands/microfocus","html_tokens":85606,"markdown_tokens":20,"pruned_tokens":20}},
{"rank":4522,"domain":"sinica.edu.tw","origin":"https://www.sinica.edu.tw","robots":{"status":"missing","http":404,"url":"https://www.sinica.edu.tw/errors/404","bytes":284,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":284},"home":{"status":"ok","http":200,"url":"https://www.sinica.edu.tw/","html_tokens":111908,"markdown_tokens":1751,"pruned_tokens":129}},
{"rank":4523,"domain":"freehostia.com","origin":"https://freehostia.com","robots":{"status":"ok","http":200,"url":"https://www.freehostia.com/robots.txt","bytes":406,"groups":1},"bots":{"GPTBot":["partial","*",14,0],"ChatGPT-User":["partial","*",14,0],"OAI-SearchBot":["partial","*",14,0],"ClaudeBot":["partial","*",14,0],"Claude-User":["partial","*",14,0],"Claude-SearchBot":["partial","*",14,0],"anthropic-ai":["partial","*",14,0],"PerplexityBot":["partial","*",14,0],"Perplexity-User":["partial","*",14,0],"Google-Extended":["partial","*",14,0],"Applebot-Extended":["partial","*",14,0],"CCBot":["partial","*",14,0],"Bytespider":["partial","*",14,0],"Amazonbot":["partial","*",14,0],"meta-externalagent":["partial","*",14,0],"cohere-ai":["partial","*",14,0],"Diffbot":["partial","*",14,0],"ContextFetchBot":["partial","*",14,0]},"llms":{"status":"missing","http":404,"bytes":633382},"home":{"status":"ok","http":200,"url":"https://www.freehostia.com/","html_tokens":190327,"markdown_tokens":868,"pruned_tokens":131}},
{"rank":4524,"domain":"bmbf.de","origin":"https://bmbf.de","robots":{"status":"ok","http":200,"url":"https://www.bmftr.bund.de/robots.txt","bytes":300,"groups":1},"bots":{"GPTBot":["partial","*",4,3],"ChatGPT-User":["partial","*",4,3],"OAI-SearchBot":["partial","*",4,3],"ClaudeBot":["partial","*",4,3],"Claude-User":["partial","*",4,3],"Claude-SearchBot":["partial","*",4,3],"anthropic-ai":["partial","*",4,3],"PerplexityBot":["partial","*",4,3],"Perplexity-User":["partial","*",4,3],"Google-Extended":["partial","*",4,3],"Applebot-Extended":["partial","*",4,3],"CCBot":["partial","*",4,3],"Bytespider":["partial","*",4,3],"Amazonbot":["partial","*",4,3],"meta-externalagent":["partial","*",4,3],"cohere-ai":["partial","*",4,3],"Diffbot":["partial","*",4,3],"ContextFetchBot":["partial","*",4,3]},"llms":{"status":"missing","http":404,"bytes":404688},"home":{"status":"timeout","error":"timed out after 15 s"}},
{"rank":4525,"domain":"livroreclamacoes.pt","origin":"https://livroreclamacoes.pt","robots":{"status":"html","http":200,"url":"https://www.livroreclamacoes.pt/Inicio/","bytes":2322,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":4526,"domain":"ccn.com","origin":"https://ccn.com","robots":{"status":"ok","http":200,"url":"https://ccn.com/robots.txt","bytes":986,"groups":1},"bots":{"GPTBot":["partial","*",31,0],"ChatGPT-User":["partial","*",31,0],"OAI-SearchBot":["partial","*",31,0],"ClaudeBot":["partial","*",31,0],"Claude-User":["partial","*",31,0],"Claude-SearchBot":["partial","*",31,0],"anthropic-ai":["partial","*",31,0],"PerplexityBot":["partial","*",31,0],"Perplexity-User":["partial","*",31,0],"Google-Extended":["partial","*",31,0],"Applebot-Extended":["partial","*",31,0],"CCBot":["partial","*",31,0],"Bytespider":["partial","*",31,0],"Amazonbot":["partial","*",31,0],"meta-externalagent":["partial","*",31,0],"cohere-ai":["partial","*",31,0],"Diffbot":["partial","*",31,0],"ContextFetchBot":["partial","*",31,0]},"llms":{"status":"present","http":200,"bytes":33205},"home":{"status":"http_error","http":403,"url":"https://ccn.com/"}},
{"rank":4527,"domain":"lithub.com","origin":"https://lithub.com","robots":{"status":"ok","http":200,"url":"https://lithub.com/robots.txt","bytes":379,"groups":1},"bots":{"GPTBot":["partial","*",8,1],"ChatGPT-User":["partial","*",8,1],"OAI-SearchBot":["partial","*",8,1],"ClaudeBot":["partial","*",8,1],"Claude-User":["partial","*",8,1],"Claude-SearchBot":["partial","*",8,1],"anthropic-ai":["partial","*",8,1],"PerplexityBot":["partial","*",8,1],"Perplexity-User":["partial","*",8,1],"Google-Extended":["partial","*",8,1],"Applebot-Extended":["partial","*",8,1],"CCBot":["partial","*",8,1],"Bytespider":["partial","*",8,1],"Amazonbot":["partial","*",8,1],"meta-externalagent":["partial","*",8,1],"cohere-ai":["partial","*",8,1],"Diffbot":["partial","*",8,1],"ContextFetchBot":["partial","*",8,1]},"llms":{"status":"timeout","error":"timed out after 10 s"},"home":{"status":"ok","http":200,"url":"https://lithub.com/","html_tokens":24768,"markdown_tokens":1237,"pruned_tokens":127}},
{"rank":4528,"domain":"hubs.ly","origin":"https://hubs.ly","robots":{"status":"ok","http":200,"url":"https://hubs.ly/robots.txt","bytes":167,"groups":5},"bots":{"GPTBot":["blocked","*",1,0],"ChatGPT-User":["blocked","*",1,0],"OAI-SearchBot":["blocked","*",1,0],"ClaudeBot":["blocked","*",1,0],"Claude-User":["blocked","*",1,0],"Claude-SearchBot":["blocked","*",1,0],"anthropic-ai":["blocked","*",1,0],"PerplexityBot":["blocked","*",1,0],"Perplexity-User":["blocked","*",1,0],"Google-Extended":["blocked","*",1,0],"Applebot-Extended":["blocked","*",1,0],"CCBot":["blocked","*",1,0],"Bytespider":["blocked","*",1,0],"Amazonbot":["blocked","*",1,0],"meta-externalagent":["blocked","*",1,0],"cohere-ai":["blocked","*",1,0],"Diffbot":["blocked","*",1,0],"ContextFetchBot":["blocked","*",1,0]},"llms":{"status":"disallowed"},"home":{"status":"disallowed"}},
{"rank":4529,"domain":"ubuntuforums.org","origin":"https://ubuntuforums.org","robots":{"status":"unreachable","http":null,"url":"https://ubuntuforums.org/robots.txt","bytes":0,"groups":0,"error":"cannot resolve ubuntuforums.org"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":4530,"domain":"ttu.edu","origin":"https://ttu.edu","robots":{"status":"missing","http":404,"url":"https://www.ttu.edu/robots.txt","bytes":27622,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":27620},"home":{"status":"ok","http":200,"url":"https://www.ttu.edu/","html_tokens":37423,"markdown_tokens":355,"pruned_tokens":167}},
{"rank":4531,"domain":"etonline.com","origin":"https://etonline.com","robots":{"status":"ok","http":200,"url":"https://www.etonline.com/robots.txt","bytes":1701,"groups":1},"bots":{"GPTBot":["partial","*",22,18],"ChatGPT-User":["partial","*",22,18],"OAI-SearchBot":["partial","*",22,18],"ClaudeBot":["partial","*",22,18],"Claude-User":["partial","*",22,18],"Claude-SearchBot":["partial","*",22,18],"anthropic-ai":["partial","*",22,18],"PerplexityBot":["partial","*",22,18],"Perplexity-User":["partial","*",22,18],"Google-Extended":["partial","*",22,18],"Applebot-Extended":["partial","*",22,18],"CCBot":["partial","*",22,18],"Bytespider":["partial","*",22,18],"Amazonbot":["partial","*",22,18],"meta-externalagent":["partial","*",22,18],"cohere-ai":["partial","*",22,18],"Diffbot":["partial","*",22,18],"ContextFetchBot":["partial","*",22,18]},"llms":{"status":"missing","http":404,"bytes":24348},"home":{"status":"ok","http":200,"url":"https://www.etonline.com","html_tokens":160367,"markdown_tokens":31,"pruned_tokens":31}},
{"rank":4532,"domain":"tvguide.com","origin":"https://tvguide.com","robots":{"status":"ok","http":200,"url":"https://www.tvguide.com/robots.txt","bytes":892,"groups":15},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["partial","*",4,0],"OAI-SearchBot":["blocked","named",1,0],"ClaudeBot":["partial","*",4,0],"Claude-User":["partial","*",4,0],"Claude-SearchBot":["partial","*",4,0],"anthropic-ai":["partial","*",4,0],"PerplexityBot":["partial","*",4,0],"Perplexity-User":["partial","*",4,0],"Google-Extended":["partial","*",4,0],"Applebot-Extended":["partial","*",4,0],"CCBot":["blocked","named",1,0],"Bytespider":["partial","*",4,0],"Amazonbot":["partial","*",4,0],"meta-externalagent":["partial","*",4,0],"cohere-ai":["partial","*",4,0],"Diffbot":["partial","*",4,0],"ContextFetchBot":["partial","*",4,0]},"llms":{"status":"missing","http":404,"bytes":113962},"home":{"status":"ok","http":200,"url":"https://www.tvguide.com/","html_tokens":217292,"markdown_tokens":1165,"pruned_tokens":150}},
{"rank":4533,"domain":"seattlepi.com","origin":"https://seattlepi.com","robots":{"status":"ok","http":200,"url":"https://www.seattlepi.com/robots.txt","bytes":1052,"groups":3},"bots":{"GPTBot":["partial","*",11,0],"ChatGPT-User":["partial","*",11,0],"OAI-SearchBot":["partial","*",11,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["blocked","named",1,0],"Claude-SearchBot":["partial","*",11,0],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["blocked","named",1,0],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["partial","*",11,0],"meta-externalagent":["partial","*",11,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",11,0]},"llms":{"status":"missing","http":404,"bytes":49658},"home":{"status":"ok","http":200,"url":"https://www.seattlepi.com/","html_tokens":196937,"markdown_tokens":22,"pruned_tokens":22}},
{"rank":4534,"domain":"eiu.com","origin":"https://eiu.com","robots":{"status":"ok","http":200,"url":"https://www.eiu.com/n/robots.txt","bytes":229,"groups":3},"bots":{"GPTBot":["partial","*",1,0],"ChatGPT-User":["partial","*",1,0],"OAI-SearchBot":["partial","*",1,0],"ClaudeBot":["partial","*",1,0],"Claude-User":["partial","*",1,0],"Claude-SearchBot":["partial","*",1,0],"anthropic-ai":["partial","*",1,0],"PerplexityBot":["partial","*",1,0],"Perplexity-User":["partial","*",1,0],"Google-Extended":["partial","*",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["partial","*",1,0],"Bytespider":["partial","*",1,0],"Amazonbot":["partial","*",1,0],"meta-externalagent":["partial","*",1,0],"cohere-ai":["partial","*",1,0],"Diffbot":["partial","*",1,0],"ContextFetchBot":["partial","*",1,0]},"llms":{"status":"html","http":200,"bytes":101532},"home":{"status":"ok","http":200,"url":"https://www.eiu.com/n/","html_tokens":25377,"markdown_tokens":94,"pruned_tokens":84}},
{"rank":4535,"domain":"uow.edu.au","origin":"https://uow.edu.au","robots":{"status":"ok","http":200,"url":"https://www.uow.edu.au/robots.txt","bytes":315,"groups":2},"bots":{"GPTBot":["partial","*",3,0],"ChatGPT-User":["partial","*",3,0],"OAI-SearchBot":["partial","*",3,0],"ClaudeBot":["partial","*",3,0],"Claude-User":["partial","*",3,0],"Claude-SearchBot":["partial","*",3,0],"anthropic-ai":["partial","*",3,0],"PerplexityBot":["partial","*",3,0],"Perplexity-User":["partial","*",3,0],"Google-Extended":["partial","*",3,0],"Applebot-Extended":["partial","*",3,0],"CCBot":["partial","*",3,0],"Bytespider":["partial","*",3,0],"Amazonbot":["partial","*",3,0],"meta-externalagent":["partial","*",3,0],"cohere-ai":["partial","*",3,0],"Diffbot":["partial","*",3,0],"ContextFetchBot":["partial","*",3,0]},"llms":{"status":"missing","http":404,"bytes":32381},"home":{"status":"ok","http":200,"url":"https://www.uow.edu.au/","html_tokens":15473,"markdown_tokens":127,"pruned_tokens":81}},
{"rank":4536,"domain":"dw.de","origin":"http://dw.de","robots":{"status":"ok","http":200,"url":"https://www.dw.com/robots.txt","bytes":2757,"groups":51},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["blocked","named",1,0],"OAI-SearchBot":["blocked","named",1,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",18,0],"Claude-SearchBot":["partial","*",18,0],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["partial","*",18,0],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",18,0]},"llms":{"status":"missing","http":404,"bytes":5283},"home":{"status":"ok","http":200,"url":"https://www.dw.com/","html_tokens":3047,"markdown_tokens":12,"pruned_tokens":12}},
{"rank":4537,"domain":"fazenda.gov.br","origin":"https://fazenda.gov.br","robots":{"status":"unreachable","http":null,"url":"https://fazenda.gov.br/robots.txt","bytes":0,"groups":0,"error":"cannot resolve fazenda.gov.br"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":4538,"domain":"pew.org","origin":"https://pew.org","robots":{"status":"ok","http":200,"url":"https://www.pew.org/robots.txt","bytes":610,"groups":2},"bots":{"GPTBot":["partial","*",10,0],"ChatGPT-User":["partial","*",10,0],"OAI-SearchBot":["partial","*",10,0],"ClaudeBot":["partial","*",10,0],"Claude-User":["partial","*",10,0],"Claude-SearchBot":["partial","*",10,0],"anthropic-ai":["partial","*",10,0],"PerplexityBot":["partial","*",10,0],"Perplexity-User":["partial","*",10,0],"Google-Extended":["partial","*",10,0],"Applebot-Extended":["partial","*",10,0],"CCBot":["partial","*",10,0],"Bytespider":["partial","*",10,0],"Amazonbot":["partial","*",10,0],"meta-externalagent":["partial","*",10,0],"cohere-ai":["partial","*",10,0],"Diffbot":["partial","*",10,0],"ContextFetchBot":["partial","*",10,0]},"llms":{"status":"missing","http":404,"bytes":175166},"home":{"status":"http_error","http":403,"url":"https://www.pew.org/"}},
{"rank":4539,"domain":"boc.cn","origin":"https://www.boc.cn","robots":{"status":"html","http":200,"url":"https://www.boc.cn/custserv/cs1/msg/200909/t20090928_846317.html","bytes":26769,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":4540,"domain":"ebay.ca","origin":"https://ebay.ca","robots":{"status":"ok","http":200,"url":"https://www.ebay.ca/robots.txt","bytes":17654,"groups":7},"bots":{"GPTBot":["partial","named",1,3],"ChatGPT-User":["partial","*",221,6],"OAI-SearchBot":["partial","*",221,6],"ClaudeBot":["partial","named",1,3],"Claude-User":["partial","*",221,6],"Claude-SearchBot":["partial","*",221,6],"anthropic-ai":["partial","named",1,3],"PerplexityBot":["partial","named",1,3],"Perplexity-User":["partial","*",221,6],"Google-Extended":["partial","*",221,6],"Applebot-Extended":["partial","named",1,3],"CCBot":["partial","named",1,3],"Bytespider":["partial","named",1,3],"Amazonbot":["partial","named",1,3],"meta-externalagent":["partial","named",1,3],"cohere-ai":["partial","*",221,6],"Diffbot":["partial","*",221,6],"ContextFetchBot":["partial","*",221,6]},"llms":{"status":"missing","http":404,"bytes":77222},"home":{"status":"http_error","http":403,"url":"https://www.ebay.ca/"}},
{"rank":4541,"domain":"distrokid.com","origin":"https://distrokid.com","robots":{"status":"ok","http":200,"url":"https://distrokid.com/robots.txt","bytes":252,"groups":1},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"http_error","http":403,"bytes":5500},"home":{"status":"http_error","http":403,"url":"https://distrokid.com/"}},
{"rank":4542,"domain":"treas.gov","origin":"https://treas.gov","robots":{"status":"timeout","http":null,"url":"https://treas.gov/robots.txt","bytes":0,"groups":0,"error":"timed out after 10 s"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":4543,"domain":"universetoday.com","origin":"https://universetoday.com","robots":{"status":"ok","http":200,"url":"https://www.universetoday.com/robots.txt","bytes":209,"groups":1},"bots":{"GPTBot":["partial","*",1,1],"ChatGPT-User":["partial","*",1,1],"OAI-SearchBot":["partial","*",1,1],"ClaudeBot":["partial","*",1,1],"Claude-User":["partial","*",1,1],"Claude-SearchBot":["partial","*",1,1],"anthropic-ai":["partial","*",1,1],"PerplexityBot":["partial","*",1,1],"Perplexity-User":["partial","*",1,1],"Google-Extended":["partial","*",1,1],"Applebot-Extended":["partial","*",1,1],"CCBot":["partial","*",1,1],"Bytespider":["partial","*",1,1],"Amazonbot":["partial","*",1,1],"meta-externalagent":["partial","*",1,1],"cohere-ai":["partial","*",1,1],"Diffbot":["partial","*",1,1],"ContextFetchBot":["partial","*",1,1]},"llms":{"status":"missing","http":404,"bytes":5366},"home":{"status":"ok","http":200,"url":"https://www.universetoday.com/","html_tokens":19349,"markdown_tokens":4909,"pruned_tokens":143}},
{"rank":4544,"domain":"lboro.ac.uk","origin":"https://lboro.ac.uk","robots":{"status":"ok","http":200,"url":"https://www.lboro.ac.uk/robots.txt","bytes":3093,"groups":1},"bots":{"GPTBot":["partial","*",55,0],"ChatGPT-User":["partial","*",55,0],"OAI-SearchBot":["partial","*",55,0],"ClaudeBot":["partial","*",55,0],"Claude-User":["partial","*",55,0],"Claude-SearchBot":["partial","*",55,0],"anthropic-ai":["partial","*",55,0],"PerplexityBot":["partial","*",55,0],"Perplexity-User":["partial","*",55,0],"Google-Extended":["partial","*",55,0],"Applebot-Extended":["partial","*",55,0],"CCBot":["partial","*",55,0],"Bytespider":["partial","*",55,0],"Amazonbot":["partial","*",55,0],"meta-externalagent":["partial","*",55,0],"cohere-ai":["partial","*",55,0],"Diffbot":["partial","*",55,0],"ContextFetchBot":["partial","*",55,0]},"llms":{"status":"missing","http":404,"bytes":36325},"home":{"status":"ok","http":200,"url":"https://www.lboro.ac.uk/","html_tokens":16693,"markdown_tokens":108,"pruned_tokens":40}},
{"rank":4545,"domain":"realmadrid.com","origin":"https://realmadrid.com","robots":{"status":"html","http":200,"url":"https://www.realmadrid.com/robots.txt","bytes":16311,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":4546,"domain":"arthritis.org","origin":"https://arthritis.org","robots":{"status":"ok","http":200,"url":"https://www.arthritis.org/robots.txt","bytes":245,"groups":1},"bots":{"GPTBot":["allowed","*",0,1],"ChatGPT-User":["allowed","*",0,1],"OAI-SearchBot":["allowed","*",0,1],"ClaudeBot":["allowed","*",0,1],"Claude-User":["allowed","*",0,1],"Claude-SearchBot":["allowed","*",0,1],"anthropic-ai":["allowed","*",0,1],"PerplexityBot":["allowed","*",0,1],"Perplexity-User":["allowed","*",0,1],"Google-Extended":["allowed","*",0,1],"Applebot-Extended":["allowed","*",0,1],"CCBot":["allowed","*",0,1],"Bytespider":["allowed","*",0,1],"Amazonbot":["allowed","*",0,1],"meta-externalagent":["allowed","*",0,1],"cohere-ai":["allowed","*",0,1],"Diffbot":["allowed","*",0,1],"ContextFetchBot":["allowed","*",0,1]},"llms":{"status":"missing","http":404,"bytes":70440},"home":{"status":"ok","http":200,"url":"https://www.arthritis.org/","html_tokens":24095,"markdown_tokens":829,"pruned_tokens":106}},
{"rank":4547,"domain":"belfasttelegraph.co.uk","origin":"https://belfasttelegraph.co.uk","robots":{"status":"ok","http":200,"url":"https://www.belfasttelegraph.co.uk/robots.txt","bytes":2749,"groups":40},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["blocked","named",1,0],"OAI-SearchBot":["blocked","named",1,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["blocked","named",1,0],"Claude-SearchBot":["blocked","named",1,0],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["blocked","named",1,0],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["partial","*",1,2],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",1,2]},"llms":{"status":"http_error","http":403,"bytes":4555},"home":{"status":"http_error","http":403,"url":"https://www.belfasttelegraph.co.uk/"}},
{"rank":4548,"domain":"montana.edu","origin":"https://montana.edu","robots":{"status":"ok","http":200,"url":"https://www.montana.edu/robots.txt","bytes":51,"groups":1},"bots":{"GPTBot":["partial","*",2,0],"ChatGPT-User":["partial","*",2,0],"OAI-SearchBot":["partial","*",2,0],"ClaudeBot":["partial","*",2,0],"Claude-User":["partial","*",2,0],"Claude-SearchBot":["partial","*",2,0],"anthropic-ai":["partial","*",2,0],"PerplexityBot":["partial","*",2,0],"Perplexity-User":["partial","*",2,0],"Google-Extended":["partial","*",2,0],"Applebot-Extended":["partial","*",2,0],"CCBot":["partial","*",2,0],"Bytespider":["partial","*",2,0],"Amazonbot":["partial","*",2,0],"meta-externalagent":["partial","*",2,0],"cohere-ai":["partial","*",2,0],"Diffbot":["partial","*",2,0],"ContextFetchBot":["partial","*",2,0]},"llms":{"status":"missing","http":404,"bytes":16845},"home":{"status":"ok","http":200,"url":"https://www.montana.edu/","html_tokens":19696,"markdown_tokens":185,"pruned_tokens":144}},
{"rank":4549,"domain":"sberbank.ru","origin":"http://sberbank.ru","robots":{"status":"html","http":200,"url":"http://sberbank.ru/robots.txt","bytes":1579,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":4550,"domain":"loopia.com","origin":"https://loopia.com","robots":{"status":"ok","http":200,"url":"https://www.loopia.com/robots.txt","bytes":219,"groups":1},"bots":{"GPTBot":["partial","*",7,0],"ChatGPT-User":["partial","*",7,0],"OAI-SearchBot":["partial","*",7,0],"ClaudeBot":["partial","*",7,0],"Claude-User":["partial","*",7,0],"Claude-SearchBot":["partial","*",7,0],"anthropic-ai":["partial","*",7,0],"PerplexityBot":["partial","*",7,0],"Perplexity-User":["partial","*",7,0],"Google-Extended":["partial","*",7,0],"Applebot-Extended":["partial","*",7,0],"CCBot":["partial","*",7,0],"Bytespider":["partial","*",7,0],"Amazonbot":["partial","*",7,0],"meta-externalagent":["partial","*",7,0],"cohere-ai":["partial","*",7,0],"Diffbot":["partial","*",7,0],"ContextFetchBot":["partial","*",7,0]},"llms":{"status":"present","http":200,"bytes":11107},"home":{"status":"ok","http":200,"url":"https://www.loopia.com/","html_tokens":12841,"markdown_tokens":1080,"pruned_tokens":116}},
{"rank":4551,"domain":"sncf.com","origin":"https://sncf.com","robots":{"status":"refused","http":403,"url":"https://www.groupe-sncf.com/","bytes":775,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":4552,"domain":"toshiba.com","origin":"https://toshiba.com","robots":{"status":"ok","http":200,"url":"https://toshiba.com/robots.txt","bytes":926,"groups":2},"bots":{"GPTBot":["partial","*",21,0],"ChatGPT-User":["partial","*",21,0],"OAI-SearchBot":["partial","*",21,0],"ClaudeBot":["partial","*",21,0],"Claude-User":["partial","*",21,0],"Claude-SearchBot":["partial","*",21,0],"anthropic-ai":["partial","*",21,0],"PerplexityBot":["partial","*",21,0],"Perplexity-User":["partial","*",21,0],"Google-Extended":["partial","*",21,0],"Applebot-Extended":["partial","*",21,0],"CCBot":["partial","*",21,0],"Bytespider":["partial","*",21,0],"Amazonbot":["partial","*",21,0],"meta-externalagent":["partial","*",21,0],"cohere-ai":["partial","*",21,0],"Diffbot":["partial","*",21,0],"ContextFetchBot":["partial","*",21,0]},"llms":{"status":"missing","http":404,"bytes":210294},"home":{"status":"ok","http":200,"url":"https://toshiba.com/tai/","html_tokens":94118,"markdown_tokens":755,"pruned_tokens":540}},
{"rank":4553,"domain":"deutschlandfunkkultur.de","origin":"https://deutschlandfunkkultur.de","robots":{"status":"ok","http":200,"url":"https://www.deutschlandfunkkultur.de/robots.txt","bytes":335,"groups":1},"bots":{"GPTBot":["partial","*",2,0],"ChatGPT-User":["partial","*",2,0],"OAI-SearchBot":["partial","*",2,0],"ClaudeBot":["partial","*",2,0],"Claude-User":["partial","*",2,0],"Claude-SearchBot":["partial","*",2,0],"anthropic-ai":["partial","*",2,0],"PerplexityBot":["partial","*",2,0],"Perplexity-User":["partial","*",2,0],"Google-Extended":["partial","*",2,0],"Applebot-Extended":["partial","*",2,0],"CCBot":["partial","*",2,0],"Bytespider":["partial","*",2,0],"Amazonbot":["partial","*",2,0],"meta-externalagent":["partial","*",2,0],"cohere-ai":["partial","*",2,0],"Diffbot":["partial","*",2,0],"ContextFetchBot":["partial","*",2,0]},"llms":{"status":"missing","http":404,"bytes":252031},"home":{"status":"ok","http":200,"url":"https://www.deutschlandfunkkultur.de/","html_tokens":895291,"markdown_tokens":79,"pruned_tokens":79}},
{"rank":4554,"domain":"phonearena.com","origin":"https://phonearena.com","robots":{"status":"ok","http":200,"url":"https://www.phonearena.com/robots.txt","bytes":2436,"groups":4},"bots":{"GPTBot":["partial","*",58,0],"ChatGPT-User":["partial","*",58,0],"OAI-SearchBot":["partial","*",58,0],"ClaudeBot":["partial","*",58,0],"Claude-User":["partial","*",58,0],"Claude-SearchBot":["partial","*",58,0],"anthropic-ai":["partial","*",58,0],"PerplexityBot":["partial","*",58,0],"Perplexity-User":["partial","*",58,0],"Google-Extended":["partial","*",58,0],"Applebot-Extended":["partial","*",58,0],"CCBot":["partial","*",58,0],"Bytespider":["partial","*",58,0],"Amazonbot":["partial","*",58,0],"meta-externalagent":["partial","*",58,0],"cohere-ai":["partial","*",58,0],"Diffbot":["partial","*",58,0],"ContextFetchBot":["partial","*",58,0]},"llms":{"status":"http_error","http":403,"bytes":5503},"home":{"status":"http_error","http":403,"url":"https://phonearena.com/"}},
{"rank":4555,"domain":"cyberleninka.ru","origin":"https://cyberleninka.ru","robots":{"status":"ok","http":200,"url":"https://cyberleninka.ru/robots.txt","bytes":342,"groups":2},"bots":{"GPTBot":["partial","*",10,0],"ChatGPT-User":["partial","*",10,0],"OAI-SearchBot":["partial","*",10,0],"ClaudeBot":["partial","*",10,0],"Claude-User":["partial","*",10,0],"Claude-SearchBot":["partial","*",10,0],"anthropic-ai":["partial","*",10,0],"PerplexityBot":["partial","*",10,0],"Perplexity-User":["partial","*",10,0],"Google-Extended":["partial","*",10,0],"Applebot-Extended":["partial","*",10,0],"CCBot":["partial","*",10,0],"Bytespider":["partial","*",10,0],"Amazonbot":["partial","*",10,0],"meta-externalagent":["partial","*",10,0],"cohere-ai":["partial","*",10,0],"Diffbot":["partial","*",10,0],"ContextFetchBot":["partial","*",10,0]},"llms":{"status":"missing","http":404,"bytes":19},"home":{"status":"ok","http":200,"url":"https://cyberleninka.ru/","html_tokens":3173,"markdown_tokens":731,"pruned_tokens":168}},
{"rank":4556,"domain":"betanews.com","origin":"https://betanews.com","robots":{"status":"ok","http":200,"url":"https://betanews.com/robots.txt","bytes":158,"groups":2},"bots":{"GPTBot":["partial","*",1,1],"ChatGPT-User":["partial","*",1,1],"OAI-SearchBot":["allowed","named",0,1],"ClaudeBot":["partial","*",1,1],"Claude-User":["partial","*",1,1],"Claude-SearchBot":["partial","*",1,1],"anthropic-ai":["partial","*",1,1],"PerplexityBot":["partial","*",1,1],"Perplexity-User":["partial","*",1,1],"Google-Extended":["partial","*",1,1],"Applebot-Extended":["partial","*",1,1],"CCBot":["partial","*",1,1],"Bytespider":["partial","*",1,1],"Amazonbot":["partial","*",1,1],"meta-externalagent":["partial","*",1,1],"cohere-ai":["partial","*",1,1],"Diffbot":["partial","*",1,1],"ContextFetchBot":["partial","*",1,1]},"llms":{"status":"missing","http":404,"bytes":61518},"home":{"status":"http_error","http":403,"url":"https://betanews.com/"}},
{"rank":4557,"domain":"academic.ru","origin":"https://academic.ru","robots":{"status":"ok","http":200,"url":"https://academic.ru/robots.txt","bytes":554,"groups":14},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"html","http":200,"bytes":1790},"home":{"status":"ok","http":200,"url":"https://academic.ru/","html_tokens":19157,"markdown_tokens":159,"pruned_tokens":153}},
{"rank":4558,"domain":"cincinnati.com","origin":"https://cincinnati.com","robots":{"status":"ok","http":200,"url":"https://www.cincinnati.com/robots.txt","bytes":12784,"groups":288},"bots":{"GPTBot":["partial","named",1,4],"ChatGPT-User":["partial","named",1,4],"OAI-SearchBot":["partial","named",1,4],"ClaudeBot":["blocked","named",1,0],"Claude-User":["blocked","named",1,0],"Claude-SearchBot":["blocked","named",1,0],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["partial","named",1,4],"Perplexity-User":["blocked","named",1,0],"Google-Extended":["partial","named",1,4],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",2,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",32,0]},"llms":{"status":"http_error","http":402,"bytes":254},"home":{"status":"http_error","http":402,"url":"https://www.cincinnati.com/"}},
{"rank":4559,"domain":"animenewsnetwork.com","origin":"https://animenewsnetwork.com","robots":{"status":"ok","http":200,"url":"https://www.animenewsnetwork.com/robots.txt","bytes":1995,"groups":4},"bots":{"GPTBot":["partial","*",39,0],"ChatGPT-User":["partial","*",39,0],"OAI-SearchBot":["partial","*",39,0],"ClaudeBot":["partial","*",39,0],"Claude-User":["partial","*",39,0],"Claude-SearchBot":["partial","*",39,0],"anthropic-ai":["partial","*",39,0],"PerplexityBot":["partial","*",39,0],"Perplexity-User":["partial","*",39,0],"Google-Extended":["partial","*",39,0],"Applebot-Extended":["partial","*",39,0],"CCBot":["partial","*",39,0],"Bytespider":["partial","*",39,0],"Amazonbot":["partial","*",39,0],"meta-externalagent":["partial","*",39,0],"cohere-ai":["partial","*",39,0],"Diffbot":["partial","*",39,0],"ContextFetchBot":["partial","*",39,0]},"llms":{"status":"missing","http":404,"bytes":86329},"home":{"status":"ok","http":200,"url":"https://www.animenewsnetwork.com/","html_tokens":59474,"markdown_tokens":6556,"pruned_tokens":105}},
{"rank":4560,"domain":"thetrainline.com","origin":"https://thetrainline.com","robots":{"status":"ok","http":200,"url":"https://www.thetrainline.com/robots.txt","bytes":5731,"groups":3},"bots":{"GPTBot":["partial","*",50,0],"ChatGPT-User":["partial","*",50,0],"OAI-SearchBot":["partial","*",50,0],"ClaudeBot":["partial","*",50,0],"Claude-User":["partial","*",50,0],"Claude-SearchBot":["partial","*",50,0],"anthropic-ai":["partial","*",50,0],"PerplexityBot":["partial","*",50,0],"Perplexity-User":["partial","*",50,0],"Google-Extended":["partial","*",50,0],"Applebot-Extended":["partial","*",50,0],"CCBot":["partial","*",50,0],"Bytespider":["partial","*",50,0],"Amazonbot":["partial","*",50,0],"meta-externalagent":["partial","*",50,0],"cohere-ai":["partial","*",50,0],"Diffbot":["partial","*",50,0],"ContextFetchBot":["partial","*",50,0]},"llms":{"status":"present","http":200,"bytes":488746},"home":{"status":"ok","http":200,"url":"https://www.thetrainline.com/","html_tokens":114671,"markdown_tokens":480,"pruned_tokens":113}},
{"rank":4561,"domain":"ioncube.com","origin":"https://ioncube.com","robots":{"status":"ok","http":200,"url":"https://www.ioncube.com/robots.txt","bytes":137,"groups":1},"bots":{"GPTBot":["partial","*",4,0],"ChatGPT-User":["partial","*",4,0],"OAI-SearchBot":["partial","*",4,0],"ClaudeBot":["partial","*",4,0],"Claude-User":["partial","*",4,0],"Claude-SearchBot":["partial","*",4,0],"anthropic-ai":["partial","*",4,0],"PerplexityBot":["partial","*",4,0],"Perplexity-User":["partial","*",4,0],"Google-Extended":["partial","*",4,0],"Applebot-Extended":["partial","*",4,0],"CCBot":["partial","*",4,0],"Bytespider":["partial","*",4,0],"Amazonbot":["partial","*",4,0],"meta-externalagent":["partial","*",4,0],"cohere-ai":["partial","*",4,0],"Diffbot":["partial","*",4,0],"ContextFetchBot":["partial","*",4,0]},"llms":{"status":"missing","http":404,"bytes":25136},"home":{"status":"ok","http":200,"url":"https://www.ioncube.com/","html_tokens":16109,"markdown_tokens":63,"pruned_tokens":63}},
{"rank":4562,"domain":"cbn.com","origin":"https://cbn.com","robots":{"status":"ok","http":200,"url":"https://cbn.com/robots.txt","bytes":2027,"groups":1},"bots":{"GPTBot":["partial","*",34,18],"ChatGPT-User":["partial","*",34,18],"OAI-SearchBot":["partial","*",34,18],"ClaudeBot":["partial","*",34,18],"Claude-User":["partial","*",34,18],"Claude-SearchBot":["partial","*",34,18],"anthropic-ai":["partial","*",34,18],"PerplexityBot":["partial","*",34,18],"Perplexity-User":["partial","*",34,18],"Google-Extended":["partial","*",34,18],"Applebot-Extended":["partial","*",34,18],"CCBot":["partial","*",34,18],"Bytespider":["partial","*",34,18],"Amazonbot":["partial","*",34,18],"meta-externalagent":["partial","*",34,18],"cohere-ai":["partial","*",34,18],"Diffbot":["partial","*",34,18],"ContextFetchBot":["partial","*",34,18]},"llms":{"status":"html","http":200,"bytes":8003},"home":{"status":"ok","http":200,"url":"https://cbn.com/","html_tokens":325679,"markdown_tokens":95,"pruned_tokens":95}},
{"rank":4563,"domain":"gda.pl","origin":"http://gda.pl","robots":{"status":"refused","http":403,"url":"http://gda.pl/robots.txt","bytes":146,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":4564,"domain":"rts.ch","origin":"https://rts.ch","robots":{"status":"ok","http":200,"url":"https://www.rts.ch/robots.txt","bytes":2645,"groups":23},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["partial","*",25,0],"OAI-SearchBot":["partial","*",25,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",25,0],"Claude-SearchBot":["partial","*",25,0],"anthropic-ai":["partial","*",25,0],"PerplexityBot":["partial","*",25,0],"Perplexity-User":["partial","*",25,0],"Google-Extended":["partial","*",25,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["partial","*",25,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",25,0]},"llms":{"status":"http_error","http":406,"bytes":2272},"home":{"status":"ok","http":200,"url":"https://www.rts.ch/","html_tokens":68295,"markdown_tokens":131,"pruned_tokens":131}},
{"rank":4565,"domain":"myfitnesspal.com","origin":"https://myfitnesspal.com","robots":{"status":"ok","http":200,"url":"https://www.myfitnesspal.com/robots.txt","bytes":1120,"groups":2},"bots":{"GPTBot":["partial","*",46,0],"ChatGPT-User":["blocked","named",1,0],"OAI-SearchBot":["partial","*",46,0],"ClaudeBot":["partial","*",46,0],"Claude-User":["partial","*",46,0],"Claude-SearchBot":["partial","*",46,0],"anthropic-ai":["partial","*",46,0],"PerplexityBot":["partial","*",46,0],"Perplexity-User":["partial","*",46,0],"Google-Extended":["partial","*",46,0],"Applebot-Extended":["partial","*",46,0],"CCBot":["partial","*",46,0],"Bytespider":["partial","*",46,0],"Amazonbot":["partial","*",46,0],"meta-externalagent":["partial","*",46,0],"cohere-ai":["partial","*",46,0],"Diffbot":["partial","*",46,0],"ContextFetchBot":["partial","*",46,0]},"llms":{"status":"missing","http":404,"bytes":2080},"home":{"status":"ok","http":200,"url":"https://www.myfitnesspal.com/","html_tokens":239745,"markdown_tokens":1753,"pruned_tokens":246}},
{"rank":4566,"domain":"harpers.org","origin":"https://harpers.org","robots":{"status":"ok","http":200,"url":"https://harpers.org/robots.txt","bytes":114,"groups":1},"bots":{"GPTBot":["partial","*",1,1],"ChatGPT-User":["partial","*",1,1],"OAI-SearchBot":["partial","*",1,1],"ClaudeBot":["partial","*",1,1],"Claude-User":["partial","*",1,1],"Claude-SearchBot":["partial","*",1,1],"anthropic-ai":["partial","*",1,1],"PerplexityBot":["partial","*",1,1],"Perplexity-User":["partial","*",1,1],"Google-Extended":["partial","*",1,1],"Applebot-Extended":["partial","*",1,1],"CCBot":["partial","*",1,1],"Bytespider":["partial","*",1,1],"Amazonbot":["partial","*",1,1],"meta-externalagent":["partial","*",1,1],"cohere-ai":["partial","*",1,1],"Diffbot":["partial","*",1,1],"ContextFetchBot":["partial","*",1,1]},"llms":{"status":"missing","http":404,"bytes":79982},"home":{"status":"ok","http":200,"url":"https://harpers.org/","html_tokens":39789,"markdown_tokens":297,"pruned_tokens":122}},
{"rank":4567,"domain":"dn.ua","origin":"https://dn.ua","robots":{"status":"timeout","http":null,"url":"https://dn.ua/robots.txt","bytes":0,"groups":0,"error":"ConnectTimeout"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":4568,"domain":"gamerant.com","origin":"https://gamerant.com","robots":{"status":"ok","http":200,"url":"https://gamerant.com/robots.txt","bytes":2029,"groups":2},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["partial","*",22,0],"OAI-SearchBot":["partial","*",22,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",22,0],"Claude-SearchBot":["partial","*",22,0],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["partial","*",22,0],"Perplexity-User":["partial","*",22,0],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",22,0]},"llms":{"status":"missing","http":404,"bytes":113197},"home":{"status":"ok","http":200,"url":"https://gamerant.com/","html_tokens":233153,"markdown_tokens":316,"pruned_tokens":141}},
{"rank":4569,"domain":"computerhope.com","origin":"https://computerhope.com","robots":{"status":"ok","http":200,"url":"https://www.computerhope.com/robots.txt","bytes":9655,"groups":184},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["blocked","named",1,0],"OAI-SearchBot":["blocked","named",1,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",37,0],"Claude-SearchBot":["partial","*",37,0],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["blocked","named",1,0],"Google-Extended":["partial","*",37,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",37,0]},"llms":{"status":"missing","http":404,"bytes":6870},"home":{"status":"http_error","http":403,"url":"https://www.computerhope.com/"}},
{"rank":4570,"domain":"rian.ru","origin":"http://rian.ru","robots":{"status":"ok","http":200,"url":"https://ria.ru/robots.txt","bytes":2216,"groups":2},"bots":{"GPTBot":["partial","*",18,2],"ChatGPT-User":["partial","*",18,2],"OAI-SearchBot":["partial","*",18,2],"ClaudeBot":["partial","*",18,2],"Claude-User":["partial","*",18,2],"Claude-SearchBot":["partial","*",18,2],"anthropic-ai":["partial","*",18,2],"PerplexityBot":["partial","*",18,2],"Perplexity-User":["partial","*",18,2],"Google-Extended":["partial","*",18,2],"Applebot-Extended":["partial","*",18,2],"CCBot":["partial","*",18,2],"Bytespider":["partial","*",18,2],"Amazonbot":["partial","*",18,2],"meta-externalagent":["partial","*",18,2],"cohere-ai":["partial","*",18,2],"Diffbot":["partial","*",18,2],"ContextFetchBot":["partial","*",18,2]},"llms":{"status":"missing","http":404,"bytes":99838},"home":{"status":"ok","http":200,"url":"https://ria.ru/","html_tokens":44610,"markdown_tokens":24,"pruned_tokens":24}},
{"rank":4571,"domain":"starlink.com","origin":"https://starlink.com","robots":{"status":"ok","http":200,"url":"https://starlink.com/robots.txt","bytes":652,"groups":1},"bots":{"GPTBot":["partial","*",17,0],"ChatGPT-User":["partial","*",17,0],"OAI-SearchBot":["partial","*",17,0],"ClaudeBot":["partial","*",17,0],"Claude-User":["partial","*",17,0],"Claude-SearchBot":["partial","*",17,0],"anthropic-ai":["partial","*",17,0],"PerplexityBot":["partial","*",17,0],"Perplexity-User":["partial","*",17,0],"Google-Extended":["partial","*",17,0],"Applebot-Extended":["partial","*",17,0],"CCBot":["partial","*",17,0],"Bytespider":["partial","*",17,0],"Amazonbot":["partial","*",17,0],"meta-externalagent":["partial","*",17,0],"cohere-ai":["partial","*",17,0],"Diffbot":["partial","*",17,0],"ContextFetchBot":["partial","*",17,0]},"llms":{"status":"html","http":200,"bytes":35064},"home":{"status":"ok","http":200,"url":"https://starlink.com/","html_tokens":8766,"markdown_tokens":28,"pruned_tokens":28}},
{"rank":4572,"domain":"nttdocomo.co.jp","origin":"https://www.nttdocomo.co.jp","robots":{"status":"missing","http":404,"url":"https://www.docomo.ne.jp/robots.txt","bytes":29184,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":29184},"home":{"status":"ok","http":200,"url":"https://www.docomo.ne.jp/","html_tokens":43596,"markdown_tokens":13,"pruned_tokens":13}},
{"rank":4573,"domain":"x.gd","origin":"https://x.gd","robots":{"status":"ok","http":200,"url":"https://x.gd/robots.txt","bytes":57,"groups":1},"bots":{"GPTBot":["allowed","*",0,1],"ChatGPT-User":["allowed","*",0,1],"OAI-SearchBot":["allowed","*",0,1],"ClaudeBot":["allowed","*",0,1],"Claude-User":["allowed","*",0,1],"Claude-SearchBot":["allowed","*",0,1],"anthropic-ai":["allowed","*",0,1],"PerplexityBot":["allowed","*",0,1],"Perplexity-User":["allowed","*",0,1],"Google-Extended":["allowed","*",0,1],"Applebot-Extended":["allowed","*",0,1],"CCBot":["allowed","*",0,1],"Bytespider":["allowed","*",0,1],"Amazonbot":["allowed","*",0,1],"meta-externalagent":["allowed","*",0,1],"cohere-ai":["allowed","*",0,1],"Diffbot":["allowed","*",0,1],"ContextFetchBot":["allowed","*",0,1]},"llms":{"status":"missing","http":404,"bytes":2987},"home":{"status":"ok","http":200,"url":"https://x.gd/","html_tokens":30196,"markdown_tokens":505,"pruned_tokens":18}},
{"rank":4574,"domain":"kaiserpermanente.org","origin":"https://kaiserpermanente.org","robots":{"status":"ok","http":200,"url":"https://healthy.kaiserpermanente.org/robots.txt?kp_shortcut_referrer=kp.org/robots.txt","bytes":6653,"groups":1},"bots":{"GPTBot":["partial","*",7,0],"ChatGPT-User":["partial","*",7,0],"OAI-SearchBot":["partial","*",7,0],"ClaudeBot":["partial","*",7,0],"Claude-User":["partial","*",7,0],"Claude-SearchBot":["partial","*",7,0],"anthropic-ai":["partial","*",7,0],"PerplexityBot":["partial","*",7,0],"Perplexity-User":["partial","*",7,0],"Google-Extended":["partial","*",7,0],"Applebot-Extended":["partial","*",7,0],"CCBot":["partial","*",7,0],"Bytespider":["partial","*",7,0],"Amazonbot":["partial","*",7,0],"meta-externalagent":["partial","*",7,0],"cohere-ai":["partial","*",7,0],"Diffbot":["partial","*",7,0],"ContextFetchBot":["partial","*",7,0]},"llms":{"status":"html","http":200,"bytes":333590},"home":{"status":"ok","http":200,"url":"https://healthy.kaiserpermanente.org/front-door","html_tokens":83358,"markdown_tokens":321,"pruned_tokens":28}},
{"rank":4575,"domain":"prntscr.com","origin":"https://prntscr.com","robots":{"status":"ok","http":200,"url":"https://prntscr.com/robots.txt","bytes":40,"groups":1},"bots":{"GPTBot":["allowed","*",0,1],"ChatGPT-User":["allowed","*",0,1],"OAI-SearchBot":["allowed","*",0,1],"ClaudeBot":["allowed","*",0,1],"Claude-User":["allowed","*",0,1],"Claude-SearchBot":["allowed","*",0,1],"anthropic-ai":["allowed","*",0,1],"PerplexityBot":["allowed","*",0,1],"Perplexity-User":["allowed","*",0,1],"Google-Extended":["allowed","*",0,1],"Applebot-Extended":["allowed","*",0,1],"CCBot":["allowed","*",0,1],"Bytespider":["allowed","*",0,1],"Amazonbot":["allowed","*",0,1],"meta-externalagent":["allowed","*",0,1],"cohere-ai":["allowed","*",0,1],"Diffbot":["allowed","*",0,1],"ContextFetchBot":["allowed","*",0,1]},"llms":{"status":"html","http":200,"bytes":16424},"home":{"status":"ok","http":200,"url":"https://prnt.sc","html_tokens":3279,"markdown_tokens":13,"pruned_tokens":13}},
{"rank":4576,"domain":"cyberciti.biz","origin":"https://cyberciti.biz","robots":{"status":"ok","http":200,"url":"https://www.cyberciti.biz/robots.txt","bytes":1729,"groups":3},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["blocked","named",1,0],"OAI-SearchBot":["blocked","named",1,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",1,0],"Claude-SearchBot":["partial","*",1,0],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["partial","*",1,0],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",1,0]},"llms":{"status":"missing","http":404,"bytes":14492},"home":{"status":"http_error","http":403,"url":"https://cyberciti.biz/"}},
{"rank":4577,"domain":"comicbook.com","origin":"https://comicbook.com","robots":{"status":"ok","http":200,"url":"https://comicbook.com/robots.txt","bytes":268,"groups":1},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"missing","http":404,"bytes":204292},"home":{"status":"ok","http":200,"url":"https://comicbook.com/","html_tokens":132038,"markdown_tokens":69,"pruned_tokens":69}},
{"rank":4578,"domain":"teamtailor.com","origin":"https://teamtailor.com","robots":{"status":"missing","http":404,"url":"https://www.teamtailor.com/robots","bytes":51274,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":51274},"home":{"status":"ok","http":200,"url":"https://www.teamtailor.com/en-us/","html_tokens":25071,"markdown_tokens":398,"pruned_tokens":299}},
{"rank":4579,"domain":"anpc.ro","origin":"https://anpc.ro","robots":{"status":"server_error","http":503,"url":"https://anpc.ro/robots.txt","bytes":8904,"groups":0,"error":"HTTP 503"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":4580,"domain":"ato.gov.au","origin":"https://ato.gov.au","robots":{"status":"refused","http":403,"url":"https://www.ato.gov.au/robots.txt","bytes":386,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":4581,"domain":"smashwords.com","origin":"https://smashwords.com","robots":{"status":"ok","http":200,"url":"https://www.smashwords.com/robots.txt","bytes":1728,"groups":11},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["blocked","named",1,0],"OAI-SearchBot":["blocked","named",1,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",4,0],"Claude-SearchBot":["partial","*",4,0],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["partial","*",4,0],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",4,0]},"llms":{"status":"missing","http":404,"bytes":1028},"home":{"status":"ok","http":200,"url":"https://www.smashwords.com/","html_tokens":13494,"markdown_tokens":2048,"pruned_tokens":165}},
{"rank":4582,"domain":"nwf.org","origin":"https://nwf.org","robots":{"status":"ok","http":200,"url":"https://nwf.org/robots.txt","bytes":98,"groups":1},"bots":{"GPTBot":["partial","*",2,0],"ChatGPT-User":["partial","*",2,0],"OAI-SearchBot":["partial","*",2,0],"ClaudeBot":["partial","*",2,0],"Claude-User":["partial","*",2,0],"Claude-SearchBot":["partial","*",2,0],"anthropic-ai":["partial","*",2,0],"PerplexityBot":["partial","*",2,0],"Perplexity-User":["partial","*",2,0],"Google-Extended":["partial","*",2,0],"Applebot-Extended":["partial","*",2,0],"CCBot":["partial","*",2,0],"Bytespider":["partial","*",2,0],"Amazonbot":["partial","*",2,0],"meta-externalagent":["partial","*",2,0],"cohere-ai":["partial","*",2,0],"Diffbot":["partial","*",2,0],"ContextFetchBot":["partial","*",2,0]},"llms":{"status":"missing","http":404,"bytes":1245},"home":{"status":"ok","http":200,"url":"https://nwf.org/","html_tokens":13532,"markdown_tokens":278,"pruned_tokens":66}},
{"rank":4583,"domain":"anaconda.com","origin":"https://anaconda.com","robots":{"status":"ok","http":200,"url":"https://www.anaconda.com/robots.txt","bytes":174,"groups":1},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"missing","http":404,"bytes":323654},"home":{"status":"ok","http":200,"url":"https://www.anaconda.com/","html_tokens":130782,"markdown_tokens":381,"pruned_tokens":85}},
{"rank":4584,"domain":"astrazeneca.com","origin":"https://www.astrazeneca.com","robots":{"status":"refused","http":403,"url":"https://www.astrazeneca.com/robots.txt","bytes":919,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":4585,"domain":"zenfolio.com","origin":"https://zenfolio.com","robots":{"status":"ok","http":200,"url":"https://zenfolio.com/robots.txt","bytes":593,"groups":1},"bots":{"GPTBot":["partial","*",6,1],"ChatGPT-User":["partial","*",6,1],"OAI-SearchBot":["partial","*",6,1],"ClaudeBot":["partial","*",6,1],"Claude-User":["partial","*",6,1],"Claude-SearchBot":["partial","*",6,1],"anthropic-ai":["partial","*",6,1],"PerplexityBot":["partial","*",6,1],"Perplexity-User":["partial","*",6,1],"Google-Extended":["partial","*",6,1],"Applebot-Extended":["partial","*",6,1],"CCBot":["partial","*",6,1],"Bytespider":["partial","*",6,1],"Amazonbot":["partial","*",6,1],"meta-externalagent":["partial","*",6,1],"cohere-ai":["partial","*",6,1],"Diffbot":["partial","*",6,1],"ContextFetchBot":["partial","*",6,1]},"llms":{"status":"other_text","http":200,"bytes":5297},"home":{"status":"ok","http":200,"url":"https://zenfolio.com/","html_tokens":129627,"markdown_tokens":181,"pruned_tokens":166}},
{"rank":4586,"domain":"savefrom.net","origin":"https://savefrom.net","robots":{"status":"ok","http":200,"url":"https://savefrom.net/robots.txt","bytes":224,"groups":2},"bots":{"GPTBot":["partial","*",3,3],"ChatGPT-User":["partial","*",3,3],"OAI-SearchBot":["partial","*",3,3],"ClaudeBot":["partial","*",3,3],"Claude-User":["partial","*",3,3],"Claude-SearchBot":["partial","*",3,3],"anthropic-ai":["partial","*",3,3],"PerplexityBot":["partial","*",3,3],"Perplexity-User":["partial","*",3,3],"Google-Extended":["partial","*",3,3],"Applebot-Extended":["partial","*",3,3],"CCBot":["partial","*",3,3],"Bytespider":["partial","*",3,3],"Amazonbot":["partial","*",3,3],"meta-externalagent":["partial","*",3,3],"cohere-ai":["partial","*",3,3],"Diffbot":["partial","*",3,3],"ContextFetchBot":["partial","*",3,3]},"llms":{"status":"present","http":200,"bytes":2642},"home":{"status":"ok","http":200,"url":"https://us.savefrom.net","html_tokens":672,"markdown_tokens":146,"pruned_tokens":127}},
{"rank":4587,"domain":"theartnewspaper.com","origin":"https://theartnewspaper.com","robots":{"status":"missing","http":404,"url":"https://www.theartnewspaper.com/robots.txt","bytes":104294,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":104294},"home":{"status":"ok","http":200,"url":"https://www.theartnewspaper.com/","html_tokens":256683,"markdown_tokens":575,"pruned_tokens":105}},
{"rank":4588,"domain":"easychair.org","origin":"https://easychair.org","robots":{"status":"ok","http":200,"url":"https://easychair.org/robots.txt","bytes":156,"groups":2},"bots":{"GPTBot":["partial","*",4,0],"ChatGPT-User":["partial","*",4,0],"OAI-SearchBot":["partial","*",4,0],"ClaudeBot":["partial","*",4,0],"Claude-User":["partial","*",4,0],"Claude-SearchBot":["partial","*",4,0],"anthropic-ai":["partial","*",4,0],"PerplexityBot":["partial","*",4,0],"Perplexity-User":["partial","*",4,0],"Google-Extended":["partial","*",4,0],"Applebot-Extended":["partial","*",4,0],"CCBot":["partial","*",4,0],"Bytespider":["partial","*",4,0],"Amazonbot":["partial","*",4,0],"meta-externalagent":["partial","*",4,0],"cohere-ai":["partial","*",4,0],"Diffbot":["partial","*",4,0],"ContextFetchBot":["partial","*",4,0]},"llms":{"status":"missing","http":404,"bytes":139},"home":{"status":"unreachable","error":"Server disconnected without sending a response."}},
{"rank":4589,"domain":"247wallst.com","origin":"https://247wallst.com","robots":{"status":"ok","http":200,"url":"https://247wallst.com/robots.txt","bytes":284,"groups":1},"bots":{"GPTBot":["partial","*",11,2],"ChatGPT-User":["partial","*",11,2],"OAI-SearchBot":["partial","*",11,2],"ClaudeBot":["partial","*",11,2],"Claude-User":["partial","*",11,2],"Claude-SearchBot":["partial","*",11,2],"anthropic-ai":["partial","*",11,2],"PerplexityBot":["partial","*",11,2],"Perplexity-User":["partial","*",11,2],"Google-Extended":["partial","*",11,2],"Applebot-Extended":["partial","*",11,2],"CCBot":["partial","*",11,2],"Bytespider":["partial","*",11,2],"Amazonbot":["partial","*",11,2],"meta-externalagent":["partial","*",11,2],"cohere-ai":["partial","*",11,2],"Diffbot":["partial","*",11,2],"ContextFetchBot":["partial","*",11,2]},"llms":{"status":"missing","http":404,"bytes":264104},"home":{"status":"ok","http":200,"url":"https://247wallst.com/","html_tokens":83201,"markdown_tokens":859,"pruned_tokens":142}},
{"rank":4590,"domain":"oasis-open.org","origin":"https://oasis-open.org","robots":{"status":"ok","http":200,"url":"https://oasis-open.org/robots.txt","bytes":1246,"groups":12},"bots":{"GPTBot":["partial","*",12,1],"ChatGPT-User":["partial","*",12,1],"OAI-SearchBot":["partial","*",12,1],"ClaudeBot":["partial","*",12,1],"Claude-User":["partial","*",12,1],"Claude-SearchBot":["partial","*",12,1],"anthropic-ai":["partial","*",12,1],"PerplexityBot":["partial","*",12,1],"Perplexity-User":["partial","*",12,1],"Google-Extended":["partial","*",12,1],"Applebot-Extended":["partial","*",12,1],"CCBot":["partial","*",12,1],"Bytespider":["partial","*",12,1],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["partial","*",12,1],"cohere-ai":["partial","*",12,1],"Diffbot":["partial","*",12,1],"ContextFetchBot":["partial","*",12,1]},"llms":{"status":"missing","http":404,"bytes":513},"home":{"status":"ok","http":200,"url":"https://www.oasis-open.org/","html_tokens":44374,"markdown_tokens":240,"pruned_tokens":190}},
{"rank":4591,"domain":"cathaypacific.com","origin":"https://cathaypacific.com","robots":{"status":"unreachable","http":null,"url":"https://cathaypacific.com/robots.txt","bytes":0,"groups":0,"error":"<StreamReset stream_id:1, error_code:2, remote_reset:True>"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":4592,"domain":"9gag.com","origin":"https://9gag.com","robots":{"status":"ok","http":200,"url":"https://9gag.com/robots.txt","bytes":358,"groups":2},"bots":{"GPTBot":["partial","*",9,0],"ChatGPT-User":["partial","*",9,0],"OAI-SearchBot":["partial","*",9,0],"ClaudeBot":["partial","*",9,0],"Claude-User":["partial","*",9,0],"Claude-SearchBot":["partial","*",9,0],"anthropic-ai":["partial","*",9,0],"PerplexityBot":["partial","*",9,0],"Perplexity-User":["partial","*",9,0],"Google-Extended":["partial","*",9,0],"Applebot-Extended":["partial","*",9,0],"CCBot":["partial","*",9,0],"Bytespider":["partial","*",9,0],"Amazonbot":["partial","*",9,0],"meta-externalagent":["partial","*",9,0],"cohere-ai":["partial","*",9,0],"Diffbot":["partial","*",9,0],"ContextFetchBot":["partial","*",9,0]},"llms":{"status":"present","http":200,"bytes":60975},"home":{"status":"ok","http":200,"url":"https://9gag.com/","html_tokens":47141,"markdown_tokens":12,"pruned_tokens":12}},
{"rank":4593,"domain":"depaul.edu","origin":"https://depaul.edu","robots":{"status":"ok","http":200,"url":"https://www.depaul.edu/robots.txt","bytes":68,"groups":1},"bots":{"GPTBot":["allowed","*",0,1],"ChatGPT-User":["allowed","*",0,1],"OAI-SearchBot":["allowed","*",0,1],"ClaudeBot":["allowed","*",0,1],"Claude-User":["allowed","*",0,1],"Claude-SearchBot":["allowed","*",0,1],"anthropic-ai":["allowed","*",0,1],"PerplexityBot":["allowed","*",0,1],"Perplexity-User":["allowed","*",0,1],"Google-Extended":["allowed","*",0,1],"Applebot-Extended":["allowed","*",0,1],"CCBot":["allowed","*",0,1],"Bytespider":["allowed","*",0,1],"Amazonbot":["allowed","*",0,1],"meta-externalagent":["allowed","*",0,1],"cohere-ai":["allowed","*",0,1],"Diffbot":["allowed","*",0,1],"ContextFetchBot":["allowed","*",0,1]},"llms":{"status":"missing","http":404,"bytes":163908},"home":{"status":"ok","http":200,"url":"https://www.depaul.edu/","html_tokens":60177,"markdown_tokens":313,"pruned_tokens":268}},
{"rank":4594,"domain":"rsna.org","origin":"https://rsna.org","robots":{"status":"ok","http":200,"url":"https://www.rsna.org/robots.txt","bytes":158,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":1245},"home":{"status":"ok","http":200,"url":"https://www.rsna.org/","html_tokens":24717,"markdown_tokens":394,"pruned_tokens":251}},
{"rank":4595,"domain":"51cto.com","origin":"https://www.51cto.com","robots":{"status":"server_error","http":567,"url":"https://www.51cto.com/robots.txt","bytes":7352,"groups":0,"error":"HTTP 567"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":4596,"domain":"tagesanzeiger.ch","origin":"https://tagesanzeiger.ch","robots":{"status":"ok","http":200,"url":"https://www.tagesanzeiger.ch/robots.txt","bytes":1468,"groups":34},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["blocked","named",1,0],"OAI-SearchBot":["partial","named",1,1],"ClaudeBot":["blocked","named",1,0],"Claude-User":["allowed","*",0,1],"Claude-SearchBot":["allowed","*",0,1],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["partial","named",1,1],"Perplexity-User":["allowed","*",0,1],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["allowed","*",0,1],"Diffbot":["allowed","*",0,1],"ContextFetchBot":["allowed","*",0,1]},"llms":{"status":"missing","http":404,"bytes":62191},"home":{"status":"ok","http":200,"url":"https://www.tagesanzeiger.ch/","html_tokens":350276,"markdown_tokens":73,"pruned_tokens":73}},
{"rank":4597,"domain":"stellantis.com","origin":"https://stellantis.com","robots":{"status":"refused","http":403,"url":"https://www.stellantis.com/robots.txt","bytes":384,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":4598,"domain":"escardio.org","origin":"https://www.escardio.org","robots":{"status":"ok","http":200,"url":"https://www.escardio.org/robots.txt","bytes":2190,"groups":17},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["allowed","named",0,1],"OAI-SearchBot":["allowed","named",0,1],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",16,0],"Claude-SearchBot":["allowed","named",0,1],"anthropic-ai":["partial","*",16,0],"PerplexityBot":["allowed","named",0,1],"Perplexity-User":["partial","*",16,0],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["partial","*",16,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["partial","*",16,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["partial","*",16,0],"Diffbot":["partial","*",16,0],"ContextFetchBot":["partial","*",16,0]},"llms":{"status":"missing","http":404,"bytes":299171},"home":{"status":"ok","http":200,"url":"https://www.escardio.org/","html_tokens":93571,"markdown_tokens":1427,"pruned_tokens":116}},
{"rank":4599,"domain":"unito.it","origin":"https://unito.it","robots":{"status":"ok","http":200,"url":"https://www.unito.it/robots.txt","bytes":2067,"groups":2},"bots":{"GPTBot":["partial","*",34,18],"ChatGPT-User":["partial","*",34,18],"OAI-SearchBot":["partial","*",34,18],"ClaudeBot":["partial","*",34,18],"Claude-User":["partial","*",34,18],"Claude-SearchBot":["partial","*",34,18],"anthropic-ai":["partial","*",34,18],"PerplexityBot":["partial","*",34,18],"Perplexity-User":["partial","*",34,18],"Google-Extended":["partial","*",34,18],"Applebot-Extended":["partial","*",34,18],"CCBot":["partial","*",34,18],"Bytespider":["partial","*",34,18],"Amazonbot":["partial","*",34,18],"meta-externalagent":["partial","*",34,18],"cohere-ai":["partial","*",34,18],"Diffbot":["partial","*",34,18],"ContextFetchBot":["partial","*",34,18]},"llms":{"status":"missing","http":404,"bytes":547},"home":{"status":"ok","http":200,"url":"https://www.unito.it/","html_tokens":56978,"markdown_tokens":109,"pruned_tokens":61}},
{"rank":4600,"domain":"keepass.info","origin":"https://keepass.info","robots":{"status":"missing","http":404,"url":"https://keepass.info/robots.txt","bytes":13,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":13},"home":{"status":"ok","http":200,"url":"https://keepass.info/","html_tokens":2554,"markdown_tokens":498,"pruned_tokens":308}},
{"rank":4601,"domain":"sejm.gov.pl","origin":"https://sejm.gov.pl","robots":{"status":"ok","http":200,"url":"https://sejm.gov.pl/robots.txt","bytes":60,"groups":2},"bots":{"GPTBot":["blocked","*",1,0],"ChatGPT-User":["blocked","*",1,0],"OAI-SearchBot":["blocked","*",1,0],"ClaudeBot":["blocked","*",1,0],"Claude-User":["blocked","*",1,0],"Claude-SearchBot":["blocked","*",1,0],"anthropic-ai":["blocked","*",1,0],"PerplexityBot":["blocked","*",1,0],"Perplexity-User":["blocked","*",1,0],"Google-Extended":["blocked","*",1,0],"Applebot-Extended":["blocked","*",1,0],"CCBot":["blocked","*",1,0],"Bytespider":["blocked","*",1,0],"Amazonbot":["blocked","*",1,0],"meta-externalagent":["blocked","*",1,0],"cohere-ai":["blocked","*",1,0],"Diffbot":["blocked","*",1,0],"ContextFetchBot":["blocked","*",1,0]},"llms":{"status":"disallowed"},"home":{"status":"disallowed"}},
{"rank":4602,"domain":"nielseniq.com","origin":"https://nielseniq.com","robots":{"status":"ok","http":200,"url":"https://nielseniq.com/robots.txt","bytes":371,"groups":9},"bots":{"GPTBot":["allowed","named",0,1],"ChatGPT-User":["allowed","named",0,1],"OAI-SearchBot":["allowed","named",0,1],"ClaudeBot":["allowed","named",0,1],"Claude-User":["allowed","named",0,1],"Claude-SearchBot":["partial","*",2,0],"anthropic-ai":["partial","*",2,0],"PerplexityBot":["allowed","named",0,1],"Perplexity-User":["partial","*",2,0],"Google-Extended":["allowed","named",0,1],"Applebot-Extended":["partial","*",2,0],"CCBot":["allowed","named",0,1],"Bytespider":["partial","*",2,0],"Amazonbot":["partial","*",2,0],"meta-externalagent":["partial","*",2,0],"cohere-ai":["partial","*",2,0],"Diffbot":["partial","*",2,0],"ContextFetchBot":["partial","*",2,0]},"llms":{"status":"other_text","http":200,"bytes":4033838},"home":{"status":"ok","http":200,"url":"https://nielseniq.com/global/en/","html_tokens":90935,"markdown_tokens":351,"pruned_tokens":105}},
{"rank":4603,"domain":"iccwbo.org","origin":"https://iccwbo.org","robots":{"status":"ok","http":200,"url":"https://iccwbo.org/robots.txt","bytes":481,"groups":1},"bots":{"GPTBot":["partial","*",11,3],"ChatGPT-User":["partial","*",11,3],"OAI-SearchBot":["partial","*",11,3],"ClaudeBot":["partial","*",11,3],"Claude-User":["partial","*",11,3],"Claude-SearchBot":["partial","*",11,3],"anthropic-ai":["partial","*",11,3],"PerplexityBot":["partial","*",11,3],"Perplexity-User":["partial","*",11,3],"Google-Extended":["partial","*",11,3],"Applebot-Extended":["partial","*",11,3],"CCBot":["partial","*",11,3],"Bytespider":["partial","*",11,3],"Amazonbot":["partial","*",11,3],"meta-externalagent":["partial","*",11,3],"cohere-ai":["partial","*",11,3],"Diffbot":["partial","*",11,3],"ContextFetchBot":["partial","*",11,3]},"llms":{"status":"missing","http":404,"bytes":113391},"home":{"status":"ok","http":200,"url":"https://iccwbo.org/","html_tokens":46257,"markdown_tokens":360,"pruned_tokens":280}},
{"rank":4604,"domain":"usbank.com","origin":"https://usbank.com","robots":{"status":"ok","http":200,"url":"https://www.usbank.com/robots.txt","bytes":1416,"groups":1},"bots":{"GPTBot":["partial","*",40,3],"ChatGPT-User":["partial","*",40,3],"OAI-SearchBot":["partial","*",40,3],"ClaudeBot":["partial","*",40,3],"Claude-User":["partial","*",40,3],"Claude-SearchBot":["partial","*",40,3],"anthropic-ai":["partial","*",40,3],"PerplexityBot":["partial","*",40,3],"Perplexity-User":["partial","*",40,3],"Google-Extended":["partial","*",40,3],"Applebot-Extended":["partial","*",40,3],"CCBot":["partial","*",40,3],"Bytespider":["partial","*",40,3],"Amazonbot":["partial","*",40,3],"meta-externalagent":["partial","*",40,3],"cohere-ai":["partial","*",40,3],"Diffbot":["partial","*",40,3],"ContextFetchBot":["partial","*",40,3]},"llms":{"status":"other_text","http":200,"bytes":962481},"home":{"status":"ok","http":200,"url":"https://www.usbank.com/index.html","html_tokens":69575,"markdown_tokens":350,"pruned_tokens":167}},
{"rank":4605,"domain":"thecrimson.com","origin":"https://thecrimson.com","robots":{"status":"ok","http":200,"url":"https://www.thecrimson.com/robots.txt","bytes":63,"groups":1},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"missing","http":404,"bytes":409},"home":{"status":"ok","http":200,"url":"https://www.thecrimson.com/","html_tokens":39080,"markdown_tokens":802,"pruned_tokens":105}},
{"rank":4606,"domain":"16personalities.com","origin":"https://16personalities.com","robots":{"status":"ok","http":200,"url":"https://www.16personalities.com/robots.txt","bytes":528,"groups":2},"bots":{"GPTBot":["allowed","named",0,5],"ChatGPT-User":["allowed","named",0,5],"OAI-SearchBot":["allowed","named",0,5],"ClaudeBot":["allowed","named",0,5],"Claude-User":["allowed","named",0,5],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","named",0,5],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","named",0,5],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","named",0,5],"Bytespider":["allowed","named",0,5],"Amazonbot":["allowed","named",0,5],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"present","http":200,"bytes":1941},"home":{"status":"ok","http":200,"url":"https://www.16personalities.com/","html_tokens":20793,"markdown_tokens":114,"pruned_tokens":114}},
{"rank":4607,"domain":"hexo.io","origin":"https://hexo.io","robots":{"status":"ok","http":200,"url":"https://hexo.io/robots.txt","bytes":104,"groups":1},"bots":{"GPTBot":["partial","*",3,0],"ChatGPT-User":["partial","*",3,0],"OAI-SearchBot":["partial","*",3,0],"ClaudeBot":["partial","*",3,0],"Claude-User":["partial","*",3,0],"Claude-SearchBot":["partial","*",3,0],"anthropic-ai":["partial","*",3,0],"PerplexityBot":["partial","*",3,0],"Perplexity-User":["partial","*",3,0],"Google-Extended":["partial","*",3,0],"Applebot-Extended":["partial","*",3,0],"CCBot":["partial","*",3,0],"Bytespider":["partial","*",3,0],"Amazonbot":["partial","*",3,0],"meta-externalagent":["partial","*",3,0],"cohere-ai":["partial","*",3,0],"Diffbot":["partial","*",3,0],"ContextFetchBot":["partial","*",3,0]},"llms":{"status":"missing","http":404,"bytes":3449},"home":{"status":"ok","http":200,"url":"https://hexo.io/","html_tokens":3695,"markdown_tokens":127,"pruned_tokens":127}},
{"rank":4608,"domain":"cvut.cz","origin":"https://www.cvut.cz","robots":{"status":"ok","http":200,"url":"https://www.cvut.cz/robots.txt","bytes":2189,"groups":1},"bots":{"GPTBot":["partial","*",36,32],"ChatGPT-User":["partial","*",36,32],"OAI-SearchBot":["partial","*",36,32],"ClaudeBot":["partial","*",36,32],"Claude-User":["partial","*",36,32],"Claude-SearchBot":["partial","*",36,32],"anthropic-ai":["partial","*",36,32],"PerplexityBot":["partial","*",36,32],"Perplexity-User":["partial","*",36,32],"Google-Extended":["partial","*",36,32],"Applebot-Extended":["partial","*",36,32],"CCBot":["partial","*",36,32],"Bytespider":["partial","*",36,32],"Amazonbot":["partial","*",36,32],"meta-externalagent":["partial","*",36,32],"cohere-ai":["partial","*",36,32],"Diffbot":["partial","*",36,32],"ContextFetchBot":["partial","*",36,32]},"llms":{"status":"missing","http":404,"bytes":291},"home":{"status":"ok","http":200,"url":"https://www.cvut.cz/","html_tokens":6766,"markdown_tokens":98,"pruned_tokens":98}},
{"rank":4609,"domain":"tcpdf.org","origin":"https://tcpdf.org","robots":{"status":"ok","http":200,"url":"https://tcpdf.org/robots.txt","bytes":95,"groups":1},"bots":{"GPTBot":["allowed","*",0,1],"ChatGPT-User":["allowed","*",0,1],"OAI-SearchBot":["allowed","*",0,1],"ClaudeBot":["allowed","*",0,1],"Claude-User":["allowed","*",0,1],"Claude-SearchBot":["allowed","*",0,1],"anthropic-ai":["allowed","*",0,1],"PerplexityBot":["allowed","*",0,1],"Perplexity-User":["allowed","*",0,1],"Google-Extended":["allowed","*",0,1],"Applebot-Extended":["allowed","*",0,1],"CCBot":["allowed","*",0,1],"Bytespider":["allowed","*",0,1],"Amazonbot":["allowed","*",0,1],"meta-externalagent":["allowed","*",0,1],"cohere-ai":["allowed","*",0,1],"Diffbot":["allowed","*",0,1],"ContextFetchBot":["allowed","*",0,1]},"llms":{"status":"present","http":200,"bytes":10979},"home":{"status":"ok","http":200,"url":"https://tcpdf.org/","html_tokens":10369,"markdown_tokens":2924,"pruned_tokens":294}},
{"rank":4610,"domain":"www.royal.uk","origin":"https://www.royal.uk","robots":{"status":"ok","http":200,"url":"https://www.royal.uk/robots.txt","bytes":2141,"groups":1},"bots":{"GPTBot":["partial","*",35,22],"ChatGPT-User":["partial","*",35,22],"OAI-SearchBot":["partial","*",35,22],"ClaudeBot":["partial","*",35,22],"Claude-User":["partial","*",35,22],"Claude-SearchBot":["partial","*",35,22],"anthropic-ai":["partial","*",35,22],"PerplexityBot":["partial","*",35,22],"Perplexity-User":["partial","*",35,22],"Google-Extended":["partial","*",35,22],"Applebot-Extended":["partial","*",35,22],"CCBot":["partial","*",35,22],"Bytespider":["partial","*",35,22],"Amazonbot":["partial","*",35,22],"meta-externalagent":["partial","*",35,22],"cohere-ai":["partial","*",35,22],"Diffbot":["partial","*",35,22],"ContextFetchBot":["partial","*",35,22]},"llms":{"status":"missing","http":404,"bytes":186},"home":{"status":"ok","http":200,"url":"https://www.royal.uk/","html_tokens":15771,"markdown_tokens":73,"pruned_tokens":73}},
{"rank":4611,"domain":"pair.com","origin":"https://pair.com","robots":{"status":"ok","http":200,"url":"https://www.pair.com/robots.txt","bytes":320,"groups":1},"bots":{"GPTBot":["partial","*",2,0],"ChatGPT-User":["partial","*",2,0],"OAI-SearchBot":["partial","*",2,0],"ClaudeBot":["partial","*",2,0],"Claude-User":["partial","*",2,0],"Claude-SearchBot":["partial","*",2,0],"anthropic-ai":["partial","*",2,0],"PerplexityBot":["partial","*",2,0],"Perplexity-User":["partial","*",2,0],"Google-Extended":["partial","*",2,0],"Applebot-Extended":["partial","*",2,0],"CCBot":["partial","*",2,0],"Bytespider":["partial","*",2,0],"Amazonbot":["partial","*",2,0],"meta-externalagent":["partial","*",2,0],"cohere-ai":["partial","*",2,0],"Diffbot":["partial","*",2,0],"ContextFetchBot":["partial","*",2,0]},"llms":{"status":"present","http":200,"bytes":3841},"home":{"status":"ok","http":200,"url":"https://www.pair.com/","html_tokens":72801,"markdown_tokens":1163,"pruned_tokens":297}},
{"rank":4612,"domain":"alternativeto.net","origin":"https://alternativeto.net","robots":{"status":"ok","http":200,"url":"https://alternativeto.net/robots.txt","bytes":1762,"groups":2},"bots":{"GPTBot":["partial","*",37,0],"ChatGPT-User":["partial","*",37,0],"OAI-SearchBot":["partial","*",37,0],"ClaudeBot":["partial","*",37,0],"Claude-User":["partial","*",37,0],"Claude-SearchBot":["partial","*",37,0],"anthropic-ai":["partial","*",37,0],"PerplexityBot":["partial","*",37,0],"Perplexity-User":["partial","*",37,0],"Google-Extended":["partial","*",37,0],"Applebot-Extended":["partial","*",37,0],"CCBot":["partial","*",37,0],"Bytespider":["partial","*",37,0],"Amazonbot":["partial","*",37,0],"meta-externalagent":["partial","*",37,0],"cohere-ai":["partial","*",37,0],"Diffbot":["partial","*",37,0],"ContextFetchBot":["partial","*",37,0]},"llms":{"status":"missing","http":404,"bytes":81668},"home":{"status":"http_error","http":403,"url":"https://alternativeto.net/"}},
{"rank":4613,"domain":"mediaite.com","origin":"https://mediaite.com","robots":{"status":"ok","http":200,"url":"https://mediaite.com/robots.txt","bytes":226,"groups":2},"bots":{"GPTBot":["partial","*",6,0],"ChatGPT-User":["partial","*",6,0],"OAI-SearchBot":["partial","*",6,0],"ClaudeBot":["partial","*",6,0],"Claude-User":["partial","*",6,0],"Claude-SearchBot":["partial","*",6,0],"anthropic-ai":["partial","*",6,0],"PerplexityBot":["partial","*",6,0],"Perplexity-User":["partial","*",6,0],"Google-Extended":["partial","*",6,0],"Applebot-Extended":["partial","*",6,0],"CCBot":["partial","*",6,0],"Bytespider":["partial","*",6,0],"Amazonbot":["partial","*",6,0],"meta-externalagent":["partial","*",6,0],"cohere-ai":["partial","*",6,0],"Diffbot":["partial","*",6,0],"ContextFetchBot":["partial","*",6,0]},"llms":{"status":"crawl_delay","error":"Crawl-delay 120 s"},"home":{"status":"crawl_delay","error":"Crawl-delay 120 s"}},
{"rank":4614,"domain":"eventbrite.de","origin":"https://eventbrite.de","robots":{"status":"ok","http":200,"url":"https://eventbrite.de/robots.txt","bytes":7499,"groups":135},"bots":{"GPTBot":["partial","*",51,2],"ChatGPT-User":["partial","*",51,2],"OAI-SearchBot":["partial","*",51,2],"ClaudeBot":["partial","*",51,2],"Claude-User":["partial","*",51,2],"Claude-SearchBot":["partial","*",51,2],"anthropic-ai":["partial","*",51,2],"PerplexityBot":["partial","*",51,2],"Perplexity-User":["partial","*",51,2],"Google-Extended":["partial","*",51,2],"Applebot-Extended":["partial","*",51,2],"CCBot":["blocked","named",1,0],"Bytespider":["partial","*",51,2],"Amazonbot":["partial","*",51,2],"meta-externalagent":["partial","*",51,2],"cohere-ai":["partial","*",51,2],"Diffbot":["partial","*",51,2],"ContextFetchBot":["partial","*",51,2]},"llms":{"status":"missing","http":404,"bytes":78232},"home":{"status":"ok","http":200,"url":"https://www.eventbrite.de","html_tokens":52482,"markdown_tokens":67,"pruned_tokens":67}},
{"rank":4615,"domain":"db.com","origin":"https://db.com","robots":{"status":"ok","http":200,"url":"https://www.db.com/robots.txt","bytes":279,"groups":1},"bots":{"GPTBot":["partial","*",4,0],"ChatGPT-User":["partial","*",4,0],"OAI-SearchBot":["partial","*",4,0],"ClaudeBot":["partial","*",4,0],"Claude-User":["partial","*",4,0],"Claude-SearchBot":["partial","*",4,0],"anthropic-ai":["partial","*",4,0],"PerplexityBot":["partial","*",4,0],"Perplexity-User":["partial","*",4,0],"Google-Extended":["partial","*",4,0],"Applebot-Extended":["partial","*",4,0],"CCBot":["partial","*",4,0],"Bytespider":["partial","*",4,0],"Amazonbot":["partial","*",4,0],"meta-externalagent":["partial","*",4,0],"cohere-ai":["partial","*",4,0],"Diffbot":["partial","*",4,0],"ContextFetchBot":["partial","*",4,0]},"llms":{"status":"present","http":200,"bytes":3787},"home":{"status":"ok","http":200,"url":"https://www.db.com/","html_tokens":37838,"markdown_tokens":59,"pruned_tokens":59}},
{"rank":4616,"domain":"bundesumweltministerium.de","origin":"https://bundesumweltministerium.de","robots":{"status":"ok","http":200,"url":"https://bundesumweltministerium.de/robots.txt","bytes":222,"groups":1},"bots":{"GPTBot":["partial","*",5,0],"ChatGPT-User":["partial","*",5,0],"OAI-SearchBot":["partial","*",5,0],"ClaudeBot":["partial","*",5,0],"Claude-User":["partial","*",5,0],"Claude-SearchBot":["partial","*",5,0],"anthropic-ai":["partial","*",5,0],"PerplexityBot":["partial","*",5,0],"Perplexity-User":["partial","*",5,0],"Google-Extended":["partial","*",5,0],"Applebot-Extended":["partial","*",5,0],"CCBot":["partial","*",5,0],"Bytespider":["partial","*",5,0],"Amazonbot":["partial","*",5,0],"meta-externalagent":["partial","*",5,0],"cohere-ai":["partial","*",5,0],"Diffbot":["partial","*",5,0],"ContextFetchBot":["partial","*",5,0]},"llms":{"status":"missing","http":404,"bytes":93329},"home":{"status":"ok","http":200,"url":"https://www.bundesumweltministerium.de/","html_tokens":48842,"markdown_tokens":145,"pruned_tokens":145}},
{"rank":4617,"domain":"ki.se","origin":"https://ki.se","robots":{"status":"ok","http":200,"url":"https://ki.se/robots.txt","bytes":2644,"groups":1},"bots":{"GPTBot":["partial","*",42,22],"ChatGPT-User":["partial","*",42,22],"OAI-SearchBot":["partial","*",42,22],"ClaudeBot":["partial","*",42,22],"Claude-User":["partial","*",42,22],"Claude-SearchBot":["partial","*",42,22],"anthropic-ai":["partial","*",42,22],"PerplexityBot":["partial","*",42,22],"Perplexity-User":["partial","*",42,22],"Google-Extended":["partial","*",42,22],"Applebot-Extended":["partial","*",42,22],"CCBot":["partial","*",42,22],"Bytespider":["partial","*",42,22],"Amazonbot":["partial","*",42,22],"meta-externalagent":["partial","*",42,22],"cohere-ai":["partial","*",42,22],"Diffbot":["partial","*",42,22],"ContextFetchBot":["partial","*",42,22]},"llms":{"status":"http_error","http":403,"bytes":153},"home":{"status":"ok","http":200,"url":"https://ki.se/","html_tokens":18334,"markdown_tokens":480,"pruned_tokens":85}},
{"rank":4618,"domain":"usal.es","origin":"https://usal.es","robots":{"status":"refused","http":403,"url":"https://usal.es/robots.txt","bytes":133,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":4619,"domain":"hackaday.io","origin":"https://hackaday.io","robots":{"status":"ok","http":200,"url":"https://hackaday.io/robots.txt","bytes":98,"groups":1},"bots":{"GPTBot":["partial","*",3,0],"ChatGPT-User":["partial","*",3,0],"OAI-SearchBot":["partial","*",3,0],"ClaudeBot":["partial","*",3,0],"Claude-User":["partial","*",3,0],"Claude-SearchBot":["partial","*",3,0],"anthropic-ai":["partial","*",3,0],"PerplexityBot":["partial","*",3,0],"Perplexity-User":["partial","*",3,0],"Google-Extended":["partial","*",3,0],"Applebot-Extended":["partial","*",3,0],"CCBot":["partial","*",3,0],"Bytespider":["partial","*",3,0],"Amazonbot":["partial","*",3,0],"meta-externalagent":["partial","*",3,0],"cohere-ai":["partial","*",3,0],"Diffbot":["partial","*",3,0],"ContextFetchBot":["partial","*",3,0]},"llms":{"status":"missing","http":404,"bytes":2960},"home":{"status":"ok","http":200,"url":"https://hackaday.io/","html_tokens":7707,"markdown_tokens":107,"pruned_tokens":51}},
{"rank":4620,"domain":"hokudai.ac.jp","origin":"https://www.hokudai.ac.jp","robots":{"status":"ok","http":200,"url":"https://www.hokudai.ac.jp/robots.txt","bytes":0,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":13398},"home":{"status":"ok","http":200,"url":"https://www.hokudai.ac.jp/","html_tokens":10509,"markdown_tokens":11,"pruned_tokens":11}},
{"rank":4621,"domain":"web.com","origin":"https://web.com","robots":{"status":"refused","http":403,"url":"https://www.web.com/robots.txt","bytes":251570,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":4622,"domain":"montrealgazette.com","origin":"https://montrealgazette.com","robots":{"status":"ok","http":200,"url":"https://montrealgazette.com/robots.txt","bytes":2305,"groups":32},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["partial","*",11,1],"OAI-SearchBot":["partial","*",11,1],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",11,1],"Claude-SearchBot":["partial","*",11,1],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["blocked","named",1,0],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["partial","*",11,1],"Diffbot":["partial","*",11,1],"ContextFetchBot":["partial","*",11,1]},"llms":{"status":"missing","http":404,"bytes":387},"home":{"status":"ok","http":200,"url":"https://montrealgazette.com/","html_tokens":87701,"markdown_tokens":196,"pruned_tokens":38}},
{"rank":4623,"domain":"aerzteblatt.de","origin":"https://aerzteblatt.de","robots":{"status":"ok","http":200,"url":"https://www.aerzteblatt.de/robots.txt","bytes":2955,"groups":30},"bots":{"GPTBot":["partial","named",1,3],"ChatGPT-User":["partial","named",1,3],"OAI-SearchBot":["partial","named",1,3],"ClaudeBot":["partial","named",1,3],"Claude-User":["partial","*",2,1],"Claude-SearchBot":["partial","*",2,1],"anthropic-ai":["partial","named",1,3],"PerplexityBot":["partial","named",1,3],"Perplexity-User":["partial","*",2,1],"Google-Extended":["partial","named",1,3],"Applebot-Extended":["partial","named",1,3],"CCBot":["partial","named",4,12],"Bytespider":["partial","*",2,1],"Amazonbot":["partial","*",2,1],"meta-externalagent":["partial","named",1,3],"cohere-ai":["partial","named",1,3],"Diffbot":["partial","*",2,1],"ContextFetchBot":["partial","*",2,1]},"llms":{"status":"missing","http":404,"bytes":82416},"home":{"status":"ok","http":200,"url":"https://www.aerzteblatt.de/","html_tokens":117030,"markdown_tokens":36,"pruned_tokens":36}},
{"rank":4624,"domain":"smallpdf.com","origin":"https://smallpdf.com","robots":{"status":"ok","http":200,"url":"https://smallpdf.com/robots.txt","bytes":58,"groups":1},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"missing","http":404,"bytes":297159},"home":{"status":"ok","http":200,"url":"https://smallpdf.com/","html_tokens":97327,"markdown_tokens":189,"pruned_tokens":108}},
{"rank":4625,"domain":"bit.do","origin":"https://bit.do","robots":{"status":"timeout","http":null,"url":"https://bit.do/robots.txt","bytes":0,"groups":0,"error":"ConnectTimeout"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":4626,"domain":"1xbet.com","origin":"https://1xbet.com","robots":{"status":"ok","http":200,"url":"https://1xbet.com/robots.txt","bytes":592,"groups":1},"bots":{"GPTBot":["partial","*",22,0],"ChatGPT-User":["partial","*",22,0],"OAI-SearchBot":["partial","*",22,0],"ClaudeBot":["partial","*",22,0],"Claude-User":["partial","*",22,0],"Claude-SearchBot":["partial","*",22,0],"anthropic-ai":["partial","*",22,0],"PerplexityBot":["partial","*",22,0],"Perplexity-User":["partial","*",22,0],"Google-Extended":["partial","*",22,0],"Applebot-Extended":["partial","*",22,0],"CCBot":["partial","*",22,0],"Bytespider":["partial","*",22,0],"Amazonbot":["partial","*",22,0],"meta-externalagent":["partial","*",22,0],"cohere-ai":["partial","*",22,0],"Diffbot":["partial","*",22,0],"ContextFetchBot":["partial","*",22,0]},"llms":{"status":"missing","http":404,"bytes":146},"home":{"status":"ok","http":203,"url":"https://1xbet.com/en/block","html_tokens":643,"markdown_tokens":0,"pruned_tokens":0}},
{"rank":4627,"domain":"yourstory.com","origin":"https://yourstory.com","robots":{"status":"ok","http":200,"url":"https://yourstory.com/robots.txt","bytes":3735,"groups":33},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["partial","*",13,2],"OAI-SearchBot":["partial","*",13,2],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",13,2],"Claude-SearchBot":["partial","*",13,2],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["partial","*",13,2],"Perplexity-User":["partial","*",13,2],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",13,2]},"llms":{"status":"html","http":200,"bytes":97752},"home":{"status":"http_error","http":403,"url":"https://yourstory.com/"}},
{"rank":4628,"domain":"google.sk","origin":"https://google.sk","robots":{"status":"ok","http":200,"url":"https://www.google.sk/robots.txt","bytes":6624,"groups":4},"bots":{"GPTBot":["partial","*",169,64],"ChatGPT-User":["partial","*",169,64],"OAI-SearchBot":["partial","*",169,64],"ClaudeBot":["partial","*",169,64],"Claude-User":["partial","*",169,64],"Claude-SearchBot":["partial","*",169,64],"anthropic-ai":["partial","*",169,64],"PerplexityBot":["partial","*",169,64],"Perplexity-User":["partial","*",169,64],"Google-Extended":["partial","*",169,64],"Applebot-Extended":["partial","*",169,64],"CCBot":["partial","*",169,64],"Bytespider":["partial","*",169,64],"Amazonbot":["partial","*",169,64],"meta-externalagent":["partial","*",169,64],"cohere-ai":["partial","*",169,64],"Diffbot":["partial","*",169,64],"ContextFetchBot":["partial","*",169,64]},"llms":{"status":"missing","http":404,"bytes":1569},"home":{"status":"ok","http":200,"url":"https://www.google.com/","html_tokens":21341,"markdown_tokens":30,"pruned_tokens":30}},
{"rank":4629,"domain":"macleans.ca","origin":"https://macleans.ca","robots":{"status":"ok","http":200,"url":"https://macleans.ca/robots.txt","bytes":144,"groups":1},"bots":{"GPTBot":["partial","*",4,0],"ChatGPT-User":["partial","*",4,0],"OAI-SearchBot":["partial","*",4,0],"ClaudeBot":["partial","*",4,0],"Claude-User":["partial","*",4,0],"Claude-SearchBot":["partial","*",4,0],"anthropic-ai":["partial","*",4,0],"PerplexityBot":["partial","*",4,0],"Perplexity-User":["partial","*",4,0],"Google-Extended":["partial","*",4,0],"Applebot-Extended":["partial","*",4,0],"CCBot":["partial","*",4,0],"Bytespider":["partial","*",4,0],"Amazonbot":["partial","*",4,0],"meta-externalagent":["partial","*",4,0],"cohere-ai":["partial","*",4,0],"Diffbot":["partial","*",4,0],"ContextFetchBot":["partial","*",4,0]},"llms":{"status":"http_error","http":403,"bytes":4905},"home":{"status":"http_error","http":403,"url":"https://macleans.ca/"}},
{"rank":4630,"domain":"rotary.org","origin":"https://rotary.org","robots":{"status":"refused","http":429,"url":"https://rotary.org/robots.txt","bytes":34222,"groups":0,"error":"HTTP 429"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":4631,"domain":"rollcall.com","origin":"https://rollcall.com","robots":{"status":"ok","http":200,"url":"https://rollcall.com/robots.txt","bytes":430,"groups":6},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["partial","*",1,2],"OAI-SearchBot":["partial","*",1,2],"ClaudeBot":["partial","*",1,2],"Claude-User":["partial","*",1,2],"Claude-SearchBot":["partial","*",1,2],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["partial","*",1,2],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["partial","*",1,2],"CCBot":["partial","*",1,2],"Bytespider":["partial","*",1,2],"Amazonbot":["partial","*",1,2],"meta-externalagent":["partial","*",1,2],"cohere-ai":["blocked","named",1,0],"Diffbot":["partial","*",1,2],"ContextFetchBot":["partial","*",1,2]},"llms":{"status":"missing","http":404,"bytes":149611},"home":{"status":"ok","http":200,"url":"https://rollcall.com/","html_tokens":61124,"markdown_tokens":39,"pruned_tokens":39}},
{"rank":4632,"domain":"cntd.ru","origin":"https://cntd.ru","robots":{"status":"ok","http":200,"url":"https://cntd.ru/robots.txt","bytes":736,"groups":1},"bots":{"GPTBot":["partial","*",21,9],"ChatGPT-User":["partial","*",21,9],"OAI-SearchBot":["partial","*",21,9],"ClaudeBot":["partial","*",21,9],"Claude-User":["partial","*",21,9],"Claude-SearchBot":["partial","*",21,9],"anthropic-ai":["partial","*",21,9],"PerplexityBot":["partial","*",21,9],"Perplexity-User":["partial","*",21,9],"Google-Extended":["partial","*",21,9],"Applebot-Extended":["partial","*",21,9],"CCBot":["partial","*",21,9],"Bytespider":["partial","*",21,9],"Amazonbot":["partial","*",21,9],"meta-externalagent":["partial","*",21,9],"cohere-ai":["partial","*",21,9],"Diffbot":["partial","*",21,9],"ContextFetchBot":["partial","*",21,9]},"llms":{"status":"missing","http":404,"bytes":10943},"home":{"status":"ok","http":200,"url":"https://cntd.ru/","html_tokens":12129,"markdown_tokens":71,"pruned_tokens":64}},
{"rank":4633,"domain":"philstar.com","origin":"https://philstar.com","robots":{"status":"ok","http":200,"url":"https://www.philstar.com/robots.txt","bytes":226,"groups":1},"bots":{"GPTBot":["partial","*",8,0],"ChatGPT-User":["partial","*",8,0],"OAI-SearchBot":["partial","*",8,0],"ClaudeBot":["partial","*",8,0],"Claude-User":["partial","*",8,0],"Claude-SearchBot":["partial","*",8,0],"anthropic-ai":["partial","*",8,0],"PerplexityBot":["partial","*",8,0],"Perplexity-User":["partial","*",8,0],"Google-Extended":["partial","*",8,0],"Applebot-Extended":["partial","*",8,0],"CCBot":["partial","*",8,0],"Bytespider":["partial","*",8,0],"Amazonbot":["partial","*",8,0],"meta-externalagent":["partial","*",8,0],"cohere-ai":["partial","*",8,0],"Diffbot":["partial","*",8,0],"ContextFetchBot":["partial","*",8,0]},"llms":{"status":"missing","http":404,"bytes":50329},"home":{"status":"ok","http":200,"url":"https://www.philstar.com/","html_tokens":67418,"markdown_tokens":69,"pruned_tokens":61}},
{"rank":4634,"domain":"aau.dk","origin":"https://aau.dk","robots":{"status":"ok","http":200,"url":"https://www.aau.dk/robots.txt","bytes":22,"groups":1},"bots":{"GPTBot":["allowed","*",0,1],"ChatGPT-User":["allowed","*",0,1],"OAI-SearchBot":["allowed","*",0,1],"ClaudeBot":["allowed","*",0,1],"Claude-User":["allowed","*",0,1],"Claude-SearchBot":["allowed","*",0,1],"anthropic-ai":["allowed","*",0,1],"PerplexityBot":["allowed","*",0,1],"Perplexity-User":["allowed","*",0,1],"Google-Extended":["allowed","*",0,1],"Applebot-Extended":["allowed","*",0,1],"CCBot":["allowed","*",0,1],"Bytespider":["allowed","*",0,1],"Amazonbot":["allowed","*",0,1],"meta-externalagent":["allowed","*",0,1],"cohere-ai":["allowed","*",0,1],"Diffbot":["allowed","*",0,1],"ContextFetchBot":["allowed","*",0,1]},"llms":{"status":"missing","http":404,"bytes":2888},"home":{"status":"ok","http":200,"url":"https://www.aau.dk/","html_tokens":67578,"markdown_tokens":56,"pruned_tokens":56}},
{"rank":4635,"domain":"pbskids.org","origin":"https://pbskids.org","robots":{"status":"ok","http":200,"url":"https://pbskids.org/robots.txt","bytes":1853,"groups":2},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["partial","*",3,0],"OAI-SearchBot":["partial","*",3,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",3,0],"Claude-SearchBot":["partial","*",3,0],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["partial","*",3,0],"Perplexity-User":["partial","*",3,0],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["partial","*",3,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",3,0]},"llms":{"status":"missing","http":404,"bytes":20682},"home":{"status":"ok","http":200,"url":"https://pbskids.org/","html_tokens":27348,"markdown_tokens":58,"pruned_tokens":58}},
{"rank":4636,"domain":"bravenet.com","origin":"https://bravenet.com","robots":{"status":"ok","http":200,"url":"https://bravenet.com/robots.txt","bytes":592,"groups":7},"bots":{"GPTBot":["partial","*",3,1],"ChatGPT-User":["partial","*",3,1],"OAI-SearchBot":["partial","*",3,1],"ClaudeBot":["partial","*",3,1],"Claude-User":["partial","*",3,1],"Claude-SearchBot":["partial","*",3,1],"anthropic-ai":["partial","*",3,1],"PerplexityBot":["partial","*",3,1],"Perplexity-User":["partial","*",3,1],"Google-Extended":["partial","*",3,1],"Applebot-Extended":["partial","*",3,1],"CCBot":["partial","*",3,1],"Bytespider":["partial","*",3,1],"Amazonbot":["partial","*",3,1],"meta-externalagent":["partial","*",3,1],"cohere-ai":["partial","*",3,1],"Diffbot":["partial","*",3,1],"ContextFetchBot":["partial","*",3,1]},"llms":{"status":"missing","http":404,"bytes":8839},"home":{"status":"ok","http":200,"url":"https://bravenet.com/","html_tokens":11702,"markdown_tokens":160,"pruned_tokens":20}},
{"rank":4637,"domain":"openreview.net","origin":"https://openreview.net","robots":{"status":"ok","http":200,"url":"https://openreview.net/robots.txt","bytes":35,"groups":1},"bots":{"GPTBot":["partial","*",1,0],"ChatGPT-User":["partial","*",1,0],"OAI-SearchBot":["partial","*",1,0],"ClaudeBot":["partial","*",1,0],"Claude-User":["partial","*",1,0],"Claude-SearchBot":["partial","*",1,0],"anthropic-ai":["partial","*",1,0],"PerplexityBot":["partial","*",1,0],"Perplexity-User":["partial","*",1,0],"Google-Extended":["partial","*",1,0],"Applebot-Extended":["partial","*",1,0],"CCBot":["partial","*",1,0],"Bytespider":["partial","*",1,0],"Amazonbot":["partial","*",1,0],"meta-externalagent":["partial","*",1,0],"cohere-ai":["partial","*",1,0],"Diffbot":["partial","*",1,0],"ContextFetchBot":["partial","*",1,0]},"llms":{"status":"missing","http":404,"bytes":11910},"home":{"status":"ok","http":200,"url":"https://openreview.net/","html_tokens":234255,"markdown_tokens":37,"pruned_tokens":36}},
{"rank":4638,"domain":"bosch.com","origin":"https://bosch.com","robots":{"status":"ok","http":200,"url":"https://www.bosch.com/robots.txt","bytes":78,"groups":1},"bots":{"GPTBot":["partial","*",1,0],"ChatGPT-User":["partial","*",1,0],"OAI-SearchBot":["partial","*",1,0],"ClaudeBot":["partial","*",1,0],"Claude-User":["partial","*",1,0],"Claude-SearchBot":["partial","*",1,0],"anthropic-ai":["partial","*",1,0],"PerplexityBot":["partial","*",1,0],"Perplexity-User":["partial","*",1,0],"Google-Extended":["partial","*",1,0],"Applebot-Extended":["partial","*",1,0],"CCBot":["partial","*",1,0],"Bytespider":["partial","*",1,0],"Amazonbot":["partial","*",1,0],"meta-externalagent":["partial","*",1,0],"cohere-ai":["partial","*",1,0],"Diffbot":["partial","*",1,0],"ContextFetchBot":["partial","*",1,0]},"llms":{"status":"missing","http":404,"bytes":94681},"home":{"status":"ok","http":200,"url":"https://www.bosch.com/","html_tokens":37242,"markdown_tokens":57,"pruned_tokens":57}},
{"rank":4639,"domain":"bybit.com","origin":"https://bybit.com","robots":{"status":"unreachable","http":null,"url":"https://bybit.com/robots.txt","bytes":0,"groups":0,"error":"<StreamReset stream_id:1, error_code:2, remote_reset:True>"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":4640,"domain":"outdooractive.com","origin":"https://outdooractive.com","robots":{"status":"ok","http":200,"url":"https://www.outdooractive.com/robots.txt","bytes":23111,"groups":651},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["partial","*",20,3],"OAI-SearchBot":["partial","*",20,3],"ClaudeBot":["partial","*",20,3],"Claude-User":["partial","*",20,3],"Claude-SearchBot":["partial","*",20,3],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["partial","*",20,3],"Perplexity-User":["partial","*",20,3],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["partial","*",20,3],"meta-externalagent":["partial","*",20,3],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",20,3]},"llms":{"status":"missing","http":404,"bytes":42772},"home":{"status":"ok","http":200,"url":"https://www.outdooractive.com/en/","html_tokens":32294,"markdown_tokens":392,"pruned_tokens":146}},
{"rank":4641,"domain":"hpage.com","origin":"https://hpage.com","robots":{"status":"ok","http":200,"url":"https://www.hpage.com/robots.txt","bytes":23,"groups":1},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"missing","http":404,"bytes":2229},"home":{"status":"ok","http":200,"url":"https://www.hpage.com/","html_tokens":9561,"markdown_tokens":1095,"pruned_tokens":310}},
{"rank":4642,"domain":"einnews.com","origin":"https://einnews.com","robots":{"status":"ok","http":200,"url":"https://www.einnews.com/robots.txt","bytes":1523,"groups":4},"bots":{"GPTBot":["partial","*",16,0],"ChatGPT-User":["partial","*",16,0],"OAI-SearchBot":["partial","*",16,0],"ClaudeBot":["partial","*",16,0],"Claude-User":["partial","*",16,0],"Claude-SearchBot":["partial","*",16,0],"anthropic-ai":["partial","*",16,0],"PerplexityBot":["partial","*",16,0],"Perplexity-User":["partial","*",16,0],"Google-Extended":["partial","*",16,0],"Applebot-Extended":["partial","*",16,0],"CCBot":["partial","*",16,0],"Bytespider":["partial","*",16,0],"Amazonbot":["partial","*",16,0],"meta-externalagent":["partial","*",16,0],"cohere-ai":["partial","*",16,0],"Diffbot":["partial","*",16,0],"ContextFetchBot":["partial","*",16,0]},"llms":{"status":"crawl_delay","error":"Crawl-delay 300 s"},"home":{"status":"crawl_delay","error":"Crawl-delay 300 s"}},
{"rank":4643,"domain":"ine.es","origin":"https://ine.es","robots":{"status":"ok","http":200,"url":"https://ine.es/robots.txt","bytes":251,"groups":1},"bots":{"GPTBot":["partial","*",5,0],"ChatGPT-User":["partial","*",5,0],"OAI-SearchBot":["partial","*",5,0],"ClaudeBot":["partial","*",5,0],"Claude-User":["partial","*",5,0],"Claude-SearchBot":["partial","*",5,0],"anthropic-ai":["partial","*",5,0],"PerplexityBot":["partial","*",5,0],"Perplexity-User":["partial","*",5,0],"Google-Extended":["partial","*",5,0],"Applebot-Extended":["partial","*",5,0],"CCBot":["partial","*",5,0],"Bytespider":["partial","*",5,0],"Amazonbot":["partial","*",5,0],"meta-externalagent":["partial","*",5,0],"cohere-ai":["partial","*",5,0],"Diffbot":["partial","*",5,0],"ContextFetchBot":["partial","*",5,0]},"llms":{"status":"missing","http":404,"bytes":19778},"home":{"status":"ok","http":200,"url":"https://ine.es/","html_tokens":18614,"markdown_tokens":88,"pruned_tokens":88}},
{"rank":4644,"domain":"issn.org","origin":"https://issn.org","robots":{"status":"ok","http":200,"url":"https://www.issn.org/robots.txt","bytes":153,"groups":1},"bots":{"GPTBot":["partial","*",1,1],"ChatGPT-User":["partial","*",1,1],"OAI-SearchBot":["partial","*",1,1],"ClaudeBot":["partial","*",1,1],"Claude-User":["partial","*",1,1],"Claude-SearchBot":["partial","*",1,1],"anthropic-ai":["partial","*",1,1],"PerplexityBot":["partial","*",1,1],"Perplexity-User":["partial","*",1,1],"Google-Extended":["partial","*",1,1],"Applebot-Extended":["partial","*",1,1],"CCBot":["partial","*",1,1],"Bytespider":["partial","*",1,1],"Amazonbot":["partial","*",1,1],"meta-externalagent":["partial","*",1,1],"cohere-ai":["partial","*",1,1],"Diffbot":["partial","*",1,1],"ContextFetchBot":["partial","*",1,1]},"llms":{"status":"missing","http":404,"bytes":90998},"home":{"status":"ok","http":200,"url":"https://www.issn.org/","html_tokens":22692,"markdown_tokens":259,"pruned_tokens":133}},
{"rank":4645,"domain":"meituan.com","origin":"https://www.meituan.com","robots":{"status":"missing","http":404,"url":"https://www.meituan.com/error/404","bytes":98799,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":98799},"home":{"status":"ok","http":200,"url":"https://www.meituan.com/","html_tokens":29485,"markdown_tokens":39,"pruned_tokens":39}},
{"rank":4646,"domain":"fourmilab.ch","origin":"https://fourmilab.ch","robots":{"status":"ok","http":200,"url":"https://fourmilab.ch/robots.txt","bytes":2500,"groups":10},"bots":{"GPTBot":["partial","*",19,0],"ChatGPT-User":["partial","*",19,0],"OAI-SearchBot":["partial","*",19,0],"ClaudeBot":["partial","*",19,0],"Claude-User":["partial","*",19,0],"Claude-SearchBot":["partial","*",19,0],"anthropic-ai":["partial","*",19,0],"PerplexityBot":["partial","*",19,0],"Perplexity-User":["partial","*",19,0],"Google-Extended":["partial","*",19,0],"Applebot-Extended":["partial","*",19,0],"CCBot":["partial","*",19,0],"Bytespider":["partial","*",19,0],"Amazonbot":["partial","*",19,0],"meta-externalagent":["partial","*",19,0],"cohere-ai":["partial","*",19,0],"Diffbot":["partial","*",19,0],"ContextFetchBot":["partial","*",19,0]},"llms":{"status":"missing","http":404,"bytes":236},"home":{"status":"ok","http":200,"url":"https://fourmilab.ch/","html_tokens":428,"markdown_tokens":0,"pruned_tokens":0}},
{"rank":4647,"domain":"discuz.net","origin":"https://discuz.net","robots":{"status":"unreachable","http":null,"url":"https://discuz.net/robots.txt","bytes":0,"groups":0,"error":"discuz.net is not a public address"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":4648,"domain":"blog.jp","origin":"https://blog.jp","robots":{"status":"missing","http":404,"url":"https://blog.jp/robots.txt","bytes":2178,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":2178},"home":{"status":"ok","http":200,"url":"https://blog.jp/","html_tokens":2649,"markdown_tokens":340,"pruned_tokens":67}},
{"rank":4649,"domain":"dn.se","origin":"https://dn.se","robots":{"status":"ok","http":200,"url":"https://www.dn.se/robots.txt","bytes":5181,"groups":33},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["blocked","named",1,0],"OAI-SearchBot":["blocked","named",1,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["blocked","named",1,0],"Claude-SearchBot":["blocked","named",1,0],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["blocked","named",1,0],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["partial","*",25,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",25,0]},"llms":{"status":"missing","http":404,"bytes":9},"home":{"status":"ok","http":200,"url":"https://www.dn.se/","html_tokens":127321,"markdown_tokens":27,"pruned_tokens":27}},
{"rank":4650,"domain":"sony.net","origin":"https://sony.net","robots":{"status":"ok","http":200,"url":"https://www.sony.co.jp/en/robots.txt","bytes":448,"groups":7},"bots":{"GPTBot":["partial","*",2,0],"ChatGPT-User":["partial","*",2,0],"OAI-SearchBot":["partial","*",2,0],"ClaudeBot":["partial","*",2,0],"Claude-User":["partial","*",2,0],"Claude-SearchBot":["partial","*",2,0],"anthropic-ai":["partial","*",2,0],"PerplexityBot":["partial","*",2,0],"Perplexity-User":["partial","*",2,0],"Google-Extended":["partial","*",2,0],"Applebot-Extended":["partial","*",2,0],"CCBot":["partial","*",2,0],"Bytespider":["partial","*",2,0],"Amazonbot":["partial","*",2,0],"meta-externalagent":["partial","*",2,0],"cohere-ai":["partial","*",2,0],"Diffbot":["partial","*",2,0],"ContextFetchBot":["partial","*",2,0]},"llms":{"status":"missing","http":404,"bytes":155897},"home":{"status":"ok","http":200,"url":"https://www.sony.co.jp/en/","html_tokens":60563,"markdown_tokens":27,"pruned_tokens":27}},
{"rank":4651,"domain":"inquisitr.com","origin":"https://inquisitr.com","robots":{"status":"ok","http":200,"url":"https://www.inquisitr.com/robots.txt","bytes":622,"groups":1},"bots":{"GPTBot":["partial","*",14,1],"ChatGPT-User":["partial","*",14,1],"OAI-SearchBot":["partial","*",14,1],"ClaudeBot":["partial","*",14,1],"Claude-User":["partial","*",14,1],"Claude-SearchBot":["partial","*",14,1],"anthropic-ai":["partial","*",14,1],"PerplexityBot":["partial","*",14,1],"Perplexity-User":["partial","*",14,1],"Google-Extended":["partial","*",14,1],"Applebot-Extended":["partial","*",14,1],"CCBot":["partial","*",14,1],"Bytespider":["partial","*",14,1],"Amazonbot":["partial","*",14,1],"meta-externalagent":["partial","*",14,1],"cohere-ai":["partial","*",14,1],"Diffbot":["partial","*",14,1],"ContextFetchBot":["partial","*",14,1]},"llms":{"status":"missing","http":404,"bytes":24755},"home":{"status":"ok","http":200,"url":"https://www.inquisitr.com/","html_tokens":28963,"markdown_tokens":30,"pruned_tokens":30}},
{"rank":4652,"domain":"pfsense.org","origin":"https://pfsense.org","robots":{"status":"ok","http":200,"url":"https://pfsense.org/robots.txt","bytes":26,"groups":1},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"missing","http":404,"bytes":19295},"home":{"status":"ok","http":200,"url":"https://pfsense.org/","html_tokens":6637,"markdown_tokens":129,"pruned_tokens":129}},
{"rank":4653,"domain":"vic.edu.au","origin":"https://vic.edu.au","robots":{"status":"unreachable","http":null,"url":"https://vic.edu.au/robots.txt","bytes":0,"groups":0,"error":"cannot resolve vic.edu.au"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":4654,"domain":"tuv.com","origin":"https://tuv.com","robots":{"status":"ok","http":200,"url":"https://www.tuv.com/robots.txt","bytes":10275,"groups":1},"bots":{"GPTBot":["partial","*",240,1],"ChatGPT-User":["partial","*",240,1],"OAI-SearchBot":["partial","*",240,1],"ClaudeBot":["partial","*",240,1],"Claude-User":["partial","*",240,1],"Claude-SearchBot":["partial","*",240,1],"anthropic-ai":["partial","*",240,1],"PerplexityBot":["partial","*",240,1],"Perplexity-User":["partial","*",240,1],"Google-Extended":["partial","*",240,1],"Applebot-Extended":["partial","*",240,1],"CCBot":["partial","*",240,1],"Bytespider":["partial","*",240,1],"Amazonbot":["partial","*",240,1],"meta-externalagent":["partial","*",240,1],"cohere-ai":["partial","*",240,1],"Diffbot":["partial","*",240,1],"ContextFetchBot":["partial","*",240,1]},"llms":{"status":"missing","http":404,"bytes":142097},"home":{"status":"ok","http":200,"url":"https://www.tuv.com/usa/en/","html_tokens":49703,"markdown_tokens":72,"pruned_tokens":72}},
{"rank":4655,"domain":"tbank.ru","origin":"https://tbank.ru","robots":{"status":"unreachable","http":null,"url":"https://tbank.ru/robots.txt","bytes":0,"groups":0,"error":"[SSL: CERTIFICATE_VERIFY_FAILED] certificate verify failed: self-signed certificate in certificate chain (_ssl.c:1028)"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":4656,"domain":"allevents.in","origin":"https://allevents.in","robots":{"status":"ok","http":200,"url":"https://allevents.in/robots.txt","bytes":4146,"groups":1},"bots":{"GPTBot":["partial","*",107,22],"ChatGPT-User":["partial","*",107,22],"OAI-SearchBot":["partial","*",107,22],"ClaudeBot":["partial","named",107,22],"Claude-User":["partial","*",107,22],"Claude-SearchBot":["partial","*",107,22],"anthropic-ai":["partial","*",107,22],"PerplexityBot":["partial","*",107,22],"Perplexity-User":["partial","*",107,22],"Google-Extended":["partial","*",107,22],"Applebot-Extended":["partial","*",107,22],"CCBot":["partial","*",107,22],"Bytespider":["partial","*",107,22],"Amazonbot":["partial","*",107,22],"meta-externalagent":["partial","*",107,22],"cohere-ai":["partial","*",107,22],"Diffbot":["partial","*",107,22],"ContextFetchBot":["partial","*",107,22]},"llms":{"status":"present","http":200,"bytes":10181},"home":{"status":"ok","http":200,"url":"https://allevents.in/","html_tokens":96609,"markdown_tokens":29,"pruned_tokens":29}},
{"rank":4657,"domain":"dnaindia.com","origin":"https://dnaindia.com","robots":{"status":"ok","http":200,"url":"https://www.dnaindia.com/robots.txt","bytes":1587,"groups":18},"bots":{"GPTBot":["allowed","named",0,1],"ChatGPT-User":["allowed","named",0,1],"OAI-SearchBot":["partial","*",4,1],"ClaudeBot":["allowed","named",0,1],"Claude-User":["partial","*",4,1],"Claude-SearchBot":["partial","*",4,1],"anthropic-ai":["partial","*",4,1],"PerplexityBot":["allowed","named",0,1],"Perplexity-User":["allowed","named",0,1],"Google-Extended":["allowed","named",0,2],"Applebot-Extended":["partial","*",4,1],"CCBot":["partial","*",4,1],"Bytespider":["blocked","named",1,0],"Amazonbot":["partial","*",4,1],"meta-externalagent":["partial","*",4,1],"cohere-ai":["partial","*",4,1],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",4,1]},"llms":{"status":"missing","http":404,"bytes":58299},"home":{"status":"ok","http":200,"url":"https://www.dnaindia.com/","html_tokens":257175,"markdown_tokens":352,"pruned_tokens":156}},
{"rank":4658,"domain":"dedecms.com","origin":"https://dedecms.com","robots":{"status":"html","http":200,"url":"https://dedecms.com/","bytes":19593,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":4659,"domain":"www.gov.ie","origin":"https://www.gov.ie","robots":{"status":"refused","http":403,"url":"https://www.gov.ie/robots.txt","bytes":919,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":4660,"domain":"espacenet.com","origin":"https://espacenet.com","robots":{"status":"refused","http":403,"url":"https://espacenet.com/robots.txt","bytes":1,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":4661,"domain":"ccgp.gov.cn","origin":"https://www.ccgp.gov.cn","robots":{"status":"missing","http":404,"url":"https://www.ccgp.gov.cn/robots.txt","bytes":3134,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":3134},"home":{"status":"ok","http":200,"url":"https://www.ccgp.gov.cn/","html_tokens":13492,"markdown_tokens":7,"pruned_tokens":7}},
{"rank":4662,"domain":"otter.ai","origin":"https://otter.ai","robots":{"status":"ok","http":200,"url":"https://otter.ai/robots.txt","bytes":226,"groups":1},"bots":{"GPTBot":["partial","*",9,1],"ChatGPT-User":["partial","*",9,1],"OAI-SearchBot":["partial","*",9,1],"ClaudeBot":["partial","*",9,1],"Claude-User":["partial","*",9,1],"Claude-SearchBot":["partial","*",9,1],"anthropic-ai":["partial","*",9,1],"PerplexityBot":["partial","*",9,1],"Perplexity-User":["partial","*",9,1],"Google-Extended":["partial","*",9,1],"Applebot-Extended":["partial","*",9,1],"CCBot":["partial","*",9,1],"Bytespider":["partial","*",9,1],"Amazonbot":["partial","*",9,1],"meta-externalagent":["partial","*",9,1],"cohere-ai":["partial","*",9,1],"Diffbot":["partial","*",9,1],"ContextFetchBot":["partial","*",9,1]},"llms":{"status":"html","http":200,"bytes":589798},"home":{"status":"ok","http":200,"url":"https://otter.ai/","html_tokens":54066,"markdown_tokens":799,"pruned_tokens":415}},
{"rank":4663,"domain":"micron.com","origin":"https://micron.com","robots":{"status":"ok","http":200,"url":"https://www.micron.com/robots.txt","bytes":243,"groups":1},"bots":{"GPTBot":["partial","*",5,0],"ChatGPT-User":["partial","*",5,0],"OAI-SearchBot":["partial","*",5,0],"ClaudeBot":["partial","*",5,0],"Claude-User":["partial","*",5,0],"Claude-SearchBot":["partial","*",5,0],"anthropic-ai":["partial","*",5,0],"PerplexityBot":["partial","*",5,0],"Perplexity-User":["partial","*",5,0],"Google-Extended":["partial","*",5,0],"Applebot-Extended":["partial","*",5,0],"CCBot":["partial","*",5,0],"Bytespider":["partial","*",5,0],"Amazonbot":["partial","*",5,0],"meta-externalagent":["partial","*",5,0],"cohere-ai":["partial","*",5,0],"Diffbot":["partial","*",5,0],"ContextFetchBot":["partial","*",5,0]},"llms":{"status":"missing","http":404,"bytes":277028},"home":{"status":"ok","http":200,"url":"https://www.micron.com/","html_tokens":79112,"markdown_tokens":96,"pruned_tokens":92}},
{"rank":4664,"domain":"mercari.com","origin":"https://mercari.com","robots":{"status":"ok","http":200,"url":"https://www.mercari.com/robots.txt","bytes":1973,"groups":7},"bots":{"GPTBot":["partial","*",35,0],"ChatGPT-User":["partial","*",35,0],"OAI-SearchBot":["partial","*",35,0],"ClaudeBot":["partial","*",35,0],"Claude-User":["partial","*",35,0],"Claude-SearchBot":["partial","*",35,0],"anthropic-ai":["partial","*",35,0],"PerplexityBot":["partial","*",35,0],"Perplexity-User":["partial","*",35,0],"Google-Extended":["partial","*",35,0],"Applebot-Extended":["partial","*",35,0],"CCBot":["partial","*",35,0],"Bytespider":["partial","*",35,0],"Amazonbot":["partial","*",35,0],"meta-externalagent":["partial","*",35,0],"cohere-ai":["partial","*",35,0],"Diffbot":["partial","*",35,0],"ContextFetchBot":["partial","*",35,0]},"llms":{"status":"missing","http":404,"bytes":19344},"home":{"status":"http_error","http":403,"url":"https://www.mercari.com/"}},
{"rank":4665,"domain":"htc.com","origin":"https://htc.com","robots":{"status":"html","http":200,"url":"https://www.htc.com/us/","bytes":55377,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":4666,"domain":"xvideos.com","origin":"https://xvideos.com","robots":{"status":"ok","http":200,"url":"https://xvideos.com/robots.txt","bytes":61,"groups":1},"bots":{"GPTBot":["partial","*",2,0],"ChatGPT-User":["partial","*",2,0],"OAI-SearchBot":["partial","*",2,0],"ClaudeBot":["partial","*",2,0],"Claude-User":["partial","*",2,0],"Claude-SearchBot":["partial","*",2,0],"anthropic-ai":["partial","*",2,0],"PerplexityBot":["partial","*",2,0],"Perplexity-User":["partial","*",2,0],"Google-Extended":["partial","*",2,0],"Applebot-Extended":["partial","*",2,0],"CCBot":["partial","*",2,0],"Bytespider":["partial","*",2,0],"Amazonbot":["partial","*",2,0],"meta-externalagent":["partial","*",2,0],"cohere-ai":["partial","*",2,0],"Diffbot":["partial","*",2,0],"ContextFetchBot":["partial","*",2,0]},"llms":{"status":"missing","http":404,"bytes":38506},"home":{"status":"ok","http":200,"url":"https://www.xvideos.com/","html_tokens":32719,"markdown_tokens":14,"pruned_tokens":14}},
{"rank":4667,"domain":"wenthemes.com","origin":"https://wenthemes.com","robots":{"status":"ok","http":200,"url":"https://wenthemes.com/robots.txt","bytes":441,"groups":2},"bots":{"GPTBot":["partial","*",6,1],"ChatGPT-User":["partial","*",6,1],"OAI-SearchBot":["partial","*",6,1],"ClaudeBot":["partial","*",6,1],"Claude-User":["partial","*",6,1],"Claude-SearchBot":["partial","*",6,1],"anthropic-ai":["partial","*",6,1],"PerplexityBot":["partial","*",6,1],"Perplexity-User":["partial","*",6,1],"Google-Extended":["partial","*",6,1],"Applebot-Extended":["partial","*",6,1],"CCBot":["partial","*",6,1],"Bytespider":["partial","*",6,1],"Amazonbot":["partial","*",6,1],"meta-externalagent":["partial","*",6,1],"cohere-ai":["partial","*",6,1],"Diffbot":["partial","*",6,1],"ContextFetchBot":["partial","*",6,1]},"llms":{"status":"missing","http":404,"bytes":35566},"home":{"status":"ok","http":200,"url":"https://wenthemes.com/","html_tokens":13373,"markdown_tokens":171,"pruned_tokens":171}},
{"rank":4668,"domain":"swarthmore.edu","origin":"https://swarthmore.edu","robots":{"status":"ok","http":200,"url":"https://www.swarthmore.edu/robots.txt","bytes":2141,"groups":1},"bots":{"GPTBot":["partial","*",35,22],"ChatGPT-User":["partial","*",35,22],"OAI-SearchBot":["partial","*",35,22],"ClaudeBot":["partial","*",35,22],"Claude-User":["partial","*",35,22],"Claude-SearchBot":["partial","*",35,22],"anthropic-ai":["partial","*",35,22],"PerplexityBot":["partial","*",35,22],"Perplexity-User":["partial","*",35,22],"Google-Extended":["partial","*",35,22],"Applebot-Extended":["partial","*",35,22],"CCBot":["partial","*",35,22],"Bytespider":["partial","*",35,22],"Amazonbot":["partial","*",35,22],"meta-externalagent":["partial","*",35,22],"cohere-ai":["partial","*",35,22],"Diffbot":["partial","*",35,22],"ContextFetchBot":["partial","*",35,22]},"llms":{"status":"missing","http":404,"bytes":173298},"home":{"status":"ok","http":200,"url":"https://www.swarthmore.edu/","html_tokens":51780,"markdown_tokens":57,"pruned_tokens":57}},
{"rank":4669,"domain":"peopleapp.com","origin":"https://www.peopleapp.com","robots":{"status":"html","http":200,"url":"https://www.peopleapp.com/home","bytes":146142,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":4670,"domain":"seznam.cz","origin":"https://seznam.cz","robots":{"status":"ok","http":200,"url":"https://www.seznam.cz/robots.txt","bytes":1024,"groups":4},"bots":{"GPTBot":["partial","*",11,0],"ChatGPT-User":["partial","*",11,0],"OAI-SearchBot":["partial","*",11,0],"ClaudeBot":["partial","*",11,0],"Claude-User":["partial","*",11,0],"Claude-SearchBot":["partial","*",11,0],"anthropic-ai":["partial","*",11,0],"PerplexityBot":["partial","*",11,0],"Perplexity-User":["partial","*",11,0],"Google-Extended":["partial","*",11,0],"Applebot-Extended":["partial","*",11,0],"CCBot":["partial","*",11,0],"Bytespider":["partial","*",11,0],"Amazonbot":["partial","*",11,0],"meta-externalagent":["partial","*",11,0],"cohere-ai":["partial","*",11,0],"Diffbot":["partial","*",11,0],"ContextFetchBot":["partial","*",11,0]},"llms":{"status":"missing","http":404,"bytes":10030},"home":{"status":"disallowed","error":"robots.txt of login.seznam.cz disallows /api/v1/autologin?service=homepage&return_url=https%3A%2F%2Fwww.seznam.cz%2F%3Fnoredirect%3D1"}},
{"rank":4671,"domain":"homeip.net","origin":"http://homeip.net","robots":{"status":"refused","http":403,"url":"https://www.oracle.com/cloud/networking/dns/","bytes":1450,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":4672,"domain":"supabase.com","origin":"https://supabase.com","robots":{"status":"ok","http":200,"url":"https://supabase.com/robots.txt","bytes":226,"groups":2},"bots":{"GPTBot":["allowed","named",0,1],"ChatGPT-User":["allowed","*",0,1],"OAI-SearchBot":["allowed","*",0,1],"ClaudeBot":["allowed","named",0,1],"Claude-User":["allowed","*",0,1],"Claude-SearchBot":["allowed","*",0,1],"anthropic-ai":["allowed","*",0,1],"PerplexityBot":["allowed","named",0,1],"Perplexity-User":["allowed","*",0,1],"Google-Extended":["allowed","named",0,1],"Applebot-Extended":["allowed","*",0,1],"CCBot":["allowed","*",0,1],"Bytespider":["allowed","*",0,1],"Amazonbot":["allowed","*",0,1],"meta-externalagent":["allowed","*",0,1],"cohere-ai":["allowed","*",0,1],"Diffbot":["allowed","*",0,1],"ContextFetchBot":["allowed","*",0,1]},"llms":{"status":"present","http":200,"bytes":3242},"home":{"status":"ok","http":200,"url":"https://supabase.com/","html_tokens":335107,"markdown_tokens":367,"pruned_tokens":71}},
{"rank":4673,"domain":"aaai.org","origin":"https://aaai.org","robots":{"status":"ok","http":200,"url":"https://aaai.org/robots.txt","bytes":229,"groups":1},"bots":{"GPTBot":["partial","*",2,0],"ChatGPT-User":["partial","*",2,0],"OAI-SearchBot":["partial","*",2,0],"ClaudeBot":["partial","*",2,0],"Claude-User":["partial","*",2,0],"Claude-SearchBot":["partial","*",2,0],"anthropic-ai":["partial","*",2,0],"PerplexityBot":["partial","*",2,0],"Perplexity-User":["partial","*",2,0],"Google-Extended":["partial","*",2,0],"Applebot-Extended":["partial","*",2,0],"CCBot":["partial","*",2,0],"Bytespider":["partial","*",2,0],"Amazonbot":["partial","*",2,0],"meta-externalagent":["partial","*",2,0],"cohere-ai":["partial","*",2,0],"Diffbot":["partial","*",2,0],"ContextFetchBot":["partial","*",2,0]},"llms":{"status":"crawl_delay","error":"Crawl-delay 43200 s"},"home":{"status":"crawl_delay","error":"Crawl-delay 43200 s"}},
{"rank":4674,"domain":"semafor.com","origin":"https://semafor.com","robots":{"status":"ok","http":200,"url":"https://www.semafor.com/robots.txt","bytes":233,"groups":2},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["partial","*",4,1],"OAI-SearchBot":["partial","*",4,1],"ClaudeBot":["partial","*",4,1],"Claude-User":["partial","*",4,1],"Claude-SearchBot":["partial","*",4,1],"anthropic-ai":["partial","*",4,1],"PerplexityBot":["partial","*",4,1],"Perplexity-User":["partial","*",4,1],"Google-Extended":["partial","*",4,1],"Applebot-Extended":["partial","*",4,1],"CCBot":["partial","*",4,1],"Bytespider":["partial","*",4,1],"Amazonbot":["partial","*",4,1],"meta-externalagent":["partial","*",4,1],"cohere-ai":["partial","*",4,1],"Diffbot":["partial","*",4,1],"ContextFetchBot":["partial","*",4,1]},"llms":{"status":"missing","http":404,"bytes":2804},"home":{"status":"ok","http":200,"url":"https://www.semafor.com/","html_tokens":418784,"markdown_tokens":219,"pruned_tokens":160}},
{"rank":4675,"domain":"hyperallergic.com","origin":"https://hyperallergic.com","robots":{"status":"ok","http":200,"url":"https://hyperallergic.com/robots.txt","bytes":216,"groups":1},"bots":{"GPTBot":["partial","*",6,0],"ChatGPT-User":["partial","*",6,0],"OAI-SearchBot":["partial","*",6,0],"ClaudeBot":["partial","*",6,0],"Claude-User":["partial","*",6,0],"Claude-SearchBot":["partial","*",6,0],"anthropic-ai":["partial","*",6,0],"PerplexityBot":["partial","*",6,0],"Perplexity-User":["partial","*",6,0],"Google-Extended":["partial","*",6,0],"Applebot-Extended":["partial","*",6,0],"CCBot":["partial","*",6,0],"Bytespider":["partial","*",6,0],"Amazonbot":["partial","*",6,0],"meta-externalagent":["partial","*",6,0],"cohere-ai":["partial","*",6,0],"Diffbot":["partial","*",6,0],"ContextFetchBot":["partial","*",6,0]},"llms":{"status":"present","http":200,"bytes":51009},"home":{"status":"ok","http":200,"url":"https://hyperallergic.com/","html_tokens":24345,"markdown_tokens":777,"pruned_tokens":16}},
{"rank":4676,"domain":"adecco.com","origin":"https://adecco.com","robots":{"status":"missing","http":404,"url":"https://adecco.com/robots.txt","bytes":103,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"empty","http":200,"bytes":0},"home":{"status":"ok","http":200,"url":"https://www.adecco.com/","html_tokens":120723,"markdown_tokens":932,"pruned_tokens":162}},
{"rank":4677,"domain":"metamask.io","origin":"https://metamask.io","robots":{"status":"ok","http":200,"url":"https://metamask.io/robots.txt","bytes":221,"groups":1},"bots":{"GPTBot":["partial","*",3,2],"ChatGPT-User":["partial","*",3,2],"OAI-SearchBot":["partial","*",3,2],"ClaudeBot":["partial","*",3,2],"Claude-User":["partial","*",3,2],"Claude-SearchBot":["partial","*",3,2],"anthropic-ai":["partial","*",3,2],"PerplexityBot":["partial","*",3,2],"Perplexity-User":["partial","*",3,2],"Google-Extended":["partial","*",3,2],"Applebot-Extended":["partial","*",3,2],"CCBot":["partial","*",3,2],"Bytespider":["partial","*",3,2],"Amazonbot":["partial","*",3,2],"meta-externalagent":["partial","*",3,2],"cohere-ai":["partial","*",3,2],"Diffbot":["partial","*",3,2],"ContextFetchBot":["partial","*",3,2]},"llms":{"status":"present","http":200,"bytes":40802},"home":{"status":"ok","http":200,"url":"https://metamask.io/","html_tokens":113904,"markdown_tokens":35,"pruned_tokens":35}},
{"rank":4678,"domain":"jetro.go.jp","origin":"https://www.jetro.go.jp","robots":{"status":"refused","http":403,"url":"https://www.jetro.go.jp/robots.txt","bytes":111,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":4679,"domain":"walkscore.com","origin":"https://walkscore.com","robots":{"status":"ok","http":200,"url":"https://www.walkscore.com/robots.txt","bytes":719,"groups":1},"bots":{"GPTBot":["partial","*",19,0],"ChatGPT-User":["partial","*",19,0],"OAI-SearchBot":["partial","*",19,0],"ClaudeBot":["partial","*",19,0],"Claude-User":["partial","*",19,0],"Claude-SearchBot":["partial","*",19,0],"anthropic-ai":["partial","*",19,0],"PerplexityBot":["partial","*",19,0],"Perplexity-User":["partial","*",19,0],"Google-Extended":["partial","*",19,0],"Applebot-Extended":["partial","*",19,0],"CCBot":["partial","*",19,0],"Bytespider":["partial","*",19,0],"Amazonbot":["partial","*",19,0],"meta-externalagent":["partial","*",19,0],"cohere-ai":["partial","*",19,0],"Diffbot":["partial","*",19,0],"ContextFetchBot":["partial","*",19,0]},"llms":{"status":"missing","http":404,"bytes":19688},"home":{"status":"ok","http":200,"url":"https://www.walkscore.com/","html_tokens":13123,"markdown_tokens":93,"pruned_tokens":93}},
{"rank":4680,"domain":"cbr.com","origin":"https://cbr.com","robots":{"status":"ok","http":200,"url":"https://www.cbr.com/robots.txt","bytes":2027,"groups":2},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["partial","*",22,0],"OAI-SearchBot":["partial","*",22,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",22,0],"Claude-SearchBot":["partial","*",22,0],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["partial","*",22,0],"Perplexity-User":["partial","*",22,0],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",22,0]},"llms":{"status":"missing","http":404,"bytes":110846},"home":{"status":"ok","http":200,"url":"https://www.cbr.com/","html_tokens":311616,"markdown_tokens":119,"pruned_tokens":10}},
{"rank":4681,"domain":"google.com.eg","origin":"https://google.com.eg","robots":{"status":"ok","http":200,"url":"https://www.google.com.eg/robots.txt","bytes":6624,"groups":4},"bots":{"GPTBot":["partial","*",169,64],"ChatGPT-User":["partial","*",169,64],"OAI-SearchBot":["partial","*",169,64],"ClaudeBot":["partial","*",169,64],"Claude-User":["partial","*",169,64],"Claude-SearchBot":["partial","*",169,64],"anthropic-ai":["partial","*",169,64],"PerplexityBot":["partial","*",169,64],"Perplexity-User":["partial","*",169,64],"Google-Extended":["partial","*",169,64],"Applebot-Extended":["partial","*",169,64],"CCBot":["partial","*",169,64],"Bytespider":["partial","*",169,64],"Amazonbot":["partial","*",169,64],"meta-externalagent":["partial","*",169,64],"cohere-ai":["partial","*",169,64],"Diffbot":["partial","*",169,64],"ContextFetchBot":["partial","*",169,64]},"llms":{"status":"missing","http":404,"bytes":1569},"home":{"status":"ok","http":200,"url":"https://www.google.com/","html_tokens":21444,"markdown_tokens":30,"pruned_tokens":30}},
{"rank":4682,"domain":"ccb.com","origin":"http://ccb.com","robots":{"status":"missing","http":404,"url":"http://ccb.com/robots.txt","bytes":204,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":204},"home":{"status":"error","http":200,"url":"http://ccb.com/","error":"unparseable HTML"}},
{"rank":4683,"domain":"dukeupress.edu","origin":"https://dukeupress.edu","robots":{"status":"ok","http":200,"url":"https://dukeupress.edu/robots.txt","bytes":23473,"groups":656},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["blocked","named",1,0],"OAI-SearchBot":["allowed","*",0,3],"ClaudeBot":["allowed","*",0,3],"Claude-User":["allowed","*",0,3],"Claude-SearchBot":["allowed","*",0,3],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["allowed","*",0,3],"Perplexity-User":["allowed","*",0,3],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["allowed","*",0,3],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["allowed","*",0,3],"meta-externalagent":["allowed","*",0,3],"cohere-ai":["blocked","named",1,0],"Diffbot":["allowed","*",0,3],"ContextFetchBot":["allowed","*",0,3]},"llms":{"status":"missing","http":404,"bytes":126318},"home":{"status":"ok","http":200,"url":"https://dukeupress.edu/","html_tokens":55555,"markdown_tokens":85,"pruned_tokens":85}},
{"rank":4684,"domain":"ielts.org","origin":"https://ielts.org","robots":{"status":"ok","http":200,"url":"https://ielts.org/robots.txt","bytes":248,"groups":1},"bots":{"GPTBot":["partial","*",4,0],"ChatGPT-User":["partial","*",4,0],"OAI-SearchBot":["partial","*",4,0],"ClaudeBot":["partial","*",4,0],"Claude-User":["partial","*",4,0],"Claude-SearchBot":["partial","*",4,0],"anthropic-ai":["partial","*",4,0],"PerplexityBot":["partial","*",4,0],"Perplexity-User":["partial","*",4,0],"Google-Extended":["partial","*",4,0],"Applebot-Extended":["partial","*",4,0],"CCBot":["partial","*",4,0],"Bytespider":["partial","*",4,0],"Amazonbot":["partial","*",4,0],"meta-externalagent":["partial","*",4,0],"cohere-ai":["partial","*",4,0],"Diffbot":["partial","*",4,0],"ContextFetchBot":["partial","*",4,0]},"llms":{"status":"missing","http":404,"bytes":154114},"home":{"status":"ok","http":200,"url":"https://ielts.org/","html_tokens":38947,"markdown_tokens":599,"pruned_tokens":135}},
{"rank":4685,"domain":"sciencesconf.org","origin":"https://sciencesconf.org","robots":{"status":"ok","http":200,"url":"https://sciencesconf.org/robots.txt","bytes":56,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":179760},"home":{"status":"ok","http":200,"url":"https://www.sciencesconf.org/","html_tokens":46907,"markdown_tokens":27,"pruned_tokens":27}},
{"rank":4686,"domain":"slashfilm.com","origin":"https://slashfilm.com","robots":{"status":"ok","http":200,"url":"https://slashfilm.com/robots.txt","bytes":283,"groups":2},"bots":{"GPTBot":["partial","*",6,0],"ChatGPT-User":["partial","*",6,0],"OAI-SearchBot":["partial","*",6,0],"ClaudeBot":["partial","*",6,0],"Claude-User":["partial","*",6,0],"Claude-SearchBot":["partial","*",6,0],"anthropic-ai":["partial","*",6,0],"PerplexityBot":["partial","*",6,0],"Perplexity-User":["partial","*",6,0],"Google-Extended":["partial","*",6,0],"Applebot-Extended":["partial","*",6,0],"CCBot":["partial","*",6,0],"Bytespider":["partial","*",6,0],"Amazonbot":["partial","*",6,0],"meta-externalagent":["partial","*",6,0],"cohere-ai":["partial","*",6,0],"Diffbot":["partial","*",6,0],"ContextFetchBot":["partial","*",6,0]},"llms":{"status":"html","http":200,"bytes":72923},"home":{"status":"ok","http":200,"url":"https://www.slashfilm.com/","html_tokens":18230,"markdown_tokens":717,"pruned_tokens":175}},
{"rank":4687,"domain":"goodrx.com","origin":"https://goodrx.com","robots":{"status":"ok","http":200,"url":"https://www.goodrx.com/robots.txt","bytes":1354,"groups":2},"bots":{"GPTBot":["partial","*",39,0],"ChatGPT-User":["partial","*",39,0],"OAI-SearchBot":["partial","*",39,0],"ClaudeBot":["partial","*",39,0],"Claude-User":["partial","*",39,0],"Claude-SearchBot":["partial","*",39,0],"anthropic-ai":["partial","*",39,0],"PerplexityBot":["partial","*",39,0],"Perplexity-User":["partial","*",39,0],"Google-Extended":["partial","*",39,0],"Applebot-Extended":["partial","*",39,0],"CCBot":["partial","*",39,0],"Bytespider":["partial","*",39,0],"Amazonbot":["partial","*",39,0],"meta-externalagent":["partial","*",39,0],"cohere-ai":["partial","*",39,0],"Diffbot":["partial","*",39,0],"ContextFetchBot":["partial","*",39,0]},"llms":{"status":"http_error","http":403,"bytes":5533},"home":{"status":"http_error","http":403,"url":"https://www.goodrx.com/"}},
{"rank":4688,"domain":"pikabu.ru","origin":"https://pikabu.ru","robots":{"status":"refused","http":403,"url":"https://pikabu.ru/robots.txt","bytes":902,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":4689,"domain":"icij.org","origin":"https://icij.org","robots":{"status":"ok","http":200,"url":"https://www.icij.org/robots.txt","bytes":116,"groups":1},"bots":{"GPTBot":["partial","*",1,1],"ChatGPT-User":["partial","*",1,1],"OAI-SearchBot":["partial","*",1,1],"ClaudeBot":["partial","*",1,1],"Claude-User":["partial","*",1,1],"Claude-SearchBot":["partial","*",1,1],"anthropic-ai":["partial","*",1,1],"PerplexityBot":["partial","*",1,1],"Perplexity-User":["partial","*",1,1],"Google-Extended":["partial","*",1,1],"Applebot-Extended":["partial","*",1,1],"CCBot":["partial","*",1,1],"Bytespider":["partial","*",1,1],"Amazonbot":["partial","*",1,1],"meta-externalagent":["partial","*",1,1],"cohere-ai":["partial","*",1,1],"Diffbot":["partial","*",1,1],"ContextFetchBot":["partial","*",1,1]},"llms":{"status":"other_text","http":200,"bytes":104819},"home":{"status":"ok","http":200,"url":"https://www.icij.org/","html_tokens":37764,"markdown_tokens":188,"pruned_tokens":147}},
{"rank":4690,"domain":"freeservers.com","origin":"http://freeservers.com","robots":{"status":"ok","http":200,"url":"http://www.freeservers.com/robots.txt","bytes":133,"groups":1},"bots":{"GPTBot":["partial","*",1,0],"ChatGPT-User":["partial","*",1,0],"OAI-SearchBot":["partial","*",1,0],"ClaudeBot":["partial","*",1,0],"Claude-User":["partial","*",1,0],"Claude-SearchBot":["partial","*",1,0],"anthropic-ai":["partial","*",1,0],"PerplexityBot":["partial","*",1,0],"Perplexity-User":["partial","*",1,0],"Google-Extended":["partial","*",1,0],"Applebot-Extended":["partial","*",1,0],"CCBot":["partial","*",1,0],"Bytespider":["partial","*",1,0],"Amazonbot":["partial","*",1,0],"meta-externalagent":["partial","*",1,0],"cohere-ai":["partial","*",1,0],"Diffbot":["partial","*",1,0],"ContextFetchBot":["partial","*",1,0]},"llms":{"status":"missing","http":404,"bytes":7486},"home":{"status":"ok","http":200,"url":"http://www.freeservers.com/","html_tokens":4754,"markdown_tokens":747,"pruned_tokens":594}},
{"rank":4691,"domain":"peerj.com","origin":"https://peerj.com","robots":{"status":"ok","http":200,"url":"https://peerj.com/robots.txt","bytes":252,"groups":3},"bots":{"GPTBot":["partial","*",3,0],"ChatGPT-User":["partial","*",3,0],"OAI-SearchBot":["partial","*",3,0],"ClaudeBot":["partial","*",3,0],"Claude-User":["partial","*",3,0],"Claude-SearchBot":["partial","*",3,0],"anthropic-ai":["partial","*",3,0],"PerplexityBot":["partial","*",3,0],"Perplexity-User":["partial","*",3,0],"Google-Extended":["partial","*",3,0],"Applebot-Extended":["partial","*",3,0],"CCBot":["partial","*",3,0],"Bytespider":["partial","*",3,0],"Amazonbot":["partial","*",3,0],"meta-externalagent":["partial","*",3,0],"cohere-ai":["partial","*",3,0],"Diffbot":["partial","*",3,0],"ContextFetchBot":["partial","*",3,0]},"llms":{"status":"missing","http":404,"bytes":34406},"home":{"status":"ok","http":200,"url":"https://peerj.com/","html_tokens":8433,"markdown_tokens":25,"pruned_tokens":25}},
{"rank":4692,"domain":"in.gov.br","origin":"https://in.gov.br","robots":{"status":"unreachable","http":null,"url":"https://in.gov.br/robots.txt","bytes":0,"groups":0,"error":"<StreamReset stream_id:1, error_code:1, remote_reset:True>"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":4693,"domain":"ebayimg.com","origin":"https://ebayimg.com","robots":{"status":"unreachable","http":null,"url":"https://ebayimg.com/robots.txt","bytes":0,"groups":0,"error":"cannot resolve ebayimg.com"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":4694,"domain":"tokopedia.com","origin":"https://tokopedia.com","robots":{"status":"ok","http":200,"url":"https://www.tokopedia.com/robots.txt","bytes":4698,"groups":2},"bots":{"GPTBot":["partial","*",110,8],"ChatGPT-User":["partial","*",110,8],"OAI-SearchBot":["partial","*",110,8],"ClaudeBot":["partial","*",110,8],"Claude-User":["partial","*",110,8],"Claude-SearchBot":["partial","*",110,8],"anthropic-ai":["partial","*",110,8],"PerplexityBot":["partial","*",110,8],"Perplexity-User":["partial","*",110,8],"Google-Extended":["partial","*",110,8],"Applebot-Extended":["partial","*",110,8],"CCBot":["partial","*",110,8],"Bytespider":["partial","*",110,8],"Amazonbot":["partial","*",110,8],"meta-externalagent":["partial","*",110,8],"cohere-ai":["partial","*",110,8],"Diffbot":["partial","*",110,8],"ContextFetchBot":["partial","*",110,8]},"llms":{"status":"unreachable","error":"<StreamReset stream_id:5, error_code:2, remote_reset:True>"},"home":{"status":"unreachable","error":"<StreamReset stream_id:7, error_code:2, remote_reset:True>"}},
{"rank":4695,"domain":"geocities.co.jp","origin":"https://geocities.co.jp","robots":{"status":"unreachable","http":null,"url":"https://geocities.co.jp/robots.txt","bytes":0,"groups":0,"error":"cannot resolve geocities.co.jp"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":4696,"domain":"festival-cannes.com","origin":"https://festival-cannes.com","robots":{"status":"html","http":200,"url":"https://www.festival-cannes.com/","bytes":156287,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":4697,"domain":"solarwinds.com","origin":"https://solarwinds.com","robots":{"status":"ok","http":200,"url":"https://www.solarwinds.com/robots.txt","bytes":3002,"groups":3},"bots":{"GPTBot":["partial","*",39,0],"ChatGPT-User":["partial","*",39,0],"OAI-SearchBot":["partial","*",39,0],"ClaudeBot":["partial","*",39,0],"Claude-User":["partial","*",39,0],"Claude-SearchBot":["partial","*",39,0],"anthropic-ai":["partial","*",39,0],"PerplexityBot":["partial","*",39,0],"Perplexity-User":["partial","*",39,0],"Google-Extended":["partial","*",39,0],"Applebot-Extended":["partial","*",39,0],"CCBot":["partial","*",39,0],"Bytespider":["partial","*",39,0],"Amazonbot":["partial","*",39,0],"meta-externalagent":["partial","*",39,0],"cohere-ai":["partial","*",39,0],"Diffbot":["partial","*",39,0],"ContextFetchBot":["partial","*",39,0]},"llms":{"status":"missing","http":404,"bytes":126234},"home":{"status":"ok","http":200,"url":"https://www.solarwinds.com/","html_tokens":444511,"markdown_tokens":920,"pruned_tokens":524}},
{"rank":4698,"domain":"gyazo.com","origin":"https://gyazo.com","robots":{"status":"ok","http":200,"url":"https://gyazo.com/robots.txt","bytes":62,"groups":1},"bots":{"GPTBot":["allowed","*",0,1],"ChatGPT-User":["allowed","*",0,1],"OAI-SearchBot":["allowed","*",0,1],"ClaudeBot":["allowed","*",0,1],"Claude-User":["allowed","*",0,1],"Claude-SearchBot":["allowed","*",0,1],"anthropic-ai":["allowed","*",0,1],"PerplexityBot":["allowed","*",0,1],"Perplexity-User":["allowed","*",0,1],"Google-Extended":["allowed","*",0,1],"Applebot-Extended":["allowed","*",0,1],"CCBot":["allowed","*",0,1],"Bytespider":["allowed","*",0,1],"Amazonbot":["allowed","*",0,1],"meta-externalagent":["allowed","*",0,1],"cohere-ai":["allowed","*",0,1],"Diffbot":["allowed","*",0,1],"ContextFetchBot":["allowed","*",0,1]},"llms":{"status":"missing","http":404,"bytes":26014},"home":{"status":"ok","http":200,"url":"https://gyazo.com/","html_tokens":7556,"markdown_tokens":665,"pruned_tokens":60}},
{"rank":4699,"domain":"myminifactory.com","origin":"https://myminifactory.com","robots":{"status":"refused","http":403,"url":"https://www.myminifactory.com/robots.txt","bytes":5588,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":4700,"domain":"kununu.com","origin":"https://kununu.com","robots":{"status":"ok","http":200,"url":"https://www.kununu.com/robots.txt","bytes":1488,"groups":6},"bots":{"GPTBot":["partial","*",49,2],"ChatGPT-User":["partial","*",49,2],"OAI-SearchBot":["partial","*",49,2],"ClaudeBot":["allowed","named",0,0],"Claude-User":["partial","*",49,2],"Claude-SearchBot":["partial","*",49,2],"anthropic-ai":["partial","*",49,2],"PerplexityBot":["partial","*",49,2],"Perplexity-User":["partial","*",49,2],"Google-Extended":["partial","*",49,2],"Applebot-Extended":["partial","*",49,2],"CCBot":["partial","*",49,2],"Bytespider":["partial","*",49,2],"Amazonbot":["partial","*",49,2],"meta-externalagent":["partial","*",49,2],"cohere-ai":["partial","*",49,2],"Diffbot":["partial","*",49,2],"ContextFetchBot":["partial","*",49,2]},"llms":{"status":"html","http":202,"bytes":2419},"home":{"status":"ok","http":202,"url":"https://www.kununu.com/","html_tokens":605,"markdown_tokens":40,"pruned_tokens":40}},
{"rank":4701,"domain":"calibre-ebook.com","origin":"https://calibre-ebook.com","robots":{"status":"missing","http":404,"url":"https://calibre-ebook.com/robots.txt","bytes":146,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":146},"home":{"status":"ok","http":200,"url":"https://calibre-ebook.com/","html_tokens":8716,"markdown_tokens":7,"pruned_tokens":7}},
{"rank":4702,"domain":"rs.gov.br","origin":"https://rs.gov.br","robots":{"status":"html","http":200,"url":"https://rs.gov.br/robots.txt","bytes":245,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":4703,"domain":"ua.es","origin":"https://www.ua.es","robots":{"status":"ok","http":200,"url":"https://www.ua.es/robots.txt","bytes":23808,"groups":664},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["blocked","named",1,0],"OAI-SearchBot":["partial","*",2,1],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",2,1],"Claude-SearchBot":["partial","*",2,1],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["partial","*",2,1],"Perplexity-User":["partial","*",2,1],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["partial","*",2,1],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["partial","*",2,1],"meta-externalagent":["partial","*",2,1],"cohere-ai":["blocked","named",1,0],"Diffbot":["partial","*",2,1],"ContextFetchBot":["partial","*",2,1]},"llms":{"status":"missing","http":404,"bytes":6530},"home":{"status":"ok","http":200,"url":"https://www.ua.es/","html_tokens":23866,"markdown_tokens":100,"pruned_tokens":44}},
{"rank":4704,"domain":"mic.com","origin":"https://mic.com","robots":{"status":"ok","http":200,"url":"https://www.mic.com/robots.txt","bytes":280,"groups":2},"bots":{"GPTBot":["partial","*",1,0],"ChatGPT-User":["partial","*",1,0],"OAI-SearchBot":["partial","*",1,0],"ClaudeBot":["partial","*",1,0],"Claude-User":["partial","*",1,0],"Claude-SearchBot":["partial","*",1,0],"anthropic-ai":["partial","*",1,0],"PerplexityBot":["partial","*",1,0],"Perplexity-User":["partial","*",1,0],"Google-Extended":["partial","*",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["partial","*",1,0],"Bytespider":["partial","*",1,0],"Amazonbot":["partial","*",1,0],"meta-externalagent":["partial","*",1,0],"cohere-ai":["partial","*",1,0],"Diffbot":["partial","*",1,0],"ContextFetchBot":["partial","*",1,0]},"llms":{"status":"missing","http":404,"bytes":27437},"home":{"status":"ok","http":200,"url":"https://www.mic.com","html_tokens":54400,"markdown_tokens":1190,"pruned_tokens":38}},
{"rank":4705,"domain":"figshare.com","origin":"https://figshare.com","robots":{"status":"html","http":202,"url":"https://figshare.com/robots.txt","bytes":1995,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":4706,"domain":"dni.gov","origin":"https://dni.gov","robots":{"status":"ok","http":200,"url":"https://www.odni.gov/robots.txt","bytes":779,"groups":3},"bots":{"GPTBot":["partial","*",2,1],"ChatGPT-User":["partial","*",2,1],"OAI-SearchBot":["partial","*",2,1],"ClaudeBot":["partial","*",2,1],"Claude-User":["partial","*",2,1],"Claude-SearchBot":["partial","*",2,1],"anthropic-ai":["partial","*",2,1],"PerplexityBot":["partial","*",2,1],"Perplexity-User":["partial","*",2,1],"Google-Extended":["partial","*",2,1],"Applebot-Extended":["partial","*",2,1],"CCBot":["partial","*",2,1],"Bytespider":["partial","*",2,1],"Amazonbot":["partial","*",2,1],"meta-externalagent":["partial","*",2,1],"cohere-ai":["partial","*",2,1],"Diffbot":["partial","*",2,1],"ContextFetchBot":["partial","*",2,1]},"llms":{"status":"timeout","error":"timed out after 10 s"},"home":{"status":"ok","http":200,"url":"https://www.odni.gov/","html_tokens":46818,"markdown_tokens":130,"pruned_tokens":130}},
{"rank":4707,"domain":"creditchina.gov.cn","origin":"https://www.creditchina.gov.cn","robots":{"status":"missing","http":404,"url":"https://www.creditchina.gov.cn/robots.txt","bytes":4747,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":4747},"home":{"status":"http_error","http":412,"url":"https://www.creditchina.gov.cn/"}},
{"rank":4708,"domain":"ufrgs.br","origin":"https://www.ufrgs.br","robots":{"status":"ok","http":200,"url":"https://www.ufrgs.br/robots.txt","bytes":415,"groups":9},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":2596},"home":{"status":"ok","http":200,"url":"https://www.ufrgs.br/site/","html_tokens":44894,"markdown_tokens":32,"pruned_tokens":32}},
{"rank":4709,"domain":"maxmind.com","origin":"https://maxmind.com","robots":{"status":"ok","http":200,"url":"https://maxmind.com/robots.txt","bytes":45,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":83},"home":{"status":"ok","http":200,"url":"https://www.maxmind.com/en/home","html_tokens":15205,"markdown_tokens":157,"pruned_tokens":30}},
{"rank":4710,"domain":"nbcwashington.com","origin":"https://nbcwashington.com","robots":{"status":"ok","http":200,"url":"https://www.nbcwashington.com/robots.txt","bytes":10598,"groups":27},"bots":{"GPTBot":["partial","named",1,6],"ChatGPT-User":["partial","named",1,6],"OAI-SearchBot":["partial","*",13,3],"ClaudeBot":["partial","named",1,6],"Claude-User":["partial","*",13,3],"Claude-SearchBot":["partial","*",13,3],"anthropic-ai":["partial","named",1,6],"PerplexityBot":["partial","named",1,6],"Perplexity-User":["partial","*",13,3],"Google-Extended":["partial","named",1,6],"Applebot-Extended":["partial","*",13,3],"CCBot":["partial","named",1,6],"Bytespider":["partial","named",1,6],"Amazonbot":["partial","named",1,8],"meta-externalagent":["partial","*",13,3],"cohere-ai":["partial","*",13,3],"Diffbot":["partial","named",1,6],"ContextFetchBot":["partial","*",13,3]},"llms":{"status":"html","http":200,"bytes":552164},"home":{"status":"ok","http":200,"url":"https://www.nbcwashington.com/","html_tokens":138024,"markdown_tokens":3,"pruned_tokens":3}},
{"rank":4711,"domain":"actu.fr","origin":"https://actu.fr","robots":{"status":"ok","http":200,"url":"https://actu.fr/robots.txt","bytes":4317,"groups":20},"bots":{"GPTBot":["partial","named",1,3],"ChatGPT-User":["partial","named",1,3],"OAI-SearchBot":["partial","named",1,3],"ClaudeBot":["partial","named",1,3],"Claude-User":["partial","named",1,3],"Claude-SearchBot":["partial","named",1,3],"anthropic-ai":["partial","named",1,3],"PerplexityBot":["partial","named",1,3],"Perplexity-User":["partial","*",5,1],"Google-Extended":["partial","named",1,3],"Applebot-Extended":["partial","named",1,3],"CCBot":["blocked","named",1,0],"Bytespider":["partial","named",1,3],"Amazonbot":["partial","named",1,3],"meta-externalagent":["partial","named",1,3],"cohere-ai":["partial","named",1,3],"Diffbot":["partial","*",5,1],"ContextFetchBot":["partial","*",5,1]},"llms":{"status":"missing","http":404,"bytes":191159},"home":{"status":"ok","http":200,"url":"https://actu.fr/","html_tokens":163632,"markdown_tokens":76,"pruned_tokens":76}},
{"rank":4712,"domain":"iit.edu","origin":"https://iit.edu","robots":{"status":"refused","http":403,"url":"https://iit.edu/robots.txt","bytes":560,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":4713,"domain":"banque-france.fr","origin":"https://banque-france.fr","robots":{"status":"ok","http":200,"url":"https://www.banque-france.fr/robots.txt","bytes":2027,"groups":1},"bots":{"GPTBot":["partial","*",34,18],"ChatGPT-User":["partial","*",34,18],"OAI-SearchBot":["partial","*",34,18],"ClaudeBot":["partial","*",34,18],"Claude-User":["partial","*",34,18],"Claude-SearchBot":["partial","*",34,18],"anthropic-ai":["partial","*",34,18],"PerplexityBot":["partial","*",34,18],"Perplexity-User":["partial","*",34,18],"Google-Extended":["partial","*",34,18],"Applebot-Extended":["partial","*",34,18],"CCBot":["partial","*",34,18],"Bytespider":["partial","*",34,18],"Amazonbot":["partial","*",34,18],"meta-externalagent":["partial","*",34,18],"cohere-ai":["partial","*",34,18],"Diffbot":["partial","*",34,18],"ContextFetchBot":["partial","*",34,18]},"llms":{"status":"missing","http":404,"bytes":194},"home":{"status":"ok","http":200,"url":"https://www.banque-france.fr/fr","html_tokens":92408,"markdown_tokens":94,"pruned_tokens":51}},
{"rank":4714,"domain":"nau.edu","origin":"https://nau.edu","robots":{"status":"ok","http":200,"url":"https://nau.edu/robots.txt","bytes":106,"groups":1},"bots":{"GPTBot":["partial","*",2,0],"ChatGPT-User":["partial","*",2,0],"OAI-SearchBot":["partial","*",2,0],"ClaudeBot":["partial","*",2,0],"Claude-User":["partial","*",2,0],"Claude-SearchBot":["partial","*",2,0],"anthropic-ai":["partial","*",2,0],"PerplexityBot":["partial","*",2,0],"Perplexity-User":["partial","*",2,0],"Google-Extended":["partial","*",2,0],"Applebot-Extended":["partial","*",2,0],"CCBot":["partial","*",2,0],"Bytespider":["partial","*",2,0],"Amazonbot":["partial","*",2,0],"meta-externalagent":["partial","*",2,0],"cohere-ai":["partial","*",2,0],"Diffbot":["partial","*",2,0],"ContextFetchBot":["partial","*",2,0]},"llms":{"status":"missing","http":404,"bytes":123785},"home":{"status":"ok","http":200,"url":"https://nau.edu/","html_tokens":40948,"markdown_tokens":84,"pruned_tokens":84}},
{"rank":4715,"domain":"hotpepper.jp","origin":"https://hotpepper.jp","robots":{"status":"ok","http":200,"url":"https://www.hotpepper.jp/robots.txt","bytes":204,"groups":2},"bots":{"GPTBot":["partial","*",4,1],"ChatGPT-User":["partial","*",4,1],"OAI-SearchBot":["partial","*",4,1],"ClaudeBot":["partial","*",4,1],"Claude-User":["partial","*",4,1],"Claude-SearchBot":["partial","*",4,1],"anthropic-ai":["partial","*",4,1],"PerplexityBot":["partial","*",4,1],"Perplexity-User":["partial","*",4,1],"Google-Extended":["partial","*",4,1],"Applebot-Extended":["partial","*",4,1],"CCBot":["partial","*",4,1],"Bytespider":["partial","*",4,1],"Amazonbot":["partial","*",4,1],"meta-externalagent":["partial","*",4,1],"cohere-ai":["partial","*",4,1],"Diffbot":["partial","*",4,1],"ContextFetchBot":["partial","*",4,1]},"llms":{"status":"html","http":200,"bytes":1017},"home":{"status":"ok","http":200,"url":"https://www.hotpepper.jp/","html_tokens":21053,"markdown_tokens":114,"pruned_tokens":11}},
{"rank":4716,"domain":"dergipark.org.tr","origin":"https://dergipark.org.tr","robots":{"status":"ok","http":200,"url":"https://dergipark.org.tr/robots.txt","bytes":166,"groups":2},"bots":{"GPTBot":["partial","*",6,0],"ChatGPT-User":["partial","*",6,0],"OAI-SearchBot":["partial","*",6,0],"ClaudeBot":["partial","*",6,0],"Claude-User":["partial","*",6,0],"Claude-SearchBot":["partial","*",6,0],"anthropic-ai":["partial","*",6,0],"PerplexityBot":["partial","*",6,0],"Perplexity-User":["partial","*",6,0],"Google-Extended":["partial","*",6,0],"Applebot-Extended":["partial","*",6,0],"CCBot":["partial","*",6,0],"Bytespider":["partial","*",6,0],"Amazonbot":["partial","*",6,0],"meta-externalagent":["partial","*",6,0],"cohere-ai":["partial","*",6,0],"Diffbot":["partial","*",6,0],"ContextFetchBot":["partial","*",6,0]},"llms":{"status":"missing","http":404,"bytes":1251},"home":{"status":"ok","http":200,"url":"https://dergipark.org.tr/tr/","html_tokens":16840,"markdown_tokens":438,"pruned_tokens":138}},
{"rank":4717,"domain":"ascopubs.org","origin":"https://ascopubs.org","robots":{"status":"ok","http":200,"url":"https://ascopubs.org/robots.txt","bytes":1503,"groups":18},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["allowed","named",0,1],"OAI-SearchBot":["allowed","named",0,1],"ClaudeBot":["blocked","named",1,0],"Claude-User":["allowed","named",0,1],"Claude-SearchBot":["allowed","named",0,1],"anthropic-ai":["partial","*",15,6],"PerplexityBot":["allowed","named",0,1],"Perplexity-User":["allowed","named",0,1],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["partial","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["partial","*",15,6],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["partial","*",15,6],"Diffbot":["partial","*",15,6],"ContextFetchBot":["partial","*",15,6]},"llms":{"status":"http_error","http":403,"bytes":5544},"home":{"status":"http_error","http":403,"url":"https://ascopubs.org/"}},
{"rank":4718,"domain":"cftc.gov","origin":"https://cftc.gov","robots":{"status":"ok","http":200,"url":"https://cftc.gov/robots.txt","bytes":2676,"groups":32},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["allowed","*",0,1],"OAI-SearchBot":["allowed","*",0,1],"ClaudeBot":["blocked","named",1,0],"Claude-User":["allowed","*",0,1],"Claude-SearchBot":["allowed","*",0,1],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["allowed","*",0,1],"Perplexity-User":["allowed","*",0,1],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["allowed","*",0,1]},"llms":{"status":"missing","http":404,"bytes":547},"home":{"status":"ok","http":200,"url":"https://www.cftc.gov/","html_tokens":14247,"markdown_tokens":243,"pruned_tokens":172}},
{"rank":4719,"domain":"mediaset.it","origin":"https://mediaset.it","robots":{"status":"ok","http":200,"url":"https://www.mediaset.it/robots.txt","bytes":30,"groups":1},"bots":{"GPTBot":["partial","*",1,0],"ChatGPT-User":["partial","*",1,0],"OAI-SearchBot":["partial","*",1,0],"ClaudeBot":["partial","*",1,0],"Claude-User":["partial","*",1,0],"Claude-SearchBot":["partial","*",1,0],"anthropic-ai":["partial","*",1,0],"PerplexityBot":["partial","*",1,0],"Perplexity-User":["partial","*",1,0],"Google-Extended":["partial","*",1,0],"Applebot-Extended":["partial","*",1,0],"CCBot":["partial","*",1,0],"Bytespider":["partial","*",1,0],"Amazonbot":["partial","*",1,0],"meta-externalagent":["partial","*",1,0],"cohere-ai":["partial","*",1,0],"Diffbot":["partial","*",1,0],"ContextFetchBot":["partial","*",1,0]},"llms":{"status":"missing","http":404,"bytes":312},"home":{"status":"too_large","error":"response larger than 5 MB"}},
{"rank":4720,"domain":"mysanantonio.com","origin":"https://mysanantonio.com","robots":{"status":"ok","http":200,"url":"https://www.mysanantonio.com/robots.txt","bytes":1029,"groups":3},"bots":{"GPTBot":["partial","*",9,0],"ChatGPT-User":["partial","*",9,0],"OAI-SearchBot":["partial","*",9,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["blocked","named",1,0],"Claude-SearchBot":["partial","*",9,0],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["blocked","named",1,0],"Google-Extended":["partial","*",9,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["partial","*",9,0],"meta-externalagent":["partial","*",9,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",9,0]},"llms":{"status":"missing","http":404,"bytes":51762},"home":{"status":"ok","http":200,"url":"https://www.mysanantonio.com/","html_tokens":270487,"markdown_tokens":21,"pruned_tokens":21}},
{"rank":4721,"domain":"mapy.cz","origin":"https://mapy.cz","robots":{"status":"ok","http":200,"url":"https://mapy.cz/robots.txt","bytes":1976,"groups":14},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["partial","*",2,0],"OAI-SearchBot":["partial","*",2,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",2,0],"Claude-SearchBot":["partial","*",2,0],"anthropic-ai":["partial","*",2,0],"PerplexityBot":["partial","*",2,0],"Perplexity-User":["partial","*",2,0],"Google-Extended":["partial","*",2,0],"Applebot-Extended":["partial","*",2,0],"CCBot":["partial","*",2,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["partial","*",2,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["partial","*",2,0],"Diffbot":["partial","*",2,0],"ContextFetchBot":["partial","*",2,0]},"llms":{"status":"html","http":200,"bytes":13662},"home":{"status":"ok","http":200,"url":"https://mapy.com/en/","html_tokens":3385,"markdown_tokens":0,"pruned_tokens":0}},
{"rank":4722,"domain":"spacenews.com","origin":"https://spacenews.com","robots":{"status":"ok","http":200,"url":"https://spacenews.com/robots.txt","bytes":702,"groups":19},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["blocked","named",1,0],"OAI-SearchBot":["blocked","named",1,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["blocked","named",1,0],"Claude-SearchBot":["blocked","named",1,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":195710},"home":{"status":"ok","http":200,"url":"https://spacenews.com/","html_tokens":79826,"markdown_tokens":47,"pruned_tokens":47}},
{"rank":4723,"domain":"earthsky.org","origin":"https://earthsky.org","robots":{"status":"ok","http":200,"url":"https://earthsky.org/robots.txt","bytes":423,"groups":8},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["blocked","named",1,0],"OAI-SearchBot":["partial","*",1,1],"ClaudeBot":["partial","*",1,1],"Claude-User":["partial","*",1,1],"Claude-SearchBot":["partial","*",1,1],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["partial","*",1,1],"Perplexity-User":["partial","*",1,1],"Google-Extended":["partial","*",1,1],"Applebot-Extended":["partial","*",1,1],"CCBot":["blocked","named",1,0],"Bytespider":["partial","*",1,1],"Amazonbot":["partial","*",1,1],"meta-externalagent":["partial","*",1,1],"cohere-ai":["partial","*",1,1],"Diffbot":["partial","*",1,1],"ContextFetchBot":["partial","*",1,1]},"llms":{"status":"other_text","http":200,"bytes":203432},"home":{"status":"ok","http":200,"url":"https://earthsky.org/","html_tokens":35554,"markdown_tokens":211,"pruned_tokens":40}},
{"rank":4724,"domain":"krone.at","origin":"https://krone.at","robots":{"status":"ok","http":200,"url":"https://www.krone.at/robots.txt","bytes":1460,"groups":1},"bots":{"GPTBot":["partial","*",15,0],"ChatGPT-User":["partial","*",15,0],"OAI-SearchBot":["partial","*",15,0],"ClaudeBot":["partial","*",15,0],"Claude-User":["partial","*",15,0],"Claude-SearchBot":["partial","*",15,0],"anthropic-ai":["partial","*",15,0],"PerplexityBot":["partial","*",15,0],"Perplexity-User":["partial","*",15,0],"Google-Extended":["partial","*",15,0],"Applebot-Extended":["partial","*",15,0],"CCBot":["partial","*",15,0],"Bytespider":["partial","*",15,0],"Amazonbot":["partial","*",15,0],"meta-externalagent":["partial","*",15,0],"cohere-ai":["partial","*",15,0],"Diffbot":["partial","*",15,0],"ContextFetchBot":["partial","*",15,0]},"llms":{"status":"missing","http":404,"bytes":239497},"home":{"status":"ok","http":200,"url":"https://www.krone.at/","html_tokens":183234,"markdown_tokens":2,"pruned_tokens":2}},
{"rank":4725,"domain":"thegameroom.org","origin":"https://thegameroom.org","robots":{"status":"refused","http":403,"url":"https://thegameroom.org/robots.txt","bytes":4548,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":4726,"domain":"wanfangdata.com.cn","origin":"https://wanfangdata.com.cn","robots":{"status":"missing","http":404,"url":"https://wanfangdata.com.cn/robots.txt","bytes":162,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":162},"home":{"status":"ok","http":200,"url":"https://wanfangdata.com.cn/","html_tokens":52613,"markdown_tokens":212,"pruned_tokens":175}},
{"rank":4727,"domain":"crazyegg.com","origin":"https://crazyegg.com","robots":{"status":"ok","http":200,"url":"https://www.crazyegg.com/robots.txt","bytes":904,"groups":2},"bots":{"GPTBot":["allowed","named",0,1],"ChatGPT-User":["partial","*",12,1],"OAI-SearchBot":["allowed","named",0,1],"ClaudeBot":["partial","*",12,1],"Claude-User":["partial","*",12,1],"Claude-SearchBot":["partial","*",12,1],"anthropic-ai":["allowed","named",0,1],"PerplexityBot":["partial","*",12,1],"Perplexity-User":["partial","*",12,1],"Google-Extended":["allowed","named",0,1],"Applebot-Extended":["allowed","named",0,1],"CCBot":["allowed","named",0,1],"Bytespider":["allowed","named",0,1],"Amazonbot":["allowed","named",0,1],"meta-externalagent":["partial","*",12,1],"cohere-ai":["partial","*",12,1],"Diffbot":["partial","*",12,1],"ContextFetchBot":["partial","*",12,1]},"llms":{"status":"present","http":200,"bytes":4274},"home":{"status":"not_html","error":"unsupported content type text/markdown"}},
{"rank":4728,"domain":"zippyshare.com","origin":"https://zippyshare.com","robots":{"status":"ok","http":200,"url":"https://zippyshare.com/robots.txt","bytes":35,"groups":1},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"html","http":200,"bytes":5114},"home":{"status":"ok","http":200,"url":"https://zippyshare.com/","html_tokens":1279,"markdown_tokens":63,"pruned_tokens":63}},
{"rank":4729,"domain":"jit.si","origin":"https://jit.si","robots":{"status":"unreachable","http":null,"url":"https://jit.si/robots.txt","bytes":0,"groups":0,"error":"cannot resolve jit.si"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":4730,"domain":"dow.com","origin":"https://dow.com","robots":{"status":"ok","http":200,"url":"https://www.dow.com/robots.txt","bytes":895,"groups":1},"bots":{"GPTBot":["partial","*",22,0],"ChatGPT-User":["partial","*",22,0],"OAI-SearchBot":["partial","*",22,0],"ClaudeBot":["partial","*",22,0],"Claude-User":["partial","*",22,0],"Claude-SearchBot":["partial","*",22,0],"anthropic-ai":["partial","*",22,0],"PerplexityBot":["partial","*",22,0],"Perplexity-User":["partial","*",22,0],"Google-Extended":["partial","*",22,0],"Applebot-Extended":["partial","*",22,0],"CCBot":["partial","*",22,0],"Bytespider":["partial","*",22,0],"Amazonbot":["partial","*",22,0],"meta-externalagent":["partial","*",22,0],"cohere-ai":["partial","*",22,0],"Diffbot":["partial","*",22,0],"ContextFetchBot":["partial","*",22,0]},"llms":{"status":"missing","http":404,"bytes":236587},"home":{"status":"ok","http":200,"url":"https://www.dow.com/en-us","html_tokens":80307,"markdown_tokens":107,"pruned_tokens":43}},
{"rank":4731,"domain":"swagger.io","origin":"https://swagger.io","robots":{"status":"ok","http":200,"url":"https://swagger.io/robots.txt","bytes":1248,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":50559},"home":{"status":"ok","http":200,"url":"https://swagger.io/","html_tokens":23999,"markdown_tokens":488,"pruned_tokens":357}},
{"rank":4732,"domain":"iihs.org","origin":"https://iihs.org","robots":{"status":"ok","http":200,"url":"https://www.iihs.org/robots.txt","bytes":115,"groups":1},"bots":{"GPTBot":["partial","*",2,0],"ChatGPT-User":["partial","*",2,0],"OAI-SearchBot":["partial","*",2,0],"ClaudeBot":["partial","*",2,0],"Claude-User":["partial","*",2,0],"Claude-SearchBot":["partial","*",2,0],"anthropic-ai":["partial","*",2,0],"PerplexityBot":["partial","*",2,0],"Perplexity-User":["partial","*",2,0],"Google-Extended":["partial","*",2,0],"Applebot-Extended":["partial","*",2,0],"CCBot":["partial","*",2,0],"Bytespider":["partial","*",2,0],"Amazonbot":["partial","*",2,0],"meta-externalagent":["partial","*",2,0],"cohere-ai":["partial","*",2,0],"Diffbot":["partial","*",2,0],"ContextFetchBot":["partial","*",2,0]},"llms":{"status":"missing","http":404,"bytes":19098},"home":{"status":"ok","http":200,"url":"https://www.iihs.org/","html_tokens":7875,"markdown_tokens":135,"pruned_tokens":82}},
{"rank":4733,"domain":"mikecrm.com","origin":"https://mikecrm.com","robots":{"status":"ok","http":200,"url":"https://mikecrm.com/robots.txt","bytes":557,"groups":1},"bots":{"GPTBot":["partial","*",7,21],"ChatGPT-User":["partial","*",7,21],"OAI-SearchBot":["partial","*",7,21],"ClaudeBot":["partial","*",7,21],"Claude-User":["partial","*",7,21],"Claude-SearchBot":["partial","*",7,21],"anthropic-ai":["partial","*",7,21],"PerplexityBot":["partial","*",7,21],"Perplexity-User":["partial","*",7,21],"Google-Extended":["partial","*",7,21],"Applebot-Extended":["partial","*",7,21],"CCBot":["partial","*",7,21],"Bytespider":["partial","*",7,21],"Amazonbot":["partial","*",7,21],"meta-externalagent":["partial","*",7,21],"cohere-ai":["partial","*",7,21],"Diffbot":["partial","*",7,21],"ContextFetchBot":["partial","*",7,21]},"llms":{"status":"disallowed"},"home":{"status":"ok","http":200,"url":"https://mikecrm.com/","html_tokens":17875,"markdown_tokens":2097,"pruned_tokens":646}},
{"rank":4734,"domain":"magcloud.com","origin":"https://magcloud.com","robots":{"status":"ok","http":200,"url":"https://magcloud.com/robots.txt","bytes":415,"groups":3},"bots":{"GPTBot":["partial","*",8,1],"ChatGPT-User":["partial","*",8,1],"OAI-SearchBot":["partial","*",8,1],"ClaudeBot":["partial","*",8,1],"Claude-User":["partial","*",8,1],"Claude-SearchBot":["partial","*",8,1],"anthropic-ai":["partial","*",8,1],"PerplexityBot":["partial","*",8,1],"Perplexity-User":["partial","*",8,1],"Google-Extended":["partial","*",8,1],"Applebot-Extended":["partial","*",8,1],"CCBot":["partial","*",8,1],"Bytespider":["partial","*",8,1],"Amazonbot":["partial","*",8,1],"meta-externalagent":["partial","*",8,1],"cohere-ai":["partial","*",8,1],"Diffbot":["partial","*",8,1],"ContextFetchBot":["partial","*",8,1]},"llms":{"status":"missing","http":404,"bytes":11796},"home":{"status":"ok","http":200,"url":"https://magcloud.com/","html_tokens":9993,"markdown_tokens":69,"pruned_tokens":69}},
{"rank":4735,"domain":"no-ip.org","origin":"http://no-ip.org","robots":{"status":"ok","http":200,"url":"https://www.noip.com/robots.txt","bytes":611,"groups":1},"bots":{"GPTBot":["partial","*",18,1],"ChatGPT-User":["partial","*",18,1],"OAI-SearchBot":["partial","*",18,1],"ClaudeBot":["partial","*",18,1],"Claude-User":["partial","*",18,1],"Claude-SearchBot":["partial","*",18,1],"anthropic-ai":["partial","*",18,1],"PerplexityBot":["partial","*",18,1],"Perplexity-User":["partial","*",18,1],"Google-Extended":["partial","*",18,1],"Applebot-Extended":["partial","*",18,1],"CCBot":["partial","*",18,1],"Bytespider":["partial","*",18,1],"Amazonbot":["partial","*",18,1],"meta-externalagent":["partial","*",18,1],"cohere-ai":["partial","*",18,1],"Diffbot":["partial","*",18,1],"ContextFetchBot":["partial","*",18,1]},"llms":{"status":"present","http":200,"bytes":8953},"home":{"status":"ok","http":200,"url":"https://www.noip.com/","html_tokens":49052,"markdown_tokens":633,"pruned_tokens":281}},
{"rank":4736,"domain":"spektrum.de","origin":"https://spektrum.de","robots":{"status":"refused","http":403,"url":"https://spektrum.de/robots.txt","bytes":919,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":4737,"domain":"goneo.de","origin":"https://goneo.de","robots":{"status":"ok","http":200,"url":"https://www.goneo.de/robots.txt","bytes":764,"groups":1},"bots":{"GPTBot":["partial","*",15,0],"ChatGPT-User":["partial","*",15,0],"OAI-SearchBot":["partial","*",15,0],"ClaudeBot":["partial","*",15,0],"Claude-User":["partial","*",15,0],"Claude-SearchBot":["partial","*",15,0],"anthropic-ai":["partial","*",15,0],"PerplexityBot":["partial","*",15,0],"Perplexity-User":["partial","*",15,0],"Google-Extended":["partial","*",15,0],"Applebot-Extended":["partial","*",15,0],"CCBot":["partial","*",15,0],"Bytespider":["partial","*",15,0],"Amazonbot":["partial","*",15,0],"meta-externalagent":["partial","*",15,0],"cohere-ai":["partial","*",15,0],"Diffbot":["partial","*",15,0],"ContextFetchBot":["partial","*",15,0]},"llms":{"status":"missing","http":404,"bytes":1630},"home":{"status":"ok","http":200,"url":"https://www.goneo.de/","html_tokens":24738,"markdown_tokens":1022,"pruned_tokens":92}},
{"rank":4738,"domain":"iii.org","origin":"https://iii.org","robots":{"status":"ok","http":200,"url":"https://www.iii.org/robots.txt","bytes":23,"groups":1},"bots":{"GPTBot":["allowed","*",0,1],"ChatGPT-User":["allowed","*",0,1],"OAI-SearchBot":["allowed","*",0,1],"ClaudeBot":["allowed","*",0,1],"Claude-User":["allowed","*",0,1],"Claude-SearchBot":["allowed","*",0,1],"anthropic-ai":["allowed","*",0,1],"PerplexityBot":["allowed","*",0,1],"Perplexity-User":["allowed","*",0,1],"Google-Extended":["allowed","*",0,1],"Applebot-Extended":["allowed","*",0,1],"CCBot":["allowed","*",0,1],"Bytespider":["allowed","*",0,1],"Amazonbot":["allowed","*",0,1],"meta-externalagent":["allowed","*",0,1],"cohere-ai":["allowed","*",0,1],"Diffbot":["allowed","*",0,1],"ContextFetchBot":["allowed","*",0,1]},"llms":{"status":"missing","http":404,"bytes":39042},"home":{"status":"ok","http":200,"url":"https://www.iii.org/","html_tokens":25387,"markdown_tokens":33,"pruned_tokens":33}},
{"rank":4739,"domain":"hs-sites.com","origin":"https://hs-sites.com","robots":{"status":"ok","http":200,"url":"https://hs-sites.com/robots.txt","bytes":171,"groups":2},"bots":{"GPTBot":["blocked","*",1,0],"ChatGPT-User":["blocked","*",1,0],"OAI-SearchBot":["blocked","*",1,0],"ClaudeBot":["blocked","*",1,0],"Claude-User":["blocked","*",1,0],"Claude-SearchBot":["blocked","*",1,0],"anthropic-ai":["blocked","*",1,0],"PerplexityBot":["blocked","*",1,0],"Perplexity-User":["blocked","*",1,0],"Google-Extended":["blocked","*",1,0],"Applebot-Extended":["blocked","*",1,0],"CCBot":["blocked","*",1,0],"Bytespider":["blocked","*",1,0],"Amazonbot":["blocked","*",1,0],"meta-externalagent":["blocked","*",1,0],"cohere-ai":["blocked","*",1,0],"Diffbot":["blocked","*",1,0],"ContextFetchBot":["blocked","*",1,0]},"llms":{"status":"disallowed"},"home":{"status":"disallowed"}},
{"rank":4740,"domain":"aparat.com","origin":"https://aparat.com","robots":{"status":"ok","http":200,"url":"https://www.aparat.com/robots.txt","bytes":1471,"groups":1},"bots":{"GPTBot":["partial","*",59,2],"ChatGPT-User":["partial","*",59,2],"OAI-SearchBot":["partial","*",59,2],"ClaudeBot":["partial","*",59,2],"Claude-User":["partial","*",59,2],"Claude-SearchBot":["partial","*",59,2],"anthropic-ai":["partial","*",59,2],"PerplexityBot":["partial","*",59,2],"Perplexity-User":["partial","*",59,2],"Google-Extended":["partial","*",59,2],"Applebot-Extended":["partial","*",59,2],"CCBot":["partial","*",59,2],"Bytespider":["partial","*",59,2],"Amazonbot":["partial","*",59,2],"meta-externalagent":["partial","*",59,2],"cohere-ai":["partial","*",59,2],"Diffbot":["partial","*",59,2],"ContextFetchBot":["partial","*",59,2]},"llms":{"status":"missing","http":404,"bytes":146},"home":{"status":"ok","http":200,"url":"https://www.aparat.com/home","html_tokens":44684,"markdown_tokens":0,"pruned_tokens":0}},
{"rank":4741,"domain":"qrz.com","origin":"https://qrz.com","robots":{"status":"ok","http":200,"url":"https://www.qrz.com/robots.txt","bytes":439,"groups":2},"bots":{"GPTBot":["partial","*",14,1],"ChatGPT-User":["partial","*",14,1],"OAI-SearchBot":["partial","*",14,1],"ClaudeBot":["partial","*",14,1],"Claude-User":["partial","*",14,1],"Claude-SearchBot":["partial","*",14,1],"anthropic-ai":["partial","*",14,1],"PerplexityBot":["partial","*",14,1],"Perplexity-User":["partial","*",14,1],"Google-Extended":["partial","*",14,1],"Applebot-Extended":["partial","*",14,1],"CCBot":["partial","*",14,1],"Bytespider":["partial","*",14,1],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["partial","*",14,1],"cohere-ai":["partial","*",14,1],"Diffbot":["partial","*",14,1],"ContextFetchBot":["partial","*",14,1]},"llms":{"status":"unreachable","error":"Server disconnected without sending a response."},"home":{"status":"ok","http":200,"url":"https://www.qrz.com/","html_tokens":50098,"markdown_tokens":128,"pruned_tokens":126}},
{"rank":4742,"domain":"huffingtonpost.fr","origin":"https://huffingtonpost.fr","robots":{"status":"ok","http":200,"url":"https://www.huffingtonpost.fr/robots.txt","bytes":7932,"groups":211},"bots":{"GPTBot":["partial","*",1,0],"ChatGPT-User":["partial","*",1,0],"OAI-SearchBot":["partial","*",1,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",1,0],"Claude-SearchBot":["partial","*",1,0],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["partial","*",1,0],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",1,0]},"llms":{"status":"missing","http":404,"bytes":130729},"home":{"status":"ok","http":200,"url":"https://www.huffingtonpost.fr/","html_tokens":61179,"markdown_tokens":93,"pruned_tokens":93}},
{"rank":4743,"domain":"euobserver.com","origin":"https://euobserver.com","robots":{"status":"ok","http":200,"url":"https://euobserver.com/robots.txt","bytes":893,"groups":14},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["partial","*",5,0],"OAI-SearchBot":["partial","*",5,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",5,0],"Claude-SearchBot":["partial","*",5,0],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["partial","*",5,0],"Perplexity-User":["partial","*",5,0],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["partial","*",5,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["partial","*",5,0],"meta-externalagent":["partial","*",5,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",5,0]},"llms":{"status":"missing","http":404,"bytes":44214},"home":{"status":"ok","http":200,"url":"https://euobserver.com/","html_tokens":23058,"markdown_tokens":837,"pruned_tokens":161}},
{"rank":4744,"domain":"shein.com","origin":"https://shein.com","robots":{"status":"ok","http":200,"url":"https://www.shein.com/robots.txt","bytes":279,"groups":1},"bots":{"GPTBot":["partial","*",7,2],"ChatGPT-User":["partial","*",7,2],"OAI-SearchBot":["partial","*",7,2],"ClaudeBot":["partial","*",7,2],"Claude-User":["partial","*",7,2],"Claude-SearchBot":["partial","*",7,2],"anthropic-ai":["partial","*",7,2],"PerplexityBot":["partial","*",7,2],"Perplexity-User":["partial","*",7,2],"Google-Extended":["partial","*",7,2],"Applebot-Extended":["partial","*",7,2],"CCBot":["partial","*",7,2],"Bytespider":["partial","*",7,2],"Amazonbot":["partial","*",7,2],"meta-externalagent":["partial","*",7,2],"cohere-ai":["partial","*",7,2],"Diffbot":["partial","*",7,2],"ContextFetchBot":["partial","*",7,2]},"llms":{"status":"present","http":200,"bytes":63855},"home":{"status":"ok","http":200,"url":"https://us.shein.com/?cdn_rsite=cf&ref=www&rep=dir&ret=us","html_tokens":277944,"markdown_tokens":92,"pruned_tokens":92}},
{"rank":4745,"domain":"google.com.pk","origin":"https://google.com.pk","robots":{"status":"ok","http":200,"url":"https://www.google.com.pk/robots.txt","bytes":6624,"groups":4},"bots":{"GPTBot":["partial","*",169,64],"ChatGPT-User":["partial","*",169,64],"OAI-SearchBot":["partial","*",169,64],"ClaudeBot":["partial","*",169,64],"Claude-User":["partial","*",169,64],"Claude-SearchBot":["partial","*",169,64],"anthropic-ai":["partial","*",169,64],"PerplexityBot":["partial","*",169,64],"Perplexity-User":["partial","*",169,64],"Google-Extended":["partial","*",169,64],"Applebot-Extended":["partial","*",169,64],"CCBot":["partial","*",169,64],"Bytespider":["partial","*",169,64],"Amazonbot":["partial","*",169,64],"meta-externalagent":["partial","*",169,64],"cohere-ai":["partial","*",169,64],"Diffbot":["partial","*",169,64],"ContextFetchBot":["partial","*",169,64]},"llms":{"status":"missing","http":404,"bytes":1569},"home":{"status":"ok","http":200,"url":"https://www.google.com/","html_tokens":21353,"markdown_tokens":30,"pruned_tokens":30}},
{"rank":4746,"domain":"autismspeaks.org","origin":"https://autismspeaks.org","robots":{"status":"ok","http":200,"url":"https://www.autismspeaks.org/robots.txt","bytes":2853,"groups":1},"bots":{"GPTBot":["partial","*",49,18],"ChatGPT-User":["partial","*",49,18],"OAI-SearchBot":["partial","*",49,18],"ClaudeBot":["partial","*",49,18],"Claude-User":["partial","*",49,18],"Claude-SearchBot":["partial","*",49,18],"anthropic-ai":["partial","*",49,18],"PerplexityBot":["partial","*",49,18],"Perplexity-User":["partial","*",49,18],"Google-Extended":["partial","*",49,18],"Applebot-Extended":["partial","*",49,18],"CCBot":["partial","*",49,18],"Bytespider":["partial","*",49,18],"Amazonbot":["partial","*",49,18],"meta-externalagent":["partial","*",49,18],"cohere-ai":["partial","*",49,18],"Diffbot":["partial","*",49,18],"ContextFetchBot":["partial","*",49,18]},"llms":{"status":"missing","http":404,"bytes":328},"home":{"status":"ok","http":200,"url":"https://www.autismspeaks.org/","html_tokens":29065,"markdown_tokens":240,"pruned_tokens":105}},
{"rank":4747,"domain":"binghamton.edu","origin":"https://binghamton.edu","robots":{"status":"ok","http":200,"url":"https://www.binghamton.edu/robots.txt","bytes":1097,"groups":3},"bots":{"GPTBot":["partial","*",36,0],"ChatGPT-User":["partial","*",36,0],"OAI-SearchBot":["partial","*",36,0],"ClaudeBot":["partial","*",36,0],"Claude-User":["partial","*",36,0],"Claude-SearchBot":["partial","*",36,0],"anthropic-ai":["partial","*",36,0],"PerplexityBot":["partial","*",36,0],"Perplexity-User":["partial","*",36,0],"Google-Extended":["partial","*",36,0],"Applebot-Extended":["partial","*",36,0],"CCBot":["partial","*",36,0],"Bytespider":["partial","*",36,0],"Amazonbot":["partial","*",36,0],"meta-externalagent":["partial","*",36,0],"cohere-ai":["partial","*",36,0],"Diffbot":["partial","*",36,0],"ContextFetchBot":["partial","*",36,0]},"llms":{"status":"missing","http":404,"bytes":10617},"home":{"status":"ok","http":200,"url":"https://www.binghamton.edu/index.php","html_tokens":6189,"markdown_tokens":67,"pruned_tokens":67}},
{"rank":4748,"domain":"sheknows.com","origin":"https://sheknows.com","robots":{"status":"ok","http":200,"url":"https://www.sheknows.com/robots.txt","bytes":1219,"groups":25},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["blocked","named",1,0],"OAI-SearchBot":["blocked","named",1,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",9,1],"Claude-SearchBot":["partial","*",9,1],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["partial","*",9,1],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["partial","*",9,1],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",9,1]},"llms":{"status":"missing","http":404,"bytes":387920},"home":{"status":"ok","http":200,"url":"https://www.sheknows.com/","html_tokens":126634,"markdown_tokens":92,"pruned_tokens":86}},
{"rank":4749,"domain":"businessoffashion.com","origin":"https://businessoffashion.com","robots":{"status":"refused","http":403,"url":"https://www.businessoffashion.com/robots.txt","bytes":5631,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":4750,"domain":"babbel.com","origin":"https://babbel.com","robots":{"status":"ok","http":200,"url":"https://www.babbel.com/robots.txt","bytes":1657,"groups":2},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["partial","*",9,0],"OAI-SearchBot":["partial","*",9,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",9,0],"Claude-SearchBot":["partial","*",9,0],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["partial","*",9,0],"Perplexity-User":["partial","*",9,0],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["partial","*",9,0],"CCBot":["partial","*",9,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["partial","*",9,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["partial","*",9,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",9,0]},"llms":{"status":"present","http":200,"bytes":42429},"home":{"status":"ok","http":200,"url":"https://www.babbel.com/","html_tokens":54481,"markdown_tokens":586,"pruned_tokens":343}},
{"rank":4751,"domain":"aidar.ru","origin":"https://aidar.ru","robots":{"status":"ok","http":200,"url":"https://www.aidar.ru/robots.txt","bytes":102,"groups":1},"bots":{"GPTBot":["partial","*",2,1],"ChatGPT-User":["partial","*",2,1],"OAI-SearchBot":["partial","*",2,1],"ClaudeBot":["partial","*",2,1],"Claude-User":["partial","*",2,1],"Claude-SearchBot":["partial","*",2,1],"anthropic-ai":["partial","*",2,1],"PerplexityBot":["partial","*",2,1],"Perplexity-User":["partial","*",2,1],"Google-Extended":["partial","*",2,1],"Applebot-Extended":["partial","*",2,1],"CCBot":["partial","*",2,1],"Bytespider":["partial","*",2,1],"Amazonbot":["partial","*",2,1],"meta-externalagent":["partial","*",2,1],"cohere-ai":["partial","*",2,1],"Diffbot":["partial","*",2,1],"ContextFetchBot":["partial","*",2,1]},"llms":{"status":"html","http":200,"bytes":40143},"home":{"status":"ok","http":200,"url":"https://www.medslv.ru/","html_tokens":7530,"markdown_tokens":2621,"pruned_tokens":121}},
{"rank":4752,"domain":"copy.ai","origin":"https://copy.ai","robots":{"status":"ok","http":200,"url":"https://www.copy.ai/robots.txt","bytes":854,"groups":1},"bots":{"GPTBot":["partial","*",25,1],"ChatGPT-User":["partial","*",25,1],"OAI-SearchBot":["partial","*",25,1],"ClaudeBot":["partial","*",25,1],"Claude-User":["partial","*",25,1],"Claude-SearchBot":["partial","*",25,1],"anthropic-ai":["partial","*",25,1],"PerplexityBot":["partial","*",25,1],"Perplexity-User":["partial","*",25,1],"Google-Extended":["partial","*",25,1],"Applebot-Extended":["partial","*",25,1],"CCBot":["partial","*",25,1],"Bytespider":["partial","*",25,1],"Amazonbot":["partial","*",25,1],"meta-externalagent":["partial","*",25,1],"cohere-ai":["partial","*",25,1],"Diffbot":["partial","*",25,1],"ContextFetchBot":["partial","*",25,1]},"llms":{"status":"missing","http":404,"bytes":22384},"home":{"status":"ok","http":200,"url":"https://www.copy.ai/","html_tokens":53200,"markdown_tokens":138,"pruned_tokens":120}},
{"rank":4753,"domain":"bundesnetzagentur.de","origin":"https://bundesnetzagentur.de","robots":{"status":"ok","http":200,"url":"https://bundesnetzagentur.de/robots.txt","bytes":36,"groups":1},"bots":{"GPTBot":["partial","*",1,0],"ChatGPT-User":["partial","*",1,0],"OAI-SearchBot":["partial","*",1,0],"ClaudeBot":["partial","*",1,0],"Claude-User":["partial","*",1,0],"Claude-SearchBot":["partial","*",1,0],"anthropic-ai":["partial","*",1,0],"PerplexityBot":["partial","*",1,0],"Perplexity-User":["partial","*",1,0],"Google-Extended":["partial","*",1,0],"Applebot-Extended":["partial","*",1,0],"CCBot":["partial","*",1,0],"Bytespider":["partial","*",1,0],"Amazonbot":["partial","*",1,0],"meta-externalagent":["partial","*",1,0],"cohere-ai":["partial","*",1,0],"Diffbot":["partial","*",1,0],"ContextFetchBot":["partial","*",1,0]},"llms":{"status":"missing","http":404,"bytes":429},"home":{"status":"ok","http":200,"url":"https://www.bundesnetzagentur.de/DE/Home/home_node.html","html_tokens":27363,"markdown_tokens":2280,"pruned_tokens":1546}},
{"rank":4754,"domain":"jwwb.nl","origin":"http://jwwb.nl","robots":{"status":"missing","http":404,"url":"http://jwwb.nl/robots.txt","bytes":5825,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":5825},"home":{"status":"http_error","http":404,"url":"http://jwwb.nl/"}},
{"rank":4755,"domain":"renren.com","origin":"https://renren.com","robots":{"status":"html","http":200,"url":"https://renren.com/robots.txt","bytes":6817,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":4756,"domain":"wdc.com","origin":"https://wdc.com","robots":{"status":"ok","http":200,"url":"https://www.westerndigital.com/robots.txt","bytes":697,"groups":3},"bots":{"GPTBot":["partial","*",5,4],"ChatGPT-User":["partial","*",5,4],"OAI-SearchBot":["partial","named",3,4],"ClaudeBot":["partial","*",5,4],"Claude-User":["partial","*",5,4],"Claude-SearchBot":["partial","*",5,4],"anthropic-ai":["partial","*",5,4],"PerplexityBot":["partial","*",5,4],"Perplexity-User":["partial","*",5,4],"Google-Extended":["partial","*",5,4],"Applebot-Extended":["partial","*",5,4],"CCBot":["partial","*",5,4],"Bytespider":["partial","*",5,4],"Amazonbot":["partial","*",5,4],"meta-externalagent":["partial","*",5,4],"cohere-ai":["partial","*",5,4],"Diffbot":["partial","*",5,4],"ContextFetchBot":["partial","*",5,4]},"llms":{"status":"present","http":200,"bytes":23109},"home":{"status":"ok","http":200,"url":"https://www.westerndigital.com/","html_tokens":298905,"markdown_tokens":174,"pruned_tokens":174}},
{"rank":4757,"domain":"subway.com","origin":"https://subway.com","robots":{"status":"ok","http":200,"url":"https://www.subway.com/robots.txt","bytes":533,"groups":1},"bots":{"GPTBot":["partial","*",7,0],"ChatGPT-User":["partial","*",7,0],"OAI-SearchBot":["partial","*",7,0],"ClaudeBot":["partial","*",7,0],"Claude-User":["partial","*",7,0],"Claude-SearchBot":["partial","*",7,0],"anthropic-ai":["partial","*",7,0],"PerplexityBot":["partial","*",7,0],"Perplexity-User":["partial","*",7,0],"Google-Extended":["partial","*",7,0],"Applebot-Extended":["partial","*",7,0],"CCBot":["partial","*",7,0],"Bytespider":["partial","*",7,0],"Amazonbot":["partial","*",7,0],"meta-externalagent":["partial","*",7,0],"cohere-ai":["partial","*",7,0],"Diffbot":["partial","*",7,0],"ContextFetchBot":["partial","*",7,0]},"llms":{"status":"timeout","error":"timed out after 10 s"},"home":{"status":"ok","http":200,"url":"https://www.subway.com/en-us","html_tokens":6174,"markdown_tokens":447,"pruned_tokens":141}},
{"rank":4758,"domain":"dignitymemorial.com","origin":"https://dignitymemorial.com","robots":{"status":"ok","http":200,"url":"https://www.dignitymemorial.com/robots.txt","bytes":1009,"groups":1},"bots":{"GPTBot":["partial","*",11,0],"ChatGPT-User":["partial","*",11,0],"OAI-SearchBot":["partial","*",11,0],"ClaudeBot":["partial","*",11,0],"Claude-User":["partial","*",11,0],"Claude-SearchBot":["partial","*",11,0],"anthropic-ai":["partial","*",11,0],"PerplexityBot":["partial","*",11,0],"Perplexity-User":["partial","*",11,0],"Google-Extended":["partial","*",11,0],"Applebot-Extended":["partial","*",11,0],"CCBot":["partial","*",11,0],"Bytespider":["partial","*",11,0],"Amazonbot":["partial","*",11,0],"meta-externalagent":["partial","*",11,0],"cohere-ai":["partial","*",11,0],"Diffbot":["partial","*",11,0],"ContextFetchBot":["partial","*",11,0]},"llms":{"status":"missing","http":404,"bytes":26180},"home":{"status":"http_error","http":403,"url":"https://dignitymemorial.com/"}},
{"rank":4759,"domain":"salary.com","origin":"https://salary.com","robots":{"status":"ok","http":200,"url":"https://www.salary.com/robots.txt","bytes":3120,"groups":2},"bots":{"GPTBot":["partial","*",49,0],"ChatGPT-User":["partial","*",49,0],"OAI-SearchBot":["partial","*",49,0],"ClaudeBot":["partial","*",49,0],"Claude-User":["partial","*",49,0],"Claude-SearchBot":["partial","*",49,0],"anthropic-ai":["partial","*",49,0],"PerplexityBot":["partial","*",49,0],"Perplexity-User":["partial","*",49,0],"Google-Extended":["partial","*",49,0],"Applebot-Extended":["partial","*",49,0],"CCBot":["partial","*",49,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["partial","*",49,0],"meta-externalagent":["partial","*",49,0],"cohere-ai":["partial","*",49,0],"Diffbot":["partial","*",49,0],"ContextFetchBot":["partial","*",49,0]},"llms":{"status":"missing","http":404,"bytes":1245},"home":{"status":"ok","http":200,"url":"https://www.salary.com/","html_tokens":25124,"markdown_tokens":170,"pruned_tokens":170}},
{"rank":4760,"domain":"tigris.org","origin":"https://tigris.org","robots":{"status":"timeout","http":null,"url":"https://tigris.org/robots.txt","bytes":0,"groups":0,"error":"ConnectTimeout"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":4761,"domain":"rtl.nl","origin":"https://rtl.nl","robots":{"status":"ok","http":200,"url":"https://www.rtl.nl/robots.txt","bytes":3757,"groups":3},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["blocked","named",1,0],"OAI-SearchBot":["blocked","named",1,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["blocked","named",1,0],"Claude-SearchBot":["blocked","named",1,0],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["blocked","named",1,0],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",10,0]},"llms":{"status":"missing","http":404,"bytes":146},"home":{"status":"ok","http":200,"url":"https://www.rtl.nl/","html_tokens":128031,"markdown_tokens":19,"pruned_tokens":19}},
{"rank":4762,"domain":"telegram.com","origin":"https://telegram.com","robots":{"status":"ok","http":200,"url":"https://www.telegram.com/robots.txt","bytes":14170,"groups":288},"bots":{"GPTBot":["partial","named",1,4],"ChatGPT-User":["partial","named",1,4],"OAI-SearchBot":["partial","named",1,4],"ClaudeBot":["blocked","named",1,0],"Claude-User":["blocked","named",1,0],"Claude-SearchBot":["blocked","named",1,0],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["partial","named",1,4],"Perplexity-User":["blocked","named",1,0],"Google-Extended":["partial","named",1,4],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",2,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",88,0]},"llms":{"status":"http_error","http":402,"bytes":254},"home":{"status":"http_error","http":402,"url":"https://www.telegram.com/"}},
{"rank":4763,"domain":"tineye.com","origin":"https://tineye.com","robots":{"status":"ok","http":200,"url":"https://tineye.com/robots.txt","bytes":372,"groups":1},"bots":{"GPTBot":["partial","*",3,0],"ChatGPT-User":["partial","*",3,0],"OAI-SearchBot":["partial","*",3,0],"ClaudeBot":["partial","*",3,0],"Claude-User":["partial","*",3,0],"Claude-SearchBot":["partial","*",3,0],"anthropic-ai":["partial","*",3,0],"PerplexityBot":["partial","*",3,0],"Perplexity-User":["partial","*",3,0],"Google-Extended":["partial","*",3,0],"Applebot-Extended":["partial","*",3,0],"CCBot":["partial","*",3,0],"Bytespider":["partial","*",3,0],"Amazonbot":["partial","*",3,0],"meta-externalagent":["partial","*",3,0],"cohere-ai":["partial","*",3,0],"Diffbot":["partial","*",3,0],"ContextFetchBot":["partial","*",3,0]},"llms":{"status":"missing","http":404,"bytes":3147},"home":{"status":"ok","http":200,"url":"https://tineye.com/","html_tokens":1062,"markdown_tokens":28,"pruned_tokens":28}},
{"rank":4764,"domain":"lichess.org","origin":"https://lichess.org","robots":{"status":"ok","http":200,"url":"https://lichess.org/robots.txt","bytes":2965,"groups":33},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["partial","*",10,3],"OAI-SearchBot":["partial","*",10,3],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",10,3],"Claude-SearchBot":["partial","*",10,3],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["partial","*",10,3],"Perplexity-User":["partial","*",10,3],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",10,3]},"llms":{"status":"missing","http":404,"bytes":13805},"home":{"status":"http_error","http":403,"url":"https://lichess.org/"}},
{"rank":4765,"domain":"redmine.org","origin":"https://redmine.org","robots":{"status":"ok","http":200,"url":"https://www.redmine.org/robots.txt","bytes":286,"groups":3},"bots":{"GPTBot":["partial","*",7,0],"ChatGPT-User":["partial","*",7,0],"OAI-SearchBot":["partial","*",7,0],"ClaudeBot":["partial","*",7,0],"Claude-User":["partial","*",7,0],"Claude-SearchBot":["partial","*",7,0],"anthropic-ai":["partial","*",7,0],"PerplexityBot":["partial","*",7,0],"Perplexity-User":["partial","*",7,0],"Google-Extended":["partial","*",7,0],"Applebot-Extended":["partial","*",7,0],"CCBot":["partial","*",7,0],"Bytespider":["partial","*",7,0],"Amazonbot":["partial","*",7,0],"meta-externalagent":["partial","*",7,0],"cohere-ai":["partial","*",7,0],"Diffbot":["partial","*",7,0],"ContextFetchBot":["partial","*",7,0]},"llms":{"status":"missing","http":404,"bytes":459},"home":{"status":"ok","http":200,"url":"https://www.redmine.org/","html_tokens":4538,"markdown_tokens":760,"pruned_tokens":313}},
{"rank":4766,"domain":"siliconangle.com","origin":"https://siliconangle.com","robots":{"status":"ok","http":200,"url":"https://siliconangle.com/robots.txt","bytes":174,"groups":1},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"missing","http":404,"bytes":122650},"home":{"status":"ok","http":200,"url":"https://siliconangle.com/","html_tokens":57191,"markdown_tokens":76,"pruned_tokens":76}},
{"rank":4767,"domain":"corsair.com","origin":"https://corsair.com","robots":{"status":"ok","http":200,"url":"https://www.corsair.com/robots.txt","bytes":2082,"groups":11},"bots":{"GPTBot":["partial","*",6,0],"ChatGPT-User":["partial","*",6,0],"OAI-SearchBot":["allowed","named",0,1],"ClaudeBot":["partial","*",6,0],"Claude-User":["partial","*",6,0],"Claude-SearchBot":["partial","*",6,0],"anthropic-ai":["partial","*",6,0],"PerplexityBot":["partial","*",6,0],"Perplexity-User":["partial","*",6,0],"Google-Extended":["partial","*",6,0],"Applebot-Extended":["partial","*",6,0],"CCBot":["partial","*",6,0],"Bytespider":["partial","*",6,0],"Amazonbot":["partial","*",6,0],"meta-externalagent":["partial","*",6,0],"cohere-ai":["partial","*",6,0],"Diffbot":["partial","*",6,0],"ContextFetchBot":["partial","*",6,0]},"llms":{"status":"missing","http":404,"bytes":234747},"home":{"status":"ok","http":200,"url":"https://www.corsair.com/us/en","html_tokens":105635,"markdown_tokens":164,"pruned_tokens":41}},
{"rank":4768,"domain":"wonderhowto.com","origin":"https://wonderhowto.com","robots":{"status":"ok","http":200,"url":"https://www.wonderhowto.com/robots.txt","bytes":177,"groups":1},"bots":{"GPTBot":["partial","*",5,0],"ChatGPT-User":["partial","*",5,0],"OAI-SearchBot":["partial","*",5,0],"ClaudeBot":["partial","*",5,0],"Claude-User":["partial","*",5,0],"Claude-SearchBot":["partial","*",5,0],"anthropic-ai":["partial","*",5,0],"PerplexityBot":["partial","*",5,0],"Perplexity-User":["partial","*",5,0],"Google-Extended":["partial","*",5,0],"Applebot-Extended":["partial","*",5,0],"CCBot":["partial","*",5,0],"Bytespider":["partial","*",5,0],"Amazonbot":["partial","*",5,0],"meta-externalagent":["partial","*",5,0],"cohere-ai":["partial","*",5,0],"Diffbot":["partial","*",5,0],"ContextFetchBot":["partial","*",5,0]},"llms":{"status":"missing","http":404,"bytes":71054},"home":{"status":"http_error","http":403,"url":"https://www.wonderhowto.com/"}},
{"rank":4769,"domain":"christianpost.com","origin":"https://christianpost.com","robots":{"status":"ok","http":200,"url":"https://www.christianpost.com/robots.txt","bytes":220,"groups":1},"bots":{"GPTBot":["partial","*",6,1],"ChatGPT-User":["partial","*",6,1],"OAI-SearchBot":["partial","*",6,1],"ClaudeBot":["partial","*",6,1],"Claude-User":["partial","*",6,1],"Claude-SearchBot":["partial","*",6,1],"anthropic-ai":["partial","*",6,1],"PerplexityBot":["partial","*",6,1],"Perplexity-User":["partial","*",6,1],"Google-Extended":["partial","*",6,1],"Applebot-Extended":["partial","*",6,1],"CCBot":["partial","*",6,1],"Bytespider":["partial","*",6,1],"Amazonbot":["partial","*",6,1],"meta-externalagent":["partial","*",6,1],"cohere-ai":["partial","*",6,1],"Diffbot":["partial","*",6,1],"ContextFetchBot":["partial","*",6,1]},"llms":{"status":"missing","http":404,"bytes":11871},"home":{"status":"ok","http":200,"url":"https://www.christianpost.com/","html_tokens":31484,"markdown_tokens":316,"pruned_tokens":104}},
{"rank":4770,"domain":"louisiana.gov","origin":"https://louisiana.gov","robots":{"status":"missing","http":404,"url":"https://www.louisiana.gov/robots.txt","bytes":1606,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":1606},"home":{"status":"ok","http":200,"url":"https://www.louisiana.gov/","html_tokens":108881,"markdown_tokens":91,"pruned_tokens":91}},
{"rank":4771,"domain":"kongregate.com","origin":"https://kongregate.com","robots":{"status":"ok","http":200,"url":"https://www.kongregate.com/robots.txt","bytes":642,"groups":1},"bots":{"GPTBot":["partial","*",21,1],"ChatGPT-User":["partial","*",21,1],"OAI-SearchBot":["partial","*",21,1],"ClaudeBot":["partial","*",21,1],"Claude-User":["partial","*",21,1],"Claude-SearchBot":["partial","*",21,1],"anthropic-ai":["partial","*",21,1],"PerplexityBot":["partial","*",21,1],"Perplexity-User":["partial","*",21,1],"Google-Extended":["partial","*",21,1],"Applebot-Extended":["partial","*",21,1],"CCBot":["partial","*",21,1],"Bytespider":["partial","*",21,1],"Amazonbot":["partial","*",21,1],"meta-externalagent":["partial","*",21,1],"cohere-ai":["partial","*",21,1],"Diffbot":["partial","*",21,1],"ContextFetchBot":["partial","*",21,1]},"llms":{"status":"missing","http":404,"bytes":3550},"home":{"status":"ok","http":200,"url":"https://www.kongregate.com/en","html_tokens":326816,"markdown_tokens":184,"pruned_tokens":160}},
{"rank":4772,"domain":"eatright.org","origin":"https://eatright.org","robots":{"status":"ok","http":200,"url":"https://www.eatright.org/robots.txt","bytes":315,"groups":1},"bots":{"GPTBot":["partial","*",10,1],"ChatGPT-User":["partial","*",10,1],"OAI-SearchBot":["partial","*",10,1],"ClaudeBot":["partial","*",10,1],"Claude-User":["partial","*",10,1],"Claude-SearchBot":["partial","*",10,1],"anthropic-ai":["partial","*",10,1],"PerplexityBot":["partial","*",10,1],"Perplexity-User":["partial","*",10,1],"Google-Extended":["partial","*",10,1],"Applebot-Extended":["partial","*",10,1],"CCBot":["partial","*",10,1],"Bytespider":["partial","*",10,1],"Amazonbot":["partial","*",10,1],"meta-externalagent":["partial","*",10,1],"cohere-ai":["partial","*",10,1],"Diffbot":["partial","*",10,1],"ContextFetchBot":["partial","*",10,1]},"llms":{"status":"missing","http":404,"bytes":1245},"home":{"status":"ok","http":200,"url":"https://www.eatright.org/","html_tokens":47356,"markdown_tokens":131,"pruned_tokens":131}},
{"rank":4773,"domain":"export.gov","origin":"https://export.gov","robots":{"status":"unreachable","http":null,"url":"https://export.gov/robots.txt","bytes":0,"groups":0,"error":"cannot resolve export.gov"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":4774,"domain":"patheos.com","origin":"https://patheos.com","robots":{"status":"ok","http":200,"url":"https://www.patheos.com/robots.txt","bytes":1210,"groups":5},"bots":{"GPTBot":["partial","*",39,0],"ChatGPT-User":["partial","*",39,0],"OAI-SearchBot":["partial","*",39,0],"ClaudeBot":["partial","*",39,0],"Claude-User":["partial","*",39,0],"Claude-SearchBot":["partial","*",39,0],"anthropic-ai":["partial","*",39,0],"PerplexityBot":["partial","*",39,0],"Perplexity-User":["partial","*",39,0],"Google-Extended":["partial","*",39,0],"Applebot-Extended":["partial","*",39,0],"CCBot":["partial","*",39,0],"Bytespider":["allowed","named",0,0],"Amazonbot":["partial","*",39,0],"meta-externalagent":["partial","*",39,0],"cohere-ai":["partial","*",39,0],"Diffbot":["partial","*",39,0],"ContextFetchBot":["partial","*",39,0]},"llms":{"status":"missing","http":404,"bytes":58230},"home":{"status":"http_error","http":403,"url":"https://www.patheos.com/"}},
{"rank":4775,"domain":"tv5monde.com","origin":"https://tv5monde.com","robots":{"status":"refused","http":403,"url":"https://www.tv5monde.com/robots.txt","bytes":384,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":4776,"domain":"tucows.com","origin":"https://tucows.com","robots":{"status":"ok","http":200,"url":"https://tucows.com/robots.txt","bytes":0,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":47911},"home":{"status":"ok","http":200,"url":"https://tucows.com/","html_tokens":21360,"markdown_tokens":69,"pruned_tokens":69}},
{"rank":4777,"domain":"buffalonews.com","origin":"https://buffalonews.com","robots":{"status":"ok","http":200,"url":"https://buffalonews.com/robots.txt","bytes":8129,"groups":31},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["blocked","named",1,0],"OAI-SearchBot":["blocked","named",1,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",103,0],"Claude-SearchBot":["partial","*",103,0],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["partial","*",103,0],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["partial","*",103,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",103,0]},"llms":{"status":"http_error","http":429,"bytes":64},"home":{"status":"ok","http":200,"url":"https://buffalonews.com/","html_tokens":157345,"markdown_tokens":863,"pruned_tokens":42}},
{"rank":4778,"domain":"beckershospitalreview.com","origin":"https://beckershospitalreview.com","robots":{"status":"ok","http":200,"url":"https://www.beckershospitalreview.com/robots.txt","bytes":781,"groups":7},"bots":{"GPTBot":["partial","*",9,1],"ChatGPT-User":["partial","*",9,1],"OAI-SearchBot":["partial","*",9,1],"ClaudeBot":["partial","*",9,1],"Claude-User":["partial","*",9,1],"Claude-SearchBot":["partial","*",9,1],"anthropic-ai":["partial","*",9,1],"PerplexityBot":["partial","*",9,1],"Perplexity-User":["partial","*",9,1],"Google-Extended":["partial","*",9,1],"Applebot-Extended":["partial","*",9,1],"CCBot":["partial","*",9,1],"Bytespider":["partial","*",9,1],"Amazonbot":["partial","*",9,1],"meta-externalagent":["partial","*",9,1],"cohere-ai":["partial","*",9,1],"Diffbot":["partial","*",9,1],"ContextFetchBot":["partial","*",9,1]},"llms":{"status":"http_error","http":403,"bytes":785},"home":{"status":"http_error","http":403,"url":"https://www.beckershospitalreview.com/"}},
{"rank":4779,"domain":"itpro.com","origin":"https://itpro.com","robots":{"status":"ok","http":200,"url":"https://www.itpro.com/robots.txt","bytes":1854,"groups":4},"bots":{"GPTBot":["partial","*",18,0],"ChatGPT-User":["partial","*",18,0],"OAI-SearchBot":["partial","*",18,0],"ClaudeBot":["partial","*",18,0],"Claude-User":["partial","*",18,0],"Claude-SearchBot":["partial","*",18,0],"anthropic-ai":["partial","*",18,0],"PerplexityBot":["partial","*",18,0],"Perplexity-User":["partial","*",18,0],"Google-Extended":["partial","*",18,0],"Applebot-Extended":["blocked","named",3,0],"CCBot":["partial","*",18,0],"Bytespider":["blocked","named",3,0],"Amazonbot":["blocked","named",3,0],"meta-externalagent":["blocked","named",3,0],"cohere-ai":["partial","*",18,0],"Diffbot":["blocked","named",3,0],"ContextFetchBot":["partial","*",18,0]},"llms":{"status":"missing","http":404,"bytes":660360},"home":{"status":"ok","http":200,"url":"https://www.itpro.com/","html_tokens":250426,"markdown_tokens":53,"pruned_tokens":53}},
{"rank":4780,"domain":"deakin.edu.au","origin":"https://deakin.edu.au","robots":{"status":"refused","http":403,"url":"https://www.deakin.edu.au/robots.txt","bytes":5555,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":4781,"domain":"auth.gr","origin":"https://auth.gr","robots":{"status":"ok","http":200,"url":"https://auth.gr/robots.txt","bytes":122,"groups":1},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":244644},"home":{"status":"ok","http":200,"url":"https://www.auth.gr/","html_tokens":123962,"markdown_tokens":251,"pruned_tokens":175}},
{"rank":4782,"domain":"climate.gov","origin":"https://climate.gov","robots":{"status":"ok","http":200,"url":"https://www.climate.gov/robots.txt","bytes":3056,"groups":1},"bots":{"GPTBot":["partial","*",48,18],"ChatGPT-User":["partial","*",48,18],"OAI-SearchBot":["partial","*",48,18],"ClaudeBot":["partial","*",48,18],"Claude-User":["partial","*",48,18],"Claude-SearchBot":["partial","*",48,18],"anthropic-ai":["partial","*",48,18],"PerplexityBot":["partial","*",48,18],"Perplexity-User":["partial","*",48,18],"Google-Extended":["partial","*",48,18],"Applebot-Extended":["partial","*",48,18],"CCBot":["partial","*",48,18],"Bytespider":["partial","*",48,18],"Amazonbot":["partial","*",48,18],"meta-externalagent":["partial","*",48,18],"cohere-ai":["partial","*",48,18],"Diffbot":["partial","*",48,18],"ContextFetchBot":["partial","*",48,18]},"llms":{"status":"missing","http":404,"bytes":189},"home":{"status":"disallowed","error":"robots.txt of www.noaa.gov not readable (refused)"}},
{"rank":4783,"domain":"bodybuilding.com","origin":"https://bodybuilding.com","robots":{"status":"ok","http":200,"url":"https://shop.bodybuilding.com/robots.txt","bytes":5429,"groups":6},"bots":{"GPTBot":["partial","*",44,0],"ChatGPT-User":["partial","*",44,0],"OAI-SearchBot":["partial","*",44,0],"ClaudeBot":["partial","*",44,0],"Claude-User":["partial","*",44,0],"Claude-SearchBot":["partial","*",44,0],"anthropic-ai":["partial","*",44,0],"PerplexityBot":["partial","*",44,0],"Perplexity-User":["partial","*",44,0],"Google-Extended":["partial","*",44,0],"Applebot-Extended":["partial","*",44,0],"CCBot":["partial","*",44,0],"Bytespider":["partial","*",44,0],"Amazonbot":["partial","*",44,0],"meta-externalagent":["partial","*",44,0],"cohere-ai":["partial","*",44,0],"Diffbot":["partial","*",44,0],"ContextFetchBot":["partial","*",44,0]},"llms":{"status":"present","http":200,"bytes":5844},"home":{"status":"ok","http":200,"url":"https://shop.bodybuilding.com/","html_tokens":112074,"markdown_tokens":737,"pruned_tokens":42}},
{"rank":4784,"domain":"arcg.is","origin":"https://arcg.is","robots":{"status":"ok","http":200,"url":"http://bit.ly/robots.txt","bytes":123,"groups":1},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"html","http":200,"bytes":158978},"home":{"status":"ok","http":200,"url":"https://bitly.com/","html_tokens":77218,"markdown_tokens":282,"pruned_tokens":173}},
{"rank":4785,"domain":"nm.us","origin":"https://nm.us","robots":{"status":"unreachable","http":null,"url":"https://nm.us/robots.txt","bytes":0,"groups":0,"error":"cannot resolve nm.us"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":4786,"domain":"wikimediafoundation.org","origin":"https://wikimediafoundation.org","robots":{"status":"ok","http":200,"url":"https://wikimediafoundation.org/robots.txt","bytes":1741,"groups":1},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"missing","http":404,"bytes":135778},"home":{"status":"ok","http":200,"url":"https://wikimediafoundation.org/","html_tokens":57452,"markdown_tokens":388,"pruned_tokens":19}},
{"rank":4787,"domain":"ntv.ru","origin":"https://ntv.ru","robots":{"status":"ok","http":200,"url":"https://www.ntv.ru/robots.txt","bytes":959,"groups":5},"bots":{"GPTBot":["partial","*",12,0],"ChatGPT-User":["partial","*",12,0],"OAI-SearchBot":["partial","*",12,0],"ClaudeBot":["partial","*",12,0],"Claude-User":["partial","*",12,0],"Claude-SearchBot":["partial","*",12,0],"anthropic-ai":["partial","*",12,0],"PerplexityBot":["partial","*",12,0],"Perplexity-User":["partial","*",12,0],"Google-Extended":["partial","*",12,0],"Applebot-Extended":["partial","*",12,0],"CCBot":["partial","*",12,0],"Bytespider":["partial","*",12,0],"Amazonbot":["partial","*",12,0],"meta-externalagent":["partial","*",12,0],"cohere-ai":["partial","*",12,0],"Diffbot":["partial","*",12,0],"ContextFetchBot":["partial","*",12,0]},"llms":{"status":"missing","http":404,"bytes":70453},"home":{"status":"ok","http":200,"url":"https://www.ntv.ru","html_tokens":55262,"markdown_tokens":21,"pruned_tokens":21}},
{"rank":4788,"domain":"cinemablend.com","origin":"https://cinemablend.com","robots":{"status":"ok","http":200,"url":"https://www.cinemablend.com/robots.txt","bytes":1637,"groups":4},"bots":{"GPTBot":["partial","*",18,0],"ChatGPT-User":["partial","*",18,0],"OAI-SearchBot":["partial","*",18,0],"ClaudeBot":["partial","*",18,0],"Claude-User":["partial","*",18,0],"Claude-SearchBot":["partial","*",18,0],"anthropic-ai":["partial","*",18,0],"PerplexityBot":["partial","*",18,0],"Perplexity-User":["partial","*",18,0],"Google-Extended":["partial","*",18,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["partial","*",18,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["partial","*",18,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",18,0]},"llms":{"status":"missing","http":404,"bytes":533014},"home":{"status":"ok","http":200,"url":"https://www.cinemablend.com/","html_tokens":317216,"markdown_tokens":132,"pruned_tokens":52}},
{"rank":4789,"domain":"mayoclinic.com","origin":"https://mayoclinic.com","robots":{"status":"refused","http":403,"url":"https://www.mayoclinic.org/robots.txt","bytes":384,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":4790,"domain":"fcbarcelona.com","origin":"https://fcbarcelona.com","robots":{"status":"ok","http":200,"url":"https://www.fcbarcelona.com/robots.txt","bytes":156,"groups":1},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"missing","http":404,"bytes":275567},"home":{"status":"ok","http":200,"url":"https://www.fcbarcelona.com/en/","html_tokens":102585,"markdown_tokens":48,"pruned_tokens":48}},
{"rank":4791,"domain":"pca.st","origin":"https://pca.st","robots":{"status":"ok","http":200,"url":"https://pca.st/robots.txt","bytes":264,"groups":1},"bots":{"GPTBot":["partial","*",4,0],"ChatGPT-User":["partial","*",4,0],"OAI-SearchBot":["partial","*",4,0],"ClaudeBot":["partial","*",4,0],"Claude-User":["partial","*",4,0],"Claude-SearchBot":["partial","*",4,0],"anthropic-ai":["partial","*",4,0],"PerplexityBot":["partial","*",4,0],"Perplexity-User":["partial","*",4,0],"Google-Extended":["partial","*",4,0],"Applebot-Extended":["partial","*",4,0],"CCBot":["partial","*",4,0],"Bytespider":["partial","*",4,0],"Amazonbot":["partial","*",4,0],"meta-externalagent":["partial","*",4,0],"cohere-ai":["partial","*",4,0],"Diffbot":["partial","*",4,0],"ContextFetchBot":["partial","*",4,0]},"llms":{"status":"http_error","http":403,"bytes":19984},"home":{"status":"http_error","http":403,"url":"https://www.pocketcasts.com"}},
{"rank":4792,"domain":"mises.org","origin":"https://mises.org","robots":{"status":"refused","http":403,"url":"https://mises.org/robots.txt","bytes":5546,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":4793,"domain":"govtech.com","origin":"https://govtech.com","robots":{"status":"ok","http":200,"url":"https://www.govtech.com/robots.txt","bytes":823,"groups":17},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["allowed","named",0,1],"OAI-SearchBot":["allowed","named",0,1],"ClaudeBot":["blocked","named",1,0],"Claude-User":["allowed","named",0,1],"Claude-SearchBot":["allowed","named",0,1],"anthropic-ai":["allowed","*",0,1],"PerplexityBot":["allowed","named",0,1],"Perplexity-User":["allowed","named",0,1],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["allowed","*",0,1],"ContextFetchBot":["allowed","*",0,1]},"llms":{"status":"present","http":200,"bytes":2187},"home":{"status":"ok","http":200,"url":"https://www.govtech.com/","html_tokens":57739,"markdown_tokens":1182,"pruned_tokens":168}},
{"rank":4794,"domain":"cas.cz","origin":"https://cas.cz","robots":{"status":"ok","http":200,"url":"https://www.avcr.cz/robots.txt","bytes":1616,"groups":1},"bots":{"GPTBot":["partial","*",21,0],"ChatGPT-User":["partial","*",21,0],"OAI-SearchBot":["partial","*",21,0],"ClaudeBot":["partial","*",21,0],"Claude-User":["partial","*",21,0],"Claude-SearchBot":["partial","*",21,0],"anthropic-ai":["partial","*",21,0],"PerplexityBot":["partial","*",21,0],"Perplexity-User":["partial","*",21,0],"Google-Extended":["partial","*",21,0],"Applebot-Extended":["partial","*",21,0],"CCBot":["partial","*",21,0],"Bytespider":["partial","*",21,0],"Amazonbot":["partial","*",21,0],"meta-externalagent":["partial","*",21,0],"cohere-ai":["partial","*",21,0],"Diffbot":["partial","*",21,0],"ContextFetchBot":["partial","*",21,0]},"llms":{"status":"missing","http":404,"bytes":3064},"home":{"status":"ok","http":200,"url":"https://www.avcr.cz/cs/","html_tokens":21760,"markdown_tokens":410,"pruned_tokens":170}},
{"rank":4795,"domain":"mindmeister.com","origin":"https://mindmeister.com","robots":{"status":"ok","http":200,"url":"https://www.mindmeister.com/robots.txt","bytes":87,"groups":1},"bots":{"GPTBot":["partial","*",1,1],"ChatGPT-User":["partial","*",1,1],"OAI-SearchBot":["partial","*",1,1],"ClaudeBot":["partial","*",1,1],"Claude-User":["partial","*",1,1],"Claude-SearchBot":["partial","*",1,1],"anthropic-ai":["partial","*",1,1],"PerplexityBot":["partial","*",1,1],"Perplexity-User":["partial","*",1,1],"Google-Extended":["partial","*",1,1],"Applebot-Extended":["partial","*",1,1],"CCBot":["partial","*",1,1],"Bytespider":["partial","*",1,1],"Amazonbot":["partial","*",1,1],"meta-externalagent":["partial","*",1,1],"cohere-ai":["partial","*",1,1],"Diffbot":["partial","*",1,1],"ContextFetchBot":["partial","*",1,1]},"llms":{"status":"missing","http":404,"bytes":0},"home":{"status":"ok","http":200,"url":"https://www.mindmeister.com/","html_tokens":203951,"markdown_tokens":1086,"pruned_tokens":505}},
{"rank":4796,"domain":"mailinabox.email","origin":"https://mailinabox.email","robots":{"status":"missing","http":404,"url":"https://mailinabox.email/robots.txt","bytes":146,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":146},"home":{"status":"ok","http":200,"url":"https://mailinabox.email/","html_tokens":5222,"markdown_tokens":1689,"pruned_tokens":33}},
{"rank":4797,"domain":"mundodeportivo.com","origin":"https://mundodeportivo.com","robots":{"status":"ok","http":200,"url":"https://www.mundodeportivo.com/robots.txt","bytes":149288,"groups":4},"bots":{"GPTBot":["partial","*",2236,19],"ChatGPT-User":["partial","*",2236,19],"OAI-SearchBot":["partial","*",2236,19],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",2236,19],"Claude-SearchBot":["partial","*",2236,19],"anthropic-ai":["partial","*",2236,19],"PerplexityBot":["partial","*",2236,19],"Perplexity-User":["partial","*",2236,19],"Google-Extended":["partial","*",2236,19],"Applebot-Extended":["partial","*",2236,19],"CCBot":["partial","*",2236,19],"Bytespider":["partial","*",2236,19],"Amazonbot":["partial","*",2236,19],"meta-externalagent":["partial","*",2236,19],"cohere-ai":["partial","*",2236,19],"Diffbot":["partial","*",2236,19],"ContextFetchBot":["partial","*",2236,19]},"llms":{"status":"missing","http":404,"bytes":59498},"home":{"status":"ok","http":200,"url":"https://www.mundodeportivo.com/","html_tokens":131566,"markdown_tokens":48,"pruned_tokens":30}},
{"rank":4798,"domain":"db.tt","origin":"https://db.tt","robots":{"status":"ok","http":200,"url":"https://db.tt/robots.txt","bytes":801,"groups":3},"bots":{"GPTBot":["partial","*",23,4],"ChatGPT-User":["partial","*",23,4],"OAI-SearchBot":["partial","*",23,4],"ClaudeBot":["partial","*",23,4],"Claude-User":["partial","*",23,4],"Claude-SearchBot":["partial","*",23,4],"anthropic-ai":["partial","*",23,4],"PerplexityBot":["partial","*",23,4],"Perplexity-User":["partial","*",23,4],"Google-Extended":["partial","*",23,4],"Applebot-Extended":["partial","*",23,4],"CCBot":["partial","*",23,4],"Bytespider":["partial","*",23,4],"Amazonbot":["partial","*",23,4],"meta-externalagent":["partial","*",23,4],"cohere-ai":["partial","*",23,4],"Diffbot":["partial","*",23,4],"ContextFetchBot":["partial","*",23,4]},"llms":{"status":"present","http":200,"bytes":209330},"home":{"status":"ok","http":200,"url":"https://www.dropbox.com","html_tokens":158217,"markdown_tokens":221,"pruned_tokens":111}},
{"rank":4799,"domain":"simplemachines.org","origin":"https://simplemachines.org","robots":{"status":"refused","http":403,"url":"https://simplemachines.org/robots.txt","bytes":5535,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":4800,"domain":"money.com","origin":"https://money.com","robots":{"status":"ok","http":200,"url":"https://money.com/robots.txt","bytes":3934,"groups":16},"bots":{"GPTBot":["partial","named",8,1],"ChatGPT-User":["allowed","named",0,1],"OAI-SearchBot":["partial","*",12,0],"ClaudeBot":["partial","named",8,1],"Claude-User":["partial","*",12,0],"Claude-SearchBot":["partial","*",12,0],"anthropic-ai":["allowed","named",0,1],"PerplexityBot":["partial","named",8,1],"Perplexity-User":["partial","*",12,0],"Google-Extended":["allowed","named",0,1],"Applebot-Extended":["allowed","named",0,1],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["partial","*",12,0],"cohere-ai":["allowed","named",0,1],"Diffbot":["partial","*",12,0],"ContextFetchBot":["partial","*",12,0]},"llms":{"status":"present","http":200,"bytes":5801},"home":{"status":"ok","http":200,"url":"https://money.com/","html_tokens":73246,"markdown_tokens":52,"pruned_tokens":52}},
{"rank":4801,"domain":"jewishvirtuallibrary.org","origin":"https://jewishvirtuallibrary.org","robots":{"status":"ok","http":200,"url":"https://jewishvirtuallibrary.org/robots.txt","bytes":24,"groups":1},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"html","http":200,"bytes":26716},"home":{"status":"ok","http":200,"url":"https://jewishvirtuallibrary.org/","html_tokens":14946,"markdown_tokens":123,"pruned_tokens":123}},
{"rank":4802,"domain":"percona.com","origin":"https://percona.com","robots":{"status":"ok","http":200,"url":"https://www.percona.com/robots.txt","bytes":251,"groups":2},"bots":{"GPTBot":["partial","*",1,0],"ChatGPT-User":["partial","*",1,0],"OAI-SearchBot":["partial","*",1,0],"ClaudeBot":["partial","*",1,0],"Claude-User":["partial","*",1,0],"Claude-SearchBot":["partial","*",1,0],"anthropic-ai":["partial","*",1,0],"PerplexityBot":["partial","*",1,0],"Perplexity-User":["partial","*",1,0],"Google-Extended":["partial","*",1,0],"Applebot-Extended":["partial","*",1,0],"CCBot":["partial","*",1,0],"Bytespider":["partial","*",1,0],"Amazonbot":["partial","*",1,0],"meta-externalagent":["partial","*",1,0],"cohere-ai":["partial","*",1,0],"Diffbot":["partial","*",1,0],"ContextFetchBot":["partial","*",1,0]},"llms":{"status":"other_text","http":200,"bytes":12170},"home":{"status":"ok","http":200,"url":"https://www.percona.com/","html_tokens":81457,"markdown_tokens":140,"pruned_tokens":42}},
{"rank":4803,"domain":"uidaho.edu","origin":"https://uidaho.edu","robots":{"status":"ok","http":200,"url":"https://www.uidaho.edu/robots.txt","bytes":70,"groups":1},"bots":{"GPTBot":["allowed","*",0,1],"ChatGPT-User":["allowed","*",0,1],"OAI-SearchBot":["allowed","*",0,1],"ClaudeBot":["allowed","*",0,1],"Claude-User":["allowed","*",0,1],"Claude-SearchBot":["allowed","*",0,1],"anthropic-ai":["allowed","*",0,1],"PerplexityBot":["allowed","*",0,1],"Perplexity-User":["allowed","*",0,1],"Google-Extended":["allowed","*",0,1],"Applebot-Extended":["allowed","*",0,1],"CCBot":["allowed","*",0,1],"Bytespider":["allowed","*",0,1],"Amazonbot":["allowed","*",0,1],"meta-externalagent":["allowed","*",0,1],"cohere-ai":["allowed","*",0,1],"Diffbot":["allowed","*",0,1],"ContextFetchBot":["allowed","*",0,1]},"llms":{"status":"http_error","http":403,"bytes":7255},"home":{"status":"ok","http":200,"url":"https://www.uidaho.edu/","html_tokens":40658,"markdown_tokens":368,"pruned_tokens":146}},
{"rank":4804,"domain":"unizar.es","origin":"https://unizar.es","robots":{"status":"ok","http":200,"url":"https://www.unizar.es/robots.txt","bytes":2027,"groups":1},"bots":{"GPTBot":["partial","*",34,18],"ChatGPT-User":["partial","*",34,18],"OAI-SearchBot":["partial","*",34,18],"ClaudeBot":["partial","*",34,18],"Claude-User":["partial","*",34,18],"Claude-SearchBot":["partial","*",34,18],"anthropic-ai":["partial","*",34,18],"PerplexityBot":["partial","*",34,18],"Perplexity-User":["partial","*",34,18],"Google-Extended":["partial","*",34,18],"Applebot-Extended":["partial","*",34,18],"CCBot":["partial","*",34,18],"Bytespider":["partial","*",34,18],"Amazonbot":["partial","*",34,18],"meta-externalagent":["partial","*",34,18],"cohere-ai":["partial","*",34,18],"Diffbot":["partial","*",34,18],"ContextFetchBot":["partial","*",34,18]},"llms":{"status":"http_error","http":403,"bytes":153},"home":{"status":"ok","http":200,"url":"https://www.unizar.es/","html_tokens":49965,"markdown_tokens":225,"pruned_tokens":90}},
{"rank":4805,"domain":"jrj.com.cn","origin":"https://jrj.com.cn","robots":{"status":"ok","http":200,"url":"https://jrj.com.cn/robots.txt","bytes":3250,"groups":20},"bots":{"GPTBot":["partial","*",16,0],"ChatGPT-User":["partial","*",16,0],"OAI-SearchBot":["partial","*",16,0],"ClaudeBot":["partial","*",16,0],"Claude-User":["partial","*",16,0],"Claude-SearchBot":["partial","*",16,0],"anthropic-ai":["partial","*",16,0],"PerplexityBot":["partial","*",16,0],"Perplexity-User":["partial","*",16,0],"Google-Extended":["partial","*",16,0],"Applebot-Extended":["partial","*",16,0],"CCBot":["partial","*",16,0],"Bytespider":["partial","named",16,0],"Amazonbot":["partial","*",16,0],"meta-externalagent":["partial","*",16,0],"cohere-ai":["partial","*",16,0],"Diffbot":["partial","*",16,0],"ContextFetchBot":["partial","*",16,0]},"llms":{"status":"missing","http":404,"bytes":315},"home":{"status":"ok","http":200,"url":"https://jrj.com.cn/","html_tokens":10517,"markdown_tokens":10,"pruned_tokens":10}},
{"rank":4806,"domain":"sherwin-williams.com","origin":"https://sherwin-williams.com","robots":{"status":"ok","http":200,"url":"https://www.sherwin-williams.com/robots.txt","bytes":3023,"groups":1},"bots":{"GPTBot":["partial","*",84,0],"ChatGPT-User":["partial","*",84,0],"OAI-SearchBot":["partial","*",84,0],"ClaudeBot":["partial","*",84,0],"Claude-User":["partial","*",84,0],"Claude-SearchBot":["partial","*",84,0],"anthropic-ai":["partial","*",84,0],"PerplexityBot":["partial","*",84,0],"Perplexity-User":["partial","*",84,0],"Google-Extended":["partial","*",84,0],"Applebot-Extended":["partial","*",84,0],"CCBot":["partial","*",84,0],"Bytespider":["partial","*",84,0],"Amazonbot":["partial","*",84,0],"meta-externalagent":["partial","*",84,0],"cohere-ai":["partial","*",84,0],"Diffbot":["partial","*",84,0],"ContextFetchBot":["partial","*",84,0]},"llms":{"status":"http_error","http":403,"bytes":345},"home":{"status":"ok","http":200,"url":"https://www.sherwin-williams.com/","html_tokens":182496,"markdown_tokens":238,"pruned_tokens":39}},
{"rank":4807,"domain":"numpy.org","origin":"https://numpy.org","robots":{"status":"ok","http":200,"url":"https://numpy.org/robots.txt","bytes":1248,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":9746},"home":{"status":"ok","http":200,"url":"https://numpy.org/","html_tokens":9353,"markdown_tokens":935,"pruned_tokens":315}},
{"rank":4808,"domain":"nippon.com","origin":"https://nippon.com","robots":{"status":"ok","http":200,"url":"https://www.nippon.com/robots.txt","bytes":1118,"groups":6},"bots":{"GPTBot":["partial","*",4,0],"ChatGPT-User":["partial","*",4,0],"OAI-SearchBot":["partial","*",4,0],"ClaudeBot":["partial","*",4,0],"Claude-User":["partial","*",4,0],"Claude-SearchBot":["partial","*",4,0],"anthropic-ai":["partial","*",4,0],"PerplexityBot":["partial","*",4,0],"Perplexity-User":["partial","*",4,0],"Google-Extended":["partial","*",4,0],"Applebot-Extended":["partial","*",4,0],"CCBot":["partial","*",4,0],"Bytespider":["partial","*",4,0],"Amazonbot":["partial","*",4,0],"meta-externalagent":["partial","*",4,0],"cohere-ai":["partial","*",4,0],"Diffbot":["partial","*",4,0],"ContextFetchBot":["partial","*",4,0]},"llms":{"status":"missing","http":404,"bytes":6560},"home":{"status":"ok","http":200,"url":"https://www.nippon.com/en/","html_tokens":7254,"markdown_tokens":57,"pruned_tokens":50}},
{"rank":4809,"domain":"aamc.org","origin":"https://aamc.org","robots":{"status":"ok","http":200,"url":"https://www.aamc.org/robots.txt","bytes":2980,"groups":2},"bots":{"GPTBot":["partial","*",63,18],"ChatGPT-User":["partial","*",63,18],"OAI-SearchBot":["partial","*",63,18],"ClaudeBot":["partial","*",63,18],"Claude-User":["partial","*",63,18],"Claude-SearchBot":["partial","*",63,18],"anthropic-ai":["partial","*",63,18],"PerplexityBot":["partial","*",63,18],"Perplexity-User":["partial","*",63,18],"Google-Extended":["partial","*",63,18],"Applebot-Extended":["partial","*",63,18],"CCBot":["partial","*",63,18],"Bytespider":["partial","*",63,18],"Amazonbot":["partial","*",63,18],"meta-externalagent":["partial","named",9,0],"cohere-ai":["partial","*",63,18],"Diffbot":["partial","*",63,18],"ContextFetchBot":["partial","*",63,18]},"llms":{"status":"missing","http":404,"bytes":186},"home":{"status":"ok","http":200,"url":"https://www.aamc.org/","html_tokens":42188,"markdown_tokens":623,"pruned_tokens":318}},
{"rank":4810,"domain":"cepal.org","origin":"https://cepal.org","robots":{"status":"ok","http":200,"url":"https://www.cepal.org/robots.txt","bytes":2324,"groups":3},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["partial","*",38,18],"OAI-SearchBot":["partial","*",38,18],"ClaudeBot":["partial","*",38,18],"Claude-User":["partial","*",38,18],"Claude-SearchBot":["partial","*",38,18],"anthropic-ai":["partial","*",38,18],"PerplexityBot":["partial","*",38,18],"Perplexity-User":["partial","*",38,18],"Google-Extended":["partial","*",38,18],"Applebot-Extended":["partial","*",38,18],"CCBot":["partial","*",38,18],"Bytespider":["partial","*",38,18],"Amazonbot":["partial","*",38,18],"meta-externalagent":["partial","*",38,18],"cohere-ai":["partial","*",38,18],"Diffbot":["partial","*",38,18],"ContextFetchBot":["partial","*",38,18]},"llms":{"status":"http_error","http":403,"bytes":146},"home":{"status":"http_error","http":403,"url":"https://www.cepal.org/"}},
{"rank":4811,"domain":"mobileslotsite.co.uk","origin":"https://mobileslotsite.co.uk","robots":{"status":"ok","http":200,"url":"https://mobileslotsite.co.uk/robots.txt","bytes":545,"groups":1},"bots":{"GPTBot":["partial","*",19,0],"ChatGPT-User":["partial","*",19,0],"OAI-SearchBot":["partial","*",19,0],"ClaudeBot":["partial","*",19,0],"Claude-User":["partial","*",19,0],"Claude-SearchBot":["partial","*",19,0],"anthropic-ai":["partial","*",19,0],"PerplexityBot":["partial","*",19,0],"Perplexity-User":["partial","*",19,0],"Google-Extended":["partial","*",19,0],"Applebot-Extended":["partial","*",19,0],"CCBot":["partial","*",19,0],"Bytespider":["partial","*",19,0],"Amazonbot":["partial","*",19,0],"meta-externalagent":["partial","*",19,0],"cohere-ai":["partial","*",19,0],"Diffbot":["partial","*",19,0],"ContextFetchBot":["partial","*",19,0]},"llms":{"status":"missing","http":404,"bytes":15385},"home":{"status":"ok","http":200,"url":"https://mobileslotsite.co.uk/","html_tokens":115520,"markdown_tokens":1454,"pruned_tokens":168}},
{"rank":4812,"domain":"spain.info","origin":"https://spain.info","robots":{"status":"ok","http":200,"url":"https://www.spain.info/robots.txt","bytes":5537,"groups":111},"bots":{"GPTBot":["partial","*",23,6],"ChatGPT-User":["partial","*",23,6],"OAI-SearchBot":["partial","*",23,6],"ClaudeBot":["partial","*",23,6],"Claude-User":["partial","*",23,6],"Claude-SearchBot":["partial","*",23,6],"anthropic-ai":["partial","*",23,6],"PerplexityBot":["partial","*",23,6],"Perplexity-User":["partial","*",23,6],"Google-Extended":["partial","*",23,6],"Applebot-Extended":["partial","*",23,6],"CCBot":["partial","*",23,6],"Bytespider":["partial","*",23,6],"Amazonbot":["partial","*",23,6],"meta-externalagent":["partial","*",23,6],"cohere-ai":["partial","*",23,6],"Diffbot":["partial","*",23,6],"ContextFetchBot":["partial","*",23,6]},"llms":{"status":"missing","http":404,"bytes":17165},"home":{"status":"ok","http":200,"url":"https://www.spain.info/","html_tokens":49247,"markdown_tokens":67,"pruned_tokens":14}},
{"rank":4813,"domain":"ozerov.de","origin":"https://ozerov.de","robots":{"status":"ok","http":200,"url":"https://www.ozerov.de/robots.txt","bytes":104,"groups":1},"bots":{"GPTBot":["partial","*",3,0],"ChatGPT-User":["partial","*",3,0],"OAI-SearchBot":["partial","*",3,0],"ClaudeBot":["partial","*",3,0],"Claude-User":["partial","*",3,0],"Claude-SearchBot":["partial","*",3,0],"anthropic-ai":["partial","*",3,0],"PerplexityBot":["partial","*",3,0],"Perplexity-User":["partial","*",3,0],"Google-Extended":["partial","*",3,0],"Applebot-Extended":["partial","*",3,0],"CCBot":["partial","*",3,0],"Bytespider":["partial","*",3,0],"Amazonbot":["partial","*",3,0],"meta-externalagent":["partial","*",3,0],"cohere-ai":["partial","*",3,0],"Diffbot":["partial","*",3,0],"ContextFetchBot":["partial","*",3,0]},"llms":{"status":"missing","http":404,"bytes":21401},"home":{"status":"ok","http":200,"url":"https://www.fully-kiosk.com","html_tokens":6229,"markdown_tokens":211,"pruned_tokens":120}},
{"rank":4814,"domain":"lancaster.ac.uk","origin":"https://lancaster.ac.uk","robots":{"status":"ok","http":200,"url":"https://www.lancaster.ac.uk/robots.txt","bytes":2283,"groups":2},"bots":{"GPTBot":["partial","*",54,6],"ChatGPT-User":["partial","*",54,6],"OAI-SearchBot":["partial","*",54,6],"ClaudeBot":["partial","*",54,6],"Claude-User":["partial","*",54,6],"Claude-SearchBot":["partial","*",54,6],"anthropic-ai":["partial","*",54,6],"PerplexityBot":["partial","*",54,6],"Perplexity-User":["partial","*",54,6],"Google-Extended":["partial","*",54,6],"Applebot-Extended":["partial","*",54,6],"CCBot":["partial","*",54,6],"Bytespider":["partial","*",54,6],"Amazonbot":["partial","*",54,6],"meta-externalagent":["partial","*",54,6],"cohere-ai":["partial","*",54,6],"Diffbot":["partial","*",54,6],"ContextFetchBot":["partial","*",54,6]},"llms":{"status":"missing","http":404,"bytes":82961},"home":{"status":"ok","http":200,"url":"https://www.lancaster.ac.uk/","html_tokens":27491,"markdown_tokens":157,"pruned_tokens":81}},
{"rank":4815,"domain":"publico.es","origin":"https://publico.es","robots":{"status":"ok","http":200,"url":"https://www.publico.es/robots.txt","bytes":11772,"groups":4},"bots":{"GPTBot":["partial","*",149,0],"ChatGPT-User":["partial","*",149,0],"OAI-SearchBot":["partial","*",149,0],"ClaudeBot":["partial","*",149,0],"Claude-User":["partial","*",149,0],"Claude-SearchBot":["partial","*",149,0],"anthropic-ai":["partial","*",149,0],"PerplexityBot":["partial","*",149,0],"Perplexity-User":["partial","*",149,0],"Google-Extended":["partial","*",149,0],"Applebot-Extended":["partial","*",149,0],"CCBot":["partial","*",149,0],"Bytespider":["partial","*",149,0],"Amazonbot":["partial","*",149,0],"meta-externalagent":["partial","*",149,0],"cohere-ai":["partial","*",149,0],"Diffbot":["partial","*",149,0],"ContextFetchBot":["partial","*",149,0]},"llms":{"status":"missing","http":404,"bytes":185117},"home":{"status":"ok","http":200,"url":"https://www.publico.es/","html_tokens":132002,"markdown_tokens":116,"pruned_tokens":65}},
{"rank":4816,"domain":"thehotline.org","origin":"https://thehotline.org","robots":{"status":"ok","http":200,"url":"https://www.thehotline.org/robots.txt","bytes":251,"groups":1},"bots":{"GPTBot":["partial","*",1,0],"ChatGPT-User":["partial","*",1,0],"OAI-SearchBot":["partial","*",1,0],"ClaudeBot":["partial","*",1,0],"Claude-User":["partial","*",1,0],"Claude-SearchBot":["partial","*",1,0],"anthropic-ai":["partial","*",1,0],"PerplexityBot":["partial","*",1,0],"Perplexity-User":["partial","*",1,0],"Google-Extended":["partial","*",1,0],"Applebot-Extended":["partial","*",1,0],"CCBot":["partial","*",1,0],"Bytespider":["partial","*",1,0],"Amazonbot":["partial","*",1,0],"meta-externalagent":["partial","*",1,0],"cohere-ai":["partial","*",1,0],"Diffbot":["partial","*",1,0],"ContextFetchBot":["partial","*",1,0]},"llms":{"status":"missing","http":404,"bytes":60},"home":{"status":"ok","http":200,"url":"https://www.thehotline.org/","html_tokens":25126,"markdown_tokens":85,"pruned_tokens":85}},
{"rank":4817,"domain":"internationalwomensday.com","origin":"https://internationalwomensday.com","robots":{"status":"ok","http":200,"url":"https://internationalwomensday.com/robots.txt","bytes":121,"groups":1},"bots":{"GPTBot":["partial","*",1,1],"ChatGPT-User":["partial","*",1,1],"OAI-SearchBot":["partial","*",1,1],"ClaudeBot":["partial","*",1,1],"Claude-User":["partial","*",1,1],"Claude-SearchBot":["partial","*",1,1],"anthropic-ai":["partial","*",1,1],"PerplexityBot":["partial","*",1,1],"Perplexity-User":["partial","*",1,1],"Google-Extended":["partial","*",1,1],"Applebot-Extended":["partial","*",1,1],"CCBot":["partial","*",1,1],"Bytespider":["partial","*",1,1],"Amazonbot":["partial","*",1,1],"meta-externalagent":["partial","*",1,1],"cohere-ai":["partial","*",1,1],"Diffbot":["partial","*",1,1],"ContextFetchBot":["partial","*",1,1]},"llms":{"status":"html","http":200,"bytes":16865},"home":{"status":"ok","http":200,"url":"https://internationalwomensday.com/","html_tokens":9385,"markdown_tokens":122,"pruned_tokens":122}},
{"rank":4818,"domain":"tomsk.ru","origin":"https://tomsk.ru","robots":{"status":"ok","http":200,"url":"http://www.tomsk.ru/robots.txt","bytes":412,"groups":1},"bots":{"GPTBot":["partial","*",9,6],"ChatGPT-User":["partial","*",9,6],"OAI-SearchBot":["partial","*",9,6],"ClaudeBot":["partial","*",9,6],"Claude-User":["partial","*",9,6],"Claude-SearchBot":["partial","*",9,6],"anthropic-ai":["partial","*",9,6],"PerplexityBot":["partial","*",9,6],"Perplexity-User":["partial","*",9,6],"Google-Extended":["partial","*",9,6],"Applebot-Extended":["partial","*",9,6],"CCBot":["partial","*",9,6],"Bytespider":["partial","*",9,6],"Amazonbot":["partial","*",9,6],"meta-externalagent":["partial","*",9,6],"cohere-ai":["partial","*",9,6],"Diffbot":["partial","*",9,6],"ContextFetchBot":["partial","*",9,6]},"llms":{"status":"missing","http":404,"bytes":46249},"home":{"status":"ok","http":200,"url":"https://www.tomsk.ru/","html_tokens":16999,"markdown_tokens":243,"pruned_tokens":85}},
{"rank":4819,"domain":"9news.com.au","origin":"https://9news.com.au","robots":{"status":"refused","http":406,"url":"https://www.9news.com.au/robots.txt","bytes":0,"groups":0,"error":"HTTP 406"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":4820,"domain":"playcasinoonline.ca","origin":"https://playcasinoonline.ca","robots":{"status":"unreachable","http":null,"url":"https://playcasinoonline.ca/robots.txt","bytes":0,"groups":0,"error":"ConnectError"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":4821,"domain":"optica.org","origin":"https://optica.org","robots":{"status":"missing","http":404,"url":"https://www.optica.org/robots.txt","bytes":90079,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"html","http":200,"bytes":120796},"home":{"status":"ok","http":200,"url":"https://www.optica.org/","html_tokens":30199,"markdown_tokens":316,"pruned_tokens":220}},
{"rank":4822,"domain":"fandango.com","origin":"https://fandango.com","robots":{"status":"ok","http":200,"url":"https://www.fandango.com/robots.txt","bytes":1835,"groups":2},"bots":{"GPTBot":["partial","*",41,2],"ChatGPT-User":["partial","*",41,2],"OAI-SearchBot":["partial","*",41,2],"ClaudeBot":["partial","*",41,2],"Claude-User":["partial","*",41,2],"Claude-SearchBot":["partial","*",41,2],"anthropic-ai":["partial","*",41,2],"PerplexityBot":["partial","*",41,2],"Perplexity-User":["partial","*",41,2],"Google-Extended":["partial","*",41,2],"Applebot-Extended":["partial","*",41,2],"CCBot":["partial","*",41,2],"Bytespider":["partial","*",41,2],"Amazonbot":["partial","*",41,2],"meta-externalagent":["partial","*",41,2],"cohere-ai":["partial","*",41,2],"Diffbot":["partial","*",41,2],"ContextFetchBot":["partial","*",41,2]},"llms":{"status":"missing","http":404,"bytes":3144},"home":{"status":"ok","http":200,"url":"https://www.fandango.com/","html_tokens":63211,"markdown_tokens":1993,"pruned_tokens":1351}},
{"rank":4823,"domain":"milb.com","origin":"https://milb.com","robots":{"status":"ok","http":200,"url":"https://www.milb.com/robots.txt","bytes":25,"groups":1},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"missing","http":404,"bytes":119890},"home":{"status":"ok","http":200,"url":"https://www.mlb.com/milb","html_tokens":272749,"markdown_tokens":48,"pruned_tokens":48}},
{"rank":4824,"domain":"4.cn","origin":"https://4.cn","robots":{"status":"html","http":200,"url":"https://www.4.cn/","bytes":47962,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":4825,"domain":"inrae.fr","origin":"https://inrae.fr","robots":{"status":"ok","http":200,"url":"https://www.inrae.fr/robots.txt","bytes":1917,"groups":1},"bots":{"GPTBot":["partial","*",27,18],"ChatGPT-User":["partial","*",27,18],"OAI-SearchBot":["partial","*",27,18],"ClaudeBot":["partial","*",27,18],"Claude-User":["partial","*",27,18],"Claude-SearchBot":["partial","*",27,18],"anthropic-ai":["partial","*",27,18],"PerplexityBot":["partial","*",27,18],"Perplexity-User":["partial","*",27,18],"Google-Extended":["partial","*",27,18],"Applebot-Extended":["partial","*",27,18],"CCBot":["partial","*",27,18],"Bytespider":["partial","*",27,18],"Amazonbot":["partial","*",27,18],"meta-externalagent":["partial","*",27,18],"cohere-ai":["partial","*",27,18],"Diffbot":["partial","*",27,18],"ContextFetchBot":["partial","*",27,18]},"llms":{"status":"missing","http":404,"bytes":186},"home":{"status":"ok","http":200,"url":"https://www.inrae.fr/","html_tokens":67997,"markdown_tokens":243,"pruned_tokens":39}},
{"rank":4826,"domain":"sanity.io","origin":"https://sanity.io","robots":{"status":"ok","http":200,"url":"https://www.sanity.io/robots.txt","bytes":296,"groups":1},"bots":{"GPTBot":["partial","*",8,1],"ChatGPT-User":["partial","*",8,1],"OAI-SearchBot":["partial","*",8,1],"ClaudeBot":["partial","*",8,1],"Claude-User":["partial","*",8,1],"Claude-SearchBot":["partial","*",8,1],"anthropic-ai":["partial","*",8,1],"PerplexityBot":["partial","*",8,1],"Perplexity-User":["partial","*",8,1],"Google-Extended":["partial","*",8,1],"Applebot-Extended":["partial","*",8,1],"CCBot":["partial","*",8,1],"Bytespider":["partial","*",8,1],"Amazonbot":["partial","*",8,1],"meta-externalagent":["partial","*",8,1],"cohere-ai":["partial","*",8,1],"Diffbot":["partial","*",8,1],"ContextFetchBot":["partial","*",8,1]},"llms":{"status":"present","http":200,"bytes":142043},"home":{"status":"ok","http":200,"url":"https://www.sanity.io/","html_tokens":97514,"markdown_tokens":1505,"pruned_tokens":76}},
{"rank":4827,"domain":"taleo.net","origin":"https://www.taleo.net","robots":{"status":"refused","http":403,"url":"https://www.oracle.com/us/products/applications/taleo/enterprise/overview/index.html","bytes":1450,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":4828,"domain":"uam.es","origin":"https://www.uam.es","robots":{"status":"ok","http":200,"url":"https://www.uam.es/robots.txt","bytes":1245,"groups":1},"bots":{"GPTBot":["partial","*",10,1],"ChatGPT-User":["partial","*",10,1],"OAI-SearchBot":["partial","*",10,1],"ClaudeBot":["partial","*",10,1],"Claude-User":["partial","*",10,1],"Claude-SearchBot":["partial","*",10,1],"anthropic-ai":["partial","*",10,1],"PerplexityBot":["partial","*",10,1],"Perplexity-User":["partial","*",10,1],"Google-Extended":["partial","*",10,1],"Applebot-Extended":["partial","*",10,1],"CCBot":["partial","*",10,1],"Bytespider":["partial","*",10,1],"Amazonbot":["partial","*",10,1],"meta-externalagent":["partial","*",10,1],"cohere-ai":["partial","*",10,1],"Diffbot":["partial","*",10,1],"ContextFetchBot":["partial","*",10,1]},"llms":{"status":"missing","http":404,"bytes":2699},"home":{"status":"ok","http":200,"url":"https://www.uam.es/uam/inicio","html_tokens":18583,"markdown_tokens":379,"pruned_tokens":87}},
{"rank":4829,"domain":"staples.com","origin":"https://staples.com","robots":{"status":"refused","http":403,"url":"https://staples.com/robots.txt","bytes":373,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":4830,"domain":"familyhandyman.com","origin":"https://familyhandyman.com","robots":{"status":"ok","http":200,"url":"https://www.familyhandyman.com/robots.txt","bytes":825,"groups":2},"bots":{"GPTBot":["partial","*",14,1],"ChatGPT-User":["partial","*",14,1],"OAI-SearchBot":["partial","*",14,1],"ClaudeBot":["partial","*",14,1],"Claude-User":["partial","*",14,1],"Claude-SearchBot":["partial","*",14,1],"anthropic-ai":["partial","*",14,1],"PerplexityBot":["partial","*",14,1],"Perplexity-User":["partial","*",14,1],"Google-Extended":["partial","*",14,1],"Applebot-Extended":["partial","*",14,1],"CCBot":["partial","*",14,1],"Bytespider":["partial","*",14,1],"Amazonbot":["partial","*",14,1],"meta-externalagent":["partial","*",14,1],"cohere-ai":["partial","*",14,1],"Diffbot":["partial","*",14,1],"ContextFetchBot":["partial","*",14,1]},"llms":{"status":"missing","http":404,"bytes":395150},"home":{"status":"ok","http":200,"url":"https://www.familyhandyman.com/","html_tokens":132239,"markdown_tokens":68,"pruned_tokens":68}},
{"rank":4831,"domain":"thedailystar.net","origin":"https://thedailystar.net","robots":{"status":"ok","http":200,"url":"https://www.thedailystar.net/robots.txt","bytes":2127,"groups":1},"bots":{"GPTBot":["partial","*",34,18],"ChatGPT-User":["partial","*",34,18],"OAI-SearchBot":["partial","*",34,18],"ClaudeBot":["partial","*",34,18],"Claude-User":["partial","*",34,18],"Claude-SearchBot":["partial","*",34,18],"anthropic-ai":["partial","*",34,18],"PerplexityBot":["partial","*",34,18],"Perplexity-User":["partial","*",34,18],"Google-Extended":["partial","*",34,18],"Applebot-Extended":["partial","*",34,18],"CCBot":["partial","*",34,18],"Bytespider":["partial","*",34,18],"Amazonbot":["partial","*",34,18],"meta-externalagent":["partial","*",34,18],"cohere-ai":["partial","*",34,18],"Diffbot":["partial","*",34,18],"ContextFetchBot":["partial","*",34,18]},"llms":{"status":"missing","http":404,"bytes":555},"home":{"status":"ok","http":200,"url":"https://www.thedailystar.net/","html_tokens":70214,"markdown_tokens":319,"pruned_tokens":174}},
{"rank":4832,"domain":"dwd.de","origin":"https://dwd.de","robots":{"status":"ok","http":200,"url":"https://dwd.de/robots.txt","bytes":226,"groups":5},"bots":{"GPTBot":["blocked","*",1,0],"ChatGPT-User":["blocked","*",1,0],"OAI-SearchBot":["blocked","*",1,0],"ClaudeBot":["blocked","*",1,0],"Claude-User":["blocked","*",1,0],"Claude-SearchBot":["blocked","*",1,0],"anthropic-ai":["blocked","*",1,0],"PerplexityBot":["blocked","*",1,0],"Perplexity-User":["blocked","*",1,0],"Google-Extended":["blocked","*",1,0],"Applebot-Extended":["blocked","*",1,0],"CCBot":["blocked","*",1,0],"Bytespider":["blocked","*",1,0],"Amazonbot":["blocked","*",1,0],"meta-externalagent":["blocked","*",1,0],"cohere-ai":["blocked","*",1,0],"Diffbot":["blocked","*",1,0],"ContextFetchBot":["blocked","*",1,0]},"llms":{"status":"disallowed"},"home":{"status":"disallowed"}},
{"rank":4833,"domain":"zj.gov.cn","origin":"https://zj.gov.cn","robots":{"status":"unreachable","http":null,"url":"https://zj.gov.cn/robots.txt","bytes":0,"groups":0,"error":"cannot resolve zj.gov.cn"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":4834,"domain":"rtl.fr","origin":"https://rtl.fr","robots":{"status":"ok","http":200,"url":"https://www.rtl.fr/robots.txt","bytes":962,"groups":5},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["blocked","named",1,0],"OAI-SearchBot":["partial","*",24,0],"ClaudeBot":["partial","*",24,0],"Claude-User":["partial","*",24,0],"Claude-SearchBot":["partial","*",24,0],"anthropic-ai":["partial","*",24,0],"PerplexityBot":["partial","*",24,0],"Perplexity-User":["partial","*",24,0],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["partial","*",24,0],"CCBot":["blocked","named",1,0],"Bytespider":["partial","*",24,0],"Amazonbot":["partial","*",24,0],"meta-externalagent":["partial","*",24,0],"cohere-ai":["partial","*",24,0],"Diffbot":["partial","*",24,0],"ContextFetchBot":["partial","*",24,0]},"llms":{"status":"missing","http":404,"bytes":1140590},"home":{"status":"ok","http":200,"url":"https://www.rtl.fr/","html_tokens":386705,"markdown_tokens":71,"pruned_tokens":71}},
{"rank":4835,"domain":"socialmediaexaminer.com","origin":"https://socialmediaexaminer.com","robots":{"status":"ok","http":200,"url":"https://socialmediaexaminer.com/robots.txt","bytes":2438,"groups":2},"bots":{"GPTBot":["partial","*",73,0],"ChatGPT-User":["partial","*",73,0],"OAI-SearchBot":["partial","*",73,0],"ClaudeBot":["partial","*",73,0],"Claude-User":["partial","*",73,0],"Claude-SearchBot":["partial","*",73,0],"anthropic-ai":["partial","*",73,0],"PerplexityBot":["partial","*",73,0],"Perplexity-User":["partial","*",73,0],"Google-Extended":["partial","*",73,0],"Applebot-Extended":["partial","*",73,0],"CCBot":["partial","*",73,0],"Bytespider":["partial","*",73,0],"Amazonbot":["partial","*",73,0],"meta-externalagent":["partial","*",73,0],"cohere-ai":["partial","*",73,0],"Diffbot":["partial","*",73,0],"ContextFetchBot":["partial","*",73,0]},"llms":{"status":"other_text","http":200,"bytes":1652},"home":{"status":"http_error","http":403,"url":"https://socialmediaexaminer.com/"}},
{"rank":4836,"domain":"palmbeachpost.com","origin":"https://palmbeachpost.com","robots":{"status":"ok","http":200,"url":"https://www.palmbeachpost.com/robots.txt","bytes":14857,"groups":288},"bots":{"GPTBot":["partial","named",1,5],"ChatGPT-User":["partial","named",1,5],"OAI-SearchBot":["partial","named",1,5],"ClaudeBot":["blocked","named",1,0],"Claude-User":["blocked","named",1,0],"Claude-SearchBot":["blocked","named",1,0],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["partial","named",1,5],"Perplexity-User":["blocked","named",1,0],"Google-Extended":["partial","named",1,5],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",2,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",88,0]},"llms":{"status":"http_error","http":402,"bytes":254},"home":{"status":"http_error","http":402,"url":"https://www.palmbeachpost.com/"}},
{"rank":4837,"domain":"librarything.com","origin":"https://librarything.com","robots":{"status":"ok","http":200,"url":"https://www.librarything.com/robots.txt","bytes":6580,"groups":8},"bots":{"GPTBot":["partial","*",104,2],"ChatGPT-User":["partial","*",104,2],"OAI-SearchBot":["partial","*",104,2],"ClaudeBot":["partial","*",104,2],"Claude-User":["partial","*",104,2],"Claude-SearchBot":["partial","*",104,2],"anthropic-ai":["partial","*",104,2],"PerplexityBot":["partial","*",104,2],"Perplexity-User":["partial","*",104,2],"Google-Extended":["partial","*",104,2],"Applebot-Extended":["partial","*",104,2],"CCBot":["partial","*",104,2],"Bytespider":["partial","*",104,2],"Amazonbot":["partial","*",104,2],"meta-externalagent":["partial","*",104,2],"cohere-ai":["partial","*",104,2],"Diffbot":["partial","*",104,2],"ContextFetchBot":["partial","*",104,2]},"llms":{"status":"missing","http":404,"bytes":153},"home":{"status":"http_error","http":403,"url":"https://librarything.com/"}},
{"rank":4838,"domain":"corel.com","origin":"https://corel.com","robots":{"status":"ok","http":200,"url":"https://www.corel.com/robots.txt","bytes":245,"groups":4},"bots":{"GPTBot":["partial","*",2,0],"ChatGPT-User":["partial","*",2,0],"OAI-SearchBot":["partial","*",2,0],"ClaudeBot":["partial","*",2,0],"Claude-User":["partial","*",2,0],"Claude-SearchBot":["partial","*",2,0],"anthropic-ai":["partial","*",2,0],"PerplexityBot":["partial","*",2,0],"Perplexity-User":["partial","*",2,0],"Google-Extended":["partial","*",2,0],"Applebot-Extended":["partial","*",2,0],"CCBot":["partial","*",2,0],"Bytespider":["partial","*",2,0],"Amazonbot":["partial","*",2,0],"meta-externalagent":["partial","*",2,0],"cohere-ai":["partial","*",2,0],"Diffbot":["partial","*",2,0],"ContextFetchBot":["partial","*",2,0]},"llms":{"status":"present","http":200,"bytes":2135},"home":{"status":"ok","http":200,"url":"https://www.corel.com/en/","html_tokens":13969,"markdown_tokens":206,"pruned_tokens":77}},
{"rank":4839,"domain":"harley-davidson.com","origin":"https://harley-davidson.com","robots":{"status":"ok","http":200,"url":"https://www.harley-davidson.com/robots.txt","bytes":5693,"groups":2},"bots":{"GPTBot":["partial","*",3,0],"ChatGPT-User":["partial","*",3,0],"OAI-SearchBot":["partial","*",3,0],"ClaudeBot":["partial","*",3,0],"Claude-User":["partial","*",3,0],"Claude-SearchBot":["partial","*",3,0],"anthropic-ai":["partial","*",3,0],"PerplexityBot":["partial","*",3,0],"Perplexity-User":["partial","*",3,0],"Google-Extended":["partial","*",3,0],"Applebot-Extended":["partial","*",3,0],"CCBot":["partial","*",3,0],"Bytespider":["partial","*",3,0],"Amazonbot":["partial","*",3,0],"meta-externalagent":["partial","*",3,0],"cohere-ai":["partial","*",3,0],"Diffbot":["partial","*",3,0],"ContextFetchBot":["partial","*",3,0]},"llms":{"status":"present","http":200,"bytes":14579},"home":{"status":"ok","http":200,"url":"https://www.harley-davidson.com/us/en/index.html","html_tokens":178145,"markdown_tokens":387,"pruned_tokens":163}},
{"rank":4840,"domain":"dhgate.com","origin":"https://dhgate.com","robots":{"status":"ok","http":200,"url":"https://www.dhgate.com/robots.txt","bytes":11054,"groups":86},"bots":{"GPTBot":["allowed","named",0,1],"ChatGPT-User":["allowed","named",0,1],"OAI-SearchBot":["allowed","named",0,1],"ClaudeBot":["partial","*",53,3],"Claude-User":["partial","*",53,3],"Claude-SearchBot":["partial","*",53,3],"anthropic-ai":["partial","*",53,3],"PerplexityBot":["partial","*",53,3],"Perplexity-User":["partial","*",53,3],"Google-Extended":["partial","*",53,3],"Applebot-Extended":["partial","*",53,3],"CCBot":["partial","*",53,3],"Bytespider":["partial","*",53,3],"Amazonbot":["partial","*",53,3],"meta-externalagent":["partial","*",53,3],"cohere-ai":["partial","*",53,3],"Diffbot":["partial","*",53,3],"ContextFetchBot":["partial","*",53,3]},"llms":{"status":"http_error","http":403,"bytes":380},"home":{"status":"http_error","http":403,"url":"https://www.dhgate.com/"}},
{"rank":4841,"domain":"chinatimes.com","origin":"https://chinatimes.com","robots":{"status":"ok","http":200,"url":"https://www.chinatimes.com/robots.txt","bytes":1805,"groups":3},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["blocked","named",1,0],"OAI-SearchBot":["blocked","named",1,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",2,0],"Claude-SearchBot":["partial","*",2,0],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["partial","*",2,0],"Google-Extended":["allowed","named",0,1],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["allowed","named",0,1],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",2,0]},"llms":{"status":"missing","http":404,"bytes":45832},"home":{"status":"http_error","http":403,"url":"https://www.chinatimes.com/"}},
{"rank":4842,"domain":"ui.ac.id","origin":"https://ui.ac.id","robots":{"status":"unreachable","http":null,"url":"https://ui.ac.id/robots.txt","bytes":0,"groups":0,"error":"Server disconnected without sending a response."},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":4843,"domain":"workday.com","origin":"https://workday.com","robots":{"status":"ok","http":200,"url":"https://www.workday.com/robots.txt","bytes":6757,"groups":1},"bots":{"GPTBot":["partial","*",121,1],"ChatGPT-User":["partial","*",121,1],"OAI-SearchBot":["partial","*",121,1],"ClaudeBot":["partial","*",121,1],"Claude-User":["partial","*",121,1],"Claude-SearchBot":["partial","*",121,1],"anthropic-ai":["partial","*",121,1],"PerplexityBot":["partial","*",121,1],"Perplexity-User":["partial","*",121,1],"Google-Extended":["partial","*",121,1],"Applebot-Extended":["partial","*",121,1],"CCBot":["partial","*",121,1],"Bytespider":["partial","*",121,1],"Amazonbot":["partial","*",121,1],"meta-externalagent":["partial","*",121,1],"cohere-ai":["partial","*",121,1],"Diffbot":["partial","*",121,1],"ContextFetchBot":["partial","*",121,1]},"llms":{"status":"missing","http":404,"bytes":38416},"home":{"status":"ok","http":200,"url":"https://www.workday.com/","html_tokens":151899,"markdown_tokens":488,"pruned_tokens":109}},
{"rank":4844,"domain":"banggood.com","origin":"https://banggood.com","robots":{"status":"refused","http":403,"url":"https://www.banggood.com/robots.txt?akmClientCountry=America&","bytes":387,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":4845,"domain":"aichi.jp","origin":"https://aichi.jp","robots":{"status":"unreachable","http":null,"url":"https://aichi.jp/robots.txt","bytes":0,"groups":0,"error":"cannot resolve aichi.jp"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":4846,"domain":"laposte.fr","origin":"https://laposte.fr","robots":{"status":"ok","http":200,"url":"https://www.laposte.fr/robots.txt","bytes":9536,"groups":1},"bots":{"GPTBot":["partial","*",231,0],"ChatGPT-User":["partial","*",231,0],"OAI-SearchBot":["partial","*",231,0],"ClaudeBot":["partial","*",231,0],"Claude-User":["partial","*",231,0],"Claude-SearchBot":["partial","*",231,0],"anthropic-ai":["partial","*",231,0],"PerplexityBot":["partial","*",231,0],"Perplexity-User":["partial","*",231,0],"Google-Extended":["partial","*",231,0],"Applebot-Extended":["partial","*",231,0],"CCBot":["partial","*",231,0],"Bytespider":["partial","*",231,0],"Amazonbot":["partial","*",231,0],"meta-externalagent":["partial","*",231,0],"cohere-ai":["partial","*",231,0],"Diffbot":["partial","*",231,0],"ContextFetchBot":["partial","*",231,0]},"llms":{"status":"missing","http":404,"bytes":22440},"home":{"status":"ok","http":200,"url":"https://www.laposte.fr/","html_tokens":86941,"markdown_tokens":123,"pruned_tokens":124}},
{"rank":4847,"domain":"comptia.org","origin":"https://comptia.org","robots":{"status":"ok","http":200,"url":"https://www.comptia.org/robots.txt","bytes":869,"groups":1},"bots":{"GPTBot":["partial","*",18,0],"ChatGPT-User":["partial","*",18,0],"OAI-SearchBot":["partial","*",18,0],"ClaudeBot":["partial","*",18,0],"Claude-User":["partial","*",18,0],"Claude-SearchBot":["partial","*",18,0],"anthropic-ai":["partial","*",18,0],"PerplexityBot":["partial","*",18,0],"Perplexity-User":["partial","*",18,0],"Google-Extended":["partial","*",18,0],"Applebot-Extended":["partial","*",18,0],"CCBot":["partial","*",18,0],"Bytespider":["partial","*",18,0],"Amazonbot":["partial","*",18,0],"meta-externalagent":["partial","*",18,0],"cohere-ai":["partial","*",18,0],"Diffbot":["partial","*",18,0],"ContextFetchBot":["partial","*",18,0]},"llms":{"status":"present","http":200,"bytes":1330},"home":{"status":"ok","http":200,"url":"https://www.comptia.org/en-us/","html_tokens":130745,"markdown_tokens":612,"pruned_tokens":228}},
{"rank":4848,"domain":"rootsweb.com","origin":"https://rootsweb.com","robots":{"status":"refused","http":403,"url":"https://rootsweb.com/robots.txt","bytes":2,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":4849,"domain":"ncaa.com","origin":"https://ncaa.com","robots":{"status":"ok","http":200,"url":"https://www.ncaa.com/robots.txt","bytes":4649,"groups":3},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["blocked","named",1,0],"OAI-SearchBot":["partial","*",44,40],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",44,40],"Claude-SearchBot":["partial","*",44,40],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["partial","*",44,40],"Perplexity-User":["partial","*",44,40],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["partial","*",44,40],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",44,40]},"llms":{"status":"http_error","http":403,"bytes":150},"home":{"status":"ok","http":200,"url":"https://www.ncaa.com/","html_tokens":35976,"markdown_tokens":48,"pruned_tokens":48}},
{"rank":4850,"domain":"10jqka.com.cn","origin":"https://10jqka.com.cn","robots":{"status":"ok","http":200,"url":"https://www.10jqka.com.cn/robots.txt","bytes":170,"groups":1},"bots":{"GPTBot":["partial","*",8,0],"ChatGPT-User":["partial","*",8,0],"OAI-SearchBot":["partial","*",8,0],"ClaudeBot":["partial","*",8,0],"Claude-User":["partial","*",8,0],"Claude-SearchBot":["partial","*",8,0],"anthropic-ai":["partial","*",8,0],"PerplexityBot":["partial","*",8,0],"Perplexity-User":["partial","*",8,0],"Google-Extended":["partial","*",8,0],"Applebot-Extended":["partial","*",8,0],"CCBot":["partial","*",8,0],"Bytespider":["partial","*",8,0],"Amazonbot":["partial","*",8,0],"meta-externalagent":["partial","*",8,0],"cohere-ai":["partial","*",8,0],"Diffbot":["partial","*",8,0],"ContextFetchBot":["partial","*",8,0]},"llms":{"status":"missing","http":404,"bytes":1159},"home":{"status":"ok","http":200,"url":"https://www.10jqka.com.cn/","html_tokens":58063,"markdown_tokens":120,"pruned_tokens":120}},
{"rank":4851,"domain":"sigmaaldrich.com","origin":"https://sigmaaldrich.com","robots":{"status":"unreachable","http":null,"url":"https://sigmaaldrich.com/robots.txt","bytes":0,"groups":0,"error":"<StreamReset stream_id:1, error_code:2, remote_reset:True>"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":4852,"domain":"lasvegassun.com","origin":"https://lasvegassun.com","robots":{"status":"ok","http":200,"url":"https://lasvegassun.com/robots.txt","bytes":5390,"groups":62},"bots":{"GPTBot":["allowed","named",0,1],"ChatGPT-User":["allowed","named",0,1],"OAI-SearchBot":["partial","*",48,0],"ClaudeBot":["allowed","named",0,1],"Claude-User":["partial","*",48,0],"Claude-SearchBot":["partial","*",48,0],"anthropic-ai":["allowed","named",0,1],"PerplexityBot":["allowed","named",0,1],"Perplexity-User":["partial","*",48,0],"Google-Extended":["allowed","named",0,1],"Applebot-Extended":["allowed","named",0,1],"CCBot":["allowed","named",0,1],"Bytespider":["partial","*",48,0],"Amazonbot":["allowed","named",0,1],"meta-externalagent":["allowed","named",0,1],"cohere-ai":["allowed","named",0,1],"Diffbot":["partial","*",48,0],"ContextFetchBot":["partial","*",48,0]},"llms":{"status":"missing","http":404,"bytes":13845},"home":{"status":"ok","http":200,"url":"https://lasvegassun.com/","html_tokens":37356,"markdown_tokens":552,"pruned_tokens":102}},
{"rank":4853,"domain":"ticksy.com","origin":"https://ticksy.com","robots":{"status":"ok","http":200,"url":"https://ticksy.com/robots.txt","bytes":111,"groups":1},"bots":{"GPTBot":["partial","*",1,1],"ChatGPT-User":["partial","*",1,1],"OAI-SearchBot":["partial","*",1,1],"ClaudeBot":["partial","*",1,1],"Claude-User":["partial","*",1,1],"Claude-SearchBot":["partial","*",1,1],"anthropic-ai":["partial","*",1,1],"PerplexityBot":["partial","*",1,1],"Perplexity-User":["partial","*",1,1],"Google-Extended":["partial","*",1,1],"Applebot-Extended":["partial","*",1,1],"CCBot":["partial","*",1,1],"Bytespider":["partial","*",1,1],"Amazonbot":["partial","*",1,1],"meta-externalagent":["partial","*",1,1],"cohere-ai":["partial","*",1,1],"Diffbot":["partial","*",1,1],"ContextFetchBot":["partial","*",1,1]},"llms":{"status":"missing","http":404,"bytes":7904},"home":{"status":"ok","http":200,"url":"https://ticksy.com/","html_tokens":13578,"markdown_tokens":601,"pruned_tokens":143}},
{"rank":4854,"domain":"primary.jwwb.nl","origin":"https://primary.jwwb.nl","robots":{"status":"missing","http":404,"url":"https://primary.jwwb.nl/robots.txt","bytes":404,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":404},"home":{"status":"error","http":200,"url":"https://primary.jwwb.nl/","error":"unparseable HTML"}},
{"rank":4855,"domain":"fresha.com","origin":"https://fresha.com","robots":{"status":"ok","http":200,"url":"https://www.fresha.com/robots.txt","bytes":1104,"groups":2},"bots":{"GPTBot":["partial","*",17,0],"ChatGPT-User":["partial","*",17,0],"OAI-SearchBot":["partial","*",17,0],"ClaudeBot":["partial","*",17,0],"Claude-User":["partial","*",17,0],"Claude-SearchBot":["partial","*",17,0],"anthropic-ai":["partial","*",17,0],"PerplexityBot":["partial","*",17,0],"Perplexity-User":["partial","*",17,0],"Google-Extended":["partial","*",17,0],"Applebot-Extended":["partial","*",17,0],"CCBot":["partial","*",17,0],"Bytespider":["partial","*",17,0],"Amazonbot":["partial","*",17,0],"meta-externalagent":["partial","*",17,0],"cohere-ai":["partial","*",17,0],"Diffbot":["partial","*",17,0],"ContextFetchBot":["partial","*",17,0]},"llms":{"status":"missing","http":410,"bytes":45140},"home":{"status":"ok","http":200,"url":"https://www.fresha.com/","html_tokens":264397,"markdown_tokens":1448,"pruned_tokens":534}},
{"rank":4856,"domain":"mprnews.org","origin":"https://mprnews.org","robots":{"status":"ok","http":200,"url":"https://www.mprnews.org/robots.txt","bytes":2095,"groups":1},"bots":{"GPTBot":["partial","*",4,0],"ChatGPT-User":["partial","*",4,0],"OAI-SearchBot":["partial","*",4,0],"ClaudeBot":["partial","*",4,0],"Claude-User":["partial","*",4,0],"Claude-SearchBot":["partial","*",4,0],"anthropic-ai":["partial","*",4,0],"PerplexityBot":["partial","*",4,0],"Perplexity-User":["partial","*",4,0],"Google-Extended":["partial","*",4,0],"Applebot-Extended":["partial","*",4,0],"CCBot":["partial","*",4,0],"Bytespider":["partial","*",4,0],"Amazonbot":["partial","*",4,0],"meta-externalagent":["partial","*",4,0],"cohere-ai":["partial","*",4,0],"Diffbot":["partial","*",4,0],"ContextFetchBot":["partial","*",4,0]},"llms":{"status":"present","http":200,"bytes":8014},"home":{"status":"ok","http":200,"url":"https://www.mprnews.org/","html_tokens":39265,"markdown_tokens":124,"pruned_tokens":124}},
{"rank":4857,"domain":"vn.ua","origin":"https://www.vn.ua","robots":{"status":"missing","http":404,"url":"https://www.vn.ua/robots.txt","bytes":272,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":272},"home":{"status":"ok","http":200,"url":"https://www.vn.ua/","html_tokens":2144,"markdown_tokens":248,"pruned_tokens":245}},
{"rank":4858,"domain":"atom.io","origin":"https://atom.io","robots":{"status":"html","http":200,"url":"https://github.blog/news-insights/product-news/sunsetting-atom/","bytes":168812,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":4859,"domain":"pr.gov.br","origin":"https://www.pr.gov.br","robots":{"status":"ok","http":200,"url":"https://www.pr.gov.br/robots.txt","bytes":81,"groups":1},"bots":{"GPTBot":["blocked","*",4,0],"ChatGPT-User":["blocked","*",4,0],"OAI-SearchBot":["blocked","*",4,0],"ClaudeBot":["blocked","*",4,0],"Claude-User":["blocked","*",4,0],"Claude-SearchBot":["blocked","*",4,0],"anthropic-ai":["blocked","*",4,0],"PerplexityBot":["blocked","*",4,0],"Perplexity-User":["blocked","*",4,0],"Google-Extended":["blocked","*",4,0],"Applebot-Extended":["blocked","*",4,0],"CCBot":["blocked","*",4,0],"Bytespider":["blocked","*",4,0],"Amazonbot":["blocked","*",4,0],"meta-externalagent":["blocked","*",4,0],"cohere-ai":["blocked","*",4,0],"Diffbot":["blocked","*",4,0],"ContextFetchBot":["blocked","*",4,0]},"llms":{"status":"disallowed"},"home":{"status":"disallowed"}},
{"rank":4860,"domain":"osd.mil","origin":"https://osd.mil","robots":{"status":"unreachable","http":null,"url":"https://osd.mil/robots.txt","bytes":0,"groups":0,"error":"cannot resolve osd.mil"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":4861,"domain":"brides.com","origin":"https://brides.com","robots":{"status":"ok","http":200,"url":"https://www.brides.com/robots.txt","bytes":3465,"groups":6},"bots":{"GPTBot":["partial","named",1,0],"ChatGPT-User":["partial","named",1,0],"OAI-SearchBot":["partial","named",1,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["blocked","named",1,0],"Claude-SearchBot":["partial","named",1,2],"anthropic-ai":["partial","named",1,2],"PerplexityBot":["partial","named",1,2],"Perplexity-User":["blocked","named",1,0],"Google-Extended":["partial","named",1,2],"Applebot-Extended":["blocked","named",1,0],"CCBot":["partial","named",1,2],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["partial","named",1,2],"cohere-ai":["partial","named",1,2],"Diffbot":["partial","*",2,0],"ContextFetchBot":["partial","*",2,0]},"llms":{"status":"http_error","http":402,"bytes":612},"home":{"status":"http_error","http":402,"url":"https://www.brides.com/"}},
{"rank":4862,"domain":"ixdf.org","origin":"https://ixdf.org","robots":{"status":"ok","http":200,"url":"https://ixdf.org/robots.txt","bytes":709,"groups":2},"bots":{"GPTBot":["partial","*",23,0],"ChatGPT-User":["partial","*",23,0],"OAI-SearchBot":["partial","*",23,0],"ClaudeBot":["partial","*",23,0],"Claude-User":["partial","*",23,0],"Claude-SearchBot":["partial","*",23,0],"anthropic-ai":["partial","*",23,0],"PerplexityBot":["partial","*",23,0],"Perplexity-User":["partial","*",23,0],"Google-Extended":["partial","*",23,0],"Applebot-Extended":["partial","*",23,0],"CCBot":["partial","*",23,0],"Bytespider":["partial","*",23,0],"Amazonbot":["partial","*",23,0],"meta-externalagent":["partial","*",23,0],"cohere-ai":["partial","*",23,0],"Diffbot":["partial","*",23,0],"ContextFetchBot":["partial","*",23,0]},"llms":{"status":"missing","http":404,"bytes":563440},"home":{"status":"ok","http":200,"url":"https://ixdf.org/","html_tokens":95310,"markdown_tokens":498,"pruned_tokens":168}},
{"rank":4863,"domain":"ecovadis.com","origin":"https://ecovadis.com","robots":{"status":"ok","http":200,"url":"https://ecovadis.com/robots.txt","bytes":889,"groups":1},"bots":{"GPTBot":["partial","*",21,4],"ChatGPT-User":["partial","*",21,4],"OAI-SearchBot":["partial","*",21,4],"ClaudeBot":["partial","*",21,4],"Claude-User":["partial","*",21,4],"Claude-SearchBot":["partial","*",21,4],"anthropic-ai":["partial","*",21,4],"PerplexityBot":["partial","*",21,4],"Perplexity-User":["partial","*",21,4],"Google-Extended":["partial","*",21,4],"Applebot-Extended":["partial","*",21,4],"CCBot":["partial","*",21,4],"Bytespider":["partial","*",21,4],"Amazonbot":["partial","*",21,4],"meta-externalagent":["partial","*",21,4],"cohere-ai":["partial","*",21,4],"Diffbot":["partial","*",21,4],"ContextFetchBot":["partial","*",21,4]},"llms":{"status":"missing","http":404,"bytes":148270},"home":{"status":"ok","http":200,"url":"https://ecovadis.com/","html_tokens":46818,"markdown_tokens":659,"pruned_tokens":182}},
{"rank":4864,"domain":"businesstimes.com.sg","origin":"https://businesstimes.com.sg","robots":{"status":"ok","http":200,"url":"https://www.businesstimes.com.sg/robots.txt","bytes":1441,"groups":18},"bots":{"GPTBot":["allowed","named",0,1],"ChatGPT-User":["allowed","named",0,1],"OAI-SearchBot":["allowed","named",0,1],"ClaudeBot":["allowed","named",0,1],"Claude-User":["allowed","named",0,1],"Claude-SearchBot":["allowed","named",0,1],"anthropic-ai":["allowed","named",0,1],"PerplexityBot":["allowed","named",0,1],"Perplexity-User":["allowed","named",0,1],"Google-Extended":["partial","named",1,1],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"missing","http":404,"bytes":9},"home":{"status":"ok","http":200,"url":"https://www.businesstimes.com.sg","html_tokens":67441,"markdown_tokens":38,"pruned_tokens":38}},
{"rank":4865,"domain":"sendibm1.com","origin":"https://sendibm1.com","robots":{"status":"unreachable","http":null,"url":"https://sendibm1.com/robots.txt","bytes":0,"groups":0,"error":"cannot resolve sendibm1.com"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":4866,"domain":"radiotimes.com","origin":"https://radiotimes.com","robots":{"status":"ok","http":200,"url":"https://www.radiotimes.com/robots.txt","bytes":1921,"groups":14},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["partial","*",39,2],"OAI-SearchBot":["partial","*",39,2],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",39,2],"Claude-SearchBot":["partial","*",39,2],"anthropic-ai":["partial","*",39,2],"PerplexityBot":["partial","*",39,2],"Perplexity-User":["partial","*",39,2],"Google-Extended":["partial","*",39,2],"Applebot-Extended":["partial","*",39,2],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["partial","*",39,2],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["partial","*",39,2],"ContextFetchBot":["partial","*",39,2]},"llms":{"status":"missing","http":404,"bytes":9},"home":{"status":"ok","http":200,"url":"https://www.radiotimes.com/","html_tokens":255715,"markdown_tokens":72,"pruned_tokens":72}},
{"rank":4867,"domain":"wwnorton.com","origin":"https://wwnorton.com","robots":{"status":"ok","http":200,"url":"https://wwnorton.com/robots.txt","bytes":606,"groups":8},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["partial","*",15,0],"OAI-SearchBot":["partial","*",15,0],"ClaudeBot":["partial","*",15,0],"Claude-User":["partial","*",15,0],"Claude-SearchBot":["partial","*",15,0],"anthropic-ai":["partial","*",15,0],"PerplexityBot":["partial","*",15,0],"Perplexity-User":["partial","*",15,0],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["partial","*",15,0],"CCBot":["blocked","named",1,0],"Bytespider":["partial","*",15,0],"Amazonbot":["partial","*",15,0],"meta-externalagent":["partial","*",15,0],"cohere-ai":["partial","*",15,0],"Diffbot":["partial","*",15,0],"ContextFetchBot":["partial","*",15,0]},"llms":{"status":"missing","http":404,"bytes":19433},"home":{"status":"ok","http":200,"url":"https://wwnorton.com/","html_tokens":18442,"markdown_tokens":12,"pruned_tokens":12}},
{"rank":4868,"domain":"nagoya-u.ac.jp","origin":"https://www.nagoya-u.ac.jp","robots":{"status":"missing","http":404,"url":"https://www.nagoya-u.ac.jp/robots.txt","bytes":315,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":315},"home":{"status":"ok","http":200,"url":"https://www.nagoya-u.ac.jp/","html_tokens":21519,"markdown_tokens":511,"pruned_tokens":173}},
{"rank":4869,"domain":"healthdata.org","origin":"https://healthdata.org","robots":{"status":"ok","http":200,"url":"https://www.healthdata.org/robots.txt","bytes":2246,"groups":1},"bots":{"GPTBot":["partial","*",35,18],"ChatGPT-User":["partial","*",35,18],"OAI-SearchBot":["partial","*",35,18],"ClaudeBot":["partial","*",35,18],"Claude-User":["partial","*",35,18],"Claude-SearchBot":["partial","*",35,18],"anthropic-ai":["partial","*",35,18],"PerplexityBot":["partial","*",35,18],"Perplexity-User":["partial","*",35,18],"Google-Extended":["partial","*",35,18],"Applebot-Extended":["partial","*",35,18],"CCBot":["partial","*",35,18],"Bytespider":["partial","*",35,18],"Amazonbot":["partial","*",35,18],"meta-externalagent":["partial","*",35,18],"cohere-ai":["partial","*",35,18],"Diffbot":["partial","*",35,18],"ContextFetchBot":["partial","*",35,18]},"llms":{"status":"missing","http":404,"bytes":192},"home":{"status":"ok","http":200,"url":"https://www.healthdata.org/","html_tokens":39121,"markdown_tokens":51,"pruned_tokens":51}},
{"rank":4870,"domain":"ns.ca","origin":"https://ns.ca","robots":{"status":"unreachable","http":null,"url":"https://ns.ca/robots.txt","bytes":0,"groups":0,"error":"cannot resolve ns.ca"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":4871,"domain":"uj.edu.pl","origin":"https://uj.edu.pl","robots":{"status":"ok","http":200,"url":"https://www.uj.edu.pl/robots.txt","bytes":2167,"groups":9},"bots":{"GPTBot":["blocked","*",1,0],"ChatGPT-User":["blocked","*",1,0],"OAI-SearchBot":["blocked","*",1,0],"ClaudeBot":["blocked","*",1,0],"Claude-User":["blocked","*",1,0],"Claude-SearchBot":["blocked","*",1,0],"anthropic-ai":["blocked","*",1,0],"PerplexityBot":["blocked","*",1,0],"Perplexity-User":["blocked","*",1,0],"Google-Extended":["blocked","*",1,0],"Applebot-Extended":["blocked","*",1,0],"CCBot":["blocked","*",1,0],"Bytespider":["blocked","*",1,0],"Amazonbot":["blocked","*",1,0],"meta-externalagent":["blocked","*",1,0],"cohere-ai":["blocked","*",1,0],"Diffbot":["blocked","*",1,0],"ContextFetchBot":["blocked","*",1,0]},"llms":{"status":"disallowed"},"home":{"status":"disallowed"}},
{"rank":4872,"domain":"royalmail.com","origin":"https://royalmail.com","robots":{"status":"refused","http":403,"url":"https://www.royalmail.com/robots.txt","bytes":385,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":4873,"domain":"canonical.com","origin":"https://canonical.com","robots":{"status":"ok","http":200,"url":"https://canonical.com/robots.txt","bytes":3269,"groups":2},"bots":{"GPTBot":["partial","named",11,16],"ChatGPT-User":["partial","named",11,16],"OAI-SearchBot":["partial","named",11,16],"ClaudeBot":["partial","named",11,16],"Claude-User":["partial","*",28,0],"Claude-SearchBot":["partial","named",11,16],"anthropic-ai":["partial","named",11,16],"PerplexityBot":["partial","named",11,16],"Perplexity-User":["partial","named",11,16],"Google-Extended":["partial","named",11,16],"Applebot-Extended":["partial","named",11,16],"CCBot":["partial","*",28,0],"Bytespider":["partial","named",11,16],"Amazonbot":["partial","*",28,0],"meta-externalagent":["partial","named",11,16],"cohere-ai":["partial","named",11,16],"Diffbot":["partial","*",28,0],"ContextFetchBot":["partial","*",28,0]},"llms":{"status":"present","http":200,"bytes":27081},"home":{"status":"ok","http":200,"url":"https://canonical.com/","html_tokens":86479,"markdown_tokens":697,"pruned_tokens":173}},
{"rank":4874,"domain":"cpj.org","origin":"https://cpj.org","robots":{"status":"ok","http":200,"url":"https://cpj.org/robots.txt","bytes":165,"groups":1},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"missing","http":404,"bytes":172609},"home":{"status":"ok","http":200,"url":"https://cpj.org/","html_tokens":53765,"markdown_tokens":227,"pruned_tokens":113}},
{"rank":4875,"domain":"indiamart.com","origin":"https://indiamart.com","robots":{"status":"ok","http":200,"url":"https://indiamart.com/robots.txt","bytes":3011,"groups":57},"bots":{"GPTBot":["allowed","named",0,1],"ChatGPT-User":["allowed","named",0,1],"OAI-SearchBot":["allowed","named",0,1],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",31,0],"Claude-SearchBot":["partial","*",31,0],"anthropic-ai":["partial","*",31,0],"PerplexityBot":["partial","*",31,0],"Perplexity-User":["partial","*",31,0],"Google-Extended":["allowed","named",0,1],"Applebot-Extended":["partial","*",31,0],"CCBot":["partial","*",31,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["partial","*",31,0],"meta-externalagent":["allowed","named",0,1],"cohere-ai":["partial","*",31,0],"Diffbot":["partial","*",31,0],"ContextFetchBot":["partial","*",31,0]},"llms":{"status":"http_error","http":429,"bytes":142},"home":{"status":"http_error","http":429,"url":"https://indiamart.com/"}},
{"rank":4876,"domain":"liverpoolecho.co.uk","origin":"https://liverpoolecho.co.uk","robots":{"status":"ok","http":200,"url":"https://www.liverpoolecho.co.uk/robots.txt","bytes":937,"groups":13},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["partial","*",9,0],"OAI-SearchBot":["blocked","named",1,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",9,0],"Claude-SearchBot":["partial","*",9,0],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["partial","*",9,0],"Google-Extended":["partial","*",9,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["partial","*",9,0],"Amazonbot":["partial","*",9,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["partial","*",9,0],"Diffbot":["partial","*",9,0],"ContextFetchBot":["partial","*",9,0]},"llms":{"status":"missing","http":404,"bytes":807602},"home":{"status":"ok","http":200,"url":"https://www.liverpoolecho.co.uk/","html_tokens":204355,"markdown_tokens":153,"pruned_tokens":153}},
{"rank":4877,"domain":"luc.edu","origin":"https://luc.edu","robots":{"status":"ok","http":200,"url":"https://www.luc.edu/robots.txt","bytes":1330,"groups":4},"bots":{"GPTBot":["partial","*",54,0],"ChatGPT-User":["partial","*",54,0],"OAI-SearchBot":["partial","*",54,0],"ClaudeBot":["partial","*",54,0],"Claude-User":["partial","*",54,0],"Claude-SearchBot":["partial","*",54,0],"anthropic-ai":["partial","*",54,0],"PerplexityBot":["partial","*",54,0],"Perplexity-User":["partial","*",54,0],"Google-Extended":["partial","*",54,0],"Applebot-Extended":["partial","*",54,0],"CCBot":["partial","*",54,0],"Bytespider":["partial","*",54,0],"Amazonbot":["partial","*",54,0],"meta-externalagent":["partial","*",54,0],"cohere-ai":["partial","*",54,0],"Diffbot":["partial","*",54,0],"ContextFetchBot":["partial","*",54,0]},"llms":{"status":"missing","http":404,"bytes":29091},"home":{"status":"ok","http":200,"url":"https://www.luc.edu/","html_tokens":18752,"markdown_tokens":787,"pruned_tokens":165}},
{"rank":4878,"domain":"webpagetest.org","origin":"https://webpagetest.org","robots":{"status":"ok","http":200,"url":"https://www.webpagetest.org/robots.txt","bytes":350,"groups":2},"bots":{"GPTBot":["partial","*",11,1],"ChatGPT-User":["partial","*",11,1],"OAI-SearchBot":["partial","*",11,1],"ClaudeBot":["partial","*",11,1],"Claude-User":["partial","*",11,1],"Claude-SearchBot":["partial","*",11,1],"anthropic-ai":["partial","*",11,1],"PerplexityBot":["partial","*",11,1],"Perplexity-User":["partial","*",11,1],"Google-Extended":["partial","*",11,1],"Applebot-Extended":["partial","*",11,1],"CCBot":["partial","*",11,1],"Bytespider":["partial","*",11,1],"Amazonbot":["partial","*",11,1],"meta-externalagent":["partial","*",11,1],"cohere-ai":["partial","*",11,1],"Diffbot":["partial","*",11,1],"ContextFetchBot":["partial","*",11,1]},"llms":{"status":"missing","http":404,"bytes":1727},"home":{"status":"http_error","http":403,"url":"https://www.webpagetest.org/"}},
{"rank":4879,"domain":"pocoo.org","origin":"http://pocoo.org","robots":{"status":"missing","http":404,"url":"http://www.pocoo.org/robots.txt","bytes":162,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":162},"home":{"status":"ok","http":200,"url":"http://www.pocoo.org/","html_tokens":494,"markdown_tokens":137,"pruned_tokens":10}},
{"rank":4880,"domain":"augsburger-allgemeine.de","origin":"https://augsburger-allgemeine.de","robots":{"status":"ok","http":200,"url":"https://www.augsburger-allgemeine.de/robots.txt","bytes":7318,"groups":109},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["blocked","named",1,0],"OAI-SearchBot":["blocked","named",1,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["blocked","named",2,0],"Claude-SearchBot":["blocked","named",2,0],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["blocked","named",2,0],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",2,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",13,1]},"llms":{"status":"missing","http":404,"bytes":112189},"home":{"status":"ok","http":200,"url":"https://www.augsburger-allgemeine.de/","html_tokens":175354,"markdown_tokens":3913,"pruned_tokens":175}},
{"rank":4881,"domain":"robinhood.com","origin":"https://robinhood.com","robots":{"status":"ok","http":200,"url":"https://robinhood.com/robots.txt","bytes":240,"groups":1},"bots":{"GPTBot":["partial","*",7,1],"ChatGPT-User":["partial","*",7,1],"OAI-SearchBot":["partial","*",7,1],"ClaudeBot":["partial","*",7,1],"Claude-User":["partial","*",7,1],"Claude-SearchBot":["partial","*",7,1],"anthropic-ai":["partial","*",7,1],"PerplexityBot":["partial","*",7,1],"Perplexity-User":["partial","*",7,1],"Google-Extended":["partial","*",7,1],"Applebot-Extended":["partial","*",7,1],"CCBot":["partial","*",7,1],"Bytespider":["partial","*",7,1],"Amazonbot":["partial","*",7,1],"meta-externalagent":["partial","*",7,1],"cohere-ai":["partial","*",7,1],"Diffbot":["partial","*",7,1],"ContextFetchBot":["partial","*",7,1]},"llms":{"status":"http_error","http":403,"bytes":111},"home":{"status":"ok","http":200,"url":"https://robinhood.com/us/en/","html_tokens":66929,"markdown_tokens":175,"pruned_tokens":153}},
{"rank":4882,"domain":"gnavi.co.jp","origin":"https://gnavi.co.jp","robots":{"status":"ok","http":200,"url":"http://www.gnavi.co.jp/robots.txt","bytes":161,"groups":1},"bots":{"GPTBot":["partial","*",7,0],"ChatGPT-User":["partial","*",7,0],"OAI-SearchBot":["partial","*",7,0],"ClaudeBot":["partial","*",7,0],"Claude-User":["partial","*",7,0],"Claude-SearchBot":["partial","*",7,0],"anthropic-ai":["partial","*",7,0],"PerplexityBot":["partial","*",7,0],"Perplexity-User":["partial","*",7,0],"Google-Extended":["partial","*",7,0],"Applebot-Extended":["partial","*",7,0],"CCBot":["partial","*",7,0],"Bytespider":["partial","*",7,0],"Amazonbot":["partial","*",7,0],"meta-externalagent":["partial","*",7,0],"cohere-ai":["partial","*",7,0],"Diffbot":["partial","*",7,0],"ContextFetchBot":["partial","*",7,0]},"llms":{"status":"missing","http":404,"bytes":10959},"home":{"status":"ok","http":200,"url":"https://www.gnavi.co.jp/","html_tokens":26626,"markdown_tokens":12,"pruned_tokens":12}},
{"rank":4883,"domain":"ulule.com","origin":"https://ulule.com","robots":{"status":"ok","http":200,"url":"https://www.ulule.com/robots.txt","bytes":222,"groups":1},"bots":{"GPTBot":["partial","*",10,0],"ChatGPT-User":["partial","*",10,0],"OAI-SearchBot":["partial","*",10,0],"ClaudeBot":["partial","*",10,0],"Claude-User":["partial","*",10,0],"Claude-SearchBot":["partial","*",10,0],"anthropic-ai":["partial","*",10,0],"PerplexityBot":["partial","*",10,0],"Perplexity-User":["partial","*",10,0],"Google-Extended":["partial","*",10,0],"Applebot-Extended":["partial","*",10,0],"CCBot":["partial","*",10,0],"Bytespider":["partial","*",10,0],"Amazonbot":["partial","*",10,0],"meta-externalagent":["partial","*",10,0],"cohere-ai":["partial","*",10,0],"Diffbot":["partial","*",10,0],"ContextFetchBot":["partial","*",10,0]},"llms":{"status":"missing","http":404,"bytes":24122},"home":{"status":"ok","http":200,"url":"https://www.ulule.com/","html_tokens":53036,"markdown_tokens":44,"pruned_tokens":41}},
{"rank":4884,"domain":"jiangsu.gov.cn","origin":"https://www.jiangsu.gov.cn","robots":{"status":"ok","http":200,"url":"https://www.jiangsu.gov.cn/robots.txt","bytes":544,"groups":15},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":146},"home":{"status":"ok","http":200,"url":"https://www.jiangsu.gov.cn/","html_tokens":19081,"markdown_tokens":9,"pruned_tokens":9}},
{"rank":4885,"domain":"chicago.gov","origin":"https://chicago.gov","robots":{"status":"refused","http":403,"url":"https://chicago.gov/robots.txt","bytes":375,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":4886,"domain":"pscp.tv","origin":"https://pscp.tv","robots":{"status":"ok","http":200,"url":"https://www.pscp.tv/robots.txt","bytes":151,"groups":2},"bots":{"GPTBot":["partial","*",4,0],"ChatGPT-User":["partial","*",4,0],"OAI-SearchBot":["partial","*",4,0],"ClaudeBot":["partial","*",4,0],"Claude-User":["partial","*",4,0],"Claude-SearchBot":["partial","*",4,0],"anthropic-ai":["partial","*",4,0],"PerplexityBot":["partial","*",4,0],"Perplexity-User":["partial","*",4,0],"Google-Extended":["partial","*",4,0],"Applebot-Extended":["partial","*",4,0],"CCBot":["partial","*",4,0],"Bytespider":["partial","*",4,0],"Amazonbot":["partial","*",4,0],"meta-externalagent":["partial","*",4,0],"cohere-ai":["partial","*",4,0],"Diffbot":["partial","*",4,0],"ContextFetchBot":["partial","*",4,0]},"llms":{"status":"http_error","http":400,"bytes":15373},"home":{"status":"ok","http":200,"url":"https://www.pscp.tv/","html_tokens":5352,"markdown_tokens":42,"pruned_tokens":42}},
{"rank":4887,"domain":"yangshipin.cn","origin":"https://yangshipin.cn","robots":{"status":"html","http":200,"url":"https://yangshipin.cn/robots.txt","bytes":9341,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":4888,"domain":"tvp.pl","origin":"https://tvp.pl","robots":{"status":"ok","http":200,"url":"https://www.tvp.pl/robots.txt","bytes":401,"groups":1},"bots":{"GPTBot":["partial","*",9,1],"ChatGPT-User":["partial","*",9,1],"OAI-SearchBot":["partial","*",9,1],"ClaudeBot":["partial","*",9,1],"Claude-User":["partial","*",9,1],"Claude-SearchBot":["partial","*",9,1],"anthropic-ai":["partial","*",9,1],"PerplexityBot":["partial","*",9,1],"Perplexity-User":["partial","*",9,1],"Google-Extended":["partial","*",9,1],"Applebot-Extended":["partial","*",9,1],"CCBot":["partial","*",9,1],"Bytespider":["partial","*",9,1],"Amazonbot":["partial","*",9,1],"meta-externalagent":["partial","*",9,1],"cohere-ai":["partial","*",9,1],"Diffbot":["partial","*",9,1],"ContextFetchBot":["partial","*",9,1]},"llms":{"status":"missing","http":404,"bytes":114891},"home":{"status":"ok","http":200,"url":"https://www.tvp.pl/","html_tokens":957449,"markdown_tokens":0,"pruned_tokens":0}},
{"rank":4889,"domain":"chevrolet.com","origin":"https://chevrolet.com","robots":{"status":"refused","http":403,"url":"https://www.chevrolet.com/robots.txt","bytes":385,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":4890,"domain":"usccb.org","origin":"https://usccb.org","robots":{"status":"refused","http":403,"url":"https://usccb.org/robots.txt","bytes":560,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":4891,"domain":"unil.ch","origin":"https://unil.ch","robots":{"status":"ok","http":200,"url":"https://www.unil.ch/robots.txt","bytes":4121,"groups":2},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["partial","*",57,1],"OAI-SearchBot":["blocked","named",1,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["blocked","named",1,0],"Claude-SearchBot":["blocked","named",1,0],"anthropic-ai":["partial","*",57,1],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["partial","*",57,1],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",57,1]},"llms":{"status":"missing","http":404,"bytes":2985},"home":{"status":"ok","http":200,"url":"https://www.unil.ch/unil/fr/home.html","html_tokens":33273,"markdown_tokens":294,"pruned_tokens":132}},
{"rank":4892,"domain":"bose.com","origin":"https://bose.com","robots":{"status":"ok","http":200,"url":"https://www.bose.com/robots.txt","bytes":546,"groups":1},"bots":{"GPTBot":["partial","*",6,0],"ChatGPT-User":["partial","*",6,0],"OAI-SearchBot":["partial","*",6,0],"ClaudeBot":["partial","*",6,0],"Claude-User":["partial","*",6,0],"Claude-SearchBot":["partial","*",6,0],"anthropic-ai":["partial","*",6,0],"PerplexityBot":["partial","*",6,0],"Perplexity-User":["partial","*",6,0],"Google-Extended":["partial","*",6,0],"Applebot-Extended":["partial","*",6,0],"CCBot":["partial","*",6,0],"Bytespider":["partial","*",6,0],"Amazonbot":["partial","*",6,0],"meta-externalagent":["partial","*",6,0],"cohere-ai":["partial","*",6,0],"Diffbot":["partial","*",6,0],"ContextFetchBot":["partial","*",6,0]},"llms":{"status":"missing","http":404,"bytes":165540},"home":{"status":"ok","http":200,"url":"https://www.bose.com/home","html_tokens":81497,"markdown_tokens":266,"pruned_tokens":133}},
{"rank":4893,"domain":"regnum.ru","origin":"https://regnum.ru","robots":{"status":"ok","http":200,"url":"https://regnum.ru/robots.txt","bytes":592,"groups":3},"bots":{"GPTBot":["partial","*",4,0],"ChatGPT-User":["partial","*",4,0],"OAI-SearchBot":["partial","*",4,0],"ClaudeBot":["partial","*",4,0],"Claude-User":["partial","*",4,0],"Claude-SearchBot":["partial","*",4,0],"anthropic-ai":["partial","*",4,0],"PerplexityBot":["partial","*",4,0],"Perplexity-User":["partial","*",4,0],"Google-Extended":["partial","*",4,0],"Applebot-Extended":["partial","*",4,0],"CCBot":["partial","*",4,0],"Bytespider":["partial","*",4,0],"Amazonbot":["partial","*",4,0],"meta-externalagent":["partial","*",4,0],"cohere-ai":["partial","*",4,0],"Diffbot":["partial","*",4,0],"ContextFetchBot":["partial","*",4,0]},"llms":{"status":"missing","http":404,"bytes":387944},"home":{"status":"http_error","http":403,"url":"https://regnum.ru/"}},
{"rank":4894,"domain":"fortnite.com","origin":"https://fortnite.com","robots":{"status":"refused","http":403,"url":"https://www.fortnite.com/robots.txt","bytes":25359,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":4895,"domain":"images-amazon.com","origin":"https://images-amazon.com","robots":{"status":"unreachable","http":null,"url":"https://images-amazon.com/robots.txt","bytes":0,"groups":0,"error":"cannot resolve images-amazon.com"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":4896,"domain":"al-monitor.com","origin":"https://al-monitor.com","robots":{"status":"refused","http":403,"url":"https://al-monitor.com/robots.txt","bytes":560,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":4897,"domain":"websupport.sk","origin":"https://websupport.sk","robots":{"status":"ok","http":200,"url":"https://www.websupport.sk/robots.txt","bytes":138,"groups":1},"bots":{"GPTBot":["partial","*",1,2],"ChatGPT-User":["partial","*",1,2],"OAI-SearchBot":["partial","*",1,2],"ClaudeBot":["partial","*",1,2],"Claude-User":["partial","*",1,2],"Claude-SearchBot":["partial","*",1,2],"anthropic-ai":["partial","*",1,2],"PerplexityBot":["partial","*",1,2],"Perplexity-User":["partial","*",1,2],"Google-Extended":["partial","*",1,2],"Applebot-Extended":["partial","*",1,2],"CCBot":["partial","*",1,2],"Bytespider":["partial","*",1,2],"Amazonbot":["partial","*",1,2],"meta-externalagent":["partial","*",1,2],"cohere-ai":["partial","*",1,2],"Diffbot":["partial","*",1,2],"ContextFetchBot":["partial","*",1,2]},"llms":{"status":"present","http":200,"bytes":4536},"home":{"status":"ok","http":200,"url":"https://www.websupport.sk/","html_tokens":71905,"markdown_tokens":353,"pruned_tokens":116}},
{"rank":4898,"domain":"avid.com","origin":"https://avid.com","robots":{"status":"ok","http":200,"url":"https://www.avid.com/robots.txt","bytes":127,"groups":1},"bots":{"GPTBot":["partial","*",3,0],"ChatGPT-User":["partial","*",3,0],"OAI-SearchBot":["partial","*",3,0],"ClaudeBot":["partial","*",3,0],"Claude-User":["partial","*",3,0],"Claude-SearchBot":["partial","*",3,0],"anthropic-ai":["partial","*",3,0],"PerplexityBot":["partial","*",3,0],"Perplexity-User":["partial","*",3,0],"Google-Extended":["partial","*",3,0],"Applebot-Extended":["partial","*",3,0],"CCBot":["partial","*",3,0],"Bytespider":["partial","*",3,0],"Amazonbot":["partial","*",3,0],"meta-externalagent":["partial","*",3,0],"cohere-ai":["partial","*",3,0],"Diffbot":["partial","*",3,0],"ContextFetchBot":["partial","*",3,0]},"llms":{"status":"missing","http":404,"bytes":197430},"home":{"status":"http_error","http":403,"url":"https://www.avid.com/"}},
{"rank":4899,"domain":"thepiratebay.org","origin":"https://thepiratebay.org","robots":{"status":"ok","http":200,"url":"https://thepiratebay.org/robots.txt","bytes":49,"groups":1},"bots":{"GPTBot":["partial","*",1,0],"ChatGPT-User":["partial","*",1,0],"OAI-SearchBot":["partial","*",1,0],"ClaudeBot":["partial","*",1,0],"Claude-User":["partial","*",1,0],"Claude-SearchBot":["partial","*",1,0],"anthropic-ai":["partial","*",1,0],"PerplexityBot":["partial","*",1,0],"Perplexity-User":["partial","*",1,0],"Google-Extended":["partial","*",1,0],"Applebot-Extended":["partial","*",1,0],"CCBot":["partial","*",1,0],"Bytespider":["partial","*",1,0],"Amazonbot":["partial","*",1,0],"meta-externalagent":["partial","*",1,0],"cohere-ai":["partial","*",1,0],"Diffbot":["partial","*",1,0],"ContextFetchBot":["partial","*",1,0]},"llms":{"status":"html","http":200,"bytes":5643},"home":{"status":"ok","http":200,"url":"https://thepiratebay.org/index.html","html_tokens":1411,"markdown_tokens":27,"pruned_tokens":27}},
{"rank":4900,"domain":"ledger.com","origin":"https://ledger.com","robots":{"status":"ok","http":200,"url":"https://www.ledger.com/robots.txt","bytes":372,"groups":5},"bots":{"GPTBot":["allowed","named",0,1],"ChatGPT-User":["partial","*",3,3],"OAI-SearchBot":["partial","*",3,3],"ClaudeBot":["allowed","named",0,1],"Claude-User":["partial","*",3,3],"Claude-SearchBot":["partial","*",3,3],"anthropic-ai":["partial","*",3,3],"PerplexityBot":["allowed","named",0,1],"Perplexity-User":["partial","*",3,3],"Google-Extended":["allowed","named",0,1],"Applebot-Extended":["partial","*",3,3],"CCBot":["partial","*",3,3],"Bytespider":["partial","*",3,3],"Amazonbot":["partial","*",3,3],"meta-externalagent":["partial","*",3,3],"cohere-ai":["partial","*",3,3],"Diffbot":["partial","*",3,3],"ContextFetchBot":["partial","*",3,3]},"llms":{"status":"missing","http":404,"bytes":189304},"home":{"status":"ok","http":200,"url":"https://www.ledger.com/","html_tokens":67818,"markdown_tokens":282,"pruned_tokens":258}},
{"rank":4901,"domain":"thewire.in","origin":"https://thewire.in","robots":{"status":"ok","http":200,"url":"https://thewire.in/robots.txt","bytes":48,"groups":1},"bots":{"GPTBot":["partial","*",1,1],"ChatGPT-User":["partial","*",1,1],"OAI-SearchBot":["partial","*",1,1],"ClaudeBot":["partial","*",1,1],"Claude-User":["partial","*",1,1],"Claude-SearchBot":["partial","*",1,1],"anthropic-ai":["partial","*",1,1],"PerplexityBot":["partial","*",1,1],"Perplexity-User":["partial","*",1,1],"Google-Extended":["partial","*",1,1],"Applebot-Extended":["partial","*",1,1],"CCBot":["partial","*",1,1],"Bytespider":["partial","*",1,1],"Amazonbot":["partial","*",1,1],"meta-externalagent":["partial","*",1,1],"cohere-ai":["partial","*",1,1],"Diffbot":["partial","*",1,1],"ContextFetchBot":["partial","*",1,1]},"llms":{"status":"html","http":200,"bytes":10436},"home":{"status":"ok","http":200,"url":"https://thewire.in/","html_tokens":2844,"markdown_tokens":0,"pruned_tokens":0}},
{"rank":4902,"domain":"ceskatelevize.cz","origin":"https://ceskatelevize.cz","robots":{"status":"refused","http":403,"url":"https://www.ceskatelevize.cz/robots.txt","bytes":245,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":4903,"domain":"piktochart.com","origin":"https://piktochart.com","robots":{"status":"ok","http":200,"url":"https://piktochart.com/robots.txt","bytes":122,"groups":1},"bots":{"GPTBot":["partial","*",1,1],"ChatGPT-User":["partial","*",1,1],"OAI-SearchBot":["partial","*",1,1],"ClaudeBot":["partial","*",1,1],"Claude-User":["partial","*",1,1],"Claude-SearchBot":["partial","*",1,1],"anthropic-ai":["partial","*",1,1],"PerplexityBot":["partial","*",1,1],"Perplexity-User":["partial","*",1,1],"Google-Extended":["partial","*",1,1],"Applebot-Extended":["partial","*",1,1],"CCBot":["partial","*",1,1],"Bytespider":["partial","*",1,1],"Amazonbot":["partial","*",1,1],"meta-externalagent":["partial","*",1,1],"cohere-ai":["partial","*",1,1],"Diffbot":["partial","*",1,1],"ContextFetchBot":["partial","*",1,1]},"llms":{"status":"present","http":200,"bytes":30401},"home":{"status":"ok","http":200,"url":"https://piktochart.com/","html_tokens":54994,"markdown_tokens":792,"pruned_tokens":276}},
{"rank":4904,"domain":"financesonline.com","origin":"https://financesonline.com","robots":{"status":"ok","http":200,"url":"https://financesonline.com/robots.txt","bytes":23,"groups":1},"bots":{"GPTBot":["allowed","*",0,1],"ChatGPT-User":["allowed","*",0,1],"OAI-SearchBot":["allowed","*",0,1],"ClaudeBot":["allowed","*",0,1],"Claude-User":["allowed","*",0,1],"Claude-SearchBot":["allowed","*",0,1],"anthropic-ai":["allowed","*",0,1],"PerplexityBot":["allowed","*",0,1],"Perplexity-User":["allowed","*",0,1],"Google-Extended":["allowed","*",0,1],"Applebot-Extended":["allowed","*",0,1],"CCBot":["allowed","*",0,1],"Bytespider":["allowed","*",0,1],"Amazonbot":["allowed","*",0,1],"meta-externalagent":["allowed","*",0,1],"cohere-ai":["allowed","*",0,1],"Diffbot":["allowed","*",0,1],"ContextFetchBot":["allowed","*",0,1]},"llms":{"status":"missing","http":404,"bytes":32259},"home":{"status":"ok","http":200,"url":"https://financesonline.com/","html_tokens":58027,"markdown_tokens":403,"pruned_tokens":300}},
{"rank":4905,"domain":"lws.fr","origin":"https://lws.fr","robots":{"status":"ok","http":200,"url":"https://www.lws.fr/robots.txt","bytes":875,"groups":2},"bots":{"GPTBot":["partial","*",29,1],"ChatGPT-User":["partial","*",29,1],"OAI-SearchBot":["partial","*",29,1],"ClaudeBot":["partial","*",29,1],"Claude-User":["partial","*",29,1],"Claude-SearchBot":["partial","*",29,1],"anthropic-ai":["partial","*",29,1],"PerplexityBot":["partial","*",29,1],"Perplexity-User":["partial","*",29,1],"Google-Extended":["partial","*",29,1],"Applebot-Extended":["partial","*",29,1],"CCBot":["partial","*",29,1],"Bytespider":["partial","*",29,1],"Amazonbot":["partial","*",29,1],"meta-externalagent":["partial","*",29,1],"cohere-ai":["partial","*",29,1],"Diffbot":["partial","*",29,1],"ContextFetchBot":["partial","*",29,1]},"llms":{"status":"missing","http":404,"bytes":76635},"home":{"status":"ok","http":200,"url":"https://www.lws.fr/","html_tokens":26137,"markdown_tokens":917,"pruned_tokens":111}},
{"rank":4906,"domain":"newspapers.com","origin":"https://newspapers.com","robots":{"status":"refused","http":403,"url":"https://newspapers.com/robots.txt","bytes":5509,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":4907,"domain":"casinonsvenska.eu","origin":"https://casinonsvenska.eu","robots":{"status":"ok","http":200,"url":"https://casinonsvenska.eu/robots.txt","bytes":490,"groups":2},"bots":{"GPTBot":["partial","*",18,0],"ChatGPT-User":["partial","*",18,0],"OAI-SearchBot":["partial","*",18,0],"ClaudeBot":["partial","*",18,0],"Claude-User":["partial","*",18,0],"Claude-SearchBot":["partial","*",18,0],"anthropic-ai":["partial","*",18,0],"PerplexityBot":["partial","*",18,0],"Perplexity-User":["partial","*",18,0],"Google-Extended":["partial","*",18,0],"Applebot-Extended":["partial","*",18,0],"CCBot":["partial","*",18,0],"Bytespider":["partial","*",18,0],"Amazonbot":["partial","*",18,0],"meta-externalagent":["partial","*",18,0],"cohere-ai":["partial","*",18,0],"Diffbot":["partial","*",18,0],"ContextFetchBot":["partial","*",18,0]},"llms":{"status":"missing","http":404,"bytes":37869},"home":{"status":"ok","http":200,"url":"https://casinonsvenska.eu/","html_tokens":35247,"markdown_tokens":1041,"pruned_tokens":110}},
{"rank":4908,"domain":"food.gov.uk","origin":"https://food.gov.uk","robots":{"status":"ok","http":200,"url":"https://food.gov.uk/robots.txt","bytes":64,"groups":1},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"missing","http":404,"bytes":1700},"home":{"status":"ok","http":200,"url":"https://www.gov.uk/government/organisations/food-standards-agency","html_tokens":33723,"markdown_tokens":81,"pruned_tokens":81}},
{"rank":4909,"domain":"opendns.com","origin":"https://opendns.com","robots":{"status":"ok","http":200,"url":"https://www.opendns.com/robots.txt","bytes":24,"groups":1},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"missing","http":404,"bytes":8064},"home":{"status":"ok","http":200,"url":"https://www.opendns.com/","html_tokens":3292,"markdown_tokens":175,"pruned_tokens":47}},
{"rank":4910,"domain":"simplilearn.com","origin":"https://simplilearn.com","robots":{"status":"ok","http":200,"url":"https://www.simplilearn.com/robots.txt","bytes":3980,"groups":9},"bots":{"GPTBot":["partial","*",86,4],"ChatGPT-User":["allowed","named",0,0],"OAI-SearchBot":["allowed","named",0,0],"ClaudeBot":["allowed","named",0,0],"Claude-User":["partial","*",86,4],"Claude-SearchBot":["partial","*",86,4],"anthropic-ai":["partial","*",86,4],"PerplexityBot":["allowed","named",0,0],"Perplexity-User":["partial","*",86,4],"Google-Extended":["partial","*",86,4],"Applebot-Extended":["partial","*",86,4],"CCBot":["partial","*",86,4],"Bytespider":["partial","*",86,4],"Amazonbot":["partial","*",86,4],"meta-externalagent":["partial","*",86,4],"cohere-ai":["partial","*",86,4],"Diffbot":["partial","*",86,4],"ContextFetchBot":["partial","*",86,4]},"llms":{"status":"other_text","http":200,"bytes":1972060},"home":{"status":"ok","http":200,"url":"https://www.simplilearn.com/","html_tokens":152352,"markdown_tokens":272,"pruned_tokens":156}},
{"rank":4911,"domain":"president.gov.ua","origin":"https://president.gov.ua","robots":{"status":"refused","http":403,"url":"https://president.gov.ua/robots.txt","bytes":384,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":4912,"domain":"go.ug","origin":"https://go.ug","robots":{"status":"unreachable","http":null,"url":"https://go.ug/robots.txt","bytes":0,"groups":0,"error":"cannot resolve go.ug"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":4913,"domain":"levi.com","origin":"https://levi.com","robots":{"status":"ok","http":200,"url":"https://www.levi.com/robots.txt","bytes":70944,"groups":5},"bots":{"GPTBot":["partial","*",1618,9],"ChatGPT-User":["partial","*",1618,9],"OAI-SearchBot":["partial","*",1618,9],"ClaudeBot":["partial","*",1618,9],"Claude-User":["partial","*",1618,9],"Claude-SearchBot":["partial","*",1618,9],"anthropic-ai":["partial","*",1618,9],"PerplexityBot":["partial","*",1618,9],"Perplexity-User":["partial","*",1618,9],"Google-Extended":["partial","*",1618,9],"Applebot-Extended":["partial","*",1618,9],"CCBot":["partial","*",1618,9],"Bytespider":["partial","*",1618,9],"Amazonbot":["partial","*",1618,9],"meta-externalagent":["partial","*",1618,9],"cohere-ai":["partial","*",1618,9],"Diffbot":["partial","*",1618,9],"ContextFetchBot":["partial","*",1618,9]},"llms":{"status":"missing","http":404,"bytes":31},"home":{"status":"http_error","http":403,"url":"https://www.levi.com/US/en_US/"}},
{"rank":4914,"domain":"manta.com","origin":"https://manta.com","robots":{"status":"ok","http":200,"url":"https://www.manta.com/robots.txt","bytes":9591,"groups":6},"bots":{"GPTBot":["partial","named",3,0],"ChatGPT-User":["partial","*",55,3],"OAI-SearchBot":["partial","*",55,3],"ClaudeBot":["partial","named",3,0],"Claude-User":["partial","*",55,3],"Claude-SearchBot":["partial","*",55,3],"anthropic-ai":["partial","*",55,3],"PerplexityBot":["partial","*",55,3],"Perplexity-User":["partial","*",55,3],"Google-Extended":["partial","*",55,3],"Applebot-Extended":["partial","*",55,3],"CCBot":["blocked","named",1,0],"Bytespider":["partial","*",55,3],"Amazonbot":["partial","named",3,0],"meta-externalagent":["partial","named",3,0],"cohere-ai":["partial","*",55,3],"Diffbot":["partial","*",55,3],"ContextFetchBot":["partial","*",55,3]},"llms":{"status":"http_error","http":403,"bytes":5566},"home":{"status":"http_error","http":403,"url":"https://www.manta.com/"}},
{"rank":4915,"domain":"valvesoftware.com","origin":"https://valvesoftware.com","robots":{"status":"ok","http":200,"url":"https://www.valvesoftware.com/en/robots.txt","bytes":90,"groups":1},"bots":{"GPTBot":["partial","*",4,0],"ChatGPT-User":["partial","*",4,0],"OAI-SearchBot":["partial","*",4,0],"ClaudeBot":["partial","*",4,0],"Claude-User":["partial","*",4,0],"Claude-SearchBot":["partial","*",4,0],"anthropic-ai":["partial","*",4,0],"PerplexityBot":["partial","*",4,0],"Perplexity-User":["partial","*",4,0],"Google-Extended":["partial","*",4,0],"Applebot-Extended":["partial","*",4,0],"CCBot":["partial","*",4,0],"Bytespider":["partial","*",4,0],"Amazonbot":["partial","*",4,0],"meta-externalagent":["partial","*",4,0],"cohere-ai":["partial","*",4,0],"Diffbot":["partial","*",4,0],"ContextFetchBot":["partial","*",4,0]},"llms":{"status":"html","http":200,"bytes":22944},"home":{"status":"ok","http":200,"url":"https://www.valvesoftware.com/en/","html_tokens":5736,"markdown_tokens":719,"pruned_tokens":168}},
{"rank":4916,"domain":"info.flagcounter.com","origin":"https://info.flagcounter.com","robots":{"status":"missing","http":404,"url":"https://info.flagcounter.com/robots.txt","bytes":196,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":196},"home":{"status":"error","http":200,"url":"https://info.flagcounter.com/","error":"unparseable HTML"}},
{"rank":4917,"domain":"danfoss.com","origin":"https://danfoss.com","robots":{"status":"refused","http":403,"url":"https://www.danfoss.com/robots.txt","bytes":7884,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":4918,"domain":"uaf.edu","origin":"https://uaf.edu","robots":{"status":"ok","http":200,"url":"https://www.uaf.edu/robots.txt","bytes":330,"groups":3},"bots":{"GPTBot":["partial","*",9,0],"ChatGPT-User":["partial","*",9,0],"OAI-SearchBot":["partial","*",9,0],"ClaudeBot":["partial","*",9,0],"Claude-User":["partial","*",9,0],"Claude-SearchBot":["partial","*",9,0],"anthropic-ai":["partial","*",9,0],"PerplexityBot":["partial","*",9,0],"Perplexity-User":["partial","*",9,0],"Google-Extended":["partial","*",9,0],"Applebot-Extended":["partial","*",9,0],"CCBot":["partial","*",9,0],"Bytespider":["partial","*",9,0],"Amazonbot":["partial","*",9,0],"meta-externalagent":["partial","*",9,0],"cohere-ai":["partial","*",9,0],"Diffbot":["partial","*",9,0],"ContextFetchBot":["partial","*",9,0]},"llms":{"status":"missing","http":404,"bytes":31274},"home":{"status":"ok","http":200,"url":"https://www.uaf.edu/uaf/","html_tokens":17980,"markdown_tokens":811,"pruned_tokens":239}},
{"rank":4919,"domain":"boosty.to","origin":"https://boosty.to","robots":{"status":"ok","http":200,"url":"https://boosty.to/robots.txt","bytes":772,"groups":3},"bots":{"GPTBot":["partial","*",20,0],"ChatGPT-User":["partial","*",20,0],"OAI-SearchBot":["partial","*",20,0],"ClaudeBot":["partial","*",20,0],"Claude-User":["partial","*",20,0],"Claude-SearchBot":["partial","*",20,0],"anthropic-ai":["partial","*",20,0],"PerplexityBot":["partial","*",20,0],"Perplexity-User":["partial","*",20,0],"Google-Extended":["partial","*",20,0],"Applebot-Extended":["partial","*",20,0],"CCBot":["partial","*",20,0],"Bytespider":["partial","*",20,0],"Amazonbot":["partial","*",20,0],"meta-externalagent":["partial","*",20,0],"cohere-ai":["partial","*",20,0],"Diffbot":["partial","*",20,0],"ContextFetchBot":["partial","*",20,0]},"llms":{"status":"missing","http":404,"bytes":728719},"home":{"status":"ok","http":200,"url":"https://boosty.to/","html_tokens":191540,"markdown_tokens":483,"pruned_tokens":6}},
{"rank":4920,"domain":"eklablog.com","origin":"https://eklablog.com","robots":{"status":"ok","http":200,"url":"https://eklablog.com/robots.txt","bytes":23,"groups":1},"bots":{"GPTBot":["allowed","*",0,1],"ChatGPT-User":["allowed","*",0,1],"OAI-SearchBot":["allowed","*",0,1],"ClaudeBot":["allowed","*",0,1],"Claude-User":["allowed","*",0,1],"Claude-SearchBot":["allowed","*",0,1],"anthropic-ai":["allowed","*",0,1],"PerplexityBot":["allowed","*",0,1],"Perplexity-User":["allowed","*",0,1],"Google-Extended":["allowed","*",0,1],"Applebot-Extended":["allowed","*",0,1],"CCBot":["allowed","*",0,1],"Bytespider":["allowed","*",0,1],"Amazonbot":["allowed","*",0,1],"meta-externalagent":["allowed","*",0,1],"cohere-ai":["allowed","*",0,1],"Diffbot":["allowed","*",0,1],"ContextFetchBot":["allowed","*",0,1]},"llms":{"status":"missing","http":404,"bytes":34050},"home":{"status":"ok","http":200,"url":"https://www.eklablog.com/","html_tokens":38855,"markdown_tokens":200,"pruned_tokens":57}},
{"rank":4921,"domain":"npa.go.jp","origin":"https://www.npa.go.jp","robots":{"status":"ok","http":200,"url":"https://www.npa.go.jp/robots.txt","bytes":75,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":3351},"home":{"status":"ok","http":200,"url":"https://www.npa.go.jp/","html_tokens":5883,"markdown_tokens":14,"pruned_tokens":14}},
{"rank":4922,"domain":"inserm.fr","origin":"https://inserm.fr","robots":{"status":"ok","http":200,"url":"https://www.inserm.fr/robots.txt","bytes":771,"groups":1},"bots":{"GPTBot":["partial","*",24,1],"ChatGPT-User":["partial","*",24,1],"OAI-SearchBot":["partial","*",24,1],"ClaudeBot":["partial","*",24,1],"Claude-User":["partial","*",24,1],"Claude-SearchBot":["partial","*",24,1],"anthropic-ai":["partial","*",24,1],"PerplexityBot":["partial","*",24,1],"Perplexity-User":["partial","*",24,1],"Google-Extended":["partial","*",24,1],"Applebot-Extended":["partial","*",24,1],"CCBot":["partial","*",24,1],"Bytespider":["partial","*",24,1],"Amazonbot":["partial","*",24,1],"meta-externalagent":["partial","*",24,1],"cohere-ai":["partial","*",24,1],"Diffbot":["partial","*",24,1],"ContextFetchBot":["partial","*",24,1]},"llms":{"status":"missing","http":404,"bytes":65459},"home":{"status":"ok","http":200,"url":"https://www.inserm.fr/","html_tokens":31180,"markdown_tokens":257,"pruned_tokens":47}},
{"rank":4923,"domain":"gitter.im","origin":"https://gitter.im","robots":{"status":"missing","http":404,"url":"https://gitter.im/robots.txt","bytes":153,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":153},"home":{"status":"ok","http":200,"url":"https://gitter.im/","html_tokens":7270,"markdown_tokens":123,"pruned_tokens":123}},
{"rank":4924,"domain":"openresty.org","origin":"https://openresty.org","robots":{"status":"ok","http":200,"url":"https://openresty.org/robots.txt","bytes":116,"groups":1},"bots":{"GPTBot":["allowed","*",0,1],"ChatGPT-User":["allowed","*",0,1],"OAI-SearchBot":["allowed","*",0,1],"ClaudeBot":["allowed","*",0,1],"Claude-User":["allowed","*",0,1],"Claude-SearchBot":["allowed","*",0,1],"anthropic-ai":["allowed","*",0,1],"PerplexityBot":["allowed","*",0,1],"Perplexity-User":["allowed","*",0,1],"Google-Extended":["allowed","*",0,1],"Applebot-Extended":["allowed","*",0,1],"CCBot":["allowed","*",0,1],"Bytespider":["allowed","*",0,1],"Amazonbot":["allowed","*",0,1],"meta-externalagent":["allowed","*",0,1],"cohere-ai":["allowed","*",0,1],"Diffbot":["allowed","*",0,1],"ContextFetchBot":["allowed","*",0,1]},"llms":{"status":"missing","http":404,"bytes":12890},"home":{"status":"ok","http":200,"url":"https://openresty.org/en/","html_tokens":10681,"markdown_tokens":709,"pruned_tokens":656}},
{"rank":4925,"domain":"okx.com","origin":"https://okx.com","robots":{"status":"ok","http":200,"url":"https://www.okx.com/robots.txt","bytes":10188,"groups":3},"bots":{"GPTBot":["partial","*",216,16],"ChatGPT-User":["partial","*",216,16],"OAI-SearchBot":["partial","*",216,16],"ClaudeBot":["partial","*",216,16],"Claude-User":["partial","*",216,16],"Claude-SearchBot":["partial","*",216,16],"anthropic-ai":["partial","*",216,16],"PerplexityBot":["partial","*",216,16],"Perplexity-User":["partial","*",216,16],"Google-Extended":["partial","*",216,16],"Applebot-Extended":["partial","*",216,16],"CCBot":["partial","*",216,16],"Bytespider":["partial","*",216,16],"Amazonbot":["partial","*",216,16],"meta-externalagent":["partial","*",216,16],"cohere-ai":["partial","*",216,16],"Diffbot":["partial","*",216,16],"ContextFetchBot":["partial","*",216,16]},"llms":{"status":"present","http":200,"bytes":8899},"home":{"status":"ok","http":200,"url":"https://www.okx.com/en-us","html_tokens":18486,"markdown_tokens":176,"pruned_tokens":39}},
{"rank":4926,"domain":"yougov.co.uk","origin":"https://yougov.co.uk","robots":{"status":"missing","http":404,"url":"https://yougov.com/en-gb/robots.txt","bytes":424156,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":424083},"home":{"status":"ok","http":200,"url":"https://yougov.com/en-gb/","html_tokens":222886,"markdown_tokens":45,"pruned_tokens":45}},
{"rank":4927,"domain":"rogerebert.com","origin":"https://rogerebert.com","robots":{"status":"refused","http":403,"url":"https://rogerebert.com/robots.txt","bytes":103,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":4928,"domain":"weizmann.ac.il","origin":"https://weizmann.ac.il","robots":{"status":"ok","http":200,"url":"https://www.weizmann.ac.il/robots.txt","bytes":2104,"groups":2},"bots":{"GPTBot":["partial","*",43,0],"ChatGPT-User":["partial","*",43,0],"OAI-SearchBot":["partial","*",43,0],"ClaudeBot":["partial","*",43,0],"Claude-User":["partial","*",43,0],"Claude-SearchBot":["partial","*",43,0],"anthropic-ai":["partial","*",43,0],"PerplexityBot":["partial","*",43,0],"Perplexity-User":["partial","*",43,0],"Google-Extended":["partial","*",43,0],"Applebot-Extended":["partial","*",43,0],"CCBot":["partial","*",43,0],"Bytespider":["partial","*",43,0],"Amazonbot":["partial","*",43,0],"meta-externalagent":["partial","*",43,0],"cohere-ai":["partial","*",43,0],"Diffbot":["partial","*",43,0],"ContextFetchBot":["partial","*",43,0]},"llms":{"status":"timeout","error":"timed out after 10 s"},"home":{"status":"timeout","error":"timed out after 15 s"}},
{"rank":4929,"domain":"hermes.com","origin":"https://hermes.com","robots":{"status":"ok","http":200,"url":"https://www.hermes.com/robots.txt","bytes":7630,"groups":10},"bots":{"GPTBot":["partial","*",181,2],"ChatGPT-User":["partial","*",181,2],"OAI-SearchBot":["partial","*",181,2],"ClaudeBot":["partial","*",181,2],"Claude-User":["partial","*",181,2],"Claude-SearchBot":["partial","*",181,2],"anthropic-ai":["partial","*",181,2],"PerplexityBot":["partial","*",181,2],"Perplexity-User":["partial","*",181,2],"Google-Extended":["partial","*",181,2],"Applebot-Extended":["partial","*",181,2],"CCBot":["partial","*",181,2],"Bytespider":["partial","*",181,2],"Amazonbot":["partial","*",181,2],"meta-externalagent":["partial","*",181,2],"cohere-ai":["partial","*",181,2],"Diffbot":["partial","*",181,2],"ContextFetchBot":["partial","*",181,2]},"llms":{"status":"html","http":200,"bytes":671805},"home":{"status":"http_error","http":403,"url":"https://www.hermes.com/"}},
{"rank":4930,"domain":"uni-due.de","origin":"https://uni-due.de","robots":{"status":"ok","http":200,"url":"https://www.uni-due.de/robots.txt","bytes":247,"groups":2},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","named",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":11672},"home":{"status":"ok","http":200,"url":"https://www.uni-due.de/","html_tokens":18431,"markdown_tokens":69,"pruned_tokens":69}},
{"rank":4931,"domain":"securelist.com","origin":"https://securelist.com","robots":{"status":"ok","http":200,"url":"https://securelist.com/robots.txt","bytes":2809,"groups":21},"bots":{"GPTBot":["allowed","named",0,1],"ChatGPT-User":["allowed","named",0,1],"OAI-SearchBot":["allowed","named",0,1],"ClaudeBot":["allowed","named",0,1],"Claude-User":["allowed","named",0,1],"Claude-SearchBot":["partial","*",15,1],"anthropic-ai":["partial","*",15,1],"PerplexityBot":["allowed","named",0,1],"Perplexity-User":["allowed","named",0,1],"Google-Extended":["allowed","named",0,1],"Applebot-Extended":["allowed","named",0,1],"CCBot":["allowed","named",0,1],"Bytespider":["partial","*",15,1],"Amazonbot":["partial","*",15,1],"meta-externalagent":["partial","*",15,1],"cohere-ai":["partial","*",15,1],"Diffbot":["allowed","named",0,1],"ContextFetchBot":["partial","*",15,1]},"llms":{"status":"missing","http":404,"bytes":120794},"home":{"status":"ok","http":200,"url":"https://securelist.com/","html_tokens":42540,"markdown_tokens":219,"pruned_tokens":96}},
{"rank":4932,"domain":"virgilio.it","origin":"https://virgilio.it","robots":{"status":"ok","http":200,"url":"https://www.virgilio.it/robots.txt","bytes":2296,"groups":3},"bots":{"GPTBot":["partial","*",42,7],"ChatGPT-User":["partial","*",42,7],"OAI-SearchBot":["partial","*",42,7],"ClaudeBot":["partial","*",42,7],"Claude-User":["partial","*",42,7],"Claude-SearchBot":["partial","*",42,7],"anthropic-ai":["partial","*",42,7],"PerplexityBot":["partial","*",42,7],"Perplexity-User":["partial","*",42,7],"Google-Extended":["partial","*",42,7],"Applebot-Extended":["partial","*",42,7],"CCBot":["partial","*",42,7],"Bytespider":["partial","*",42,7],"Amazonbot":["partial","*",42,7],"meta-externalagent":["partial","*",42,7],"cohere-ai":["partial","*",42,7],"Diffbot":["partial","*",42,7],"ContextFetchBot":["partial","*",42,7]},"llms":{"status":"missing","http":404,"bytes":152316},"home":{"status":"ok","http":200,"url":"https://www.virgilio.it/","html_tokens":173346,"markdown_tokens":18,"pruned_tokens":18}},
{"rank":4933,"domain":"womenshealth.gov","origin":"https://womenshealth.gov","robots":{"status":"ok","http":200,"url":"https://womenshealth.gov/robots.txt","bytes":2027,"groups":1},"bots":{"GPTBot":["partial","*",34,18],"ChatGPT-User":["partial","*",34,18],"OAI-SearchBot":["partial","*",34,18],"ClaudeBot":["partial","*",34,18],"Claude-User":["partial","*",34,18],"Claude-SearchBot":["partial","*",34,18],"anthropic-ai":["partial","*",34,18],"PerplexityBot":["partial","*",34,18],"Perplexity-User":["partial","*",34,18],"Google-Extended":["partial","*",34,18],"Applebot-Extended":["partial","*",34,18],"CCBot":["partial","*",34,18],"Bytespider":["partial","*",34,18],"Amazonbot":["partial","*",34,18],"meta-externalagent":["partial","*",34,18],"cohere-ai":["partial","*",34,18],"Diffbot":["partial","*",34,18],"ContextFetchBot":["partial","*",34,18]},"llms":{"status":"missing","http":404,"bytes":291},"home":{"status":"ok","http":200,"url":"https://womenshealth.gov/","html_tokens":16996,"markdown_tokens":195,"pruned_tokens":152}},
{"rank":4934,"domain":"au.com","origin":"https://au.com","robots":{"status":"ok","http":200,"url":"https://www.au.com/robots.txt","bytes":267,"groups":1},"bots":{"GPTBot":["partial","*",7,0],"ChatGPT-User":["partial","*",7,0],"OAI-SearchBot":["partial","*",7,0],"ClaudeBot":["partial","*",7,0],"Claude-User":["partial","*",7,0],"Claude-SearchBot":["partial","*",7,0],"anthropic-ai":["partial","*",7,0],"PerplexityBot":["partial","*",7,0],"Perplexity-User":["partial","*",7,0],"Google-Extended":["partial","*",7,0],"Applebot-Extended":["partial","*",7,0],"CCBot":["partial","*",7,0],"Bytespider":["partial","*",7,0],"Amazonbot":["partial","*",7,0],"meta-externalagent":["partial","*",7,0],"cohere-ai":["partial","*",7,0],"Diffbot":["partial","*",7,0],"ContextFetchBot":["partial","*",7,0]},"llms":{"status":"missing","http":404,"bytes":89126},"home":{"status":"ok","http":200,"url":"https://www.au.com/","html_tokens":33119,"markdown_tokens":32,"pruned_tokens":32}},
{"rank":4935,"domain":"articulate.com","origin":"https://articulate.com","robots":{"status":"ok","http":200,"url":"https://www.articulate.com/robots.txt","bytes":9825,"groups":2},"bots":{"GPTBot":["partial","*",14,295],"ChatGPT-User":["partial","*",14,295],"OAI-SearchBot":["partial","*",14,295],"ClaudeBot":["partial","*",14,295],"Claude-User":["partial","*",14,295],"Claude-SearchBot":["partial","*",14,295],"anthropic-ai":["partial","*",14,295],"PerplexityBot":["partial","*",14,295],"Perplexity-User":["partial","*",14,295],"Google-Extended":["partial","*",14,295],"Applebot-Extended":["partial","*",14,295],"CCBot":["partial","*",14,295],"Bytespider":["partial","*",14,295],"Amazonbot":["partial","*",14,295],"meta-externalagent":["partial","*",14,295],"cohere-ai":["partial","*",14,295],"Diffbot":["partial","*",14,295],"ContextFetchBot":["partial","*",14,295]},"llms":{"status":"http_error","http":403,"bytes":5571},"home":{"status":"http_error","http":403,"url":"https://www.articulate.com/"}},
{"rank":4936,"domain":"ambafrance.org","origin":"https://ambafrance.org","robots":{"status":"server_error","http":500,"url":"https://ambafrance.org/robots.txt","bytes":19356,"groups":0,"error":"HTTP 500"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":4937,"domain":"farm3.static.flickr.com","origin":"https://farm3.static.flickr.com","robots":{"status":"ok","http":200,"url":"https://farm3.static.flickr.com/robots.txt","bytes":21,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"http_error","http":403,"bytes":93},"home":{"status":"http_error","http":403,"url":"https://farm3.static.flickr.com/"}},
{"rank":4938,"domain":"bcb.gov.br","origin":"https://bcb.gov.br","robots":{"status":"ok","http":200,"url":"https://bcb.gov.br/robots.txt","bytes":1147,"groups":9},"bots":{"GPTBot":["partial","*",6,1],"ChatGPT-User":["partial","*",6,1],"OAI-SearchBot":["partial","*",6,1],"ClaudeBot":["partial","*",6,1],"Claude-User":["partial","*",6,1],"Claude-SearchBot":["partial","*",6,1],"anthropic-ai":["partial","*",6,1],"PerplexityBot":["partial","*",6,1],"Perplexity-User":["partial","*",6,1],"Google-Extended":["partial","*",6,1],"Applebot-Extended":["partial","*",6,1],"CCBot":["partial","*",6,1],"Bytespider":["partial","*",6,1],"Amazonbot":["partial","*",6,1],"meta-externalagent":["partial","*",6,1],"cohere-ai":["partial","*",6,1],"Diffbot":["partial","*",6,1],"ContextFetchBot":["partial","*",6,1]},"llms":{"status":"html","http":200,"bytes":2871},"home":{"status":"ok","http":200,"url":"https://bcb.gov.br/","html_tokens":717,"markdown_tokens":23,"pruned_tokens":23}},
{"rank":4939,"domain":"cninfo.com.cn","origin":"https://www.cninfo.com.cn","robots":{"status":"missing","http":404,"url":"https://www.cninfo.com.cn/robots.txt","bytes":573,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":573},"home":{"status":"ok","http":200,"url":"https://www.cninfo.com.cn/","html_tokens":26323,"markdown_tokens":76,"pruned_tokens":76}},
{"rank":4940,"domain":"infosecurity-magazine.com","origin":"https://infosecurity-magazine.com","robots":{"status":"ok","http":200,"url":"https://www.infosecurity-magazine.com/robots.txt","bytes":128,"groups":1},"bots":{"GPTBot":["partial","*",2,0],"ChatGPT-User":["partial","*",2,0],"OAI-SearchBot":["partial","*",2,0],"ClaudeBot":["partial","*",2,0],"Claude-User":["partial","*",2,0],"Claude-SearchBot":["partial","*",2,0],"anthropic-ai":["partial","*",2,0],"PerplexityBot":["partial","*",2,0],"Perplexity-User":["partial","*",2,0],"Google-Extended":["partial","*",2,0],"Applebot-Extended":["partial","*",2,0],"CCBot":["partial","*",2,0],"Bytespider":["partial","*",2,0],"Amazonbot":["partial","*",2,0],"meta-externalagent":["partial","*",2,0],"cohere-ai":["partial","*",2,0],"Diffbot":["partial","*",2,0],"ContextFetchBot":["partial","*",2,0]},"llms":{"status":"missing","http":404,"bytes":12399},"home":{"status":"ok","http":200,"url":"https://www.infosecurity-magazine.com/","html_tokens":35445,"markdown_tokens":51,"pruned_tokens":51}},
{"rank":4941,"domain":"opb.org","origin":"https://opb.org","robots":{"status":"ok","http":200,"url":"https://www.opb.org/robots.txt","bytes":217,"groups":1},"bots":{"GPTBot":["partial","*",8,1],"ChatGPT-User":["partial","*",8,1],"OAI-SearchBot":["partial","*",8,1],"ClaudeBot":["partial","*",8,1],"Claude-User":["partial","*",8,1],"Claude-SearchBot":["partial","*",8,1],"anthropic-ai":["partial","*",8,1],"PerplexityBot":["partial","*",8,1],"Perplexity-User":["partial","*",8,1],"Google-Extended":["partial","*",8,1],"Applebot-Extended":["partial","*",8,1],"CCBot":["partial","*",8,1],"Bytespider":["partial","*",8,1],"Amazonbot":["partial","*",8,1],"meta-externalagent":["partial","*",8,1],"cohere-ai":["partial","*",8,1],"Diffbot":["partial","*",8,1],"ContextFetchBot":["partial","*",8,1]},"llms":{"status":"missing","http":404,"bytes":166549},"home":{"status":"ok","http":200,"url":"https://www.opb.org/","html_tokens":207441,"markdown_tokens":47,"pruned_tokens":47}},
{"rank":4942,"domain":"open.edu","origin":"https://open.edu","robots":{"status":"refused","http":403,"url":"https://www.open.edu/robots.txt","bytes":5529,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":4943,"domain":"thewaltdisneycompany.com","origin":"https://thewaltdisneycompany.com","robots":{"status":"ok","http":200,"url":"https://thewaltdisneycompany.com/robots.txt","bytes":149,"groups":1},"bots":{"GPTBot":["partial","*",2,1],"ChatGPT-User":["partial","*",2,1],"OAI-SearchBot":["partial","*",2,1],"ClaudeBot":["partial","*",2,1],"Claude-User":["partial","*",2,1],"Claude-SearchBot":["partial","*",2,1],"anthropic-ai":["partial","*",2,1],"PerplexityBot":["partial","*",2,1],"Perplexity-User":["partial","*",2,1],"Google-Extended":["partial","*",2,1],"Applebot-Extended":["partial","*",2,1],"CCBot":["partial","*",2,1],"Bytespider":["partial","*",2,1],"Amazonbot":["partial","*",2,1],"meta-externalagent":["partial","*",2,1],"cohere-ai":["partial","*",2,1],"Diffbot":["partial","*",2,1],"ContextFetchBot":["partial","*",2,1]},"llms":{"status":"missing","http":404,"bytes":119593},"home":{"status":"ok","http":200,"url":"https://thewaltdisneycompany.com/","html_tokens":58576,"markdown_tokens":118,"pruned_tokens":118}},
{"rank":4944,"domain":"goodmorningamerica.com","origin":"https://goodmorningamerica.com","robots":{"status":"ok","http":200,"url":"https://goodmorningamerica.com/robots.txt","bytes":733,"groups":11},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["blocked","named",1,0],"OAI-SearchBot":["partial","*",7,0],"ClaudeBot":["partial","*",7,0],"Claude-User":["partial","*",7,0],"Claude-SearchBot":["partial","*",7,0],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["partial","*",7,0],"Perplexity-User":["partial","*",7,0],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["partial","*",7,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["partial","*",7,0],"meta-externalagent":["partial","*",7,0],"cohere-ai":["partial","*",7,0],"Diffbot":["partial","*",7,0],"ContextFetchBot":["partial","*",7,0]},"llms":{"status":"missing","http":404,"bytes":1},"home":{"status":"ok","http":200,"url":"https://goodmorningamerica.com/","html_tokens":67197,"markdown_tokens":9,"pruned_tokens":9}},
{"rank":4945,"domain":"dzwww.com","origin":"https://dzwww.com","robots":{"status":"ok","http":200,"url":"https://dzwww.com/robots.txt","bytes":72,"groups":1},"bots":{"GPTBot":["partial","*",1,1],"ChatGPT-User":["partial","*",1,1],"OAI-SearchBot":["partial","*",1,1],"ClaudeBot":["partial","*",1,1],"Claude-User":["partial","*",1,1],"Claude-SearchBot":["partial","*",1,1],"anthropic-ai":["partial","*",1,1],"PerplexityBot":["partial","*",1,1],"Perplexity-User":["partial","*",1,1],"Google-Extended":["partial","*",1,1],"Applebot-Extended":["partial","*",1,1],"CCBot":["partial","*",1,1],"Bytespider":["partial","*",1,1],"Amazonbot":["partial","*",1,1],"meta-externalagent":["partial","*",1,1],"cohere-ai":["partial","*",1,1],"Diffbot":["partial","*",1,1],"ContextFetchBot":["partial","*",1,1]},"llms":{"status":"missing","http":404,"bytes":146},"home":{"status":"ok","http":200,"url":"https://dzwww.com/","html_tokens":25336,"markdown_tokens":64,"pruned_tokens":64}},
{"rank":4946,"domain":"jw.org","origin":"https://jw.org","robots":{"status":"ok","http":200,"url":"https://www.jw.org/robots.txt","bytes":228,"groups":1},"bots":{"GPTBot":["partial","*",7,0],"ChatGPT-User":["partial","*",7,0],"OAI-SearchBot":["partial","*",7,0],"ClaudeBot":["partial","*",7,0],"Claude-User":["partial","*",7,0],"Claude-SearchBot":["partial","*",7,0],"anthropic-ai":["partial","*",7,0],"PerplexityBot":["partial","*",7,0],"Perplexity-User":["partial","*",7,0],"Google-Extended":["partial","*",7,0],"Applebot-Extended":["partial","*",7,0],"CCBot":["partial","*",7,0],"Bytespider":["partial","*",7,0],"Amazonbot":["partial","*",7,0],"meta-externalagent":["partial","*",7,0],"cohere-ai":["partial","*",7,0],"Diffbot":["partial","*",7,0],"ContextFetchBot":["partial","*",7,0]},"llms":{"status":"missing","http":404,"bytes":165146},"home":{"status":"ok","http":200,"url":"https://www.jw.org/en/","html_tokens":44664,"markdown_tokens":132,"pruned_tokens":132}},
{"rank":4947,"domain":"japantoday.com","origin":"https://japantoday.com","robots":{"status":"ok","http":200,"url":"https://japantoday.com/robots.txt","bytes":5619,"groups":136},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":33661},"home":{"status":"ok","http":200,"url":"https://japantoday.com/","html_tokens":638,"markdown_tokens":8,"pruned_tokens":8}},
{"rank":4948,"domain":"northjersey.com","origin":"https://northjersey.com","robots":{"status":"ok","http":200,"url":"https://www.northjersey.com/robots.txt","bytes":12671,"groups":288},"bots":{"GPTBot":["partial","named",1,4],"ChatGPT-User":["partial","named",1,4],"OAI-SearchBot":["partial","named",1,4],"ClaudeBot":["blocked","named",1,0],"Claude-User":["blocked","named",1,0],"Claude-SearchBot":["blocked","named",1,0],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["partial","named",1,4],"Perplexity-User":["blocked","named",1,0],"Google-Extended":["partial","named",1,4],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",2,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",32,0]},"llms":{"status":"http_error","http":402,"bytes":254},"home":{"status":"http_error","http":402,"url":"https://www.northjersey.com/"}},
{"rank":4949,"domain":"google.com.pe","origin":"https://google.com.pe","robots":{"status":"ok","http":200,"url":"https://www.google.com.pe/robots.txt","bytes":6624,"groups":4},"bots":{"GPTBot":["partial","*",169,64],"ChatGPT-User":["partial","*",169,64],"OAI-SearchBot":["partial","*",169,64],"ClaudeBot":["partial","*",169,64],"Claude-User":["partial","*",169,64],"Claude-SearchBot":["partial","*",169,64],"anthropic-ai":["partial","*",169,64],"PerplexityBot":["partial","*",169,64],"Perplexity-User":["partial","*",169,64],"Google-Extended":["partial","*",169,64],"Applebot-Extended":["partial","*",169,64],"CCBot":["partial","*",169,64],"Bytespider":["partial","*",169,64],"Amazonbot":["partial","*",169,64],"meta-externalagent":["partial","*",169,64],"cohere-ai":["partial","*",169,64],"Diffbot":["partial","*",169,64],"ContextFetchBot":["partial","*",169,64]},"llms":{"status":"missing","http":404,"bytes":1569},"home":{"status":"ok","http":200,"url":"https://www.google.com/","html_tokens":21479,"markdown_tokens":30,"pruned_tokens":30}},
{"rank":4950,"domain":"uni-regensburg.de","origin":"https://uni-regensburg.de","robots":{"status":"ok","http":200,"url":"https://www.uni-regensburg.de/robots.txt","bytes":48,"groups":1},"bots":{"GPTBot":["allowed","*",0,1],"ChatGPT-User":["allowed","*",0,1],"OAI-SearchBot":["allowed","*",0,1],"ClaudeBot":["allowed","*",0,1],"Claude-User":["allowed","*",0,1],"Claude-SearchBot":["allowed","*",0,1],"anthropic-ai":["allowed","*",0,1],"PerplexityBot":["allowed","*",0,1],"Perplexity-User":["allowed","*",0,1],"Google-Extended":["allowed","*",0,1],"Applebot-Extended":["allowed","*",0,1],"CCBot":["allowed","*",0,1],"Bytespider":["allowed","*",0,1],"Amazonbot":["allowed","*",0,1],"meta-externalagent":["allowed","*",0,1],"cohere-ai":["allowed","*",0,1],"Diffbot":["allowed","*",0,1],"ContextFetchBot":["allowed","*",0,1]},"llms":{"status":"missing","http":404,"bytes":93833},"home":{"status":"ok","http":200,"url":"https://www.uni-regensburg.de/","html_tokens":35443,"markdown_tokens":43,"pruned_tokens":43}},
{"rank":4951,"domain":"adn.com","origin":"https://adn.com","robots":{"status":"ok","http":200,"url":"https://www.adn.com/robots.txt","bytes":813,"groups":10},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["blocked","named",1,0],"OAI-SearchBot":["blocked","named",1,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["blocked","named",1,0],"Claude-SearchBot":["partial","*",2,0],"anthropic-ai":["partial","*",2,0],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["blocked","named",1,0],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["partial","*",2,0],"CCBot":["blocked","named",1,0],"Bytespider":["partial","*",2,0],"Amazonbot":["partial","*",2,0],"meta-externalagent":["partial","*",2,0],"cohere-ai":["partial","*",2,0],"Diffbot":["partial","*",2,0],"ContextFetchBot":["partial","*",2,0]},"llms":{"status":"missing","http":404,"bytes":127755},"home":{"status":"ok","http":200,"url":"https://www.adn.com/","html_tokens":88358,"markdown_tokens":748,"pruned_tokens":162}},
{"rank":4952,"domain":"manutd.com","origin":"https://manutd.com","robots":{"status":"ok","http":200,"url":"https://www.manutd.com/robots.txt","bytes":143,"groups":2},"bots":{"GPTBot":["partial","*",1,1],"ChatGPT-User":["partial","*",1,1],"OAI-SearchBot":["partial","*",1,1],"ClaudeBot":["partial","*",1,1],"Claude-User":["partial","*",1,1],"Claude-SearchBot":["partial","*",1,1],"anthropic-ai":["partial","*",1,1],"PerplexityBot":["partial","*",1,1],"Perplexity-User":["partial","*",1,1],"Google-Extended":["partial","*",1,1],"Applebot-Extended":["partial","*",1,1],"CCBot":["partial","*",1,1],"Bytespider":["partial","*",1,1],"Amazonbot":["partial","*",1,1],"meta-externalagent":["partial","*",1,1],"cohere-ai":["partial","*",1,1],"Diffbot":["partial","*",1,1],"ContextFetchBot":["partial","*",1,1]},"llms":{"status":"missing","http":404,"bytes":280917},"home":{"status":"ok","http":200,"url":"https://www.manutd.com/en","html_tokens":340137,"markdown_tokens":237,"pruned_tokens":146}},
{"rank":4953,"domain":"mandarinoriental.com","origin":"https://mandarinoriental.com","robots":{"status":"ok","http":200,"url":"https://www.mandarinoriental.com/robots.txt","bytes":500,"groups":3},"bots":{"GPTBot":["partial","*",14,1],"ChatGPT-User":["partial","*",14,1],"OAI-SearchBot":["partial","*",14,1],"ClaudeBot":["partial","*",14,1],"Claude-User":["partial","*",14,1],"Claude-SearchBot":["partial","*",14,1],"anthropic-ai":["partial","*",14,1],"PerplexityBot":["partial","*",14,1],"Perplexity-User":["partial","*",14,1],"Google-Extended":["partial","*",14,1],"Applebot-Extended":["partial","*",14,1],"CCBot":["partial","*",14,1],"Bytespider":["partial","*",14,1],"Amazonbot":["partial","*",14,1],"meta-externalagent":["partial","*",14,1],"cohere-ai":["partial","*",14,1],"Diffbot":["partial","*",14,1],"ContextFetchBot":["partial","*",14,1]},"llms":{"status":"missing","http":404,"bytes":0},"home":{"status":"ok","http":200,"url":"https://www.mandarinoriental.com/en","html_tokens":344393,"markdown_tokens":73,"pruned_tokens":73}},
{"rank":4954,"domain":"rj.gov.br","origin":"https://rj.gov.br","robots":{"status":"server_error","http":500,"url":"https://www.rj.gov.br/robots.txt","bytes":39115,"groups":0,"error":"HTTP 500"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":4955,"domain":"zeiss.com","origin":"https://zeiss.com","robots":{"status":"ok","http":200,"url":"https://www.zeiss.com/robots.txt","bytes":6651,"groups":1},"bots":{"GPTBot":["partial","*",2,1],"ChatGPT-User":["partial","*",2,1],"OAI-SearchBot":["partial","*",2,1],"ClaudeBot":["partial","*",2,1],"Claude-User":["partial","*",2,1],"Claude-SearchBot":["partial","*",2,1],"anthropic-ai":["partial","*",2,1],"PerplexityBot":["partial","*",2,1],"Perplexity-User":["partial","*",2,1],"Google-Extended":["partial","*",2,1],"Applebot-Extended":["partial","*",2,1],"CCBot":["partial","*",2,1],"Bytespider":["partial","*",2,1],"Amazonbot":["partial","*",2,1],"meta-externalagent":["partial","*",2,1],"cohere-ai":["partial","*",2,1],"Diffbot":["partial","*",2,1],"ContextFetchBot":["partial","*",2,1]},"llms":{"status":"present","http":200,"bytes":2070},"home":{"status":"ok","http":200,"url":"https://www.zeiss.com/corporate/en/home.html","html_tokens":165257,"markdown_tokens":442,"pruned_tokens":299}},
{"rank":4956,"domain":"rmg.co.uk","origin":"https://rmg.co.uk","robots":{"status":"refused","http":403,"url":"https://rmg.co.uk/robots.txt","bytes":560,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":4957,"domain":"um.es","origin":"https://um.es","robots":{"status":"ok","http":200,"url":"https://www.um.es/robots.txt","bytes":889,"groups":2},"bots":{"GPTBot":["partial","*",16,0],"ChatGPT-User":["partial","*",16,0],"OAI-SearchBot":["partial","*",16,0],"ClaudeBot":["partial","*",16,0],"Claude-User":["partial","*",16,0],"Claude-SearchBot":["partial","*",16,0],"anthropic-ai":["partial","*",16,0],"PerplexityBot":["partial","*",16,0],"Perplexity-User":["partial","*",16,0],"Google-Extended":["partial","*",16,0],"Applebot-Extended":["partial","*",16,0],"CCBot":["partial","*",16,0],"Bytespider":["partial","*",16,0],"Amazonbot":["partial","*",16,0],"meta-externalagent":["partial","*",16,0],"cohere-ai":["partial","*",16,0],"Diffbot":["partial","*",16,0],"ContextFetchBot":["partial","*",16,0]},"llms":{"status":"missing","http":404,"bytes":183822},"home":{"status":"ok","http":200,"url":"https://www.um.es/","html_tokens":79724,"markdown_tokens":11,"pruned_tokens":11}},
{"rank":4958,"domain":"akkadia.org","origin":"https://akkadia.org","robots":{"status":"ok","http":200,"url":"https://akkadia.org/robots.txt","bytes":52,"groups":2},"bots":{"GPTBot":["blocked","*",2,0],"ChatGPT-User":["blocked","*",2,0],"OAI-SearchBot":["blocked","*",2,0],"ClaudeBot":["blocked","*",2,0],"Claude-User":["blocked","*",2,0],"Claude-SearchBot":["blocked","*",2,0],"anthropic-ai":["blocked","*",2,0],"PerplexityBot":["blocked","*",2,0],"Perplexity-User":["blocked","*",2,0],"Google-Extended":["blocked","*",2,0],"Applebot-Extended":["blocked","*",2,0],"CCBot":["blocked","*",2,0],"Bytespider":["blocked","*",2,0],"Amazonbot":["blocked","*",2,0],"meta-externalagent":["blocked","*",2,0],"cohere-ai":["blocked","*",2,0],"Diffbot":["blocked","*",2,0],"ContextFetchBot":["blocked","*",2,0]},"llms":{"status":"disallowed"},"home":{"status":"disallowed"}},
{"rank":4959,"domain":"kddi.com","origin":"https://kddi.com","robots":{"status":"missing","http":404,"url":"https://www.kddi.com/robots.txt","bytes":43568,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":43568},"home":{"status":"ok","http":200,"url":"https://www.kddi.com/","html_tokens":21803,"markdown_tokens":18,"pruned_tokens":18}},
{"rank":4960,"domain":"letemps.ch","origin":"https://letemps.ch","robots":{"status":"ok","http":200,"url":"https://letemps.ch/robots.txt","bytes":3544,"groups":93},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["partial","*",9,0],"OAI-SearchBot":["partial","*",9,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",9,0],"Claude-SearchBot":["partial","*",9,0],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["partial","*",9,0],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",2,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",9,0]},"llms":{"status":"missing","http":404,"bytes":1526},"home":{"status":"ok","http":200,"url":"https://www.letemps.ch","html_tokens":78698,"markdown_tokens":1129,"pruned_tokens":124}},
{"rank":4961,"domain":"arrl.org","origin":"https://arrl.org","robots":{"status":"ok","http":200,"url":"https://www.arrl.org/robots.txt","bytes":333,"groups":1},"bots":{"GPTBot":["partial","*",10,0],"ChatGPT-User":["partial","*",10,0],"OAI-SearchBot":["partial","*",10,0],"ClaudeBot":["partial","*",10,0],"Claude-User":["partial","*",10,0],"Claude-SearchBot":["partial","*",10,0],"anthropic-ai":["partial","*",10,0],"PerplexityBot":["partial","*",10,0],"Perplexity-User":["partial","*",10,0],"Google-Extended":["partial","*",10,0],"Applebot-Extended":["partial","*",10,0],"CCBot":["partial","*",10,0],"Bytespider":["partial","*",10,0],"Amazonbot":["partial","*",10,0],"meta-externalagent":["partial","*",10,0],"cohere-ai":["partial","*",10,0],"Diffbot":["partial","*",10,0],"ContextFetchBot":["partial","*",10,0]},"llms":{"status":"html","http":200,"bytes":35860},"home":{"status":"ok","http":200,"url":"https://www.arrl.org/","html_tokens":11659,"markdown_tokens":175,"pruned_tokens":173}},
{"rank":4962,"domain":"leafletjs.com","origin":"https://leafletjs.com","robots":{"status":"missing","http":404,"url":"https://leafletjs.com/robots.txt","bytes":4950,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":4950},"home":{"status":"ok","http":200,"url":"https://leafletjs.com/","html_tokens":3527,"markdown_tokens":917,"pruned_tokens":151}},
{"rank":4963,"domain":"sam.gov","origin":"https://sam.gov","robots":{"status":"ok","http":200,"url":"https://sam.gov/robots.txt","bytes":2027,"groups":1},"bots":{"GPTBot":["partial","*",34,18],"ChatGPT-User":["partial","*",34,18],"OAI-SearchBot":["partial","*",34,18],"ClaudeBot":["partial","*",34,18],"Claude-User":["partial","*",34,18],"Claude-SearchBot":["partial","*",34,18],"anthropic-ai":["partial","*",34,18],"PerplexityBot":["partial","*",34,18],"Perplexity-User":["partial","*",34,18],"Google-Extended":["partial","*",34,18],"Applebot-Extended":["partial","*",34,18],"CCBot":["partial","*",34,18],"Bytespider":["partial","*",34,18],"Amazonbot":["partial","*",34,18],"meta-externalagent":["partial","*",34,18],"cohere-ai":["partial","*",34,18],"Diffbot":["partial","*",34,18],"ContextFetchBot":["partial","*",34,18]},"llms":{"status":"missing","http":404,"bytes":204},"home":{"status":"ok","http":200,"url":"https://sam.gov/","html_tokens":13868,"markdown_tokens":71,"pruned_tokens":71}},
{"rank":4964,"domain":"contently.com","origin":"https://contently.com","robots":{"status":"ok","http":200,"url":"https://contently.com/robots.txt","bytes":81,"groups":1},"bots":{"GPTBot":["partial","*",1,0],"ChatGPT-User":["partial","*",1,0],"OAI-SearchBot":["partial","*",1,0],"ClaudeBot":["partial","*",1,0],"Claude-User":["partial","*",1,0],"Claude-SearchBot":["partial","*",1,0],"anthropic-ai":["partial","*",1,0],"PerplexityBot":["partial","*",1,0],"Perplexity-User":["partial","*",1,0],"Google-Extended":["partial","*",1,0],"Applebot-Extended":["partial","*",1,0],"CCBot":["partial","*",1,0],"Bytespider":["partial","*",1,0],"Amazonbot":["partial","*",1,0],"meta-externalagent":["partial","*",1,0],"cohere-ai":["partial","*",1,0],"Diffbot":["partial","*",1,0],"ContextFetchBot":["partial","*",1,0]},"llms":{"status":"missing","http":404,"bytes":60},"home":{"status":"ok","http":200,"url":"https://contently.com/","html_tokens":39734,"markdown_tokens":664,"pruned_tokens":148}},
{"rank":4965,"domain":"lynda.com","origin":"https://lynda.com","robots":{"status":"missing","http":404,"url":"https://www.linkedin.com/learning/page-not-found?trk=lynda_redirect_learning","bytes":115162,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"disallowed","error":"robots.txt of www.linkedin.com disallows /learning/page-not-found?trk=lynda_redirect_learning"},"home":{"status":"disallowed","error":"robots.txt of www.linkedin.com disallows /learning/?trk=lynda_redirect_learning"}},
{"rank":4966,"domain":"kyushu-u.ac.jp","origin":"https://www.kyushu-u.ac.jp","robots":{"status":"missing","http":404,"url":"https://www.kyushu-u.ac.jp/robots.txt","bytes":58903,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":58899},"home":{"status":"ok","http":200,"url":"https://www.kyushu-u.ac.jp/ja/","html_tokens":28356,"markdown_tokens":268,"pruned_tokens":99}},
{"rank":4967,"domain":"jamesclear.com","origin":"https://jamesclear.com","robots":{"status":"ok","http":200,"url":"https://jamesclear.com/robots.txt","bytes":172,"groups":1},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"missing","http":404,"bytes":48671},"home":{"status":"ok","http":200,"url":"https://jamesclear.com/","html_tokens":14110,"markdown_tokens":360,"pruned_tokens":144}},
{"rank":4968,"domain":"qlik.com","origin":"https://qlik.com","robots":{"status":"ok","http":200,"url":"https://www.qlik.com/robots.txt","bytes":57,"groups":1},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"present","http":200,"bytes":12607},"home":{"status":"ok","http":200,"url":"https://www.qlik.com/us","html_tokens":252023,"markdown_tokens":164,"pruned_tokens":164}},
{"rank":4969,"domain":"lublin.pl","origin":"http://lublin.pl","robots":{"status":"missing","http":404,"url":"http://lublin.pl/robots.txt","bytes":267,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":265},"home":{"status":"ok","http":200,"url":"http://lublin.pl/","html_tokens":419,"markdown_tokens":0,"pruned_tokens":0}},
{"rank":4970,"domain":"curseforge.com","origin":"https://curseforge.com","robots":{"status":"refused","http":403,"url":"https://www.curseforge.com/robots.txt","bytes":5560,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":4971,"domain":"thieme-connect.com","origin":"https://thieme-connect.com","robots":{"status":"ok","http":200,"url":"https://thieme-connect.com/robots.txt","bytes":1931,"groups":43},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["blocked","named",1,0],"OAI-SearchBot":["blocked","named",1,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",8,0],"Claude-SearchBot":["partial","*",8,0],"anthropic-ai":["partial","*",8,0],"PerplexityBot":["partial","*",8,0],"Perplexity-User":["partial","*",8,0],"Google-Extended":["partial","*",8,0],"Applebot-Extended":["partial","*",8,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",2,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["partial","*",8,0],"Diffbot":["partial","*",8,0],"ContextFetchBot":["partial","*",8,0]},"llms":{"status":"html","http":200,"bytes":4767},"home":{"status":"ok","http":200,"url":"https://thieme-connect.com/","html_tokens":1192,"markdown_tokens":94,"pruned_tokens":66}},
{"rank":4972,"domain":"nirsoft.net","origin":"https://nirsoft.net","robots":{"status":"ok","http":200,"url":"https://www.nirsoft.net/robots.txt","bytes":32,"groups":1},"bots":{"GPTBot":["partial","*",1,0],"ChatGPT-User":["partial","*",1,0],"OAI-SearchBot":["partial","*",1,0],"ClaudeBot":["partial","*",1,0],"Claude-User":["partial","*",1,0],"Claude-SearchBot":["partial","*",1,0],"anthropic-ai":["partial","*",1,0],"PerplexityBot":["partial","*",1,0],"Perplexity-User":["partial","*",1,0],"Google-Extended":["partial","*",1,0],"Applebot-Extended":["partial","*",1,0],"CCBot":["partial","*",1,0],"Bytespider":["partial","*",1,0],"Amazonbot":["partial","*",1,0],"meta-externalagent":["partial","*",1,0],"cohere-ai":["partial","*",1,0],"Diffbot":["partial","*",1,0],"ContextFetchBot":["partial","*",1,0]},"llms":{"status":"missing","http":404,"bytes":355},"home":{"status":"ok","http":200,"url":"https://www.nirsoft.net/","html_tokens":11060,"markdown_tokens":1736,"pruned_tokens":174}},
{"rank":4973,"domain":"ufsc.br","origin":"https://ufsc.br","robots":{"status":"missing","http":404,"url":"https://ufsc.br/robots.txt","bytes":178,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":19605},"home":{"status":"ok","http":200,"url":"https://ufsc.br/","html_tokens":19181,"markdown_tokens":181,"pruned_tokens":181}},
{"rank":4974,"domain":"arrow.com","origin":"https://arrow.com","robots":{"status":"unreachable","http":null,"url":"https://arrow.com/robots.txt","bytes":0,"groups":0,"error":"<StreamReset stream_id:1, error_code:2, remote_reset:True>"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":4975,"domain":"spokesman.com","origin":"https://spokesman.com","robots":{"status":"ok","http":200,"url":"https://www.spokesman.com/robots.txt","bytes":6498,"groups":21},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["partial","named",20,1],"OAI-SearchBot":["partial","named",20,1],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","named",20,1],"Claude-SearchBot":["partial","named",20,1],"anthropic-ai":["partial","*",20,1],"PerplexityBot":["partial","named",20,1],"Perplexity-User":["partial","named",20,1],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["partial","named",20,1],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["partial","*",20,1],"Diffbot":["partial","*",20,1],"ContextFetchBot":["partial","*",20,1]},"llms":{"status":"missing","http":404,"bytes":124457},"home":{"status":"ok","http":200,"url":"https://www.spokesman.com/","html_tokens":72508,"markdown_tokens":24,"pruned_tokens":24}},
{"rank":4976,"domain":"vg.no","origin":"https://vg.no","robots":{"status":"ok","http":200,"url":"https://vg.no/robots.txt","bytes":2174,"groups":3},"bots":{"GPTBot":["partial","*",11,1],"ChatGPT-User":["partial","*",11,1],"OAI-SearchBot":["partial","named",11,1],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",11,1],"Claude-SearchBot":["blocked","named",1,0],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["partial","*",11,1],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",11,1]},"llms":{"status":"missing","http":404,"bytes":105530},"home":{"status":"ok","http":200,"url":"https://www.vg.no/","html_tokens":116724,"markdown_tokens":20,"pruned_tokens":20}},
{"rank":4977,"domain":"2k.com","origin":"https://2k.com","robots":{"status":"ok","http":200,"url":"https://2k.com/robots.txt","bytes":99,"groups":1},"bots":{"GPTBot":["partial","*",2,0],"ChatGPT-User":["partial","*",2,0],"OAI-SearchBot":["partial","*",2,0],"ClaudeBot":["partial","*",2,0],"Claude-User":["partial","*",2,0],"Claude-SearchBot":["partial","*",2,0],"anthropic-ai":["partial","*",2,0],"PerplexityBot":["partial","*",2,0],"Perplexity-User":["partial","*",2,0],"Google-Extended":["partial","*",2,0],"Applebot-Extended":["partial","*",2,0],"CCBot":["partial","*",2,0],"Bytespider":["partial","*",2,0],"Amazonbot":["partial","*",2,0],"meta-externalagent":["partial","*",2,0],"cohere-ai":["partial","*",2,0],"Diffbot":["partial","*",2,0],"ContextFetchBot":["partial","*",2,0]},"llms":{"status":"missing","http":404,"bytes":308},"home":{"status":"ok","http":200,"url":"https://2k.com/","html_tokens":115997,"markdown_tokens":132,"pruned_tokens":94}},
{"rank":4978,"domain":"townandcountrymag.com","origin":"https://townandcountrymag.com","robots":{"status":"ok","http":200,"url":"https://townandcountrymag.com/robots.txt","bytes":979,"groups":3},"bots":{"GPTBot":["partial","named",32,0],"ChatGPT-User":["partial","named",32,0],"OAI-SearchBot":["partial","named",32,0],"ClaudeBot":["partial","*",19,0],"Claude-User":["partial","*",19,0],"Claude-SearchBot":["partial","*",19,0],"anthropic-ai":["partial","*",19,0],"PerplexityBot":["partial","*",19,0],"Perplexity-User":["partial","*",19,0],"Google-Extended":["partial","*",19,0],"Applebot-Extended":["partial","*",19,0],"CCBot":["partial","*",19,0],"Bytespider":["partial","*",19,0],"Amazonbot":["partial","*",19,0],"meta-externalagent":["partial","*",19,0],"cohere-ai":["partial","*",19,0],"Diffbot":["partial","*",19,0],"ContextFetchBot":["partial","*",19,0]},"llms":{"status":"missing","http":404,"bytes":84},"home":{"status":"ok","http":200,"url":"https://www.townandcountrymag.com/","html_tokens":176029,"markdown_tokens":108,"pruned_tokens":108}},
{"rank":4979,"domain":"inmotionhosting.com","origin":"https://inmotionhosting.com","robots":{"status":"ok","http":200,"url":"https://www.inmotionhosting.com/robots.txt","bytes":1626,"groups":1},"bots":{"GPTBot":["partial","*",36,2],"ChatGPT-User":["partial","*",36,2],"OAI-SearchBot":["partial","*",36,2],"ClaudeBot":["partial","*",36,2],"Claude-User":["partial","*",36,2],"Claude-SearchBot":["partial","*",36,2],"anthropic-ai":["partial","*",36,2],"PerplexityBot":["partial","*",36,2],"Perplexity-User":["partial","*",36,2],"Google-Extended":["partial","*",36,2],"Applebot-Extended":["partial","*",36,2],"CCBot":["partial","*",36,2],"Bytespider":["partial","*",36,2],"Amazonbot":["partial","*",36,2],"meta-externalagent":["partial","*",36,2],"cohere-ai":["partial","*",36,2],"Diffbot":["partial","*",36,2],"ContextFetchBot":["partial","*",36,2]},"llms":{"status":"other_text","http":200,"bytes":18577},"home":{"status":"http_error","http":403,"url":"https://inmotionhosting.com/"}},
{"rank":4980,"domain":"livedoor.com","origin":"https://livedoor.com","robots":{"status":"ok","http":200,"url":"https://www.livedoor.com/robots.txt","bytes":33,"groups":1},"bots":{"GPTBot":["partial","*",1,0],"ChatGPT-User":["partial","*",1,0],"OAI-SearchBot":["partial","*",1,0],"ClaudeBot":["partial","*",1,0],"Claude-User":["partial","*",1,0],"Claude-SearchBot":["partial","*",1,0],"anthropic-ai":["partial","*",1,0],"PerplexityBot":["partial","*",1,0],"Perplexity-User":["partial","*",1,0],"Google-Extended":["partial","*",1,0],"Applebot-Extended":["partial","*",1,0],"CCBot":["partial","*",1,0],"Bytespider":["partial","*",1,0],"Amazonbot":["partial","*",1,0],"meta-externalagent":["partial","*",1,0],"cohere-ai":["partial","*",1,0],"Diffbot":["partial","*",1,0],"ContextFetchBot":["partial","*",1,0]},"llms":{"status":"missing","http":404,"bytes":674},"home":{"status":"ok","http":200,"url":"https://www.livedoor.com/","html_tokens":6567,"markdown_tokens":232,"pruned_tokens":114}},
{"rank":4981,"domain":"statefarm.com","origin":"https://statefarm.com","robots":{"status":"ok","http":200,"url":"https://www.statefarm.com/robots.txt","bytes":1005,"groups":2},"bots":{"GPTBot":["partial","*",19,1],"ChatGPT-User":["partial","*",19,1],"OAI-SearchBot":["partial","*",19,1],"ClaudeBot":["partial","*",19,1],"Claude-User":["partial","*",19,1],"Claude-SearchBot":["partial","*",19,1],"anthropic-ai":["partial","*",19,1],"PerplexityBot":["partial","*",19,1],"Perplexity-User":["partial","*",19,1],"Google-Extended":["partial","*",19,1],"Applebot-Extended":["partial","*",19,1],"CCBot":["partial","*",19,1],"Bytespider":["partial","*",19,1],"Amazonbot":["partial","*",19,1],"meta-externalagent":["partial","*",19,1],"cohere-ai":["partial","*",19,1],"Diffbot":["partial","*",19,1],"ContextFetchBot":["partial","*",19,1]},"llms":{"status":"missing","http":404,"bytes":42406},"home":{"status":"ok","http":200,"url":"https://www.statefarm.com/","html_tokens":90794,"markdown_tokens":810,"pruned_tokens":107}},
{"rank":4982,"domain":"temu.com","origin":"https://temu.com","robots":{"status":"ok","http":200,"url":"https://www.temu.com/robots.txt","bytes":8239,"groups":5},"bots":{"GPTBot":["partial","named",103,0],"ChatGPT-User":["partial","named",103,0],"OAI-SearchBot":["partial","named",103,0],"ClaudeBot":["partial","named",103,0],"Claude-User":["partial","named",103,0],"Claude-SearchBot":["partial","named",103,0],"anthropic-ai":["allowed","*",0,1],"PerplexityBot":["partial","named",103,0],"Perplexity-User":["allowed","*",0,1],"Google-Extended":["partial","named",103,0],"Applebot-Extended":["allowed","*",0,1],"CCBot":["allowed","*",0,1],"Bytespider":["blocked","named",1,0],"Amazonbot":["allowed","*",0,1],"meta-externalagent":["allowed","*",0,1],"cohere-ai":["allowed","*",0,1],"Diffbot":["allowed","*",0,1],"ContextFetchBot":["allowed","*",0,1]},"llms":{"status":"missing","http":404,"bytes":0},"home":{"status":"ok","http":200,"url":"https://www.temu.com/","html_tokens":118587,"markdown_tokens":69,"pruned_tokens":69}},
{"rank":4983,"domain":"deadspin.com","origin":"https://deadspin.com","robots":{"status":"ok","http":200,"url":"https://deadspin.com/robots.txt","bytes":317,"groups":3},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["partial","*",10,0],"OAI-SearchBot":["partial","*",10,0],"ClaudeBot":["partial","*",10,0],"Claude-User":["partial","*",10,0],"Claude-SearchBot":["partial","*",10,0],"anthropic-ai":["partial","*",10,0],"PerplexityBot":["partial","*",10,0],"Perplexity-User":["partial","*",10,0],"Google-Extended":["partial","*",10,0],"Applebot-Extended":["partial","*",10,0],"CCBot":["blocked","named",1,0],"Bytespider":["partial","*",10,0],"Amazonbot":["partial","*",10,0],"meta-externalagent":["partial","*",10,0],"cohere-ai":["partial","*",10,0],"Diffbot":["partial","*",10,0],"ContextFetchBot":["partial","*",10,0]},"llms":{"status":"missing","http":404,"bytes":272877},"home":{"status":"ok","http":200,"url":"https://deadspin.com/","html_tokens":131018,"markdown_tokens":373,"pruned_tokens":91}},
{"rank":4984,"domain":"italia.it","origin":"https://italia.it","robots":{"status":"ok","http":200,"url":"https://www.italia.it/robots.txt","bytes":1196,"groups":1},"bots":{"GPTBot":["partial","*",9,5],"ChatGPT-User":["partial","*",9,5],"OAI-SearchBot":["partial","*",9,5],"ClaudeBot":["partial","*",9,5],"Claude-User":["partial","*",9,5],"Claude-SearchBot":["partial","*",9,5],"anthropic-ai":["partial","*",9,5],"PerplexityBot":["partial","*",9,5],"Perplexity-User":["partial","*",9,5],"Google-Extended":["partial","*",9,5],"Applebot-Extended":["partial","*",9,5],"CCBot":["partial","*",9,5],"Bytespider":["partial","*",9,5],"Amazonbot":["partial","*",9,5],"meta-externalagent":["partial","*",9,5],"cohere-ai":["partial","*",9,5],"Diffbot":["partial","*",9,5],"ContextFetchBot":["partial","*",9,5]},"llms":{"status":"missing","http":404,"bytes":189397},"home":{"status":"ok","http":200,"url":"https://www.italia.it/en","html_tokens":352951,"markdown_tokens":81,"pruned_tokens":81}},
{"rank":4985,"domain":"service-now.com","origin":"https://service-now.com","robots":{"status":"refused","http":403,"url":"http://www.servicenow.com/","bytes":372,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":4986,"domain":"cloudflare.vc","origin":"https://cloudflare.vc","robots":{"status":"timeout","http":null,"url":"https://cloudflare.vc/robots.txt","bytes":0,"groups":0,"error":"ConnectTimeout"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":4987,"domain":"samaritans.org","origin":"https://samaritans.org","robots":{"status":"ok","http":200,"url":"https://www.samaritans.org/robots.txt","bytes":63,"groups":1},"bots":{"GPTBot":["partial","*",2,1],"ChatGPT-User":["partial","*",2,1],"OAI-SearchBot":["partial","*",2,1],"ClaudeBot":["partial","*",2,1],"Claude-User":["partial","*",2,1],"Claude-SearchBot":["partial","*",2,1],"anthropic-ai":["partial","*",2,1],"PerplexityBot":["partial","*",2,1],"Perplexity-User":["partial","*",2,1],"Google-Extended":["partial","*",2,1],"Applebot-Extended":["partial","*",2,1],"CCBot":["partial","*",2,1],"Bytespider":["partial","*",2,1],"Amazonbot":["partial","*",2,1],"meta-externalagent":["partial","*",2,1],"cohere-ai":["partial","*",2,1],"Diffbot":["partial","*",2,1],"ContextFetchBot":["partial","*",2,1]},"llms":{"status":"missing","http":404,"bytes":216360},"home":{"status":"ok","http":200,"url":"https://www.samaritans.org/","html_tokens":120463,"markdown_tokens":299,"pruned_tokens":80}},
{"rank":4988,"domain":"dw-world.de","origin":"http://dw-world.de","robots":{"status":"ok","http":200,"url":"https://www.dw.com/robots.txt","bytes":2757,"groups":51},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["blocked","named",1,0],"OAI-SearchBot":["blocked","named",1,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",18,0],"Claude-SearchBot":["partial","*",18,0],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["partial","*",18,0],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",18,0]},"llms":{"status":"missing","http":404,"bytes":5283},"home":{"status":"ok","http":200,"url":"https://www.dw.com/","html_tokens":3047,"markdown_tokens":12,"pruned_tokens":12}},
{"rank":4989,"domain":"stjude.org","origin":"https://stjude.org","robots":{"status":"ok","http":200,"url":"https://www.stjude.org/robots.txt","bytes":1430,"groups":1},"bots":{"GPTBot":["partial","*",21,0],"ChatGPT-User":["partial","*",21,0],"OAI-SearchBot":["partial","*",21,0],"ClaudeBot":["partial","*",21,0],"Claude-User":["partial","*",21,0],"Claude-SearchBot":["partial","*",21,0],"anthropic-ai":["partial","*",21,0],"PerplexityBot":["partial","*",21,0],"Perplexity-User":["partial","*",21,0],"Google-Extended":["partial","*",21,0],"Applebot-Extended":["partial","*",21,0],"CCBot":["partial","*",21,0],"Bytespider":["partial","*",21,0],"Amazonbot":["partial","*",21,0],"meta-externalagent":["partial","*",21,0],"cohere-ai":["partial","*",21,0],"Diffbot":["partial","*",21,0],"ContextFetchBot":["partial","*",21,0]},"llms":{"status":"present","http":200,"bytes":5956},"home":{"status":"ok","http":200,"url":"https://www.stjude.org/","html_tokens":74443,"markdown_tokens":31,"pruned_tokens":31}},
{"rank":4990,"domain":"divephotoguide.com","origin":"https://divephotoguide.com","robots":{"status":"ok","http":200,"url":"https://www.divephotoguide.com/robots.txt","bytes":113,"groups":1},"bots":{"GPTBot":["partial","*",2,0],"ChatGPT-User":["partial","*",2,0],"OAI-SearchBot":["partial","*",2,0],"ClaudeBot":["partial","*",2,0],"Claude-User":["partial","*",2,0],"Claude-SearchBot":["partial","*",2,0],"anthropic-ai":["partial","*",2,0],"PerplexityBot":["partial","*",2,0],"Perplexity-User":["partial","*",2,0],"Google-Extended":["partial","*",2,0],"Applebot-Extended":["partial","*",2,0],"CCBot":["partial","*",2,0],"Bytespider":["partial","*",2,0],"Amazonbot":["partial","*",2,0],"meta-externalagent":["partial","*",2,0],"cohere-ai":["partial","*",2,0],"Diffbot":["partial","*",2,0],"ContextFetchBot":["partial","*",2,0]},"llms":{"status":"missing","http":404,"bytes":17877},"home":{"status":"ok","http":200,"url":"https://www.divephotoguide.com/","html_tokens":5648,"markdown_tokens":103,"pruned_tokens":29}},
{"rank":4991,"domain":"8m.com","origin":"https://8m.com","robots":{"status":"unreachable","http":null,"url":"https://8m.com/robots.txt","bytes":0,"groups":0,"error":"cannot resolve 8m.com"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":4992,"domain":"thomasnet.com","origin":"https://thomasnet.com","robots":{"status":"refused","http":403,"url":"https://www.thomasnet.com/robots.txt","bytes":773,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":4993,"domain":"vie-publique.fr","origin":"https://vie-publique.fr","robots":{"status":"html","http":200,"url":"https://vie-publique.fr/robots.txt","bytes":232,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":4994,"domain":"list.ly","origin":"https://list.ly","robots":{"status":"missing","http":404,"url":"https://list.ly/robots.txt","bytes":0,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":0},"home":{"status":"ok","http":200,"url":"https://list.ly/","html_tokens":832,"markdown_tokens":355,"pruned_tokens":161}},
{"rank":4995,"domain":"hubblesite.org","origin":"https://hubblesite.org","robots":{"status":"html","http":200,"url":"https://science.nasa.gov/mission/hubble/","bytes":397730,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":4996,"domain":"nju.edu.cn","origin":"https://www.nju.edu.cn","robots":{"status":"missing","http":404,"url":"https://www.nju.edu.cn/robots.txt","bytes":1693,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":1693},"home":{"status":"ok","http":200,"url":"https://www.nju.edu.cn/","html_tokens":73559,"markdown_tokens":29,"pruned_tokens":29}},
{"rank":4997,"domain":"s-sols.com","origin":"https://s-sols.com","robots":{"status":"ok","http":200,"url":"https://www.s-sols.com/robots.txt","bytes":1889,"groups":3},"bots":{"GPTBot":["partial","*",15,3],"ChatGPT-User":["partial","*",15,3],"OAI-SearchBot":["partial","*",15,3],"ClaudeBot":["partial","*",15,3],"Claude-User":["partial","*",15,3],"Claude-SearchBot":["partial","*",15,3],"anthropic-ai":["partial","*",15,3],"PerplexityBot":["partial","*",15,3],"Perplexity-User":["partial","*",15,3],"Google-Extended":["partial","*",15,3],"Applebot-Extended":["partial","*",15,3],"CCBot":["partial","*",15,3],"Bytespider":["partial","*",15,3],"Amazonbot":["partial","*",15,3],"meta-externalagent":["partial","*",15,3],"cohere-ai":["partial","*",15,3],"Diffbot":["partial","*",15,3],"ContextFetchBot":["partial","*",15,3]},"llms":{"status":"missing","http":404,"bytes":69359},"home":{"status":"ok","http":200,"url":"https://www.s-sols.com/","html_tokens":76930,"markdown_tokens":402,"pruned_tokens":133}},
{"rank":4998,"domain":"continental.com","origin":"https://continental.com","robots":{"status":"ok","http":200,"url":"https://www.continental.com//robots.txt","bytes":2381,"groups":2},"bots":{"GPTBot":["partial","*",15,1],"ChatGPT-User":["partial","*",15,1],"OAI-SearchBot":["partial","*",15,1],"ClaudeBot":["partial","*",15,1],"Claude-User":["partial","*",15,1],"Claude-SearchBot":["partial","*",15,1],"anthropic-ai":["partial","*",15,1],"PerplexityBot":["partial","*",15,1],"Perplexity-User":["partial","*",15,1],"Google-Extended":["partial","*",15,1],"Applebot-Extended":["partial","*",15,1],"CCBot":["partial","*",15,1],"Bytespider":["partial","*",15,1],"Amazonbot":["partial","*",15,1],"meta-externalagent":["partial","*",15,1],"cohere-ai":["partial","*",15,1],"Diffbot":["partial","*",15,1],"ContextFetchBot":["partial","*",15,1]},"llms":{"status":"missing","http":404,"bytes":175393},"home":{"status":"ok","http":200,"url":"https://www.continental.com/en/","html_tokens":55189,"markdown_tokens":410,"pruned_tokens":149}},
{"rank":4999,"domain":"legiscan.com","origin":"https://legiscan.com","robots":{"status":"refused","http":403,"url":"https://legiscan.com/robots.txt","bytes":5507,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":5000,"domain":"surfshark.com","origin":"https://surfshark.com","robots":{"status":"ok","http":200,"url":"https://surfshark.com/robots.txt","bytes":397,"groups":1},"bots":{"GPTBot":["partial","*",8,1],"ChatGPT-User":["partial","*",8,1],"OAI-SearchBot":["partial","*",8,1],"ClaudeBot":["partial","*",8,1],"Claude-User":["partial","*",8,1],"Claude-SearchBot":["partial","*",8,1],"anthropic-ai":["partial","*",8,1],"PerplexityBot":["partial","*",8,1],"Perplexity-User":["partial","*",8,1],"Google-Extended":["partial","*",8,1],"Applebot-Extended":["partial","*",8,1],"CCBot":["partial","*",8,1],"Bytespider":["partial","*",8,1],"Amazonbot":["partial","*",8,1],"meta-externalagent":["partial","*",8,1],"cohere-ai":["partial","*",8,1],"Diffbot":["partial","*",8,1],"ContextFetchBot":["partial","*",8,1]},"llms":{"status":"present","http":200,"bytes":13352},"home":{"status":"ok","http":200,"url":"https://surfshark.com/","html_tokens":53354,"markdown_tokens":1059,"pruned_tokens":249}},
{"rank":5001,"domain":"petmd.com","origin":"https://petmd.com","robots":{"status":"ok","http":200,"url":"https://www.petmd.com/robots.txt","bytes":1597,"groups":1},"bots":{"GPTBot":["partial","*",23,18],"ChatGPT-User":["partial","*",23,18],"OAI-SearchBot":["partial","*",23,18],"ClaudeBot":["partial","*",23,18],"Claude-User":["partial","*",23,18],"Claude-SearchBot":["partial","*",23,18],"anthropic-ai":["partial","*",23,18],"PerplexityBot":["partial","*",23,18],"Perplexity-User":["partial","*",23,18],"Google-Extended":["partial","*",23,18],"Applebot-Extended":["partial","*",23,18],"CCBot":["partial","*",23,18],"Bytespider":["partial","*",23,18],"Amazonbot":["partial","*",23,18],"meta-externalagent":["partial","*",23,18],"cohere-ai":["partial","*",23,18],"Diffbot":["partial","*",23,18],"ContextFetchBot":["partial","*",23,18]},"llms":{"status":"missing","http":404,"bytes":73228},"home":{"status":"ok","http":200,"url":"https://www.petmd.com/","html_tokens":35056,"markdown_tokens":65,"pruned_tokens":47}},
{"rank":5002,"domain":"humanitix.com","origin":"https://humanitix.com","robots":{"status":"ok","http":200,"url":"https://humanitix.com/robots.txt","bytes":67,"groups":1},"bots":{"GPTBot":["allowed","*",0,1],"ChatGPT-User":["allowed","*",0,1],"OAI-SearchBot":["allowed","*",0,1],"ClaudeBot":["allowed","*",0,1],"Claude-User":["allowed","*",0,1],"Claude-SearchBot":["allowed","*",0,1],"anthropic-ai":["allowed","*",0,1],"PerplexityBot":["allowed","*",0,1],"Perplexity-User":["allowed","*",0,1],"Google-Extended":["allowed","*",0,1],"Applebot-Extended":["allowed","*",0,1],"CCBot":["allowed","*",0,1],"Bytespider":["allowed","*",0,1],"Amazonbot":["allowed","*",0,1],"meta-externalagent":["allowed","*",0,1],"cohere-ai":["allowed","*",0,1],"Diffbot":["allowed","*",0,1],"ContextFetchBot":["allowed","*",0,1]},"llms":{"status":"missing","http":404,"bytes":62148},"home":{"status":"ok","http":200,"url":"https://humanitix.com/us","html_tokens":66930,"markdown_tokens":202,"pruned_tokens":111}},
{"rank":5003,"domain":"interaction-design.org","origin":"https://interaction-design.org","robots":{"status":"missing","http":404,"url":"https://interaction-design.org/robots.txt","bytes":146,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":563440},"home":{"status":"ok","http":200,"url":"https://ixdf.org/","html_tokens":95310,"markdown_tokens":498,"pruned_tokens":38}},
{"rank":5004,"domain":"eyoucms.com","origin":"https://eyoucms.com","robots":{"status":"ok","http":200,"url":"https://www.eyoucms.com/robots.txt","bytes":266,"groups":2},"bots":{"GPTBot":["partial","*",9,0],"ChatGPT-User":["partial","*",9,0],"OAI-SearchBot":["partial","*",9,0],"ClaudeBot":["partial","*",9,0],"Claude-User":["partial","*",9,0],"Claude-SearchBot":["partial","*",9,0],"anthropic-ai":["partial","*",9,0],"PerplexityBot":["partial","*",9,0],"Perplexity-User":["partial","*",9,0],"Google-Extended":["partial","*",9,0],"Applebot-Extended":["partial","*",9,0],"CCBot":["partial","*",9,0],"Bytespider":["partial","*",9,0],"Amazonbot":["partial","*",9,0],"meta-externalagent":["partial","*",9,0],"cohere-ai":["partial","*",9,0],"Diffbot":["partial","*",9,0],"ContextFetchBot":["partial","*",9,0]},"llms":{"status":"missing","http":404,"bytes":3924},"home":{"status":"ok","http":200,"url":"https://www.eyoucms.com/","html_tokens":21260,"markdown_tokens":331,"pruned_tokens":25}},
{"rank":5005,"domain":"embedgooglemap.net","origin":"https://embedgooglemap.net","robots":{"status":"html","http":200,"url":"https://app-board.net/","bytes":94487,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":5006,"domain":"rivm.nl","origin":"https://rivm.nl","robots":{"status":"ok","http":200,"url":"https://www.rivm.nl/robots.txt","bytes":5287,"groups":9},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["partial","*",72,22],"OAI-SearchBot":["partial","*",72,22],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",72,22],"Claude-SearchBot":["partial","*",72,22],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["partial","*",72,22],"Perplexity-User":["partial","*",72,22],"Google-Extended":["partial","*",72,22],"Applebot-Extended":["partial","*",72,22],"CCBot":["partial","*",72,22],"Bytespider":["blocked","named",1,0],"Amazonbot":["partial","*",72,22],"meta-externalagent":["blocked","named",2,0],"cohere-ai":["partial","*",72,22],"Diffbot":["partial","*",72,22],"ContextFetchBot":["partial","*",72,22]},"llms":{"status":"missing","http":404,"bytes":185},"home":{"status":"ok","http":200,"url":"https://www.rivm.nl/","html_tokens":9032,"markdown_tokens":96,"pruned_tokens":96}},
{"rank":5007,"domain":"econsultancy.com","origin":"https://econsultancy.com","robots":{"status":"ok","http":200,"url":"https://econsultancy.com/robots.txt","bytes":2890,"groups":33},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["partial","*",6,1],"OAI-SearchBot":["partial","*",6,1],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",6,1],"Claude-SearchBot":["partial","*",6,1],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["partial","*",6,1],"Perplexity-User":["partial","*",6,1],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",6,1]},"llms":{"status":"missing","http":404,"bytes":64916},"home":{"status":"ok","http":200,"url":"https://econsultancy.com/","html_tokens":31502,"markdown_tokens":956,"pruned_tokens":266}},
{"rank":5008,"domain":"astro.build","origin":"https://astro.build","robots":{"status":"ok","http":200,"url":"https://astro.build/robots.txt","bytes":120,"groups":1},"bots":{"GPTBot":["allowed","*",0,1],"ChatGPT-User":["allowed","*",0,1],"OAI-SearchBot":["allowed","*",0,1],"ClaudeBot":["allowed","*",0,1],"Claude-User":["allowed","*",0,1],"Claude-SearchBot":["allowed","*",0,1],"anthropic-ai":["allowed","*",0,1],"PerplexityBot":["allowed","*",0,1],"Perplexity-User":["allowed","*",0,1],"Google-Extended":["allowed","*",0,1],"Applebot-Extended":["allowed","*",0,1],"CCBot":["allowed","*",0,1],"Bytespider":["allowed","*",0,1],"Amazonbot":["allowed","*",0,1],"meta-externalagent":["allowed","*",0,1],"cohere-ai":["allowed","*",0,1],"Diffbot":["allowed","*",0,1],"ContextFetchBot":["allowed","*",0,1]},"llms":{"status":"missing","http":404,"bytes":0},"home":{"status":"ok","http":200,"url":"https://astro.build/","html_tokens":78953,"markdown_tokens":515,"pruned_tokens":213}},
{"rank":5009,"domain":"wikivoyage.org","origin":"https://wikivoyage.org","robots":{"status":"ok","http":200,"url":"https://en.wikivoyage.org/robots.txt","bytes":15254,"groups":33},"bots":{"GPTBot":["partial","*",200,4],"ChatGPT-User":["partial","*",200,4],"OAI-SearchBot":["partial","*",200,4],"ClaudeBot":["partial","*",200,4],"Claude-User":["partial","*",200,4],"Claude-SearchBot":["partial","*",200,4],"anthropic-ai":["partial","*",200,4],"PerplexityBot":["partial","*",200,4],"Perplexity-User":["partial","*",200,4],"Google-Extended":["partial","*",200,4],"Applebot-Extended":["partial","*",200,4],"CCBot":["partial","*",200,4],"Bytespider":["partial","*",200,4],"Amazonbot":["partial","*",200,4],"meta-externalagent":["partial","*",200,4],"cohere-ai":["partial","*",200,4],"Diffbot":["partial","*",200,4],"ContextFetchBot":["partial","*",200,4]},"llms":{"status":"missing","http":404,"bytes":1868},"home":{"status":"ok","http":200,"url":"https://www.wikivoyage.org/","html_tokens":15726,"markdown_tokens":9,"pruned_tokens":9}},
{"rank":5010,"domain":"wa.us","origin":"https://wa.us","robots":{"status":"unreachable","http":null,"url":"https://wa.us/robots.txt","bytes":0,"groups":0,"error":"cannot resolve wa.us"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":5011,"domain":"nicepage.com","origin":"https://nicepage.com","robots":{"status":"ok","http":200,"url":"https://nicepage.com/robots.txt","bytes":1174,"groups":4},"bots":{"GPTBot":["partial","*",30,0],"ChatGPT-User":["partial","*",30,0],"OAI-SearchBot":["partial","*",30,0],"ClaudeBot":["partial","*",30,0],"Claude-User":["partial","*",30,0],"Claude-SearchBot":["partial","*",30,0],"anthropic-ai":["partial","*",30,0],"PerplexityBot":["partial","*",30,0],"Perplexity-User":["partial","*",30,0],"Google-Extended":["partial","*",30,0],"Applebot-Extended":["partial","*",30,0],"CCBot":["blocked","named",1,0],"Bytespider":["partial","*",30,0],"Amazonbot":["partial","*",30,0],"meta-externalagent":["partial","*",30,0],"cohere-ai":["partial","*",30,0],"Diffbot":["partial","*",30,0],"ContextFetchBot":["partial","*",30,0]},"llms":{"status":"missing","http":404,"bytes":4962},"home":{"status":"ok","http":200,"url":"https://nicepage.com/","html_tokens":34988,"markdown_tokens":88,"pruned_tokens":88}},
{"rank":5012,"domain":"cureus.com","origin":"https://cureus.com","robots":{"status":"ok","http":200,"url":"https://www.cureus.com/robots.txt","bytes":2185,"groups":3},"bots":{"GPTBot":["partial","named",12,0],"ChatGPT-User":["partial","named",11,1],"OAI-SearchBot":["partial","named",11,1],"ClaudeBot":["partial","named",12,0],"Claude-User":["partial","*",11,0],"Claude-SearchBot":["partial","*",11,0],"anthropic-ai":["partial","named",12,0],"PerplexityBot":["partial","named",11,1],"Perplexity-User":["partial","*",11,0],"Google-Extended":["partial","named",12,0],"Applebot-Extended":["partial","named",12,0],"CCBot":["partial","named",12,0],"Bytespider":["partial","named",12,0],"Amazonbot":["partial","*",11,0],"meta-externalagent":["partial","named",12,0],"cohere-ai":["partial","named",12,0],"Diffbot":["partial","named",12,0],"ContextFetchBot":["partial","*",11,0]},"llms":{"status":"html","http":200,"bytes":162375},"home":{"status":"http_error","http":403,"url":"https://cureus.com/"}},
{"rank":5013,"domain":"telerik.com","origin":"https://telerik.com","robots":{"status":"ok","http":200,"url":"https://www.telerik.com/robots.txt","bytes":3036,"groups":6},"bots":{"GPTBot":["partial","*",14,0],"ChatGPT-User":["partial","*",14,0],"OAI-SearchBot":["partial","*",14,0],"ClaudeBot":["partial","*",14,0],"Claude-User":["partial","*",14,0],"Claude-SearchBot":["partial","*",14,0],"anthropic-ai":["partial","*",14,0],"PerplexityBot":["partial","*",14,0],"Perplexity-User":["partial","*",14,0],"Google-Extended":["partial","*",14,0],"Applebot-Extended":["partial","*",14,0],"CCBot":["partial","*",14,0],"Bytespider":["partial","*",14,0],"Amazonbot":["partial","*",14,0],"meta-externalagent":["partial","*",14,0],"cohere-ai":["partial","*",14,0],"Diffbot":["partial","*",14,0],"ContextFetchBot":["partial","*",14,0]},"llms":{"status":"present","http":200,"bytes":13410},"home":{"status":"ok","http":200,"url":"https://www.telerik.com/","html_tokens":46436,"markdown_tokens":495,"pruned_tokens":295}},
{"rank":5014,"domain":"oppomobile.com","origin":"https://oppomobile.com","robots":{"status":"missing","http":404,"url":"https://oppomobile.com/robots.txt","bytes":2259,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":2259},"home":{"status":"ok","http":200,"url":"https://oppomobile.com/","html_tokens":3476,"markdown_tokens":30,"pruned_tokens":30}},
{"rank":5015,"domain":"humanesociety.org","origin":"https://humanesociety.org","robots":{"status":"refused","http":403,"url":"https://humanesociety.org/robots.txt","bytes":560,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":5016,"domain":"newschool.edu","origin":"https://newschool.edu","robots":{"status":"refused","http":429,"url":"https://www.newschool.edu/robots.txt","bytes":34224,"groups":0,"error":"HTTP 429"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":5017,"domain":"yfrog.com","origin":"http://yfrog.com","robots":{"status":"html","http":200,"url":"https://imageshack.com","bytes":7707,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":5018,"domain":"wakwak.com","origin":"https://www.wakwak.com","robots":{"status":"html","http":200,"url":"https://www.wakwak.com/prl/robots.txt","bytes":8722,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":5019,"domain":"scotusblog.com","origin":"https://scotusblog.com","robots":{"status":"ok","http":200,"url":"https://www.scotusblog.com/robots.txt","bytes":2026,"groups":49},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["blocked","named",1,0],"OAI-SearchBot":["blocked","named",1,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",11,1],"Claude-SearchBot":["partial","*",11,1],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["blocked","named",1,0],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",11,1]},"llms":{"status":"missing","http":404,"bytes":74680},"home":{"status":"ok","http":200,"url":"https://www.scotusblog.com/","html_tokens":111583,"markdown_tokens":19,"pruned_tokens":19}},
{"rank":5020,"domain":"amazon.com.mx","origin":"https://amazon.com.mx","robots":{"status":"ok","http":200,"url":"https://www.amazon.com.mx/robots.txt","bytes":6983,"groups":101},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["blocked","named",1,0],"OAI-SearchBot":["blocked","named",1,0],"ClaudeBot":["blocked","named",2,0],"Claude-User":["blocked","named",1,0],"Claude-SearchBot":["blocked","named",1,0],"anthropic-ai":["partial","*",87,6],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["blocked","named",1,0],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["partial","*",87,6],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["partial","*",87,6],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",87,6]},"llms":{"status":"missing","http":404,"bytes":2471},"home":{"status":"ok","http":200,"url":"https://www.amazon.com.mx/","html_tokens":131859,"markdown_tokens":986,"pruned_tokens":175}},
{"rank":5021,"domain":"aon.com","origin":"https://aon.com","robots":{"status":"html","http":200,"url":"https://www.aon.com/en/","bytes":383338,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":5022,"domain":"curtin.edu.au","origin":"https://curtin.edu.au","robots":{"status":"ok","http":200,"url":"https://www.curtin.edu.au/robots.txt","bytes":1872,"groups":1},"bots":{"GPTBot":["partial","*",14,26],"ChatGPT-User":["partial","*",14,26],"OAI-SearchBot":["partial","*",14,26],"ClaudeBot":["partial","*",14,26],"Claude-User":["partial","*",14,26],"Claude-SearchBot":["partial","*",14,26],"anthropic-ai":["partial","*",14,26],"PerplexityBot":["partial","*",14,26],"Perplexity-User":["partial","*",14,26],"Google-Extended":["partial","*",14,26],"Applebot-Extended":["partial","*",14,26],"CCBot":["partial","*",14,26],"Bytespider":["partial","*",14,26],"Amazonbot":["partial","*",14,26],"meta-externalagent":["partial","*",14,26],"cohere-ai":["partial","*",14,26],"Diffbot":["partial","*",14,26],"ContextFetchBot":["partial","*",14,26]},"llms":{"status":"missing","http":404,"bytes":222172},"home":{"status":"ok","http":200,"url":"https://www.curtin.edu.au/","html_tokens":85385,"markdown_tokens":156,"pruned_tokens":72}},
{"rank":5023,"domain":"sankei.com","origin":"https://sankei.com","robots":{"status":"ok","http":200,"url":"https://www.sankei.com/robots.txt","bytes":2131,"groups":3},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["blocked","named",1,0],"OAI-SearchBot":["blocked","named",1,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",13,0],"Claude-SearchBot":["partial","*",13,0],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["blocked","named",1,0],"Google-Extended":["partial","named",1,1],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["partial","*",13,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",13,0]},"llms":{"status":"missing","http":404,"bytes":52720},"home":{"status":"ok","http":200,"url":"https://www.sankei.com/","html_tokens":108076,"markdown_tokens":38,"pruned_tokens":38}},
{"rank":5024,"domain":"discover.com","origin":"https://discover.com","robots":{"status":"ok","http":200,"url":"https://www.discover.com/robots.txt","bytes":2107,"groups":1},"bots":{"GPTBot":["partial","*",35,0],"ChatGPT-User":["partial","*",35,0],"OAI-SearchBot":["partial","*",35,0],"ClaudeBot":["partial","*",35,0],"Claude-User":["partial","*",35,0],"Claude-SearchBot":["partial","*",35,0],"anthropic-ai":["partial","*",35,0],"PerplexityBot":["partial","*",35,0],"Perplexity-User":["partial","*",35,0],"Google-Extended":["partial","*",35,0],"Applebot-Extended":["partial","*",35,0],"CCBot":["partial","*",35,0],"Bytespider":["partial","*",35,0],"Amazonbot":["partial","*",35,0],"meta-externalagent":["partial","*",35,0],"cohere-ai":["partial","*",35,0],"Diffbot":["partial","*",35,0],"ContextFetchBot":["partial","*",35,0]},"llms":{"status":"missing","http":404,"bytes":3151},"home":{"status":"ok","http":200,"url":"https://www.discover.com/","html_tokens":30791,"markdown_tokens":424,"pruned_tokens":249}},
{"rank":5025,"domain":"niemanlab.org","origin":"https://niemanlab.org","robots":{"status":"ok","http":200,"url":"https://niemanlab.org/robots.txt","bytes":2676,"groups":32},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["allowed","*",0,1],"OAI-SearchBot":["allowed","*",0,1],"ClaudeBot":["blocked","named",1,0],"Claude-User":["allowed","*",0,1],"Claude-SearchBot":["allowed","*",0,1],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["allowed","*",0,1],"Perplexity-User":["allowed","*",0,1],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["allowed","*",0,1]},"llms":{"status":"http_error","http":403,"bytes":4832},"home":{"status":"http_error","http":403,"url":"https://niemanlab.org/"}},
{"rank":5026,"domain":"uni-halle.de","origin":"https://uni-halle.de","robots":{"status":"ok","http":200,"url":"https://www.uni-halle.de/robots.txt","bytes":39,"groups":1},"bots":{"GPTBot":["allowed","*",1,0],"ChatGPT-User":["allowed","*",1,0],"OAI-SearchBot":["allowed","*",1,0],"ClaudeBot":["allowed","*",1,0],"Claude-User":["allowed","*",1,0],"Claude-SearchBot":["allowed","*",1,0],"anthropic-ai":["allowed","*",1,0],"PerplexityBot":["allowed","*",1,0],"Perplexity-User":["allowed","*",1,0],"Google-Extended":["allowed","*",1,0],"Applebot-Extended":["allowed","*",1,0],"CCBot":["allowed","*",1,0],"Bytespider":["allowed","*",1,0],"Amazonbot":["allowed","*",1,0],"meta-externalagent":["allowed","*",1,0],"cohere-ai":["allowed","*",1,0],"Diffbot":["allowed","*",1,0],"ContextFetchBot":["allowed","*",1,0]},"llms":{"status":"missing","http":404,"bytes":16071},"home":{"status":"ok","http":200,"url":"https://www.uni-halle.de/","html_tokens":5900,"markdown_tokens":682,"pruned_tokens":41}},
{"rank":5027,"domain":"theblaze.com","origin":"https://theblaze.com","robots":{"status":"ok","http":200,"url":"https://www.theblaze.com/robots.txt","bytes":318,"groups":1},"bots":{"GPTBot":["partial","*",3,0],"ChatGPT-User":["partial","*",3,0],"OAI-SearchBot":["partial","*",3,0],"ClaudeBot":["partial","*",3,0],"Claude-User":["partial","*",3,0],"Claude-SearchBot":["partial","*",3,0],"anthropic-ai":["partial","*",3,0],"PerplexityBot":["partial","*",3,0],"Perplexity-User":["partial","*",3,0],"Google-Extended":["partial","*",3,0],"Applebot-Extended":["partial","*",3,0],"CCBot":["partial","*",3,0],"Bytespider":["partial","*",3,0],"Amazonbot":["partial","*",3,0],"meta-externalagent":["partial","*",3,0],"cohere-ai":["partial","*",3,0],"Diffbot":["partial","*",3,0],"ContextFetchBot":["partial","*",3,0]},"llms":{"status":"present","http":200,"bytes":5884},"home":{"status":"ok","http":200,"url":"https://www.theblaze.com/","html_tokens":101700,"markdown_tokens":484,"pruned_tokens":472}},
{"rank":5028,"domain":"handbrake.fr","origin":"https://handbrake.fr","robots":{"status":"ok","http":200,"url":"https://handbrake.fr/robots.txt","bytes":29,"groups":1},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"missing","http":404,"bytes":196},"home":{"status":"ok","http":200,"url":"https://handbrake.fr/","html_tokens":765,"markdown_tokens":84,"pruned_tokens":83}},
{"rank":5029,"domain":"pressherald.com","origin":"https://pressherald.com","robots":{"status":"ok","http":200,"url":"https://www.pressherald.com/robots.txt","bytes":120,"groups":1},"bots":{"GPTBot":["partial","*",1,1],"ChatGPT-User":["partial","*",1,1],"OAI-SearchBot":["partial","*",1,1],"ClaudeBot":["partial","*",1,1],"Claude-User":["partial","*",1,1],"Claude-SearchBot":["partial","*",1,1],"anthropic-ai":["partial","*",1,1],"PerplexityBot":["partial","*",1,1],"Perplexity-User":["partial","*",1,1],"Google-Extended":["partial","*",1,1],"Applebot-Extended":["partial","*",1,1],"CCBot":["partial","*",1,1],"Bytespider":["partial","*",1,1],"Amazonbot":["partial","*",1,1],"meta-externalagent":["partial","*",1,1],"cohere-ai":["partial","*",1,1],"Diffbot":["partial","*",1,1],"ContextFetchBot":["partial","*",1,1]},"llms":{"status":"missing","http":404,"bytes":210315},"home":{"status":"ok","http":200,"url":"https://www.pressherald.com/","html_tokens":151848,"markdown_tokens":135,"pruned_tokens":135}},
{"rank":5030,"domain":"insurancejournal.com","origin":"https://insurancejournal.com","robots":{"status":"ok","http":200,"url":"https://www.insurancejournal.com/robots.txt","bytes":1845,"groups":4},"bots":{"GPTBot":["partial","*",43,1],"ChatGPT-User":["partial","*",43,1],"OAI-SearchBot":["partial","*",43,1],"ClaudeBot":["partial","*",43,1],"Claude-User":["partial","*",43,1],"Claude-SearchBot":["partial","*",43,1],"anthropic-ai":["partial","*",43,1],"PerplexityBot":["partial","*",43,1],"Perplexity-User":["partial","*",43,1],"Google-Extended":["partial","*",43,1],"Applebot-Extended":["partial","*",43,1],"CCBot":["partial","*",43,1],"Bytespider":["partial","*",43,1],"Amazonbot":["blocked","named",2,0],"meta-externalagent":["partial","*",43,1],"cohere-ai":["partial","*",43,1],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",43,1]},"llms":{"status":"missing","http":404,"bytes":80636},"home":{"status":"ok","http":200,"url":"https://www.insurancejournal.com/","html_tokens":30118,"markdown_tokens":196,"pruned_tokens":95}},
{"rank":5031,"domain":"angieslist.com","origin":"https://angieslist.com","robots":{"status":"ok","http":200,"url":"https://www.angi.com/robots.txt","bytes":3435,"groups":5},"bots":{"GPTBot":["partial","*",13,1],"ChatGPT-User":["partial","*",13,1],"OAI-SearchBot":["partial","*",13,1],"ClaudeBot":["partial","*",13,1],"Claude-User":["partial","*",13,1],"Claude-SearchBot":["partial","*",13,1],"anthropic-ai":["partial","*",13,1],"PerplexityBot":["partial","*",13,1],"Perplexity-User":["partial","*",13,1],"Google-Extended":["partial","*",13,1],"Applebot-Extended":["partial","*",13,1],"CCBot":["partial","*",13,1],"Bytespider":["partial","*",13,1],"Amazonbot":["partial","*",13,1],"meta-externalagent":["partial","*",13,1],"cohere-ai":["partial","*",13,1],"Diffbot":["partial","*",13,1],"ContextFetchBot":["partial","*",13,1]},"llms":{"status":"missing","http":404,"bytes":40315},"home":{"status":"http_error","http":403,"url":"https://www.angi.com/"}},
{"rank":5032,"domain":"skf.com","origin":"https://skf.com","robots":{"status":"ok","http":200,"url":"https://www.skf.com/robots.txt","bytes":768,"groups":2},"bots":{"GPTBot":["partial","*",18,0],"ChatGPT-User":["partial","*",18,0],"OAI-SearchBot":["partial","*",18,0],"ClaudeBot":["partial","*",18,0],"Claude-User":["partial","*",18,0],"Claude-SearchBot":["partial","*",18,0],"anthropic-ai":["partial","*",18,0],"PerplexityBot":["partial","*",18,0],"Perplexity-User":["partial","*",18,0],"Google-Extended":["partial","*",18,0],"Applebot-Extended":["partial","*",18,0],"CCBot":["partial","*",18,0],"Bytespider":["partial","*",18,0],"Amazonbot":["partial","*",18,0],"meta-externalagent":["partial","*",18,0],"cohere-ai":["partial","*",18,0],"Diffbot":["partial","*",18,0],"ContextFetchBot":["partial","*",18,0]},"llms":{"status":"html","http":200,"bytes":19306},"home":{"status":"ok","http":200,"url":"https://www.skf.com/us","html_tokens":6030,"markdown_tokens":0,"pruned_tokens":0}},
{"rank":5033,"domain":"fatf-gafi.org","origin":"https://fatf-gafi.org","robots":{"status":"refused","http":403,"url":"https://fatf-gafi.org/robots.txt","bytes":0,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":5034,"domain":"xataka.com","origin":"https://xataka.com","robots":{"status":"ok","http":200,"url":"https://www.xataka.com/robots.txt","bytes":4231,"groups":43},"bots":{"GPTBot":["partial","*",51,1],"ChatGPT-User":["partial","*",51,1],"OAI-SearchBot":["partial","*",51,1],"ClaudeBot":["partial","*",51,1],"Claude-User":["partial","*",51,1],"Claude-SearchBot":["partial","*",51,1],"anthropic-ai":["partial","*",51,1],"PerplexityBot":["partial","*",51,1],"Perplexity-User":["partial","*",51,1],"Google-Extended":["partial","*",51,1],"Applebot-Extended":["partial","*",51,1],"CCBot":["partial","*",51,1],"Bytespider":["partial","*",51,1],"Amazonbot":["partial","*",51,1],"meta-externalagent":["partial","*",51,1],"cohere-ai":["partial","*",51,1],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",51,1]},"llms":{"status":"missing","http":404,"bytes":71813},"home":{"status":"ok","http":200,"url":"https://www.xataka.com/","html_tokens":52097,"markdown_tokens":19,"pruned_tokens":19}},
{"rank":5035,"domain":"thenewstack.io","origin":"https://thenewstack.io","robots":{"status":"ok","http":200,"url":"https://thenewstack.io/robots.txt","bytes":1045,"groups":16},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["blocked","named",1,0],"OAI-SearchBot":["partial","*",15,1],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",15,1],"Claude-SearchBot":["partial","*",15,1],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["partial","*",15,1],"Google-Extended":["partial","*",15,1],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["partial","*",15,1],"ContextFetchBot":["partial","*",15,1]},"llms":{"status":"missing","http":404,"bytes":1161051},"home":{"status":"ok","http":200,"url":"https://thenewstack.io/","html_tokens":323208,"markdown_tokens":115,"pruned_tokens":115}},
{"rank":5036,"domain":"dazn.com","origin":"https://dazn.com","robots":{"status":"ok","http":200,"url":"https://www.dazn.com/robots.txt","bytes":124,"groups":1},"bots":{"GPTBot":["partial","*",4,0],"ChatGPT-User":["partial","*",4,0],"OAI-SearchBot":["partial","*",4,0],"ClaudeBot":["partial","*",4,0],"Claude-User":["partial","*",4,0],"Claude-SearchBot":["partial","*",4,0],"anthropic-ai":["partial","*",4,0],"PerplexityBot":["partial","*",4,0],"Perplexity-User":["partial","*",4,0],"Google-Extended":["partial","*",4,0],"Applebot-Extended":["partial","*",4,0],"CCBot":["partial","*",4,0],"Bytespider":["partial","*",4,0],"Amazonbot":["partial","*",4,0],"meta-externalagent":["partial","*",4,0],"cohere-ai":["partial","*",4,0],"Diffbot":["partial","*",4,0],"ContextFetchBot":["partial","*",4,0]},"llms":{"status":"html","http":200,"bytes":102423},"home":{"status":"ok","http":200,"url":"https://www.dazn.com/en-US/welcome","html_tokens":284837,"markdown_tokens":101,"pruned_tokens":87}},
{"rank":5037,"domain":"eplus.jp","origin":"https://eplus.jp","robots":{"status":"ok","http":200,"url":"https://eplus.jp/robots.txt","bytes":551,"groups":1},"bots":{"GPTBot":["partial","*",6,0],"ChatGPT-User":["partial","*",6,0],"OAI-SearchBot":["partial","*",6,0],"ClaudeBot":["partial","*",6,0],"Claude-User":["partial","*",6,0],"Claude-SearchBot":["partial","*",6,0],"anthropic-ai":["partial","*",6,0],"PerplexityBot":["partial","*",6,0],"Perplexity-User":["partial","*",6,0],"Google-Extended":["partial","*",6,0],"Applebot-Extended":["partial","*",6,0],"CCBot":["partial","*",6,0],"Bytespider":["partial","*",6,0],"Amazonbot":["partial","*",6,0],"meta-externalagent":["partial","*",6,0],"cohere-ai":["partial","*",6,0],"Diffbot":["partial","*",6,0],"ContextFetchBot":["partial","*",6,0]},"llms":{"status":"missing","http":404,"bytes":2575},"home":{"status":"ok","http":200,"url":"https://eplus.jp/","html_tokens":24329,"markdown_tokens":32,"pruned_tokens":32}},
{"rank":5038,"domain":"justwatch.com","origin":"https://justwatch.com","robots":{"status":"ok","http":200,"url":"https://www.justwatch.com/robots.txt","bytes":23,"groups":1},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"missing","http":404,"bytes":19},"home":{"status":"too_large","error":"response larger than 5 MB"}},
{"rank":5039,"domain":"mdanderson.org","origin":"https://mdanderson.org","robots":{"status":"ok","http":200,"url":"https://www.mdanderson.org/robots.txt","bytes":3587,"groups":3},"bots":{"GPTBot":["partial","*",23,0],"ChatGPT-User":["partial","*",23,0],"OAI-SearchBot":["partial","*",23,0],"ClaudeBot":["partial","*",23,0],"Claude-User":["partial","*",23,0],"Claude-SearchBot":["partial","*",23,0],"anthropic-ai":["partial","*",23,0],"PerplexityBot":["partial","*",23,0],"Perplexity-User":["partial","*",23,0],"Google-Extended":["partial","*",23,0],"Applebot-Extended":["partial","*",23,0],"CCBot":["partial","*",23,0],"Bytespider":["partial","*",23,0],"Amazonbot":["partial","*",23,0],"meta-externalagent":["partial","*",23,0],"cohere-ai":["partial","*",23,0],"Diffbot":["partial","*",23,0],"ContextFetchBot":["partial","*",23,0]},"llms":{"status":"missing","http":404,"bytes":149653},"home":{"status":"ok","http":200,"url":"https://www.mdanderson.org/","html_tokens":41836,"markdown_tokens":226,"pruned_tokens":104}},
{"rank":5040,"domain":"healthgrades.com","origin":"https://healthgrades.com","robots":{"status":"ok","http":200,"url":"https://healthgrades.com/robots.txt","bytes":3097,"groups":21},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["partial","*",56,1],"OAI-SearchBot":["partial","named",56,1],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",56,1],"Claude-SearchBot":["partial","*",56,1],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["partial","*",56,1],"Perplexity-User":["partial","*",56,1],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["partial","*",56,1],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",56,1]},"llms":{"status":"present","http":200,"bytes":3085},"home":{"status":"ok","http":200,"url":"https://www.healthgrades.com/","html_tokens":391797,"markdown_tokens":293,"pruned_tokens":129}},
{"rank":5041,"domain":"kyivpost.com","origin":"https://kyivpost.com","robots":{"status":"ok","http":200,"url":"https://www.kyivpost.com/robots.txt","bytes":398,"groups":1},"bots":{"GPTBot":["partial","*",14,1],"ChatGPT-User":["partial","*",14,1],"OAI-SearchBot":["partial","*",14,1],"ClaudeBot":["partial","*",14,1],"Claude-User":["partial","*",14,1],"Claude-SearchBot":["partial","*",14,1],"anthropic-ai":["partial","*",14,1],"PerplexityBot":["partial","*",14,1],"Perplexity-User":["partial","*",14,1],"Google-Extended":["partial","*",14,1],"Applebot-Extended":["partial","*",14,1],"CCBot":["partial","*",14,1],"Bytespider":["partial","*",14,1],"Amazonbot":["partial","*",14,1],"meta-externalagent":["partial","*",14,1],"cohere-ai":["partial","*",14,1],"Diffbot":["partial","*",14,1],"ContextFetchBot":["partial","*",14,1]},"llms":{"status":"missing","http":404,"bytes":49841},"home":{"status":"ok","http":200,"url":"https://www.kyivpost.com/","html_tokens":45906,"markdown_tokens":153,"pruned_tokens":87}},
{"rank":5042,"domain":"answerthepublic.com","origin":"https://answerthepublic.com","robots":{"status":"ok","http":200,"url":"https://answerthepublic.com/robots.txt","bytes":477,"groups":2},"bots":{"GPTBot":["partial","named",4,1],"ChatGPT-User":["partial","named",4,1],"OAI-SearchBot":["partial","named",4,1],"ClaudeBot":["partial","named",4,1],"Claude-User":["partial","*",4,1],"Claude-SearchBot":["partial","*",4,1],"anthropic-ai":["partial","named",4,1],"PerplexityBot":["partial","named",4,1],"Perplexity-User":["partial","*",4,1],"Google-Extended":["partial","named",4,1],"Applebot-Extended":["partial","named",4,1],"CCBot":["partial","named",4,1],"Bytespider":["partial","*",4,1],"Amazonbot":["partial","*",4,1],"meta-externalagent":["partial","*",4,1],"cohere-ai":["partial","*",4,1],"Diffbot":["partial","*",4,1],"ContextFetchBot":["partial","*",4,1]},"llms":{"status":"present","http":200,"bytes":6765},"home":{"status":"not_html","error":"unsupported content type text/markdown"}},
{"rank":5043,"domain":"movember.com","origin":"https://movember.com","robots":{"status":"ok","http":200,"url":"https://movember.com/robots.txt","bytes":2081,"groups":1},"bots":{"GPTBot":["partial","*",3,1],"ChatGPT-User":["partial","*",3,1],"OAI-SearchBot":["partial","*",3,1],"ClaudeBot":["partial","*",3,1],"Claude-User":["partial","*",3,1],"Claude-SearchBot":["partial","*",3,1],"anthropic-ai":["partial","*",3,1],"PerplexityBot":["partial","*",3,1],"Perplexity-User":["partial","*",3,1],"Google-Extended":["partial","*",3,1],"Applebot-Extended":["partial","*",3,1],"CCBot":["partial","*",3,1],"Bytespider":["partial","*",3,1],"Amazonbot":["partial","*",3,1],"meta-externalagent":["partial","*",3,1],"cohere-ai":["partial","*",3,1],"Diffbot":["partial","*",3,1],"ContextFetchBot":["partial","*",3,1]},"llms":{"status":"missing","http":404,"bytes":9500},"home":{"status":"ok","http":200,"url":"https://movember.com/","html_tokens":57052,"markdown_tokens":105,"pruned_tokens":32}},
{"rank":5044,"domain":"city.ac.uk","origin":"https://city.ac.uk","robots":{"status":"refused","http":403,"url":"https://www.citystgeorges.ac.uk/robots.txt","bytes":5561,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":5045,"domain":"xml-sitemaps.com","origin":"https://xml-sitemaps.com","robots":{"status":"ok","http":200,"url":"https://www.xml-sitemaps.com/robots.txt","bytes":707,"groups":1},"bots":{"GPTBot":["partial","*",30,0],"ChatGPT-User":["partial","*",30,0],"OAI-SearchBot":["partial","*",30,0],"ClaudeBot":["partial","*",30,0],"Claude-User":["partial","*",30,0],"Claude-SearchBot":["partial","*",30,0],"anthropic-ai":["partial","*",30,0],"PerplexityBot":["partial","*",30,0],"Perplexity-User":["partial","*",30,0],"Google-Extended":["partial","*",30,0],"Applebot-Extended":["partial","*",30,0],"CCBot":["partial","*",30,0],"Bytespider":["partial","*",30,0],"Amazonbot":["partial","*",30,0],"meta-externalagent":["partial","*",30,0],"cohere-ai":["partial","*",30,0],"Diffbot":["partial","*",30,0],"ContextFetchBot":["partial","*",30,0]},"llms":{"status":"missing","http":404,"bytes":94},"home":{"status":"ok","http":200,"url":"https://www.xml-sitemaps.com/","html_tokens":5962,"markdown_tokens":162,"pruned_tokens":100}},
{"rank":5046,"domain":"callofduty.com","origin":"https://callofduty.com","robots":{"status":"timeout","http":null,"url":"https://callofduty.com/robots.txt","bytes":0,"groups":0,"error":"timed out after 10 s"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":5047,"domain":"pinterest.com.au","origin":"https://pinterest.com.au","robots":{"status":"ok","http":200,"url":"https://au.pinterest.com/robots.txt","bytes":29335,"groups":8},"bots":{"GPTBot":["blocked","*",1,0],"ChatGPT-User":["blocked","*",1,0],"OAI-SearchBot":["blocked","*",1,0],"ClaudeBot":["blocked","*",1,0],"Claude-User":["blocked","*",1,0],"Claude-SearchBot":["blocked","*",1,0],"anthropic-ai":["blocked","*",1,0],"PerplexityBot":["blocked","*",1,0],"Perplexity-User":["blocked","*",1,0],"Google-Extended":["blocked","*",1,0],"Applebot-Extended":["blocked","*",1,0],"CCBot":["blocked","*",1,0],"Bytespider":["blocked","*",1,0],"Amazonbot":["blocked","*",1,0],"meta-externalagent":["blocked","*",1,0],"cohere-ai":["blocked","*",1,0],"Diffbot":["blocked","*",1,0],"ContextFetchBot":["blocked","*",1,0]},"llms":{"status":"disallowed"},"home":{"status":"disallowed"}},
{"rank":5048,"domain":"rockpapershotgun.com","origin":"https://rockpapershotgun.com","robots":{"status":"ok","http":200,"url":"https://www.rockpapershotgun.com/robots.txt","bytes":5663,"groups":2},"bots":{"GPTBot":["partial","named",1,1],"ChatGPT-User":["partial","named",1,1],"OAI-SearchBot":["partial","named",1,1],"ClaudeBot":["partial","named",1,1],"Claude-User":["partial","named",1,1],"Claude-SearchBot":["partial","named",1,1],"anthropic-ai":["partial","named",1,1],"PerplexityBot":["partial","named",1,1],"Perplexity-User":["partial","named",1,1],"Google-Extended":["partial","*",9,0],"Applebot-Extended":["partial","named",1,1],"CCBot":["partial","named",1,1],"Bytespider":["partial","named",1,1],"Amazonbot":["partial","named",1,1],"meta-externalagent":["partial","named",1,1],"cohere-ai":["partial","named",1,1],"Diffbot":["partial","named",1,1],"ContextFetchBot":["partial","*",9,0]},"llms":{"status":"missing","http":404,"bytes":2300},"home":{"status":"ok","http":200,"url":"https://www.rockpapershotgun.com/","html_tokens":77476,"markdown_tokens":199,"pruned_tokens":162}},
{"rank":5049,"domain":"he.net","origin":"https://he.net","robots":{"status":"ok","http":200,"url":"https://he.net/robots.txt","bytes":60,"groups":1},"bots":{"GPTBot":["allowed","*",0,1],"ChatGPT-User":["allowed","*",0,1],"OAI-SearchBot":["allowed","*",0,1],"ClaudeBot":["allowed","*",0,1],"Claude-User":["allowed","*",0,1],"Claude-SearchBot":["allowed","*",0,1],"anthropic-ai":["allowed","*",0,1],"PerplexityBot":["allowed","*",0,1],"Perplexity-User":["allowed","*",0,1],"Google-Extended":["allowed","*",0,1],"Applebot-Extended":["allowed","*",0,1],"CCBot":["allowed","*",0,1],"Bytespider":["allowed","*",0,1],"Amazonbot":["allowed","*",0,1],"meta-externalagent":["allowed","*",0,1],"cohere-ai":["allowed","*",0,1],"Diffbot":["allowed","*",0,1],"ContextFetchBot":["allowed","*",0,1]},"llms":{"status":"missing","http":404,"bytes":18878},"home":{"status":"ok","http":200,"url":"https://he.net/","html_tokens":4941,"markdown_tokens":57,"pruned_tokens":57}},
{"rank":5050,"domain":"comcast.com","origin":"https://www.comcast.com","robots":{"status":"ok","http":200,"url":"https://www.comcast.com/robots.txt","bytes":99,"groups":1},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"present","http":200,"bytes":40769},"home":{"status":"http_error","http":403,"url":"https://www.xfinity.com/"}},
{"rank":5051,"domain":"otago.ac.nz","origin":"https://otago.ac.nz","robots":{"status":"refused","http":403,"url":"https://otago.ac.nz/robots.txt","bytes":5762,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":5052,"domain":"simplesite.com","origin":"https://simplesite.com","robots":{"status":"unreachable","http":null,"url":"https://simplesite.com/robots.txt","bytes":0,"groups":0,"error":"cannot resolve robots-txt.simplesite.com"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":5053,"domain":"iupui.edu","origin":"https://iupui.edu","robots":{"status":"html","http":200,"url":"https://indianapolis.iu.edu","bytes":314106,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":5054,"domain":"sixflags.com","origin":"https://sixflags.com","robots":{"status":"ok","http":200,"url":"https://www.sixflags.com/robots.txt","bytes":1681,"groups":1},"bots":{"GPTBot":["partial","*",50,1],"ChatGPT-User":["partial","*",50,1],"OAI-SearchBot":["partial","*",50,1],"ClaudeBot":["partial","*",50,1],"Claude-User":["partial","*",50,1],"Claude-SearchBot":["partial","*",50,1],"anthropic-ai":["partial","*",50,1],"PerplexityBot":["partial","*",50,1],"Perplexity-User":["partial","*",50,1],"Google-Extended":["partial","*",50,1],"Applebot-Extended":["partial","*",50,1],"CCBot":["partial","*",50,1],"Bytespider":["partial","*",50,1],"Amazonbot":["partial","*",50,1],"meta-externalagent":["partial","*",50,1],"cohere-ai":["partial","*",50,1],"Diffbot":["partial","*",50,1],"ContextFetchBot":["partial","*",50,1]},"llms":{"status":"missing","http":404,"bytes":89767},"home":{"status":"ok","http":200,"url":"https://www.sixflags.com/","html_tokens":72241,"markdown_tokens":289,"pruned_tokens":171}},
{"rank":5055,"domain":"shangri-la.com","origin":"https://shangri-la.com","robots":{"status":"html","http":200,"url":"https://www.shangri-la.com/en/","bytes":121580,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":5056,"domain":"startpage.com","origin":"https://startpage.com","robots":{"status":"ok","http":200,"url":"https://www.startpage.com/robots.txt","bytes":188,"groups":1},"bots":{"GPTBot":["partial","*",4,2],"ChatGPT-User":["partial","*",4,2],"OAI-SearchBot":["partial","*",4,2],"ClaudeBot":["partial","*",4,2],"Claude-User":["partial","*",4,2],"Claude-SearchBot":["partial","*",4,2],"anthropic-ai":["partial","*",4,2],"PerplexityBot":["partial","*",4,2],"Perplexity-User":["partial","*",4,2],"Google-Extended":["partial","*",4,2],"Applebot-Extended":["partial","*",4,2],"CCBot":["partial","*",4,2],"Bytespider":["partial","*",4,2],"Amazonbot":["partial","*",4,2],"meta-externalagent":["partial","*",4,2],"cohere-ai":["partial","*",4,2],"Diffbot":["partial","*",4,2],"ContextFetchBot":["partial","*",4,2]},"llms":{"status":"html","http":200,"bytes":70297},"home":{"status":"disallowed","error":"robots.txt of www.startpage.com disallows /sp/captcha-block?bc=US&bi=Pavlov+Media&be=3409671ad6913f947ff804bf16bce2cc&bds_t=1792374588T48c9a12"}},
{"rank":5057,"domain":"timepad.ru","origin":"https://timepad.ru","robots":{"status":"ok","http":200,"url":"https://timepad.ru/robots.txt","bytes":355,"groups":1},"bots":{"GPTBot":["partial","*",14,0],"ChatGPT-User":["partial","*",14,0],"OAI-SearchBot":["partial","*",14,0],"ClaudeBot":["partial","*",14,0],"Claude-User":["partial","*",14,0],"Claude-SearchBot":["partial","*",14,0],"anthropic-ai":["partial","*",14,0],"PerplexityBot":["partial","*",14,0],"Perplexity-User":["partial","*",14,0],"Google-Extended":["partial","*",14,0],"Applebot-Extended":["partial","*",14,0],"CCBot":["partial","*",14,0],"Bytespider":["partial","*",14,0],"Amazonbot":["partial","*",14,0],"meta-externalagent":["partial","*",14,0],"cohere-ai":["partial","*",14,0],"Diffbot":["partial","*",14,0],"ContextFetchBot":["partial","*",14,0]},"llms":{"status":"missing","http":404,"bytes":2885},"home":{"status":"ok","http":200,"url":"https://afisha.timepad.ru/","html_tokens":429,"markdown_tokens":18,"pruned_tokens":18}},
{"rank":5058,"domain":"fau.edu","origin":"https://fau.edu","robots":{"status":"ok","http":200,"url":"https://www.fau.edu/robots.txt","bytes":666,"groups":1},"bots":{"GPTBot":["partial","*",22,1],"ChatGPT-User":["partial","*",22,1],"OAI-SearchBot":["partial","*",22,1],"ClaudeBot":["partial","*",22,1],"Claude-User":["partial","*",22,1],"Claude-SearchBot":["partial","*",22,1],"anthropic-ai":["partial","*",22,1],"PerplexityBot":["partial","*",22,1],"Perplexity-User":["partial","*",22,1],"Google-Extended":["partial","*",22,1],"Applebot-Extended":["partial","*",22,1],"CCBot":["partial","*",22,1],"Bytespider":["partial","*",22,1],"Amazonbot":["partial","*",22,1],"meta-externalagent":["partial","*",22,1],"cohere-ai":["partial","*",22,1],"Diffbot":["partial","*",22,1],"ContextFetchBot":["partial","*",22,1]},"llms":{"status":"missing","http":404,"bytes":16705},"home":{"status":"ok","http":200,"url":"https://www.fau.edu/","html_tokens":7600,"markdown_tokens":315,"pruned_tokens":115}},
{"rank":5059,"domain":"questionpro.com","origin":"https://questionpro.com","robots":{"status":"ok","http":200,"url":"https://www.questionpro.com/robots.txt","bytes":330,"groups":1},"bots":{"GPTBot":["partial","*",6,3],"ChatGPT-User":["partial","*",6,3],"OAI-SearchBot":["partial","*",6,3],"ClaudeBot":["partial","*",6,3],"Claude-User":["partial","*",6,3],"Claude-SearchBot":["partial","*",6,3],"anthropic-ai":["partial","*",6,3],"PerplexityBot":["partial","*",6,3],"Perplexity-User":["partial","*",6,3],"Google-Extended":["partial","*",6,3],"Applebot-Extended":["partial","*",6,3],"CCBot":["partial","*",6,3],"Bytespider":["partial","*",6,3],"Amazonbot":["partial","*",6,3],"meta-externalagent":["partial","*",6,3],"cohere-ai":["partial","*",6,3],"Diffbot":["partial","*",6,3],"ContextFetchBot":["partial","*",6,3]},"llms":{"status":"disallowed"},"home":{"status":"ok","http":200,"url":"https://www.questionpro.com/us/","html_tokens":36250,"markdown_tokens":882,"pruned_tokens":300}},
{"rank":5060,"domain":"sched.com","origin":"https://sched.com","robots":{"status":"ok","http":200,"url":"https://sched.com/robots.txt","bytes":58,"groups":1},"bots":{"GPTBot":["partial","*",2,0],"ChatGPT-User":["partial","*",2,0],"OAI-SearchBot":["partial","*",2,0],"ClaudeBot":["partial","*",2,0],"Claude-User":["partial","*",2,0],"Claude-SearchBot":["partial","*",2,0],"anthropic-ai":["partial","*",2,0],"PerplexityBot":["partial","*",2,0],"Perplexity-User":["partial","*",2,0],"Google-Extended":["partial","*",2,0],"Applebot-Extended":["partial","*",2,0],"CCBot":["partial","*",2,0],"Bytespider":["partial","*",2,0],"Amazonbot":["partial","*",2,0],"meta-externalagent":["partial","*",2,0],"cohere-ai":["partial","*",2,0],"Diffbot":["partial","*",2,0],"ContextFetchBot":["partial","*",2,0]},"llms":{"status":"missing","http":404,"bytes":65447},"home":{"status":"ok","http":200,"url":"https://sched.com/","html_tokens":30261,"markdown_tokens":1124,"pruned_tokens":370}},
{"rank":5061,"domain":"allaboutbirds.org","origin":"https://allaboutbirds.org","robots":{"status":"refused","http":403,"url":"https://allaboutbirds.org/robots.txt","bytes":4550,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":5062,"domain":"prlog.org","origin":"https://prlog.org","robots":{"status":"ok","http":200,"url":"https://prlog.org/robots.txt","bytes":289,"groups":1},"bots":{"GPTBot":["partial","*",9,0],"ChatGPT-User":["partial","*",9,0],"OAI-SearchBot":["partial","*",9,0],"ClaudeBot":["partial","*",9,0],"Claude-User":["partial","*",9,0],"Claude-SearchBot":["partial","*",9,0],"anthropic-ai":["partial","*",9,0],"PerplexityBot":["partial","*",9,0],"Perplexity-User":["partial","*",9,0],"Google-Extended":["partial","*",9,0],"Applebot-Extended":["partial","*",9,0],"CCBot":["partial","*",9,0],"Bytespider":["partial","*",9,0],"Amazonbot":["partial","*",9,0],"meta-externalagent":["partial","*",9,0],"cohere-ai":["partial","*",9,0],"Diffbot":["partial","*",9,0],"ContextFetchBot":["partial","*",9,0]},"llms":{"status":"missing","http":404,"bytes":10646},"home":{"status":"ok","http":200,"url":"https://www.prlog.org/","html_tokens":9597,"markdown_tokens":2817,"pruned_tokens":19}},
{"rank":5063,"domain":"bobvila.com","origin":"https://bobvila.com","robots":{"status":"ok","http":200,"url":"https://www.bobvila.com/robots.txt","bytes":387,"groups":2},"bots":{"GPTBot":["partial","*",7,1],"ChatGPT-User":["partial","*",7,1],"OAI-SearchBot":["partial","*",7,1],"ClaudeBot":["partial","*",7,1],"Claude-User":["partial","*",7,1],"Claude-SearchBot":["partial","*",7,1],"anthropic-ai":["partial","*",7,1],"PerplexityBot":["partial","*",7,1],"Perplexity-User":["partial","*",7,1],"Google-Extended":["partial","*",7,1],"Applebot-Extended":["partial","*",7,1],"CCBot":["partial","*",7,1],"Bytespider":["partial","*",7,1],"Amazonbot":["partial","*",7,1],"meta-externalagent":["partial","*",7,1],"cohere-ai":["partial","*",7,1],"Diffbot":["partial","*",7,1],"ContextFetchBot":["partial","*",7,1]},"llms":{"status":"missing","http":404,"bytes":114749},"home":{"status":"ok","http":200,"url":"https://www.bobvila.com/","html_tokens":58942,"markdown_tokens":511,"pruned_tokens":320}},
{"rank":5064,"domain":"hurriyetdailynews.com","origin":"https://hurriyetdailynews.com","robots":{"status":"ok","http":200,"url":"https://www.hurriyetdailynews.com/robots.txt","bytes":162,"groups":1},"bots":{"GPTBot":["partial","*",1,0],"ChatGPT-User":["partial","*",1,0],"OAI-SearchBot":["partial","*",1,0],"ClaudeBot":["partial","*",1,0],"Claude-User":["partial","*",1,0],"Claude-SearchBot":["partial","*",1,0],"anthropic-ai":["partial","*",1,0],"PerplexityBot":["partial","*",1,0],"Perplexity-User":["partial","*",1,0],"Google-Extended":["partial","*",1,0],"Applebot-Extended":["partial","*",1,0],"CCBot":["partial","*",1,0],"Bytespider":["partial","*",1,0],"Amazonbot":["partial","*",1,0],"meta-externalagent":["partial","*",1,0],"cohere-ai":["partial","*",1,0],"Diffbot":["partial","*",1,0],"ContextFetchBot":["partial","*",1,0]},"llms":{"status":"missing","http":404,"bytes":8451},"home":{"status":"ok","http":200,"url":"https://www.hurriyetdailynews.com/","html_tokens":22248,"markdown_tokens":66,"pruned_tokens":66}},
{"rank":5065,"domain":"photopea.com","origin":"https://photopea.com","robots":{"status":"ok","http":200,"url":"https://www.photopea.com/robots.txt","bytes":1248,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":1389},"home":{"status":"ok","http":200,"url":"https://www.photopea.com/","html_tokens":4202,"markdown_tokens":544,"pruned_tokens":289}},
{"rank":5066,"domain":"kitco.com","origin":"https://kitco.com","robots":{"status":"ok","http":200,"url":"http://www.kitco.com/robots.txt","bytes":406,"groups":1},"bots":{"GPTBot":["allowed","*",0,1],"ChatGPT-User":["allowed","*",0,1],"OAI-SearchBot":["allowed","*",0,1],"ClaudeBot":["allowed","*",0,1],"Claude-User":["allowed","*",0,1],"Claude-SearchBot":["allowed","*",0,1],"anthropic-ai":["allowed","*",0,1],"PerplexityBot":["allowed","*",0,1],"Perplexity-User":["allowed","*",0,1],"Google-Extended":["allowed","*",0,1],"Applebot-Extended":["allowed","*",0,1],"CCBot":["allowed","*",0,1],"Bytespider":["allowed","*",0,1],"Amazonbot":["allowed","*",0,1],"meta-externalagent":["allowed","*",0,1],"cohere-ai":["allowed","*",0,1],"Diffbot":["allowed","*",0,1],"ContextFetchBot":["allowed","*",0,1]},"llms":{"status":"missing","http":404,"bytes":48927},"home":{"status":"ok","http":200,"url":"http://www.kitco.com/","html_tokens":67115,"markdown_tokens":328,"pruned_tokens":163}},
{"rank":5067,"domain":"researchsquare.com","origin":"https://researchsquare.com","robots":{"status":"ok","http":200,"url":"https://www.researchsquare.com/robots.txt","bytes":929,"groups":23},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["blocked","named",1,0],"OAI-SearchBot":["partial","*",1,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",1,0],"Claude-SearchBot":["partial","*",1,0],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["partial","*",1,0],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",1,0]},"llms":{"status":"missing","http":404,"bytes":5389},"home":{"status":"ok","http":200,"url":"https://www.researchsquare.com/","html_tokens":5588,"markdown_tokens":219,"pruned_tokens":115}},
{"rank":5068,"domain":"travis-ci.com","origin":"https://travis-ci.com","robots":{"status":"ok","http":200,"url":"https://www.travis-ci.com/robots.txt","bytes":0,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":51512},"home":{"status":"ok","http":200,"url":"https://www.travis-ci.com/","html_tokens":41949,"markdown_tokens":3939,"pruned_tokens":143}},
{"rank":5069,"domain":"juraforum.de","origin":"https://juraforum.de","robots":{"status":"ok","http":200,"url":"https://www.juraforum.de/robots.txt","bytes":18240,"groups":30},"bots":{"GPTBot":["partial","*",510,1],"ChatGPT-User":["blocked","named",1,0],"OAI-SearchBot":["partial","*",510,1],"ClaudeBot":["partial","*",510,1],"Claude-User":["partial","*",510,1],"Claude-SearchBot":["partial","*",510,1],"anthropic-ai":["partial","*",510,1],"PerplexityBot":["partial","*",510,1],"Perplexity-User":["partial","*",510,1],"Google-Extended":["partial","*",510,1],"Applebot-Extended":["partial","*",510,1],"CCBot":["blocked","named",1,0],"Bytespider":["partial","*",510,1],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["partial","*",510,1],"cohere-ai":["partial","*",510,1],"Diffbot":["partial","*",510,1],"ContextFetchBot":["partial","*",510,1]},"llms":{"status":"missing","http":404,"bytes":303},"home":{"status":"ok","http":200,"url":"https://www.juraforum.de/","html_tokens":29832,"markdown_tokens":3093,"pruned_tokens":32}},
{"rank":5070,"domain":"centrepompidou.fr","origin":"https://centrepompidou.fr","robots":{"status":"ok","http":200,"url":"https://www.centrepompidou.fr/robots.txt","bytes":200,"groups":2},"bots":{"GPTBot":["partial","*",3,3],"ChatGPT-User":["partial","*",3,3],"OAI-SearchBot":["partial","*",3,3],"ClaudeBot":["partial","*",3,3],"Claude-User":["partial","*",3,3],"Claude-SearchBot":["partial","*",3,3],"anthropic-ai":["partial","*",3,3],"PerplexityBot":["partial","*",3,3],"Perplexity-User":["partial","*",3,3],"Google-Extended":["partial","*",3,3],"Applebot-Extended":["partial","*",3,3],"CCBot":["partial","*",3,3],"Bytespider":["partial","*",3,3],"Amazonbot":["partial","*",3,3],"meta-externalagent":["partial","*",3,3],"cohere-ai":["partial","*",3,3],"Diffbot":["partial","*",3,3],"ContextFetchBot":["partial","*",3,3]},"llms":{"status":"missing","http":404,"bytes":51274},"home":{"status":"ok","http":200,"url":"https://www.centrepompidou.fr/en/","html_tokens":40312,"markdown_tokens":864,"pruned_tokens":104}},
{"rank":5071,"domain":"food.com","origin":"https://food.com","robots":{"status":"html","http":200,"url":"https://www.food.com/","bytes":193746,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":5072,"domain":"yuque.com","origin":"https://yuque.com","robots":{"status":"ok","http":200,"url":"https://yuque.com/robots.txt","bytes":184,"groups":1},"bots":{"GPTBot":["partial","*",1,1],"ChatGPT-User":["partial","*",1,1],"OAI-SearchBot":["partial","*",1,1],"ClaudeBot":["partial","*",1,1],"Claude-User":["partial","*",1,1],"Claude-SearchBot":["partial","*",1,1],"anthropic-ai":["partial","*",1,1],"PerplexityBot":["partial","*",1,1],"Perplexity-User":["partial","*",1,1],"Google-Extended":["partial","*",1,1],"Applebot-Extended":["partial","*",1,1],"CCBot":["partial","*",1,1],"Bytespider":["partial","*",1,1],"Amazonbot":["partial","*",1,1],"meta-externalagent":["partial","*",1,1],"cohere-ai":["partial","*",1,1],"Diffbot":["partial","*",1,1],"ContextFetchBot":["partial","*",1,1]},"llms":{"status":"missing","http":404,"bytes":28},"home":{"status":"ok","http":200,"url":"https://www.yuque.com/","html_tokens":8441,"markdown_tokens":0,"pruned_tokens":0}},
{"rank":5073,"domain":"screencast.com","origin":"https://screencast.com","robots":{"status":"html","http":200,"url":"https://app.screencast.com/","bytes":25055,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":5074,"domain":"overstock.com","origin":"https://overstock.com","robots":{"status":"ok","http":200,"url":"https://www.overstock.com/robots.txt","bytes":1282,"groups":8},"bots":{"GPTBot":["partial","named",1,0],"ChatGPT-User":["partial","*",25,0],"OAI-SearchBot":["partial","*",25,0],"ClaudeBot":["partial","*",25,0],"Claude-User":["partial","*",25,0],"Claude-SearchBot":["partial","*",25,0],"anthropic-ai":["partial","*",25,0],"PerplexityBot":["partial","*",25,0],"Perplexity-User":["partial","*",25,0],"Google-Extended":["partial","*",25,0],"Applebot-Extended":["partial","*",25,0],"CCBot":["partial","*",25,0],"Bytespider":["partial","*",25,0],"Amazonbot":["partial","named",6,0],"meta-externalagent":["partial","named",1,0],"cohere-ai":["partial","*",25,0],"Diffbot":["partial","*",25,0],"ContextFetchBot":["partial","*",25,0]},"llms":{"status":"missing","http":404,"bytes":73492},"home":{"status":"ok","http":200,"url":"https://www.overstock.com/","html_tokens":91187,"markdown_tokens":26,"pruned_tokens":26}},
{"rank":5075,"domain":"tinkercad.com","origin":"https://tinkercad.com","robots":{"status":"ok","http":200,"url":"https://www.tinkercad.com/robots.txt","bytes":232,"groups":1},"bots":{"GPTBot":["partial","*",7,1],"ChatGPT-User":["partial","*",7,1],"OAI-SearchBot":["partial","*",7,1],"ClaudeBot":["partial","*",7,1],"Claude-User":["partial","*",7,1],"Claude-SearchBot":["partial","*",7,1],"anthropic-ai":["partial","*",7,1],"PerplexityBot":["partial","*",7,1],"Perplexity-User":["partial","*",7,1],"Google-Extended":["partial","*",7,1],"Applebot-Extended":["partial","*",7,1],"CCBot":["partial","*",7,1],"Bytespider":["partial","*",7,1],"Amazonbot":["partial","*",7,1],"meta-externalagent":["partial","*",7,1],"cohere-ai":["partial","*",7,1],"Diffbot":["partial","*",7,1],"ContextFetchBot":["partial","*",7,1]},"llms":{"status":"missing","http":404,"bytes":10101},"home":{"status":"ok","http":200,"url":"https://www.tinkercad.com/","html_tokens":2798,"markdown_tokens":0,"pruned_tokens":0}},
{"rank":5076,"domain":"ispmanager.com","origin":"https://ispmanager.com","robots":{"status":"ok","http":200,"url":"https://www.ispmanager.com/robots.txt","bytes":703,"groups":1},"bots":{"GPTBot":["partial","*",23,8],"ChatGPT-User":["partial","*",23,8],"OAI-SearchBot":["partial","*",23,8],"ClaudeBot":["partial","*",23,8],"Claude-User":["partial","*",23,8],"Claude-SearchBot":["partial","*",23,8],"anthropic-ai":["partial","*",23,8],"PerplexityBot":["partial","*",23,8],"Perplexity-User":["partial","*",23,8],"Google-Extended":["partial","*",23,8],"Applebot-Extended":["partial","*",23,8],"CCBot":["partial","*",23,8],"Bytespider":["partial","*",23,8],"Amazonbot":["partial","*",23,8],"meta-externalagent":["partial","*",23,8],"cohere-ai":["partial","*",23,8],"Diffbot":["partial","*",23,8],"ContextFetchBot":["partial","*",23,8]},"llms":{"status":"missing","http":404,"bytes":28080},"home":{"status":"ok","http":200,"url":"https://www.ispmanager.com/","html_tokens":13538,"markdown_tokens":175,"pruned_tokens":132}},
{"rank":5077,"domain":"flsenate.gov","origin":"https://flsenate.gov","robots":{"status":"ok","http":200,"url":"https://flsenate.gov/robots.txt","bytes":968,"groups":1},"bots":{"GPTBot":["partial","*",22,11],"ChatGPT-User":["partial","*",22,11],"OAI-SearchBot":["partial","*",22,11],"ClaudeBot":["partial","*",22,11],"Claude-User":["partial","*",22,11],"Claude-SearchBot":["partial","*",22,11],"anthropic-ai":["partial","*",22,11],"PerplexityBot":["partial","*",22,11],"Perplexity-User":["partial","*",22,11],"Google-Extended":["partial","*",22,11],"Applebot-Extended":["partial","*",22,11],"CCBot":["partial","*",22,11],"Bytespider":["partial","*",22,11],"Amazonbot":["partial","*",22,11],"meta-externalagent":["partial","*",22,11],"cohere-ai":["partial","*",22,11],"Diffbot":["partial","*",22,11],"ContextFetchBot":["partial","*",22,11]},"llms":{"status":"missing","http":404,"bytes":21308},"home":{"status":"ok","http":200,"url":"https://flsenate.gov/","html_tokens":14518,"markdown_tokens":303,"pruned_tokens":155}},
{"rank":5078,"domain":"domaineasy.com","origin":"https://domaineasy.com","robots":{"status":"missing","http":404,"url":"https://www.domaineasy.com/robots.txt","bytes":19818,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":19818},"home":{"status":"http_error","http":403,"url":"https://domaineasy.com/"}},
{"rank":5079,"domain":"wn.com","origin":"https://wn.com","robots":{"status":"ok","http":200,"url":"https://wn.com/robots.txt","bytes":0,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":955},"home":{"status":"ok","http":200,"url":"https://wn.com/","html_tokens":36699,"markdown_tokens":2876,"pruned_tokens":175}},
{"rank":5080,"domain":"ath.cx","origin":"http://ath.cx","robots":{"status":"refused","http":403,"url":"https://www.oracle.com/cloud/networking/dns/","bytes":1450,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":5081,"domain":"kffhealthnews.org","origin":"https://kffhealthnews.org","robots":{"status":"ok","http":200,"url":"https://kffhealthnews.org/robots.txt","bytes":331,"groups":1},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"missing","http":404,"bytes":105013},"home":{"status":"ok","http":200,"url":"https://kffhealthnews.org/","html_tokens":59752,"markdown_tokens":759,"pruned_tokens":166}},
{"rank":5082,"domain":"imgflip.com","origin":"https://imgflip.com","robots":{"status":"ok","http":200,"url":"https://imgflip.com/robots.txt","bytes":90,"groups":1},"bots":{"GPTBot":["partial","*",2,0],"ChatGPT-User":["partial","*",2,0],"OAI-SearchBot":["partial","*",2,0],"ClaudeBot":["partial","*",2,0],"Claude-User":["partial","*",2,0],"Claude-SearchBot":["partial","*",2,0],"anthropic-ai":["partial","*",2,0],"PerplexityBot":["partial","*",2,0],"Perplexity-User":["partial","*",2,0],"Google-Extended":["partial","*",2,0],"Applebot-Extended":["partial","*",2,0],"CCBot":["partial","*",2,0],"Bytespider":["partial","*",2,0],"Amazonbot":["partial","*",2,0],"meta-externalagent":["partial","*",2,0],"cohere-ai":["partial","*",2,0],"Diffbot":["partial","*",2,0],"ContextFetchBot":["partial","*",2,0]},"llms":{"status":"missing","http":404,"bytes":7380},"home":{"status":"ok","http":200,"url":"https://imgflip.com/","html_tokens":4022,"markdown_tokens":323,"pruned_tokens":64}},
{"rank":5083,"domain":"ina.fr","origin":"https://ina.fr","robots":{"status":"ok","http":200,"url":"https://www.ina.fr/robots.txt","bytes":5716,"groups":2},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["blocked","named",1,0],"OAI-SearchBot":["blocked","named",1,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["blocked","named",1,0],"Claude-SearchBot":["blocked","named",1,0],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["blocked","named",1,0],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",26,30]},"llms":{"status":"missing","http":404,"bytes":53813},"home":{"status":"ok","http":200,"url":"https://www.ina.fr/","html_tokens":59749,"markdown_tokens":86,"pruned_tokens":86}},
{"rank":5084,"domain":"thejournal.ie","origin":"https://thejournal.ie","robots":{"status":"ok","http":200,"url":"https://www.thejournal.ie/robots.txt","bytes":1092,"groups":5},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["blocked","named",1,0],"OAI-SearchBot":["partial","*",24,0],"ClaudeBot":["partial","*",24,0],"Claude-User":["partial","*",24,0],"Claude-SearchBot":["partial","*",24,0],"anthropic-ai":["partial","*",24,0],"PerplexityBot":["partial","*",24,0],"Perplexity-User":["partial","*",24,0],"Google-Extended":["partial","*",24,0],"Applebot-Extended":["partial","*",24,0],"CCBot":["partial","*",24,0],"Bytespider":["partial","*",24,0],"Amazonbot":["partial","*",24,0],"meta-externalagent":["partial","*",24,0],"cohere-ai":["partial","*",24,0],"Diffbot":["partial","*",24,0],"ContextFetchBot":["partial","*",24,0]},"llms":{"status":"missing","http":404,"bytes":192041},"home":{"status":"ok","http":200,"url":"https://www.thejournal.ie/","html_tokens":90823,"markdown_tokens":1514,"pruned_tokens":149}},
{"rank":5085,"domain":"hktdc.com","origin":"https://hktdc.com","robots":{"status":"ok","http":200,"url":"https://www.hktdc.com/robots.txt","bytes":26814,"groups":14},"bots":{"GPTBot":["partial","*",117,0],"ChatGPT-User":["partial","*",117,0],"OAI-SearchBot":["partial","*",117,0],"ClaudeBot":["partial","*",117,0],"Claude-User":["partial","*",117,0],"Claude-SearchBot":["partial","*",117,0],"anthropic-ai":["partial","*",117,0],"PerplexityBot":["partial","*",117,0],"Perplexity-User":["partial","*",117,0],"Google-Extended":["partial","*",117,0],"Applebot-Extended":["partial","*",117,0],"CCBot":["partial","*",117,0],"Bytespider":["partial","*",117,0],"Amazonbot":["partial","*",117,0],"meta-externalagent":["partial","*",117,0],"cohere-ai":["partial","*",117,0],"Diffbot":["partial","*",117,0],"ContextFetchBot":["partial","*",117,0]},"llms":{"status":"missing","http":404,"bytes":5620},"home":{"status":"ok","http":200,"url":"https://www.hktdc.com/","html_tokens":2086,"markdown_tokens":12,"pruned_tokens":12}},
{"rank":5086,"domain":"agilent.com","origin":"https://agilent.com","robots":{"status":"missing","http":404,"url":"https://agilent.com/robots.txt","bytes":236,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"http_error","http":403,"bytes":381},"home":{"status":"http_error","http":403,"url":"https://www.agilent.com/"}},
{"rank":5087,"domain":"chez.com","origin":"http://chez.com","robots":{"status":"missing","http":404,"url":"http://chez.com/robots.txt","bytes":345,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":345},"home":{"status":"ok","http":200,"url":"http://chez.com/","html_tokens":1966,"markdown_tokens":175,"pruned_tokens":126}},
{"rank":5088,"domain":"conac.cn","origin":"http://conac.cn","robots":{"status":"missing","http":404,"url":"http://conac.cn/robots.txt","bytes":146,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":146},"home":{"status":"ok","http":200,"url":"http://conac.cn/","html_tokens":3461,"markdown_tokens":75,"pruned_tokens":75}},
{"rank":5089,"domain":"royalcaribbean.com","origin":"https://royalcaribbean.com","robots":{"status":"ok","http":200,"url":"https://www.royalcaribbean.com/robots.txt","bytes":1351,"groups":2},"bots":{"GPTBot":["partial","*",40,2],"ChatGPT-User":["partial","*",40,2],"OAI-SearchBot":["partial","*",40,2],"ClaudeBot":["partial","*",40,2],"Claude-User":["partial","*",40,2],"Claude-SearchBot":["partial","*",40,2],"anthropic-ai":["partial","*",40,2],"PerplexityBot":["partial","*",40,2],"Perplexity-User":["partial","*",40,2],"Google-Extended":["partial","*",40,2],"Applebot-Extended":["partial","*",40,2],"CCBot":["partial","*",40,2],"Bytespider":["partial","*",40,2],"Amazonbot":["partial","*",40,2],"meta-externalagent":["partial","*",40,2],"cohere-ai":["partial","*",40,2],"Diffbot":["partial","*",40,2],"ContextFetchBot":["partial","*",40,2]},"llms":{"status":"missing","http":404,"bytes":9536},"home":{"status":"ok","http":200,"url":"https://www.royalcaribbean.com/","html_tokens":194790,"markdown_tokens":455,"pruned_tokens":19}},
{"rank":5090,"domain":"kantar.com","origin":"https://kantar.com","robots":{"status":"ok","http":200,"url":"https://www.kantar.com/robots.txt","bytes":1377,"groups":1},"bots":{"GPTBot":["partial","*",18,0],"ChatGPT-User":["partial","*",18,0],"OAI-SearchBot":["partial","*",18,0],"ClaudeBot":["partial","*",18,0],"Claude-User":["partial","*",18,0],"Claude-SearchBot":["partial","*",18,0],"anthropic-ai":["partial","*",18,0],"PerplexityBot":["partial","*",18,0],"Perplexity-User":["partial","*",18,0],"Google-Extended":["partial","*",18,0],"Applebot-Extended":["partial","*",18,0],"CCBot":["partial","*",18,0],"Bytespider":["partial","*",18,0],"Amazonbot":["partial","*",18,0],"meta-externalagent":["partial","*",18,0],"cohere-ai":["partial","*",18,0],"Diffbot":["partial","*",18,0],"ContextFetchBot":["partial","*",18,0]},"llms":{"status":"missing","http":404,"bytes":103},"home":{"status":"ok","http":200,"url":"https://www.kantar.com/","html_tokens":22804,"markdown_tokens":193,"pruned_tokens":45}},
{"rank":5091,"domain":"piwigo.org","origin":"https://piwigo.org","robots":{"status":"missing","http":404,"url":"https://piwigo.org/robots.txt","bytes":3599,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":3599},"home":{"status":"ok","http":200,"url":"https://piwigo.org/","html_tokens":27957,"markdown_tokens":1140,"pruned_tokens":832}},
{"rank":5092,"domain":"veoh.com","origin":"https://veoh.com","robots":{"status":"ok","http":200,"url":"https://veoh.com/robots.txt","bytes":579,"groups":1},"bots":{"GPTBot":["partial","*",27,0],"ChatGPT-User":["partial","*",27,0],"OAI-SearchBot":["partial","*",27,0],"ClaudeBot":["partial","*",27,0],"Claude-User":["partial","*",27,0],"Claude-SearchBot":["partial","*",27,0],"anthropic-ai":["partial","*",27,0],"PerplexityBot":["partial","*",27,0],"Perplexity-User":["partial","*",27,0],"Google-Extended":["partial","*",27,0],"Applebot-Extended":["partial","*",27,0],"CCBot":["partial","*",27,0],"Bytespider":["partial","*",27,0],"Amazonbot":["partial","*",27,0],"meta-externalagent":["partial","*",27,0],"cohere-ai":["partial","*",27,0],"Diffbot":["partial","*",27,0],"ContextFetchBot":["partial","*",27,0]},"llms":{"status":"present","http":200,"bytes":937},"home":{"status":"ok","http":200,"url":"https://veoh.com/","html_tokens":64658,"markdown_tokens":168,"pruned_tokens":168}},
{"rank":5093,"domain":"liverpoolfc.com","origin":"https://liverpoolfc.com","robots":{"status":"ok","http":200,"url":"https://www.liverpoolfc.com/robots.txt","bytes":161,"groups":1},"bots":{"GPTBot":["allowed","*",0,1],"ChatGPT-User":["allowed","*",0,1],"OAI-SearchBot":["allowed","*",0,1],"ClaudeBot":["allowed","*",0,1],"Claude-User":["allowed","*",0,1],"Claude-SearchBot":["allowed","*",0,1],"anthropic-ai":["allowed","*",0,1],"PerplexityBot":["allowed","*",0,1],"Perplexity-User":["allowed","*",0,1],"Google-Extended":["allowed","*",0,1],"Applebot-Extended":["allowed","*",0,1],"CCBot":["allowed","*",0,1],"Bytespider":["allowed","*",0,1],"Amazonbot":["allowed","*",0,1],"meta-externalagent":["allowed","*",0,1],"cohere-ai":["allowed","*",0,1],"Diffbot":["allowed","*",0,1],"ContextFetchBot":["allowed","*",0,1]},"llms":{"status":"missing","http":404,"bytes":72116},"home":{"status":"ok","http":200,"url":"https://www.liverpoolfc.com/","html_tokens":280797,"markdown_tokens":55,"pruned_tokens":55}},
{"rank":5094,"domain":"creativemarket.com","origin":"https://creativemarket.com","robots":{"status":"refused","http":403,"url":"https://creativemarket.com/robots.txt","bytes":5535,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":5095,"domain":"psl.eu","origin":"https://psl.eu","robots":{"status":"refused","http":403,"url":"https://psl.eu/robots.txt","bytes":3654,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":5096,"domain":"us04web.zoom.us","origin":"https://us04web.zoom.us","robots":{"status":"ok","http":200,"url":"https://us04web.zoom.us/robots.txt","bytes":2005,"groups":13},"bots":{"GPTBot":["partial","*",28,0],"ChatGPT-User":["partial","*",28,0],"OAI-SearchBot":["partial","*",28,0],"ClaudeBot":["partial","*",28,0],"Claude-User":["partial","*",28,0],"Claude-SearchBot":["partial","*",28,0],"anthropic-ai":["partial","*",28,0],"PerplexityBot":["partial","*",28,0],"Perplexity-User":["partial","*",28,0],"Google-Extended":["partial","*",28,0],"Applebot-Extended":["partial","*",28,0],"CCBot":["partial","*",28,0],"Bytespider":["partial","*",28,0],"Amazonbot":["partial","*",28,0],"meta-externalagent":["partial","*",28,0],"cohere-ai":["partial","*",28,0],"Diffbot":["partial","*",28,0],"ContextFetchBot":["partial","*",28,0]},"llms":{"status":"missing","http":404,"bytes":526514},"home":{"status":"ok","http":200,"url":"https://www.zoom.com/","html_tokens":79701,"markdown_tokens":938,"pruned_tokens":156}},
{"rank":5097,"domain":"treccani.it","origin":"https://treccani.it","robots":{"status":"ok","http":200,"url":"https://www.treccani.it/robots.txt","bytes":12801,"groups":47},"bots":{"GPTBot":["partial","named",7,0],"ChatGPT-User":["partial","named",7,1],"OAI-SearchBot":["partial","named",7,1],"ClaudeBot":["partial","named",7,0],"Claude-User":["partial","named",7,1],"Claude-SearchBot":["partial","named",7,1],"anthropic-ai":["partial","named",7,0],"PerplexityBot":["partial","named",7,1],"Perplexity-User":["partial","named",7,1],"Google-Extended":["partial","named",7,0],"Applebot-Extended":["blocked","named",8,0],"CCBot":["partial","named",7,0],"Bytespider":["blocked","named",8,0],"Amazonbot":["blocked","named",8,0],"meta-externalagent":["blocked","named",8,0],"cohere-ai":["blocked","named",8,0],"Diffbot":["blocked","named",8,0],"ContextFetchBot":["partial","*",48,0]},"llms":{"status":"html","http":200,"bytes":4221880},"home":{"status":"ok","http":200,"url":"https://www.treccani.it/","html_tokens":1052944,"markdown_tokens":499,"pruned_tokens":101}},
{"rank":5098,"domain":"brnostavi.cz","origin":"https://brnostavi.cz","robots":{"status":"ok","http":200,"url":"https://brnostavi.cz/robots.txt","bytes":129,"groups":1},"bots":{"GPTBot":["partial","*",2,1],"ChatGPT-User":["partial","*",2,1],"OAI-SearchBot":["partial","*",2,1],"ClaudeBot":["partial","*",2,1],"Claude-User":["partial","*",2,1],"Claude-SearchBot":["partial","*",2,1],"anthropic-ai":["partial","*",2,1],"PerplexityBot":["partial","*",2,1],"Perplexity-User":["partial","*",2,1],"Google-Extended":["partial","*",2,1],"Applebot-Extended":["partial","*",2,1],"CCBot":["partial","*",2,1],"Bytespider":["partial","*",2,1],"Amazonbot":["partial","*",2,1],"meta-externalagent":["partial","*",2,1],"cohere-ai":["partial","*",2,1],"Diffbot":["partial","*",2,1],"ContextFetchBot":["partial","*",2,1]},"llms":{"status":"missing","http":404,"bytes":2017},"home":{"status":"ok","http":200,"url":"https://brnostavi.cz/","html_tokens":7708,"markdown_tokens":2643,"pruned_tokens":134}},
{"rank":5099,"domain":"netapp.com","origin":"https://netapp.com","robots":{"status":"refused","http":403,"url":"https://www.netapp.com/robots.txt","bytes":382,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":5100,"domain":"buzzsumo.com","origin":"https://buzzsumo.com","robots":{"status":"ok","http":200,"url":"https://buzzsumo.com/robots.txt","bytes":113,"groups":1},"bots":{"GPTBot":["partial","*",1,1],"ChatGPT-User":["partial","*",1,1],"OAI-SearchBot":["partial","*",1,1],"ClaudeBot":["partial","*",1,1],"Claude-User":["partial","*",1,1],"Claude-SearchBot":["partial","*",1,1],"anthropic-ai":["partial","*",1,1],"PerplexityBot":["partial","*",1,1],"Perplexity-User":["partial","*",1,1],"Google-Extended":["partial","*",1,1],"Applebot-Extended":["partial","*",1,1],"CCBot":["partial","*",1,1],"Bytespider":["partial","*",1,1],"Amazonbot":["partial","*",1,1],"meta-externalagent":["partial","*",1,1],"cohere-ai":["partial","*",1,1],"Diffbot":["partial","*",1,1],"ContextFetchBot":["partial","*",1,1]},"llms":{"status":"missing","http":404,"bytes":109530},"home":{"status":"ok","http":200,"url":"https://buzzsumo.com/","html_tokens":43071,"markdown_tokens":92,"pruned_tokens":92}},
{"rank":5101,"domain":"rp.pl","origin":"https://rp.pl","robots":{"status":"ok","http":200,"url":"https://www.rp.pl/robots.txt","bytes":1044,"groups":1},"bots":{"GPTBot":["partial","*",23,1],"ChatGPT-User":["partial","*",23,1],"OAI-SearchBot":["partial","*",23,1],"ClaudeBot":["partial","*",23,1],"Claude-User":["partial","*",23,1],"Claude-SearchBot":["partial","*",23,1],"anthropic-ai":["partial","*",23,1],"PerplexityBot":["partial","*",23,1],"Perplexity-User":["partial","*",23,1],"Google-Extended":["partial","*",23,1],"Applebot-Extended":["partial","*",23,1],"CCBot":["partial","*",23,1],"Bytespider":["partial","*",23,1],"Amazonbot":["partial","*",23,1],"meta-externalagent":["partial","*",23,1],"cohere-ai":["partial","*",23,1],"Diffbot":["partial","*",23,1],"ContextFetchBot":["partial","*",23,1]},"llms":{"status":"missing","http":404,"bytes":81724},"home":{"status":"ok","http":200,"url":"https://www.rp.pl/","html_tokens":191701,"markdown_tokens":275,"pruned_tokens":141}},
{"rank":5102,"domain":"michaels.com","origin":"https://michaels.com","robots":{"status":"refused","http":403,"url":"https://www.michaels.com/robots.txt","bytes":382,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":5103,"domain":"telekom.com","origin":"https://telekom.com","robots":{"status":"ok","http":200,"url":"https://www.telekom.com/robots.txt","bytes":167,"groups":2},"bots":{"GPTBot":["partial","*",1,0],"ChatGPT-User":["partial","*",1,0],"OAI-SearchBot":["partial","*",1,0],"ClaudeBot":["partial","*",1,0],"Claude-User":["partial","*",1,0],"Claude-SearchBot":["partial","*",1,0],"anthropic-ai":["partial","*",1,0],"PerplexityBot":["partial","*",1,0],"Perplexity-User":["partial","*",1,0],"Google-Extended":["partial","*",1,0],"Applebot-Extended":["partial","*",1,0],"CCBot":["partial","*",1,0],"Bytespider":["partial","*",1,0],"Amazonbot":["partial","*",1,0],"meta-externalagent":["partial","*",1,0],"cohere-ai":["partial","*",1,0],"Diffbot":["partial","*",1,0],"ContextFetchBot":["partial","*",1,0]},"llms":{"status":"missing","http":404,"bytes":128639},"home":{"status":"ok","http":200,"url":"https://www.telekom.com/en","html_tokens":55954,"markdown_tokens":177,"pruned_tokens":115}},
{"rank":5104,"domain":"keysight.com","origin":"https://keysight.com","robots":{"status":"ok","http":200,"url":"https://www.keysight.com/robots.txt","bytes":1255,"groups":1},"bots":{"GPTBot":["partial","*",10,0],"ChatGPT-User":["partial","*",10,0],"OAI-SearchBot":["partial","*",10,0],"ClaudeBot":["partial","*",10,0],"Claude-User":["partial","*",10,0],"Claude-SearchBot":["partial","*",10,0],"anthropic-ai":["partial","*",10,0],"PerplexityBot":["partial","*",10,0],"Perplexity-User":["partial","*",10,0],"Google-Extended":["partial","*",10,0],"Applebot-Extended":["partial","*",10,0],"CCBot":["partial","*",10,0],"Bytespider":["partial","*",10,0],"Amazonbot":["partial","*",10,0],"meta-externalagent":["partial","*",10,0],"cohere-ai":["partial","*",10,0],"Diffbot":["partial","*",10,0],"ContextFetchBot":["partial","*",10,0]},"llms":{"status":"missing","http":404,"bytes":667},"home":{"status":"http_error","http":403,"url":"https://www.keysight.com/us/en/home.html"}},
{"rank":5105,"domain":"courier-journal.com","origin":"https://courier-journal.com","robots":{"status":"ok","http":200,"url":"https://www.courier-journal.com/robots.txt","bytes":12687,"groups":288},"bots":{"GPTBot":["partial","named",1,4],"ChatGPT-User":["partial","named",1,4],"OAI-SearchBot":["partial","named",1,4],"ClaudeBot":["blocked","named",1,0],"Claude-User":["blocked","named",1,0],"Claude-SearchBot":["blocked","named",1,0],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["partial","named",1,4],"Perplexity-User":["blocked","named",1,0],"Google-Extended":["partial","named",1,4],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",2,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",32,0]},"llms":{"status":"http_error","http":402,"bytes":254},"home":{"status":"http_error","http":402,"url":"https://www.courier-journal.com/"}},
{"rank":5106,"domain":"twincities.com","origin":"https://twincities.com","robots":{"status":"ok","http":200,"url":"https://www.twincities.com/robots.txt","bytes":4480,"groups":68},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["blocked","named",1,0],"OAI-SearchBot":["blocked","named",1,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["blocked","named",1,0],"Claude-SearchBot":["blocked","named",1,0],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["blocked","named",1,0],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["partial","*",8,1],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",8,1]},"llms":{"status":"missing","http":404,"bytes":99686},"home":{"status":"ok","http":200,"url":"https://www.twincities.com/","html_tokens":63313,"markdown_tokens":1069,"pruned_tokens":175}},
{"rank":5107,"domain":"cmbchina.com","origin":"https://cmbchina.com","robots":{"status":"html","http":200,"url":"https://cmbchina.com/robots.txt","bytes":7588,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":5108,"domain":"hihonor.com","origin":"https://hihonor.com","robots":{"status":"ok","http":200,"url":"https://www.honor.com/robots.txt","bytes":1921,"groups":1},"bots":{"GPTBot":["partial","*",66,2],"ChatGPT-User":["partial","*",66,2],"OAI-SearchBot":["partial","*",66,2],"ClaudeBot":["partial","*",66,2],"Claude-User":["partial","*",66,2],"Claude-SearchBot":["partial","*",66,2],"anthropic-ai":["partial","*",66,2],"PerplexityBot":["partial","*",66,2],"Perplexity-User":["partial","*",66,2],"Google-Extended":["partial","*",66,2],"Applebot-Extended":["partial","*",66,2],"CCBot":["partial","*",66,2],"Bytespider":["partial","*",66,2],"Amazonbot":["partial","*",66,2],"meta-externalagent":["partial","*",66,2],"cohere-ai":["partial","*",66,2],"Diffbot":["partial","*",66,2],"ContextFetchBot":["partial","*",66,2]},"llms":{"status":"present","http":200,"bytes":1960},"home":{"status":"timeout","error":"timed out after 15 s"}},
{"rank":5109,"domain":"ms.us","origin":"https://ms.us","robots":{"status":"unreachable","http":null,"url":"https://ms.us/robots.txt","bytes":0,"groups":0,"error":"cannot resolve ms.us"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":5110,"domain":"birminghammail.co.uk","origin":"https://birminghammail.co.uk","robots":{"status":"ok","http":200,"url":"https://www.birminghammail.co.uk/robots.txt","bytes":939,"groups":14},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["partial","*",9,0],"OAI-SearchBot":["blocked","named",1,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",9,0],"Claude-SearchBot":["partial","*",9,0],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["partial","*",9,0],"Google-Extended":["partial","*",9,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["partial","*",9,0],"Amazonbot":["partial","*",9,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["partial","*",9,0],"Diffbot":["partial","*",9,0],"ContextFetchBot":["partial","*",9,0]},"llms":{"status":"missing","http":404,"bytes":472351},"home":{"status":"ok","http":200,"url":"https://www.birminghammail.co.uk/","html_tokens":249623,"markdown_tokens":153,"pruned_tokens":153}},
{"rank":5111,"domain":"wyborcza.pl","origin":"https://wyborcza.pl","robots":{"status":"ok","http":200,"url":"https://wyborcza.pl/robots.txt","bytes":14241,"groups":304},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["blocked","named",2,0],"OAI-SearchBot":["partial","*",24,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",24,0],"Claude-SearchBot":["partial","*",24,0],"anthropic-ai":["blocked","named",2,0],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["partial","*",24,0],"Google-Extended":["partial","*",24,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["partial","*",24,0],"meta-externalagent":["partial","*",24,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["partial","*",24,0],"ContextFetchBot":["partial","*",24,0]},"llms":{"status":"missing","http":404,"bytes":3849},"home":{"status":"ok","http":200,"url":"https://wyborcza.pl/0,0.html","html_tokens":155665,"markdown_tokens":20,"pruned_tokens":20}},
{"rank":5112,"domain":"istat.it","origin":"https://www.istat.it","robots":{"status":"ok","http":200,"url":"https://www.istat.it/robots.txt","bytes":13,"groups":1},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"http_error","http":403,"bytes":568},"home":{"status":"ok","http":200,"url":"https://www.istat.it/","html_tokens":70094,"markdown_tokens":1175,"pruned_tokens":124}},
{"rank":5113,"domain":"ccleaner.com","origin":"https://ccleaner.com","robots":{"status":"refused","http":403,"url":"https://www.ccleaner.com/robots.txt","bytes":384,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":5114,"domain":"zenbusiness.com","origin":"https://zenbusiness.com","robots":{"status":"ok","http":200,"url":"https://www.zenbusiness.com/robots.txt","bytes":2483,"groups":1},"bots":{"GPTBot":["partial","*",83,3],"ChatGPT-User":["partial","*",83,3],"OAI-SearchBot":["partial","*",83,3],"ClaudeBot":["partial","*",83,3],"Claude-User":["partial","*",83,3],"Claude-SearchBot":["partial","*",83,3],"anthropic-ai":["partial","*",83,3],"PerplexityBot":["partial","*",83,3],"Perplexity-User":["partial","*",83,3],"Google-Extended":["partial","*",83,3],"Applebot-Extended":["partial","*",83,3],"CCBot":["partial","*",83,3],"Bytespider":["partial","*",83,3],"Amazonbot":["partial","*",83,3],"meta-externalagent":["partial","*",83,3],"cohere-ai":["partial","*",83,3],"Diffbot":["partial","*",83,3],"ContextFetchBot":["partial","*",83,3]},"llms":{"status":"present","http":200,"bytes":31406},"home":{"status":"ok","http":200,"url":"https://www.zenbusiness.com/","html_tokens":61683,"markdown_tokens":1762,"pruned_tokens":1328}},
{"rank":5115,"domain":"atw.hu","origin":"https://atw.hu","robots":{"status":"ok","http":200,"url":"https://atw.hu/robots.txt","bytes":24,"groups":1},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"missing","http":404,"bytes":23710},"home":{"status":"ok","http":200,"url":"https://atw.hu/","html_tokens":21360,"markdown_tokens":423,"pruned_tokens":135}},
{"rank":5116,"domain":"avvo.com","origin":"https://avvo.com","robots":{"status":"ok","http":200,"url":"https://www.avvo.com/robots.txt","bytes":6299,"groups":7},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["partial","*",73,0],"OAI-SearchBot":["partial","*",73,0],"ClaudeBot":["partial","*",73,0],"Claude-User":["partial","*",73,0],"Claude-SearchBot":["partial","*",73,0],"anthropic-ai":["partial","*",73,0],"PerplexityBot":["partial","*",73,0],"Perplexity-User":["partial","*",73,0],"Google-Extended":["partial","*",73,0],"Applebot-Extended":["partial","*",73,0],"CCBot":["partial","*",73,0],"Bytespider":["partial","*",73,0],"Amazonbot":["partial","*",73,0],"meta-externalagent":["partial","*",73,0],"cohere-ai":["partial","*",73,0],"Diffbot":["partial","*",73,0],"ContextFetchBot":["partial","*",73,0]},"llms":{"status":"http_error","http":403,"bytes":5501},"home":{"status":"http_error","http":403,"url":"https://www.avvo.com/"}},
{"rank":5117,"domain":"google.hr","origin":"https://google.hr","robots":{"status":"ok","http":200,"url":"https://www.google.hr/robots.txt","bytes":6624,"groups":4},"bots":{"GPTBot":["partial","*",169,64],"ChatGPT-User":["partial","*",169,64],"OAI-SearchBot":["partial","*",169,64],"ClaudeBot":["partial","*",169,64],"Claude-User":["partial","*",169,64],"Claude-SearchBot":["partial","*",169,64],"anthropic-ai":["partial","*",169,64],"PerplexityBot":["partial","*",169,64],"Perplexity-User":["partial","*",169,64],"Google-Extended":["partial","*",169,64],"Applebot-Extended":["partial","*",169,64],"CCBot":["partial","*",169,64],"Bytespider":["partial","*",169,64],"Amazonbot":["partial","*",169,64],"meta-externalagent":["partial","*",169,64],"cohere-ai":["partial","*",169,64],"Diffbot":["partial","*",169,64],"ContextFetchBot":["partial","*",169,64]},"llms":{"status":"missing","http":404,"bytes":1569},"home":{"status":"ok","http":200,"url":"https://www.google.com/","html_tokens":21313,"markdown_tokens":30,"pruned_tokens":30}},
{"rank":5118,"domain":"bigbadwolf-slot.com","origin":"https://bigbadwolf-slot.com","robots":{"status":"ok","http":200,"url":"https://bigbadwolf-slot.com/robots.txt","bytes":542,"groups":1},"bots":{"GPTBot":["partial","*",19,0],"ChatGPT-User":["partial","*",19,0],"OAI-SearchBot":["partial","*",19,0],"ClaudeBot":["partial","*",19,0],"Claude-User":["partial","*",19,0],"Claude-SearchBot":["partial","*",19,0],"anthropic-ai":["partial","*",19,0],"PerplexityBot":["partial","*",19,0],"Perplexity-User":["partial","*",19,0],"Google-Extended":["partial","*",19,0],"Applebot-Extended":["partial","*",19,0],"CCBot":["partial","*",19,0],"Bytespider":["partial","*",19,0],"Amazonbot":["partial","*",19,0],"meta-externalagent":["partial","*",19,0],"cohere-ai":["partial","*",19,0],"Diffbot":["partial","*",19,0],"ContextFetchBot":["partial","*",19,0]},"llms":{"status":"missing","http":404,"bytes":24697},"home":{"status":"ok","http":200,"url":"https://bigbadwolf-slot.com/","html_tokens":20876,"markdown_tokens":1426,"pruned_tokens":30}},
{"rank":5119,"domain":"elitedaily.com","origin":"https://elitedaily.com","robots":{"status":"ok","http":200,"url":"https://www.elitedaily.com/robots.txt","bytes":294,"groups":2},"bots":{"GPTBot":["partial","*",1,0],"ChatGPT-User":["partial","*",1,0],"OAI-SearchBot":["partial","*",1,0],"ClaudeBot":["partial","*",1,0],"Claude-User":["partial","*",1,0],"Claude-SearchBot":["partial","*",1,0],"anthropic-ai":["partial","*",1,0],"PerplexityBot":["partial","*",1,0],"Perplexity-User":["partial","*",1,0],"Google-Extended":["partial","*",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["partial","*",1,0],"Bytespider":["partial","*",1,0],"Amazonbot":["partial","*",1,0],"meta-externalagent":["partial","*",1,0],"cohere-ai":["partial","*",1,0],"Diffbot":["partial","*",1,0],"ContextFetchBot":["partial","*",1,0]},"llms":{"status":"missing","http":404,"bytes":71213},"home":{"status":"ok","http":200,"url":"https://www.elitedaily.com","html_tokens":83684,"markdown_tokens":395,"pruned_tokens":30}},
{"rank":5120,"domain":"bookdepository.com","origin":"https://bookdepository.com","robots":{"status":"html","http":200,"url":"https://www.amazon.com/amz-books/store?ou=psf&ref_=usbk_surl_books","bytes":393441,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":5121,"domain":"cci.fr","origin":"https://www.cci.fr","robots":{"status":"refused","http":403,"url":"https://www.cci.fr/robots.txt","bytes":5547,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":5122,"domain":"cnpq.br","origin":"http://cnpq.br","robots":{"status":"html","http":200,"url":"https://www.gov.br/cnpq/pt-br","bytes":254304,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":5123,"domain":"netdna-cdn.com","origin":"https://netdna-cdn.com","robots":{"status":"unreachable","http":null,"url":"https://netdna-cdn.com/robots.txt","bytes":0,"groups":0,"error":"cannot resolve netdna-cdn.com"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":5124,"domain":"rr.com","origin":"https://rr.com","robots":{"status":"timeout","http":null,"url":"https://rr.com/robots.txt","bytes":0,"groups":0,"error":"ConnectTimeout"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":5125,"domain":"isaca.org","origin":"https://isaca.org","robots":{"status":"ok","http":200,"url":"https://www.isaca.org/robots.txt","bytes":1073,"groups":9},"bots":{"GPTBot":["partial","*",28,0],"ChatGPT-User":["partial","*",28,0],"OAI-SearchBot":["partial","*",28,0],"ClaudeBot":["partial","*",28,0],"Claude-User":["partial","*",28,0],"Claude-SearchBot":["partial","*",28,0],"anthropic-ai":["partial","*",28,0],"PerplexityBot":["partial","*",28,0],"Perplexity-User":["partial","*",28,0],"Google-Extended":["partial","*",28,0],"Applebot-Extended":["partial","*",28,0],"CCBot":["blocked","named",1,0],"Bytespider":["partial","*",28,0],"Amazonbot":["partial","*",28,0],"meta-externalagent":["partial","*",28,0],"cohere-ai":["partial","*",28,0],"Diffbot":["partial","*",28,0],"ContextFetchBot":["partial","*",28,0]},"llms":{"status":"missing","http":404,"bytes":118771},"home":{"status":"ok","http":200,"url":"https://www.isaca.org/","html_tokens":40931,"markdown_tokens":392,"pruned_tokens":176}},
{"rank":5126,"domain":"tucson.com","origin":"https://tucson.com","robots":{"status":"ok","http":200,"url":"https://tucson.com/robots.txt","bytes":6821,"groups":1},"bots":{"GPTBot":["partial","*",85,0],"ChatGPT-User":["partial","*",85,0],"OAI-SearchBot":["partial","*",85,0],"ClaudeBot":["partial","*",85,0],"Claude-User":["partial","*",85,0],"Claude-SearchBot":["partial","*",85,0],"anthropic-ai":["partial","*",85,0],"PerplexityBot":["partial","*",85,0],"Perplexity-User":["partial","*",85,0],"Google-Extended":["partial","*",85,0],"Applebot-Extended":["partial","*",85,0],"CCBot":["partial","*",85,0],"Bytespider":["partial","*",85,0],"Amazonbot":["partial","*",85,0],"meta-externalagent":["partial","*",85,0],"cohere-ai":["partial","*",85,0],"Diffbot":["partial","*",85,0],"ContextFetchBot":["partial","*",85,0]},"llms":{"status":"http_error","http":429,"bytes":64},"home":{"status":"ok","http":200,"url":"https://tucson.com/","html_tokens":102743,"markdown_tokens":937,"pruned_tokens":92}},
{"rank":5127,"domain":"libretexts.org","origin":"https://libretexts.org","robots":{"status":"ok","http":200,"url":"https://libretexts.org/robots.txt","bytes":2027,"groups":1},"bots":{"GPTBot":["partial","*",34,18],"ChatGPT-User":["partial","*",34,18],"OAI-SearchBot":["partial","*",34,18],"ClaudeBot":["partial","*",34,18],"Claude-User":["partial","*",34,18],"Claude-SearchBot":["partial","*",34,18],"anthropic-ai":["partial","*",34,18],"PerplexityBot":["partial","*",34,18],"Perplexity-User":["partial","*",34,18],"Google-Extended":["partial","*",34,18],"Applebot-Extended":["partial","*",34,18],"CCBot":["partial","*",34,18],"Bytespider":["partial","*",34,18],"Amazonbot":["partial","*",34,18],"meta-externalagent":["partial","*",34,18],"cohere-ai":["partial","*",34,18],"Diffbot":["partial","*",34,18],"ContextFetchBot":["partial","*",34,18]},"llms":{"status":"missing","http":404,"bytes":188},"home":{"status":"ok","http":200,"url":"https://libretexts.org/","html_tokens":22051,"markdown_tokens":3009,"pruned_tokens":528}},
{"rank":5128,"domain":"visme.co","origin":"https://visme.co","robots":{"status":"ok","http":200,"url":"https://visme.co/robots.txt","bytes":2791,"groups":7},"bots":{"GPTBot":["allowed","named",0,1],"ChatGPT-User":["partial","*",32,0],"OAI-SearchBot":["allowed","named",0,1],"ClaudeBot":["allowed","named",0,1],"Claude-User":["partial","*",32,0],"Claude-SearchBot":["partial","*",32,0],"anthropic-ai":["partial","*",32,0],"PerplexityBot":["allowed","named",0,1],"Perplexity-User":["partial","*",32,0],"Google-Extended":["partial","*",32,0],"Applebot-Extended":["partial","*",32,0],"CCBot":["allowed","named",0,1],"Bytespider":["partial","*",32,0],"Amazonbot":["allowed","named",0,1],"meta-externalagent":["partial","*",32,0],"cohere-ai":["partial","*",32,0],"Diffbot":["partial","*",32,0],"ContextFetchBot":["partial","*",32,0]},"llms":{"status":"present","http":200,"bytes":4053},"home":{"status":"ok","http":200,"url":"https://www.visme.co/","html_tokens":89139,"markdown_tokens":806,"pruned_tokens":103}},
{"rank":5129,"domain":"kick.com","origin":"https://kick.com","robots":{"status":"ok","http":200,"url":"https://kick.com/robots.txt","bytes":355,"groups":1},"bots":{"GPTBot":["partial","*",14,0],"ChatGPT-User":["partial","*",14,0],"OAI-SearchBot":["partial","*",14,0],"ClaudeBot":["partial","*",14,0],"Claude-User":["partial","*",14,0],"Claude-SearchBot":["partial","*",14,0],"anthropic-ai":["partial","*",14,0],"PerplexityBot":["partial","*",14,0],"Perplexity-User":["partial","*",14,0],"Google-Extended":["partial","*",14,0],"Applebot-Extended":["partial","*",14,0],"CCBot":["partial","*",14,0],"Bytespider":["partial","*",14,0],"Amazonbot":["partial","*",14,0],"meta-externalagent":["partial","*",14,0],"cohere-ai":["partial","*",14,0],"Diffbot":["partial","*",14,0],"ContextFetchBot":["partial","*",14,0]},"llms":{"status":"missing","http":404,"bytes":13285},"home":{"status":"ok","http":200,"url":"https://kick.com/","html_tokens":82937,"markdown_tokens":115,"pruned_tokens":115}},
{"rank":5130,"domain":"cfainstitute.org","origin":"https://cfainstitute.org","robots":{"status":"ok","http":200,"url":"https://www.cfainstitute.org/robots.txt","bytes":210,"groups":2},"bots":{"GPTBot":["partial","*",6,0],"ChatGPT-User":["partial","*",6,0],"OAI-SearchBot":["partial","*",6,0],"ClaudeBot":["partial","*",6,0],"Claude-User":["partial","*",6,0],"Claude-SearchBot":["partial","*",6,0],"anthropic-ai":["partial","*",6,0],"PerplexityBot":["partial","*",6,0],"Perplexity-User":["partial","*",6,0],"Google-Extended":["partial","*",6,0],"Applebot-Extended":["partial","*",6,0],"CCBot":["partial","*",6,0],"Bytespider":["partial","*",6,0],"Amazonbot":["partial","*",6,0],"meta-externalagent":["partial","*",6,0],"cohere-ai":["partial","*",6,0],"Diffbot":["partial","*",6,0],"ContextFetchBot":["partial","*",6,0]},"llms":{"status":"missing","http":404,"bytes":681},"home":{"status":"ok","http":200,"url":"https://www.cfainstitute.org/","html_tokens":120794,"markdown_tokens":407,"pruned_tokens":250}},
{"rank":5131,"domain":"tsukuba.ac.jp","origin":"https://www.tsukuba.ac.jp","robots":{"status":"ok","http":200,"url":"https://www.tsukuba.ac.jp/robots.txt","bytes":1306,"groups":1},"bots":{"GPTBot":["partial","*",3,21],"ChatGPT-User":["partial","*",3,21],"OAI-SearchBot":["partial","*",3,21],"ClaudeBot":["partial","*",3,21],"Claude-User":["partial","*",3,21],"Claude-SearchBot":["partial","*",3,21],"anthropic-ai":["partial","*",3,21],"PerplexityBot":["partial","*",3,21],"Perplexity-User":["partial","*",3,21],"Google-Extended":["partial","*",3,21],"Applebot-Extended":["partial","*",3,21],"CCBot":["partial","*",3,21],"Bytespider":["partial","*",3,21],"Amazonbot":["partial","*",3,21],"meta-externalagent":["partial","*",3,21],"cohere-ai":["partial","*",3,21],"Diffbot":["partial","*",3,21],"ContextFetchBot":["partial","*",3,21]},"llms":{"status":"missing","http":404,"bytes":185494},"home":{"status":"ok","http":200,"url":"https://www.tsukuba.ac.jp/","html_tokens":46337,"markdown_tokens":18,"pruned_tokens":18}},
{"rank":5132,"domain":"astronomy.com","origin":"https://astronomy.com","robots":{"status":"ok","http":200,"url":"https://www.astronomy.com/robots.txt","bytes":569,"groups":3},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"missing","http":404,"bytes":166787},"home":{"status":"ok","http":200,"url":"https://www.astronomy.com/","html_tokens":65651,"markdown_tokens":152,"pruned_tokens":25}},
{"rank":5133,"domain":"art.br","origin":"https://art.br","robots":{"status":"unreachable","http":null,"url":"https://art.br/robots.txt","bytes":0,"groups":0,"error":"cannot resolve art.br"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":5134,"domain":"alliedmarketresearch.com","origin":"https://alliedmarketresearch.com","robots":{"status":"ok","http":200,"url":"https://www.alliedmarketresearch.com/robots.txt","bytes":1856,"groups":20},"bots":{"GPTBot":["partial","*",42,0],"ChatGPT-User":["partial","*",42,0],"OAI-SearchBot":["partial","*",42,0],"ClaudeBot":["partial","*",42,0],"Claude-User":["partial","*",42,0],"Claude-SearchBot":["partial","*",42,0],"anthropic-ai":["partial","*",42,0],"PerplexityBot":["partial","*",42,0],"Perplexity-User":["partial","*",42,0],"Google-Extended":["partial","*",42,0],"Applebot-Extended":["partial","*",42,0],"CCBot":["partial","*",42,0],"Bytespider":["partial","*",42,0],"Amazonbot":["partial","*",42,0],"meta-externalagent":["partial","*",42,0],"cohere-ai":["partial","*",42,0],"Diffbot":["partial","*",42,0],"ContextFetchBot":["partial","*",42,0]},"llms":{"status":"present","http":200,"bytes":6771},"home":{"status":"http_error","http":403,"url":"https://alliedmarketresearch.com/"}},
{"rank":5135,"domain":"ilga.gov","origin":"https://ilga.gov","robots":{"status":"unreachable","http":null,"url":"https://ilga.gov/robots.txt","bytes":0,"groups":0,"error":"[SSL: CERTIFICATE_VERIFY_FAILED] certificate verify failed: unable to get local issuer certificate (_ssl.c:1028)"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":5136,"domain":"shaw.ca","origin":"https://shaw.ca","robots":{"status":"ok","http":200,"url":"https://www.rogers.com/robots.txt","bytes":2494,"groups":1},"bots":{"GPTBot":["partial","*",9,1],"ChatGPT-User":["partial","*",9,1],"OAI-SearchBot":["partial","*",9,1],"ClaudeBot":["partial","*",9,1],"Claude-User":["partial","*",9,1],"Claude-SearchBot":["partial","*",9,1],"anthropic-ai":["partial","*",9,1],"PerplexityBot":["partial","*",9,1],"Perplexity-User":["partial","*",9,1],"Google-Extended":["partial","*",9,1],"Applebot-Extended":["partial","*",9,1],"CCBot":["partial","*",9,1],"Bytespider":["partial","*",9,1],"Amazonbot":["partial","*",9,1],"meta-externalagent":["partial","*",9,1],"cohere-ai":["partial","*",9,1],"Diffbot":["partial","*",9,1],"ContextFetchBot":["partial","*",9,1]},"llms":{"status":"missing","http":404,"bytes":2339},"home":{"status":"ok","http":200,"url":"https://www.rogers.com/","html_tokens":63031,"markdown_tokens":244,"pruned_tokens":169}},
{"rank":5137,"domain":"quillbot.com","origin":"https://quillbot.com","robots":{"status":"refused","http":403,"url":"https://quillbot.com/robots.txt","bytes":5550,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":5138,"domain":"brandwatch.com","origin":"https://brandwatch.com","robots":{"status":"ok","http":200,"url":"https://brandwatch.com/robots.txt","bytes":1117,"groups":2},"bots":{"GPTBot":["partial","*",18,1],"ChatGPT-User":["partial","*",18,1],"OAI-SearchBot":["partial","*",18,1],"ClaudeBot":["partial","*",18,1],"Claude-User":["partial","*",18,1],"Claude-SearchBot":["partial","*",18,1],"anthropic-ai":["partial","*",18,1],"PerplexityBot":["partial","*",18,1],"Perplexity-User":["partial","*",18,1],"Google-Extended":["partial","*",18,1],"Applebot-Extended":["partial","*",18,1],"CCBot":["partial","*",18,1],"Bytespider":["partial","*",18,1],"Amazonbot":["partial","*",18,1],"meta-externalagent":["partial","*",18,1],"cohere-ai":["partial","*",18,1],"Diffbot":["partial","*",18,1],"ContextFetchBot":["partial","*",18,1]},"llms":{"status":"missing","http":404,"bytes":279295},"home":{"status":"ok","http":200,"url":"https://www.brandwatch.com/","html_tokens":153163,"markdown_tokens":262,"pruned_tokens":43}},
{"rank":5139,"domain":"ilfattoquotidiano.it","origin":"https://ilfattoquotidiano.it","robots":{"status":"ok","http":200,"url":"https://www.ilfattoquotidiano.it/robots.txt","bytes":1064,"groups":7},"bots":{"GPTBot":["partial","*",17,0],"ChatGPT-User":["partial","*",17,0],"OAI-SearchBot":["partial","*",17,0],"ClaudeBot":["partial","*",17,0],"Claude-User":["partial","*",17,0],"Claude-SearchBot":["partial","*",17,0],"anthropic-ai":["partial","*",17,0],"PerplexityBot":["partial","*",17,0],"Perplexity-User":["partial","*",17,0],"Google-Extended":["partial","*",17,0],"Applebot-Extended":["partial","*",17,0],"CCBot":["partial","*",17,0],"Bytespider":["partial","*",17,0],"Amazonbot":["partial","*",17,0],"meta-externalagent":["partial","*",17,0],"cohere-ai":["partial","*",17,0],"Diffbot":["partial","*",17,0],"ContextFetchBot":["partial","*",17,0]},"llms":{"status":"missing","http":404,"bytes":143777},"home":{"status":"ok","http":200,"url":"https://www.ilfattoquotidiano.it/","html_tokens":216865,"markdown_tokens":199,"pruned_tokens":138}},
{"rank":5140,"domain":"chewy.com","origin":"https://chewy.com","robots":{"status":"ok","http":200,"url":"https://www.chewy.com/robots.txt","bytes":2513,"groups":4},"bots":{"GPTBot":["partial","*",44,0],"ChatGPT-User":["partial","*",44,0],"OAI-SearchBot":["partial","*",44,0],"ClaudeBot":["partial","*",44,0],"Claude-User":["partial","*",44,0],"Claude-SearchBot":["partial","*",44,0],"anthropic-ai":["partial","*",44,0],"PerplexityBot":["partial","*",44,0],"Perplexity-User":["partial","*",44,0],"Google-Extended":["partial","*",44,0],"Applebot-Extended":["partial","*",44,0],"CCBot":["partial","*",44,0],"Bytespider":["partial","*",44,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["partial","*",44,0],"cohere-ai":["partial","*",44,0],"Diffbot":["partial","*",44,0],"ContextFetchBot":["partial","*",44,0]},"llms":{"status":"missing","http":404,"bytes":0},"home":{"status":"http_error","http":429,"url":"https://www.chewy.com/"}},
{"rank":5141,"domain":"intercontinental.com","origin":"https://intercontinental.com","robots":{"status":"refused","http":403,"url":"https://intercontinental.com/robots.txt","bytes":384,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":5142,"domain":"jci.org","origin":"https://jci.org","robots":{"status":"ok","http":200,"url":"https://jci.org/robots.txt","bytes":67,"groups":1},"bots":{"GPTBot":["partial","*",1,0],"ChatGPT-User":["partial","*",1,0],"OAI-SearchBot":["partial","*",1,0],"ClaudeBot":["partial","*",1,0],"Claude-User":["partial","*",1,0],"Claude-SearchBot":["partial","*",1,0],"anthropic-ai":["partial","*",1,0],"PerplexityBot":["partial","*",1,0],"Perplexity-User":["partial","*",1,0],"Google-Extended":["partial","*",1,0],"Applebot-Extended":["partial","*",1,0],"CCBot":["partial","*",1,0],"Bytespider":["partial","*",1,0],"Amazonbot":["partial","*",1,0],"meta-externalagent":["partial","*",1,0],"cohere-ai":["partial","*",1,0],"Diffbot":["partial","*",1,0],"ContextFetchBot":["partial","*",1,0]},"llms":{"status":"timeout","error":"timed out after 10 s"},"home":{"status":"ok","http":200,"url":"https://jci.org/","html_tokens":50066,"markdown_tokens":2280,"pruned_tokens":100}},
{"rank":5143,"domain":"rappler.com","origin":"https://rappler.com","robots":{"status":"ok","http":200,"url":"https://www.rappler.com/robots.txt","bytes":1498,"groups":36},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["blocked","named",1,0],"OAI-SearchBot":["blocked","named",1,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["blocked","named",1,0],"Claude-SearchBot":["blocked","named",1,0],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["blocked","named",1,0],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",2,0]},"llms":{"status":"missing","http":404,"bytes":153},"home":{"status":"ok","http":200,"url":"https://www.rappler.com/","html_tokens":201288,"markdown_tokens":54,"pruned_tokens":53}},
{"rank":5144,"domain":"ottawacitizen.com","origin":"https://ottawacitizen.com","robots":{"status":"ok","http":200,"url":"https://ottawacitizen.com/robots.txt","bytes":1233,"groups":15},"bots":{"GPTBot":["partial","*",3,6],"ChatGPT-User":["partial","*",3,6],"OAI-SearchBot":["partial","*",3,6],"ClaudeBot":["partial","*",3,6],"Claude-User":["partial","*",3,6],"Claude-SearchBot":["partial","*",3,6],"anthropic-ai":["partial","*",3,6],"PerplexityBot":["partial","*",3,6],"Perplexity-User":["partial","*",3,6],"Google-Extended":["partial","*",3,6],"Applebot-Extended":["partial","*",3,6],"CCBot":["partial","*",3,6],"Bytespider":["partial","*",3,6],"Amazonbot":["partial","*",3,6],"meta-externalagent":["partial","*",3,6],"cohere-ai":["partial","*",3,6],"Diffbot":["partial","*",3,6],"ContextFetchBot":["partial","*",3,6]},"llms":{"status":"missing","http":404,"bytes":245512},"home":{"status":"ok","http":200,"url":"https://ottawacitizen.com/","html_tokens":154510,"markdown_tokens":239,"pruned_tokens":122}},
{"rank":5145,"domain":"dti.ne.jp","origin":"https://dti.ne.jp","robots":{"status":"unreachable","http":null,"url":"https://dti.ne.jp/robots.txt","bytes":0,"groups":0,"error":"cannot resolve dti.ne.jp"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":5146,"domain":"committees.parliament.uk","origin":"https://committees.parliament.uk","robots":{"status":"refused","http":403,"url":"https://committees.parliament.uk/robots.txt","bytes":5562,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":5147,"domain":"casetext.com","origin":"https://casetext.com","robots":{"status":"missing","http":410,"url":"https://casetext.com/robots.txt","bytes":640,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":410,"bytes":640},"home":{"status":"ok","http":200,"url":"https://www.thomsonreuters.com/en/cocounsel","html_tokens":111236,"markdown_tokens":1400,"pruned_tokens":158}},
{"rank":5148,"domain":"newswise.com","origin":"https://newswise.com","robots":{"status":"ok","http":200,"url":"https://www.newswise.com/robots.txt","bytes":2654,"groups":22},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["partial","*",18,5],"OAI-SearchBot":["partial","*",18,5],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",18,5],"Claude-SearchBot":["partial","*",18,5],"anthropic-ai":["partial","*",18,5],"PerplexityBot":["partial","*",18,5],"Perplexity-User":["partial","*",18,5],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["partial","*",18,5],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",2,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["partial","*",18,5],"Diffbot":["partial","*",18,5],"ContextFetchBot":["partial","*",18,5]},"llms":{"status":"timeout","error":"timed out after 10 s"},"home":{"status":"timeout","error":"timed out after 15 s"}},
{"rank":5149,"domain":"ntlworld.com","origin":"http://ntlworld.com","robots":{"status":"html","http":200,"url":"https://www.virginmedia.com/","bytes":689791,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":5150,"domain":"bmwi.de","origin":"https://www.bmwi.de","robots":{"status":"missing","http":404,"url":"https://www.bundeswirtschaftsministerium.de/robots.txt","bytes":133493,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":133493},"home":{"status":"ok","http":200,"url":"https://www.bundeswirtschaftsministerium.de/Navigation/DE/Home/home.html","html_tokens":29603,"markdown_tokens":13,"pruned_tokens":13}},
{"rank":5151,"domain":"agilemanifesto.org","origin":"https://agilemanifesto.org","robots":{"status":"missing","http":404,"url":"https://agilemanifesto.org/robots.txt","bytes":9379,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":9379},"home":{"status":"ok","http":200,"url":"https://agilemanifesto.org/","html_tokens":1547,"markdown_tokens":170,"pruned_tokens":169}},
{"rank":5152,"domain":"jamendo.com","origin":"https://jamendo.com","robots":{"status":"ok","http":200,"url":"https://www.jamendo.com/robots.txt","bytes":98,"groups":1},"bots":{"GPTBot":["partial","*",4,0],"ChatGPT-User":["partial","*",4,0],"OAI-SearchBot":["partial","*",4,0],"ClaudeBot":["partial","*",4,0],"Claude-User":["partial","*",4,0],"Claude-SearchBot":["partial","*",4,0],"anthropic-ai":["partial","*",4,0],"PerplexityBot":["partial","*",4,0],"Perplexity-User":["partial","*",4,0],"Google-Extended":["partial","*",4,0],"Applebot-Extended":["partial","*",4,0],"CCBot":["partial","*",4,0],"Bytespider":["partial","*",4,0],"Amazonbot":["partial","*",4,0],"meta-externalagent":["partial","*",4,0],"cohere-ai":["partial","*",4,0],"Diffbot":["partial","*",4,0],"ContextFetchBot":["partial","*",4,0]},"llms":{"status":"missing","http":404,"bytes":29836},"home":{"status":"ok","http":200,"url":"https://www.jamendo.com/","html_tokens":8290,"markdown_tokens":0,"pruned_tokens":0}},
{"rank":5153,"domain":"lycos.fr","origin":"https://lycos.fr","robots":{"status":"ok","http":200,"url":"https://lycos.fr/robots.txt","bytes":1248,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":7051},"home":{"status":"ok","http":200,"url":"https://lycos.fr/","html_tokens":6919,"markdown_tokens":172,"pruned_tokens":172}},
{"rank":5154,"domain":"caa.go.jp","origin":"https://www.caa.go.jp","robots":{"status":"ok","http":200,"url":"https://www.caa.go.jp/robots.txt","bytes":94,"groups":2},"bots":{"GPTBot":["partial","*",1,0],"ChatGPT-User":["partial","*",1,0],"OAI-SearchBot":["partial","*",1,0],"ClaudeBot":["partial","*",1,0],"Claude-User":["partial","*",1,0],"Claude-SearchBot":["partial","*",1,0],"anthropic-ai":["partial","*",1,0],"PerplexityBot":["partial","*",1,0],"Perplexity-User":["partial","*",1,0],"Google-Extended":["partial","*",1,0],"Applebot-Extended":["partial","*",1,0],"CCBot":["partial","*",1,0],"Bytespider":["partial","*",1,0],"Amazonbot":["partial","*",1,0],"meta-externalagent":["partial","*",1,0],"cohere-ai":["partial","*",1,0],"Diffbot":["partial","*",1,0],"ContextFetchBot":["partial","*",1,0]},"llms":{"status":"http_error","http":403,"bytes":40597},"home":{"status":"ok","http":200,"url":"https://www.caa.go.jp/","html_tokens":12216,"markdown_tokens":29,"pruned_tokens":29}},
{"rank":5155,"domain":"uni-jena.de","origin":"https://uni-jena.de","robots":{"status":"ok","http":200,"url":"https://www.uni-jena.de/robots.txt","bytes":661,"groups":1},"bots":{"GPTBot":["partial","*",8,13],"ChatGPT-User":["partial","*",8,13],"OAI-SearchBot":["partial","*",8,13],"ClaudeBot":["partial","*",8,13],"Claude-User":["partial","*",8,13],"Claude-SearchBot":["partial","*",8,13],"anthropic-ai":["partial","*",8,13],"PerplexityBot":["partial","*",8,13],"Perplexity-User":["partial","*",8,13],"Google-Extended":["partial","*",8,13],"Applebot-Extended":["partial","*",8,13],"CCBot":["partial","*",8,13],"Bytespider":["partial","*",8,13],"Amazonbot":["partial","*",8,13],"meta-externalagent":["partial","*",8,13],"cohere-ai":["partial","*",8,13],"Diffbot":["partial","*",8,13],"ContextFetchBot":["partial","*",8,13]},"llms":{"status":"missing","http":404,"bytes":29784},"home":{"status":"ok","http":200,"url":"https://www.uni-jena.de/","html_tokens":36600,"markdown_tokens":191,"pruned_tokens":154}},
{"rank":5156,"domain":"kroger.com","origin":"https://kroger.com","robots":{"status":"unreachable","http":null,"url":"https://kroger.com/robots.txt","bytes":0,"groups":0,"error":"<StreamReset stream_id:1, error_code:2, remote_reset:True>"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":5157,"domain":"thumbtack.com","origin":"https://thumbtack.com","robots":{"status":"ok","http":200,"url":"https://www.thumbtack.com/robots.txt","bytes":2477,"groups":1},"bots":{"GPTBot":["partial","*",32,3],"ChatGPT-User":["partial","*",32,3],"OAI-SearchBot":["partial","*",32,3],"ClaudeBot":["partial","*",32,3],"Claude-User":["partial","*",32,3],"Claude-SearchBot":["partial","*",32,3],"anthropic-ai":["partial","*",32,3],"PerplexityBot":["partial","*",32,3],"Perplexity-User":["partial","*",32,3],"Google-Extended":["partial","*",32,3],"Applebot-Extended":["partial","*",32,3],"CCBot":["partial","*",32,3],"Bytespider":["partial","*",32,3],"Amazonbot":["partial","*",32,3],"meta-externalagent":["partial","*",32,3],"cohere-ai":["partial","*",32,3],"Diffbot":["partial","*",32,3],"ContextFetchBot":["partial","*",32,3]},"llms":{"status":"missing","http":404,"bytes":20046},"home":{"status":"ok","http":200,"url":"https://www.thumbtack.com/","html_tokens":50587,"markdown_tokens":128,"pruned_tokens":35}},
{"rank":5158,"domain":"php-fig.org","origin":"https://php-fig.org","robots":{"status":"missing","http":404,"url":"https://www.php-fig.org/robots.txt","bytes":684,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":684},"home":{"status":"ok","http":200,"url":"https://www.php-fig.org/","html_tokens":3834,"markdown_tokens":126,"pruned_tokens":103}},
{"rank":5159,"domain":"lazada.co.id","origin":"https://lazada.co.id","robots":{"status":"ok","http":200,"url":"https://www.lazada.co.id/robots.txt","bytes":153,"groups":2},"bots":{"GPTBot":["partial","*",4,0],"ChatGPT-User":["partial","*",4,0],"OAI-SearchBot":["partial","*",4,0],"ClaudeBot":["partial","*",4,0],"Claude-User":["partial","*",4,0],"Claude-SearchBot":["partial","*",4,0],"anthropic-ai":["partial","*",4,0],"PerplexityBot":["partial","*",4,0],"Perplexity-User":["partial","*",4,0],"Google-Extended":["partial","*",4,0],"Applebot-Extended":["partial","*",4,0],"CCBot":["partial","*",4,0],"Bytespider":["partial","*",4,0],"Amazonbot":["partial","*",4,0],"meta-externalagent":["partial","*",4,0],"cohere-ai":["partial","*",4,0],"Diffbot":["partial","*",4,0],"ContextFetchBot":["partial","*",4,0]},"llms":{"status":"html","http":200,"bytes":81988},"home":{"status":"ok","http":200,"url":"https://www.lazada.co.id/","html_tokens":111056,"markdown_tokens":774,"pruned_tokens":147}},
{"rank":5160,"domain":"city-data.com","origin":"https://city-data.com","robots":{"status":"ok","http":200,"url":"https://www.city-data.com/robots.txt","bytes":2376,"groups":19},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["partial","*",30,0],"OAI-SearchBot":["partial","*",30,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",30,0],"Claude-SearchBot":["partial","*",30,0],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["partial","*",30,0],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["partial","*",30,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",30,0]},"llms":{"status":"missing","http":404,"bytes":108662},"home":{"status":"ok","http":200,"url":"https://www.city-data.com/","html_tokens":153017,"markdown_tokens":1086,"pruned_tokens":434}},
{"rank":5161,"domain":"td.com","origin":"https://td.com","robots":{"status":"ok","http":200,"url":"https://www.td.com/robots.txt","bytes":364,"groups":4},"bots":{"GPTBot":["allowed","*",0,1],"ChatGPT-User":["allowed","*",0,1],"OAI-SearchBot":["allowed","*",0,1],"ClaudeBot":["allowed","*",0,1],"Claude-User":["allowed","*",0,1],"Claude-SearchBot":["allowed","*",0,1],"anthropic-ai":["allowed","*",0,1],"PerplexityBot":["allowed","*",0,1],"Perplexity-User":["allowed","*",0,1],"Google-Extended":["allowed","*",0,1],"Applebot-Extended":["allowed","*",0,1],"CCBot":["allowed","*",0,1],"Bytespider":["allowed","*",0,1],"Amazonbot":["allowed","*",0,1],"meta-externalagent":["allowed","*",0,1],"cohere-ai":["allowed","*",0,1],"Diffbot":["allowed","*",0,1],"ContextFetchBot":["allowed","*",0,1]},"llms":{"status":"present","http":200,"bytes":7318},"home":{"status":"ok","http":200,"url":"https://www.td.com/us/en/personal-banking","html_tokens":60249,"markdown_tokens":123,"pruned_tokens":50}},
{"rank":5162,"domain":"latercera.com","origin":"https://latercera.com","robots":{"status":"ok","http":200,"url":"https://www.latercera.com/robots.txt","bytes":318,"groups":1},"bots":{"GPTBot":["partial","*",2,1],"ChatGPT-User":["partial","*",2,1],"OAI-SearchBot":["partial","*",2,1],"ClaudeBot":["partial","*",2,1],"Claude-User":["partial","*",2,1],"Claude-SearchBot":["partial","*",2,1],"anthropic-ai":["partial","*",2,1],"PerplexityBot":["partial","*",2,1],"Perplexity-User":["partial","*",2,1],"Google-Extended":["partial","*",2,1],"Applebot-Extended":["partial","*",2,1],"CCBot":["partial","*",2,1],"Bytespider":["partial","*",2,1],"Amazonbot":["partial","*",2,1],"meta-externalagent":["partial","*",2,1],"cohere-ai":["partial","*",2,1],"Diffbot":["partial","*",2,1],"ContextFetchBot":["partial","*",2,1]},"llms":{"status":"missing","http":404,"bytes":141381},"home":{"status":"ok","http":200,"url":"https://www.latercera.com/","html_tokens":308973,"markdown_tokens":261,"pruned_tokens":107}},
{"rank":5163,"domain":"pchome.com.tw","origin":"https://pchome.com.tw","robots":{"status":"ok","http":200,"url":"https://www.pchome.com.tw/robots.txt","bytes":69,"groups":1},"bots":{"GPTBot":["allowed","*",0,1],"ChatGPT-User":["allowed","*",0,1],"OAI-SearchBot":["allowed","*",0,1],"ClaudeBot":["allowed","*",0,1],"Claude-User":["allowed","*",0,1],"Claude-SearchBot":["allowed","*",0,1],"anthropic-ai":["allowed","*",0,1],"PerplexityBot":["allowed","*",0,1],"Perplexity-User":["allowed","*",0,1],"Google-Extended":["allowed","*",0,1],"Applebot-Extended":["allowed","*",0,1],"CCBot":["allowed","*",0,1],"Bytespider":["allowed","*",0,1],"Amazonbot":["allowed","*",0,1],"meta-externalagent":["allowed","*",0,1],"cohere-ai":["allowed","*",0,1],"Diffbot":["allowed","*",0,1],"ContextFetchBot":["allowed","*",0,1]},"llms":{"status":"html","http":200,"bytes":256829},"home":{"status":"ok","http":200,"url":"https://www.pchome.com.tw/","html_tokens":58837,"markdown_tokens":50,"pruned_tokens":50}},
{"rank":5164,"domain":"kimi.com","origin":"https://kimi.com","robots":{"status":"ok","http":200,"url":"https://www.kimi.com/robots.txt","bytes":278,"groups":1},"bots":{"GPTBot":["partial","*",5,2],"ChatGPT-User":["partial","*",5,2],"OAI-SearchBot":["partial","*",5,2],"ClaudeBot":["partial","*",5,2],"Claude-User":["partial","*",5,2],"Claude-SearchBot":["partial","*",5,2],"anthropic-ai":["partial","*",5,2],"PerplexityBot":["partial","*",5,2],"Perplexity-User":["partial","*",5,2],"Google-Extended":["partial","*",5,2],"Applebot-Extended":["partial","*",5,2],"CCBot":["partial","*",5,2],"Bytespider":["partial","*",5,2],"Amazonbot":["partial","*",5,2],"meta-externalagent":["partial","*",5,2],"cohere-ai":["partial","*",5,2],"Diffbot":["partial","*",5,2],"ContextFetchBot":["partial","*",5,2]},"llms":{"status":"html","http":200,"bytes":143100},"home":{"status":"ok","http":200,"url":"https://www.kimi.com/","html_tokens":35773,"markdown_tokens":8,"pruned_tokens":8}},
{"rank":5165,"domain":"amazon.jobs","origin":"https://amazon.jobs","robots":{"status":"ok","http":200,"url":"https://amazon.jobs/robots.txt","bytes":324,"groups":2},"bots":{"GPTBot":["partial","*",12,0],"ChatGPT-User":["partial","*",12,0],"OAI-SearchBot":["partial","*",12,0],"ClaudeBot":["partial","*",12,0],"Claude-User":["partial","*",12,0],"Claude-SearchBot":["partial","*",12,0],"anthropic-ai":["partial","*",12,0],"PerplexityBot":["partial","*",12,0],"Perplexity-User":["partial","*",12,0],"Google-Extended":["partial","*",12,0],"Applebot-Extended":["partial","*",12,0],"CCBot":["partial","*",12,0],"Bytespider":["partial","*",12,0],"Amazonbot":["partial","*",12,0],"meta-externalagent":["partial","*",12,0],"cohere-ai":["partial","*",12,0],"Diffbot":["partial","*",12,0],"ContextFetchBot":["partial","*",12,0]},"llms":{"status":"missing","http":404,"bytes":0},"home":{"status":"ok","http":200,"url":"https://amazon.jobs/en/","html_tokens":8266,"markdown_tokens":110,"pruned_tokens":110}},
{"rank":5166,"domain":"crates.io","origin":"https://crates.io","robots":{"status":"ok","http":200,"url":"https://crates.io/robots.txt","bytes":63,"groups":1},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"html","http":200,"bytes":5399},"home":{"status":"ok","http":200,"url":"https://crates.io/","html_tokens":1350,"markdown_tokens":19,"pruned_tokens":19}},
{"rank":5167,"domain":"randomhouse.com","origin":"https://randomhouse.com","robots":{"status":"ok","http":200,"url":"http://www.randomhouse.com/robots.txt","bytes":8303,"groups":1},"bots":{"GPTBot":["partial","*",181,1],"ChatGPT-User":["partial","*",181,1],"OAI-SearchBot":["partial","*",181,1],"ClaudeBot":["partial","*",181,1],"Claude-User":["partial","*",181,1],"Claude-SearchBot":["partial","*",181,1],"anthropic-ai":["partial","*",181,1],"PerplexityBot":["partial","*",181,1],"Perplexity-User":["partial","*",181,1],"Google-Extended":["partial","*",181,1],"Applebot-Extended":["partial","*",181,1],"CCBot":["partial","*",181,1],"Bytespider":["partial","*",181,1],"Amazonbot":["partial","*",181,1],"meta-externalagent":["partial","*",181,1],"cohere-ai":["partial","*",181,1],"Diffbot":["partial","*",181,1],"ContextFetchBot":["partial","*",181,1]},"llms":{"status":"missing","http":404,"bytes":19462},"home":{"status":"ok","http":200,"url":"https://www.penguinrandomhouse.com/","html_tokens":57880,"markdown_tokens":11,"pruned_tokens":11}},
{"rank":5168,"domain":"fontsquirrel.com","origin":"https://fontsquirrel.com","robots":{"status":"html","http":202,"url":"https://www.fontsquirrel.com/robots.txt","bytes":2475,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":5169,"domain":"leidenuniv.nl","origin":"https://leidenuniv.nl","robots":{"status":"html","http":200,"url":"https://www.universiteitleiden.nl/robots.txt","bytes":21994,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":5170,"domain":"yr.no","origin":"https://yr.no","robots":{"status":"ok","http":200,"url":"https://www.yr.no/robots.txt","bytes":984,"groups":24},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["blocked","named",1,0],"OAI-SearchBot":["blocked","named",1,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",4,0],"Claude-SearchBot":["partial","*",4,0],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["partial","*",4,0],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",4,0]},"llms":{"status":"missing","http":404,"bytes":91881},"home":{"status":"ok","http":200,"url":"https://www.yr.no/nb","html_tokens":73017,"markdown_tokens":247,"pruned_tokens":73}},
{"rank":5171,"domain":"delltechnologies.com","origin":"https://delltechnologies.com","robots":{"status":"refused","http":403,"url":"https://www.delltechnologies.com/robots.txt","bytes":274,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":5172,"domain":"vlaanderen.be","origin":"https://vlaanderen.be","robots":{"status":"refused","http":403,"url":"https://vlaanderen.be/robots.txt","bytes":573,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":5173,"domain":"kohm.ru","origin":"https://kohm.ru","robots":{"status":"ok","http":200,"url":"https://kohm.ru/robots.txt","bytes":119,"groups":1},"bots":{"GPTBot":["partial","*",2,1],"ChatGPT-User":["partial","*",2,1],"OAI-SearchBot":["partial","*",2,1],"ClaudeBot":["partial","*",2,1],"Claude-User":["partial","*",2,1],"Claude-SearchBot":["partial","*",2,1],"anthropic-ai":["partial","*",2,1],"PerplexityBot":["partial","*",2,1],"Perplexity-User":["partial","*",2,1],"Google-Extended":["partial","*",2,1],"Applebot-Extended":["partial","*",2,1],"CCBot":["partial","*",2,1],"Bytespider":["partial","*",2,1],"Amazonbot":["partial","*",2,1],"meta-externalagent":["partial","*",2,1],"cohere-ai":["partial","*",2,1],"Diffbot":["partial","*",2,1],"ContextFetchBot":["partial","*",2,1]},"llms":{"status":"html","http":200,"bytes":39635},"home":{"status":"ok","http":200,"url":"https://www.foxchasefarm.org/","html_tokens":7807,"markdown_tokens":2548,"pruned_tokens":144}},
{"rank":5174,"domain":"businessinsider.de","origin":"https://businessinsider.de","robots":{"status":"ok","http":200,"url":"https://www.businessinsider.de/robots.txt","bytes":2603,"groups":4},"bots":{"GPTBot":["allowed","named",0,1],"ChatGPT-User":["partial","*",2,1],"OAI-SearchBot":["partial","*",2,1],"ClaudeBot":["partial","*",2,1],"Claude-User":["partial","*",2,1],"Claude-SearchBot":["partial","*",2,1],"anthropic-ai":["partial","*",2,1],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["blocked","named",1,0],"Google-Extended":["partial","*",2,1],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["partial","*",2,1],"meta-externalagent":["partial","*",2,1],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",2,1]},"llms":{"status":"missing","http":404,"bytes":379075},"home":{"status":"ok","http":200,"url":"https://www.businessinsider.de/","html_tokens":129140,"markdown_tokens":135,"pruned_tokens":135}},
{"rank":5175,"domain":"cryptonews.com","origin":"https://cryptonews.com","robots":{"status":"ok","http":200,"url":"https://cryptonews.com/robots.txt","bytes":2113,"groups":1},"bots":{"GPTBot":["partial","*",51,0],"ChatGPT-User":["partial","*",51,0],"OAI-SearchBot":["partial","*",51,0],"ClaudeBot":["partial","*",51,0],"Claude-User":["partial","*",51,0],"Claude-SearchBot":["partial","*",51,0],"anthropic-ai":["partial","*",51,0],"PerplexityBot":["partial","*",51,0],"Perplexity-User":["partial","*",51,0],"Google-Extended":["partial","*",51,0],"Applebot-Extended":["partial","*",51,0],"CCBot":["partial","*",51,0],"Bytespider":["partial","*",51,0],"Amazonbot":["partial","*",51,0],"meta-externalagent":["partial","*",51,0],"cohere-ai":["partial","*",51,0],"Diffbot":["partial","*",51,0],"ContextFetchBot":["partial","*",51,0]},"llms":{"status":"missing","http":404,"bytes":170165},"home":{"status":"ok","http":200,"url":"https://cryptonews.com/","html_tokens":88700,"markdown_tokens":45,"pruned_tokens":45}},
{"rank":5176,"domain":"the-independent.com","origin":"https://the-independent.com","robots":{"status":"ok","http":200,"url":"https://www.the-independent.com/robots.txt","bytes":1368,"groups":2},"bots":{"GPTBot":["partial","*",43,0],"ChatGPT-User":["partial","*",43,0],"OAI-SearchBot":["partial","*",43,0],"ClaudeBot":["partial","*",43,0],"Claude-User":["partial","*",43,0],"Claude-SearchBot":["partial","*",43,0],"anthropic-ai":["partial","*",43,0],"PerplexityBot":["partial","*",43,0],"Perplexity-User":["partial","*",43,0],"Google-Extended":["partial","*",43,0],"Applebot-Extended":["partial","*",43,0],"CCBot":["partial","*",43,0],"Bytespider":["partial","*",43,0],"Amazonbot":["partial","*",43,0],"meta-externalagent":["partial","*",43,0],"cohere-ai":["partial","*",43,0],"Diffbot":["partial","*",43,0],"ContextFetchBot":["partial","*",43,0]},"llms":{"status":"missing","http":410,"bytes":410},"home":{"status":"ok","http":200,"url":"https://www.the-independent.com/","html_tokens":86481,"markdown_tokens":34,"pruned_tokens":34}},
{"rank":5177,"domain":"comparitech.com","origin":"https://comparitech.com","robots":{"status":"ok","http":200,"url":"https://www.comparitech.com/robots.txt","bytes":8786,"groups":30},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["partial","named",6,0],"OAI-SearchBot":["partial","named",6,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","named",6,0],"Claude-SearchBot":["partial","named",6,0],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["partial","named",6,0],"Perplexity-User":["partial","named",6,0],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["partial","*",6,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",6,0]},"llms":{"status":"present","http":200,"bytes":18655},"home":{"status":"ok","http":200,"url":"https://www.comparitech.com/","html_tokens":72650,"markdown_tokens":20,"pruned_tokens":20}},
{"rank":5178,"domain":"southcn.com","origin":"https://southcn.com","robots":{"status":"html","http":200,"url":"https://www.southcn.com","bytes":287815,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":5179,"domain":"bethesda.net","origin":"https://bethesda.net","robots":{"status":"ok","http":200,"url":"https://bethesda.net/robots.txt","bytes":128,"groups":1},"bots":{"GPTBot":["partial","*",3,0],"ChatGPT-User":["partial","*",3,0],"OAI-SearchBot":["partial","*",3,0],"ClaudeBot":["partial","*",3,0],"Claude-User":["partial","*",3,0],"Claude-SearchBot":["partial","*",3,0],"anthropic-ai":["partial","*",3,0],"PerplexityBot":["partial","*",3,0],"Perplexity-User":["partial","*",3,0],"Google-Extended":["partial","*",3,0],"Applebot-Extended":["partial","*",3,0],"CCBot":["partial","*",3,0],"Bytespider":["partial","*",3,0],"Amazonbot":["partial","*",3,0],"meta-externalagent":["partial","*",3,0],"cohere-ai":["partial","*",3,0],"Diffbot":["partial","*",3,0],"ContextFetchBot":["partial","*",3,0]},"llms":{"status":"missing","http":404,"bytes":64293},"home":{"status":"ok","http":200,"url":"https://bethesda.net/en-US","html_tokens":64221,"markdown_tokens":479,"pruned_tokens":69}},
{"rank":5180,"domain":"dingtalk.com","origin":"https://dingtalk.com","robots":{"status":"ok","http":200,"url":"https://www.dingtalk.com/robots.txt","bytes":829,"groups":2},"bots":{"GPTBot":["partial","named",1,0],"ChatGPT-User":["partial","*",14,0],"OAI-SearchBot":["partial","*",14,0],"ClaudeBot":["partial","named",1,0],"Claude-User":["partial","*",14,0],"Claude-SearchBot":["partial","*",14,0],"anthropic-ai":["partial","*",14,0],"PerplexityBot":["partial","*",14,0],"Perplexity-User":["partial","*",14,0],"Google-Extended":["partial","*",14,0],"Applebot-Extended":["partial","*",14,0],"CCBot":["partial","named",1,0],"Bytespider":["partial","*",14,0],"Amazonbot":["partial","*",14,0],"meta-externalagent":["partial","*",14,0],"cohere-ai":["partial","*",14,0],"Diffbot":["partial","*",14,0],"ContextFetchBot":["partial","*",14,0]},"llms":{"status":"html","http":200,"bytes":50295},"home":{"status":"ok","http":200,"url":"https://www.dingtalk.com/","html_tokens":12316,"markdown_tokens":27,"pruned_tokens":27}},
{"rank":5181,"domain":"topgear.com","origin":"https://topgear.com","robots":{"status":"refused","http":403,"url":"https://www.topgear.com/robots.txt","bytes":381,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":5182,"domain":"gminsights.com","origin":"https://gminsights.com","robots":{"status":"ok","http":200,"url":"https://www.gminsights.com/robots.txt","bytes":297,"groups":2},"bots":{"GPTBot":["partial","*",6,1],"ChatGPT-User":["partial","*",6,1],"OAI-SearchBot":["partial","*",6,1],"ClaudeBot":["partial","*",6,1],"Claude-User":["partial","*",6,1],"Claude-SearchBot":["partial","*",6,1],"anthropic-ai":["partial","*",6,1],"PerplexityBot":["partial","*",6,1],"Perplexity-User":["partial","*",6,1],"Google-Extended":["allowed","named",0,1],"Applebot-Extended":["partial","*",6,1],"CCBot":["partial","*",6,1],"Bytespider":["partial","*",6,1],"Amazonbot":["partial","*",6,1],"meta-externalagent":["partial","*",6,1],"cohere-ai":["partial","*",6,1],"Diffbot":["partial","*",6,1],"ContextFetchBot":["partial","*",6,1]},"llms":{"status":"missing","http":404,"bytes":26593},"home":{"status":"ok","http":200,"url":"https://www.gminsights.com/","html_tokens":62984,"markdown_tokens":1349,"pruned_tokens":118}},
{"rank":5183,"domain":"unlp.edu.ar","origin":"https://unlp.edu.ar","robots":{"status":"ok","http":200,"url":"https://unlp.edu.ar/robots.txt","bytes":155,"groups":1},"bots":{"GPTBot":["partial","*",1,1],"ChatGPT-User":["partial","*",1,1],"OAI-SearchBot":["partial","*",1,1],"ClaudeBot":["partial","*",1,1],"Claude-User":["partial","*",1,1],"Claude-SearchBot":["partial","*",1,1],"anthropic-ai":["partial","*",1,1],"PerplexityBot":["partial","*",1,1],"Perplexity-User":["partial","*",1,1],"Google-Extended":["partial","*",1,1],"Applebot-Extended":["partial","*",1,1],"CCBot":["partial","*",1,1],"Bytespider":["partial","*",1,1],"Amazonbot":["partial","*",1,1],"meta-externalagent":["partial","*",1,1],"cohere-ai":["partial","*",1,1],"Diffbot":["partial","*",1,1],"ContextFetchBot":["partial","*",1,1]},"llms":{"status":"missing","http":404,"bytes":146779},"home":{"status":"ok","http":200,"url":"https://unlp.edu.ar/","html_tokens":80587,"markdown_tokens":138,"pruned_tokens":138}},
{"rank":5184,"domain":"google.com.sa","origin":"https://google.com.sa","robots":{"status":"ok","http":200,"url":"https://www.google.com.sa/robots.txt","bytes":6624,"groups":4},"bots":{"GPTBot":["partial","*",169,64],"ChatGPT-User":["partial","*",169,64],"OAI-SearchBot":["partial","*",169,64],"ClaudeBot":["partial","*",169,64],"Claude-User":["partial","*",169,64],"Claude-SearchBot":["partial","*",169,64],"anthropic-ai":["partial","*",169,64],"PerplexityBot":["partial","*",169,64],"Perplexity-User":["partial","*",169,64],"Google-Extended":["partial","*",169,64],"Applebot-Extended":["partial","*",169,64],"CCBot":["partial","*",169,64],"Bytespider":["partial","*",169,64],"Amazonbot":["partial","*",169,64],"meta-externalagent":["partial","*",169,64],"cohere-ai":["partial","*",169,64],"Diffbot":["partial","*",169,64],"ContextFetchBot":["partial","*",169,64]},"llms":{"status":"missing","http":404,"bytes":1569},"home":{"status":"ok","http":200,"url":"https://www.google.com/","html_tokens":21291,"markdown_tokens":30,"pruned_tokens":30}},
{"rank":5185,"domain":"unhabitat.org","origin":"https://unhabitat.org","robots":{"status":"refused","http":403,"url":"https://unhabitat.org/robots.txt","bytes":560,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":5186,"domain":"mun.ca","origin":"https://mun.ca","robots":{"status":"ok","http":200,"url":"https://mun.ca/robots.txt","bytes":89,"groups":1},"bots":{"GPTBot":["partial","*",3,0],"ChatGPT-User":["partial","*",3,0],"OAI-SearchBot":["partial","*",3,0],"ClaudeBot":["partial","*",3,0],"Claude-User":["partial","*",3,0],"Claude-SearchBot":["partial","*",3,0],"anthropic-ai":["partial","*",3,0],"PerplexityBot":["partial","*",3,0],"Perplexity-User":["partial","*",3,0],"Google-Extended":["partial","*",3,0],"Applebot-Extended":["partial","*",3,0],"CCBot":["partial","*",3,0],"Bytespider":["partial","*",3,0],"Amazonbot":["partial","*",3,0],"meta-externalagent":["partial","*",3,0],"cohere-ai":["partial","*",3,0],"Diffbot":["partial","*",3,0],"ContextFetchBot":["partial","*",3,0]},"llms":{"status":"missing","http":404,"bytes":26516},"home":{"status":"ok","http":200,"url":"https://mun.ca/","html_tokens":9740,"markdown_tokens":65,"pruned_tokens":65}},
{"rank":5187,"domain":"brennancenter.org","origin":"https://brennancenter.org","robots":{"status":"ok","http":200,"url":"https://www.brennancenter.org/robots.txt","bytes":1804,"groups":1},"bots":{"GPTBot":["partial","*",30,18],"ChatGPT-User":["partial","*",30,18],"OAI-SearchBot":["partial","*",30,18],"ClaudeBot":["partial","*",30,18],"Claude-User":["partial","*",30,18],"Claude-SearchBot":["partial","*",30,18],"anthropic-ai":["partial","*",30,18],"PerplexityBot":["partial","*",30,18],"Perplexity-User":["partial","*",30,18],"Google-Extended":["partial","*",30,18],"Applebot-Extended":["partial","*",30,18],"CCBot":["partial","*",30,18],"Bytespider":["partial","*",30,18],"Amazonbot":["partial","*",30,18],"meta-externalagent":["partial","*",30,18],"cohere-ai":["partial","*",30,18],"Diffbot":["partial","*",30,18],"ContextFetchBot":["partial","*",30,18]},"llms":{"status":"missing","http":404,"bytes":205},"home":{"status":"ok","http":200,"url":"https://www.brennancenter.org/","html_tokens":39760,"markdown_tokens":66,"pruned_tokens":51}},
{"rank":5188,"domain":"ncaa.org","origin":"https://ncaa.org","robots":{"status":"ok","http":200,"url":"https://www.ncaa.org/robots.txt","bytes":186,"groups":1},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"http_error","http":403,"bytes":5539},"home":{"status":"http_error","http":403,"url":"https://ncaa.org/"}},
{"rank":5189,"domain":"ebay.fr","origin":"https://ebay.fr","robots":{"status":"ok","http":200,"url":"https://www.ebay.fr/robots.txt","bytes":16643,"groups":7},"bots":{"GPTBot":["partial","named",1,3],"ChatGPT-User":["partial","*",205,7],"OAI-SearchBot":["partial","*",205,7],"ClaudeBot":["partial","named",1,3],"Claude-User":["partial","*",205,7],"Claude-SearchBot":["partial","*",205,7],"anthropic-ai":["partial","named",1,3],"PerplexityBot":["partial","named",1,3],"Perplexity-User":["partial","*",205,7],"Google-Extended":["partial","*",205,7],"Applebot-Extended":["partial","named",1,3],"CCBot":["partial","named",1,3],"Bytespider":["partial","named",1,3],"Amazonbot":["partial","named",1,3],"meta-externalagent":["partial","named",1,3],"cohere-ai":["partial","*",205,7],"Diffbot":["partial","*",205,7],"ContextFetchBot":["partial","*",205,7]},"llms":{"status":"missing","http":404,"bytes":80489},"home":{"status":"http_error","http":403,"url":"https://www.ebay.fr/"}},
{"rank":5190,"domain":"citi.com","origin":"https://citi.com","robots":{"status":"ok","http":200,"url":"https://www.citi.com/robots.txt","bytes":955,"groups":1},"bots":{"GPTBot":["partial","*",6,1],"ChatGPT-User":["partial","*",6,1],"OAI-SearchBot":["partial","*",6,1],"ClaudeBot":["partial","*",6,1],"Claude-User":["partial","*",6,1],"Claude-SearchBot":["partial","*",6,1],"anthropic-ai":["partial","*",6,1],"PerplexityBot":["partial","*",6,1],"Perplexity-User":["partial","*",6,1],"Google-Extended":["partial","*",6,1],"Applebot-Extended":["partial","*",6,1],"CCBot":["partial","*",6,1],"Bytespider":["partial","*",6,1],"Amazonbot":["partial","*",6,1],"meta-externalagent":["partial","*",6,1],"cohere-ai":["partial","*",6,1],"Diffbot":["partial","*",6,1],"ContextFetchBot":["partial","*",6,1]},"llms":{"status":"missing","http":404,"bytes":188},"home":{"status":"ok","http":200,"url":"https://www.citi.com/","html_tokens":63351,"markdown_tokens":1043,"pruned_tokens":461}},
{"rank":5191,"domain":"uni-konstanz.de","origin":"https://uni-konstanz.de","robots":{"status":"ok","http":200,"url":"https://www.uni-konstanz.de/robots.txt","bytes":2105,"groups":56},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["blocked","named",1,0],"OAI-SearchBot":["blocked","named",1,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["blocked","*",1,0],"Claude-SearchBot":["blocked","named",1,0],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["blocked","named",1,0],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["blocked","*",1,0]},"llms":{"status":"disallowed"},"home":{"status":"disallowed"}},
{"rank":5192,"domain":"netzpolitik.org","origin":"https://netzpolitik.org","robots":{"status":"ok","http":200,"url":"https://netzpolitik.org/robots.txt","bytes":392,"groups":1},"bots":{"GPTBot":["partial","*",16,0],"ChatGPT-User":["partial","*",16,0],"OAI-SearchBot":["partial","*",16,0],"ClaudeBot":["partial","*",16,0],"Claude-User":["partial","*",16,0],"Claude-SearchBot":["partial","*",16,0],"anthropic-ai":["partial","*",16,0],"PerplexityBot":["partial","*",16,0],"Perplexity-User":["partial","*",16,0],"Google-Extended":["partial","*",16,0],"Applebot-Extended":["partial","*",16,0],"CCBot":["partial","*",16,0],"Bytespider":["partial","*",16,0],"Amazonbot":["partial","*",16,0],"meta-externalagent":["partial","*",16,0],"cohere-ai":["partial","*",16,0],"Diffbot":["partial","*",16,0],"ContextFetchBot":["partial","*",16,0]},"llms":{"status":"missing","http":404,"bytes":105833},"home":{"status":"ok","http":200,"url":"https://netzpolitik.org/","html_tokens":87366,"markdown_tokens":1118,"pruned_tokens":323}},
{"rank":5193,"domain":"spbu.ru","origin":"https://spbu.ru","robots":{"status":"ok","http":200,"url":"https://spbu.ru/robots.txt","bytes":3355,"groups":2},"bots":{"GPTBot":["partial","*",31,18],"ChatGPT-User":["partial","*",31,18],"OAI-SearchBot":["partial","*",31,18],"ClaudeBot":["partial","*",31,18],"Claude-User":["partial","*",31,18],"Claude-SearchBot":["partial","*",31,18],"anthropic-ai":["partial","*",31,18],"PerplexityBot":["partial","*",31,18],"Perplexity-User":["partial","*",31,18],"Google-Extended":["partial","*",31,18],"Applebot-Extended":["partial","*",31,18],"CCBot":["partial","*",31,18],"Bytespider":["partial","*",31,18],"Amazonbot":["partial","*",31,18],"meta-externalagent":["partial","*",31,18],"cohere-ai":["partial","*",31,18],"Diffbot":["partial","*",31,18],"ContextFetchBot":["partial","*",31,18]},"llms":{"status":"missing","http":404,"bytes":291},"home":{"status":"ok","http":200,"url":"https://spbu.ru/","html_tokens":33305,"markdown_tokens":298,"pruned_tokens":123}},
{"rank":5194,"domain":"klaviyo.com","origin":"https://klaviyo.com","robots":{"status":"ok","http":200,"url":"https://www.klaviyo.com/robots.txt","bytes":3530,"groups":2},"bots":{"GPTBot":["partial","named",58,1],"ChatGPT-User":["partial","named",58,1],"OAI-SearchBot":["partial","named",58,1],"ClaudeBot":["partial","named",58,1],"Claude-User":["partial","named",58,1],"Claude-SearchBot":["partial","named",58,1],"anthropic-ai":["partial","*",58,1],"PerplexityBot":["partial","named",58,1],"Perplexity-User":["partial","named",58,1],"Google-Extended":["partial","*",58,1],"Applebot-Extended":["partial","*",58,1],"CCBot":["partial","*",58,1],"Bytespider":["partial","*",58,1],"Amazonbot":["partial","*",58,1],"meta-externalagent":["partial","named",58,1],"cohere-ai":["partial","*",58,1],"Diffbot":["partial","*",58,1],"ContextFetchBot":["partial","*",58,1]},"llms":{"status":"present","http":200,"bytes":20277},"home":{"status":"ok","http":200,"url":"https://www.klaviyo.com/","html_tokens":145156,"markdown_tokens":945,"pruned_tokens":755}},
{"rank":5195,"domain":"star-telegram.com","origin":"https://star-telegram.com","robots":{"status":"unreachable","http":null,"url":"https://star-telegram.com/robots.txt","bytes":0,"groups":0,"error":"<StreamReset stream_id:1, error_code:2, remote_reset:True>"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":5196,"domain":"freshbooks.com","origin":"https://freshbooks.com","robots":{"status":"ok","http":200,"url":"https://www.freshbooks.com/robots.txt","bytes":866,"groups":4},"bots":{"GPTBot":["partial","*",15,1],"ChatGPT-User":["allowed","named",0,1],"OAI-SearchBot":["partial","*",15,1],"ClaudeBot":["allowed","named",0,1],"Claude-User":["partial","*",15,1],"Claude-SearchBot":["partial","*",15,1],"anthropic-ai":["partial","*",15,1],"PerplexityBot":["partial","*",15,1],"Perplexity-User":["partial","*",15,1],"Google-Extended":["partial","*",15,1],"Applebot-Extended":["partial","*",15,1],"CCBot":["partial","*",15,1],"Bytespider":["blocked","named",1,0],"Amazonbot":["allowed","named",0,1],"meta-externalagent":["partial","*",15,1],"cohere-ai":["partial","*",15,1],"Diffbot":["partial","*",15,1],"ContextFetchBot":["partial","*",15,1]},"llms":{"status":"missing","http":404,"bytes":502501},"home":{"status":"ok","http":200,"url":"https://www.freshbooks.com/","html_tokens":344255,"markdown_tokens":1786,"pruned_tokens":524}},
{"rank":5197,"domain":"ecwid.com","origin":"https://ecwid.com","robots":{"status":"ok","http":200,"url":"https://www.ecwid.com/robots.txt","bytes":616,"groups":5},"bots":{"GPTBot":["partial","*",11,1],"ChatGPT-User":["partial","*",11,1],"OAI-SearchBot":["partial","*",11,1],"ClaudeBot":["partial","*",11,1],"Claude-User":["partial","*",11,1],"Claude-SearchBot":["partial","*",11,1],"anthropic-ai":["partial","*",11,1],"PerplexityBot":["partial","*",11,1],"Perplexity-User":["partial","*",11,1],"Google-Extended":["partial","*",11,1],"Applebot-Extended":["partial","*",11,1],"CCBot":["partial","*",11,1],"Bytespider":["partial","*",11,1],"Amazonbot":["partial","*",11,1],"meta-externalagent":["partial","*",11,1],"cohere-ai":["partial","*",11,1],"Diffbot":["partial","*",11,1],"ContextFetchBot":["partial","*",11,1]},"llms":{"status":"missing","http":404,"bytes":119980},"home":{"status":"ok","http":200,"url":"https://www.ecwid.com/","html_tokens":39274,"markdown_tokens":521,"pruned_tokens":92}},
{"rank":5198,"domain":"markmonitor.com","origin":"https://markmonitor.com","robots":{"status":"refused","http":403,"url":"https://markmonitor.com/robots.txt","bytes":5552,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":5199,"domain":"dlink.com","origin":"https://dlink.com","robots":{"status":"ok","http":200,"url":"https://www.dlink.com/robots.txt","bytes":262,"groups":1},"bots":{"GPTBot":["partial","*",1,11],"ChatGPT-User":["partial","*",1,11],"OAI-SearchBot":["partial","*",1,11],"ClaudeBot":["partial","*",1,11],"Claude-User":["partial","*",1,11],"Claude-SearchBot":["partial","*",1,11],"anthropic-ai":["partial","*",1,11],"PerplexityBot":["partial","*",1,11],"Perplexity-User":["partial","*",1,11],"Google-Extended":["partial","*",1,11],"Applebot-Extended":["partial","*",1,11],"CCBot":["partial","*",1,11],"Bytespider":["partial","*",1,11],"Amazonbot":["partial","*",1,11],"meta-externalagent":["partial","*",1,11],"cohere-ai":["partial","*",1,11],"Diffbot":["partial","*",1,11],"ContextFetchBot":["partial","*",1,11]},"llms":{"status":"missing","http":404,"bytes":1245},"home":{"status":"ok","http":200,"url":"https://www.dlink.com/us/en","html_tokens":29022,"markdown_tokens":85,"pruned_tokens":85}},
{"rank":5200,"domain":"agu.org","origin":"https://agu.org","robots":{"status":"missing","http":404,"url":"https://www.agu.org/robots.txt","bytes":1606,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":1606},"home":{"status":"http_error","http":403,"url":"https://agu.org/"}},
{"rank":5201,"domain":"anses.fr","origin":"https://anses.fr","robots":{"status":"html","http":200,"url":"https://www.anses.fr/fr","bytes":100085,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":5202,"domain":"retaildive.com","origin":"https://retaildive.com","robots":{"status":"ok","http":200,"url":"https://www.retaildive.com/robots.txt","bytes":4954,"groups":13},"bots":{"GPTBot":["partial","*",15,2],"ChatGPT-User":["partial","*",15,2],"OAI-SearchBot":["partial","*",15,2],"ClaudeBot":["partial","*",15,2],"Claude-User":["partial","*",15,2],"Claude-SearchBot":["partial","*",15,2],"anthropic-ai":["partial","*",15,2],"PerplexityBot":["partial","*",15,2],"Perplexity-User":["partial","*",15,2],"Google-Extended":["partial","*",15,2],"Applebot-Extended":["partial","*",15,2],"CCBot":["partial","*",15,2],"Bytespider":["partial","*",15,2],"Amazonbot":["partial","named",17,2],"meta-externalagent":["partial","*",15,2],"cohere-ai":["partial","*",15,2],"Diffbot":["partial","*",15,2],"ContextFetchBot":["partial","*",15,2]},"llms":{"status":"missing","http":404,"bytes":140071},"home":{"status":"ok","http":200,"url":"https://www.retaildive.com/","html_tokens":72484,"markdown_tokens":533,"pruned_tokens":166}},
{"rank":5203,"domain":"ok.gov","origin":"https://ok.gov","robots":{"status":"ok","http":200,"url":"https://www.ok.gov/robots.txt","bytes":1255,"groups":2},"bots":{"GPTBot":["partial","*",50,0],"ChatGPT-User":["partial","*",50,0],"OAI-SearchBot":["partial","*",50,0],"ClaudeBot":["partial","*",50,0],"Claude-User":["partial","*",50,0],"Claude-SearchBot":["partial","*",50,0],"anthropic-ai":["partial","*",50,0],"PerplexityBot":["partial","*",50,0],"Perplexity-User":["partial","*",50,0],"Google-Extended":["partial","*",50,0],"Applebot-Extended":["partial","*",50,0],"CCBot":["partial","*",50,0],"Bytespider":["partial","*",50,0],"Amazonbot":["partial","*",50,0],"meta-externalagent":["partial","*",50,0],"cohere-ai":["partial","*",50,0],"Diffbot":["partial","*",50,0],"ContextFetchBot":["partial","*",50,0]},"llms":{"status":"missing","http":404,"bytes":385},"home":{"status":"ok","http":200,"url":"https://oklahoma.gov","html_tokens":25821,"markdown_tokens":425,"pruned_tokens":140}},
{"rank":5204,"domain":"12306.cn","origin":"https://www.12306.cn","robots":{"status":"ok","http":200,"url":"https://www.12306.cn/mormhweb/logFiles/error.html","bytes":2042,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"html","http":200,"bytes":2042},"home":{"status":"ok","http":200,"url":"https://www.12306.cn/index/","html_tokens":18806,"markdown_tokens":16,"pruned_tokens":12}},
{"rank":5205,"domain":"forexby.com","origin":"https://forexby.com","robots":{"status":"ok","http":200,"url":"https://forexby.com/robots.txt","bytes":345,"groups":1},"bots":{"GPTBot":["partial","*",5,9],"ChatGPT-User":["partial","*",5,9],"OAI-SearchBot":["partial","*",5,9],"ClaudeBot":["partial","*",5,9],"Claude-User":["partial","*",5,9],"Claude-SearchBot":["partial","*",5,9],"anthropic-ai":["partial","*",5,9],"PerplexityBot":["partial","*",5,9],"Perplexity-User":["partial","*",5,9],"Google-Extended":["partial","*",5,9],"Applebot-Extended":["partial","*",5,9],"CCBot":["partial","*",5,9],"Bytespider":["partial","*",5,9],"Amazonbot":["partial","*",5,9],"meta-externalagent":["partial","*",5,9],"cohere-ai":["partial","*",5,9],"Diffbot":["partial","*",5,9],"ContextFetchBot":["partial","*",5,9]},"llms":{"status":"missing","http":404,"bytes":48663},"home":{"status":"ok","http":200,"url":"https://forexby.com/","html_tokens":14814,"markdown_tokens":471,"pruned_tokens":84}},
{"rank":5206,"domain":"openclipart.org","origin":"https://openclipart.org","robots":{"status":"ok","http":200,"url":"https://openclipart.org/robots.txt","bytes":59,"groups":1},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"html","http":200,"bytes":23736},"home":{"status":"ok","http":200,"url":"https://openclipart.org/","html_tokens":5932,"markdown_tokens":4,"pruned_tokens":4}},
{"rank":5207,"domain":"volcengine.com","origin":"https://volcengine.com","robots":{"status":"ok","http":200,"url":"https://www.volcengine.com/robots.txt","bytes":1219,"groups":2},"bots":{"GPTBot":["allowed","named",0,1],"ChatGPT-User":["allowed","named",0,1],"OAI-SearchBot":["allowed","named",0,1],"ClaudeBot":["allowed","named",0,1],"Claude-User":["allowed","named",0,1],"Claude-SearchBot":["allowed","named",0,1],"anthropic-ai":["partial","*",12,0],"PerplexityBot":["allowed","named",0,1],"Perplexity-User":["allowed","named",0,1],"Google-Extended":["allowed","named",0,1],"Applebot-Extended":["allowed","named",0,1],"CCBot":["allowed","named",0,1],"Bytespider":["allowed","named",0,1],"Amazonbot":["partial","*",12,0],"meta-externalagent":["partial","*",12,0],"cohere-ai":["partial","*",12,0],"Diffbot":["partial","*",12,0],"ContextFetchBot":["partial","*",12,0]},"llms":{"status":"html","http":200,"bytes":330354},"home":{"status":"ok","http":200,"url":"https://www.volcengine.com/","html_tokens":26576,"markdown_tokens":67,"pruned_tokens":67}},
{"rank":5208,"domain":"robbreport.com","origin":"https://robbreport.com","robots":{"status":"ok","http":200,"url":"https://robbreport.com/robots.txt","bytes":2352,"groups":27},"bots":{"GPTBot":["partial","named",1,1],"ChatGPT-User":["partial","named",1,1],"OAI-SearchBot":["partial","named",1,1],"ClaudeBot":["partial","named",1,1],"Claude-User":["partial","*",10,1],"Claude-SearchBot":["partial","*",10,1],"anthropic-ai":["partial","named",1,1],"PerplexityBot":["partial","named",1,1],"Perplexity-User":["partial","*",10,1],"Google-Extended":["partial","named",1,1],"Applebot-Extended":["partial","named",1,1],"CCBot":["partial","named",1,1],"Bytespider":["partial","named",1,1],"Amazonbot":["partial","*",10,1],"meta-externalagent":["partial","named",1,1],"cohere-ai":["partial","named",1,1],"Diffbot":["partial","named",1,1],"ContextFetchBot":["partial","*",10,1]},"llms":{"status":"missing","http":404,"bytes":384619},"home":{"status":"ok","http":200,"url":"https://robbreport.com/","html_tokens":119978,"markdown_tokens":84,"pruned_tokens":84}},
{"rank":5209,"domain":"rarlab.com","origin":"https://rarlab.com","robots":{"status":"ok","http":200,"url":"https://www.rarlab.com/robots.txt","bytes":60,"groups":1},"bots":{"GPTBot":["partial","*",2,0],"ChatGPT-User":["partial","*",2,0],"OAI-SearchBot":["partial","*",2,0],"ClaudeBot":["partial","*",2,0],"Claude-User":["partial","*",2,0],"Claude-SearchBot":["partial","*",2,0],"anthropic-ai":["partial","*",2,0],"PerplexityBot":["partial","*",2,0],"Perplexity-User":["partial","*",2,0],"Google-Extended":["partial","*",2,0],"Applebot-Extended":["partial","*",2,0],"CCBot":["partial","*",2,0],"Bytespider":["partial","*",2,0],"Amazonbot":["partial","*",2,0],"meta-externalagent":["partial","*",2,0],"cohere-ai":["partial","*",2,0],"Diffbot":["partial","*",2,0],"ContextFetchBot":["partial","*",2,0]},"llms":{"status":"missing","http":404,"bytes":196},"home":{"status":"ok","http":200,"url":"https://www.rarlab.com/","html_tokens":2919,"markdown_tokens":400,"pruned_tokens":227}},
{"rank":5210,"domain":"cirquedusoleil.com","origin":"https://cirquedusoleil.com","robots":{"status":"ok","http":200,"url":"https://www.cirquedusoleil.com/robots.txt","bytes":416,"groups":1},"bots":{"GPTBot":["partial","*",3,0],"ChatGPT-User":["partial","*",3,0],"OAI-SearchBot":["partial","*",3,0],"ClaudeBot":["partial","*",3,0],"Claude-User":["partial","*",3,0],"Claude-SearchBot":["partial","*",3,0],"anthropic-ai":["partial","*",3,0],"PerplexityBot":["partial","*",3,0],"Perplexity-User":["partial","*",3,0],"Google-Extended":["partial","*",3,0],"Applebot-Extended":["partial","*",3,0],"CCBot":["partial","*",3,0],"Bytespider":["partial","*",3,0],"Amazonbot":["partial","*",3,0],"meta-externalagent":["partial","*",3,0],"cohere-ai":["partial","*",3,0],"Diffbot":["partial","*",3,0],"ContextFetchBot":["partial","*",3,0]},"llms":{"status":"present","http":200,"bytes":9954},"home":{"status":"ok","http":200,"url":"https://www.cirquedusoleil.com/","html_tokens":37106,"markdown_tokens":728,"pruned_tokens":90}},
{"rank":5211,"domain":"skyscanner.net","origin":"https://skyscanner.net","robots":{"status":"ok","http":200,"url":"https://www.skyscanner.net/robots.txt","bytes":10882,"groups":11},"bots":{"GPTBot":["partial","*",324,1],"ChatGPT-User":["partial","*",324,1],"OAI-SearchBot":["partial","*",324,1],"ClaudeBot":["partial","*",324,1],"Claude-User":["partial","*",324,1],"Claude-SearchBot":["partial","*",324,1],"anthropic-ai":["partial","*",324,1],"PerplexityBot":["partial","*",324,1],"Perplexity-User":["partial","*",324,1],"Google-Extended":["partial","*",324,1],"Applebot-Extended":["partial","*",324,1],"CCBot":["partial","*",324,1],"Bytespider":["partial","*",324,1],"Amazonbot":["partial","*",324,1],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["partial","*",324,1],"Diffbot":["partial","*",324,1],"ContextFetchBot":["partial","*",324,1]},"llms":{"status":"missing","http":404,"bytes":147},"home":{"status":"ok","http":200,"url":"https://www.skyscanner.com/?previousCultureSource=GEO_LOCATION&redirectedFrom=www.skyscanner.net&userSessionDataId=50671d02-d4e9-4423-888c-de38ecf40853&session_handoff=eyJhbGciOiJFUzI1NiIsImtpZCI6InNlc3Npb24tc2VydmljZS1oYW5kb2ZmLWtleS0yMDI2LTA1IiwidHlwIjoiSldUIn0.eyJodHRwczovL3NreXNjYW5uZXIubmV0L3Nlc3Npb25faWQiOiIwMWExNTFkOS1mY2Q2LTgwMDEtODg4OC00Mzg4ZjRkZGY3ODEiLCJleHAiOjE3OTIzNzQ2MTN9.t04hClRrna7PMn-KWh0ujrDxz4baYRbGcoiG6ImBF40a1OT_I1-S2Unz-0fQB9K2NXsfe7-MSdySjcKY-TpWBA","html_tokens":66058,"markdown_tokens":0,"pruned_tokens":0}},
{"rank":5212,"domain":"linternaute.com","origin":"https://linternaute.com","robots":{"status":"ok","http":200,"url":"http://www.linternaute.com/robots.txt","bytes":3103,"groups":8},"bots":{"GPTBot":["partial","*",102,0],"ChatGPT-User":["partial","*",102,0],"OAI-SearchBot":["partial","*",102,0],"ClaudeBot":["partial","*",102,0],"Claude-User":["partial","*",102,0],"Claude-SearchBot":["partial","*",102,0],"anthropic-ai":["partial","*",102,0],"PerplexityBot":["partial","*",102,0],"Perplexity-User":["partial","*",102,0],"Google-Extended":["partial","*",102,0],"Applebot-Extended":["partial","*",102,0],"CCBot":["partial","*",102,0],"Bytespider":["partial","*",102,0],"Amazonbot":["partial","*",102,0],"meta-externalagent":["partial","*",102,0],"cohere-ai":["partial","*",102,0],"Diffbot":["partial","*",102,0],"ContextFetchBot":["partial","*",102,0]},"llms":{"status":"missing","http":404,"bytes":14731},"home":{"status":"ok","http":200,"url":"https://www.linternaute.com/","html_tokens":47348,"markdown_tokens":269,"pruned_tokens":155}},
{"rank":5213,"domain":"abendblatt.de","origin":"https://abendblatt.de","robots":{"status":"ok","http":200,"url":"https://www.abendblatt.de/robots.txt","bytes":6366,"groups":7},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["partial","named",27,0],"OAI-SearchBot":["partial","*",26,3],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",26,3],"Claude-SearchBot":["partial","named",27,0],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["partial","named",27,0],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",26,3]},"llms":{"status":"missing","http":404,"bytes":278994},"home":{"status":"ok","http":200,"url":"https://www.abendblatt.de/","html_tokens":278696,"markdown_tokens":3784,"pruned_tokens":137}},
{"rank":5214,"domain":"taboola.com","origin":"https://taboola.com","robots":{"status":"ok","http":200,"url":"https://www.taboola.com/robots.txt","bytes":2350,"groups":1},"bots":{"GPTBot":["partial","*",40,32],"ChatGPT-User":["partial","*",40,32],"OAI-SearchBot":["partial","*",40,32],"ClaudeBot":["partial","*",40,32],"Claude-User":["partial","*",40,32],"Claude-SearchBot":["partial","*",40,32],"anthropic-ai":["partial","*",40,32],"PerplexityBot":["partial","*",40,32],"Perplexity-User":["partial","*",40,32],"Google-Extended":["partial","*",40,32],"Applebot-Extended":["partial","*",40,32],"CCBot":["partial","*",40,32],"Bytespider":["partial","*",40,32],"Amazonbot":["partial","*",40,32],"meta-externalagent":["partial","*",40,32],"cohere-ai":["partial","*",40,32],"Diffbot":["partial","*",40,32],"ContextFetchBot":["partial","*",40,32]},"llms":{"status":"present","http":200,"bytes":278000},"home":{"status":"ok","http":200,"url":"https://www.taboola.com/","html_tokens":39385,"markdown_tokens":392,"pruned_tokens":44}},
{"rank":5215,"domain":"veed.io","origin":"https://veed.io","robots":{"status":"ok","http":200,"url":"https://www.veed.io/robots.txt","bytes":244,"groups":2},"bots":{"GPTBot":["partial","*",7,1],"ChatGPT-User":["partial","*",7,1],"OAI-SearchBot":["partial","*",7,1],"ClaudeBot":["partial","*",7,1],"Claude-User":["partial","*",7,1],"Claude-SearchBot":["partial","*",7,1],"anthropic-ai":["partial","*",7,1],"PerplexityBot":["partial","*",7,1],"Perplexity-User":["partial","*",7,1],"Google-Extended":["partial","*",7,1],"Applebot-Extended":["partial","*",7,1],"CCBot":["partial","*",7,1],"Bytespider":["partial","*",7,1],"Amazonbot":["partial","*",7,1],"meta-externalagent":["partial","*",7,1],"cohere-ai":["partial","*",7,1],"Diffbot":["partial","*",7,1],"ContextFetchBot":["partial","*",7,1]},"llms":{"status":"missing","http":404,"bytes":507},"home":{"status":"ok","http":200,"url":"https://www.veed.io/","html_tokens":168573,"markdown_tokens":224,"pruned_tokens":173}},
{"rank":5216,"domain":"feverup.com","origin":"https://feverup.com","robots":{"status":"ok","http":200,"url":"https://feverup.com/robots.txt","bytes":5220,"groups":106},"bots":{"GPTBot":["partial","*",13,1],"ChatGPT-User":["partial","*",13,1],"OAI-SearchBot":["partial","*",13,1],"ClaudeBot":["partial","*",13,1],"Claude-User":["partial","*",13,1],"Claude-SearchBot":["partial","*",13,1],"anthropic-ai":["partial","*",13,1],"PerplexityBot":["partial","*",13,1],"Perplexity-User":["partial","*",13,1],"Google-Extended":["partial","*",13,1],"Applebot-Extended":["partial","*",13,1],"CCBot":["partial","*",13,1],"Bytespider":["partial","*",13,1],"Amazonbot":["partial","*",13,1],"meta-externalagent":["partial","*",13,1],"cohere-ai":["partial","*",13,1],"Diffbot":["partial","*",13,1],"ContextFetchBot":["partial","*",13,1]},"llms":{"status":"missing","http":404,"bytes":0},"home":{"status":"ok","http":200,"url":"https://feverup.com/en","html_tokens":59651,"markdown_tokens":229,"pruned_tokens":42}},
{"rank":5217,"domain":"jwplayer.com","origin":"https://jwplayer.com","robots":{"status":"ok","http":200,"url":"https://jwplayer.com/robots.txt","bytes":170,"groups":1},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"missing","http":404,"bytes":58335},"home":{"status":"ok","http":200,"url":"https://jwx.com/","html_tokens":43563,"markdown_tokens":412,"pruned_tokens":287}},
{"rank":5218,"domain":"stat.go.jp","origin":"https://www.stat.go.jp","robots":{"status":"ok","http":200,"url":"https://www.stat.go.jp/robots.txt","bytes":39,"groups":1},"bots":{"GPTBot":["partial","*",1,0],"ChatGPT-User":["partial","*",1,0],"OAI-SearchBot":["partial","*",1,0],"ClaudeBot":["partial","*",1,0],"Claude-User":["partial","*",1,0],"Claude-SearchBot":["partial","*",1,0],"anthropic-ai":["partial","*",1,0],"PerplexityBot":["partial","*",1,0],"Perplexity-User":["partial","*",1,0],"Google-Extended":["partial","*",1,0],"Applebot-Extended":["partial","*",1,0],"CCBot":["partial","*",1,0],"Bytespider":["partial","*",1,0],"Amazonbot":["partial","*",1,0],"meta-externalagent":["partial","*",1,0],"cohere-ai":["partial","*",1,0],"Diffbot":["partial","*",1,0],"ContextFetchBot":["partial","*",1,0]},"llms":{"status":"missing","http":404,"bytes":4691},"home":{"status":"ok","http":200,"url":"https://www.stat.go.jp/","html_tokens":9172,"markdown_tokens":35,"pruned_tokens":35}},
{"rank":5219,"domain":"f6s.com","origin":"https://f6s.com","robots":{"status":"refused","http":403,"url":"https://f6s.com/robots.txt","bytes":2657,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":5220,"domain":"toggl.com","origin":"https://toggl.com","robots":{"status":"ok","http":200,"url":"https://toggl.com/robots.txt","bytes":191,"groups":1},"bots":{"GPTBot":["partial","*",2,2],"ChatGPT-User":["partial","*",2,2],"OAI-SearchBot":["partial","*",2,2],"ClaudeBot":["partial","*",2,2],"Claude-User":["partial","*",2,2],"Claude-SearchBot":["partial","*",2,2],"anthropic-ai":["partial","*",2,2],"PerplexityBot":["partial","*",2,2],"Perplexity-User":["partial","*",2,2],"Google-Extended":["partial","*",2,2],"Applebot-Extended":["partial","*",2,2],"CCBot":["partial","*",2,2],"Bytespider":["partial","*",2,2],"Amazonbot":["partial","*",2,2],"meta-externalagent":["partial","*",2,2],"cohere-ai":["partial","*",2,2],"Diffbot":["partial","*",2,2],"ContextFetchBot":["partial","*",2,2]},"llms":{"status":"missing","http":404,"bytes":227410},"home":{"status":"ok","http":200,"url":"https://toggl.com/","html_tokens":109132,"markdown_tokens":1523,"pruned_tokens":280}},
{"rank":5221,"domain":"belkin.com","origin":"https://belkin.com","robots":{"status":"ok","http":200,"url":"https://www.belkin.com/robots.txt","bytes":3031,"groups":21},"bots":{"GPTBot":["partial","*",32,0],"ChatGPT-User":["partial","*",32,0],"OAI-SearchBot":["partial","*",32,0],"ClaudeBot":["partial","*",32,0],"Claude-User":["partial","*",32,0],"Claude-SearchBot":["partial","*",32,0],"anthropic-ai":["partial","*",32,0],"PerplexityBot":["partial","*",32,0],"Perplexity-User":["partial","*",32,0],"Google-Extended":["partial","*",32,0],"Applebot-Extended":["partial","*",32,0],"CCBot":["partial","*",32,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["allowed","named",0,1],"meta-externalagent":["partial","*",32,0],"cohere-ai":["partial","*",32,0],"Diffbot":["partial","*",32,0],"ContextFetchBot":["partial","*",32,0]},"llms":{"status":"missing","http":404,"bytes":540387},"home":{"status":"ok","http":200,"url":"https://www.belkin.com/","html_tokens":183608,"markdown_tokens":27,"pruned_tokens":27}},
{"rank":5222,"domain":"mapy.com","origin":"https://mapy.com","robots":{"status":"ok","http":200,"url":"https://mapy.com/robots.txt","bytes":20639,"groups":14},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["partial","*",2,0],"OAI-SearchBot":["partial","*",2,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",2,0],"Claude-SearchBot":["partial","*",2,0],"anthropic-ai":["partial","*",2,0],"PerplexityBot":["partial","*",2,0],"Perplexity-User":["partial","*",2,0],"Google-Extended":["partial","*",2,0],"Applebot-Extended":["partial","*",2,0],"CCBot":["partial","*",2,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["partial","*",2,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["partial","*",2,0],"Diffbot":["partial","*",2,0],"ContextFetchBot":["partial","*",2,0]},"llms":{"status":"html","http":200,"bytes":13662},"home":{"status":"ok","http":200,"url":"https://mapy.com/en/","html_tokens":3385,"markdown_tokens":0,"pruned_tokens":0}},
{"rank":5223,"domain":"slideserve.com","origin":"https://slideserve.com","robots":{"status":"ok","http":200,"url":"https://www.slideserve.com/robots.txt","bytes":1158,"groups":10},"bots":{"GPTBot":["partial","*",37,0],"ChatGPT-User":["partial","*",37,0],"OAI-SearchBot":["partial","*",37,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",37,0],"Claude-SearchBot":["partial","*",37,0],"anthropic-ai":["partial","*",37,0],"PerplexityBot":["partial","*",37,0],"Perplexity-User":["partial","*",37,0],"Google-Extended":["partial","*",37,0],"Applebot-Extended":["partial","*",37,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["partial","*",37,0],"meta-externalagent":["partial","*",37,0],"cohere-ai":["partial","*",37,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",37,0]},"llms":{"status":"present","http":200,"bytes":4782},"home":{"status":"ok","http":200,"url":"https://www.slideserve.com/","html_tokens":19230,"markdown_tokens":216,"pruned_tokens":179}},
{"rank":5224,"domain":"creative.com","origin":"https://creative.com","robots":{"status":"ok","http":200,"url":"https://www.creative.com/robots.txt","bytes":32,"groups":1},"bots":{"GPTBot":["partial","*",1,0],"ChatGPT-User":["partial","*",1,0],"OAI-SearchBot":["partial","*",1,0],"ClaudeBot":["partial","*",1,0],"Claude-User":["partial","*",1,0],"Claude-SearchBot":["partial","*",1,0],"anthropic-ai":["partial","*",1,0],"PerplexityBot":["partial","*",1,0],"Perplexity-User":["partial","*",1,0],"Google-Extended":["partial","*",1,0],"Applebot-Extended":["partial","*",1,0],"CCBot":["partial","*",1,0],"Bytespider":["partial","*",1,0],"Amazonbot":["partial","*",1,0],"meta-externalagent":["partial","*",1,0],"cohere-ai":["partial","*",1,0],"Diffbot":["partial","*",1,0],"ContextFetchBot":["partial","*",1,0]},"llms":{"status":"missing","http":404,"bytes":1245},"home":{"status":"ok","http":200,"url":"https://us.creative.com/","html_tokens":64830,"markdown_tokens":58,"pruned_tokens":58}},
{"rank":5225,"domain":"techxplore.com","origin":"https://techxplore.com","robots":{"status":"ok","http":200,"url":"https://techxplore.com/robots.txt","bytes":1153,"groups":15},"bots":{"GPTBot":["partial","named",4,0],"ChatGPT-User":["partial","*",5,1],"OAI-SearchBot":["partial","*",5,1],"ClaudeBot":["partial","named",4,0],"Claude-User":["partial","*",5,1],"Claude-SearchBot":["partial","*",5,1],"anthropic-ai":["partial","*",5,1],"PerplexityBot":["partial","*",5,1],"Perplexity-User":["partial","*",5,1],"Google-Extended":["partial","named",2,0],"Applebot-Extended":["partial","named",2,0],"CCBot":["partial","named",4,0],"Bytespider":["partial","named",4,0],"Amazonbot":["partial","*",5,1],"meta-externalagent":["partial","*",5,1],"cohere-ai":["partial","named",4,0],"Diffbot":["partial","*",5,1],"ContextFetchBot":["partial","*",5,1]},"llms":{"status":"missing","http":404,"bytes":6948},"home":{"status":"ok","http":200,"url":"https://techxplore.com/","html_tokens":25568,"markdown_tokens":600,"pruned_tokens":130}},
{"rank":5226,"domain":"dmm.com","origin":"https://dmm.com","robots":{"status":"ok","http":200,"url":"https://www.dmm.com/robots.txt","bytes":1368,"groups":3},"bots":{"GPTBot":["partial","*",39,0],"ChatGPT-User":["partial","*",39,0],"OAI-SearchBot":["partial","*",39,0],"ClaudeBot":["partial","*",39,0],"Claude-User":["partial","*",39,0],"Claude-SearchBot":["partial","*",39,0],"anthropic-ai":["partial","*",39,0],"PerplexityBot":["partial","*",39,0],"Perplexity-User":["partial","*",39,0],"Google-Extended":["partial","named",3,0],"Applebot-Extended":["partial","named",3,0],"CCBot":["partial","*",39,0],"Bytespider":["partial","*",39,0],"Amazonbot":["partial","*",39,0],"meta-externalagent":["partial","*",39,0],"cohere-ai":["partial","*",39,0],"Diffbot":["partial","*",39,0],"ContextFetchBot":["partial","*",39,0]},"llms":{"status":"missing","http":404,"bytes":7205},"home":{"status":"ok","http":200,"url":"https://www.dmm.com/","html_tokens":111299,"markdown_tokens":25,"pruned_tokens":24}},
{"rank":5227,"domain":"wfu.edu","origin":"https://wfu.edu","robots":{"status":"ok","http":200,"url":"https://www.wfu.edu/robots.txt","bytes":137,"groups":1},"bots":{"GPTBot":["partial","*",2,0],"ChatGPT-User":["partial","*",2,0],"OAI-SearchBot":["partial","*",2,0],"ClaudeBot":["partial","*",2,0],"Claude-User":["partial","*",2,0],"Claude-SearchBot":["partial","*",2,0],"anthropic-ai":["partial","*",2,0],"PerplexityBot":["partial","*",2,0],"Perplexity-User":["partial","*",2,0],"Google-Extended":["partial","*",2,0],"Applebot-Extended":["partial","*",2,0],"CCBot":["partial","*",2,0],"Bytespider":["partial","*",2,0],"Amazonbot":["partial","*",2,0],"meta-externalagent":["partial","*",2,0],"cohere-ai":["partial","*",2,0],"Diffbot":["partial","*",2,0],"ContextFetchBot":["partial","*",2,0]},"llms":{"status":"present","http":200,"bytes":118},"home":{"status":"ok","http":200,"url":"https://www.wfu.edu/","html_tokens":66817,"markdown_tokens":84,"pruned_tokens":84}},
{"rank":5228,"domain":"godotengine.org","origin":"https://godotengine.org","robots":{"status":"ok","http":200,"url":"https://godotengine.org/robots.txt","bytes":1248,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":4368},"home":{"status":"ok","http":200,"url":"https://godotengine.org/","html_tokens":10075,"markdown_tokens":397,"pruned_tokens":159}},
{"rank":5229,"domain":"adaa.org","origin":"https://adaa.org","robots":{"status":"ok","http":200,"url":"https://adaa.org/robots.txt","bytes":2362,"groups":1},"bots":{"GPTBot":["partial","*",44,18],"ChatGPT-User":["partial","*",44,18],"OAI-SearchBot":["partial","*",44,18],"ClaudeBot":["partial","*",44,18],"Claude-User":["partial","*",44,18],"Claude-SearchBot":["partial","*",44,18],"anthropic-ai":["partial","*",44,18],"PerplexityBot":["partial","*",44,18],"Perplexity-User":["partial","*",44,18],"Google-Extended":["partial","*",44,18],"Applebot-Extended":["partial","*",44,18],"CCBot":["partial","*",44,18],"Bytespider":["partial","*",44,18],"Amazonbot":["partial","*",44,18],"meta-externalagent":["partial","*",44,18],"cohere-ai":["partial","*",44,18],"Diffbot":["partial","*",44,18],"ContextFetchBot":["partial","*",44,18]},"llms":{"status":"missing","http":404,"bytes":182},"home":{"status":"http_error","http":403,"url":"https://adaa.org/"}},
{"rank":5230,"domain":"feedingamerica.org","origin":"https://feedingamerica.org","robots":{"status":"refused","http":403,"url":"https://feedingamerica.org/robots.txt","bytes":560,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":5231,"domain":"hln.be","origin":"https://hln.be","robots":{"status":"ok","http":200,"url":"https://www.hln.be/robots.txt","bytes":2689,"groups":30},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["blocked","named",1,0],"OAI-SearchBot":["blocked","named",1,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",13,0],"Claude-SearchBot":["partial","*",13,0],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["blocked","named",1,0],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["partial","*",13,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",13,0]},"llms":{"status":"missing","http":404,"bytes":192754},"home":{"status":"ok","http":200,"url":"https://www.hln.be/","html_tokens":201210,"markdown_tokens":17,"pruned_tokens":17}},
{"rank":5232,"domain":"kas.de","origin":"https://kas.de","robots":{"status":"ok","http":200,"url":"https://www.kas.de/robots.txt","bytes":64,"groups":1},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"missing","http":404,"bytes":238963},"home":{"status":"ok","http":200,"url":"https://www.kas.de/de/","html_tokens":93271,"markdown_tokens":929,"pruned_tokens":69}},
{"rank":5233,"domain":"uwyo.edu","origin":"https://www.uwyo.edu","robots":{"status":"ok","http":200,"url":"https://www.uwyo.edu/robots.txt","bytes":1956,"groups":1},"bots":{"GPTBot":["partial","*",75,0],"ChatGPT-User":["partial","*",75,0],"OAI-SearchBot":["partial","*",75,0],"ClaudeBot":["partial","*",75,0],"Claude-User":["partial","*",75,0],"Claude-SearchBot":["partial","*",75,0],"anthropic-ai":["partial","*",75,0],"PerplexityBot":["partial","*",75,0],"Perplexity-User":["partial","*",75,0],"Google-Extended":["partial","*",75,0],"Applebot-Extended":["partial","*",75,0],"CCBot":["partial","*",75,0],"Bytespider":["partial","*",75,0],"Amazonbot":["partial","*",75,0],"meta-externalagent":["partial","*",75,0],"cohere-ai":["partial","*",75,0],"Diffbot":["partial","*",75,0],"ContextFetchBot":["partial","*",75,0]},"llms":{"status":"html","http":200,"bytes":27641},"home":{"status":"ok","http":200,"url":"https://www.uwyo.edu/","html_tokens":17359,"markdown_tokens":108,"pruned_tokens":108}},
{"rank":5234,"domain":"domainnameshop.com","origin":"https://domainnameshop.com","robots":{"status":"ok","http":200,"url":"https://domainnameshop.com/robots.txt","bytes":703,"groups":2},"bots":{"GPTBot":["partial","*",32,0],"ChatGPT-User":["partial","*",32,0],"OAI-SearchBot":["partial","*",32,0],"ClaudeBot":["partial","*",32,0],"Claude-User":["partial","*",32,0],"Claude-SearchBot":["partial","*",32,0],"anthropic-ai":["partial","*",32,0],"PerplexityBot":["partial","*",32,0],"Perplexity-User":["partial","*",32,0],"Google-Extended":["partial","*",32,0],"Applebot-Extended":["partial","*",32,0],"CCBot":["partial","*",32,0],"Bytespider":["partial","*",32,0],"Amazonbot":["partial","*",32,0],"meta-externalagent":["partial","*",32,0],"cohere-ai":["partial","*",32,0],"Diffbot":["partial","*",32,0],"ContextFetchBot":["partial","*",32,0]},"llms":{"status":"missing","http":404,"bytes":0},"home":{"status":"ok","http":200,"url":"https://domainname.shop/","html_tokens":6976,"markdown_tokens":65,"pruned_tokens":65}},
{"rank":5235,"domain":"oxfordlearnersdictionaries.com","origin":"https://oxfordlearnersdictionaries.com","robots":{"status":"ok","http":200,"url":"https://www.oxfordlearnersdictionaries.com/robots.txt","bytes":7008,"groups":4},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["blocked","named",1,0],"OAI-SearchBot":["partial","*",60,0],"ClaudeBot":["partial","*",60,0],"Claude-User":["partial","*",60,0],"Claude-SearchBot":["partial","*",60,0],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["partial","*",60,0],"Perplexity-User":["partial","*",60,0],"Google-Extended":["partial","*",60,0],"Applebot-Extended":["partial","*",60,0],"CCBot":["blocked","named",1,0],"Bytespider":["partial","*",60,0],"Amazonbot":["partial","*",60,0],"meta-externalagent":["partial","*",60,0],"cohere-ai":["partial","*",60,0],"Diffbot":["partial","*",60,0],"ContextFetchBot":["partial","*",60,0]},"llms":{"status":"missing","http":404,"bytes":0},"home":{"status":"ok","http":200,"url":"https://www.oxfordlearnersdictionaries.com/us/","html_tokens":17626,"markdown_tokens":343,"pruned_tokens":52}},
{"rank":5236,"domain":"shandong.gov.cn","origin":"https://shandong.gov.cn","robots":{"status":"unreachable","http":null,"url":"https://shandong.gov.cn/robots.txt","bytes":0,"groups":0,"error":"cannot resolve shandong.gov.cn"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":5237,"domain":"openssh.org","origin":"https://openssh.org","robots":{"status":"missing","http":404,"url":"https://www.openssh.org/robots.txt","bytes":494,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":494},"home":{"status":"ok","http":200,"url":"https://www.openssh.org/","html_tokens":765,"markdown_tokens":227,"pruned_tokens":227}},
{"rank":5238,"domain":"mga.org.mt","origin":"https://mga.org.mt","robots":{"status":"ok","http":200,"url":"https://mga.org.mt/robots.txt","bytes":176,"groups":1},"bots":{"GPTBot":["partial","*",6,1],"ChatGPT-User":["partial","*",6,1],"OAI-SearchBot":["partial","*",6,1],"ClaudeBot":["partial","*",6,1],"Claude-User":["partial","*",6,1],"Claude-SearchBot":["partial","*",6,1],"anthropic-ai":["partial","*",6,1],"PerplexityBot":["partial","*",6,1],"Perplexity-User":["partial","*",6,1],"Google-Extended":["partial","*",6,1],"Applebot-Extended":["partial","*",6,1],"CCBot":["partial","*",6,1],"Bytespider":["partial","*",6,1],"Amazonbot":["partial","*",6,1],"meta-externalagent":["partial","*",6,1],"cohere-ai":["partial","*",6,1],"Diffbot":["partial","*",6,1],"ContextFetchBot":["partial","*",6,1]},"llms":{"status":"timeout","error":"timed out after 10 s"},"home":{"status":"ok","http":200,"url":"https://www.mga.org.mt/","html_tokens":24843,"markdown_tokens":105,"pruned_tokens":105}},
{"rank":5239,"domain":"appsflyer.com","origin":"https://appsflyer.com","robots":{"status":"ok","http":200,"url":"https://www.appsflyer.com/robots.txt","bytes":3116,"groups":1},"bots":{"GPTBot":["partial","*",74,24],"ChatGPT-User":["partial","*",74,24],"OAI-SearchBot":["partial","*",74,24],"ClaudeBot":["partial","*",74,24],"Claude-User":["partial","*",74,24],"Claude-SearchBot":["partial","*",74,24],"anthropic-ai":["partial","*",74,24],"PerplexityBot":["partial","*",74,24],"Perplexity-User":["partial","*",74,24],"Google-Extended":["partial","*",74,24],"Applebot-Extended":["partial","*",74,24],"CCBot":["partial","*",74,24],"Bytespider":["partial","*",74,24],"Amazonbot":["partial","*",74,24],"meta-externalagent":["partial","*",74,24],"cohere-ai":["partial","*",74,24],"Diffbot":["partial","*",74,24],"ContextFetchBot":["partial","*",74,24]},"llms":{"status":"missing","http":404,"bytes":146},"home":{"status":"ok","http":200,"url":"https://www.appsflyer.com/","html_tokens":106735,"markdown_tokens":617,"pruned_tokens":474}},
{"rank":5240,"domain":"hunan.gov.cn","origin":"http://hunan.gov.cn","robots":{"status":"missing","http":404,"url":"http://hunan.gov.cn/robots.txt","bytes":146,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":146},"home":{"status":"ok","http":200,"url":"http://hunan.gov.cn/","html_tokens":25720,"markdown_tokens":8,"pruned_tokens":8}},
{"rank":5241,"domain":"triblive.com","origin":"https://triblive.com","robots":{"status":"ok","http":200,"url":"https://triblive.com/robots.txt","bytes":241,"groups":4},"bots":{"GPTBot":["allowed","*",0,1],"ChatGPT-User":["allowed","*",0,1],"OAI-SearchBot":["allowed","*",0,1],"ClaudeBot":["allowed","*",0,1],"Claude-User":["allowed","*",0,1],"Claude-SearchBot":["allowed","*",0,1],"anthropic-ai":["allowed","*",0,1],"PerplexityBot":["allowed","*",0,1],"Perplexity-User":["allowed","*",0,1],"Google-Extended":["allowed","*",0,1],"Applebot-Extended":["allowed","*",0,1],"CCBot":["allowed","*",0,1],"Bytespider":["allowed","*",0,1],"Amazonbot":["allowed","named",0,1],"meta-externalagent":["allowed","*",0,1],"cohere-ai":["allowed","*",0,1],"Diffbot":["allowed","*",0,1],"ContextFetchBot":["allowed","*",0,1]},"llms":{"status":"missing","http":404,"bytes":118636},"home":{"status":"ok","http":200,"url":"https://triblive.com/privacy/","html_tokens":3526,"markdown_tokens":204,"pruned_tokens":170}},
{"rank":5242,"domain":"brandenburg.de","origin":"https://brandenburg.de","robots":{"status":"missing","http":404,"url":"https://brandenburg.de/robots.txt","bytes":196,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":196},"home":{"status":"ok","http":200,"url":"https://brandenburg.de/de/portal/bb1.c.473964.de","html_tokens":7777,"markdown_tokens":177,"pruned_tokens":144}},
{"rank":5243,"domain":"mediamatters.org","origin":"https://mediamatters.org","robots":{"status":"refused","http":403,"url":"https://mediamatters.org/robots.txt","bytes":560,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":5244,"domain":"bundesbank.de","origin":"https://bundesbank.de","robots":{"status":"ok","http":200,"url":"https://www.bundesbank.de/robots.txt","bytes":1280,"groups":2},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"crawl_delay","error":"Crawl-delay 200 s"},"home":{"status":"crawl_delay","error":"Crawl-delay 200 s"}},
{"rank":5245,"domain":"globalsign.com","origin":"https://globalsign.com","robots":{"status":"ok","http":200,"url":"https://www.globalsign.com/robots.txt","bytes":1727,"groups":12},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["partial","named",5,1],"OAI-SearchBot":["partial","named",5,1],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",14,0],"Claude-SearchBot":["partial","named",5,1],"anthropic-ai":["partial","*",14,0],"PerplexityBot":["partial","named",5,1],"Perplexity-User":["partial","*",14,0],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["partial","*",14,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["partial","*",14,0],"Diffbot":["partial","*",14,0],"ContextFetchBot":["partial","*",14,0]},"llms":{"status":"missing","http":404,"bytes":88753},"home":{"status":"ok","http":200,"url":"https://www.globalsign.com/en","html_tokens":34132,"markdown_tokens":110,"pruned_tokens":110}},
{"rank":5246,"domain":"usni.org","origin":"https://usni.org","robots":{"status":"ok","http":200,"url":"https://www.usni.org/robots.txt","bytes":2027,"groups":1},"bots":{"GPTBot":["partial","*",34,18],"ChatGPT-User":["partial","*",34,18],"OAI-SearchBot":["partial","*",34,18],"ClaudeBot":["partial","*",34,18],"Claude-User":["partial","*",34,18],"Claude-SearchBot":["partial","*",34,18],"anthropic-ai":["partial","*",34,18],"PerplexityBot":["partial","*",34,18],"Perplexity-User":["partial","*",34,18],"Google-Extended":["partial","*",34,18],"Applebot-Extended":["partial","*",34,18],"CCBot":["partial","*",34,18],"Bytespider":["partial","*",34,18],"Amazonbot":["partial","*",34,18],"meta-externalagent":["partial","*",34,18],"cohere-ai":["partial","*",34,18],"Diffbot":["partial","*",34,18],"ContextFetchBot":["partial","*",34,18]},"llms":{"status":"missing","http":404,"bytes":186},"home":{"status":"http_error","http":403,"url":"https://usni.org/"}},
{"rank":5247,"domain":"codinghorror.com","origin":"https://codinghorror.com","robots":{"status":"timeout","http":null,"url":"https://codinghorror.com/robots.txt","bytes":0,"groups":0,"error":"timed out after 10 s"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":5248,"domain":"sanofi.com","origin":"https://sanofi.com","robots":{"status":"ok","http":200,"url":"https://www.sanofi.com/robots.txt","bytes":74,"groups":1},"bots":{"GPTBot":["allowed","*",0,1],"ChatGPT-User":["allowed","*",0,1],"OAI-SearchBot":["allowed","*",0,1],"ClaudeBot":["allowed","*",0,1],"Claude-User":["allowed","*",0,1],"Claude-SearchBot":["allowed","*",0,1],"anthropic-ai":["allowed","*",0,1],"PerplexityBot":["allowed","*",0,1],"Perplexity-User":["allowed","*",0,1],"Google-Extended":["allowed","*",0,1],"Applebot-Extended":["allowed","*",0,1],"CCBot":["allowed","*",0,1],"Bytespider":["allowed","*",0,1],"Amazonbot":["allowed","*",0,1],"meta-externalagent":["allowed","*",0,1],"cohere-ai":["allowed","*",0,1],"Diffbot":["allowed","*",0,1],"ContextFetchBot":["allowed","*",0,1]},"llms":{"status":"missing","http":404,"bytes":124443},"home":{"status":"ok","http":200,"url":"https://www.sanofi.com/en","html_tokens":67751,"markdown_tokens":308,"pruned_tokens":136}},
{"rank":5249,"domain":"nic.ru","origin":"https://nic.ru","robots":{"status":"ok","http":200,"url":"https://www.nic.ru/robots.txt","bytes":8772,"groups":2},"bots":{"GPTBot":["partial","*",127,25],"ChatGPT-User":["partial","*",127,25],"OAI-SearchBot":["partial","*",127,25],"ClaudeBot":["partial","*",127,25],"Claude-User":["partial","*",127,25],"Claude-SearchBot":["partial","*",127,25],"anthropic-ai":["partial","*",127,25],"PerplexityBot":["partial","*",127,25],"Perplexity-User":["partial","*",127,25],"Google-Extended":["partial","*",127,25],"Applebot-Extended":["partial","*",127,25],"CCBot":["partial","*",127,25],"Bytespider":["partial","*",127,25],"Amazonbot":["partial","*",127,25],"meta-externalagent":["partial","*",127,25],"cohere-ai":["partial","*",127,25],"Diffbot":["partial","*",127,25],"ContextFetchBot":["partial","*",127,25]},"llms":{"status":"missing","http":404,"bytes":580427},"home":{"status":"ok","http":200,"url":"https://www.nic.ru/","html_tokens":157954,"markdown_tokens":115,"pruned_tokens":115}},
{"rank":5250,"domain":"vultr.com","origin":"https://vultr.com","robots":{"status":"ok","http":200,"url":"https://www.vultr.com/robots.txt","bytes":753,"groups":2},"bots":{"GPTBot":["partial","named",1,0],"ChatGPT-User":["partial","*",27,0],"OAI-SearchBot":["partial","*",27,0],"ClaudeBot":["partial","*",27,0],"Claude-User":["partial","*",27,0],"Claude-SearchBot":["partial","*",27,0],"anthropic-ai":["partial","*",27,0],"PerplexityBot":["partial","*",27,0],"Perplexity-User":["partial","*",27,0],"Google-Extended":["partial","*",27,0],"Applebot-Extended":["partial","*",27,0],"CCBot":["partial","*",27,0],"Bytespider":["partial","*",27,0],"Amazonbot":["partial","*",27,0],"meta-externalagent":["partial","*",27,0],"cohere-ai":["partial","*",27,0],"Diffbot":["partial","*",27,0],"ContextFetchBot":["partial","*",27,0]},"llms":{"status":"missing","http":404,"bytes":196855},"home":{"status":"http_error","http":403,"url":"https://www.vultr.com/"}},
{"rank":5251,"domain":"roma.it","origin":"https://roma.it","robots":{"status":"unreachable","http":null,"url":"https://roma.it/robots.txt","bytes":0,"groups":0,"error":"cannot resolve roma.it"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":5252,"domain":"rufus.ie","origin":"https://rufus.ie","robots":{"status":"ok","http":200,"url":"https://rufus.ie/robots.txt","bytes":24,"groups":1},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"missing","http":404,"bytes":9379},"home":{"status":"ok","http":200,"url":"https://rufus.ie/","html_tokens":478,"markdown_tokens":0,"pruned_tokens":0}},
{"rank":5253,"domain":"atptour.com","origin":"https://atptour.com","robots":{"status":"ok","http":200,"url":"https://www.atptour.com/robots.txt","bytes":347,"groups":1},"bots":{"GPTBot":["partial","*",9,0],"ChatGPT-User":["partial","*",9,0],"OAI-SearchBot":["partial","*",9,0],"ClaudeBot":["partial","*",9,0],"Claude-User":["partial","*",9,0],"Claude-SearchBot":["partial","*",9,0],"anthropic-ai":["partial","*",9,0],"PerplexityBot":["partial","*",9,0],"Perplexity-User":["partial","*",9,0],"Google-Extended":["partial","*",9,0],"Applebot-Extended":["partial","*",9,0],"CCBot":["partial","*",9,0],"Bytespider":["partial","*",9,0],"Amazonbot":["partial","*",9,0],"meta-externalagent":["partial","*",9,0],"cohere-ai":["partial","*",9,0],"Diffbot":["partial","*",9,0],"ContextFetchBot":["partial","*",9,0]},"llms":{"status":"missing","http":404,"bytes":1606},"home":{"status":"http_error","http":403,"url":"https://www.atptour.com/"}},
{"rank":5254,"domain":"grc.com","origin":"https://grc.com","robots":{"status":"ok","http":200,"url":"https://www.grc.com/robots.txt","bytes":96,"groups":1},"bots":{"GPTBot":["partial","*",4,0],"ChatGPT-User":["partial","*",4,0],"OAI-SearchBot":["partial","*",4,0],"ClaudeBot":["partial","*",4,0],"Claude-User":["partial","*",4,0],"Claude-SearchBot":["partial","*",4,0],"anthropic-ai":["partial","*",4,0],"PerplexityBot":["partial","*",4,0],"Perplexity-User":["partial","*",4,0],"Google-Extended":["partial","*",4,0],"Applebot-Extended":["partial","*",4,0],"CCBot":["partial","*",4,0],"Bytespider":["partial","*",4,0],"Amazonbot":["partial","*",4,0],"meta-externalagent":["partial","*",4,0],"cohere-ai":["partial","*",4,0],"Diffbot":["partial","*",4,0],"ContextFetchBot":["partial","*",4,0]},"llms":{"status":"missing","http":404,"bytes":1245},"home":{"status":"ok","http":200,"url":"https://www.grc.com/intro.htm","html_tokens":3150,"markdown_tokens":47,"pruned_tokens":47}},
{"rank":5255,"domain":"ncpgambling.org","origin":"https://ncpgambling.org","robots":{"status":"ok","http":200,"url":"https://www.ncpgambling.org/robots.txt","bytes":878,"groups":1},"bots":{"GPTBot":["partial","*",24,0],"ChatGPT-User":["partial","*",24,0],"OAI-SearchBot":["partial","*",24,0],"ClaudeBot":["partial","*",24,0],"Claude-User":["partial","*",24,0],"Claude-SearchBot":["partial","*",24,0],"anthropic-ai":["partial","*",24,0],"PerplexityBot":["partial","*",24,0],"Perplexity-User":["partial","*",24,0],"Google-Extended":["partial","*",24,0],"Applebot-Extended":["partial","*",24,0],"CCBot":["partial","*",24,0],"Bytespider":["partial","*",24,0],"Amazonbot":["partial","*",24,0],"meta-externalagent":["partial","*",24,0],"cohere-ai":["partial","*",24,0],"Diffbot":["partial","*",24,0],"ContextFetchBot":["partial","*",24,0]},"llms":{"status":"http_error","http":403,"bytes":5568},"home":{"status":"http_error","http":403,"url":"https://ncpgambling.org/"}},
{"rank":5256,"domain":"agenziaentrate.gov.it","origin":"https://agenziaentrate.gov.it","robots":{"status":"refused","http":403,"url":"https://agenziaentrate.gov.it/robots.txt","bytes":387,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":5257,"domain":"topsy.com","origin":"https://topsy.com","robots":{"status":"unreachable","http":null,"url":"https://topsy.com/robots.txt","bytes":0,"groups":0,"error":"cannot resolve topsy.com"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":5258,"domain":"global.toyota","origin":"https://global.toyota","robots":{"status":"missing","http":404,"url":"https://global.toyota/robots.txt/","bytes":4658,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":4658},"home":{"status":"ok","http":200,"url":"https://global.toyota/","html_tokens":4275,"markdown_tokens":27,"pruned_tokens":27}},
{"rank":5259,"domain":"advocate.com","origin":"https://advocate.com","robots":{"status":"ok","http":200,"url":"https://www.advocate.com/robots.txt","bytes":320,"groups":1},"bots":{"GPTBot":["partial","*",3,0],"ChatGPT-User":["partial","*",3,0],"OAI-SearchBot":["partial","*",3,0],"ClaudeBot":["partial","*",3,0],"Claude-User":["partial","*",3,0],"Claude-SearchBot":["partial","*",3,0],"anthropic-ai":["partial","*",3,0],"PerplexityBot":["partial","*",3,0],"Perplexity-User":["partial","*",3,0],"Google-Extended":["partial","*",3,0],"Applebot-Extended":["partial","*",3,0],"CCBot":["partial","*",3,0],"Bytespider":["partial","*",3,0],"Amazonbot":["partial","*",3,0],"meta-externalagent":["partial","*",3,0],"cohere-ai":["partial","*",3,0],"Diffbot":["partial","*",3,0],"ContextFetchBot":["partial","*",3,0]},"llms":{"status":"missing","http":404,"bytes":361345},"home":{"status":"ok","http":200,"url":"https://www.advocate.com/","html_tokens":111341,"markdown_tokens":171,"pruned_tokens":171}},
{"rank":5260,"domain":"mastercard.us","origin":"https://mastercard.us","robots":{"status":"refused","http":403,"url":"https://mastercard.us/robots.txt","bytes":375,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":5261,"domain":"laist.com","origin":"https://laist.com","robots":{"status":"ok","http":200,"url":"https://laist.com/robots.txt","bytes":555,"groups":4},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["blocked","named",1,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["blocked","named",1,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":186686},"home":{"status":"ok","http":200,"url":"https://laist.com/","html_tokens":113922,"markdown_tokens":1013,"pruned_tokens":147}},
{"rank":5262,"domain":"casinolead.ca","origin":"https://casinolead.ca","robots":{"status":"ok","http":200,"url":"https://casinolead.ca/robots.txt","bytes":531,"groups":1},"bots":{"GPTBot":["partial","*",19,0],"ChatGPT-User":["partial","*",19,0],"OAI-SearchBot":["partial","*",19,0],"ClaudeBot":["partial","*",19,0],"Claude-User":["partial","*",19,0],"Claude-SearchBot":["partial","*",19,0],"anthropic-ai":["partial","*",19,0],"PerplexityBot":["partial","*",19,0],"Perplexity-User":["partial","*",19,0],"Google-Extended":["partial","*",19,0],"Applebot-Extended":["partial","*",19,0],"CCBot":["partial","*",19,0],"Bytespider":["partial","*",19,0],"Amazonbot":["partial","*",19,0],"meta-externalagent":["partial","*",19,0],"cohere-ai":["partial","*",19,0],"Diffbot":["partial","*",19,0],"ContextFetchBot":["partial","*",19,0]},"llms":{"status":"missing","http":404,"bytes":39146},"home":{"status":"ok","http":200,"url":"https://casinolead.ca/","html_tokens":27240,"markdown_tokens":1450,"pruned_tokens":138}},
{"rank":5263,"domain":"odu.edu","origin":"https://odu.edu","robots":{"status":"refused","http":403,"url":"https://odu.edu/robots.txt","bytes":560,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":5264,"domain":"viadeo.com","origin":"https://viadeo.com","robots":{"status":"ok","http":200,"url":"https://viadeo.journaldunet.com/robots.txt","bytes":388,"groups":5},"bots":{"GPTBot":["partial","*",6,0],"ChatGPT-User":["partial","*",6,0],"OAI-SearchBot":["partial","*",6,0],"ClaudeBot":["partial","*",6,0],"Claude-User":["partial","*",6,0],"Claude-SearchBot":["partial","*",6,0],"anthropic-ai":["partial","*",6,0],"PerplexityBot":["partial","*",6,0],"Perplexity-User":["partial","*",6,0],"Google-Extended":["partial","*",6,0],"Applebot-Extended":["partial","*",6,0],"CCBot":["partial","*",6,0],"Bytespider":["partial","*",6,0],"Amazonbot":["partial","*",6,0],"meta-externalagent":["partial","*",6,0],"cohere-ai":["partial","*",6,0],"Diffbot":["partial","*",6,0],"ContextFetchBot":["partial","*",6,0]},"llms":{"status":"missing","http":404,"bytes":39457},"home":{"status":"ok","http":200,"url":"https://viadeo.journaldunet.com/","html_tokens":27442,"markdown_tokens":1028,"pruned_tokens":170}},
{"rank":5265,"domain":"urbanoutfitters.com","origin":"https://urbanoutfitters.com","robots":{"status":"refused","http":403,"url":"https://www.urbanoutfitters.com/robots.txt","bytes":779,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":5266,"domain":"grab.com","origin":"https://grab.com","robots":{"status":"ok","http":200,"url":"https://grab.com/robots.txt","bytes":5788,"groups":1},"bots":{"GPTBot":["partial","*",70,11],"ChatGPT-User":["partial","*",70,11],"OAI-SearchBot":["partial","*",70,11],"ClaudeBot":["partial","*",70,11],"Claude-User":["partial","*",70,11],"Claude-SearchBot":["partial","*",70,11],"anthropic-ai":["partial","*",70,11],"PerplexityBot":["partial","*",70,11],"Perplexity-User":["partial","*",70,11],"Google-Extended":["partial","*",70,11],"Applebot-Extended":["partial","*",70,11],"CCBot":["partial","*",70,11],"Bytespider":["partial","*",70,11],"Amazonbot":["partial","*",70,11],"meta-externalagent":["partial","*",70,11],"cohere-ai":["partial","*",70,11],"Diffbot":["partial","*",70,11],"ContextFetchBot":["partial","*",70,11]},"llms":{"status":"missing","http":404,"bytes":146},"home":{"status":"ok","http":200,"url":"https://www.grab.com/sg/","html_tokens":117743,"markdown_tokens":885,"pruned_tokens":312}},
{"rank":5267,"domain":"fodors.com","origin":"https://fodors.com","robots":{"status":"refused","http":403,"url":"https://fodors.com/robots.txt","bytes":5530,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":5268,"domain":"htmlcodex.com","origin":"https://htmlcodex.com","robots":{"status":"ok","http":200,"url":"https://htmlcodex.com/robots.txt","bytes":249,"groups":2},"bots":{"GPTBot":["partial","*",3,0],"ChatGPT-User":["partial","*",3,0],"OAI-SearchBot":["partial","*",3,0],"ClaudeBot":["partial","*",3,0],"Claude-User":["partial","*",3,0],"Claude-SearchBot":["partial","*",3,0],"anthropic-ai":["partial","*",3,0],"PerplexityBot":["partial","*",3,0],"Perplexity-User":["partial","*",3,0],"Google-Extended":["partial","*",3,0],"Applebot-Extended":["partial","*",3,0],"CCBot":["partial","*",3,0],"Bytespider":["partial","*",3,0],"Amazonbot":["partial","*",3,0],"meta-externalagent":["partial","*",3,0],"cohere-ai":["partial","*",3,0],"Diffbot":["partial","*",3,0],"ContextFetchBot":["partial","*",3,0]},"llms":{"status":"missing","http":404,"bytes":65201},"home":{"status":"ok","http":200,"url":"https://htmlcodex.com/","html_tokens":36844,"markdown_tokens":108,"pruned_tokens":108}},
{"rank":5269,"domain":"sacred-texts.com","origin":"https://sacred-texts.com","robots":{"status":"ok","http":200,"url":"https://sacred-texts.com/robots.txt","bytes":392,"groups":4},"bots":{"GPTBot":["partial","*",3,1],"ChatGPT-User":["partial","*",3,1],"OAI-SearchBot":["partial","*",3,1],"ClaudeBot":["partial","*",3,1],"Claude-User":["partial","*",3,1],"Claude-SearchBot":["partial","*",3,1],"anthropic-ai":["partial","*",3,1],"PerplexityBot":["partial","*",3,1],"Perplexity-User":["partial","*",3,1],"Google-Extended":["partial","*",3,1],"Applebot-Extended":["partial","*",3,1],"CCBot":["partial","*",3,1],"Bytespider":["partial","*",3,1],"Amazonbot":["partial","*",3,1],"meta-externalagent":["partial","*",3,1],"cohere-ai":["partial","*",3,1],"Diffbot":["partial","*",3,1],"ContextFetchBot":["partial","*",3,1]},"llms":{"status":"http_error","http":403,"bytes":4910},"home":{"status":"ok","http":200,"url":"https://sacred-texts.com/","html_tokens":75075,"markdown_tokens":172,"pruned_tokens":131}},
{"rank":5270,"domain":"sgi.com","origin":"https://sgi.com","robots":{"status":"timeout","http":null,"url":"https://sgi.com/robots.txt","bytes":0,"groups":0,"error":"ConnectTimeout"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":5271,"domain":"chop.edu","origin":"https://chop.edu","robots":{"status":"refused","http":403,"url":"https://chop.edu/robots.txt","bytes":560,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":5272,"domain":"mercer.com","origin":"https://mercer.com","robots":{"status":"ok","http":200,"url":"https://www.mercer.com/robots.txt","bytes":285338,"groups":1},"bots":{"GPTBot":["partial","*",3589,1],"ChatGPT-User":["partial","*",3589,1],"OAI-SearchBot":["partial","*",3589,1],"ClaudeBot":["partial","*",3589,1],"Claude-User":["partial","*",3589,1],"Claude-SearchBot":["partial","*",3589,1],"anthropic-ai":["partial","*",3589,1],"PerplexityBot":["partial","*",3589,1],"Perplexity-User":["partial","*",3589,1],"Google-Extended":["partial","*",3589,1],"Applebot-Extended":["partial","*",3589,1],"CCBot":["partial","*",3589,1],"Bytespider":["partial","*",3589,1],"Amazonbot":["partial","*",3589,1],"meta-externalagent":["partial","*",3589,1],"cohere-ai":["partial","*",3589,1],"Diffbot":["partial","*",3589,1],"ContextFetchBot":["partial","*",3589,1]},"llms":{"status":"missing","http":404,"bytes":109728},"home":{"status":"ok","http":200,"url":"https://www.mercer.com/en-us/","html_tokens":38943,"markdown_tokens":73,"pruned_tokens":73}},
{"rank":5273,"domain":"wpi.edu","origin":"https://www.wpi.edu","robots":{"status":"ok","http":200,"url":"https://www.wpi.edu/robots.txt","bytes":3303,"groups":13},"bots":{"GPTBot":["allowed","named",0,2],"ChatGPT-User":["allowed","named",0,2],"OAI-SearchBot":["partial","*",38,20],"ClaudeBot":["partial","*",38,20],"Claude-User":["partial","*",38,20],"Claude-SearchBot":["partial","*",38,20],"anthropic-ai":["allowed","named",0,2],"PerplexityBot":["allowed","named",0,2],"Perplexity-User":["partial","*",38,20],"Google-Extended":["partial","*",38,20],"Applebot-Extended":["partial","*",38,20],"CCBot":["partial","*",38,20],"Bytespider":["partial","*",38,20],"Amazonbot":["partial","named",5,0],"meta-externalagent":["allowed","named",0,2],"cohere-ai":["allowed","named",0,2],"Diffbot":["partial","*",38,20],"ContextFetchBot":["partial","*",38,20]},"llms":{"status":"present","http":200,"bytes":5929},"home":{"status":"ok","http":200,"url":"https://www.wpi.edu/","html_tokens":63400,"markdown_tokens":64,"pruned_tokens":64}},
{"rank":5274,"domain":"kadokawa.co.jp","origin":"https://kadokawa.co.jp","robots":{"status":"ok","http":200,"url":"https://www.kadokawa.co.jp/robots.txt","bytes":376,"groups":2},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"missing","http":404,"bytes":103608},"home":{"status":"ok","http":200,"url":"https://www.kadokawa.co.jp/","html_tokens":39293,"markdown_tokens":123,"pruned_tokens":123}},
{"rank":5275,"domain":"nabu.de","origin":"https://nabu.de","robots":{"status":"ok","http":200,"url":"https://www.nabu.de/robots.txt","bytes":646,"groups":1},"bots":{"GPTBot":["partial","*",13,6],"ChatGPT-User":["partial","*",13,6],"OAI-SearchBot":["partial","*",13,6],"ClaudeBot":["partial","*",13,6],"Claude-User":["partial","*",13,6],"Claude-SearchBot":["partial","*",13,6],"anthropic-ai":["partial","*",13,6],"PerplexityBot":["partial","*",13,6],"Perplexity-User":["partial","*",13,6],"Google-Extended":["partial","*",13,6],"Applebot-Extended":["partial","*",13,6],"CCBot":["partial","*",13,6],"Bytespider":["partial","*",13,6],"Amazonbot":["partial","*",13,6],"meta-externalagent":["partial","*",13,6],"cohere-ai":["partial","*",13,6],"Diffbot":["partial","*",13,6],"ContextFetchBot":["partial","*",13,6]},"llms":{"status":"missing","http":404,"bytes":65269},"home":{"status":"ok","http":200,"url":"https://www.nabu.de/","html_tokens":26542,"markdown_tokens":202,"pruned_tokens":52}},
{"rank":5276,"domain":"saude.gov.br","origin":"https://saude.gov.br","robots":{"status":"html","http":200,"url":"https://www.gov.br/saude/pt-br","bytes":159218,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":5277,"domain":"descript.com","origin":"https://descript.com","robots":{"status":"error","http":null,"url":"https://descript.com/robots.txt","bytes":0,"groups":0,"error":"unsupported content encoding br"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":5278,"domain":"what3words.com","origin":"https://what3words.com","robots":{"status":"ok","http":200,"url":"https://what3words.com/robots.txt","bytes":1462,"groups":1},"bots":{"GPTBot":["partial","*",37,1],"ChatGPT-User":["partial","*",37,1],"OAI-SearchBot":["partial","*",37,1],"ClaudeBot":["partial","*",37,1],"Claude-User":["partial","*",37,1],"Claude-SearchBot":["partial","*",37,1],"anthropic-ai":["partial","*",37,1],"PerplexityBot":["partial","*",37,1],"Perplexity-User":["partial","*",37,1],"Google-Extended":["partial","*",37,1],"Applebot-Extended":["partial","*",37,1],"CCBot":["partial","*",37,1],"Bytespider":["partial","*",37,1],"Amazonbot":["partial","*",37,1],"meta-externalagent":["partial","*",37,1],"cohere-ai":["partial","*",37,1],"Diffbot":["partial","*",37,1],"ContextFetchBot":["partial","*",37,1]},"llms":{"status":"missing","http":404,"bytes":92462},"home":{"status":"http_error","http":403,"url":"https://what3words.com/clip.apples.leap?redirect=true"}},
{"rank":5279,"domain":"arcor.de","origin":"https://arcor.de","robots":{"status":"ok","http":200,"url":"https://arcor.de/robots.txt","bytes":25,"groups":1},"bots":{"GPTBot":["blocked","*",1,0],"ChatGPT-User":["blocked","*",1,0],"OAI-SearchBot":["blocked","*",1,0],"ClaudeBot":["blocked","*",1,0],"Claude-User":["blocked","*",1,0],"Claude-SearchBot":["blocked","*",1,0],"anthropic-ai":["blocked","*",1,0],"PerplexityBot":["blocked","*",1,0],"Perplexity-User":["blocked","*",1,0],"Google-Extended":["blocked","*",1,0],"Applebot-Extended":["blocked","*",1,0],"CCBot":["blocked","*",1,0],"Bytespider":["blocked","*",1,0],"Amazonbot":["blocked","*",1,0],"meta-externalagent":["blocked","*",1,0],"cohere-ai":["blocked","*",1,0],"Diffbot":["blocked","*",1,0],"ContextFetchBot":["blocked","*",1,0]},"llms":{"status":"disallowed"},"home":{"status":"disallowed"}},
{"rank":5280,"domain":"sse.com.cn","origin":"https://sse.com.cn","robots":{"status":"timeout","http":null,"url":"https://sse.com.cn/robots.txt","bytes":0,"groups":0,"error":"ConnectTimeout"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":5281,"domain":"antaranews.com","origin":"https://antaranews.com","robots":{"status":"ok","http":200,"url":"https://www.antaranews.com/robots.txt","bytes":294,"groups":6},"bots":{"GPTBot":["partial","*",2,1],"ChatGPT-User":["partial","*",2,1],"OAI-SearchBot":["partial","*",2,1],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",2,1],"Claude-SearchBot":["partial","*",2,1],"anthropic-ai":["partial","*",2,1],"PerplexityBot":["partial","*",2,1],"Perplexity-User":["partial","*",2,1],"Google-Extended":["partial","*",2,1],"Applebot-Extended":["partial","*",2,1],"CCBot":["partial","*",2,1],"Bytespider":["partial","*",2,1],"Amazonbot":["partial","*",2,1],"meta-externalagent":["partial","*",2,1],"cohere-ai":["partial","*",2,1],"Diffbot":["partial","*",2,1],"ContextFetchBot":["partial","*",2,1]},"llms":{"status":"missing","http":404,"bytes":795395},"home":{"status":"ok","http":200,"url":"https://www.antaranews.com/","html_tokens":198889,"markdown_tokens":3,"pruned_tokens":3}},
{"rank":5282,"domain":"bt.cn","origin":"https://bt.cn","robots":{"status":"ok","http":200,"url":"https://www.bt.cn/robots.txt","bytes":59,"groups":1},"bots":{"GPTBot":["partial","*",1,1],"ChatGPT-User":["partial","*",1,1],"OAI-SearchBot":["partial","*",1,1],"ClaudeBot":["partial","*",1,1],"Claude-User":["partial","*",1,1],"Claude-SearchBot":["partial","*",1,1],"anthropic-ai":["partial","*",1,1],"PerplexityBot":["partial","*",1,1],"Perplexity-User":["partial","*",1,1],"Google-Extended":["partial","*",1,1],"Applebot-Extended":["partial","*",1,1],"CCBot":["partial","*",1,1],"Bytespider":["partial","*",1,1],"Amazonbot":["partial","*",1,1],"meta-externalagent":["partial","*",1,1],"cohere-ai":["partial","*",1,1],"Diffbot":["partial","*",1,1],"ContextFetchBot":["partial","*",1,1]},"llms":{"status":"present","http":200,"bytes":7709},"home":{"status":"ok","http":200,"url":"https://www.bt.cn/new/index.html","html_tokens":22390,"markdown_tokens":105,"pruned_tokens":105}},
{"rank":5283,"domain":"pia.jp","origin":"https://pia.jp","robots":{"status":"html","http":200,"url":"http://sorry.pia.jp/piajp/index.html","bytes":1035,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":5284,"domain":"betway.com","origin":"https://betway.com","robots":{"status":"ok","http":200,"url":"https://betway.com/robots.txt","bytes":3126,"groups":1},"bots":{"GPTBot":["partial","*",7,74],"ChatGPT-User":["partial","*",7,74],"OAI-SearchBot":["partial","*",7,74],"ClaudeBot":["partial","*",7,74],"Claude-User":["partial","*",7,74],"Claude-SearchBot":["partial","*",7,74],"anthropic-ai":["partial","*",7,74],"PerplexityBot":["partial","*",7,74],"Perplexity-User":["partial","*",7,74],"Google-Extended":["partial","*",7,74],"Applebot-Extended":["partial","*",7,74],"CCBot":["partial","*",7,74],"Bytespider":["partial","*",7,74],"Amazonbot":["partial","*",7,74],"meta-externalagent":["partial","*",7,74],"cohere-ai":["partial","*",7,74],"Diffbot":["partial","*",7,74],"ContextFetchBot":["partial","*",7,74]},"llms":{"status":"missing","http":404,"bytes":9},"home":{"status":"ok","http":200,"url":"https://betway.com/","html_tokens":21177,"markdown_tokens":263,"pruned_tokens":137}},
{"rank":5285,"domain":"quebec.ca","origin":"https://quebec.ca","robots":{"status":"ok","http":200,"url":"https://www.quebec.ca/robots.txt","bytes":863,"groups":1},"bots":{"GPTBot":["partial","*",15,8],"ChatGPT-User":["partial","*",15,8],"OAI-SearchBot":["partial","*",15,8],"ClaudeBot":["partial","*",15,8],"Claude-User":["partial","*",15,8],"Claude-SearchBot":["partial","*",15,8],"anthropic-ai":["partial","*",15,8],"PerplexityBot":["partial","*",15,8],"Perplexity-User":["partial","*",15,8],"Google-Extended":["partial","*",15,8],"Applebot-Extended":["partial","*",15,8],"CCBot":["partial","*",15,8],"Bytespider":["partial","*",15,8],"Amazonbot":["partial","*",15,8],"meta-externalagent":["partial","*",15,8],"cohere-ai":["partial","*",15,8],"Diffbot":["partial","*",15,8],"ContextFetchBot":["partial","*",15,8]},"llms":{"status":"present","http":200,"bytes":2586},"home":{"status":"ok","http":200,"url":"https://www.quebec.ca/","html_tokens":14486,"markdown_tokens":51,"pruned_tokens":51}},
{"rank":5286,"domain":"mg.co.za","origin":"https://mg.co.za","robots":{"status":"ok","http":200,"url":"https://mg.co.za/robots.txt","bytes":164,"groups":2},"bots":{"GPTBot":["partial","*",2,1],"ChatGPT-User":["partial","*",2,1],"OAI-SearchBot":["partial","*",2,1],"ClaudeBot":["partial","*",2,1],"Claude-User":["partial","*",2,1],"Claude-SearchBot":["partial","*",2,1],"anthropic-ai":["partial","*",2,1],"PerplexityBot":["partial","*",2,1],"Perplexity-User":["partial","*",2,1],"Google-Extended":["partial","named",1,1],"Applebot-Extended":["partial","*",2,1],"CCBot":["partial","*",2,1],"Bytespider":["partial","*",2,1],"Amazonbot":["partial","*",2,1],"meta-externalagent":["partial","*",2,1],"cohere-ai":["partial","*",2,1],"Diffbot":["partial","*",2,1],"ContextFetchBot":["partial","*",2,1]},"llms":{"status":"missing","http":404,"bytes":91803},"home":{"status":"ok","http":200,"url":"https://mg.co.za/","html_tokens":208132,"markdown_tokens":78,"pruned_tokens":78}},
{"rank":5287,"domain":"carecredit.com","origin":"https://carecredit.com","robots":{"status":"ok","http":200,"url":"https://www.carecredit.com/robots.txt","bytes":1072,"groups":1},"bots":{"GPTBot":["partial","*",24,0],"ChatGPT-User":["partial","*",24,0],"OAI-SearchBot":["partial","*",24,0],"ClaudeBot":["partial","*",24,0],"Claude-User":["partial","*",24,0],"Claude-SearchBot":["partial","*",24,0],"anthropic-ai":["partial","*",24,0],"PerplexityBot":["partial","*",24,0],"Perplexity-User":["partial","*",24,0],"Google-Extended":["partial","*",24,0],"Applebot-Extended":["partial","*",24,0],"CCBot":["partial","*",24,0],"Bytespider":["partial","*",24,0],"Amazonbot":["partial","*",24,0],"meta-externalagent":["partial","*",24,0],"cohere-ai":["partial","*",24,0],"Diffbot":["partial","*",24,0],"ContextFetchBot":["partial","*",24,0]},"llms":{"status":"missing","http":404,"bytes":115849},"home":{"status":"ok","http":200,"url":"https://www.carecredit.com/","html_tokens":43513,"markdown_tokens":729,"pruned_tokens":57}},
{"rank":5288,"domain":"err.ee","origin":"https://err.ee","robots":{"status":"ok","http":200,"url":"https://www.err.ee/robots.txt","bytes":279,"groups":7},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"html","http":200,"bytes":362231},"home":{"status":"ok","http":200,"url":"https://www.err.ee/","html_tokens":90455,"markdown_tokens":71,"pruned_tokens":71}},
{"rank":5289,"domain":"audioboom.com","origin":"https://audioboom.com","robots":{"status":"ok","http":200,"url":"https://audioboom.com/robots.txt","bytes":205,"groups":1},"bots":{"GPTBot":["partial","*",7,0],"ChatGPT-User":["partial","*",7,0],"OAI-SearchBot":["partial","*",7,0],"ClaudeBot":["partial","*",7,0],"Claude-User":["partial","*",7,0],"Claude-SearchBot":["partial","*",7,0],"anthropic-ai":["partial","*",7,0],"PerplexityBot":["partial","*",7,0],"Perplexity-User":["partial","*",7,0],"Google-Extended":["partial","*",7,0],"Applebot-Extended":["partial","*",7,0],"CCBot":["partial","*",7,0],"Bytespider":["partial","*",7,0],"Amazonbot":["partial","*",7,0],"meta-externalagent":["partial","*",7,0],"cohere-ai":["partial","*",7,0],"Diffbot":["partial","*",7,0],"ContextFetchBot":["partial","*",7,0]},"llms":{"status":"missing","http":404,"bytes":4860},"home":{"status":"ok","http":200,"url":"https://audioboom.com/","html_tokens":6653,"markdown_tokens":235,"pruned_tokens":49}},
{"rank":5290,"domain":"aviationweek.com","origin":"https://aviationweek.com","robots":{"status":"ok","http":200,"url":"https://aviationweek.com/robots.txt","bytes":1750,"groups":1},"bots":{"GPTBot":["partial","*",26,18],"ChatGPT-User":["partial","*",26,18],"OAI-SearchBot":["partial","*",26,18],"ClaudeBot":["partial","*",26,18],"Claude-User":["partial","*",26,18],"Claude-SearchBot":["partial","*",26,18],"anthropic-ai":["partial","*",26,18],"PerplexityBot":["partial","*",26,18],"Perplexity-User":["partial","*",26,18],"Google-Extended":["partial","*",26,18],"Applebot-Extended":["partial","*",26,18],"CCBot":["partial","*",26,18],"Bytespider":["partial","*",26,18],"Amazonbot":["partial","*",26,18],"meta-externalagent":["partial","*",26,18],"cohere-ai":["partial","*",26,18],"Diffbot":["partial","*",26,18],"ContextFetchBot":["partial","*",26,18]},"llms":{"status":"missing","http":404,"bytes":190},"home":{"status":"ok","http":200,"url":"https://aviationweek.com/","html_tokens":53683,"markdown_tokens":175,"pruned_tokens":174}},
{"rank":5291,"domain":"tripadvisor.de","origin":"https://tripadvisor.de","robots":{"status":"ok","http":200,"url":"https://www.tripadvisor.de/robots.txt","bytes":22865,"groups":9},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["partial","named",623,8],"OAI-SearchBot":["partial","named",623,8],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",621,8],"Claude-SearchBot":["partial","*",621,8],"anthropic-ai":["partial","*",621,8],"PerplexityBot":["partial","named",625,8],"Perplexity-User":["partial","*",621,8],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["partial","*",621,8],"ContextFetchBot":["partial","*",621,8]},"llms":{"status":"http_error","http":403,"bytes":774},"home":{"status":"http_error","http":403,"url":"https://www.tripadvisor.de/"}},
{"rank":5292,"domain":"cplusplus.com","origin":"https://cplusplus.com","robots":{"status":"ok","http":200,"url":"https://cplusplus.com/robots.txt","bytes":32,"groups":1},"bots":{"GPTBot":["partial","*",1,0],"ChatGPT-User":["partial","*",1,0],"OAI-SearchBot":["partial","*",1,0],"ClaudeBot":["partial","*",1,0],"Claude-User":["partial","*",1,0],"Claude-SearchBot":["partial","*",1,0],"anthropic-ai":["partial","*",1,0],"PerplexityBot":["partial","*",1,0],"Perplexity-User":["partial","*",1,0],"Google-Extended":["partial","*",1,0],"Applebot-Extended":["partial","*",1,0],"CCBot":["partial","*",1,0],"Bytespider":["partial","*",1,0],"Amazonbot":["partial","*",1,0],"meta-externalagent":["partial","*",1,0],"cohere-ai":["partial","*",1,0],"Diffbot":["partial","*",1,0],"ContextFetchBot":["partial","*",1,0]},"llms":{"status":"html","http":200,"bytes":1473},"home":{"status":"ok","http":200,"url":"https://cplusplus.com/","html_tokens":4151,"markdown_tokens":319,"pruned_tokens":160}},
{"rank":5293,"domain":"johnlewis.com","origin":"https://johnlewis.com","robots":{"status":"ok","http":200,"url":"https://www.johnlewis.com/robots.txt","bytes":1404,"groups":1},"bots":{"GPTBot":["partial","*",34,4],"ChatGPT-User":["partial","*",34,4],"OAI-SearchBot":["partial","*",34,4],"ClaudeBot":["partial","*",34,4],"Claude-User":["partial","*",34,4],"Claude-SearchBot":["partial","*",34,4],"anthropic-ai":["partial","*",34,4],"PerplexityBot":["partial","*",34,4],"Perplexity-User":["partial","*",34,4],"Google-Extended":["partial","*",34,4],"Applebot-Extended":["partial","*",34,4],"CCBot":["partial","*",34,4],"Bytespider":["partial","*",34,4],"Amazonbot":["partial","*",34,4],"meta-externalagent":["partial","*",34,4],"cohere-ai":["partial","*",34,4],"Diffbot":["partial","*",34,4],"ContextFetchBot":["partial","*",34,4]},"llms":{"status":"missing","http":404,"bytes":302989},"home":{"status":"ok","http":200,"url":"https://www.johnlewis.com/","html_tokens":138589,"markdown_tokens":22,"pruned_tokens":22}},
{"rank":5294,"domain":"highsnobiety.com","origin":"https://highsnobiety.com","robots":{"status":"ok","http":200,"url":"https://www.highsnobiety.com/robots.txt","bytes":1209,"groups":4},"bots":{"GPTBot":["partial","*",15,0],"ChatGPT-User":["partial","*",15,0],"OAI-SearchBot":["partial","*",15,0],"ClaudeBot":["partial","*",15,0],"Claude-User":["partial","*",15,0],"Claude-SearchBot":["partial","*",15,0],"anthropic-ai":["partial","*",15,0],"PerplexityBot":["partial","*",15,0],"Perplexity-User":["partial","*",15,0],"Google-Extended":["partial","*",15,0],"Applebot-Extended":["partial","*",15,0],"CCBot":["partial","*",15,0],"Bytespider":["partial","*",15,0],"Amazonbot":["partial","*",15,0],"meta-externalagent":["partial","*",15,0],"cohere-ai":["partial","*",15,0],"Diffbot":["partial","*",15,0],"ContextFetchBot":["partial","*",15,0]},"llms":{"status":"missing","http":404,"bytes":186942},"home":{"status":"ok","http":200,"url":"https://www.highsnobiety.com/","html_tokens":187791,"markdown_tokens":691,"pruned_tokens":19}},
{"rank":5295,"domain":"skift.com","origin":"https://skift.com","robots":{"status":"ok","http":200,"url":"https://skift.com/robots.txt","bytes":2895,"groups":34},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["partial","*",1,1],"OAI-SearchBot":["partial","*",1,1],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",1,1],"Claude-SearchBot":["partial","*",1,1],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["partial","*",1,1],"Perplexity-User":["partial","*",1,1],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",1,1]},"llms":{"status":"missing","http":404,"bytes":132019},"home":{"status":"ok","http":200,"url":"https://skift.com/","html_tokens":72788,"markdown_tokens":589,"pruned_tokens":314}},
{"rank":5296,"domain":"linkr.bio","origin":"https://linkr.bio","robots":{"status":"missing","http":404,"url":"https://linkr.bio/robots.txt","bytes":127,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":127},"home":{"status":"ok","http":200,"url":"https://linkr.bio/","html_tokens":770,"markdown_tokens":11,"pruned_tokens":11}},
{"rank":5297,"domain":"aok.de","origin":"https://aok.de","robots":{"status":"ok","http":200,"url":"https://www.aok.de/robots.txt","bytes":1478,"groups":1},"bots":{"GPTBot":["partial","*",33,2],"ChatGPT-User":["partial","*",33,2],"OAI-SearchBot":["partial","*",33,2],"ClaudeBot":["partial","*",33,2],"Claude-User":["partial","*",33,2],"Claude-SearchBot":["partial","*",33,2],"anthropic-ai":["partial","*",33,2],"PerplexityBot":["partial","*",33,2],"Perplexity-User":["partial","*",33,2],"Google-Extended":["partial","*",33,2],"Applebot-Extended":["partial","*",33,2],"CCBot":["partial","*",33,2],"Bytespider":["partial","*",33,2],"Amazonbot":["partial","*",33,2],"meta-externalagent":["partial","*",33,2],"cohere-ai":["partial","*",33,2],"Diffbot":["partial","*",33,2],"ContextFetchBot":["partial","*",33,2]},"llms":{"status":"missing","http":404,"bytes":5331},"home":{"status":"ok","http":200,"url":"https://www.aok.de/pk/","html_tokens":125558,"markdown_tokens":185,"pruned_tokens":157}},
{"rank":5298,"domain":"sympla.com.br","origin":"https://sympla.com.br","robots":{"status":"ok","http":200,"url":"https://www.sympla.com.br/robots.txt","bytes":22,"groups":1},"bots":{"GPTBot":["allowed","*",0,1],"ChatGPT-User":["allowed","*",0,1],"OAI-SearchBot":["allowed","*",0,1],"ClaudeBot":["allowed","*",0,1],"Claude-User":["allowed","*",0,1],"Claude-SearchBot":["allowed","*",0,1],"anthropic-ai":["allowed","*",0,1],"PerplexityBot":["allowed","*",0,1],"Perplexity-User":["allowed","*",0,1],"Google-Extended":["allowed","*",0,1],"Applebot-Extended":["allowed","*",0,1],"CCBot":["allowed","*",0,1],"Bytespider":["allowed","*",0,1],"Amazonbot":["allowed","*",0,1],"meta-externalagent":["allowed","*",0,1],"cohere-ai":["allowed","*",0,1],"Diffbot":["allowed","*",0,1],"ContextFetchBot":["allowed","*",0,1]},"llms":{"status":"missing","http":404,"bytes":74433},"home":{"status":"ok","http":200,"url":"https://www.sympla.com.br/","html_tokens":80377,"markdown_tokens":46,"pruned_tokens":46}},
{"rank":5299,"domain":"denic.de","origin":"https://denic.de","robots":{"status":"missing","http":404,"url":"https://www.denic.de/robots.txt","bytes":210913,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":210913},"home":{"status":"ok","http":200,"url":"https://www.denic.de/","html_tokens":95278,"markdown_tokens":486,"pruned_tokens":21}},
{"rank":5300,"domain":"euromonitor.com","origin":"https://euromonitor.com","robots":{"status":"ok","http":200,"url":"https://www.euromonitor.com/robots.txt","bytes":219,"groups":1},"bots":{"GPTBot":["partial","*",7,0],"ChatGPT-User":["partial","*",7,0],"OAI-SearchBot":["partial","*",7,0],"ClaudeBot":["partial","*",7,0],"Claude-User":["partial","*",7,0],"Claude-SearchBot":["partial","*",7,0],"anthropic-ai":["partial","*",7,0],"PerplexityBot":["partial","*",7,0],"Perplexity-User":["partial","*",7,0],"Google-Extended":["partial","*",7,0],"Applebot-Extended":["partial","*",7,0],"CCBot":["partial","*",7,0],"Bytespider":["partial","*",7,0],"Amazonbot":["partial","*",7,0],"meta-externalagent":["partial","*",7,0],"cohere-ai":["partial","*",7,0],"Diffbot":["partial","*",7,0],"ContextFetchBot":["partial","*",7,0]},"llms":{"status":"present","http":200,"bytes":1873},"home":{"status":"ok","http":200,"url":"https://www.euromonitor.com/","html_tokens":48134,"markdown_tokens":1538,"pruned_tokens":189}},
{"rank":5301,"domain":"farnell.com","origin":"https://farnell.com","robots":{"status":"unreachable","http":null,"url":"https://farnell.com/robots.txt","bytes":0,"groups":0,"error":"[SSL: CERTIFICATE_VERIFY_FAILED] certificate verify failed: unable to get local issuer certificate (_ssl.c:1028)"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":5302,"domain":"conta.cc","origin":"https://conta.cc","robots":{"status":"ok","http":200,"url":"https://conta.cc/robots.txt","bytes":123,"groups":1},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"missing","http":404,"bytes":6075},"home":{"status":"http_error","http":403,"url":"https://www.constantcontact.com"}},
{"rank":5303,"domain":"recreation.gov","origin":"https://recreation.gov","robots":{"status":"ok","http":200,"url":"https://www.recreation.gov/robots.txt","bytes":326,"groups":1},"bots":{"GPTBot":["partial","*",5,0],"ChatGPT-User":["partial","*",5,0],"OAI-SearchBot":["partial","*",5,0],"ClaudeBot":["partial","*",5,0],"Claude-User":["partial","*",5,0],"Claude-SearchBot":["partial","*",5,0],"anthropic-ai":["partial","*",5,0],"PerplexityBot":["partial","*",5,0],"Perplexity-User":["partial","*",5,0],"Google-Extended":["partial","*",5,0],"Applebot-Extended":["partial","*",5,0],"CCBot":["partial","*",5,0],"Bytespider":["partial","*",5,0],"Amazonbot":["partial","*",5,0],"meta-externalagent":["partial","*",5,0],"cohere-ai":["partial","*",5,0],"Diffbot":["partial","*",5,0],"ContextFetchBot":["partial","*",5,0]},"llms":{"status":"html","http":200,"bytes":3765},"home":{"status":"ok","http":200,"url":"https://www.recreation.gov/","html_tokens":9639,"markdown_tokens":2206,"pruned_tokens":220}},
{"rank":5304,"domain":"calpoly.edu","origin":"https://calpoly.edu","robots":{"status":"refused","http":403,"url":"https://calpoly.edu/robots.txt","bytes":560,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":5305,"domain":"unbounce.com","origin":"https://unbounce.com","robots":{"status":"ok","http":200,"url":"https://unbounce.com/robots.txt","bytes":775,"groups":2},"bots":{"GPTBot":["partial","named",6,4],"ChatGPT-User":["partial","named",6,4],"OAI-SearchBot":["partial","named",6,4],"ClaudeBot":["partial","named",6,4],"Claude-User":["partial","named",6,4],"Claude-SearchBot":["partial","named",6,4],"anthropic-ai":["partial","*",6,4],"PerplexityBot":["partial","named",6,4],"Perplexity-User":["partial","named",6,4],"Google-Extended":["partial","named",6,4],"Applebot-Extended":["partial","named",6,4],"CCBot":["partial","named",6,4],"Bytespider":["partial","*",6,4],"Amazonbot":["partial","*",6,4],"meta-externalagent":["partial","*",6,4],"cohere-ai":["partial","*",6,4],"Diffbot":["partial","*",6,4],"ContextFetchBot":["partial","*",6,4]},"llms":{"status":"missing","http":404,"bytes":31768},"home":{"status":"ok","http":200,"url":"https://unbounce.com/","html_tokens":75855,"markdown_tokens":605,"pruned_tokens":231}},
{"rank":5306,"domain":"sass-lang.com","origin":"https://sass-lang.com","robots":{"status":"ok","http":200,"url":"https://sass-lang.com/robots.txt","bytes":121,"groups":1},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"missing","http":404,"bytes":9511},"home":{"status":"ok","http":200,"url":"https://sass-lang.com/","html_tokens":3027,"markdown_tokens":234,"pruned_tokens":234}},
{"rank":5307,"domain":"qsl.net","origin":"https://qsl.net","robots":{"status":"ok","http":200,"url":"https://qsl.net/robots.txt","bytes":1782,"groups":48},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["partial","*",4,0],"OAI-SearchBot":["partial","*",4,0],"ClaudeBot":["partial","*",4,0],"Claude-User":["partial","*",4,0],"Claude-SearchBot":["partial","*",4,0],"anthropic-ai":["partial","*",4,0],"PerplexityBot":["partial","*",4,0],"Perplexity-User":["partial","*",4,0],"Google-Extended":["partial","*",4,0],"Applebot-Extended":["partial","*",4,0],"CCBot":["partial","*",4,0],"Bytespider":["partial","*",4,0],"Amazonbot":["partial","*",4,0],"meta-externalagent":["partial","*",4,0],"cohere-ai":["partial","*",4,0],"Diffbot":["partial","*",4,0],"ContextFetchBot":["partial","*",4,0]},"llms":{"status":"html","http":200,"bytes":5439},"home":{"status":"ok","http":200,"url":"https://admin.qsl.net/index.php","html_tokens":2346,"markdown_tokens":645,"pruned_tokens":515}},
{"rank":5308,"domain":"dropcatch.ai","origin":"https://dropcatch.ai","robots":{"status":"ok","http":200,"url":"https://dropcatch.ai/robots.txt","bytes":131,"groups":1},"bots":{"GPTBot":["partial","*",4,0],"ChatGPT-User":["partial","*",4,0],"OAI-SearchBot":["partial","*",4,0],"ClaudeBot":["partial","*",4,0],"Claude-User":["partial","*",4,0],"Claude-SearchBot":["partial","*",4,0],"anthropic-ai":["partial","*",4,0],"PerplexityBot":["partial","*",4,0],"Perplexity-User":["partial","*",4,0],"Google-Extended":["partial","*",4,0],"Applebot-Extended":["partial","*",4,0],"CCBot":["partial","*",4,0],"Bytespider":["partial","*",4,0],"Amazonbot":["partial","*",4,0],"meta-externalagent":["partial","*",4,0],"cohere-ai":["partial","*",4,0],"Diffbot":["partial","*",4,0],"ContextFetchBot":["partial","*",4,0]},"llms":{"status":"missing","http":404,"bytes":4387},"home":{"status":"ok","http":200,"url":"https://dropcatch.ai/","html_tokens":3600,"markdown_tokens":0,"pruned_tokens":0}},
{"rank":5309,"domain":"boots.com","origin":"https://boots.com","robots":{"status":"ok","http":200,"url":"https://www.boots.com/robots.txt","bytes":1008,"groups":2},"bots":{"GPTBot":["partial","*",31,0],"ChatGPT-User":["partial","*",31,0],"OAI-SearchBot":["partial","*",31,0],"ClaudeBot":["partial","*",31,0],"Claude-User":["partial","*",31,0],"Claude-SearchBot":["partial","*",31,0],"anthropic-ai":["partial","*",31,0],"PerplexityBot":["partial","*",31,0],"Perplexity-User":["partial","*",31,0],"Google-Extended":["partial","*",31,0],"Applebot-Extended":["partial","*",31,0],"CCBot":["partial","*",31,0],"Bytespider":["partial","*",31,0],"Amazonbot":["partial","*",31,0],"meta-externalagent":["partial","*",31,0],"cohere-ai":["partial","*",31,0],"Diffbot":["partial","*",31,0],"ContextFetchBot":["partial","*",31,0]},"llms":{"status":"missing","http":404,"bytes":863608},"home":{"status":"http_error","http":403,"url":"https://www.boots.com/"}},
{"rank":5310,"domain":"unifi.it","origin":"https://unifi.it","robots":{"status":"missing","http":404,"url":"https://www.unifi.it/itrobots.txt","bytes":551,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":549},"home":{"status":"ok","http":200,"url":"https://www.unifi.it/it","html_tokens":51028,"markdown_tokens":616,"pruned_tokens":67}},
{"rank":5311,"domain":"konami.com","origin":"https://www.konami.com","robots":{"status":"ok","http":200,"url":"https://img.konami.com/robots.txt","bytes":487,"groups":3},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":2829},"home":{"status":"ok","http":200,"url":"https://www.konami.com/en/","html_tokens":2340,"markdown_tokens":283,"pruned_tokens":284}},
{"rank":5312,"domain":"weather.com.cn","origin":"https://weather.com.cn","robots":{"status":"ok","http":200,"url":"https://www.weather.com.cn/robots.txt","bytes":23,"groups":1},"bots":{"GPTBot":["allowed","*",0,1],"ChatGPT-User":["allowed","*",0,1],"OAI-SearchBot":["allowed","*",0,1],"ClaudeBot":["allowed","*",0,1],"Claude-User":["allowed","*",0,1],"Claude-SearchBot":["allowed","*",0,1],"anthropic-ai":["allowed","*",0,1],"PerplexityBot":["allowed","*",0,1],"Perplexity-User":["allowed","*",0,1],"Google-Extended":["allowed","*",0,1],"Applebot-Extended":["allowed","*",0,1],"CCBot":["allowed","*",0,1],"Bytespider":["allowed","*",0,1],"Amazonbot":["allowed","*",0,1],"meta-externalagent":["allowed","*",0,1],"cohere-ai":["allowed","*",0,1],"Diffbot":["allowed","*",0,1],"ContextFetchBot":["allowed","*",0,1]},"llms":{"status":"html","http":200,"bytes":10453},"home":{"status":"ok","http":200,"url":"https://www.weather.com.cn/","html_tokens":33384,"markdown_tokens":61,"pruned_tokens":61}},
{"rank":5313,"domain":"mta.info","origin":"https://mta.info","robots":{"status":"ok","http":200,"url":"https://www.mta.info/robots.txt","bytes":1629,"groups":1},"bots":{"GPTBot":["partial","*",24,18],"ChatGPT-User":["partial","*",24,18],"OAI-SearchBot":["partial","*",24,18],"ClaudeBot":["partial","*",24,18],"Claude-User":["partial","*",24,18],"Claude-SearchBot":["partial","*",24,18],"anthropic-ai":["partial","*",24,18],"PerplexityBot":["partial","*",24,18],"Perplexity-User":["partial","*",24,18],"Google-Extended":["partial","*",24,18],"Applebot-Extended":["partial","*",24,18],"CCBot":["partial","*",24,18],"Bytespider":["partial","*",24,18],"Amazonbot":["partial","*",24,18],"meta-externalagent":["partial","*",24,18],"cohere-ai":["partial","*",24,18],"Diffbot":["partial","*",24,18],"ContextFetchBot":["partial","*",24,18]},"llms":{"status":"missing","http":404,"bytes":186},"home":{"status":"ok","http":200,"url":"https://www.mta.info/","html_tokens":18837,"markdown_tokens":52,"pruned_tokens":52}},
{"rank":5314,"domain":"umaine.edu","origin":"https://umaine.edu","robots":{"status":"ok","http":200,"url":"https://umaine.edu/robots.txt","bytes":108,"groups":1},"bots":{"GPTBot":["partial","*",1,1],"ChatGPT-User":["partial","*",1,1],"OAI-SearchBot":["partial","*",1,1],"ClaudeBot":["partial","*",1,1],"Claude-User":["partial","*",1,1],"Claude-SearchBot":["partial","*",1,1],"anthropic-ai":["partial","*",1,1],"PerplexityBot":["partial","*",1,1],"Perplexity-User":["partial","*",1,1],"Google-Extended":["partial","*",1,1],"Applebot-Extended":["partial","*",1,1],"CCBot":["partial","*",1,1],"Bytespider":["partial","*",1,1],"Amazonbot":["partial","*",1,1],"meta-externalagent":["partial","*",1,1],"cohere-ai":["partial","*",1,1],"Diffbot":["partial","*",1,1],"ContextFetchBot":["partial","*",1,1]},"llms":{"status":"missing","http":404,"bytes":455471},"home":{"status":"ok","http":200,"url":"https://umaine.edu/","html_tokens":174347,"markdown_tokens":103,"pruned_tokens":103}},
{"rank":5315,"domain":"eventbrite.it","origin":"https://eventbrite.it","robots":{"status":"ok","http":200,"url":"https://eventbrite.it/robots.txt","bytes":7499,"groups":135},"bots":{"GPTBot":["partial","*",51,2],"ChatGPT-User":["partial","*",51,2],"OAI-SearchBot":["partial","*",51,2],"ClaudeBot":["partial","*",51,2],"Claude-User":["partial","*",51,2],"Claude-SearchBot":["partial","*",51,2],"anthropic-ai":["partial","*",51,2],"PerplexityBot":["partial","*",51,2],"Perplexity-User":["partial","*",51,2],"Google-Extended":["partial","*",51,2],"Applebot-Extended":["partial","*",51,2],"CCBot":["blocked","named",1,0],"Bytespider":["partial","*",51,2],"Amazonbot":["partial","*",51,2],"meta-externalagent":["partial","*",51,2],"cohere-ai":["partial","*",51,2],"Diffbot":["partial","*",51,2],"ContextFetchBot":["partial","*",51,2]},"llms":{"status":"missing","http":404,"bytes":76917},"home":{"status":"ok","http":200,"url":"https://www.eventbrite.it","html_tokens":52560,"markdown_tokens":62,"pruned_tokens":62}},
{"rank":5316,"domain":"snazzymaps.com","origin":"https://snazzymaps.com","robots":{"status":"ok","http":200,"url":"https://snazzymaps.com/robots.txt","bytes":70,"groups":1},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"missing","http":404,"bytes":0},"home":{"status":"ok","http":200,"url":"https://snazzymaps.com/","html_tokens":7227,"markdown_tokens":4,"pruned_tokens":4}},
{"rank":5317,"domain":"r2.dev","origin":"https://www.r2.dev","robots":{"status":"server_error","http":500,"url":"https://www.r2.dev/robots.txt","bytes":13,"groups":0,"error":"HTTP 500"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":5318,"domain":"listennotes.com","origin":"https://listennotes.com","robots":{"status":"ok","http":200,"url":"https://www.listennotes.com/robots.txt","bytes":2442,"groups":21},"bots":{"GPTBot":["partial","*",31,14],"ChatGPT-User":["partial","*",31,14],"OAI-SearchBot":["partial","*",31,14],"ClaudeBot":["partial","*",31,14],"Claude-User":["partial","*",31,14],"Claude-SearchBot":["partial","*",31,14],"anthropic-ai":["partial","*",31,14],"PerplexityBot":["partial","*",31,14],"Perplexity-User":["partial","*",31,14],"Google-Extended":["partial","*",31,14],"Applebot-Extended":["partial","*",31,14],"CCBot":["blocked","named",1,0],"Bytespider":["partial","*",31,14],"Amazonbot":["partial","*",31,14],"meta-externalagent":["partial","*",31,14],"cohere-ai":["partial","*",31,14],"Diffbot":["partial","*",31,14],"ContextFetchBot":["partial","*",31,14]},"llms":{"status":"present","http":200,"bytes":13100},"home":{"status":"ok","http":200,"url":"https://www.listennotes.com/","html_tokens":59289,"markdown_tokens":309,"pruned_tokens":154}},
{"rank":5319,"domain":"shopee.co.id","origin":"https://shopee.co.id","robots":{"status":"ok","http":200,"url":"https://shopee.co.id/robots.txt","bytes":2795,"groups":2},"bots":{"GPTBot":["partial","*",16,0],"ChatGPT-User":["partial","*",16,0],"OAI-SearchBot":["partial","*",16,0],"ClaudeBot":["partial","*",16,0],"Claude-User":["partial","*",16,0],"Claude-SearchBot":["partial","*",16,0],"anthropic-ai":["partial","*",16,0],"PerplexityBot":["partial","*",16,0],"Perplexity-User":["partial","*",16,0],"Google-Extended":["partial","*",16,0],"Applebot-Extended":["partial","*",16,0],"CCBot":["partial","*",16,0],"Bytespider":["partial","*",16,0],"Amazonbot":["partial","*",16,0],"meta-externalagent":["partial","*",16,0],"cohere-ai":["partial","*",16,0],"Diffbot":["partial","*",16,0],"ContextFetchBot":["partial","*",16,0]},"llms":{"status":"html","http":200,"bytes":198497},"home":{"status":"ok","http":200,"url":"https://shopee.co.id/","html_tokens":49625,"markdown_tokens":11,"pruned_tokens":11}},
{"rank":5320,"domain":"builtwith.com","origin":"https://builtwith.com","robots":{"status":"ok","http":200,"url":"https://builtwith.com/robots.txt","bytes":180,"groups":1},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"present","http":200,"bytes":6505},"home":{"status":"ok","http":200,"url":"https://builtwith.com/","html_tokens":56500,"markdown_tokens":140,"pruned_tokens":140}},
{"rank":5321,"domain":"japan-guide.com","origin":"https://japan-guide.com","robots":{"status":"ok","http":200,"url":"https://japan-guide.com/robots.txt","bytes":109,"groups":3},"bots":{"GPTBot":["partial","*",1,0],"ChatGPT-User":["partial","*",1,0],"OAI-SearchBot":["partial","*",1,0],"ClaudeBot":["partial","*",1,0],"Claude-User":["partial","*",1,0],"Claude-SearchBot":["partial","*",1,0],"anthropic-ai":["partial","*",1,0],"PerplexityBot":["partial","*",1,0],"Perplexity-User":["partial","*",1,0],"Google-Extended":["partial","*",1,0],"Applebot-Extended":["partial","*",1,0],"CCBot":["partial","*",1,0],"Bytespider":["partial","*",1,0],"Amazonbot":["partial","*",1,0],"meta-externalagent":["partial","*",1,0],"cohere-ai":["partial","*",1,0],"Diffbot":["partial","*",1,0],"ContextFetchBot":["partial","*",1,0]},"llms":{"status":"missing","http":404,"bytes":74879},"home":{"status":"ok","http":200,"url":"https://www.japan-guide.com/","html_tokens":56909,"markdown_tokens":37,"pruned_tokens":37}},
{"rank":5322,"domain":"ai.google","origin":"https://ai.google","robots":{"status":"ok","http":200,"url":"https://ai.google/robots.txt","bytes":63,"groups":1},"bots":{"GPTBot":["allowed","*",0,1],"ChatGPT-User":["allowed","*",0,1],"OAI-SearchBot":["allowed","*",0,1],"ClaudeBot":["allowed","*",0,1],"Claude-User":["allowed","*",0,1],"Claude-SearchBot":["allowed","*",0,1],"anthropic-ai":["allowed","*",0,1],"PerplexityBot":["allowed","*",0,1],"Perplexity-User":["allowed","*",0,1],"Google-Extended":["allowed","*",0,1],"Applebot-Extended":["allowed","*",0,1],"CCBot":["allowed","*",0,1],"Bytespider":["allowed","*",0,1],"Amazonbot":["allowed","*",0,1],"meta-externalagent":["allowed","*",0,1],"cohere-ai":["allowed","*",0,1],"Diffbot":["allowed","*",0,1],"ContextFetchBot":["allowed","*",0,1]},"llms":{"status":"missing","http":404,"bytes":226093},"home":{"status":"ok","http":200,"url":"https://ai.google/","html_tokens":116780,"markdown_tokens":2661,"pruned_tokens":164}},
{"rank":5323,"domain":"katowice.pl","origin":"https://katowice.pl","robots":{"status":"unreachable","http":null,"url":"https://katowice.pl/robots.txt","bytes":0,"groups":0,"error":"cannot resolve katowice.pl"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":5324,"domain":"777spinslots.com","origin":"https://777spinslots.com","robots":{"status":"ok","http":200,"url":"https://777spinslots.com/robots.txt","bytes":598,"groups":2},"bots":{"GPTBot":["partial","*",18,2],"ChatGPT-User":["partial","*",18,2],"OAI-SearchBot":["partial","*",18,2],"ClaudeBot":["partial","*",18,2],"Claude-User":["partial","*",18,2],"Claude-SearchBot":["partial","*",18,2],"anthropic-ai":["partial","*",18,2],"PerplexityBot":["partial","*",18,2],"Perplexity-User":["partial","*",18,2],"Google-Extended":["partial","*",18,2],"Applebot-Extended":["partial","*",18,2],"CCBot":["partial","*",18,2],"Bytespider":["partial","*",18,2],"Amazonbot":["partial","*",18,2],"meta-externalagent":["partial","*",18,2],"cohere-ai":["partial","*",18,2],"Diffbot":["partial","*",18,2],"ContextFetchBot":["partial","*",18,2]},"llms":{"status":"missing","http":404,"bytes":39766},"home":{"status":"ok","http":200,"url":"https://777spinslots.com/","html_tokens":63136,"markdown_tokens":8221,"pruned_tokens":171}},
{"rank":5325,"domain":"umkc.edu","origin":"https://umkc.edu","robots":{"status":"missing","http":404,"url":"https://www.umkc.edu/robots.txt","bytes":62052,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":62052},"home":{"status":"ok","http":200,"url":"https://www.umkc.edu/","html_tokens":24966,"markdown_tokens":1170,"pruned_tokens":663}},
{"rank":5326,"domain":"uni-marburg.de","origin":"https://uni-marburg.de","robots":{"status":"ok","http":200,"url":"https://www.uni-marburg.de/robots.txt","bytes":590,"groups":2},"bots":{"GPTBot":["partial","*",7,0],"ChatGPT-User":["partial","*",7,0],"OAI-SearchBot":["partial","*",7,0],"ClaudeBot":["partial","*",7,0],"Claude-User":["partial","*",7,0],"Claude-SearchBot":["partial","*",7,0],"anthropic-ai":["partial","*",7,0],"PerplexityBot":["partial","*",7,0],"Perplexity-User":["partial","*",7,0],"Google-Extended":["partial","*",7,0],"Applebot-Extended":["partial","*",7,0],"CCBot":["partial","*",7,0],"Bytespider":["partial","*",7,0],"Amazonbot":["partial","*",7,0],"meta-externalagent":["partial","*",7,0],"cohere-ai":["partial","*",7,0],"Diffbot":["partial","*",7,0],"ContextFetchBot":["partial","*",7,0]},"llms":{"status":"missing","http":404,"bytes":69120},"home":{"status":"ok","http":200,"url":"https://www.uni-marburg.de/de","html_tokens":35676,"markdown_tokens":9,"pruned_tokens":9}},
{"rank":5327,"domain":"nysed.gov","origin":"https://nysed.gov","robots":{"status":"unreachable","http":null,"url":"https://nysed.gov/robots.txt","bytes":0,"groups":0,"error":"[SSL: CERTIFICATE_VERIFY_FAILED] certificate verify failed: unable to get local issuer certificate (_ssl.c:1028)"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":5328,"domain":"ng.ru","origin":"https://ng.ru","robots":{"status":"ok","http":200,"url":"https://www.ng.ru/robots.txt","bytes":1492,"groups":1},"bots":{"GPTBot":["partial","*",51,4],"ChatGPT-User":["partial","*",51,4],"OAI-SearchBot":["partial","*",51,4],"ClaudeBot":["partial","*",51,4],"Claude-User":["partial","*",51,4],"Claude-SearchBot":["partial","*",51,4],"anthropic-ai":["partial","*",51,4],"PerplexityBot":["partial","*",51,4],"Perplexity-User":["partial","*",51,4],"Google-Extended":["partial","*",51,4],"Applebot-Extended":["partial","*",51,4],"CCBot":["partial","*",51,4],"Bytespider":["partial","*",51,4],"Amazonbot":["partial","*",51,4],"meta-externalagent":["partial","*",51,4],"cohere-ai":["partial","*",51,4],"Diffbot":["partial","*",51,4],"ContextFetchBot":["partial","*",51,4]},"llms":{"status":"missing","http":404,"bytes":146},"home":{"status":"ok","http":200,"url":"https://www.ng.ru/","html_tokens":72064,"markdown_tokens":27,"pruned_tokens":26}},
{"rank":5329,"domain":"sina.com","origin":"https://sina.com","robots":{"status":"missing","http":404,"url":"https://sina.com/robots.txt","bytes":168,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":168},"home":{"status":"ok","http":200,"url":"https://sina.com/","html_tokens":36,"markdown_tokens":0,"pruned_tokens":0}},
{"rank":5330,"domain":"invideo.io","origin":"https://invideo.io","robots":{"status":"ok","http":200,"url":"https://invideo.io/robots.txt","bytes":1276,"groups":4},"bots":{"GPTBot":["partial","named",3,0],"ChatGPT-User":["partial","named",12,1],"OAI-SearchBot":["partial","named",12,1],"ClaudeBot":["partial","named",3,0],"Claude-User":["partial","named",12,1],"Claude-SearchBot":["partial","named",12,1],"anthropic-ai":["partial","*",3,4],"PerplexityBot":["partial","named",12,1],"Perplexity-User":["partial","named",12,1],"Google-Extended":["partial","named",3,0],"Applebot-Extended":["partial","named",3,0],"CCBot":["partial","named",3,0],"Bytespider":["partial","*",3,4],"Amazonbot":["partial","named",3,0],"meta-externalagent":["partial","*",3,4],"cohere-ai":["partial","*",3,4],"Diffbot":["partial","*",3,4],"ContextFetchBot":["partial","*",3,4]},"llms":{"status":"present","http":200,"bytes":16207},"home":{"status":"ok","http":200,"url":"https://invideo.io/","html_tokens":91438,"markdown_tokens":220,"pruned_tokens":113}},
{"rank":5331,"domain":"allstate.com","origin":"https://allstate.com","robots":{"status":"unreachable","http":null,"url":"https://allstate.com/robots.txt","bytes":0,"groups":0,"error":"<StreamReset stream_id:1, error_code:2, remote_reset:True>"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":5332,"domain":"dailynews.com","origin":"https://dailynews.com","robots":{"status":"ok","http":200,"url":"https://www.dailynews.com/robots.txt","bytes":4477,"groups":68},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["blocked","named",1,0],"OAI-SearchBot":["blocked","named",1,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["blocked","named",1,0],"Claude-SearchBot":["blocked","named",1,0],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["blocked","named",1,0],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["partial","*",8,1],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",8,1]},"llms":{"status":"missing","http":404,"bytes":81496},"home":{"status":"ok","http":200,"url":"https://www.dailynews.com/","html_tokens":59127,"markdown_tokens":258,"pruned_tokens":95}},
{"rank":5333,"domain":"squarespace-cdn.com","origin":"https://squarespace-cdn.com","robots":{"status":"unreachable","http":null,"url":"https://squarespace-cdn.com/robots.txt","bytes":0,"groups":0,"error":"cannot resolve squarespace-cdn.com"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":5334,"domain":"internetsociety.org","origin":"https://internetsociety.org","robots":{"status":"ok","http":200,"url":"https://internetsociety.org/robots.txt","bytes":130,"groups":1},"bots":{"GPTBot":["partial","*",1,1],"ChatGPT-User":["partial","*",1,1],"OAI-SearchBot":["partial","*",1,1],"ClaudeBot":["partial","*",1,1],"Claude-User":["partial","*",1,1],"Claude-SearchBot":["partial","*",1,1],"anthropic-ai":["partial","*",1,1],"PerplexityBot":["partial","*",1,1],"Perplexity-User":["partial","*",1,1],"Google-Extended":["partial","*",1,1],"Applebot-Extended":["partial","*",1,1],"CCBot":["partial","*",1,1],"Bytespider":["partial","*",1,1],"Amazonbot":["partial","*",1,1],"meta-externalagent":["partial","*",1,1],"cohere-ai":["partial","*",1,1],"Diffbot":["partial","*",1,1],"ContextFetchBot":["partial","*",1,1]},"llms":{"status":"missing","http":404,"bytes":174370},"home":{"status":"ok","http":200,"url":"https://www.internetsociety.org/","html_tokens":62939,"markdown_tokens":515,"pruned_tokens":101}},
{"rank":5335,"domain":"sc.com","origin":"https://sc.com","robots":{"status":"ok","http":200,"url":"https://www.sc.com/robots.txt","bytes":2106,"groups":1},"bots":{"GPTBot":["partial","*",28,0],"ChatGPT-User":["partial","*",28,0],"OAI-SearchBot":["partial","*",28,0],"ClaudeBot":["partial","*",28,0],"Claude-User":["partial","*",28,0],"Claude-SearchBot":["partial","*",28,0],"anthropic-ai":["partial","*",28,0],"PerplexityBot":["partial","*",28,0],"Perplexity-User":["partial","*",28,0],"Google-Extended":["partial","*",28,0],"Applebot-Extended":["partial","*",28,0],"CCBot":["partial","*",28,0],"Bytespider":["partial","*",28,0],"Amazonbot":["partial","*",28,0],"meta-externalagent":["partial","*",28,0],"cohere-ai":["partial","*",28,0],"Diffbot":["partial","*",28,0],"ContextFetchBot":["partial","*",28,0]},"llms":{"status":"missing","http":404,"bytes":115504},"home":{"status":"ok","http":200,"url":"https://www.sc.com/","html_tokens":177,"markdown_tokens":0,"pruned_tokens":0}},
{"rank":5336,"domain":"scienceblogs.com","origin":"https://scienceblogs.com","robots":{"status":"ok","http":200,"url":"https://scienceblogs.com/robots.txt","bytes":2027,"groups":1},"bots":{"GPTBot":["partial","*",34,18],"ChatGPT-User":["partial","*",34,18],"OAI-SearchBot":["partial","*",34,18],"ClaudeBot":["partial","*",34,18],"Claude-User":["partial","*",34,18],"Claude-SearchBot":["partial","*",34,18],"anthropic-ai":["partial","*",34,18],"PerplexityBot":["partial","*",34,18],"Perplexity-User":["partial","*",34,18],"Google-Extended":["partial","*",34,18],"Applebot-Extended":["partial","*",34,18],"CCBot":["partial","*",34,18],"Bytespider":["partial","*",34,18],"Amazonbot":["partial","*",34,18],"meta-externalagent":["partial","*",34,18],"cohere-ai":["partial","*",34,18],"Diffbot":["partial","*",34,18],"ContextFetchBot":["partial","*",34,18]},"llms":{"status":"missing","http":404,"bytes":190},"home":{"status":"ok","http":200,"url":"https://scienceblogs.com/","html_tokens":17064,"markdown_tokens":3459,"pruned_tokens":69}},
{"rank":5337,"domain":"barchart.com","origin":"https://barchart.com","robots":{"status":"ok","http":200,"url":"https://www.barchart.com/robots.txt","bytes":371,"groups":3},"bots":{"GPTBot":["partial","*",6,0],"ChatGPT-User":["partial","*",6,0],"OAI-SearchBot":["partial","*",6,0],"ClaudeBot":["partial","*",6,0],"Claude-User":["partial","*",6,0],"Claude-SearchBot":["partial","*",6,0],"anthropic-ai":["partial","*",6,0],"PerplexityBot":["partial","*",6,0],"Perplexity-User":["partial","*",6,0],"Google-Extended":["partial","*",6,0],"Applebot-Extended":["partial","*",6,0],"CCBot":["partial","*",6,0],"Bytespider":["partial","*",6,0],"Amazonbot":["partial","*",6,0],"meta-externalagent":["partial","*",6,0],"cohere-ai":["partial","*",6,0],"Diffbot":["partial","*",6,0],"ContextFetchBot":["partial","*",6,0]},"llms":{"status":"missing","http":404,"bytes":1497058},"home":{"status":"ok","http":202,"url":"https://barchart.com/","html_tokens":500,"markdown_tokens":40,"pruned_tokens":40}},
{"rank":5338,"domain":"mubi.com","origin":"https://mubi.com","robots":{"status":"ok","http":200,"url":"https://mubi.com/robots.txt","bytes":694,"groups":1},"bots":{"GPTBot":["partial","*",6,13],"ChatGPT-User":["partial","*",6,13],"OAI-SearchBot":["partial","*",6,13],"ClaudeBot":["partial","*",6,13],"Claude-User":["partial","*",6,13],"Claude-SearchBot":["partial","*",6,13],"anthropic-ai":["partial","*",6,13],"PerplexityBot":["partial","*",6,13],"Perplexity-User":["partial","*",6,13],"Google-Extended":["partial","*",6,13],"Applebot-Extended":["partial","*",6,13],"CCBot":["partial","*",6,13],"Bytespider":["partial","*",6,13],"Amazonbot":["partial","*",6,13],"meta-externalagent":["partial","*",6,13],"cohere-ai":["partial","*",6,13],"Diffbot":["partial","*",6,13],"ContextFetchBot":["partial","*",6,13]},"llms":{"status":"missing","http":404,"bytes":96758},"home":{"status":"ok","http":200,"url":"https://mubi.com/en/us","html_tokens":77318,"markdown_tokens":70,"pruned_tokens":70}},
{"rank":5339,"domain":"thisismoney.co.uk","origin":"https://thisismoney.co.uk","robots":{"status":"ok","http":200,"url":"https://www.thisismoney.co.uk/robots.txt","bytes":6312,"groups":5},"bots":{"GPTBot":["partial","named",1,1],"ChatGPT-User":["partial","named",1,1],"OAI-SearchBot":["partial","named",1,1],"ClaudeBot":["partial","named",1,1],"Claude-User":["partial","named",1,1],"Claude-SearchBot":["partial","named",1,1],"anthropic-ai":["partial","named",1,1],"PerplexityBot":["partial","named",1,1],"Perplexity-User":["partial","named",1,1],"Google-Extended":["partial","named",1,1],"Applebot-Extended":["partial","named",1,1],"CCBot":["partial","named",1,1],"Bytespider":["partial","named",1,1],"Amazonbot":["partial","named",1,1],"meta-externalagent":["partial","named",1,1],"cohere-ai":["partial","named",1,1],"Diffbot":["partial","named",1,1],"ContextFetchBot":["partial","*",42,0]},"llms":{"status":"missing","http":404,"bytes":41103},"home":{"status":"ok","http":200,"url":"https://www.thisismoney.co.uk/money/index.html","html_tokens":134419,"markdown_tokens":850,"pruned_tokens":175}},
{"rank":5340,"domain":"uned.es","origin":"https://uned.es","robots":{"status":"html","http":200,"url":"https://www.uned.es/universidad/inicio.html","bytes":87405,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":5341,"domain":"random.org","origin":"https://random.org","robots":{"status":"ok","http":200,"url":"https://www.random.org/robots.txt","bytes":738,"groups":1},"bots":{"GPTBot":["partial","*",17,9],"ChatGPT-User":["partial","*",17,9],"OAI-SearchBot":["partial","*",17,9],"ClaudeBot":["partial","*",17,9],"Claude-User":["partial","*",17,9],"Claude-SearchBot":["partial","*",17,9],"anthropic-ai":["partial","*",17,9],"PerplexityBot":["partial","*",17,9],"Perplexity-User":["partial","*",17,9],"Google-Extended":["partial","*",17,9],"Applebot-Extended":["partial","*",17,9],"CCBot":["partial","*",17,9],"Bytespider":["partial","*",17,9],"Amazonbot":["partial","*",17,9],"meta-externalagent":["partial","*",17,9],"cohere-ai":["partial","*",17,9],"Diffbot":["partial","*",17,9],"ContextFetchBot":["partial","*",17,9]},"llms":{"status":"missing","http":404,"bytes":27253},"home":{"status":"ok","http":200,"url":"https://www.random.org/","html_tokens":5983,"markdown_tokens":1130,"pruned_tokens":193}},
{"rank":5342,"domain":"polyu.edu.hk","origin":"https://www.polyu.edu.hk","robots":{"status":"ok","http":200,"url":"https://www.polyu.edu.hk/robots.txt","bytes":3036,"groups":1},"bots":{"GPTBot":["partial","*",97,0],"ChatGPT-User":["partial","*",97,0],"OAI-SearchBot":["partial","*",97,0],"ClaudeBot":["partial","*",97,0],"Claude-User":["partial","*",97,0],"Claude-SearchBot":["partial","*",97,0],"anthropic-ai":["partial","*",97,0],"PerplexityBot":["partial","*",97,0],"Perplexity-User":["partial","*",97,0],"Google-Extended":["partial","*",97,0],"Applebot-Extended":["partial","*",97,0],"CCBot":["partial","*",97,0],"Bytespider":["partial","*",97,0],"Amazonbot":["partial","*",97,0],"meta-externalagent":["partial","*",97,0],"cohere-ai":["partial","*",97,0],"Diffbot":["partial","*",97,0],"ContextFetchBot":["partial","*",97,0]},"llms":{"status":"missing","http":404,"bytes":1749},"home":{"status":"ok","http":200,"url":"https://www.polyu.edu.hk/","html_tokens":127727,"markdown_tokens":71,"pruned_tokens":70}},
{"rank":5343,"domain":"cdn-website.com","origin":"https://www.cdn-website.com","robots":{"status":"refused","http":403,"url":"https://www.cdn-website.com/robots.txt","bytes":263,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":5344,"domain":"acc.org","origin":"https://acc.org","robots":{"status":"html","http":200,"url":"https://acc.org/robots.txt","bytes":3095,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":5345,"domain":"renfe.com","origin":"https://renfe.com","robots":{"status":"html","http":200,"url":"https://www.renfe.com/es/en","bytes":99524,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":5346,"domain":"adblockplus.org","origin":"https://adblockplus.org","robots":{"status":"ok","http":200,"url":"https://adblockplus.org/robots-function","bytes":793,"groups":6},"bots":{"GPTBot":["partial","*",23,0],"ChatGPT-User":["partial","*",23,0],"OAI-SearchBot":["partial","*",23,0],"ClaudeBot":["partial","*",23,0],"Claude-User":["partial","*",23,0],"Claude-SearchBot":["partial","*",23,0],"anthropic-ai":["partial","*",23,0],"PerplexityBot":["partial","*",23,0],"Perplexity-User":["partial","*",23,0],"Google-Extended":["partial","*",23,0],"Applebot-Extended":["partial","*",23,0],"CCBot":["partial","*",23,0],"Bytespider":["partial","*",23,0],"Amazonbot":["partial","*",23,0],"meta-externalagent":["partial","*",23,0],"cohere-ai":["partial","*",23,0],"Diffbot":["partial","*",23,0],"ContextFetchBot":["partial","*",23,0]},"llms":{"status":"missing","http":404,"bytes":14},"home":{"status":"ok","http":200,"url":"https://adblockplus.org/","html_tokens":12333,"markdown_tokens":241,"pruned_tokens":85}},
{"rank":5347,"domain":"lazada.com.ph","origin":"https://lazada.com.ph","robots":{"status":"ok","http":200,"url":"https://www.lazada.com.ph/robots.txt","bytes":222,"groups":4},"bots":{"GPTBot":["allowed","named",0,1],"ChatGPT-User":["allowed","named",0,1],"OAI-SearchBot":["allowed","named",0,1],"ClaudeBot":["partial","*",4,0],"Claude-User":["partial","*",4,0],"Claude-SearchBot":["partial","*",4,0],"anthropic-ai":["partial","*",4,0],"PerplexityBot":["partial","*",4,0],"Perplexity-User":["partial","*",4,0],"Google-Extended":["partial","*",4,0],"Applebot-Extended":["partial","*",4,0],"CCBot":["partial","*",4,0],"Bytespider":["partial","*",4,0],"Amazonbot":["partial","*",4,0],"meta-externalagent":["partial","*",4,0],"cohere-ai":["partial","*",4,0],"Diffbot":["partial","*",4,0],"ContextFetchBot":["partial","*",4,0]},"llms":{"status":"present","http":200,"bytes":2812},"home":{"status":"ok","http":200,"url":"https://www.lazada.com.ph/","html_tokens":108076,"markdown_tokens":1711,"pruned_tokens":953}},
{"rank":5348,"domain":"artnews.com","origin":"https://artnews.com","robots":{"status":"ok","http":200,"url":"https://www.artnews.com/robots.txt","bytes":1254,"groups":26},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["blocked","named",1,0],"OAI-SearchBot":["blocked","named",1,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",9,1],"Claude-SearchBot":["partial","*",9,1],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["partial","*",9,1],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["partial","*",9,1],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",9,1]},"llms":{"status":"missing","http":404,"bytes":4301596},"home":{"status":"ok","http":200,"url":"https://www.artnews.com/","html_tokens":1119226,"markdown_tokens":101,"pruned_tokens":92}},
{"rank":5349,"domain":"gratisafhalen.be","origin":"https://gratisafhalen.be","robots":{"status":"ok","http":200,"url":"https://gratisafhalen.be/robots.txt","bytes":1248,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":46076},"home":{"status":"ok","http":200,"url":"https://gratisafhalen.be/","html_tokens":123887,"markdown_tokens":565,"pruned_tokens":81}},
{"rank":5350,"domain":"theathletic.com","origin":"https://theathletic.com","robots":{"status":"ok","http":200,"url":"https://theathletic.com/robots.txt","bytes":3196,"groups":25},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["blocked","named",1,0],"OAI-SearchBot":["partial","*",40,7],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",40,7],"Claude-SearchBot":["partial","*",40,7],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["partial","*",40,7],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["partial","*",40,7],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["partial","*",40,7],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",40,7]},"llms":{"status":"missing","http":404,"bytes":366},"home":{"status":"http_error","http":403,"url":"https://www.nytimes.com/athletic/"}},
{"rank":5351,"domain":"seroundtable.com","origin":"https://seroundtable.com","robots":{"status":"ok","http":200,"url":"https://www.seroundtable.com/robots.txt","bytes":90,"groups":1},"bots":{"GPTBot":["partial","*",3,0],"ChatGPT-User":["partial","*",3,0],"OAI-SearchBot":["partial","*",3,0],"ClaudeBot":["partial","*",3,0],"Claude-User":["partial","*",3,0],"Claude-SearchBot":["partial","*",3,0],"anthropic-ai":["partial","*",3,0],"PerplexityBot":["partial","*",3,0],"Perplexity-User":["partial","*",3,0],"Google-Extended":["partial","*",3,0],"Applebot-Extended":["partial","*",3,0],"CCBot":["partial","*",3,0],"Bytespider":["partial","*",3,0],"Amazonbot":["partial","*",3,0],"meta-externalagent":["partial","*",3,0],"cohere-ai":["partial","*",3,0],"Diffbot":["partial","*",3,0],"ContextFetchBot":["partial","*",3,0]},"llms":{"status":"missing","http":404,"bytes":179375},"home":{"status":"ok","http":200,"url":"https://www.seroundtable.com/","html_tokens":24808,"markdown_tokens":456,"pruned_tokens":171}},
{"rank":5352,"domain":"dfat.gov.au","origin":"https://dfat.gov.au","robots":{"status":"unreachable","http":null,"url":"https://dfat.gov.au/robots.txt","bytes":0,"groups":0,"error":"<StreamReset stream_id:1, error_code:2, remote_reset:True>"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":5353,"domain":"dilbert.com","origin":"https://dilbert.com","robots":{"status":"ok","http":200,"url":"https://dilbert.com/robots.txt","bytes":69,"groups":1},"bots":{"GPTBot":["partial","*",1,0],"ChatGPT-User":["partial","*",1,0],"OAI-SearchBot":["partial","*",1,0],"ClaudeBot":["partial","*",1,0],"Claude-User":["partial","*",1,0],"Claude-SearchBot":["partial","*",1,0],"anthropic-ai":["partial","*",1,0],"PerplexityBot":["partial","*",1,0],"Perplexity-User":["partial","*",1,0],"Google-Extended":["partial","*",1,0],"Applebot-Extended":["partial","*",1,0],"CCBot":["partial","*",1,0],"Bytespider":["partial","*",1,0],"Amazonbot":["partial","*",1,0],"meta-externalagent":["partial","*",1,0],"cohere-ai":["partial","*",1,0],"Diffbot":["partial","*",1,0],"ContextFetchBot":["partial","*",1,0]},"llms":{"status":"present","http":200,"bytes":329},"home":{"status":"ok","http":200,"url":"https://dilbert.com/","html_tokens":37443,"markdown_tokens":823,"pruned_tokens":479}},
{"rank":5354,"domain":"nbcdfw.com","origin":"https://nbcdfw.com","robots":{"status":"ok","http":200,"url":"https://www.nbcdfw.com/robots.txt","bytes":13162,"groups":27},"bots":{"GPTBot":["partial","named",1,7],"ChatGPT-User":["partial","named",1,7],"OAI-SearchBot":["partial","*",13,3],"ClaudeBot":["partial","named",1,7],"Claude-User":["partial","*",13,3],"Claude-SearchBot":["partial","*",13,3],"anthropic-ai":["partial","named",1,7],"PerplexityBot":["partial","named",1,7],"Perplexity-User":["partial","*",13,3],"Google-Extended":["partial","named",1,7],"Applebot-Extended":["partial","*",13,3],"CCBot":["partial","named",1,7],"Bytespider":["partial","named",1,7],"Amazonbot":["partial","named",1,9],"meta-externalagent":["partial","*",13,3],"cohere-ai":["partial","*",13,3],"Diffbot":["partial","named",1,7],"ContextFetchBot":["partial","*",13,3]},"llms":{"status":"html","http":200,"bytes":650091},"home":{"status":"ok","http":200,"url":"https://www.nbcdfw.com/","html_tokens":162455,"markdown_tokens":3,"pruned_tokens":3}},
{"rank":5355,"domain":"suning.com","origin":"https://www.suning.com","robots":{"status":"missing","http":404,"url":"https://www.suning.com/robots.txt","bytes":20,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":20},"home":{"status":"ok","http":200,"url":"https://www.suning.com/","html_tokens":50495,"markdown_tokens":26,"pruned_tokens":26}},
{"rank":5356,"domain":"citigroup.com","origin":"https://citigroup.com","robots":{"status":"ok","http":200,"url":"https://www.citigroup.com/robots.txt","bytes":135,"groups":1},"bots":{"GPTBot":["partial","*",3,1],"ChatGPT-User":["partial","*",3,1],"OAI-SearchBot":["partial","*",3,1],"ClaudeBot":["partial","*",3,1],"Claude-User":["partial","*",3,1],"Claude-SearchBot":["partial","*",3,1],"anthropic-ai":["partial","*",3,1],"PerplexityBot":["partial","*",3,1],"Perplexity-User":["partial","*",3,1],"Google-Extended":["partial","*",3,1],"Applebot-Extended":["partial","*",3,1],"CCBot":["partial","*",3,1],"Bytespider":["partial","*",3,1],"Amazonbot":["partial","*",3,1],"meta-externalagent":["partial","*",3,1],"cohere-ai":["partial","*",3,1],"Diffbot":["partial","*",3,1],"ContextFetchBot":["partial","*",3,1]},"llms":{"status":"missing","http":404,"bytes":189},"home":{"status":"ok","http":200,"url":"https://www.citigroup.com/global","html_tokens":400486,"markdown_tokens":55,"pruned_tokens":55}},
{"rank":5357,"domain":"ilovepdf.com","origin":"https://ilovepdf.com","robots":{"status":"ok","http":200,"url":"https://www.ilovepdf.com/robots.txt","bytes":32,"groups":1},"bots":{"GPTBot":["partial","*",1,0],"ChatGPT-User":["partial","*",1,0],"OAI-SearchBot":["partial","*",1,0],"ClaudeBot":["partial","*",1,0],"Claude-User":["partial","*",1,0],"Claude-SearchBot":["partial","*",1,0],"anthropic-ai":["partial","*",1,0],"PerplexityBot":["partial","*",1,0],"Perplexity-User":["partial","*",1,0],"Google-Extended":["partial","*",1,0],"Applebot-Extended":["partial","*",1,0],"CCBot":["partial","*",1,0],"Bytespider":["partial","*",1,0],"Amazonbot":["partial","*",1,0],"meta-externalagent":["partial","*",1,0],"cohere-ai":["partial","*",1,0],"Diffbot":["partial","*",1,0],"ContextFetchBot":["partial","*",1,0]},"llms":{"status":"missing","http":404,"bytes":34677},"home":{"status":"ok","http":200,"url":"https://www.ilovepdf.com/","html_tokens":30299,"markdown_tokens":157,"pruned_tokens":157}},
{"rank":5358,"domain":"life.ru","origin":"https://life.ru","robots":{"status":"ok","http":200,"url":"https://life.ru/robots.txt","bytes":1587,"groups":8},"bots":{"GPTBot":["partial","*",4,7],"ChatGPT-User":["partial","*",4,7],"OAI-SearchBot":["partial","*",4,7],"ClaudeBot":["partial","*",4,7],"Claude-User":["partial","*",4,7],"Claude-SearchBot":["partial","*",4,7],"anthropic-ai":["partial","*",4,7],"PerplexityBot":["partial","*",4,7],"Perplexity-User":["partial","*",4,7],"Google-Extended":["partial","*",4,7],"Applebot-Extended":["partial","*",4,7],"CCBot":["partial","*",4,7],"Bytespider":["blocked","named",1,0],"Amazonbot":["partial","*",4,7],"meta-externalagent":["partial","*",4,7],"cohere-ai":["partial","*",4,7],"Diffbot":["partial","*",4,7],"ContextFetchBot":["partial","*",4,7]},"llms":{"status":"missing","http":404,"bytes":148464},"home":{"status":"ok","http":200,"url":"https://life.ru/","html_tokens":141484,"markdown_tokens":31,"pruned_tokens":31}},
{"rank":5359,"domain":"internet.com","origin":"https://internet.com","robots":{"status":"refused","http":403,"url":"https://internet.com/robots.txt","bytes":278,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":5360,"domain":"fr.trustpilot.com","origin":"https://fr.trustpilot.com","robots":{"status":"ok","http":200,"url":"https://fr.trustpilot.com/robots.txt","bytes":25101,"groups":46},"bots":{"GPTBot":["partial","named",33,1],"ChatGPT-User":["partial","named",33,1],"OAI-SearchBot":["partial","named",33,1],"ClaudeBot":["partial","named",33,1],"Claude-User":["partial","named",33,1],"Claude-SearchBot":["partial","named",33,1],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["partial","named",33,1],"Perplexity-User":["blocked","*",1,0],"Google-Extended":["partial","named",33,1],"Applebot-Extended":["blocked","named",1,0],"CCBot":["partial","named",33,1],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","*",1,0],"meta-externalagent":["partial","named",33,1],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["blocked","*",1,0]},"llms":{"status":"disallowed"},"home":{"status":"disallowed"}},
{"rank":5361,"domain":"leboncoin.fr","origin":"https://leboncoin.fr","robots":{"status":"ok","http":200,"url":"https://www.leboncoin.fr/robots.txt","bytes":12745,"groups":8},"bots":{"GPTBot":["partial","named",11,1],"ChatGPT-User":["partial","named",17,2],"OAI-SearchBot":["partial","named",17,2],"ClaudeBot":["partial","named",11,1],"Claude-User":["partial","named",17,2],"Claude-SearchBot":["partial","named",17,2],"anthropic-ai":["partial","named",11,1],"PerplexityBot":["partial","named",17,2],"Perplexity-User":["partial","named",17,2],"Google-Extended":["partial","named",11,1],"Applebot-Extended":["partial","named",11,1],"CCBot":["partial","named",11,1],"Bytespider":["blocked","named",1,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"http_error","http":403,"bytes":771},"home":{"status":"http_error","http":403,"url":"https://www.leboncoin.fr/"}},
{"rank":5362,"domain":"mobirise.info","origin":"https://mobirise.info","robots":{"status":"ok","http":200,"url":"https://mobirise.info/robots.txt","bytes":1248,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":355},"home":{"status":"ok","http":200,"url":"https://cu3ox.com/ai-website-generator.html","html_tokens":28758,"markdown_tokens":4482,"pruned_tokens":562}},
{"rank":5363,"domain":"uclouvain.be","origin":"https://uclouvain.be","robots":{"status":"ok","http":200,"url":"https://uclouvain.be/robots.txt","bytes":2005,"groups":1},"bots":{"GPTBot":["partial","*",33,18],"ChatGPT-User":["partial","*",33,18],"OAI-SearchBot":["partial","*",33,18],"ClaudeBot":["partial","*",33,18],"Claude-User":["partial","*",33,18],"Claude-SearchBot":["partial","*",33,18],"anthropic-ai":["partial","*",33,18],"PerplexityBot":["partial","*",33,18],"Perplexity-User":["partial","*",33,18],"Google-Extended":["partial","*",33,18],"Applebot-Extended":["partial","*",33,18],"CCBot":["partial","*",33,18],"Bytespider":["partial","*",33,18],"Amazonbot":["partial","*",33,18],"meta-externalagent":["partial","*",33,18],"cohere-ai":["partial","*",33,18],"Diffbot":["partial","*",33,18],"ContextFetchBot":["partial","*",33,18]},"llms":{"status":"missing","http":404,"bytes":186},"home":{"status":"ok","http":200,"url":"https://uclouvain.be/fr","html_tokens":121448,"markdown_tokens":354,"pruned_tokens":70}},
{"rank":5364,"domain":"talkingpointsmemo.com","origin":"https://talkingpointsmemo.com","robots":{"status":"ok","http":200,"url":"https://talkingpointsmemo.com/robots.txt","bytes":119,"groups":1},"bots":{"GPTBot":["partial","*",1,1],"ChatGPT-User":["partial","*",1,1],"OAI-SearchBot":["partial","*",1,1],"ClaudeBot":["partial","*",1,1],"Claude-User":["partial","*",1,1],"Claude-SearchBot":["partial","*",1,1],"anthropic-ai":["partial","*",1,1],"PerplexityBot":["partial","*",1,1],"Perplexity-User":["partial","*",1,1],"Google-Extended":["partial","*",1,1],"Applebot-Extended":["partial","*",1,1],"CCBot":["partial","*",1,1],"Bytespider":["partial","*",1,1],"Amazonbot":["partial","*",1,1],"meta-externalagent":["partial","*",1,1],"cohere-ai":["partial","*",1,1],"Diffbot":["partial","*",1,1],"ContextFetchBot":["partial","*",1,1]},"llms":{"status":"missing","http":404,"bytes":75073},"home":{"status":"ok","http":200,"url":"https://talkingpointsmemo.com/","html_tokens":51978,"markdown_tokens":343,"pruned_tokens":75}},
{"rank":5365,"domain":"orange.com","origin":"https://orange.com","robots":{"status":"html","http":200,"url":"https://www.orange.com/","bytes":27162,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":5366,"domain":"elledecor.com","origin":"https://elledecor.com","robots":{"status":"ok","http":200,"url":"https://elledecor.com/robots.txt","bytes":963,"groups":3},"bots":{"GPTBot":["partial","named",32,0],"ChatGPT-User":["partial","named",32,0],"OAI-SearchBot":["partial","named",32,0],"ClaudeBot":["partial","*",19,0],"Claude-User":["partial","*",19,0],"Claude-SearchBot":["partial","*",19,0],"anthropic-ai":["partial","*",19,0],"PerplexityBot":["partial","*",19,0],"Perplexity-User":["partial","*",19,0],"Google-Extended":["partial","*",19,0],"Applebot-Extended":["partial","*",19,0],"CCBot":["partial","*",19,0],"Bytespider":["partial","*",19,0],"Amazonbot":["partial","*",19,0],"meta-externalagent":["partial","*",19,0],"cohere-ai":["partial","*",19,0],"Diffbot":["partial","*",19,0],"ContextFetchBot":["partial","*",19,0]},"llms":{"status":"missing","http":404,"bytes":84},"home":{"status":"ok","http":200,"url":"https://www.elledecor.com/","html_tokens":197161,"markdown_tokens":149,"pruned_tokens":149}},
{"rank":5367,"domain":"tenable.com","origin":"https://tenable.com","robots":{"status":"ok","http":200,"url":"https://www.tenable.com/robots.txt","bytes":1872,"groups":1},"bots":{"GPTBot":["partial","*",33,18],"ChatGPT-User":["partial","*",33,18],"OAI-SearchBot":["partial","*",33,18],"ClaudeBot":["partial","*",33,18],"Claude-User":["partial","*",33,18],"Claude-SearchBot":["partial","*",33,18],"anthropic-ai":["partial","*",33,18],"PerplexityBot":["partial","*",33,18],"Perplexity-User":["partial","*",33,18],"Google-Extended":["partial","*",33,18],"Applebot-Extended":["partial","*",33,18],"CCBot":["partial","*",33,18],"Bytespider":["partial","*",33,18],"Amazonbot":["partial","*",33,18],"meta-externalagent":["partial","*",33,18],"cohere-ai":["partial","*",33,18],"Diffbot":["partial","*",33,18],"ContextFetchBot":["partial","*",33,18]},"llms":{"status":"present","http":200,"bytes":8936},"home":{"status":"ok","http":200,"url":"https://www.tenable.com/","html_tokens":68094,"markdown_tokens":737,"pruned_tokens":389}},
{"rank":5368,"domain":"ilmessaggero.it","origin":"https://ilmessaggero.it","robots":{"status":"ok","http":200,"url":"https://www.ilmessaggero.it/robots.txt","bytes":4018,"groups":24},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["blocked","named",1,0],"OAI-SearchBot":["blocked","named",1,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["blocked","named",1,0],"Claude-SearchBot":["blocked","named",1,0],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["partial","*",111,0],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["partial","*",111,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["partial","*",111,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",111,0]},"llms":{"status":"missing","http":404,"bytes":196},"home":{"status":"ok","http":200,"url":"https://www.ilmessaggero.it/","html_tokens":189436,"markdown_tokens":5479,"pruned_tokens":140}},
{"rank":5369,"domain":"ae.org","origin":"https://ae.org","robots":{"status":"ok","http":200,"url":"https://ae.org/robots.txt","bytes":0,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":8017},"home":{"status":"ok","http":200,"url":"https://ae.org/","html_tokens":9789,"markdown_tokens":836,"pruned_tokens":818}},
{"rank":5370,"domain":"packtpub.com","origin":"https://packtpub.com","robots":{"status":"ok","http":200,"url":"https://packtpub.com/robots.txt","bytes":2676,"groups":32},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["allowed","*",0,1],"OAI-SearchBot":["allowed","*",0,1],"ClaudeBot":["blocked","named",1,0],"Claude-User":["allowed","*",0,1],"Claude-SearchBot":["allowed","*",0,1],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["allowed","*",0,1],"Perplexity-User":["allowed","*",0,1],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["allowed","*",0,1]},"llms":{"status":"http_error","http":403,"bytes":5501},"home":{"status":"http_error","http":403,"url":"https://packtpub.com/"}},
{"rank":5371,"domain":"goo.su","origin":"https://goo.su","robots":{"status":"ok","http":200,"url":"https://goo.su/robots.txt","bytes":213,"groups":2},"bots":{"GPTBot":["partial","*",3,1],"ChatGPT-User":["partial","*",3,1],"OAI-SearchBot":["partial","*",3,1],"ClaudeBot":["partial","*",3,1],"Claude-User":["partial","*",3,1],"Claude-SearchBot":["partial","*",3,1],"anthropic-ai":["partial","*",3,1],"PerplexityBot":["partial","*",3,1],"Perplexity-User":["partial","*",3,1],"Google-Extended":["partial","*",3,1],"Applebot-Extended":["partial","*",3,1],"CCBot":["partial","*",3,1],"Bytespider":["partial","*",3,1],"Amazonbot":["partial","*",3,1],"meta-externalagent":["partial","*",3,1],"cohere-ai":["partial","*",3,1],"Diffbot":["partial","*",3,1],"ContextFetchBot":["partial","*",3,1]},"llms":{"status":"missing","http":404,"bytes":13835},"home":{"status":"ok","http":200,"url":"https://goo.su/","html_tokens":22505,"markdown_tokens":99,"pruned_tokens":99}},
{"rank":5372,"domain":"khronos.org","origin":"https://khronos.org","robots":{"status":"ok","http":200,"url":"https://www.khronos.org/robots.txt","bytes":1512,"groups":2},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["blocked","named",1,0],"OAI-SearchBot":["partial","*",11,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",11,0],"Claude-SearchBot":["partial","*",11,0],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["partial","*",11,0],"Google-Extended":["partial","*",11,0],"Applebot-Extended":["partial","*",11,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["partial","*",11,0],"ContextFetchBot":["partial","*",11,0]},"llms":{"status":"missing","http":404,"bytes":26464},"home":{"status":"ok","http":200,"url":"https://www.khronos.org/","html_tokens":13090,"markdown_tokens":1020,"pruned_tokens":342}},
{"rank":5373,"domain":"weiyun.com","origin":"https://weiyun.com","robots":{"status":"html","http":200,"url":"https://www.weiyun.com/","bytes":40600,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":5374,"domain":"cvs.com","origin":"https://cvs.com","robots":{"status":"ok","http":200,"url":"https://www.cvs.com/robots.txt","bytes":7836,"groups":13},"bots":{"GPTBot":["partial","named",1,0],"ChatGPT-User":["partial","named",1,0],"OAI-SearchBot":["partial","*",113,11],"ClaudeBot":["partial","named",1,0],"Claude-User":["partial","*",113,11],"Claude-SearchBot":["partial","*",113,11],"anthropic-ai":["partial","named",1,0],"PerplexityBot":["partial","*",113,11],"Perplexity-User":["partial","*",113,11],"Google-Extended":["partial","named",1,0],"Applebot-Extended":["partial","*",113,11],"CCBot":["partial","named",1,0],"Bytespider":["partial","*",113,11],"Amazonbot":["partial","*",113,11],"meta-externalagent":["partial","*",113,11],"cohere-ai":["partial","named",1,0],"Diffbot":["partial","*",113,11],"ContextFetchBot":["partial","*",113,11]},"llms":{"status":"other_text","http":200,"bytes":4842},"home":{"status":"too_large","error":"response larger than 5 MB"}},
{"rank":5375,"domain":"habitat.org","origin":"https://habitat.org","robots":{"status":"ok","http":200,"url":"https://www.habitat.org/robots.txt","bytes":2189,"groups":1},"bots":{"GPTBot":["partial","*",36,22],"ChatGPT-User":["partial","*",36,22],"OAI-SearchBot":["partial","*",36,22],"ClaudeBot":["partial","*",36,22],"Claude-User":["partial","*",36,22],"Claude-SearchBot":["partial","*",36,22],"anthropic-ai":["partial","*",36,22],"PerplexityBot":["partial","*",36,22],"Perplexity-User":["partial","*",36,22],"Google-Extended":["partial","*",36,22],"Applebot-Extended":["partial","*",36,22],"CCBot":["partial","*",36,22],"Bytespider":["partial","*",36,22],"Amazonbot":["partial","*",36,22],"meta-externalagent":["partial","*",36,22],"cohere-ai":["partial","*",36,22],"Diffbot":["partial","*",36,22],"ContextFetchBot":["partial","*",36,22]},"llms":{"status":"missing","http":404,"bytes":189},"home":{"status":"ok","http":200,"url":"https://www.habitat.org/","html_tokens":29702,"markdown_tokens":84,"pruned_tokens":84}},
{"rank":5376,"domain":"kiva.org","origin":"https://kiva.org","robots":{"status":"refused","http":403,"url":"https://kiva.org/robots.txt","bytes":118,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":5377,"domain":"deviantart.net","origin":"https://deviantart.net","robots":{"status":"missing","http":404,"url":"https://deviantart.net/robots.txt","bytes":6754,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":6754},"home":{"status":"http_error","http":404,"url":"https://deviantart.net/"}},
{"rank":5378,"domain":"helpscout.com","origin":"https://helpscout.com","robots":{"status":"ok","http":200,"url":"https://www.helpscout.com/robots.txt","bytes":117,"groups":1},"bots":{"GPTBot":["partial","*",1,1],"ChatGPT-User":["partial","*",1,1],"OAI-SearchBot":["partial","*",1,1],"ClaudeBot":["partial","*",1,1],"Claude-User":["partial","*",1,1],"Claude-SearchBot":["partial","*",1,1],"anthropic-ai":["partial","*",1,1],"PerplexityBot":["partial","*",1,1],"Perplexity-User":["partial","*",1,1],"Google-Extended":["partial","*",1,1],"Applebot-Extended":["partial","*",1,1],"CCBot":["partial","*",1,1],"Bytespider":["partial","*",1,1],"Amazonbot":["partial","*",1,1],"meta-externalagent":["partial","*",1,1],"cohere-ai":["partial","*",1,1],"Diffbot":["partial","*",1,1],"ContextFetchBot":["partial","*",1,1]},"llms":{"status":"missing","http":404,"bytes":524080},"home":{"status":"ok","http":200,"url":"https://www.helpscout.com/","html_tokens":176310,"markdown_tokens":748,"pruned_tokens":149}},
{"rank":5379,"domain":"wallethub.com","origin":"https://wallethub.com","robots":{"status":"ok","http":200,"url":"https://wallethub.com/robots.txt","bytes":3296,"groups":27},"bots":{"GPTBot":["partial","*",48,2],"ChatGPT-User":["partial","*",48,2],"OAI-SearchBot":["partial","*",48,2],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",48,2],"Claude-SearchBot":["partial","*",48,2],"anthropic-ai":["partial","*",48,2],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["blocked","named",1,0],"Google-Extended":["partial","*",48,2],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["partial","*",48,2],"ContextFetchBot":["partial","*",48,2]},"llms":{"status":"missing","http":404,"bytes":0},"home":{"status":"ok","http":200,"url":"https://wallethub.com/","html_tokens":40810,"markdown_tokens":1774,"pruned_tokens":330}},
{"rank":5380,"domain":"flexjobs.com","origin":"https://flexjobs.com","robots":{"status":"unreachable","http":null,"url":"https://flexjobs.com/robots.txt","bytes":0,"groups":0,"error":"<StreamReset stream_id:1, error_code:2, remote_reset:True>"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":5381,"domain":"wps.cn","origin":"https://wps.cn","robots":{"status":"ok","http":200,"url":"https://www.wps.cn/robots.txt","bytes":67,"groups":1},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"missing","http":404,"bytes":153},"home":{"status":"ok","http":200,"url":"https://www.wps.cn/","html_tokens":1135,"markdown_tokens":0,"pruned_tokens":0}},
{"rank":5382,"domain":"google.lv","origin":"https://google.lv","robots":{"status":"ok","http":200,"url":"https://www.google.lv/robots.txt","bytes":6624,"groups":4},"bots":{"GPTBot":["partial","*",169,64],"ChatGPT-User":["partial","*",169,64],"OAI-SearchBot":["partial","*",169,64],"ClaudeBot":["partial","*",169,64],"Claude-User":["partial","*",169,64],"Claude-SearchBot":["partial","*",169,64],"anthropic-ai":["partial","*",169,64],"PerplexityBot":["partial","*",169,64],"Perplexity-User":["partial","*",169,64],"Google-Extended":["partial","*",169,64],"Applebot-Extended":["partial","*",169,64],"CCBot":["partial","*",169,64],"Bytespider":["partial","*",169,64],"Amazonbot":["partial","*",169,64],"meta-externalagent":["partial","*",169,64],"cohere-ai":["partial","*",169,64],"Diffbot":["partial","*",169,64],"ContextFetchBot":["partial","*",169,64]},"llms":{"status":"missing","http":404,"bytes":1569},"home":{"status":"ok","http":200,"url":"https://www.google.com/","html_tokens":21294,"markdown_tokens":30,"pruned_tokens":30}},
{"rank":5383,"domain":"ixigua.com","origin":"https://ixigua.com","robots":{"status":"ok","http":200,"url":"https://www.ixigua.com/robots.txt","bytes":169,"groups":3},"bots":{"GPTBot":["partial","*",3,0],"ChatGPT-User":["partial","*",3,0],"OAI-SearchBot":["partial","*",3,0],"ClaudeBot":["partial","*",3,0],"Claude-User":["partial","*",3,0],"Claude-SearchBot":["partial","*",3,0],"anthropic-ai":["partial","*",3,0],"PerplexityBot":["partial","*",3,0],"Perplexity-User":["partial","*",3,0],"Google-Extended":["partial","*",3,0],"Applebot-Extended":["partial","*",3,0],"CCBot":["partial","*",3,0],"Bytespider":["partial","*",3,0],"Amazonbot":["partial","*",3,0],"meta-externalagent":["partial","*",3,0],"cohere-ai":["partial","*",3,0],"Diffbot":["partial","*",3,0],"ContextFetchBot":["partial","*",3,0]},"llms":{"status":"html","http":200,"bytes":7765},"home":{"status":"ok","http":200,"url":"https://www.ixigua.com/","html_tokens":1930,"markdown_tokens":0,"pruned_tokens":0}},
{"rank":5384,"domain":"segmentfault.com","origin":"https://segmentfault.com","robots":{"status":"missing","http":404,"url":"https://segmentfault.com/robots.txt","bytes":2685,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":2685},"home":{"status":"ok","http":200,"url":"https://segmentfault.com/","html_tokens":31428,"markdown_tokens":17,"pruned_tokens":17}},
{"rank":5385,"domain":"typecho.org","origin":"https://typecho.org","robots":{"status":"ok","http":200,"url":"https://typecho.org/robots.txt","bytes":1248,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":1812},"home":{"status":"ok","http":200,"url":"https://typecho.org/","html_tokens":2483,"markdown_tokens":86,"pruned_tokens":51}},
{"rank":5386,"domain":"mufg.jp","origin":"https://www.mufg.jp","robots":{"status":"missing","http":404,"url":"https://www.mufg.jp/robots.txt","bytes":3347,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":3347},"home":{"status":"ok","http":200,"url":"https://www.mufg.jp/","html_tokens":10721,"markdown_tokens":22,"pruned_tokens":22}},
{"rank":5387,"domain":"balena.io","origin":"https://balena.io","robots":{"status":"ok","http":200,"url":"https://www.balena.io/robots.txt","bytes":67,"groups":1},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"missing","http":404,"bytes":27147},"home":{"status":"ok","http":200,"url":"https://www.balena.io/","html_tokens":17386,"markdown_tokens":630,"pruned_tokens":357}},
{"rank":5388,"domain":"paylocity.com","origin":"https://paylocity.com","robots":{"status":"ok","http":200,"url":"https://www.paylocity.com/robots.txt","bytes":426,"groups":1},"bots":{"GPTBot":["partial","*",17,0],"ChatGPT-User":["partial","*",17,0],"OAI-SearchBot":["partial","*",17,0],"ClaudeBot":["partial","*",17,0],"Claude-User":["partial","*",17,0],"Claude-SearchBot":["partial","*",17,0],"anthropic-ai":["partial","*",17,0],"PerplexityBot":["partial","*",17,0],"Perplexity-User":["partial","*",17,0],"Google-Extended":["partial","*",17,0],"Applebot-Extended":["partial","*",17,0],"CCBot":["partial","*",17,0],"Bytespider":["partial","*",17,0],"Amazonbot":["partial","*",17,0],"meta-externalagent":["partial","*",17,0],"cohere-ai":["partial","*",17,0],"Diffbot":["partial","*",17,0],"ContextFetchBot":["partial","*",17,0]},"llms":{"status":"present","http":200,"bytes":4076},"home":{"status":"ok","http":200,"url":"https://www.paylocity.com/","html_tokens":413,"markdown_tokens":31,"pruned_tokens":12}},
{"rank":5389,"domain":"movieweb.com","origin":"https://movieweb.com","robots":{"status":"ok","http":200,"url":"https://movieweb.com/robots.txt","bytes":2029,"groups":2},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["partial","*",22,0],"OAI-SearchBot":["partial","*",22,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",22,0],"Claude-SearchBot":["partial","*",22,0],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["partial","*",22,0],"Perplexity-User":["partial","*",22,0],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",22,0]},"llms":{"status":"missing","http":404,"bytes":111503},"home":{"status":"ok","http":200,"url":"https://movieweb.com/","html_tokens":223397,"markdown_tokens":111,"pruned_tokens":8}},
{"rank":5390,"domain":"plausible.io","origin":"https://plausible.io","robots":{"status":"ok","http":200,"url":"https://plausible.io/robots.txt","bytes":41,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":1895},"home":{"status":"ok","http":200,"url":"https://plausible.io/","html_tokens":19209,"markdown_tokens":1323,"pruned_tokens":385}},
{"rank":5391,"domain":"ifrs.org","origin":"https://ifrs.org","robots":{"status":"ok","http":200,"url":"https://www.ifrs.org/robots.txt","bytes":403,"groups":1},"bots":{"GPTBot":["partial","*",12,2],"ChatGPT-User":["partial","*",12,2],"OAI-SearchBot":["partial","*",12,2],"ClaudeBot":["partial","*",12,2],"Claude-User":["partial","*",12,2],"Claude-SearchBot":["partial","*",12,2],"anthropic-ai":["partial","*",12,2],"PerplexityBot":["partial","*",12,2],"Perplexity-User":["partial","*",12,2],"Google-Extended":["partial","*",12,2],"Applebot-Extended":["partial","*",12,2],"CCBot":["partial","*",12,2],"Bytespider":["partial","*",12,2],"Amazonbot":["partial","*",12,2],"meta-externalagent":["partial","*",12,2],"cohere-ai":["partial","*",12,2],"Diffbot":["partial","*",12,2],"ContextFetchBot":["partial","*",12,2]},"llms":{"status":"missing","http":404,"bytes":100611},"home":{"status":"ok","http":200,"url":"https://www.ifrs.org/","html_tokens":31683,"markdown_tokens":276,"pruned_tokens":222}},
{"rank":5392,"domain":"kyivindependent.com","origin":"https://kyivindependent.com","robots":{"status":"ok","http":200,"url":"https://kyivindependent.com/robots.txt","bytes":1061,"groups":4},"bots":{"GPTBot":["partial","*",12,0],"ChatGPT-User":["partial","*",12,0],"OAI-SearchBot":["partial","*",12,0],"ClaudeBot":["partial","*",12,0],"Claude-User":["partial","*",12,0],"Claude-SearchBot":["partial","*",12,0],"anthropic-ai":["partial","*",12,0],"PerplexityBot":["partial","*",12,0],"Perplexity-User":["partial","*",12,0],"Google-Extended":["partial","*",12,0],"Applebot-Extended":["partial","*",12,0],"CCBot":["partial","*",12,0],"Bytespider":["partial","*",12,0],"Amazonbot":["partial","*",12,0],"meta-externalagent":["partial","*",12,0],"cohere-ai":["partial","*",12,0],"Diffbot":["partial","*",12,0],"ContextFetchBot":["partial","*",12,0]},"llms":{"status":"present","http":200,"bytes":7536},"home":{"status":"ok","http":200,"url":"https://kyivindependent.com/","html_tokens":98300,"markdown_tokens":717,"pruned_tokens":161}},
{"rank":5393,"domain":"trustedreviews.com","origin":"https://trustedreviews.com","robots":{"status":"ok","http":200,"url":"https://www.trustedreviews.com/robots.txt","bytes":527,"groups":3},"bots":{"GPTBot":["partial","*",4,1],"ChatGPT-User":["partial","*",4,1],"OAI-SearchBot":["partial","*",4,1],"ClaudeBot":["partial","*",4,1],"Claude-User":["partial","*",4,1],"Claude-SearchBot":["partial","*",4,1],"anthropic-ai":["partial","*",4,1],"PerplexityBot":["partial","*",4,1],"Perplexity-User":["partial","*",4,1],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["partial","*",4,1],"CCBot":["partial","*",4,1],"Bytespider":["partial","*",4,1],"Amazonbot":["partial","*",4,1],"meta-externalagent":["partial","*",4,1],"cohere-ai":["partial","*",4,1],"Diffbot":["partial","*",4,1],"ContextFetchBot":["partial","*",4,1]},"llms":{"status":"missing","http":404,"bytes":83662},"home":{"status":"ok","http":200,"url":"https://www.trustedreviews.com/","html_tokens":50750,"markdown_tokens":137,"pruned_tokens":137}},
{"rank":5394,"domain":"seetickets.com","origin":"https://seetickets.com","robots":{"status":"refused","http":403,"url":"https://www.seetickets.com/robots.txt","bytes":2873,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":5395,"domain":"bmw.com","origin":"https://bmw.com","robots":{"status":"unreachable","http":null,"url":"https://bmw.com/robots.txt","bytes":0,"groups":0,"error":"<StreamReset stream_id:1, error_code:2, remote_reset:True>"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":5396,"domain":"qgis.org","origin":"https://qgis.org","robots":{"status":"ok","http":200,"url":"https://qgis.org/robots.txt","bytes":1439,"groups":18},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["blocked","named",1,0],"OAI-SearchBot":["partial","*",16,5],"ClaudeBot":["partial","*",16,5],"Claude-User":["partial","*",16,5],"Claude-SearchBot":["partial","*",16,5],"anthropic-ai":["partial","*",16,5],"PerplexityBot":["partial","*",16,5],"Perplexity-User":["partial","*",16,5],"Google-Extended":["partial","*",16,5],"Applebot-Extended":["partial","*",16,5],"CCBot":["blocked","named",1,0],"Bytespider":["partial","*",16,5],"Amazonbot":["partial","*",16,5],"meta-externalagent":["partial","*",16,5],"cohere-ai":["partial","*",16,5],"Diffbot":["partial","*",16,5],"ContextFetchBot":["partial","*",16,5]},"llms":{"status":"missing","http":404,"bytes":21603},"home":{"status":"ok","http":200,"url":"https://qgis.org/","html_tokens":8692,"markdown_tokens":493,"pruned_tokens":91}},
{"rank":5397,"domain":"au.int","origin":"https://au.int","robots":{"status":"ok","http":200,"url":"https://au.int/robots.txt","bytes":6624,"groups":33},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["partial","*",54,23],"OAI-SearchBot":["partial","*",54,23],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",54,23],"Claude-SearchBot":["partial","*",54,23],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["partial","*",54,23],"Perplexity-User":["partial","*",54,23],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",54,23]},"llms":{"status":"missing","http":404,"bytes":547},"home":{"status":"ok","http":200,"url":"https://au.int/en","html_tokens":35845,"markdown_tokens":98,"pruned_tokens":98}},
{"rank":5398,"domain":"telkomuniversity.ac.id","origin":"https://telkomuniversity.ac.id","robots":{"status":"ok","http":200,"url":"https://telkomuniversity.ac.id/robots.txt","bytes":148,"groups":1},"bots":{"GPTBot":["partial","*",1,0],"ChatGPT-User":["partial","*",1,0],"OAI-SearchBot":["partial","*",1,0],"ClaudeBot":["partial","*",1,0],"Claude-User":["partial","*",1,0],"Claude-SearchBot":["partial","*",1,0],"anthropic-ai":["partial","*",1,0],"PerplexityBot":["partial","*",1,0],"Perplexity-User":["partial","*",1,0],"Google-Extended":["partial","*",1,0],"Applebot-Extended":["partial","*",1,0],"CCBot":["partial","*",1,0],"Bytespider":["partial","*",1,0],"Amazonbot":["partial","*",1,0],"meta-externalagent":["partial","*",1,0],"cohere-ai":["partial","*",1,0],"Diffbot":["partial","*",1,0],"ContextFetchBot":["partial","*",1,0]},"llms":{"status":"missing","http":404,"bytes":169745},"home":{"status":"ok","http":200,"url":"https://telkomuniversity.ac.id/","html_tokens":94550,"markdown_tokens":343,"pruned_tokens":72}},
{"rank":5399,"domain":"dvidshub.net","origin":"https://dvidshub.net","robots":{"status":"server_error","http":502,"url":"https://dvidshub.net/robots.txt","bytes":932,"groups":0,"error":"HTTP 502"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":5400,"domain":"google.lt","origin":"https://google.lt","robots":{"status":"ok","http":200,"url":"https://www.google.lt/robots.txt","bytes":6624,"groups":4},"bots":{"GPTBot":["partial","*",169,64],"ChatGPT-User":["partial","*",169,64],"OAI-SearchBot":["partial","*",169,64],"ClaudeBot":["partial","*",169,64],"Claude-User":["partial","*",169,64],"Claude-SearchBot":["partial","*",169,64],"anthropic-ai":["partial","*",169,64],"PerplexityBot":["partial","*",169,64],"Perplexity-User":["partial","*",169,64],"Google-Extended":["partial","*",169,64],"Applebot-Extended":["partial","*",169,64],"CCBot":["partial","*",169,64],"Bytespider":["partial","*",169,64],"Amazonbot":["partial","*",169,64],"meta-externalagent":["partial","*",169,64],"cohere-ai":["partial","*",169,64],"Diffbot":["partial","*",169,64],"ContextFetchBot":["partial","*",169,64]},"llms":{"status":"missing","http":404,"bytes":1569},"home":{"status":"ok","http":200,"url":"https://www.google.com/","html_tokens":21310,"markdown_tokens":30,"pruned_tokens":30}},
{"rank":5401,"domain":"fotor.com","origin":"https://fotor.com","robots":{"status":"ok","http":200,"url":"https://www.fotor.com/robots.txt","bytes":3055,"groups":3},"bots":{"GPTBot":["partial","*",91,2],"ChatGPT-User":["partial","*",91,2],"OAI-SearchBot":["partial","*",91,2],"ClaudeBot":["partial","*",91,2],"Claude-User":["partial","*",91,2],"Claude-SearchBot":["partial","*",91,2],"anthropic-ai":["partial","*",91,2],"PerplexityBot":["partial","*",91,2],"Perplexity-User":["partial","*",91,2],"Google-Extended":["partial","*",91,2],"Applebot-Extended":["partial","*",91,2],"CCBot":["partial","*",91,2],"Bytespider":["partial","*",91,2],"Amazonbot":["partial","*",91,2],"meta-externalagent":["partial","*",91,2],"cohere-ai":["partial","*",91,2],"Diffbot":["partial","*",91,2],"ContextFetchBot":["partial","*",91,2]},"llms":{"status":"present","http":200,"bytes":7428},"home":{"status":"ok","http":200,"url":"https://www.fotor.com/","html_tokens":59131,"markdown_tokens":1442,"pruned_tokens":355}},
{"rank":5402,"domain":"bmas.de","origin":"https://bmas.de","robots":{"status":"ok","http":200,"url":"https://bmas.de/robots.txt","bytes":773,"groups":2},"bots":{"GPTBot":["partial","*",10,1],"ChatGPT-User":["partial","*",10,1],"OAI-SearchBot":["partial","*",10,1],"ClaudeBot":["partial","*",10,1],"Claude-User":["partial","*",10,1],"Claude-SearchBot":["partial","*",10,1],"anthropic-ai":["partial","*",10,1],"PerplexityBot":["partial","*",10,1],"Perplexity-User":["partial","*",10,1],"Google-Extended":["partial","*",10,1],"Applebot-Extended":["partial","*",10,1],"CCBot":["partial","*",10,1],"Bytespider":["partial","*",10,1],"Amazonbot":["partial","*",10,1],"meta-externalagent":["partial","*",10,1],"cohere-ai":["partial","*",10,1],"Diffbot":["partial","*",10,1],"ContextFetchBot":["partial","*",10,1]},"llms":{"status":"crawl_delay","error":"Crawl-delay 180 s"},"home":{"status":"crawl_delay","error":"Crawl-delay 180 s"}},
{"rank":5403,"domain":"freenet.de","origin":"https://freenet.de","robots":{"status":"ok","http":200,"url":"https://www.freenet.de/robots.txt","bytes":929,"groups":1},"bots":{"GPTBot":["partial","*",28,0],"ChatGPT-User":["partial","*",28,0],"OAI-SearchBot":["partial","*",28,0],"ClaudeBot":["partial","*",28,0],"Claude-User":["partial","*",28,0],"Claude-SearchBot":["partial","*",28,0],"anthropic-ai":["partial","*",28,0],"PerplexityBot":["partial","*",28,0],"Perplexity-User":["partial","*",28,0],"Google-Extended":["partial","*",28,0],"Applebot-Extended":["partial","*",28,0],"CCBot":["partial","*",28,0],"Bytespider":["partial","*",28,0],"Amazonbot":["partial","*",28,0],"meta-externalagent":["partial","*",28,0],"cohere-ai":["partial","*",28,0],"Diffbot":["partial","*",28,0],"ContextFetchBot":["partial","*",28,0]},"llms":{"status":"missing","http":404,"bytes":140162},"home":{"status":"ok","http":200,"url":"https://www.freenet.de/","html_tokens":91027,"markdown_tokens":194,"pruned_tokens":26}},
{"rank":5404,"domain":"griffith.edu.au","origin":"https://griffith.edu.au","robots":{"status":"refused","http":403,"url":"https://www.griffith.edu.au/robots.txt","bytes":5557,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":5405,"domain":"uni-saarland.de","origin":"https://uni-saarland.de","robots":{"status":"refused","http":403,"url":"https://uni-saarland.de/robots.txt","bytes":239,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":5406,"domain":"heygen.com","origin":"https://heygen.com","robots":{"status":"ok","http":200,"url":"https://www.heygen.com/robots.txt","bytes":2999,"groups":30},"bots":{"GPTBot":["allowed","named",0,1],"ChatGPT-User":["allowed","named",0,1],"OAI-SearchBot":["allowed","named",0,1],"ClaudeBot":["allowed","named",0,1],"Claude-User":["allowed","named",0,1],"Claude-SearchBot":["allowed","named",0,1],"anthropic-ai":["partial","*",2,0],"PerplexityBot":["allowed","named",0,1],"Perplexity-User":["allowed","named",0,1],"Google-Extended":["allowed","named",0,1],"Applebot-Extended":["allowed","named",0,1],"CCBot":["allowed","named",0,1],"Bytespider":["partial","*",2,0],"Amazonbot":["allowed","named",0,1],"meta-externalagent":["allowed","named",0,1],"cohere-ai":["partial","*",2,0],"Diffbot":["partial","*",2,0],"ContextFetchBot":["partial","*",2,0]},"llms":{"status":"present","http":200,"bytes":8254},"home":{"status":"ok","http":200,"url":"https://www.heygen.com/","html_tokens":111362,"markdown_tokens":1948,"pruned_tokens":476}},
{"rank":5407,"domain":"booksy.com","origin":"https://booksy.com","robots":{"status":"ok","http":200,"url":"https://booksy.com/robots.txt","bytes":10616,"groups":1},"bots":{"GPTBot":["partial","*",391,8],"ChatGPT-User":["partial","*",391,8],"OAI-SearchBot":["partial","*",391,8],"ClaudeBot":["partial","*",391,8],"Claude-User":["partial","*",391,8],"Claude-SearchBot":["partial","*",391,8],"anthropic-ai":["partial","*",391,8],"PerplexityBot":["partial","*",391,8],"Perplexity-User":["partial","*",391,8],"Google-Extended":["partial","*",391,8],"Applebot-Extended":["partial","*",391,8],"CCBot":["partial","*",391,8],"Bytespider":["partial","*",391,8],"Amazonbot":["partial","*",391,8],"meta-externalagent":["partial","*",391,8],"cohere-ai":["partial","*",391,8],"Diffbot":["partial","*",391,8],"ContextFetchBot":["partial","*",391,8]},"llms":{"status":"present","http":200,"bytes":12086},"home":{"status":"ok","http":200,"url":"https://booksy.com/en-us/","html_tokens":173502,"markdown_tokens":330,"pruned_tokens":215}},
{"rank":5408,"domain":"msf.org","origin":"https://msf.org","robots":{"status":"ok","http":200,"url":"https://www.msf.org/robots.txt","bytes":2497,"groups":1},"bots":{"GPTBot":["partial","*",50,18],"ChatGPT-User":["partial","*",50,18],"OAI-SearchBot":["partial","*",50,18],"ClaudeBot":["partial","*",50,18],"Claude-User":["partial","*",50,18],"Claude-SearchBot":["partial","*",50,18],"anthropic-ai":["partial","*",50,18],"PerplexityBot":["partial","*",50,18],"Perplexity-User":["partial","*",50,18],"Google-Extended":["partial","*",50,18],"Applebot-Extended":["partial","*",50,18],"CCBot":["partial","*",50,18],"Bytespider":["partial","*",50,18],"Amazonbot":["partial","*",50,18],"meta-externalagent":["partial","*",50,18],"cohere-ai":["partial","*",50,18],"Diffbot":["partial","*",50,18],"ContextFetchBot":["partial","*",50,18]},"llms":{"status":"missing","http":404,"bytes":546},"home":{"status":"ok","http":200,"url":"https://www.msf.org/","html_tokens":88519,"markdown_tokens":201,"pruned_tokens":95}},
{"rank":5409,"domain":"nicsell.com","origin":"https://nicsell.com","robots":{"status":"ok","http":200,"url":"https://nicsell.com/robots.txt","bytes":61,"groups":1},"bots":{"GPTBot":["allowed","*",0,1],"ChatGPT-User":["allowed","*",0,1],"OAI-SearchBot":["allowed","*",0,1],"ClaudeBot":["allowed","*",0,1],"Claude-User":["allowed","*",0,1],"Claude-SearchBot":["allowed","*",0,1],"anthropic-ai":["allowed","*",0,1],"PerplexityBot":["allowed","*",0,1],"Perplexity-User":["allowed","*",0,1],"Google-Extended":["allowed","*",0,1],"Applebot-Extended":["allowed","*",0,1],"CCBot":["allowed","*",0,1],"Bytespider":["allowed","*",0,1],"Amazonbot":["allowed","*",0,1],"meta-externalagent":["allowed","*",0,1],"cohere-ai":["allowed","*",0,1],"Diffbot":["allowed","*",0,1],"ContextFetchBot":["allowed","*",0,1]},"llms":{"status":"http_error","http":503,"bytes":62},"home":{"status":"ok","http":200,"url":"https://nicsell.com/","html_tokens":22315,"markdown_tokens":291,"pruned_tokens":161}},
{"rank":5410,"domain":"kaercher.com","origin":"https://kaercher.com","robots":{"status":"ok","http":200,"url":"https://www.kaercher.com/robots.txt","bytes":336,"groups":2},"bots":{"GPTBot":["partial","*",8,0],"ChatGPT-User":["partial","*",8,0],"OAI-SearchBot":["partial","*",8,0],"ClaudeBot":["partial","*",8,0],"Claude-User":["partial","*",8,0],"Claude-SearchBot":["partial","*",8,0],"anthropic-ai":["partial","*",8,0],"PerplexityBot":["partial","*",8,0],"Perplexity-User":["partial","*",8,0],"Google-Extended":["partial","*",8,0],"Applebot-Extended":["partial","*",8,0],"CCBot":["partial","*",8,0],"Bytespider":["partial","*",8,0],"Amazonbot":["partial","*",8,0],"meta-externalagent":["partial","*",8,0],"cohere-ai":["partial","*",8,0],"Diffbot":["partial","*",8,0],"ContextFetchBot":["partial","*",8,0]},"llms":{"status":"present","http":200,"bytes":92209},"home":{"status":"ok","http":200,"url":"https://www.kaercher.com/int/","html_tokens":16627,"markdown_tokens":114,"pruned_tokens":114}},
{"rank":5411,"domain":"ifema.es","origin":"https://ifema.es","robots":{"status":"ok","http":200,"url":"https://www.ifema.es/robots.txt","bytes":461,"groups":1},"bots":{"GPTBot":["partial","*",11,0],"ChatGPT-User":["partial","*",11,0],"OAI-SearchBot":["partial","*",11,0],"ClaudeBot":["partial","*",11,0],"Claude-User":["partial","*",11,0],"Claude-SearchBot":["partial","*",11,0],"anthropic-ai":["partial","*",11,0],"PerplexityBot":["partial","*",11,0],"Perplexity-User":["partial","*",11,0],"Google-Extended":["partial","*",11,0],"Applebot-Extended":["partial","*",11,0],"CCBot":["partial","*",11,0],"Bytespider":["partial","*",11,0],"Amazonbot":["partial","*",11,0],"meta-externalagent":["partial","*",11,0],"cohere-ai":["partial","*",11,0],"Diffbot":["partial","*",11,0],"ContextFetchBot":["partial","*",11,0]},"llms":{"status":"other_text","http":200,"bytes":24733},"home":{"status":"ok","http":200,"url":"https://www.ifema.es/","html_tokens":53367,"markdown_tokens":199,"pruned_tokens":153}},
{"rank":5412,"domain":"vk.cc","origin":"https://vk.cc","robots":{"status":"ok","http":200,"url":"https://vk.cc/robots.txt","bytes":35,"groups":1},"bots":{"GPTBot":["partial","*",1,1],"ChatGPT-User":["partial","*",1,1],"OAI-SearchBot":["partial","*",1,1],"ClaudeBot":["partial","*",1,1],"Claude-User":["partial","*",1,1],"Claude-SearchBot":["partial","*",1,1],"anthropic-ai":["partial","*",1,1],"PerplexityBot":["partial","*",1,1],"Perplexity-User":["partial","*",1,1],"Google-Extended":["partial","*",1,1],"Applebot-Extended":["partial","*",1,1],"CCBot":["partial","*",1,1],"Bytespider":["partial","*",1,1],"Amazonbot":["partial","*",1,1],"meta-externalagent":["partial","*",1,1],"cohere-ai":["partial","*",1,1],"Diffbot":["partial","*",1,1],"ContextFetchBot":["partial","*",1,1]},"llms":{"status":"disallowed"},"home":{"status":"disallowed","error":"robots.txt of vkontakte.ru disallows /cc"}},
{"rank":5413,"domain":"milenio.com","origin":"https://milenio.com","robots":{"status":"refused","http":403,"url":"https://www.milenio.com/robots.txt","bytes":919,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":5414,"domain":"yesky.com","origin":"https://yesky.com","robots":{"status":"missing","http":404,"url":"https://www.yesky.com/robots.txt","bytes":146,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":146},"home":{"status":"ok","http":200,"url":"https://www.yesky.com/","html_tokens":46675,"markdown_tokens":161,"pruned_tokens":160}},
{"rank":5415,"domain":"whatismyipaddress.com","origin":"https://whatismyipaddress.com","robots":{"status":"ok","http":200,"url":"https://whatismyipaddress.com/robots.txt","bytes":249,"groups":2},"bots":{"GPTBot":["partial","*",8,1],"ChatGPT-User":["partial","*",8,1],"OAI-SearchBot":["partial","*",8,1],"ClaudeBot":["partial","*",8,1],"Claude-User":["partial","*",8,1],"Claude-SearchBot":["partial","*",8,1],"anthropic-ai":["partial","*",8,1],"PerplexityBot":["partial","*",8,1],"Perplexity-User":["partial","*",8,1],"Google-Extended":["partial","*",8,1],"Applebot-Extended":["partial","*",8,1],"CCBot":["partial","*",8,1],"Bytespider":["partial","*",8,1],"Amazonbot":["partial","*",8,1],"meta-externalagent":["partial","*",8,1],"cohere-ai":["partial","*",8,1],"Diffbot":["partial","*",8,1],"ContextFetchBot":["partial","*",8,1]},"llms":{"status":"http_error","http":403,"bytes":5574},"home":{"status":"http_error","http":403,"url":"https://whatismyipaddress.com/"}},
{"rank":5416,"domain":"gamblersanonymous.org","origin":"https://gamblersanonymous.org","robots":{"status":"ok","http":200,"url":"https://gamblersanonymous.org/robots.txt","bytes":449,"groups":2},"bots":{"GPTBot":["partial","*",6,1],"ChatGPT-User":["partial","*",6,1],"OAI-SearchBot":["partial","*",6,1],"ClaudeBot":["partial","*",6,1],"Claude-User":["partial","*",6,1],"Claude-SearchBot":["partial","*",6,1],"anthropic-ai":["partial","*",6,1],"PerplexityBot":["partial","*",6,1],"Perplexity-User":["partial","*",6,1],"Google-Extended":["partial","*",6,1],"Applebot-Extended":["partial","*",6,1],"CCBot":["partial","*",6,1],"Bytespider":["partial","*",6,1],"Amazonbot":["partial","*",6,1],"meta-externalagent":["partial","*",6,1],"cohere-ai":["partial","*",6,1],"Diffbot":["partial","*",6,1],"ContextFetchBot":["partial","*",6,1]},"llms":{"status":"missing","http":404,"bytes":104255},"home":{"status":"ok","http":200,"url":"https://gamblersanonymous.org/","html_tokens":29133,"markdown_tokens":287,"pruned_tokens":145}},
{"rank":5417,"domain":"rockwellautomation.com","origin":"https://rockwellautomation.com","robots":{"status":"ok","http":200,"url":"https://www.rockwellautomation.com/robots.txt","bytes":5064,"groups":1},"bots":{"GPTBot":["partial","*",50,0],"ChatGPT-User":["partial","*",50,0],"OAI-SearchBot":["partial","*",50,0],"ClaudeBot":["partial","*",50,0],"Claude-User":["partial","*",50,0],"Claude-SearchBot":["partial","*",50,0],"anthropic-ai":["partial","*",50,0],"PerplexityBot":["partial","*",50,0],"Perplexity-User":["partial","*",50,0],"Google-Extended":["partial","*",50,0],"Applebot-Extended":["partial","*",50,0],"CCBot":["partial","*",50,0],"Bytespider":["partial","*",50,0],"Amazonbot":["partial","*",50,0],"meta-externalagent":["partial","*",50,0],"cohere-ai":["partial","*",50,0],"Diffbot":["partial","*",50,0],"ContextFetchBot":["partial","*",50,0]},"llms":{"status":"present","http":200,"bytes":569245},"home":{"status":"ok","http":200,"url":"https://www.rockwellautomation.com","html_tokens":27645,"markdown_tokens":4,"pruned_tokens":4}},
{"rank":5418,"domain":"uoa.gr","origin":"https://uoa.gr","robots":{"status":"ok","http":200,"url":"https://www.uoa.gr/robots.txt","bytes":540,"groups":1},"bots":{"GPTBot":["partial","*",10,7],"ChatGPT-User":["partial","*",10,7],"OAI-SearchBot":["partial","*",10,7],"ClaudeBot":["partial","*",10,7],"Claude-User":["partial","*",10,7],"Claude-SearchBot":["partial","*",10,7],"anthropic-ai":["partial","*",10,7],"PerplexityBot":["partial","*",10,7],"Perplexity-User":["partial","*",10,7],"Google-Extended":["partial","*",10,7],"Applebot-Extended":["partial","*",10,7],"CCBot":["partial","*",10,7],"Bytespider":["partial","*",10,7],"Amazonbot":["partial","*",10,7],"meta-externalagent":["partial","*",10,7],"cohere-ai":["partial","*",10,7],"Diffbot":["partial","*",10,7],"ContextFetchBot":["partial","*",10,7]},"llms":{"status":"missing","http":404,"bytes":79412},"home":{"status":"ok","http":200,"url":"https://www.uoa.gr/","html_tokens":27637,"markdown_tokens":96,"pruned_tokens":74}},
{"rank":5419,"domain":"winnipegfreepress.com","origin":"https://winnipegfreepress.com","robots":{"status":"ok","http":200,"url":"https://www.winnipegfreepress.com/robots.txt","bytes":302,"groups":5},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["blocked","named",1,0],"OAI-SearchBot":["partial","*",1,1],"ClaudeBot":["partial","*",1,1],"Claude-User":["partial","*",1,1],"Claude-SearchBot":["partial","*",1,1],"anthropic-ai":["partial","*",1,1],"PerplexityBot":["partial","*",1,1],"Perplexity-User":["partial","*",1,1],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["partial","*",1,1],"CCBot":["blocked","named",1,0],"Bytespider":["partial","*",1,1],"Amazonbot":["partial","*",1,1],"meta-externalagent":["partial","*",1,1],"cohere-ai":["partial","*",1,1],"Diffbot":["partial","*",1,1],"ContextFetchBot":["partial","*",1,1]},"llms":{"status":"missing","http":404,"bytes":135246},"home":{"status":"ok","http":200,"url":"https://www.winnipegfreepress.com/","html_tokens":52207,"markdown_tokens":286,"pruned_tokens":142}},
{"rank":5420,"domain":"adminer.org","origin":"https://adminer.org","robots":{"status":"ok","http":200,"url":"https://www.adminer.org/robots.txt","bytes":45,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":2242},"home":{"status":"ok","http":200,"url":"https://www.adminer.org/","html_tokens":7748,"markdown_tokens":945,"pruned_tokens":400}},
{"rank":5421,"domain":"medicaldaily.com","origin":"https://medicaldaily.com","robots":{"status":"ok","http":200,"url":"https://www.medicaldaily.com/robots.txt","bytes":437,"groups":1},"bots":{"GPTBot":["partial","*",12,0],"ChatGPT-User":["partial","*",12,0],"OAI-SearchBot":["partial","*",12,0],"ClaudeBot":["partial","*",12,0],"Claude-User":["partial","*",12,0],"Claude-SearchBot":["partial","*",12,0],"anthropic-ai":["partial","*",12,0],"PerplexityBot":["partial","*",12,0],"Perplexity-User":["partial","*",12,0],"Google-Extended":["partial","*",12,0],"Applebot-Extended":["partial","*",12,0],"CCBot":["partial","*",12,0],"Bytespider":["partial","*",12,0],"Amazonbot":["partial","*",12,0],"meta-externalagent":["partial","*",12,0],"cohere-ai":["partial","*",12,0],"Diffbot":["partial","*",12,0],"ContextFetchBot":["partial","*",12,0]},"llms":{"status":"missing","http":404,"bytes":104},"home":{"status":"ok","http":200,"url":"https://www.medicaldaily.com/","html_tokens":62136,"markdown_tokens":567,"pruned_tokens":153}},
{"rank":5422,"domain":"hani.co.kr","origin":"https://hani.co.kr","robots":{"status":"ok","http":200,"url":"https://www.hani.co.kr/robots.txt","bytes":3825,"groups":3},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["partial","named",3,2],"OAI-SearchBot":["partial","named",3,2],"ClaudeBot":["blocked","named",1,0],"Claude-User":["blocked","named",1,0],"Claude-SearchBot":["partial","named",3,2],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["partial","*",5,0],"Perplexity-User":["partial","*",5,0],"Google-Extended":["partial","*",5,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",5,0]},"llms":{"status":"missing","http":404,"bytes":10088},"home":{"status":"ok","http":200,"url":"https://www.hani.co.kr/","html_tokens":43972,"markdown_tokens":61,"pruned_tokens":61}},
{"rank":5423,"domain":"printfriendly.com","origin":"https://printfriendly.com","robots":{"status":"ok","http":200,"url":"https://www.printfriendly.com/robots.txt","bytes":4128,"groups":2},"bots":{"GPTBot":["partial","*",53,1],"ChatGPT-User":["partial","*",53,1],"OAI-SearchBot":["partial","*",53,1],"ClaudeBot":["partial","*",53,1],"Claude-User":["partial","*",53,1],"Claude-SearchBot":["partial","*",53,1],"anthropic-ai":["partial","*",53,1],"PerplexityBot":["partial","*",53,1],"Perplexity-User":["partial","*",53,1],"Google-Extended":["partial","*",53,1],"Applebot-Extended":["partial","*",53,1],"CCBot":["partial","*",53,1],"Bytespider":["partial","*",53,1],"Amazonbot":["partial","*",53,1],"meta-externalagent":["partial","*",53,1],"cohere-ai":["partial","*",53,1],"Diffbot":["partial","*",53,1],"ContextFetchBot":["partial","*",53,1]},"llms":{"status":"present","http":200,"bytes":7984},"home":{"status":"http_error","http":403,"url":"https://www.printfriendly.com/"}},
{"rank":5424,"domain":"nuance.com","origin":"https://nuance.com","robots":{"status":"html","http":200,"url":"https://www.microsoft.com/en-us/health-solutions","bytes":168289,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":5425,"domain":"rapid7.com","origin":"https://rapid7.com","robots":{"status":"ok","http":200,"url":"https://www.rapid7.com/robots.txt","bytes":205,"groups":1},"bots":{"GPTBot":["partial","*",5,0],"ChatGPT-User":["partial","*",5,0],"OAI-SearchBot":["partial","*",5,0],"ClaudeBot":["partial","*",5,0],"Claude-User":["partial","*",5,0],"Claude-SearchBot":["partial","*",5,0],"anthropic-ai":["partial","*",5,0],"PerplexityBot":["partial","*",5,0],"Perplexity-User":["partial","*",5,0],"Google-Extended":["partial","*",5,0],"Applebot-Extended":["partial","*",5,0],"CCBot":["partial","*",5,0],"Bytespider":["partial","*",5,0],"Amazonbot":["partial","*",5,0],"meta-externalagent":["partial","*",5,0],"cohere-ai":["partial","*",5,0],"Diffbot":["partial","*",5,0],"ContextFetchBot":["partial","*",5,0]},"llms":{"status":"present","http":200,"bytes":2271},"home":{"status":"ok","http":200,"url":"https://www.rapid7.com/","html_tokens":184258,"markdown_tokens":259,"pruned_tokens":100}},
{"rank":5426,"domain":"google.by","origin":"https://google.by","robots":{"status":"ok","http":200,"url":"https://www.google.by/robots.txt","bytes":6624,"groups":4},"bots":{"GPTBot":["partial","*",169,64],"ChatGPT-User":["partial","*",169,64],"OAI-SearchBot":["partial","*",169,64],"ClaudeBot":["partial","*",169,64],"Claude-User":["partial","*",169,64],"Claude-SearchBot":["partial","*",169,64],"anthropic-ai":["partial","*",169,64],"PerplexityBot":["partial","*",169,64],"Perplexity-User":["partial","*",169,64],"Google-Extended":["partial","*",169,64],"Applebot-Extended":["partial","*",169,64],"CCBot":["partial","*",169,64],"Bytespider":["partial","*",169,64],"Amazonbot":["partial","*",169,64],"meta-externalagent":["partial","*",169,64],"cohere-ai":["partial","*",169,64],"Diffbot":["partial","*",169,64],"ContextFetchBot":["partial","*",169,64]},"llms":{"status":"missing","http":404,"bytes":1569},"home":{"status":"ok","http":200,"url":"https://www.google.com/","html_tokens":21289,"markdown_tokens":30,"pruned_tokens":30}},
{"rank":5427,"domain":"cgiar.org","origin":"https://cgiar.org","robots":{"status":"ok","http":200,"url":"https://www.cgiar.org/robots.txt","bytes":2881,"groups":1},"bots":{"GPTBot":["partial","*",58,23],"ChatGPT-User":["partial","*",58,23],"OAI-SearchBot":["partial","*",58,23],"ClaudeBot":["partial","*",58,23],"Claude-User":["partial","*",58,23],"Claude-SearchBot":["partial","*",58,23],"anthropic-ai":["partial","*",58,23],"PerplexityBot":["partial","*",58,23],"Perplexity-User":["partial","*",58,23],"Google-Extended":["partial","*",58,23],"Applebot-Extended":["partial","*",58,23],"CCBot":["partial","*",58,23],"Bytespider":["partial","*",58,23],"Amazonbot":["partial","*",58,23],"meta-externalagent":["partial","*",58,23],"cohere-ai":["partial","*",58,23],"Diffbot":["partial","*",58,23],"ContextFetchBot":["partial","*",58,23]},"llms":{"status":"missing","http":404,"bytes":187},"home":{"status":"ok","http":200,"url":"https://www.cgiar.org/","html_tokens":43672,"markdown_tokens":167,"pruned_tokens":167}},
{"rank":5428,"domain":"theringer.com","origin":"https://theringer.com","robots":{"status":"ok","http":200,"url":"https://www.theringer.com/robots.txt","bytes":172,"groups":2},"bots":{"GPTBot":["partial","*",3,2],"ChatGPT-User":["partial","*",3,2],"OAI-SearchBot":["partial","*",3,2],"ClaudeBot":["partial","*",3,2],"Claude-User":["partial","*",3,2],"Claude-SearchBot":["partial","*",3,2],"anthropic-ai":["partial","*",3,2],"PerplexityBot":["partial","*",3,2],"Perplexity-User":["partial","*",3,2],"Google-Extended":["partial","*",3,2],"Applebot-Extended":["partial","*",3,2],"CCBot":["partial","*",3,2],"Bytespider":["partial","*",3,2],"Amazonbot":["partial","*",3,2],"meta-externalagent":["partial","*",3,2],"cohere-ai":["partial","*",3,2],"Diffbot":["partial","*",3,2],"ContextFetchBot":["partial","*",3,2]},"llms":{"status":"missing","http":404,"bytes":64670},"home":{"status":"ok","http":200,"url":"https://www.theringer.com","html_tokens":439196,"markdown_tokens":30,"pruned_tokens":30}},
{"rank":5429,"domain":"score.org","origin":"https://score.org","robots":{"status":"ok","http":200,"url":"https://www.score.org/robots.txt","bytes":117,"groups":1},"bots":{"GPTBot":["partial","*",1,1],"ChatGPT-User":["partial","*",1,1],"OAI-SearchBot":["partial","*",1,1],"ClaudeBot":["partial","*",1,1],"Claude-User":["partial","*",1,1],"Claude-SearchBot":["partial","*",1,1],"anthropic-ai":["partial","*",1,1],"PerplexityBot":["partial","*",1,1],"Perplexity-User":["partial","*",1,1],"Google-Extended":["partial","*",1,1],"Applebot-Extended":["partial","*",1,1],"CCBot":["partial","*",1,1],"Bytespider":["partial","*",1,1],"Amazonbot":["partial","*",1,1],"meta-externalagent":["partial","*",1,1],"cohere-ai":["partial","*",1,1],"Diffbot":["partial","*",1,1],"ContextFetchBot":["partial","*",1,1]},"llms":{"status":"missing","http":404,"bytes":2804},"home":{"status":"http_error","http":403,"url":"https://www.score.org/"}},
{"rank":5430,"domain":"unina.it","origin":"https://www.unina.it","robots":{"status":"ok","http":200,"url":"https://www.unina.it/robots.txt","bytes":69,"groups":1},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"disallowed","error":"robots.txt of www.old.unina.it not readable (server_error)"},"home":{"status":"ok","http":200,"url":"https://www.unina.it/it/","html_tokens":71869,"markdown_tokens":139,"pruned_tokens":139}},
{"rank":5431,"domain":"wwe.com","origin":"https://wwe.com","robots":{"status":"ok","http":200,"url":"https://www.wwe.com/robots.txt","bytes":2027,"groups":1},"bots":{"GPTBot":["partial","*",34,18],"ChatGPT-User":["partial","*",34,18],"OAI-SearchBot":["partial","*",34,18],"ClaudeBot":["partial","*",34,18],"Claude-User":["partial","*",34,18],"Claude-SearchBot":["partial","*",34,18],"anthropic-ai":["partial","*",34,18],"PerplexityBot":["partial","*",34,18],"Perplexity-User":["partial","*",34,18],"Google-Extended":["partial","*",34,18],"Applebot-Extended":["partial","*",34,18],"CCBot":["partial","*",34,18],"Bytespider":["partial","*",34,18],"Amazonbot":["partial","*",34,18],"meta-externalagent":["partial","*",34,18],"cohere-ai":["partial","*",34,18],"Diffbot":["partial","*",34,18],"ContextFetchBot":["partial","*",34,18]},"llms":{"status":"html","http":200,"bytes":433258},"home":{"status":"ok","http":200,"url":"https://www.wwe.com/","html_tokens":108295,"markdown_tokens":26,"pruned_tokens":21}},
{"rank":5432,"domain":"wv.gov","origin":"https://wv.gov","robots":{"status":"html","http":200,"url":"https://www.wv.gov/","bytes":40915,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":5433,"domain":"50megs.com","origin":"http://50megs.com","robots":{"status":"ok","http":200,"url":"http://www.50megs.com/robots.txt","bytes":76,"groups":1},"bots":{"GPTBot":["partial","*",3,0],"ChatGPT-User":["partial","*",3,0],"OAI-SearchBot":["partial","*",3,0],"ClaudeBot":["partial","*",3,0],"Claude-User":["partial","*",3,0],"Claude-SearchBot":["partial","*",3,0],"anthropic-ai":["partial","*",3,0],"PerplexityBot":["partial","*",3,0],"Perplexity-User":["partial","*",3,0],"Google-Extended":["partial","*",3,0],"Applebot-Extended":["partial","*",3,0],"CCBot":["partial","*",3,0],"Bytespider":["partial","*",3,0],"Amazonbot":["partial","*",3,0],"meta-externalagent":["partial","*",3,0],"cohere-ai":["partial","*",3,0],"Diffbot":["partial","*",3,0],"ContextFetchBot":["partial","*",3,0]},"llms":{"status":"missing","http":404,"bytes":7751},"home":{"status":"ok","http":200,"url":"http://www.50megs.com/","html_tokens":4080,"markdown_tokens":554,"pruned_tokens":395}},
{"rank":5434,"domain":"sofi.com","origin":"https://sofi.com","robots":{"status":"ok","http":200,"url":"https://www.sofi.com/robots.txt","bytes":940,"groups":3},"bots":{"GPTBot":["partial","*",19,2],"ChatGPT-User":["partial","*",19,2],"OAI-SearchBot":["partial","*",19,2],"ClaudeBot":["partial","*",19,2],"Claude-User":["partial","*",19,2],"Claude-SearchBot":["partial","*",19,2],"anthropic-ai":["partial","*",19,2],"PerplexityBot":["partial","*",19,2],"Perplexity-User":["partial","*",19,2],"Google-Extended":["partial","*",19,2],"Applebot-Extended":["partial","*",19,2],"CCBot":["partial","*",19,2],"Bytespider":["partial","*",19,2],"Amazonbot":["partial","*",19,2],"meta-externalagent":["partial","*",19,2],"cohere-ai":["partial","*",19,2],"Diffbot":["partial","*",19,2],"ContextFetchBot":["partial","*",19,2]},"llms":{"status":"missing","http":404,"bytes":175990},"home":{"status":"http_error","http":403,"url":"https://www.sofi.com/"}},
{"rank":5435,"domain":"realmoney-casino.ca","origin":"https://realmoney-casino.ca","robots":{"status":"ok","http":200,"url":"https://realmoney-casino.ca/robots.txt","bytes":543,"groups":1},"bots":{"GPTBot":["partial","*",19,0],"ChatGPT-User":["partial","*",19,0],"OAI-SearchBot":["partial","*",19,0],"ClaudeBot":["partial","*",19,0],"Claude-User":["partial","*",19,0],"Claude-SearchBot":["partial","*",19,0],"anthropic-ai":["partial","*",19,0],"PerplexityBot":["partial","*",19,0],"Perplexity-User":["partial","*",19,0],"Google-Extended":["partial","*",19,0],"Applebot-Extended":["partial","*",19,0],"CCBot":["partial","*",19,0],"Bytespider":["partial","*",19,0],"Amazonbot":["partial","*",19,0],"meta-externalagent":["partial","*",19,0],"cohere-ai":["partial","*",19,0],"Diffbot":["partial","*",19,0],"ContextFetchBot":["partial","*",19,0]},"llms":{"status":"missing","http":404,"bytes":44075},"home":{"status":"ok","http":200,"url":"https://realmoney-casino.ca/","html_tokens":54893,"markdown_tokens":5075,"pruned_tokens":166}},
{"rank":5436,"domain":"tugraz.at","origin":"https://tugraz.at","robots":{"status":"ok","http":200,"url":"https://www.tugraz.at/robots.txt","bytes":372,"groups":1},"bots":{"GPTBot":["partial","*",11,0],"ChatGPT-User":["partial","*",11,0],"OAI-SearchBot":["partial","*",11,0],"ClaudeBot":["partial","*",11,0],"Claude-User":["partial","*",11,0],"Claude-SearchBot":["partial","*",11,0],"anthropic-ai":["partial","*",11,0],"PerplexityBot":["partial","*",11,0],"Perplexity-User":["partial","*",11,0],"Google-Extended":["partial","*",11,0],"Applebot-Extended":["partial","*",11,0],"CCBot":["partial","*",11,0],"Bytespider":["partial","*",11,0],"Amazonbot":["partial","*",11,0],"meta-externalagent":["partial","*",11,0],"cohere-ai":["partial","*",11,0],"Diffbot":["partial","*",11,0],"ContextFetchBot":["partial","*",11,0]},"llms":{"status":"missing","http":404,"bytes":71346},"home":{"status":"ok","http":200,"url":"https://www.tugraz.at/home","html_tokens":16927,"markdown_tokens":884,"pruned_tokens":169}},
{"rank":5437,"domain":"migrationpolicy.org","origin":"https://migrationpolicy.org","robots":{"status":"ok","http":200,"url":"https://www.migrationpolicy.org/robots.txt","bytes":2141,"groups":1},"bots":{"GPTBot":["partial","*",35,22],"ChatGPT-User":["partial","*",35,22],"OAI-SearchBot":["partial","*",35,22],"ClaudeBot":["partial","*",35,22],"Claude-User":["partial","*",35,22],"Claude-SearchBot":["partial","*",35,22],"anthropic-ai":["partial","*",35,22],"PerplexityBot":["partial","*",35,22],"Perplexity-User":["partial","*",35,22],"Google-Extended":["partial","*",35,22],"Applebot-Extended":["partial","*",35,22],"CCBot":["partial","*",35,22],"Bytespider":["partial","*",35,22],"Amazonbot":["partial","*",35,22],"meta-externalagent":["partial","*",35,22],"cohere-ai":["partial","*",35,22],"Diffbot":["partial","*",35,22],"ContextFetchBot":["partial","*",35,22]},"llms":{"status":"missing","http":404,"bytes":2154},"home":{"status":"http_error","http":403,"url":"https://migrationpolicy.org/"}},
{"rank":5438,"domain":"hertz.com","origin":"https://hertz.com","robots":{"status":"ok","http":200,"url":"https://www.hertz.com/robots.txt","bytes":6275,"groups":12},"bots":{"GPTBot":["partial","named",14,0],"ChatGPT-User":["partial","named",14,0],"OAI-SearchBot":["partial","named",14,0],"ClaudeBot":["partial","*",14,0],"Claude-User":["partial","named",14,0],"Claude-SearchBot":["partial","named",14,0],"anthropic-ai":["partial","*",14,0],"PerplexityBot":["partial","named",14,0],"Perplexity-User":["partial","named",14,0],"Google-Extended":["partial","*",14,0],"Applebot-Extended":["partial","*",14,0],"CCBot":["partial","named",14,0],"Bytespider":["partial","*",14,0],"Amazonbot":["partial","*",14,0],"meta-externalagent":["partial","*",14,0],"cohere-ai":["partial","*",14,0],"Diffbot":["partial","*",14,0],"ContextFetchBot":["partial","*",14,0]},"llms":{"status":"missing","http":404,"bytes":9630},"home":{"status":"ok","http":200,"url":"https://www.hertz.com/us/en","html_tokens":4412,"markdown_tokens":0,"pruned_tokens":0}},
{"rank":5439,"domain":"ckeditor.com","origin":"https://ckeditor.com","robots":{"status":"ok","http":200,"url":"https://ckeditor.com/robots.txt","bytes":745,"groups":1},"bots":{"GPTBot":["partial","*",19,0],"ChatGPT-User":["partial","*",19,0],"OAI-SearchBot":["partial","*",19,0],"ClaudeBot":["partial","*",19,0],"Claude-User":["partial","*",19,0],"Claude-SearchBot":["partial","*",19,0],"anthropic-ai":["partial","*",19,0],"PerplexityBot":["partial","*",19,0],"Perplexity-User":["partial","*",19,0],"Google-Extended":["partial","*",19,0],"Applebot-Extended":["partial","*",19,0],"CCBot":["partial","*",19,0],"Bytespider":["partial","*",19,0],"Amazonbot":["partial","*",19,0],"meta-externalagent":["partial","*",19,0],"cohere-ai":["partial","*",19,0],"Diffbot":["partial","*",19,0],"ContextFetchBot":["partial","*",19,0]},"llms":{"status":"present","http":200,"bytes":18207},"home":{"status":"not_html","error":"unsupported content type text/markdown"}},
{"rank":5440,"domain":"browserstack.com","origin":"https://browserstack.com","robots":{"status":"ok","http":200,"url":"https://www.browserstack.com/robots.txt","bytes":1619,"groups":1},"bots":{"GPTBot":["partial","*",32,3],"ChatGPT-User":["partial","*",32,3],"OAI-SearchBot":["partial","*",32,3],"ClaudeBot":["partial","*",32,3],"Claude-User":["partial","*",32,3],"Claude-SearchBot":["partial","*",32,3],"anthropic-ai":["partial","*",32,3],"PerplexityBot":["partial","*",32,3],"Perplexity-User":["partial","*",32,3],"Google-Extended":["partial","*",32,3],"Applebot-Extended":["partial","*",32,3],"CCBot":["partial","*",32,3],"Bytespider":["partial","*",32,3],"Amazonbot":["partial","*",32,3],"meta-externalagent":["partial","*",32,3],"cohere-ai":["partial","*",32,3],"Diffbot":["partial","*",32,3],"ContextFetchBot":["partial","*",32,3]},"llms":{"status":"missing","http":404,"bytes":173869},"home":{"status":"ok","http":200,"url":"https://www.browserstack.com/","html_tokens":154314,"markdown_tokens":291,"pruned_tokens":135}},
{"rank":5441,"domain":"lovetoknow.com","origin":"https://lovetoknow.com","robots":{"status":"ok","http":200,"url":"https://www.lovetoknow.com/robots.txt","bytes":163,"groups":1},"bots":{"GPTBot":["partial","*",2,0],"ChatGPT-User":["partial","*",2,0],"OAI-SearchBot":["partial","*",2,0],"ClaudeBot":["partial","*",2,0],"Claude-User":["partial","*",2,0],"Claude-SearchBot":["partial","*",2,0],"anthropic-ai":["partial","*",2,0],"PerplexityBot":["partial","*",2,0],"Perplexity-User":["partial","*",2,0],"Google-Extended":["partial","*",2,0],"Applebot-Extended":["partial","*",2,0],"CCBot":["partial","*",2,0],"Bytespider":["partial","*",2,0],"Amazonbot":["partial","*",2,0],"meta-externalagent":["partial","*",2,0],"cohere-ai":["partial","*",2,0],"Diffbot":["partial","*",2,0],"ContextFetchBot":["partial","*",2,0]},"llms":{"status":"missing","http":404,"bytes":51675},"home":{"status":"ok","http":200,"url":"https://www.lovetoknow.com/","html_tokens":49965,"markdown_tokens":64,"pruned_tokens":64}},
{"rank":5442,"domain":"99designs.com","origin":"https://99designs.com","robots":{"status":"ok","http":200,"url":"https://99designs.com/robots.txt","bytes":12917,"groups":3},"bots":{"GPTBot":["partial","*",211,1],"ChatGPT-User":["partial","*",211,1],"OAI-SearchBot":["partial","*",211,1],"ClaudeBot":["partial","*",211,1],"Claude-User":["partial","*",211,1],"Claude-SearchBot":["partial","*",211,1],"anthropic-ai":["partial","*",211,1],"PerplexityBot":["partial","*",211,1],"Perplexity-User":["partial","*",211,1],"Google-Extended":["partial","*",211,1],"Applebot-Extended":["partial","*",211,1],"CCBot":["partial","*",211,1],"Bytespider":["partial","*",211,1],"Amazonbot":["partial","*",211,1],"meta-externalagent":["partial","*",211,1],"cohere-ai":["partial","*",211,1],"Diffbot":["partial","*",211,1],"ContextFetchBot":["partial","*",211,1]},"llms":{"status":"missing","http":404,"bytes":13014},"home":{"status":"ok","http":200,"url":"https://99designs.com/","html_tokens":40781,"markdown_tokens":711,"pruned_tokens":181}},
{"rank":5443,"domain":"fanfiction.net","origin":"https://www.fanfiction.net","robots":{"status":"ok","http":200,"url":"https://www.fanfiction.net/robots.txt","bytes":173,"groups":2},"bots":{"GPTBot":["partial","*",6,1],"ChatGPT-User":["partial","*",6,1],"OAI-SearchBot":["partial","*",6,1],"ClaudeBot":["partial","*",6,1],"Claude-User":["partial","*",6,1],"Claude-SearchBot":["partial","*",6,1],"anthropic-ai":["partial","*",6,1],"PerplexityBot":["partial","*",6,1],"Perplexity-User":["partial","*",6,1],"Google-Extended":["partial","*",6,1],"Applebot-Extended":["partial","*",6,1],"CCBot":["partial","*",6,1],"Bytespider":["partial","*",6,1],"Amazonbot":["partial","*",6,1],"meta-externalagent":["partial","*",6,1],"cohere-ai":["partial","*",6,1],"Diffbot":["partial","*",6,1],"ContextFetchBot":["partial","*",6,1]},"llms":{"status":"missing","http":404,"bytes":100},"home":{"status":"http_error","http":403,"url":"https://www.fanfiction.net/"}},
{"rank":5444,"domain":"ba.gov.br","origin":"https://ba.gov.br","robots":{"status":"server_error","http":500,"url":"https://ba.gov.br/robots.txt","bytes":1732,"groups":0,"error":"HTTP 500"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":5445,"domain":"fpf.org","origin":"https://fpf.org","robots":{"status":"ok","http":200,"url":"https://fpf.org/robots.txt","bytes":200,"groups":1},"bots":{"GPTBot":["partial","*",6,1],"ChatGPT-User":["partial","*",6,1],"OAI-SearchBot":["partial","*",6,1],"ClaudeBot":["partial","*",6,1],"Claude-User":["partial","*",6,1],"Claude-SearchBot":["partial","*",6,1],"anthropic-ai":["partial","*",6,1],"PerplexityBot":["partial","*",6,1],"Perplexity-User":["partial","*",6,1],"Google-Extended":["partial","*",6,1],"Applebot-Extended":["partial","*",6,1],"CCBot":["partial","*",6,1],"Bytespider":["partial","*",6,1],"Amazonbot":["partial","*",6,1],"meta-externalagent":["partial","*",6,1],"cohere-ai":["partial","*",6,1],"Diffbot":["partial","*",6,1],"ContextFetchBot":["partial","*",6,1]},"llms":{"status":"missing","http":404,"bytes":63574},"home":{"status":"ok","http":200,"url":"https://fpf.org/","html_tokens":25209,"markdown_tokens":157,"pruned_tokens":157}},
{"rank":5446,"domain":"demilked.com","origin":"https://demilked.com","robots":{"status":"ok","http":200,"url":"https://demilked.com/robots.txt","bytes":2676,"groups":32},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["allowed","*",0,1],"OAI-SearchBot":["allowed","*",0,1],"ClaudeBot":["blocked","named",1,0],"Claude-User":["allowed","*",0,1],"Claude-SearchBot":["allowed","*",0,1],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["allowed","*",0,1],"Perplexity-User":["allowed","*",0,1],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["allowed","*",0,1]},"llms":{"status":"missing","http":404,"bytes":246965},"home":{"status":"ok","http":200,"url":"https://www.demilked.com/","html_tokens":118533,"markdown_tokens":425,"pruned_tokens":71}},
{"rank":5447,"domain":"intercom.help","origin":"https://intercom.help","robots":{"status":"ok","http":200,"url":"https://intercom.help/robots.txt","bytes":111,"groups":1},"bots":{"GPTBot":["partial","*",3,0],"ChatGPT-User":["partial","*",3,0],"OAI-SearchBot":["partial","*",3,0],"ClaudeBot":["partial","*",3,0],"Claude-User":["partial","*",3,0],"Claude-SearchBot":["partial","*",3,0],"anthropic-ai":["partial","*",3,0],"PerplexityBot":["partial","*",3,0],"Perplexity-User":["partial","*",3,0],"Google-Extended":["partial","*",3,0],"Applebot-Extended":["partial","*",3,0],"CCBot":["partial","*",3,0],"Bytespider":["partial","*",3,0],"Amazonbot":["partial","*",3,0],"meta-externalagent":["partial","*",3,0],"cohere-ai":["partial","*",3,0],"Diffbot":["partial","*",3,0],"ContextFetchBot":["partial","*",3,0]},"llms":{"status":"missing","http":404,"bytes":9},"home":{"status":"not_html","error":"unsupported content type text/markdown"}},
{"rank":5448,"domain":"thefreelibrary.com","origin":"https://thefreelibrary.com","robots":{"status":"ok","http":200,"url":"https://www.thefreelibrary.com/robots.txt","bytes":997,"groups":4},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["blocked","named",1,0],"OAI-SearchBot":["blocked","named",1,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["blocked","named",1,0],"Claude-SearchBot":["blocked","named",1,0],"anthropic-ai":["partial","*",3,0],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["partial","*",3,0],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["partial","*",3,0],"Diffbot":["partial","*",3,0],"ContextFetchBot":["partial","*",3,0]},"llms":{"status":"missing","http":404,"bytes":2684},"home":{"status":"http_error","http":403,"url":"https://thefreelibrary.com/"}},
{"rank":5449,"domain":"uni-potsdam.de","origin":"https://uni-potsdam.de","robots":{"status":"ok","http":200,"url":"https://www.uni-potsdam.de/de/robots.txt","bytes":107,"groups":1},"bots":{"GPTBot":["partial","*",2,1],"ChatGPT-User":["partial","*",2,1],"OAI-SearchBot":["partial","*",2,1],"ClaudeBot":["partial","*",2,1],"Claude-User":["partial","*",2,1],"Claude-SearchBot":["partial","*",2,1],"anthropic-ai":["partial","*",2,1],"PerplexityBot":["partial","*",2,1],"Perplexity-User":["partial","*",2,1],"Google-Extended":["partial","*",2,1],"Applebot-Extended":["partial","*",2,1],"CCBot":["partial","*",2,1],"Bytespider":["partial","*",2,1],"Amazonbot":["partial","*",2,1],"meta-externalagent":["partial","*",2,1],"cohere-ai":["partial","*",2,1],"Diffbot":["partial","*",2,1],"ContextFetchBot":["partial","*",2,1]},"llms":{"status":"missing","http":404,"bytes":5598},"home":{"status":"ok","http":200,"url":"https://www.uni-potsdam.de/de","html_tokens":19425,"markdown_tokens":22,"pruned_tokens":22}},
{"rank":5450,"domain":"museodelprado.es","origin":"https://museodelprado.es","robots":{"status":"refused","http":403,"url":"https://www.museodelprado.es/robots.txt","bytes":5430,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":5451,"domain":"solidworks.com","origin":"https://solidworks.com","robots":{"status":"timeout","http":null,"url":"https://solidworks.com/robots.txt","bytes":0,"groups":0,"error":"ConnectTimeout"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":5452,"domain":"nbcphiladelphia.com","origin":"https://nbcphiladelphia.com","robots":{"status":"ok","http":200,"url":"https://www.nbcphiladelphia.com/robots.txt","bytes":10990,"groups":27},"bots":{"GPTBot":["partial","named",1,6],"ChatGPT-User":["partial","named",1,6],"OAI-SearchBot":["partial","*",13,3],"ClaudeBot":["partial","named",1,6],"Claude-User":["partial","*",13,3],"Claude-SearchBot":["partial","*",13,3],"anthropic-ai":["partial","named",1,6],"PerplexityBot":["partial","named",1,6],"Perplexity-User":["partial","*",13,3],"Google-Extended":["partial","named",1,6],"Applebot-Extended":["partial","*",13,3],"CCBot":["partial","named",1,6],"Bytespider":["partial","named",1,6],"Amazonbot":["partial","named",1,8],"meta-externalagent":["partial","*",13,3],"cohere-ai":["partial","*",13,3],"Diffbot":["partial","named",1,6],"ContextFetchBot":["partial","*",13,3]},"llms":{"status":"html","http":200,"bytes":618362},"home":{"status":"ok","http":200,"url":"https://www.nbcphiladelphia.com/","html_tokens":147214,"markdown_tokens":3,"pruned_tokens":3}},
{"rank":5453,"domain":"9news.com","origin":"https://9news.com","robots":{"status":"ok","http":200,"url":"https://www.9news.com/robots.txt","bytes":447,"groups":2},"bots":{"GPTBot":["partial","*",10,0],"ChatGPT-User":["partial","*",10,0],"OAI-SearchBot":["partial","*",10,0],"ClaudeBot":["partial","*",10,0],"Claude-User":["partial","*",10,0],"Claude-SearchBot":["partial","*",10,0],"anthropic-ai":["partial","*",10,0],"PerplexityBot":["partial","*",10,0],"Perplexity-User":["partial","*",10,0],"Google-Extended":["partial","*",10,0],"Applebot-Extended":["partial","*",10,0],"CCBot":["partial","*",10,0],"Bytespider":["partial","*",10,0],"Amazonbot":["partial","*",10,0],"meta-externalagent":["partial","*",10,0],"cohere-ai":["partial","*",10,0],"Diffbot":["partial","*",10,0],"ContextFetchBot":["partial","*",10,0]},"llms":{"status":"missing","http":404,"bytes":6837},"home":{"status":"ok","http":200,"url":"https://www.9news.com/","html_tokens":100700,"markdown_tokens":1639,"pruned_tokens":168}},
{"rank":5454,"domain":"indiebound.org","origin":"https://indiebound.org","robots":{"status":"ok","http":200,"url":"https://www.indiebound.org/robots.txt","bytes":2027,"groups":1},"bots":{"GPTBot":["partial","*",34,18],"ChatGPT-User":["partial","*",34,18],"OAI-SearchBot":["partial","*",34,18],"ClaudeBot":["partial","*",34,18],"Claude-User":["partial","*",34,18],"Claude-SearchBot":["partial","*",34,18],"anthropic-ai":["partial","*",34,18],"PerplexityBot":["partial","*",34,18],"Perplexity-User":["partial","*",34,18],"Google-Extended":["partial","*",34,18],"Applebot-Extended":["partial","*",34,18],"CCBot":["partial","*",34,18],"Bytespider":["partial","*",34,18],"Amazonbot":["partial","*",34,18],"meta-externalagent":["partial","*",34,18],"cohere-ai":["partial","*",34,18],"Diffbot":["partial","*",34,18],"ContextFetchBot":["partial","*",34,18]},"llms":{"status":"missing","http":404,"bytes":192},"home":{"status":"ok","http":200,"url":"https://www.indiebound.org/","html_tokens":61536,"markdown_tokens":300,"pruned_tokens":36}},
{"rank":5455,"domain":"nst.com.my","origin":"https://nst.com.my","robots":{"status":"ok","http":200,"url":"https://nst.com.my/robots.txt","bytes":1065,"groups":22},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["partial","named",1,1],"OAI-SearchBot":["partial","named",1,1],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","named",1,1],"Claude-SearchBot":["partial","named",1,1],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["partial","named",1,1],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","named",0,1],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"missing","http":404,"bytes":51660},"home":{"status":"ok","http":200,"url":"https://www.nst.com.my/","html_tokens":15110,"markdown_tokens":1,"pruned_tokens":1}},
{"rank":5456,"domain":"merckmanuals.com","origin":"https://merckmanuals.com","robots":{"status":"ok","http":200,"url":"https://www.merckmanuals.com/robots.txt","bytes":1123,"groups":4},"bots":{"GPTBot":["partial","*",24,0],"ChatGPT-User":["partial","*",24,0],"OAI-SearchBot":["partial","*",24,0],"ClaudeBot":["partial","*",24,0],"Claude-User":["partial","*",24,0],"Claude-SearchBot":["partial","*",24,0],"anthropic-ai":["partial","*",24,0],"PerplexityBot":["partial","*",24,0],"Perplexity-User":["partial","*",24,0],"Google-Extended":["partial","*",24,0],"Applebot-Extended":["partial","*",24,0],"CCBot":["partial","*",24,0],"Bytespider":["partial","*",24,0],"Amazonbot":["partial","*",24,0],"meta-externalagent":["partial","*",24,0],"cohere-ai":["partial","*",24,0],"Diffbot":["partial","*",24,0],"ContextFetchBot":["partial","*",24,0]},"llms":{"status":"missing","http":404,"bytes":123431},"home":{"status":"ok","http":200,"url":"https://www.merckmanuals.com/","html_tokens":33645,"markdown_tokens":642,"pruned_tokens":128}},
{"rank":5457,"domain":"pirelli.com","origin":"https://pirelli.com","robots":{"status":"ok","http":200,"url":"https://www.pirelli.com/robots.txt","bytes":5623,"groups":3},"bots":{"GPTBot":["partial","*",29,0],"ChatGPT-User":["partial","*",29,0],"OAI-SearchBot":["partial","*",29,0],"ClaudeBot":["partial","*",29,0],"Claude-User":["partial","*",29,0],"Claude-SearchBot":["partial","*",29,0],"anthropic-ai":["partial","*",29,0],"PerplexityBot":["partial","*",29,0],"Perplexity-User":["partial","*",29,0],"Google-Extended":["partial","*",29,0],"Applebot-Extended":["partial","*",29,0],"CCBot":["partial","*",29,0],"Bytespider":["partial","*",29,0],"Amazonbot":["partial","*",29,0],"meta-externalagent":["partial","*",29,0],"cohere-ai":["partial","*",29,0],"Diffbot":["partial","*",29,0],"ContextFetchBot":["partial","*",29,0]},"llms":{"status":"present","http":200,"bytes":3698},"home":{"status":"timeout","error":"timed out after 15 s"}},
{"rank":5458,"domain":"ipsnews.net","origin":"https://ipsnews.net","robots":{"status":"ok","http":200,"url":"https://ipsnews.net/robots.txt","bytes":1415,"groups":21},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["partial","*",3,1],"OAI-SearchBot":["partial","*",3,1],"ClaudeBot":["allowed","named",0,0],"Claude-User":["partial","*",3,1],"Claude-SearchBot":["partial","*",3,1],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["allowed","named",0,0],"Perplexity-User":["partial","*",3,1],"Google-Extended":["allowed","named",0,0],"Applebot-Extended":["allowed","named",0,0],"CCBot":["blocked","named",1,0],"Bytespider":["partial","*",3,1],"Amazonbot":["partial","*",3,1],"meta-externalagent":["partial","*",3,1],"cohere-ai":["blocked","named",1,0],"Diffbot":["partial","*",3,1],"ContextFetchBot":["partial","*",3,1]},"llms":{"status":"other_text","http":200,"bytes":6661},"home":{"status":"ok","http":200,"url":"https://www.ipsnews.net/","html_tokens":26404,"markdown_tokens":159,"pruned_tokens":159}},
{"rank":5459,"domain":"bloggersdelight.dk","origin":"https://bloggersdelight.dk","robots":{"status":"ok","http":200,"url":"https://bloggersdelight.dk/robots.txt","bytes":154,"groups":2},"bots":{"GPTBot":["blocked","*",1,0],"ChatGPT-User":["blocked","*",1,0],"OAI-SearchBot":["blocked","*",1,0],"ClaudeBot":["blocked","*",1,0],"Claude-User":["blocked","*",1,0],"Claude-SearchBot":["blocked","*",1,0],"anthropic-ai":["blocked","*",1,0],"PerplexityBot":["blocked","*",1,0],"Perplexity-User":["blocked","*",1,0],"Google-Extended":["blocked","*",1,0],"Applebot-Extended":["blocked","*",1,0],"CCBot":["blocked","*",1,0],"Bytespider":["blocked","*",1,0],"Amazonbot":["blocked","*",1,0],"meta-externalagent":["blocked","*",1,0],"cohere-ai":["blocked","*",1,0],"Diffbot":["blocked","*",1,0],"ContextFetchBot":["blocked","*",1,0]},"llms":{"status":"disallowed"},"home":{"status":"disallowed"}},
{"rank":5460,"domain":"livestorm.co","origin":"https://livestorm.co","robots":{"status":"ok","http":200,"url":"https://livestorm.co/robots.txt","bytes":174,"groups":1},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"present","http":200,"bytes":202021},"home":{"status":"ok","http":200,"url":"https://livestorm.co/","html_tokens":76283,"markdown_tokens":440,"pruned_tokens":81}},
{"rank":5461,"domain":"olemiss.edu","origin":"https://olemiss.edu","robots":{"status":"ok","http":200,"url":"https://olemiss.edu/robots.txt","bytes":367,"groups":1},"bots":{"GPTBot":["partial","*",12,0],"ChatGPT-User":["partial","*",12,0],"OAI-SearchBot":["partial","*",12,0],"ClaudeBot":["partial","*",12,0],"Claude-User":["partial","*",12,0],"Claude-SearchBot":["partial","*",12,0],"anthropic-ai":["partial","*",12,0],"PerplexityBot":["partial","*",12,0],"Perplexity-User":["partial","*",12,0],"Google-Extended":["partial","*",12,0],"Applebot-Extended":["partial","*",12,0],"CCBot":["partial","*",12,0],"Bytespider":["partial","*",12,0],"Amazonbot":["partial","*",12,0],"meta-externalagent":["partial","*",12,0],"cohere-ai":["partial","*",12,0],"Diffbot":["partial","*",12,0],"ContextFetchBot":["partial","*",12,0]},"llms":{"status":"missing","http":404,"bytes":80057},"home":{"status":"ok","http":200,"url":"https://olemiss.edu/","html_tokens":47980,"markdown_tokens":390,"pruned_tokens":159}},
{"rank":5462,"domain":"ens-lyon.fr","origin":"https://www.ens-lyon.fr","robots":{"status":"ok","http":200,"url":"https://www.ens-lyon.fr/robots.txt","bytes":1882,"groups":1},"bots":{"GPTBot":["partial","*",29,18],"ChatGPT-User":["partial","*",29,18],"OAI-SearchBot":["partial","*",29,18],"ClaudeBot":["partial","*",29,18],"Claude-User":["partial","*",29,18],"Claude-SearchBot":["partial","*",29,18],"anthropic-ai":["partial","*",29,18],"PerplexityBot":["partial","*",29,18],"Perplexity-User":["partial","*",29,18],"Google-Extended":["partial","*",29,18],"Applebot-Extended":["partial","*",29,18],"CCBot":["partial","*",29,18],"Bytespider":["partial","*",29,18],"Amazonbot":["partial","*",29,18],"meta-externalagent":["partial","*",29,18],"cohere-ai":["partial","*",29,18],"Diffbot":["partial","*",29,18],"ContextFetchBot":["partial","*",29,18]},"llms":{"status":"missing","http":404,"bytes":189},"home":{"status":"ok","http":200,"url":"https://www.ens-lyon.fr/","html_tokens":50368,"markdown_tokens":2264,"pruned_tokens":135}},
{"rank":5463,"domain":"39.net","origin":"https://39.net","robots":{"status":"html","http":200,"url":"https://www.39.net/web/404.html","bytes":19911,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":5464,"domain":"softwareadvice.com","origin":"https://softwareadvice.com","robots":{"status":"ok","http":200,"url":"https://www.softwareadvice.com/robots.txt","bytes":1713,"groups":4},"bots":{"GPTBot":["partial","*",24,2],"ChatGPT-User":["partial","*",24,2],"OAI-SearchBot":["partial","*",24,2],"ClaudeBot":["partial","*",24,2],"Claude-User":["partial","*",24,2],"Claude-SearchBot":["partial","*",24,2],"anthropic-ai":["partial","*",24,2],"PerplexityBot":["partial","*",24,2],"Perplexity-User":["partial","*",24,2],"Google-Extended":["partial","*",24,2],"Applebot-Extended":["partial","*",24,2],"CCBot":["partial","*",24,2],"Bytespider":["blocked","named",1,0],"Amazonbot":["partial","*",24,2],"meta-externalagent":["partial","*",24,2],"cohere-ai":["partial","*",24,2],"Diffbot":["partial","*",24,2],"ContextFetchBot":["partial","*",24,2]},"llms":{"status":"present","http":200,"bytes":1401},"home":{"status":"http_error","http":403,"url":"https://www.softwareadvice.com/"}},
{"rank":5465,"domain":"bandsintown.com","origin":"https://bandsintown.com","robots":{"status":"refused","http":403,"url":"https://www.bandsintown.com/robots.txt","bytes":4548,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":5466,"domain":"transistor.fm","origin":"https://transistor.fm","robots":{"status":"ok","http":200,"url":"https://transistor.fm/robots.txt","bytes":68,"groups":1},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"missing","http":404,"bytes":82775},"home":{"status":"ok","http":200,"url":"https://transistor.fm/","html_tokens":67269,"markdown_tokens":302,"pruned_tokens":17}},
{"rank":5467,"domain":"mykajabi.com","origin":"https://www.mykajabi.com","robots":{"status":"ok","http":200,"url":"https://www.mykajabi.com/robots.txt","bytes":121,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":9415},"home":{"status":"http_error","http":404,"url":"https://www.mykajabi.com/"}},
{"rank":5468,"domain":"africanews.com","origin":"https://africanews.com","robots":{"status":"ok","http":200,"url":"https://www.africanews.com/robots.txt","bytes":534,"groups":2},"bots":{"GPTBot":["partial","*",16,0],"ChatGPT-User":["partial","*",16,0],"OAI-SearchBot":["partial","*",16,0],"ClaudeBot":["partial","*",16,0],"Claude-User":["partial","*",16,0],"Claude-SearchBot":["partial","*",16,0],"anthropic-ai":["partial","*",16,0],"PerplexityBot":["partial","*",16,0],"Perplexity-User":["partial","*",16,0],"Google-Extended":["partial","*",16,0],"Applebot-Extended":["partial","*",16,0],"CCBot":["partial","*",16,0],"Bytespider":["partial","*",16,0],"Amazonbot":["partial","*",16,0],"meta-externalagent":["partial","*",16,0],"cohere-ai":["partial","*",16,0],"Diffbot":["partial","*",16,0],"ContextFetchBot":["partial","*",16,0]},"llms":{"status":"missing","http":404,"bytes":81878},"home":{"status":"ok","http":200,"url":"https://www.africanews.com/","html_tokens":64156,"markdown_tokens":27,"pruned_tokens":27}},
{"rank":5469,"domain":"thenews.com.pk","origin":"https://thenews.com.pk","robots":{"status":"ok","http":200,"url":"https://www.thenews.com.pk/robots.txt","bytes":227,"groups":1},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"missing","http":404,"bytes":1560},"home":{"status":"ok","http":200,"url":"https://www.thenews.com.pk/","html_tokens":21681,"markdown_tokens":31,"pruned_tokens":31}},
{"rank":5470,"domain":"daaz.com","origin":"https://daaz.com","robots":{"status":"ok","http":200,"url":"https://daaz.com/robots.txt","bytes":1348,"groups":16},"bots":{"GPTBot":["allowed","named",0,1],"ChatGPT-User":["allowed","named",0,1],"OAI-SearchBot":["partial","*",21,1],"ClaudeBot":["allowed","named",0,1],"Claude-User":["partial","*",21,1],"Claude-SearchBot":["partial","*",21,1],"anthropic-ai":["partial","*",21,1],"PerplexityBot":["allowed","named",0,1],"Perplexity-User":["partial","*",21,1],"Google-Extended":["allowed","named",0,1],"Applebot-Extended":["partial","*",21,1],"CCBot":["partial","*",21,1],"Bytespider":["partial","*",21,1],"Amazonbot":["partial","*",21,1],"meta-externalagent":["partial","*",21,1],"cohere-ai":["partial","*",21,1],"Diffbot":["partial","*",21,1],"ContextFetchBot":["partial","*",21,1]},"llms":{"status":"present","http":200,"bytes":5229},"home":{"status":"http_error","http":403,"url":"https://daaz.com/"}},
{"rank":5471,"domain":"cyol.com","origin":"https://cyol.com","robots":{"status":"missing","http":404,"url":"https://cyol.com/robots.txt","bytes":1754,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":1754},"home":{"status":"ok","http":200,"url":"https://cyol.com/","html_tokens":16214,"markdown_tokens":8,"pruned_tokens":8}},
{"rank":5472,"domain":"etoro.com","origin":"https://etoro.com","robots":{"status":"ok","http":200,"url":"https://www.etoro.com/robots.txt","bytes":2962,"groups":3},"bots":{"GPTBot":["partial","*",32,1],"ChatGPT-User":["partial","*",32,1],"OAI-SearchBot":["partial","*",32,1],"ClaudeBot":["partial","*",32,1],"Claude-User":["partial","*",32,1],"Claude-SearchBot":["partial","*",32,1],"anthropic-ai":["partial","*",32,1],"PerplexityBot":["partial","*",32,1],"Perplexity-User":["partial","*",32,1],"Google-Extended":["partial","*",32,1],"Applebot-Extended":["partial","*",32,1],"CCBot":["partial","*",32,1],"Bytespider":["partial","*",32,1],"Amazonbot":["partial","*",32,1],"meta-externalagent":["partial","*",32,1],"cohere-ai":["partial","*",32,1],"Diffbot":["partial","*",32,1],"ContextFetchBot":["partial","*",32,1]},"llms":{"status":"missing","http":404,"bytes":1245},"home":{"status":"ok","http":200,"url":"https://www.etoro.com/","html_tokens":102601,"markdown_tokens":189,"pruned_tokens":135}},
{"rank":5473,"domain":"index.hu","origin":"https://index.hu","robots":{"status":"ok","http":200,"url":"https://index.hu/robots.txt","bytes":570,"groups":12},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["blocked","named",1,0],"OAI-SearchBot":["partial","*",6,0],"ClaudeBot":["partial","*",6,0],"Claude-User":["partial","*",6,0],"Claude-SearchBot":["partial","*",6,0],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["partial","*",6,0],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["partial","*",6,0],"meta-externalagent":["partial","*",6,0],"cohere-ai":["partial","*",6,0],"Diffbot":["partial","*",6,0],"ContextFetchBot":["partial","*",6,0]},"llms":{"status":"missing","http":404,"bytes":63749},"home":{"status":"ok","http":200,"url":"https://index.hu/","html_tokens":143367,"markdown_tokens":2470,"pruned_tokens":173}},
{"rank":5474,"domain":"ohsu.edu","origin":"https://ohsu.edu","robots":{"status":"ok","http":200,"url":"https://www.ohsu.edu/robots.txt","bytes":2995,"groups":1},"bots":{"GPTBot":["partial","*",55,22],"ChatGPT-User":["partial","*",55,22],"OAI-SearchBot":["partial","*",55,22],"ClaudeBot":["partial","*",55,22],"Claude-User":["partial","*",55,22],"Claude-SearchBot":["partial","*",55,22],"anthropic-ai":["partial","*",55,22],"PerplexityBot":["partial","*",55,22],"Perplexity-User":["partial","*",55,22],"Google-Extended":["partial","*",55,22],"Applebot-Extended":["partial","*",55,22],"CCBot":["partial","*",55,22],"Bytespider":["partial","*",55,22],"Amazonbot":["partial","*",55,22],"meta-externalagent":["partial","*",55,22],"cohere-ai":["partial","*",55,22],"Diffbot":["partial","*",55,22],"ContextFetchBot":["partial","*",55,22]},"llms":{"status":"missing","http":404,"bytes":291},"home":{"status":"ok","http":200,"url":"https://www.ohsu.edu/","html_tokens":13980,"markdown_tokens":55,"pruned_tokens":55}},
{"rank":5475,"domain":"netbsd.org","origin":"https://netbsd.org","robots":{"status":"ok","http":200,"url":"https://netbsd.org/robots.txt","bytes":186,"groups":1},"bots":{"GPTBot":["partial","*",7,0],"ChatGPT-User":["partial","*",7,0],"OAI-SearchBot":["partial","*",7,0],"ClaudeBot":["partial","*",7,0],"Claude-User":["partial","*",7,0],"Claude-SearchBot":["partial","*",7,0],"anthropic-ai":["partial","*",7,0],"PerplexityBot":["partial","*",7,0],"Perplexity-User":["partial","*",7,0],"Google-Extended":["partial","*",7,0],"Applebot-Extended":["partial","*",7,0],"CCBot":["partial","*",7,0],"Bytespider":["partial","*",7,0],"Amazonbot":["partial","*",7,0],"meta-externalagent":["partial","*",7,0],"cohere-ai":["partial","*",7,0],"Diffbot":["partial","*",7,0],"ContextFetchBot":["partial","*",7,0]},"llms":{"status":"missing","http":404,"bytes":6665},"home":{"status":"ok","http":200,"url":"https://netbsd.org/","html_tokens":3160,"markdown_tokens":433,"pruned_tokens":111}},
{"rank":5476,"domain":"dataprotection.ie","origin":"https://dataprotection.ie","robots":{"status":"ok","http":200,"url":"https://dataprotection.ie/robots.txt","bytes":2027,"groups":1},"bots":{"GPTBot":["partial","*",34,18],"ChatGPT-User":["partial","*",34,18],"OAI-SearchBot":["partial","*",34,18],"ClaudeBot":["partial","*",34,18],"Claude-User":["partial","*",34,18],"Claude-SearchBot":["partial","*",34,18],"anthropic-ai":["partial","*",34,18],"PerplexityBot":["partial","*",34,18],"Perplexity-User":["partial","*",34,18],"Google-Extended":["partial","*",34,18],"Applebot-Extended":["partial","*",34,18],"CCBot":["partial","*",34,18],"Bytespider":["partial","*",34,18],"Amazonbot":["partial","*",34,18],"meta-externalagent":["partial","*",34,18],"cohere-ai":["partial","*",34,18],"Diffbot":["partial","*",34,18],"ContextFetchBot":["partial","*",34,18]},"llms":{"status":"missing","http":404,"bytes":191},"home":{"status":"ok","http":200,"url":"https://dataprotection.ie/en","html_tokens":9700,"markdown_tokens":127,"pruned_tokens":127}},
{"rank":5477,"domain":"omicsonline.org","origin":"https://www.omicsonline.org","robots":{"status":"ok","http":200,"url":"https://www.omicsonline.org/robots.txt","bytes":5998,"groups":3},"bots":{"GPTBot":["allowed","*",0,1],"ChatGPT-User":["allowed","*",0,1],"OAI-SearchBot":["allowed","*",0,1],"ClaudeBot":["allowed","*",0,1],"Claude-User":["allowed","*",0,1],"Claude-SearchBot":["allowed","*",0,1],"anthropic-ai":["allowed","*",0,1],"PerplexityBot":["allowed","*",0,1],"Perplexity-User":["allowed","*",0,1],"Google-Extended":["allowed","*",0,1],"Applebot-Extended":["allowed","*",0,1],"CCBot":["allowed","*",0,1],"Bytespider":["allowed","*",0,1],"Amazonbot":["allowed","*",0,1],"meta-externalagent":["allowed","*",0,1],"cohere-ai":["allowed","*",0,1],"Diffbot":["allowed","*",0,1],"ContextFetchBot":["allowed","*",0,1]},"llms":{"status":"timeout","error":"timed out after 10 s"},"home":{"status":"ok","http":200,"url":"https://www.omicsonline.org/","html_tokens":30584,"markdown_tokens":492,"pruned_tokens":78}},
{"rank":5478,"domain":"sciam.com","origin":"https://sciam.com","robots":{"status":"ok","http":200,"url":"https://www.scientificamerican.com/robots.txt","bytes":642,"groups":3},"bots":{"GPTBot":["partial","named",1,2],"ChatGPT-User":["partial","named",1,2],"OAI-SearchBot":["partial","named",1,2],"ClaudeBot":["partial","named",1,2],"Claude-User":["partial","named",1,2],"Claude-SearchBot":["partial","named",1,2],"anthropic-ai":["partial","named",1,2],"PerplexityBot":["partial","named",1,2],"Perplexity-User":["partial","*",5,0],"Google-Extended":["partial","named",1,2],"Applebot-Extended":["partial","*",5,0],"CCBot":["partial","named",1,2],"Bytespider":["partial","*",5,0],"Amazonbot":["partial","*",5,0],"meta-externalagent":["partial","*",5,0],"cohere-ai":["partial","named",1,2],"Diffbot":["partial","*",5,0],"ContextFetchBot":["partial","*",5,0]},"llms":{"status":"missing","http":404,"bytes":89542},"home":{"status":"ok","http":200,"url":"https://www.scientificamerican.com/","html_tokens":88370,"markdown_tokens":1050,"pruned_tokens":171}},
{"rank":5479,"domain":"grubhub.com","origin":"https://grubhub.com","robots":{"status":"ok","http":200,"url":"https://www.grubhub.com/robots.txt","bytes":463,"groups":1},"bots":{"GPTBot":["partial","*",17,0],"ChatGPT-User":["partial","*",17,0],"OAI-SearchBot":["partial","*",17,0],"ClaudeBot":["partial","*",17,0],"Claude-User":["partial","*",17,0],"Claude-SearchBot":["partial","*",17,0],"anthropic-ai":["partial","*",17,0],"PerplexityBot":["partial","*",17,0],"Perplexity-User":["partial","*",17,0],"Google-Extended":["partial","*",17,0],"Applebot-Extended":["partial","*",17,0],"CCBot":["partial","*",17,0],"Bytespider":["partial","*",17,0],"Amazonbot":["partial","*",17,0],"meta-externalagent":["partial","*",17,0],"cohere-ai":["partial","*",17,0],"Diffbot":["partial","*",17,0],"ContextFetchBot":["partial","*",17,0]},"llms":{"status":"html","http":200,"bytes":14626},"home":{"status":"ok","http":200,"url":"https://www.grubhub.com/","html_tokens":3657,"markdown_tokens":39,"pruned_tokens":39}},
{"rank":5480,"domain":"swarovski.com","origin":"https://swarovski.com","robots":{"status":"ok","http":200,"url":"https://swarovski.com/robots.txt","bytes":3575,"groups":18},"bots":{"GPTBot":["partial","named",7,1],"ChatGPT-User":["partial","named",7,1],"OAI-SearchBot":["partial","named",7,1],"ClaudeBot":["partial","named",7,1],"Claude-User":["partial","*",7,0],"Claude-SearchBot":["partial","named",7,1],"anthropic-ai":["partial","*",7,0],"PerplexityBot":["partial","named",7,1],"Perplexity-User":["partial","named",7,1],"Google-Extended":["partial","named",7,1],"Applebot-Extended":["partial","*",7,0],"CCBot":["partial","named",7,1],"Bytespider":["partial","*",7,0],"Amazonbot":["partial","*",7,0],"meta-externalagent":["partial","*",7,0],"cohere-ai":["partial","*",7,0],"Diffbot":["partial","*",7,0],"ContextFetchBot":["partial","*",7,0]},"llms":{"status":"present","http":200,"bytes":2642},"home":{"status":"disallowed","error":"robots.txt of www.swarovski.com not readable (html)"}},
{"rank":5481,"domain":"qrco.de","origin":"https://qrco.de","robots":{"status":"ok","http":200,"url":"https://qrco.de/robots.txt","bytes":26,"groups":1},"bots":{"GPTBot":["blocked","*",1,0],"ChatGPT-User":["blocked","*",1,0],"OAI-SearchBot":["blocked","*",1,0],"ClaudeBot":["blocked","*",1,0],"Claude-User":["blocked","*",1,0],"Claude-SearchBot":["blocked","*",1,0],"anthropic-ai":["blocked","*",1,0],"PerplexityBot":["blocked","*",1,0],"Perplexity-User":["blocked","*",1,0],"Google-Extended":["blocked","*",1,0],"Applebot-Extended":["blocked","*",1,0],"CCBot":["blocked","*",1,0],"Bytespider":["blocked","*",1,0],"Amazonbot":["blocked","*",1,0],"meta-externalagent":["blocked","*",1,0],"cohere-ai":["blocked","*",1,0],"Diffbot":["blocked","*",1,0],"ContextFetchBot":["blocked","*",1,0]},"llms":{"status":"disallowed"},"home":{"status":"disallowed"}},
{"rank":5482,"domain":"pasteur.fr","origin":"https://pasteur.fr","robots":{"status":"ok","http":200,"url":"https://www.pasteur.fr/robots.txt","bytes":2162,"groups":1},"bots":{"GPTBot":["partial","*",37,18],"ChatGPT-User":["partial","*",37,18],"OAI-SearchBot":["partial","*",37,18],"ClaudeBot":["partial","*",37,18],"Claude-User":["partial","*",37,18],"Claude-SearchBot":["partial","*",37,18],"anthropic-ai":["partial","*",37,18],"PerplexityBot":["partial","*",37,18],"Perplexity-User":["partial","*",37,18],"Google-Extended":["partial","*",37,18],"Applebot-Extended":["partial","*",37,18],"CCBot":["partial","*",37,18],"Bytespider":["partial","*",37,18],"Amazonbot":["partial","*",37,18],"meta-externalagent":["partial","*",37,18],"cohere-ai":["partial","*",37,18],"Diffbot":["partial","*",37,18],"ContextFetchBot":["partial","*",37,18]},"llms":{"status":"missing","http":404,"bytes":188},"home":{"status":"ok","http":200,"url":"https://www.pasteur.fr/fr","html_tokens":33599,"markdown_tokens":158,"pruned_tokens":158}},
{"rank":5483,"domain":"riotgames.com","origin":"https://riotgames.com","robots":{"status":"ok","http":200,"url":"https://www.riotgames.com/robots.txt","bytes":34,"groups":1},"bots":{"GPTBot":["partial","*",1,0],"ChatGPT-User":["partial","*",1,0],"OAI-SearchBot":["partial","*",1,0],"ClaudeBot":["partial","*",1,0],"Claude-User":["partial","*",1,0],"Claude-SearchBot":["partial","*",1,0],"anthropic-ai":["partial","*",1,0],"PerplexityBot":["partial","*",1,0],"Perplexity-User":["partial","*",1,0],"Google-Extended":["partial","*",1,0],"Applebot-Extended":["partial","*",1,0],"CCBot":["partial","*",1,0],"Bytespider":["partial","*",1,0],"Amazonbot":["partial","*",1,0],"meta-externalagent":["partial","*",1,0],"cohere-ai":["partial","*",1,0],"Diffbot":["partial","*",1,0],"ContextFetchBot":["partial","*",1,0]},"llms":{"status":"html","http":200,"bytes":223283},"home":{"status":"ok","http":200,"url":"https://www.riotgames.com/en","html_tokens":55815,"markdown_tokens":74,"pruned_tokens":74}},
{"rank":5484,"domain":"deepmind.com","origin":"https://deepmind.com","robots":{"status":"ok","http":200,"url":"https://deepmind.google/robots.txt","bytes":69,"groups":1},"bots":{"GPTBot":["allowed","*",0,1],"ChatGPT-User":["allowed","*",0,1],"OAI-SearchBot":["allowed","*",0,1],"ClaudeBot":["allowed","*",0,1],"Claude-User":["allowed","*",0,1],"Claude-SearchBot":["allowed","*",0,1],"anthropic-ai":["allowed","*",0,1],"PerplexityBot":["allowed","*",0,1],"Perplexity-User":["allowed","*",0,1],"Google-Extended":["allowed","*",0,1],"Applebot-Extended":["allowed","*",0,1],"CCBot":["allowed","*",0,1],"Bytespider":["allowed","*",0,1],"Amazonbot":["allowed","*",0,1],"meta-externalagent":["allowed","*",0,1],"cohere-ai":["allowed","*",0,1],"Diffbot":["allowed","*",0,1],"ContextFetchBot":["allowed","*",0,1]},"llms":{"status":"present","http":200,"bytes":6928},"home":{"status":"ok","http":200,"url":"https://deepmind.google/","html_tokens":66637,"markdown_tokens":227,"pruned_tokens":146}},
{"rank":5485,"domain":"net-a-porter.com","origin":"https://net-a-porter.com","robots":{"status":"refused","http":403,"url":"https://net-a-porter.com/robots.txt","bytes":386,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":5486,"domain":"zerodepositcasino.co.uk","origin":"https://zerodepositcasino.co.uk","robots":{"status":"ok","http":200,"url":"https://zerodepositcasino.co.uk/robots.txt","bytes":551,"groups":1},"bots":{"GPTBot":["partial","*",19,0],"ChatGPT-User":["partial","*",19,0],"OAI-SearchBot":["partial","*",19,0],"ClaudeBot":["partial","*",19,0],"Claude-User":["partial","*",19,0],"Claude-SearchBot":["partial","*",19,0],"anthropic-ai":["partial","*",19,0],"PerplexityBot":["partial","*",19,0],"Perplexity-User":["partial","*",19,0],"Google-Extended":["partial","*",19,0],"Applebot-Extended":["partial","*",19,0],"CCBot":["partial","*",19,0],"Bytespider":["partial","*",19,0],"Amazonbot":["partial","*",19,0],"meta-externalagent":["partial","*",19,0],"cohere-ai":["partial","*",19,0],"Diffbot":["partial","*",19,0],"ContextFetchBot":["partial","*",19,0]},"llms":{"status":"missing","http":404,"bytes":32298},"home":{"status":"ok","http":200,"url":"https://zerodepositcasino.co.uk/","html_tokens":27758,"markdown_tokens":3972,"pruned_tokens":105}},
{"rank":5487,"domain":"page.tl","origin":"https://page.tl","robots":{"status":"ok","http":200,"url":"https://www.own-free-website.com/robots.txt","bytes":78,"groups":1},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"missing","http":404,"bytes":206},"home":{"status":"ok","http":200,"url":"https://www.own-free-website.com/","html_tokens":6166,"markdown_tokens":271,"pruned_tokens":170}},
{"rank":5488,"domain":"meteofrance.com","origin":"https://meteofrance.com","robots":{"status":"ok","http":200,"url":"https://meteofrance.com/robots.txt","bytes":2027,"groups":1},"bots":{"GPTBot":["partial","*",34,18],"ChatGPT-User":["partial","*",34,18],"OAI-SearchBot":["partial","*",34,18],"ClaudeBot":["partial","*",34,18],"Claude-User":["partial","*",34,18],"Claude-SearchBot":["partial","*",34,18],"anthropic-ai":["partial","*",34,18],"PerplexityBot":["partial","*",34,18],"Perplexity-User":["partial","*",34,18],"Google-Extended":["partial","*",34,18],"Applebot-Extended":["partial","*",34,18],"CCBot":["partial","*",34,18],"Bytespider":["partial","*",34,18],"Amazonbot":["partial","*",34,18],"meta-externalagent":["partial","*",34,18],"cohere-ai":["partial","*",34,18],"Diffbot":["partial","*",34,18],"ContextFetchBot":["partial","*",34,18]},"llms":{"status":"http_error","http":403,"bytes":548},"home":{"status":"ok","http":200,"url":"https://meteofrance.com/","html_tokens":63884,"markdown_tokens":21,"pruned_tokens":21}},
{"rank":5489,"domain":"hankyung.com","origin":"https://hankyung.com","robots":{"status":"ok","http":200,"url":"https://www.hankyung.com/robots.txt","bytes":634,"groups":6},"bots":{"GPTBot":["partial","*",15,0],"ChatGPT-User":["partial","*",15,0],"OAI-SearchBot":["partial","*",15,0],"ClaudeBot":["partial","*",15,0],"Claude-User":["partial","*",15,0],"Claude-SearchBot":["partial","*",15,0],"anthropic-ai":["partial","*",15,0],"PerplexityBot":["partial","*",15,0],"Perplexity-User":["partial","*",15,0],"Google-Extended":["partial","*",15,0],"Applebot-Extended":["partial","*",15,0],"CCBot":["partial","*",15,0],"Bytespider":["partial","*",15,0],"Amazonbot":["partial","*",15,0],"meta-externalagent":["partial","*",15,0],"cohere-ai":["partial","*",15,0],"Diffbot":["partial","*",15,0],"ContextFetchBot":["partial","*",15,0]},"llms":{"status":"missing","http":404,"bytes":15046},"home":{"status":"ok","http":200,"url":"https://www.hankyung.com/","html_tokens":40310,"markdown_tokens":144,"pruned_tokens":144}},
{"rank":5490,"domain":"wz.cz","origin":"http://wz.cz","robots":{"status":"html","http":200,"url":"https://www.webzdarma.cz/","bytes":59469,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":5491,"domain":"uhc.com","origin":"https://uhc.com","robots":{"status":"refused","http":403,"url":"https://www.uhc.com/robots.txt","bytes":919,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":5492,"domain":"thomann.de","origin":"https://thomann.de","robots":{"status":"ok","http":200,"url":"https://www.thomann.de/robots.txt","bytes":817,"groups":4},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["partial","*",18,1],"OAI-SearchBot":["partial","*",18,1],"ClaudeBot":["partial","*",18,1],"Claude-User":["partial","*",18,1],"Claude-SearchBot":["partial","*",18,1],"anthropic-ai":["partial","*",18,1],"PerplexityBot":["partial","*",18,1],"Perplexity-User":["partial","*",18,1],"Google-Extended":["partial","*",18,1],"Applebot-Extended":["partial","*",18,1],"CCBot":["partial","*",18,1],"Bytespider":["partial","*",18,1],"Amazonbot":["partial","*",18,1],"meta-externalagent":["partial","*",18,1],"cohere-ai":["partial","*",18,1],"Diffbot":["partial","*",18,1],"ContextFetchBot":["partial","*",18,1]},"llms":{"status":"missing","http":404,"bytes":757},"home":{"status":"http_error","http":429,"url":"https://www.thomann.de/"}},
{"rank":5493,"domain":"fec.gov","origin":"https://fec.gov","robots":{"status":"html","http":200,"url":"https://www.fec.gov","bytes":39553,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":5494,"domain":"aftenposten.no","origin":"https://aftenposten.no","robots":{"status":"ok","http":200,"url":"https://aftenposten.no/robots.txt","bytes":2028,"groups":2},"bots":{"GPTBot":["partial","*",18,1],"ChatGPT-User":["partial","*",18,1],"OAI-SearchBot":["partial","named",18,1],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",18,1],"Claude-SearchBot":["blocked","named",1,0],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["partial","*",18,1],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",18,1]},"llms":{"status":"missing","http":404,"bytes":2988},"home":{"status":"ok","http":200,"url":"https://www.aftenposten.no/","html_tokens":91237,"markdown_tokens":35,"pruned_tokens":30}},
{"rank":5495,"domain":"afdb.org","origin":"https://afdb.org","robots":{"status":"refused","http":403,"url":"https://www.afdb.org/","bytes":5499,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":5496,"domain":"opencv.org","origin":"https://opencv.org","robots":{"status":"ok","http":200,"url":"https://opencv.org/robots.txt","bytes":171,"groups":1},"bots":{"GPTBot":["partial","*",4,1],"ChatGPT-User":["partial","*",4,1],"OAI-SearchBot":["partial","*",4,1],"ClaudeBot":["partial","*",4,1],"Claude-User":["partial","*",4,1],"Claude-SearchBot":["partial","*",4,1],"anthropic-ai":["partial","*",4,1],"PerplexityBot":["partial","*",4,1],"Perplexity-User":["partial","*",4,1],"Google-Extended":["partial","*",4,1],"Applebot-Extended":["partial","*",4,1],"CCBot":["partial","*",4,1],"Bytespider":["partial","*",4,1],"Amazonbot":["partial","*",4,1],"meta-externalagent":["partial","*",4,1],"cohere-ai":["partial","*",4,1],"Diffbot":["partial","*",4,1],"ContextFetchBot":["partial","*",4,1]},"llms":{"status":"missing","http":404,"bytes":104523},"home":{"status":"http_error","http":403,"url":"https://opencv.org/"}},
{"rank":5497,"domain":"fast.com","origin":"https://fast.com","robots":{"status":"ok","http":200,"url":"https://fast.com/robots.txt","bytes":41,"groups":1},"bots":{"GPTBot":["partial","*",1,1],"ChatGPT-User":["partial","*",1,1],"OAI-SearchBot":["partial","*",1,1],"ClaudeBot":["partial","*",1,1],"Claude-User":["partial","*",1,1],"Claude-SearchBot":["partial","*",1,1],"anthropic-ai":["partial","*",1,1],"PerplexityBot":["partial","*",1,1],"Perplexity-User":["partial","*",1,1],"Google-Extended":["partial","*",1,1],"Applebot-Extended":["partial","*",1,1],"CCBot":["partial","*",1,1],"Bytespider":["partial","*",1,1],"Amazonbot":["partial","*",1,1],"meta-externalagent":["partial","*",1,1],"cohere-ai":["partial","*",1,1],"Diffbot":["partial","*",1,1],"ContextFetchBot":["partial","*",1,1]},"llms":{"status":"html","http":200,"bytes":4006},"home":{"status":"ok","http":200,"url":"https://fast.com/","html_tokens":6561,"markdown_tokens":204,"pruned_tokens":169}},
{"rank":5498,"domain":"umb.edu","origin":"https://umb.edu","robots":{"status":"ok","http":200,"url":"https://www.umb.edu/robots.txt","bytes":21948,"groups":2},"bots":{"GPTBot":["partial","*",368,0],"ChatGPT-User":["partial","*",368,0],"OAI-SearchBot":["partial","*",368,0],"ClaudeBot":["partial","*",368,0],"Claude-User":["partial","*",368,0],"Claude-SearchBot":["partial","*",368,0],"anthropic-ai":["partial","*",368,0],"PerplexityBot":["partial","*",368,0],"Perplexity-User":["partial","*",368,0],"Google-Extended":["partial","*",368,0],"Applebot-Extended":["partial","*",368,0],"CCBot":["partial","*",368,0],"Bytespider":["partial","*",368,0],"Amazonbot":["partial","*",368,0],"meta-externalagent":["partial","*",368,0],"cohere-ai":["partial","*",368,0],"Diffbot":["partial","*",368,0],"ContextFetchBot":["partial","*",368,0]},"llms":{"status":"missing","http":404,"bytes":29445},"home":{"status":"ok","http":200,"url":"https://www.umb.edu/","html_tokens":17731,"markdown_tokens":161,"pruned_tokens":161}},
{"rank":5499,"domain":"whyy.org","origin":"https://whyy.org","robots":{"status":"refused","http":403,"url":"https://whyy.org/robots.txt","bytes":560,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":5500,"domain":"stocktwits.com","origin":"https://stocktwits.com","robots":{"status":"ok","http":200,"url":"https://stocktwits.com/robots.txt","bytes":2202,"groups":3},"bots":{"GPTBot":["partial","*",15,2],"ChatGPT-User":["partial","*",15,2],"OAI-SearchBot":["partial","*",15,2],"ClaudeBot":["partial","*",15,2],"Claude-User":["partial","*",15,2],"Claude-SearchBot":["partial","*",15,2],"anthropic-ai":["partial","*",15,2],"PerplexityBot":["partial","*",15,2],"Perplexity-User":["partial","*",15,2],"Google-Extended":["partial","*",15,2],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["partial","*",15,2],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",15,2]},"llms":{"status":"missing","http":404,"bytes":50675},"home":{"status":"ok","http":200,"url":"https://stocktwits.com/","html_tokens":113661,"markdown_tokens":77,"pruned_tokens":77}},
{"rank":5501,"domain":"globalcitizen.org","origin":"https://globalcitizen.org","robots":{"status":"ok","http":200,"url":"https://www.globalcitizen.org/robots.txt","bytes":187,"groups":1},"bots":{"GPTBot":["partial","*",4,0],"ChatGPT-User":["partial","*",4,0],"OAI-SearchBot":["partial","*",4,0],"ClaudeBot":["partial","*",4,0],"Claude-User":["partial","*",4,0],"Claude-SearchBot":["partial","*",4,0],"anthropic-ai":["partial","*",4,0],"PerplexityBot":["partial","*",4,0],"Perplexity-User":["partial","*",4,0],"Google-Extended":["partial","*",4,0],"Applebot-Extended":["partial","*",4,0],"CCBot":["partial","*",4,0],"Bytespider":["partial","*",4,0],"Amazonbot":["partial","*",4,0],"meta-externalagent":["partial","*",4,0],"cohere-ai":["partial","*",4,0],"Diffbot":["partial","*",4,0],"ContextFetchBot":["partial","*",4,0]},"llms":{"status":"missing","http":404,"bytes":91380},"home":{"status":"ok","http":200,"url":"https://www.globalcitizen.org/en/","html_tokens":49715,"markdown_tokens":99,"pruned_tokens":99}},
{"rank":5502,"domain":"chainalysis.com","origin":"https://chainalysis.com","robots":{"status":"ok","http":200,"url":"https://www.chainalysis.com/robots.txt","bytes":393,"groups":2},"bots":{"GPTBot":["partial","*",2,0],"ChatGPT-User":["partial","*",2,0],"OAI-SearchBot":["partial","*",2,0],"ClaudeBot":["partial","*",2,0],"Claude-User":["partial","*",2,0],"Claude-SearchBot":["partial","*",2,0],"anthropic-ai":["partial","*",2,0],"PerplexityBot":["partial","*",2,0],"Perplexity-User":["partial","*",2,0],"Google-Extended":["partial","*",2,0],"Applebot-Extended":["partial","*",2,0],"CCBot":["partial","*",2,0],"Bytespider":["partial","*",2,0],"Amazonbot":["partial","*",2,0],"meta-externalagent":["partial","*",2,0],"cohere-ai":["partial","*",2,0],"Diffbot":["partial","*",2,0],"ContextFetchBot":["partial","*",2,0]},"llms":{"status":"other_text","http":200,"bytes":563074},"home":{"status":"ok","http":200,"url":"https://www.chainalysis.com/","html_tokens":114989,"markdown_tokens":78,"pruned_tokens":53}},
{"rank":5503,"domain":"neu.edu","origin":"https://neu.edu","robots":{"status":"ok","http":200,"url":"https://www.northeastern.edu/robots.txt","bytes":366,"groups":2},"bots":{"GPTBot":["partial","*",1,0],"ChatGPT-User":["partial","*",1,0],"OAI-SearchBot":["partial","*",1,0],"ClaudeBot":["partial","*",1,0],"Claude-User":["partial","*",1,0],"Claude-SearchBot":["partial","*",1,0],"anthropic-ai":["partial","*",1,0],"PerplexityBot":["partial","*",1,0],"Perplexity-User":["partial","*",1,0],"Google-Extended":["partial","*",1,0],"Applebot-Extended":["partial","*",1,0],"CCBot":["partial","*",1,0],"Bytespider":["partial","*",1,0],"Amazonbot":["partial","*",1,0],"meta-externalagent":["partial","*",1,0],"cohere-ai":["partial","*",1,0],"Diffbot":["partial","*",1,0],"ContextFetchBot":["partial","*",1,0]},"llms":{"status":"missing","http":404,"bytes":60},"home":{"status":"ok","http":200,"url":"https://www.northeastern.edu/","html_tokens":63174,"markdown_tokens":52,"pruned_tokens":52}},
{"rank":5504,"domain":"lancs.ac.uk","origin":"https://lancs.ac.uk","robots":{"status":"ok","http":200,"url":"https://www.lancaster.ac.uk/robots.txt","bytes":2283,"groups":2},"bots":{"GPTBot":["partial","*",54,6],"ChatGPT-User":["partial","*",54,6],"OAI-SearchBot":["partial","*",54,6],"ClaudeBot":["partial","*",54,6],"Claude-User":["partial","*",54,6],"Claude-SearchBot":["partial","*",54,6],"anthropic-ai":["partial","*",54,6],"PerplexityBot":["partial","*",54,6],"Perplexity-User":["partial","*",54,6],"Google-Extended":["partial","*",54,6],"Applebot-Extended":["partial","*",54,6],"CCBot":["partial","*",54,6],"Bytespider":["partial","*",54,6],"Amazonbot":["partial","*",54,6],"meta-externalagent":["partial","*",54,6],"cohere-ai":["partial","*",54,6],"Diffbot":["partial","*",54,6],"ContextFetchBot":["partial","*",54,6]},"llms":{"status":"missing","http":404,"bytes":82963},"home":{"status":"ok","http":200,"url":"https://www.lancaster.ac.uk/","html_tokens":27492,"markdown_tokens":157,"pruned_tokens":81}},
{"rank":5505,"domain":"sz.gov.cn","origin":"https://sz.gov.cn","robots":{"status":"missing","http":404,"url":"https://sz.gov.cn/robots.txt","bytes":146,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":146},"home":{"status":"http_error","http":404,"url":"https://sz.gov.cn/"}},
{"rank":5506,"domain":"shape.com","origin":"https://shape.com","robots":{"status":"ok","http":200,"url":"https://www.shape.com/robots.txt","bytes":3463,"groups":6},"bots":{"GPTBot":["partial","named",1,0],"ChatGPT-User":["partial","named",1,0],"OAI-SearchBot":["partial","named",1,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["blocked","named",1,0],"Claude-SearchBot":["partial","named",1,2],"anthropic-ai":["partial","named",1,2],"PerplexityBot":["partial","named",1,2],"Perplexity-User":["blocked","named",1,0],"Google-Extended":["partial","named",1,2],"Applebot-Extended":["blocked","named",1,0],"CCBot":["partial","named",1,2],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["partial","named",1,2],"cohere-ai":["partial","named",1,2],"Diffbot":["partial","*",2,0],"ContextFetchBot":["partial","*",2,0]},"llms":{"status":"http_error","http":402,"bytes":612},"home":{"status":"http_error","http":402,"url":"https://www.shape.com/"}},
{"rank":5507,"domain":"qstheory.cn","origin":"https://www.qstheory.cn","robots":{"status":"missing","http":404,"url":"https://www.qstheory.cn/robots.txt","bytes":146,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":146},"home":{"status":"ok","http":200,"url":"https://www.qstheory.cn/","html_tokens":29310,"markdown_tokens":122,"pruned_tokens":122}},
{"rank":5508,"domain":"nova.edu","origin":"https://nova.edu","robots":{"status":"ok","http":200,"url":"https://nova.edu/robots.txt","bytes":1389,"groups":1},"bots":{"GPTBot":["partial","*",49,4],"ChatGPT-User":["partial","*",49,4],"OAI-SearchBot":["partial","*",49,4],"ClaudeBot":["partial","*",49,4],"Claude-User":["partial","*",49,4],"Claude-SearchBot":["partial","*",49,4],"anthropic-ai":["partial","*",49,4],"PerplexityBot":["partial","*",49,4],"Perplexity-User":["partial","*",49,4],"Google-Extended":["partial","*",49,4],"Applebot-Extended":["partial","*",49,4],"CCBot":["partial","*",49,4],"Bytespider":["partial","*",49,4],"Amazonbot":["partial","*",49,4],"meta-externalagent":["partial","*",49,4],"cohere-ai":["partial","*",49,4],"Diffbot":["partial","*",49,4],"ContextFetchBot":["partial","*",49,4]},"llms":{"status":"missing","http":404,"bytes":62650},"home":{"status":"ok","http":200,"url":"https://nova.edu/","html_tokens":27090,"markdown_tokens":1055,"pruned_tokens":118}},
{"rank":5509,"domain":"spectrumlocalnews.com","origin":"https://spectrumlocalnews.com","robots":{"status":"ok","http":200,"url":"https://spectrumlocalnews.com/robots.txt","bytes":1570,"groups":24},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["partial","*",22,1],"OAI-SearchBot":["partial","*",22,1],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",22,1],"Claude-SearchBot":["partial","*",22,1],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["partial","*",22,1],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",22,1]},"llms":{"status":"missing","http":404,"bytes":34823},"home":{"status":"ok","http":200,"url":"https://spectrumlocalnews.com/","html_tokens":2988,"markdown_tokens":0,"pruned_tokens":0}},
{"rank":5510,"domain":"t3n.de","origin":"https://t3n.de","robots":{"status":"ok","http":200,"url":"https://t3n.de/robots.txt","bytes":14886,"groups":334},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["partial","named",5,1],"OAI-SearchBot":["partial","*",5,1],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","named",5,1],"Claude-SearchBot":["partial","*",5,1],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["partial","named",5,1],"Perplexity-User":["partial","*",5,1],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",5,1]},"llms":{"status":"missing","http":404,"bytes":2936},"home":{"status":"ok","http":200,"url":"https://t3n.de/","html_tokens":81348,"markdown_tokens":24,"pruned_tokens":24}},
{"rank":5511,"domain":"sonos.com","origin":"https://sonos.com","robots":{"status":"ok","http":200,"url":"https://www.sonos.com/robots.txt","bytes":415,"groups":2},"bots":{"GPTBot":["partial","*",9,0],"ChatGPT-User":["partial","*",9,0],"OAI-SearchBot":["partial","*",9,0],"ClaudeBot":["partial","*",9,0],"Claude-User":["partial","*",9,0],"Claude-SearchBot":["partial","*",9,0],"anthropic-ai":["partial","*",9,0],"PerplexityBot":["partial","*",9,0],"Perplexity-User":["partial","*",9,0],"Google-Extended":["partial","*",9,0],"Applebot-Extended":["partial","*",9,0],"CCBot":["partial","*",9,0],"Bytespider":["partial","*",9,0],"Amazonbot":["partial","*",9,0],"meta-externalagent":["partial","*",9,0],"cohere-ai":["partial","*",9,0],"Diffbot":["partial","*",9,0],"ContextFetchBot":["partial","*",9,0]},"llms":{"status":"missing","http":404,"bytes":5467},"home":{"status":"ok","http":200,"url":"https://www.sonos.com/en-us/home","html_tokens":294807,"markdown_tokens":19,"pruned_tokens":19}},
{"rank":5512,"domain":"speedrun.com","origin":"https://speedrun.com","robots":{"status":"ok","http":200,"url":"https://www.speedrun.com/robots.txt","bytes":30,"groups":1},"bots":{"GPTBot":["partial","*",1,0],"ChatGPT-User":["partial","*",1,0],"OAI-SearchBot":["partial","*",1,0],"ClaudeBot":["partial","*",1,0],"Claude-User":["partial","*",1,0],"Claude-SearchBot":["partial","*",1,0],"anthropic-ai":["partial","*",1,0],"PerplexityBot":["partial","*",1,0],"Perplexity-User":["partial","*",1,0],"Google-Extended":["partial","*",1,0],"Applebot-Extended":["partial","*",1,0],"CCBot":["partial","*",1,0],"Bytespider":["partial","*",1,0],"Amazonbot":["partial","*",1,0],"meta-externalagent":["partial","*",1,0],"cohere-ai":["partial","*",1,0],"Diffbot":["partial","*",1,0],"ContextFetchBot":["partial","*",1,0]},"llms":{"status":"missing","http":404,"bytes":53484},"home":{"status":"http_error","http":403,"url":"https://www.speedrun.com/"}},
{"rank":5513,"domain":"distrowatch.com","origin":"https://distrowatch.com","robots":{"status":"ok","http":200,"url":"https://distrowatch.com/robots.txt","bytes":3883,"groups":103},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["partial","*",4,0],"OAI-SearchBot":["blocked","named",1,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",4,0],"Claude-SearchBot":["blocked","named",1,0],"anthropic-ai":["partial","*",4,0],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["partial","*",4,0],"Google-Extended":["partial","*",4,0],"Applebot-Extended":["partial","*",4,0],"CCBot":["partial","*",4,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["partial","*",4,0],"cohere-ai":["partial","*",4,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",4,0]},"llms":{"status":"timeout","error":"timed out after 10 s"},"home":{"status":"ok","http":200,"url":"https://distrowatch.com/","html_tokens":56570,"markdown_tokens":4596,"pruned_tokens":305}},
{"rank":5514,"domain":"mypixieset.com","origin":"https://mypixieset.com","robots":{"status":"html","http":200,"url":"https://pixieset.com/","bytes":735918,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":5515,"domain":"ntua.gr","origin":"https://ntua.gr","robots":{"status":"ok","http":200,"url":"https://ntua.gr/robots.txt","bytes":842,"groups":1},"bots":{"GPTBot":["partial","*",14,0],"ChatGPT-User":["partial","*",14,0],"OAI-SearchBot":["partial","*",14,0],"ClaudeBot":["partial","*",14,0],"Claude-User":["partial","*",14,0],"Claude-SearchBot":["partial","*",14,0],"anthropic-ai":["partial","*",14,0],"PerplexityBot":["partial","*",14,0],"Perplexity-User":["partial","*",14,0],"Google-Extended":["partial","*",14,0],"Applebot-Extended":["partial","*",14,0],"CCBot":["partial","*",14,0],"Bytespider":["partial","*",14,0],"Amazonbot":["partial","*",14,0],"meta-externalagent":["partial","*",14,0],"cohere-ai":["partial","*",14,0],"Diffbot":["partial","*",14,0],"ContextFetchBot":["partial","*",14,0]},"llms":{"status":"missing","http":404,"bytes":1013},"home":{"status":"ok","http":200,"url":"https://ntua.gr/el/","html_tokens":25446,"markdown_tokens":52,"pruned_tokens":52}},
{"rank":5516,"domain":"ig.com","origin":"https://ig.com","robots":{"status":"ok","http":200,"url":"https://www.ig.com/robots.txt","bytes":7182,"groups":1},"bots":{"GPTBot":["partial","*",237,5],"ChatGPT-User":["partial","*",237,5],"OAI-SearchBot":["partial","*",237,5],"ClaudeBot":["partial","*",237,5],"Claude-User":["partial","*",237,5],"Claude-SearchBot":["partial","*",237,5],"anthropic-ai":["partial","*",237,5],"PerplexityBot":["partial","*",237,5],"Perplexity-User":["partial","*",237,5],"Google-Extended":["partial","*",237,5],"Applebot-Extended":["partial","*",237,5],"CCBot":["partial","*",237,5],"Bytespider":["partial","*",237,5],"Amazonbot":["partial","*",237,5],"meta-externalagent":["partial","*",237,5],"cohere-ai":["partial","*",237,5],"Diffbot":["partial","*",237,5],"ContextFetchBot":["partial","*",237,5]},"llms":{"status":"present","http":200,"bytes":14819},"home":{"status":"ok","http":200,"url":"https://www.ig.com/","html_tokens":11289,"markdown_tokens":37,"pruned_tokens":33}},
{"rank":5517,"domain":"te.com","origin":"https://te.com","robots":{"status":"ok","http":200,"url":"https://www.te.com/robots.txt","bytes":3396,"groups":6},"bots":{"GPTBot":["partial","named",6,0],"ChatGPT-User":["partial","*",47,0],"OAI-SearchBot":["partial","*",47,0],"ClaudeBot":["partial","*",47,0],"Claude-User":["partial","*",47,0],"Claude-SearchBot":["partial","*",47,0],"anthropic-ai":["partial","*",47,0],"PerplexityBot":["partial","*",47,0],"Perplexity-User":["partial","*",47,0],"Google-Extended":["partial","*",47,0],"Applebot-Extended":["partial","*",47,0],"CCBot":["partial","*",47,0],"Bytespider":["partial","*",47,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["partial","*",47,0],"Diffbot":["partial","*",47,0],"ContextFetchBot":["partial","*",47,0]},"llms":{"status":"http_error","http":500,"bytes":2913},"home":{"status":"ok","http":200,"url":"https://www.te.com/en/home.html","html_tokens":85220,"markdown_tokens":464,"pruned_tokens":169}},
{"rank":5518,"domain":"u.to","origin":"https://u.to","robots":{"status":"ok","http":200,"url":"https://u.to/robots.txt","bytes":41,"groups":1},"bots":{"GPTBot":["allowed","*",0,1],"ChatGPT-User":["allowed","*",0,1],"OAI-SearchBot":["allowed","*",0,1],"ClaudeBot":["allowed","*",0,1],"Claude-User":["allowed","*",0,1],"Claude-SearchBot":["allowed","*",0,1],"anthropic-ai":["allowed","*",0,1],"PerplexityBot":["allowed","*",0,1],"Perplexity-User":["allowed","*",0,1],"Google-Extended":["allowed","*",0,1],"Applebot-Extended":["allowed","*",0,1],"CCBot":["allowed","*",0,1],"Bytespider":["allowed","*",0,1],"Amazonbot":["allowed","*",0,1],"meta-externalagent":["allowed","*",0,1],"cohere-ai":["allowed","*",0,1],"Diffbot":["allowed","*",0,1],"ContextFetchBot":["allowed","*",0,1]},"llms":{"status":"html","http":200,"bytes":2251},"home":{"status":"ok","http":200,"url":"https://u.to/","html_tokens":560,"markdown_tokens":0,"pruned_tokens":0}},
{"rank":5519,"domain":"kakaku.com","origin":"https://kakaku.com","robots":{"status":"ok","http":200,"url":"https://kakaku.com/robots.txt","bytes":1637,"groups":5},"bots":{"GPTBot":["partial","*",26,1],"ChatGPT-User":["partial","*",26,1],"OAI-SearchBot":["partial","*",26,1],"ClaudeBot":["partial","*",26,1],"Claude-User":["partial","*",26,1],"Claude-SearchBot":["partial","*",26,1],"anthropic-ai":["partial","*",26,1],"PerplexityBot":["partial","*",26,1],"Perplexity-User":["partial","*",26,1],"Google-Extended":["partial","*",26,1],"Applebot-Extended":["partial","*",26,1],"CCBot":["partial","*",26,1],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["partial","*",26,1],"cohere-ai":["partial","*",26,1],"Diffbot":["partial","*",26,1],"ContextFetchBot":["partial","*",26,1]},"llms":{"status":"missing","http":404,"bytes":1988},"home":{"status":"ok","http":200,"url":"https://kakaku.com/","html_tokens":27146,"markdown_tokens":76,"pruned_tokens":52}},
{"rank":5520,"domain":"quote.eastmoney.com","origin":"https://quote.eastmoney.com","robots":{"status":"missing","http":404,"url":"https://quote.eastmoney.com/robots.txt","bytes":767,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":767},"home":{"status":"ok","http":200,"url":"https://quote.eastmoney.com/","html_tokens":13123,"markdown_tokens":154,"pruned_tokens":154}},
{"rank":5521,"domain":"pocket-lint.com","origin":"https://pocket-lint.com","robots":{"status":"ok","http":200,"url":"https://www.pocket-lint.com/robots.txt","bytes":2043,"groups":2},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["partial","*",22,0],"OAI-SearchBot":["partial","*",22,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",22,0],"Claude-SearchBot":["partial","*",22,0],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["partial","*",22,0],"Perplexity-User":["partial","*",22,0],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",22,0]},"llms":{"status":"missing","http":404,"bytes":105495},"home":{"status":"ok","http":200,"url":"https://www.pocket-lint.com/","html_tokens":238705,"markdown_tokens":70,"pruned_tokens":70}},
{"rank":5522,"domain":"apc.com","origin":"https://apc.com","robots":{"status":"ok","http":200,"url":"https://www.se.com/robots.txt","bytes":46313,"groups":1},"bots":{"GPTBot":["partial","*",69,86],"ChatGPT-User":["partial","*",69,86],"OAI-SearchBot":["partial","*",69,86],"ClaudeBot":["partial","*",69,86],"Claude-User":["partial","*",69,86],"Claude-SearchBot":["partial","*",69,86],"anthropic-ai":["partial","*",69,86],"PerplexityBot":["partial","*",69,86],"Perplexity-User":["partial","*",69,86],"Google-Extended":["partial","*",69,86],"Applebot-Extended":["partial","*",69,86],"CCBot":["partial","*",69,86],"Bytespider":["partial","*",69,86],"Amazonbot":["partial","*",69,86],"meta-externalagent":["partial","*",69,86],"cohere-ai":["partial","*",69,86],"Diffbot":["partial","*",69,86],"ContextFetchBot":["partial","*",69,86]},"llms":{"status":"html","http":200,"bytes":2190289},"home":{"status":"ok","http":200,"url":"https://www.se.com/ww/en/brands/apc/?ref_url=/","html_tokens":529917,"markdown_tokens":592,"pruned_tokens":240}},
{"rank":5523,"domain":"easeus.com","origin":"https://easeus.com","robots":{"status":"ok","http":200,"url":"https://www.easeus.com/robots.txt","bytes":858,"groups":1},"bots":{"GPTBot":["partial","*",34,0],"ChatGPT-User":["partial","*",34,0],"OAI-SearchBot":["partial","*",34,0],"ClaudeBot":["partial","*",34,0],"Claude-User":["partial","*",34,0],"Claude-SearchBot":["partial","*",34,0],"anthropic-ai":["partial","*",34,0],"PerplexityBot":["partial","*",34,0],"Perplexity-User":["partial","*",34,0],"Google-Extended":["partial","*",34,0],"Applebot-Extended":["partial","*",34,0],"CCBot":["partial","*",34,0],"Bytespider":["partial","*",34,0],"Amazonbot":["partial","*",34,0],"meta-externalagent":["partial","*",34,0],"cohere-ai":["partial","*",34,0],"Diffbot":["partial","*",34,0],"ContextFetchBot":["partial","*",34,0]},"llms":{"status":"present","http":200,"bytes":2453306},"home":{"status":"ok","http":200,"url":"https://www.easeus.com/","html_tokens":23516,"markdown_tokens":643,"pruned_tokens":292}},
{"rank":5524,"domain":"victoriassecret.com","origin":"https://victoriassecret.com","robots":{"status":"ok","http":200,"url":"https://www.victoriassecret.com/robots.txt","bytes":2953,"groups":2},"bots":{"GPTBot":["partial","*",185,0],"ChatGPT-User":["partial","*",185,0],"OAI-SearchBot":["partial","*",185,0],"ClaudeBot":["partial","*",185,0],"Claude-User":["partial","*",185,0],"Claude-SearchBot":["partial","*",185,0],"anthropic-ai":["partial","*",185,0],"PerplexityBot":["partial","*",185,0],"Perplexity-User":["partial","*",185,0],"Google-Extended":["partial","*",185,0],"Applebot-Extended":["partial","*",185,0],"CCBot":["partial","*",185,0],"Bytespider":["partial","*",185,0],"Amazonbot":["partial","*",185,0],"meta-externalagent":["partial","*",185,0],"cohere-ai":["partial","*",185,0],"Diffbot":["partial","*",185,0],"ContextFetchBot":["partial","*",185,0]},"llms":{"status":"present","http":200,"bytes":232270},"home":{"status":"ok","http":200,"url":"https://www.victoriassecret.com/us/","html_tokens":110011,"markdown_tokens":1025,"pruned_tokens":173}},
{"rank":5525,"domain":"fontanka.ru","origin":"https://fontanka.ru","robots":{"status":"ok","http":200,"url":"https://www.fontanka.ru/robots.txt","bytes":4130,"groups":7},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["blocked","named",1,0],"OAI-SearchBot":["partial","*",48,7],"ClaudeBot":["partial","*",48,7],"Claude-User":["partial","*",48,7],"Claude-SearchBot":["partial","*",48,7],"anthropic-ai":["partial","*",48,7],"PerplexityBot":["partial","*",48,7],"Perplexity-User":["partial","*",48,7],"Google-Extended":["partial","*",48,7],"Applebot-Extended":["partial","*",48,7],"CCBot":["partial","*",48,7],"Bytespider":["partial","*",48,7],"Amazonbot":["partial","*",48,7],"meta-externalagent":["partial","*",48,7],"cohere-ai":["partial","*",48,7],"Diffbot":["partial","*",48,7],"ContextFetchBot":["partial","*",48,7]},"llms":{"status":"missing","http":404,"bytes":3120},"home":{"status":"ok","http":200,"url":"https://www.fontanka.ru/","html_tokens":163314,"markdown_tokens":23,"pruned_tokens":23}},
{"rank":5526,"domain":"tu-chemnitz.de","origin":"https://tu-chemnitz.de","robots":{"status":"ok","http":200,"url":"https://www.tu-chemnitz.de//robots.txt","bytes":399,"groups":1},"bots":{"GPTBot":["partial","*",14,0],"ChatGPT-User":["partial","*",14,0],"OAI-SearchBot":["partial","*",14,0],"ClaudeBot":["partial","*",14,0],"Claude-User":["partial","*",14,0],"Claude-SearchBot":["partial","*",14,0],"anthropic-ai":["partial","*",14,0],"PerplexityBot":["partial","*",14,0],"Perplexity-User":["partial","*",14,0],"Google-Extended":["partial","*",14,0],"Applebot-Extended":["partial","*",14,0],"CCBot":["partial","*",14,0],"Bytespider":["partial","*",14,0],"Amazonbot":["partial","*",14,0],"meta-externalagent":["partial","*",14,0],"cohere-ai":["partial","*",14,0],"Diffbot":["partial","*",14,0],"ContextFetchBot":["partial","*",14,0]},"llms":{"status":"missing","http":404,"bytes":27423},"home":{"status":"ok","http":200,"url":"https://www.tu-chemnitz.de/index.html.en","html_tokens":9804,"markdown_tokens":232,"pruned_tokens":121}},
{"rank":5527,"domain":"tiscali.it","origin":"https://tiscali.it","robots":{"status":"unreachable","http":null,"url":"https://tiscali.it/robots.txt","bytes":0,"groups":0,"error":"[SSL: SSLV3_ALERT_HANDSHAKE_FAILURE] sslv3 alert handshake failure (_ssl.c:1028)"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":5528,"domain":"lucid.co","origin":"https://lucid.co","robots":{"status":"ok","http":200,"url":"https://lucid.co/robots.txt","bytes":990,"groups":1},"bots":{"GPTBot":["partial","*",1,1],"ChatGPT-User":["partial","*",1,1],"OAI-SearchBot":["partial","*",1,1],"ClaudeBot":["partial","*",1,1],"Claude-User":["partial","*",1,1],"Claude-SearchBot":["partial","*",1,1],"anthropic-ai":["partial","*",1,1],"PerplexityBot":["partial","*",1,1],"Perplexity-User":["partial","*",1,1],"Google-Extended":["partial","*",1,1],"Applebot-Extended":["partial","*",1,1],"CCBot":["partial","*",1,1],"Bytespider":["partial","*",1,1],"Amazonbot":["partial","*",1,1],"meta-externalagent":["partial","*",1,1],"cohere-ai":["partial","*",1,1],"Diffbot":["partial","*",1,1],"ContextFetchBot":["partial","*",1,1]},"llms":{"status":"missing","http":404,"bytes":449783},"home":{"status":"ok","http":200,"url":"https://lucid.co/","html_tokens":187486,"markdown_tokens":791,"pruned_tokens":126}},
{"rank":5529,"domain":"bitkom.org","origin":"https://bitkom.org","robots":{"status":"ok","http":200,"url":"https://www.bitkom.org/robots.txt","bytes":892,"groups":1},"bots":{"GPTBot":["partial","*",17,19],"ChatGPT-User":["partial","*",17,19],"OAI-SearchBot":["partial","*",17,19],"ClaudeBot":["partial","*",17,19],"Claude-User":["partial","*",17,19],"Claude-SearchBot":["partial","*",17,19],"anthropic-ai":["partial","*",17,19],"PerplexityBot":["partial","*",17,19],"Perplexity-User":["partial","*",17,19],"Google-Extended":["partial","*",17,19],"Applebot-Extended":["partial","*",17,19],"CCBot":["partial","*",17,19],"Bytespider":["partial","*",17,19],"Amazonbot":["partial","*",17,19],"meta-externalagent":["partial","*",17,19],"cohere-ai":["partial","*",17,19],"Diffbot":["partial","*",17,19],"ContextFetchBot":["partial","*",17,19]},"llms":{"status":"present","http":200,"bytes":11949},"home":{"status":"ok","http":200,"url":"https://www.bitkom.org/","html_tokens":30074,"markdown_tokens":110,"pruned_tokens":110}},
{"rank":5530,"domain":"downdetector.com","origin":"https://downdetector.com","robots":{"status":"ok","http":200,"url":"https://downdetector.com/robots.txt","bytes":1683,"groups":37},"bots":{"GPTBot":["allowed","named",0,1],"ChatGPT-User":["allowed","named",0,1],"OAI-SearchBot":["allowed","named",0,1],"ClaudeBot":["allowed","named",0,1],"Claude-User":["partial","*",2,1],"Claude-SearchBot":["partial","*",2,1],"anthropic-ai":["allowed","named",0,1],"PerplexityBot":["allowed","named",0,1],"Perplexity-User":["allowed","named",0,1],"Google-Extended":["partial","*",2,1],"Applebot-Extended":["allowed","named",0,1],"CCBot":["allowed","named",0,1],"Bytespider":["allowed","named",0,1],"Amazonbot":["allowed","named",0,1],"meta-externalagent":["allowed","named",0,1],"cohere-ai":["allowed","named",0,1],"Diffbot":["allowed","named",0,1],"ContextFetchBot":["partial","*",2,1]},"llms":{"status":"disallowed"},"home":{"status":"disallowed"}},
{"rank":5531,"domain":"allianz.com","origin":"https://allianz.com","robots":{"status":"refused","http":403,"url":"https://www.allianz.com/robots.txt","bytes":5532,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":5532,"domain":"bab.la","origin":"https://bab.la","robots":{"status":"ok","http":200,"url":"https://bab.la/robots.txt","bytes":374,"groups":3},"bots":{"GPTBot":["partial","*",2,0],"ChatGPT-User":["partial","*",2,0],"OAI-SearchBot":["partial","*",2,0],"ClaudeBot":["partial","*",2,0],"Claude-User":["partial","*",2,0],"Claude-SearchBot":["partial","*",2,0],"anthropic-ai":["partial","*",2,0],"PerplexityBot":["partial","*",2,0],"Perplexity-User":["partial","*",2,0],"Google-Extended":["partial","*",2,0],"Applebot-Extended":["partial","*",2,0],"CCBot":["partial","*",2,0],"Bytespider":["partial","*",2,0],"Amazonbot":["partial","*",2,0],"meta-externalagent":["partial","*",2,0],"cohere-ai":["partial","*",2,0],"Diffbot":["partial","*",2,0],"ContextFetchBot":["partial","*",2,0]},"llms":{"status":"missing","http":404,"bytes":511676},"home":{"status":"http_error","http":403,"url":"https://bab.la/"}},
{"rank":5533,"domain":"hola.com","origin":"https://hola.com","robots":{"status":"ok","http":200,"url":"https://www.hola.com/robots.txt","bytes":5987,"groups":13},"bots":{"GPTBot":["partial","*",91,0],"ChatGPT-User":["partial","*",91,0],"OAI-SearchBot":["partial","*",91,0],"ClaudeBot":["partial","*",91,0],"Claude-User":["partial","*",91,0],"Claude-SearchBot":["partial","*",91,0],"anthropic-ai":["partial","*",91,0],"PerplexityBot":["partial","named",1,4],"Perplexity-User":["partial","named",1,4],"Google-Extended":["partial","*",91,0],"Applebot-Extended":["partial","named",1,4],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["partial","*",91,0],"meta-externalagent":["partial","*",91,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["partial","*",91,0],"ContextFetchBot":["partial","*",91,0]},"llms":{"status":"missing","http":404,"bytes":63615},"home":{"status":"ok","http":200,"url":"https://www.hola.com/us/","html_tokens":95713,"markdown_tokens":164,"pruned_tokens":164}},
{"rank":5534,"domain":"blackhat.com","origin":"https://blackhat.com","robots":{"status":"ok","http":200,"url":"https://blackhat.com/robots.txt","bytes":41,"groups":1},"bots":{"GPTBot":["partial","*",1,1],"ChatGPT-User":["partial","*",1,1],"OAI-SearchBot":["partial","*",1,1],"ClaudeBot":["partial","*",1,1],"Claude-User":["partial","*",1,1],"Claude-SearchBot":["partial","*",1,1],"anthropic-ai":["partial","*",1,1],"PerplexityBot":["partial","*",1,1],"Perplexity-User":["partial","*",1,1],"Google-Extended":["partial","*",1,1],"Applebot-Extended":["partial","*",1,1],"CCBot":["partial","*",1,1],"Bytespider":["partial","*",1,1],"Amazonbot":["partial","*",1,1],"meta-externalagent":["partial","*",1,1],"cohere-ai":["partial","*",1,1],"Diffbot":["partial","*",1,1],"ContextFetchBot":["partial","*",1,1]},"llms":{"status":"missing","http":404,"bytes":11513},"home":{"status":"http_error","http":403,"url":"https://blackhat.com/"}},
{"rank":5535,"domain":"sandia.gov","origin":"https://sandia.gov","robots":{"status":"ok","http":200,"url":"https://www.sandia.gov/robots.txt","bytes":13914,"groups":1},"bots":{"GPTBot":["partial","*",11,1],"ChatGPT-User":["partial","*",11,1],"OAI-SearchBot":["partial","*",11,1],"ClaudeBot":["partial","*",11,1],"Claude-User":["partial","*",11,1],"Claude-SearchBot":["partial","*",11,1],"anthropic-ai":["partial","*",11,1],"PerplexityBot":["partial","*",11,1],"Perplexity-User":["partial","*",11,1],"Google-Extended":["partial","*",11,1],"Applebot-Extended":["partial","*",11,1],"CCBot":["partial","*",11,1],"Bytespider":["partial","*",11,1],"Amazonbot":["partial","*",11,1],"meta-externalagent":["partial","*",11,1],"cohere-ai":["partial","*",11,1],"Diffbot":["partial","*",11,1],"ContextFetchBot":["partial","*",11,1]},"llms":{"status":"missing","http":404,"bytes":131412},"home":{"status":"ok","http":200,"url":"https://www.sandia.gov/","html_tokens":39733,"markdown_tokens":364,"pruned_tokens":223}},
{"rank":5536,"domain":"tracxn.com","origin":"https://tracxn.com","robots":{"status":"ok","http":200,"url":"https://tracxn.com/robots.txt","bytes":1390,"groups":2},"bots":{"GPTBot":["partial","named",3,27],"ChatGPT-User":["partial","named",3,27],"OAI-SearchBot":["partial","named",3,27],"ClaudeBot":["partial","named",3,27],"Claude-User":["blocked","*",1,0],"Claude-SearchBot":["blocked","*",1,0],"anthropic-ai":["partial","named",3,27],"PerplexityBot":["partial","named",3,27],"Perplexity-User":["blocked","*",1,0],"Google-Extended":["partial","named",3,27],"Applebot-Extended":["blocked","*",1,0],"CCBot":["blocked","*",1,0],"Bytespider":["blocked","*",1,0],"Amazonbot":["blocked","*",1,0],"meta-externalagent":["blocked","*",1,0],"cohere-ai":["partial","named",3,27],"Diffbot":["blocked","*",1,0],"ContextFetchBot":["blocked","*",1,0]},"llms":{"status":"disallowed"},"home":{"status":"disallowed"}},
{"rank":5537,"domain":"culture.fr","origin":"https://www.culture.fr","robots":{"status":"ok","http":200,"url":"https://www.culture.fr/robots.txt","bytes":115,"groups":1},"bots":{"GPTBot":["partial","*",2,0],"ChatGPT-User":["partial","*",2,0],"OAI-SearchBot":["partial","*",2,0],"ClaudeBot":["partial","*",2,0],"Claude-User":["partial","*",2,0],"Claude-SearchBot":["partial","*",2,0],"anthropic-ai":["partial","*",2,0],"PerplexityBot":["partial","*",2,0],"Perplexity-User":["partial","*",2,0],"Google-Extended":["partial","*",2,0],"Applebot-Extended":["partial","*",2,0],"CCBot":["partial","*",2,0],"Bytespider":["partial","*",2,0],"Amazonbot":["partial","*",2,0],"meta-externalagent":["partial","*",2,0],"cohere-ai":["partial","*",2,0],"Diffbot":["partial","*",2,0],"ContextFetchBot":["partial","*",2,0]},"llms":{"status":"missing","http":404,"bytes":1193},"home":{"status":"ok","http":200,"url":"https://www.culture.gouv.fr","html_tokens":50089,"markdown_tokens":246,"pruned_tokens":63}},
{"rank":5538,"domain":"panasonic.jp","origin":"https://panasonic.jp","robots":{"status":"ok","http":200,"url":"https://panasonic.jp/robots.txt","bytes":110,"groups":1},"bots":{"GPTBot":["partial","*",2,0],"ChatGPT-User":["partial","*",2,0],"OAI-SearchBot":["partial","*",2,0],"ClaudeBot":["partial","*",2,0],"Claude-User":["partial","*",2,0],"Claude-SearchBot":["partial","*",2,0],"anthropic-ai":["partial","*",2,0],"PerplexityBot":["partial","*",2,0],"Perplexity-User":["partial","*",2,0],"Google-Extended":["partial","*",2,0],"Applebot-Extended":["partial","*",2,0],"CCBot":["partial","*",2,0],"Bytespider":["partial","*",2,0],"Amazonbot":["partial","*",2,0],"meta-externalagent":["partial","*",2,0],"cohere-ai":["partial","*",2,0],"Diffbot":["partial","*",2,0],"ContextFetchBot":["partial","*",2,0]},"llms":{"status":"missing","http":404,"bytes":177406},"home":{"status":"ok","http":200,"url":"https://panasonic.jp/","html_tokens":141898,"markdown_tokens":762,"pruned_tokens":99}},
{"rank":5539,"domain":"umt.edu","origin":"https://umt.edu","robots":{"status":"ok","http":200,"url":"https://www.umt.edu/robots.txt","bytes":68,"groups":1},"bots":{"GPTBot":["allowed","*",0,1],"ChatGPT-User":["allowed","*",0,1],"OAI-SearchBot":["allowed","*",0,1],"ClaudeBot":["allowed","*",0,1],"Claude-User":["allowed","*",0,1],"Claude-SearchBot":["allowed","*",0,1],"anthropic-ai":["allowed","*",0,1],"PerplexityBot":["allowed","*",0,1],"Perplexity-User":["allowed","*",0,1],"Google-Extended":["allowed","*",0,1],"Applebot-Extended":["allowed","*",0,1],"CCBot":["allowed","*",0,1],"Bytespider":["allowed","*",0,1],"Amazonbot":["allowed","*",0,1],"meta-externalagent":["allowed","*",0,1],"cohere-ai":["allowed","*",0,1],"Diffbot":["allowed","*",0,1],"ContextFetchBot":["allowed","*",0,1]},"llms":{"status":"present","http":200,"bytes":3087},"home":{"status":"ok","http":200,"url":"https://www.umt.edu/","html_tokens":36698,"markdown_tokens":697,"pruned_tokens":170}},
{"rank":5540,"domain":"usercontent.one","origin":"https://usercontent.one","robots":{"status":"refused","http":400,"url":"https://usercontent.one/robots.txt","bytes":11,"groups":0,"error":"HTTP 400"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":5541,"domain":"hs.fi","origin":"https://hs.fi","robots":{"status":"ok","http":200,"url":"https://www.hs.fi/robots.txt","bytes":1516,"groups":2},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["blocked","named",1,0],"OAI-SearchBot":["blocked","named",1,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["blocked","named",1,0],"Claude-SearchBot":["blocked","named",1,0],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["blocked","named",1,0],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",6,0]},"llms":{"status":"missing","http":404,"bytes":526358},"home":{"status":"ok","http":200,"url":"https://www.hs.fi/","html_tokens":168236,"markdown_tokens":734,"pruned_tokens":52}},
{"rank":5542,"domain":"infowars.com","origin":"https://infowars.com","robots":{"status":"html","http":200,"url":"https://infowars.com/robots.txt","bytes":694,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":5543,"domain":"riseup.net","origin":"https://riseup.net","robots":{"status":"missing","http":404,"url":"https://riseup.net/robots.txt","bytes":236,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":236},"home":{"status":"ok","http":200,"url":"https://riseup.net/","html_tokens":3849,"markdown_tokens":732,"pruned_tokens":168}},
{"rank":5544,"domain":"sympatico.ca","origin":"https://sympatico.ca","robots":{"status":"html","http":200,"url":"https://www.noovomoi.ca/","bytes":261479,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":5545,"domain":"activision.com","origin":"https://activision.com","robots":{"status":"timeout","http":null,"url":"https://activision.com/robots.txt","bytes":0,"groups":0,"error":"timed out after 10 s"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":5546,"domain":"polito.it","origin":"https://www.polito.it","robots":{"status":"ok","http":200,"url":"https://www.polito.it/robots.txt","bytes":2027,"groups":1},"bots":{"GPTBot":["partial","*",34,18],"ChatGPT-User":["partial","*",34,18],"OAI-SearchBot":["partial","*",34,18],"ClaudeBot":["partial","*",34,18],"Claude-User":["partial","*",34,18],"Claude-SearchBot":["partial","*",34,18],"anthropic-ai":["partial","*",34,18],"PerplexityBot":["partial","*",34,18],"Perplexity-User":["partial","*",34,18],"Google-Extended":["partial","*",34,18],"Applebot-Extended":["partial","*",34,18],"CCBot":["partial","*",34,18],"Bytespider":["partial","*",34,18],"Amazonbot":["partial","*",34,18],"meta-externalagent":["partial","*",34,18],"cohere-ai":["partial","*",34,18],"Diffbot":["partial","*",34,18],"ContextFetchBot":["partial","*",34,18]},"llms":{"status":"missing","http":404,"bytes":187},"home":{"status":"ok","http":200,"url":"https://www.polito.it/","html_tokens":56173,"markdown_tokens":90,"pruned_tokens":78}},
{"rank":5547,"domain":"jsbin.com","origin":"https://jsbin.com","robots":{"status":"refused","http":403,"url":"https://jsbin.com/robots.txt","bytes":5397,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":5548,"domain":"conecti.me","origin":"https://conecti.me","robots":{"status":"ok","http":200,"url":"https://conecti.me/robots.txt","bytes":24,"groups":1},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"missing","http":404,"bytes":25753},"home":{"status":"ok","http":200,"url":"https://conecti.me/","html_tokens":22997,"markdown_tokens":706,"pruned_tokens":136}},
{"rank":5549,"domain":"store.line.me","origin":"https://store.line.me","robots":{"status":"ok","http":200,"url":"https://store.line.me/robots.txt","bytes":439,"groups":10},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["partial","*",5,0],"OAI-SearchBot":["partial","*",5,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",5,0],"Claude-SearchBot":["partial","*",5,0],"anthropic-ai":["partial","*",5,0],"PerplexityBot":["partial","*",5,0],"Perplexity-User":["partial","*",5,0],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["partial","*",5,0],"Diffbot":["partial","*",5,0],"ContextFetchBot":["partial","*",5,0]},"llms":{"status":"missing","http":404,"bytes":12730},"home":{"status":"ok","http":200,"url":"https://store.line.me/","html_tokens":13363,"markdown_tokens":27,"pruned_tokens":27}},
{"rank":5550,"domain":"sendgrid.com","origin":"https://sendgrid.com","robots":{"status":"ok","http":200,"url":"https://www.twilio.com/robots.txt","bytes":976,"groups":2},"bots":{"GPTBot":["partial","*",13,6],"ChatGPT-User":["partial","*",13,6],"OAI-SearchBot":["partial","*",13,6],"ClaudeBot":["partial","*",13,6],"Claude-User":["partial","*",13,6],"Claude-SearchBot":["partial","*",13,6],"anthropic-ai":["partial","*",13,6],"PerplexityBot":["partial","*",13,6],"Perplexity-User":["partial","*",13,6],"Google-Extended":["partial","*",13,6],"Applebot-Extended":["partial","*",13,6],"CCBot":["partial","*",13,6],"Bytespider":["partial","*",13,6],"Amazonbot":["partial","*",13,6],"meta-externalagent":["partial","*",13,6],"cohere-ai":["partial","*",13,6],"Diffbot":["partial","*",13,6],"ContextFetchBot":["partial","*",13,6]},"llms":{"status":"html","http":200,"bytes":631818},"home":{"status":"ok","http":200,"url":"https://www.twilio.com/en-us/sendgrid","html_tokens":157944,"markdown_tokens":137,"pruned_tokens":53}},
{"rank":5551,"domain":"doctolib.fr","origin":"https://doctolib.fr","robots":{"status":"ok","http":200,"url":"https://www.doctolib.fr/robots.txt","bytes":1626,"groups":3},"bots":{"GPTBot":["partial","*",45,2],"ChatGPT-User":["partial","*",45,2],"OAI-SearchBot":["partial","*",45,2],"ClaudeBot":["partial","*",45,2],"Claude-User":["partial","*",45,2],"Claude-SearchBot":["partial","*",45,2],"anthropic-ai":["partial","*",45,2],"PerplexityBot":["partial","*",45,2],"Perplexity-User":["partial","*",45,2],"Google-Extended":["partial","*",45,2],"Applebot-Extended":["partial","*",45,2],"CCBot":["partial","*",45,2],"Bytespider":["partial","*",45,2],"Amazonbot":["partial","*",45,2],"meta-externalagent":["partial","named",3,5],"cohere-ai":["partial","*",45,2],"Diffbot":["partial","*",45,2],"ContextFetchBot":["partial","*",45,2]},"llms":{"status":"missing","http":404,"bytes":0},"home":{"status":"http_error","http":403,"url":"https://www.doctolib.fr/"}},
{"rank":5552,"domain":"tuxfamily.org","origin":"https://tuxfamily.org","robots":{"status":"missing","http":404,"url":"https://www.tuxfamily.org/robots.txt","bytes":6006,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":6006},"home":{"status":"ok","http":200,"url":"https://www.tuxfamily.org/","html_tokens":1501,"markdown_tokens":234,"pruned_tokens":47}},
{"rank":5553,"domain":"thebalancesmb.com","origin":"https://thebalancesmb.com","robots":{"status":"ok","http":200,"url":"https://www.thebalancemoney.com/robots.txt","bytes":3483,"groups":6},"bots":{"GPTBot":["partial","named",1,0],"ChatGPT-User":["partial","named",1,0],"OAI-SearchBot":["partial","named",1,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["blocked","named",1,0],"Claude-SearchBot":["partial","named",1,2],"anthropic-ai":["partial","named",1,2],"PerplexityBot":["partial","named",1,2],"Perplexity-User":["blocked","named",1,0],"Google-Extended":["partial","named",1,2],"Applebot-Extended":["blocked","named",1,0],"CCBot":["partial","named",1,2],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["partial","named",1,2],"cohere-ai":["partial","named",1,2],"Diffbot":["partial","*",2,0],"ContextFetchBot":["partial","*",2,0]},"llms":{"status":"http_error","http":402,"bytes":612},"home":{"status":"http_error","http":402,"url":"https://www.thebalancemoney.com/small-business-5181174"}},
{"rank":5554,"domain":"hackster.io","origin":"https://hackster.io","robots":{"status":"ok","http":200,"url":"https://www.hackster.io/robots.txt","bytes":7388,"groups":142},"bots":{"GPTBot":["partial","*",19,2],"ChatGPT-User":["partial","*",19,2],"OAI-SearchBot":["partial","*",19,2],"ClaudeBot":["partial","*",19,2],"Claude-User":["partial","*",19,2],"Claude-SearchBot":["partial","*",19,2],"anthropic-ai":["partial","*",19,2],"PerplexityBot":["partial","*",19,2],"Perplexity-User":["partial","*",19,2],"Google-Extended":["partial","*",19,2],"Applebot-Extended":["partial","*",19,2],"CCBot":["partial","*",19,2],"Bytespider":["partial","*",19,2],"Amazonbot":["partial","*",19,2],"meta-externalagent":["partial","*",19,2],"cohere-ai":["partial","*",19,2],"Diffbot":["partial","*",19,2],"ContextFetchBot":["partial","*",19,2]},"llms":{"status":"missing","http":404,"bytes":41805},"home":{"status":"ok","http":200,"url":"https://www.hackster.io/","html_tokens":27319,"markdown_tokens":19,"pruned_tokens":19}},
{"rank":5555,"domain":"000webhostapp.com","origin":"https://000webhostapp.com","robots":{"status":"timeout","http":null,"url":"https://000webhostapp.com/robots.txt","bytes":0,"groups":0,"error":"timed out after 10 s"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":5556,"domain":"massgeneral.org","origin":"https://massgeneral.org","robots":{"status":"html","http":200,"url":"https://www.massgeneralbrigham.org/en/about/massachusetts-general-hospital","bytes":27850,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":5557,"domain":"citylab.com","origin":"https://citylab.com","robots":{"status":"html","http":200,"url":"https://www.bloomberg.com/citylab","bytes":2892955,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":5558,"domain":"hb.afl.rakuten.co.jp","origin":"https://hb.afl.rakuten.co.jp","robots":{"status":"missing","http":404,"url":"https://hb.afl.rakuten.co.jp/robots.txt","bytes":2370,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":2370},"home":{"status":"http_error","http":404,"url":"https://hb.afl.rakuten.co.jp/"}},
{"rank":5559,"domain":"insideevs.com","origin":"https://insideevs.com","robots":{"status":"ok","http":200,"url":"https://insideevs.com/robots.txt","bytes":467,"groups":5},"bots":{"GPTBot":["partial","*",6,0],"ChatGPT-User":["partial","*",6,0],"OAI-SearchBot":["partial","*",6,0],"ClaudeBot":["partial","*",6,0],"Claude-User":["partial","*",6,0],"Claude-SearchBot":["partial","*",6,0],"anthropic-ai":["partial","*",6,0],"PerplexityBot":["partial","*",6,0],"Perplexity-User":["partial","*",6,0],"Google-Extended":["partial","*",6,0],"Applebot-Extended":["partial","*",6,0],"CCBot":["partial","*",6,0],"Bytespider":["partial","*",6,0],"Amazonbot":["partial","*",6,0],"meta-externalagent":["partial","*",6,0],"cohere-ai":["partial","*",6,0],"Diffbot":["partial","*",6,0],"ContextFetchBot":["partial","*",6,0]},"llms":{"status":"missing","http":404,"bytes":169},"home":{"status":"http_error","http":403,"url":"https://insideevs.com/"}},
{"rank":5560,"domain":"traderzo.com","origin":"http://traderzo.com","robots":{"status":"ok","http":200,"url":"http://traderzo.com/robots.txt","bytes":1878,"groups":34},"bots":{"GPTBot":["allowed","named",0,1],"ChatGPT-User":["allowed","named",0,1],"OAI-SearchBot":["allowed","named",0,1],"ClaudeBot":["allowed","named",0,1],"Claude-User":["allowed","named",0,1],"Claude-SearchBot":["allowed","named",0,1],"anthropic-ai":["allowed","named",0,1],"PerplexityBot":["allowed","named",0,1],"Perplexity-User":["allowed","named",0,1],"Google-Extended":["allowed","named",0,1],"Applebot-Extended":["allowed","named",0,1],"CCBot":["allowed","named",0,1],"Bytespider":["allowed","named",0,1],"Amazonbot":["allowed","named",0,1],"meta-externalagent":["allowed","named",0,1],"cohere-ai":["allowed","named",0,1],"Diffbot":["partial","*",6,2],"ContextFetchBot":["partial","*",6,2]},"llms":{"status":"other_text","http":200,"bytes":28905},"home":{"status":"ok","http":200,"url":"http://traderzo.com/","html_tokens":39040,"markdown_tokens":454,"pruned_tokens":133}},
{"rank":5561,"domain":"dict.cc","origin":"https://dict.cc","robots":{"status":"ok","http":200,"url":"https://dict.cc/robots.txt","bytes":677,"groups":14},"bots":{"GPTBot":["partial","*",4,0],"ChatGPT-User":["partial","*",4,0],"OAI-SearchBot":["partial","*",4,0],"ClaudeBot":["partial","*",4,0],"Claude-User":["partial","*",4,0],"Claude-SearchBot":["partial","*",4,0],"anthropic-ai":["partial","*",4,0],"PerplexityBot":["partial","*",4,0],"Perplexity-User":["partial","*",4,0],"Google-Extended":["partial","*",4,0],"Applebot-Extended":["partial","*",4,0],"CCBot":["partial","*",4,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["partial","*",4,0],"meta-externalagent":["partial","*",4,0],"cohere-ai":["partial","*",4,0],"Diffbot":["partial","*",4,0],"ContextFetchBot":["partial","*",4,0]},"llms":{"status":"missing","http":404,"bytes":40},"home":{"status":"ok","http":200,"url":"https://www.dict.cc/","html_tokens":11905,"markdown_tokens":1177,"pruned_tokens":339}},
{"rank":5562,"domain":"daserste.de","origin":"https://daserste.de","robots":{"status":"html","http":200,"url":"https://www.ardmediathek.de/daserste","bytes":914091,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":5563,"domain":"rv.ua","origin":"https://rv.ua","robots":{"status":"unreachable","http":null,"url":"https://rv.ua/robots.txt","bytes":0,"groups":0,"error":"cannot resolve rv.ua"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":5564,"domain":"mariadb.com","origin":"https://mariadb.com","robots":{"status":"ok","http":200,"url":"https://mariadb.com/robots.txt","bytes":546,"groups":1},"bots":{"GPTBot":["partial","*",8,1],"ChatGPT-User":["partial","*",8,1],"OAI-SearchBot":["partial","*",8,1],"ClaudeBot":["partial","*",8,1],"Claude-User":["partial","*",8,1],"Claude-SearchBot":["partial","*",8,1],"anthropic-ai":["partial","*",8,1],"PerplexityBot":["partial","*",8,1],"Perplexity-User":["partial","*",8,1],"Google-Extended":["partial","*",8,1],"Applebot-Extended":["partial","*",8,1],"CCBot":["partial","*",8,1],"Bytespider":["partial","*",8,1],"Amazonbot":["partial","*",8,1],"meta-externalagent":["partial","*",8,1],"cohere-ai":["partial","*",8,1],"Diffbot":["partial","*",8,1],"ContextFetchBot":["partial","*",8,1]},"llms":{"status":"present","http":200,"bytes":14452},"home":{"status":"ok","http":200,"url":"https://mariadb.com/","html_tokens":427201,"markdown_tokens":944,"pruned_tokens":578}},
{"rank":5565,"domain":"about.ads.microsoft.com","origin":"https://about.ads.microsoft.com","robots":{"status":"ok","http":200,"url":"https://about.ads.microsoft.com/robots.txt","bytes":429,"groups":1},"bots":{"GPTBot":["partial","*",7,0],"ChatGPT-User":["partial","*",7,0],"OAI-SearchBot":["partial","*",7,0],"ClaudeBot":["partial","*",7,0],"Claude-User":["partial","*",7,0],"Claude-SearchBot":["partial","*",7,0],"anthropic-ai":["partial","*",7,0],"PerplexityBot":["partial","*",7,0],"Perplexity-User":["partial","*",7,0],"Google-Extended":["partial","*",7,0],"Applebot-Extended":["partial","*",7,0],"CCBot":["partial","*",7,0],"Bytespider":["partial","*",7,0],"Amazonbot":["partial","*",7,0],"meta-externalagent":["partial","*",7,0],"cohere-ai":["partial","*",7,0],"Diffbot":["partial","*",7,0],"ContextFetchBot":["partial","*",7,0]},"llms":{"status":"present","http":200,"bytes":10604},"home":{"status":"ok","http":200,"url":"https://about.ads.microsoft.com/en","html_tokens":47163,"markdown_tokens":156,"pruned_tokens":117}},
{"rank":5566,"domain":"meteoblue.com","origin":"https://meteoblue.com","robots":{"status":"ok","http":200,"url":"https://www.meteoblue.com/robots.txt","bytes":24,"groups":1},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"missing","http":404,"bytes":507},"home":{"status":"ok","http":200,"url":"https://www.meteoblue.com/en/weather/week/orlando_united-states_4167147","html_tokens":58716,"markdown_tokens":1345,"pruned_tokens":25}},
{"rank":5567,"domain":"chevron.com","origin":"https://chevron.com","robots":{"status":"refused","http":403,"url":"https://www.chevron.com/robots.txt","bytes":1484,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":5568,"domain":"doodleordie.com","origin":"https://doodleordie.com","robots":{"status":"unreachable","http":null,"url":"https://doodleordie.com/robots.txt","bytes":0,"groups":0,"error":"All connection attempts failed"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":5569,"domain":"lazada.co.th","origin":"https://lazada.co.th","robots":{"status":"ok","http":200,"url":"https://www.lazada.co.th/robots.txt","bytes":223,"groups":4},"bots":{"GPTBot":["allowed","named",0,1],"ChatGPT-User":["allowed","named",0,1],"OAI-SearchBot":["allowed","named",0,1],"ClaudeBot":["partial","*",4,0],"Claude-User":["partial","*",4,0],"Claude-SearchBot":["partial","*",4,0],"anthropic-ai":["partial","*",4,0],"PerplexityBot":["partial","*",4,0],"Perplexity-User":["partial","*",4,0],"Google-Extended":["partial","*",4,0],"Applebot-Extended":["partial","*",4,0],"CCBot":["partial","*",4,0],"Bytespider":["partial","*",4,0],"Amazonbot":["partial","*",4,0],"meta-externalagent":["partial","*",4,0],"cohere-ai":["partial","*",4,0],"Diffbot":["partial","*",4,0],"ContextFetchBot":["partial","*",4,0]},"llms":{"status":"present","http":200,"bytes":2715},"home":{"status":"ok","http":200,"url":"https://www.lazada.co.th/","html_tokens":116720,"markdown_tokens":640,"pruned_tokens":226}},
{"rank":5570,"domain":"later.com","origin":"https://later.com","robots":{"status":"ok","http":200,"url":"https://later.com/robots.txt","bytes":68,"groups":1},"bots":{"GPTBot":["partial","*",1,0],"ChatGPT-User":["partial","*",1,0],"OAI-SearchBot":["partial","*",1,0],"ClaudeBot":["partial","*",1,0],"Claude-User":["partial","*",1,0],"Claude-SearchBot":["partial","*",1,0],"anthropic-ai":["partial","*",1,0],"PerplexityBot":["partial","*",1,0],"Perplexity-User":["partial","*",1,0],"Google-Extended":["partial","*",1,0],"Applebot-Extended":["partial","*",1,0],"CCBot":["partial","*",1,0],"Bytespider":["partial","*",1,0],"Amazonbot":["partial","*",1,0],"meta-externalagent":["partial","*",1,0],"cohere-ai":["partial","*",1,0],"Diffbot":["partial","*",1,0],"ContextFetchBot":["partial","*",1,0]},"llms":{"status":"missing","http":404,"bytes":765843},"home":{"status":"ok","http":200,"url":"https://later.com/","html_tokens":216610,"markdown_tokens":550,"pruned_tokens":25}},
{"rank":5571,"domain":"vector.co.jp","origin":"https://vector.co.jp","robots":{"status":"ok","http":200,"url":"https://vector.co.jp/robots.txt","bytes":115,"groups":2},"bots":{"GPTBot":["partial","*",2,0],"ChatGPT-User":["partial","*",2,0],"OAI-SearchBot":["partial","*",2,0],"ClaudeBot":["partial","*",2,0],"Claude-User":["partial","*",2,0],"Claude-SearchBot":["partial","*",2,0],"anthropic-ai":["partial","*",2,0],"PerplexityBot":["partial","*",2,0],"Perplexity-User":["partial","*",2,0],"Google-Extended":["partial","*",2,0],"Applebot-Extended":["partial","*",2,0],"CCBot":["partial","*",2,0],"Bytespider":["partial","*",2,0],"Amazonbot":["partial","*",2,0],"meta-externalagent":["partial","*",2,0],"cohere-ai":["partial","*",2,0],"Diffbot":["partial","*",2,0],"ContextFetchBot":["partial","*",2,0]},"llms":{"status":"missing","http":404,"bytes":1280},"home":{"status":"ok","http":200,"url":"https://vector.co.jp/","html_tokens":6383,"markdown_tokens":378,"pruned_tokens":57}},
{"rank":5572,"domain":"placeit.net","origin":"https://placeit.net","robots":{"status":"ok","http":200,"url":"https://placeit.net/robots.txt","bytes":3247,"groups":6},"bots":{"GPTBot":["partial","*",44,0],"ChatGPT-User":["partial","*",44,0],"OAI-SearchBot":["partial","*",44,0],"ClaudeBot":["partial","*",44,0],"Claude-User":["partial","*",44,0],"Claude-SearchBot":["partial","*",44,0],"anthropic-ai":["partial","*",44,0],"PerplexityBot":["partial","*",44,0],"Perplexity-User":["partial","*",44,0],"Google-Extended":["partial","*",44,0],"Applebot-Extended":["partial","*",44,0],"CCBot":["blocked","named",1,0],"Bytespider":["partial","*",44,0],"Amazonbot":["partial","*",44,0],"meta-externalagent":["allowed","named",0,1],"cohere-ai":["partial","*",44,0],"Diffbot":["partial","*",44,0],"ContextFetchBot":["partial","*",44,0]},"llms":{"status":"missing","http":404,"bytes":351206},"home":{"status":"ok","http":200,"url":"https://placeit.net/","html_tokens":154888,"markdown_tokens":13,"pruned_tokens":13}},
{"rank":5573,"domain":"gazeta.pl","origin":"https://gazeta.pl","robots":{"status":"ok","http":200,"url":"https://www.gazeta.pl/robots.txt","bytes":12060,"groups":276},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["partial","*",19,0],"OAI-SearchBot":["partial","*",19,0],"ClaudeBot":["partial","*",19,0],"Claude-User":["partial","*",19,0],"Claude-SearchBot":["partial","*",19,0],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["partial","*",19,0],"Perplexity-User":["partial","*",19,0],"Google-Extended":["partial","*",19,0],"Applebot-Extended":["partial","*",19,0],"CCBot":["blocked","named",1,0],"Bytespider":["partial","*",19,0],"Amazonbot":["partial","*",19,0],"meta-externalagent":["partial","*",19,0],"cohere-ai":["partial","*",19,0],"Diffbot":["partial","*",19,0],"ContextFetchBot":["partial","*",19,0]},"llms":{"status":"missing","http":404,"bytes":13670},"home":{"status":"ok","http":200,"url":"https://www.gazeta.pl/0,0.html","html_tokens":156487,"markdown_tokens":253,"pruned_tokens":148}},
{"rank":5574,"domain":"dns.google","origin":"https://dns.google","robots":{"status":"missing","http":404,"url":"https://dns.google/robots.txt","bytes":1571,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":1569},"home":{"status":"ok","http":200,"url":"https://dns.google/","html_tokens":346,"markdown_tokens":30,"pruned_tokens":30}},
{"rank":5575,"domain":"sensortower.com","origin":"https://sensortower.com","robots":{"status":"ok","http":200,"url":"https://sensortower.com/robots.txt","bytes":40,"groups":1},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"missing","http":404,"bytes":8686},"home":{"status":"ok","http":200,"url":"https://sensortower.com/","html_tokens":66621,"markdown_tokens":883,"pruned_tokens":186}},
{"rank":5576,"domain":"metal-archives.com","origin":"https://metal-archives.com","robots":{"status":"ok","http":200,"url":"https://metal-archives.com/robots.txt","bytes":181,"groups":2},"bots":{"GPTBot":["partial","*",5,0],"ChatGPT-User":["partial","*",5,0],"OAI-SearchBot":["partial","*",5,0],"ClaudeBot":["partial","*",5,0],"Claude-User":["partial","*",5,0],"Claude-SearchBot":["partial","*",5,0],"anthropic-ai":["partial","*",5,0],"PerplexityBot":["partial","*",5,0],"Perplexity-User":["partial","*",5,0],"Google-Extended":["partial","*",5,0],"Applebot-Extended":["partial","*",5,0],"CCBot":["partial","*",5,0],"Bytespider":["partial","*",5,0],"Amazonbot":["partial","*",5,0],"meta-externalagent":["partial","*",5,0],"cohere-ai":["partial","*",5,0],"Diffbot":["partial","*",5,0],"ContextFetchBot":["partial","*",5,0]},"llms":{"status":"missing","http":404,"bytes":8433},"home":{"status":"ok","http":200,"url":"https://metal-archives.com/","html_tokens":12164,"markdown_tokens":2768,"pruned_tokens":94}},
{"rank":5577,"domain":"novayagazeta.ru","origin":"https://novayagazeta.ru","robots":{"status":"ok","http":200,"url":"https://novayagazeta.ru/robots.txt","bytes":291,"groups":3},"bots":{"GPTBot":["partial","*",3,0],"ChatGPT-User":["partial","*",3,0],"OAI-SearchBot":["partial","*",3,0],"ClaudeBot":["partial","*",3,0],"Claude-User":["partial","*",3,0],"Claude-SearchBot":["partial","*",3,0],"anthropic-ai":["partial","*",3,0],"PerplexityBot":["partial","*",3,0],"Perplexity-User":["partial","*",3,0],"Google-Extended":["partial","*",3,0],"Applebot-Extended":["partial","*",3,0],"CCBot":["partial","*",3,0],"Bytespider":["partial","*",3,0],"Amazonbot":["partial","*",3,0],"meta-externalagent":["partial","*",3,0],"cohere-ai":["partial","*",3,0],"Diffbot":["partial","*",3,0],"ContextFetchBot":["partial","*",3,0]},"llms":{"status":"html","http":200,"bytes":10099},"home":{"status":"ok","http":200,"url":"https://novayagazeta.ru/","html_tokens":2334,"markdown_tokens":0,"pruned_tokens":0}},
{"rank":5578,"domain":"scipy.org","origin":"https://scipy.org","robots":{"status":"missing","http":404,"url":"https://scipy.org/robots.txt","bytes":3449,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":3449},"home":{"status":"ok","http":200,"url":"https://scipy.org/","html_tokens":4680,"markdown_tokens":192,"pruned_tokens":151}},
{"rank":5579,"domain":"verywellfit.com","origin":"https://verywellfit.com","robots":{"status":"ok","http":200,"url":"https://www.verywellfit.com/robots.txt","bytes":3475,"groups":6},"bots":{"GPTBot":["partial","named",1,0],"ChatGPT-User":["partial","named",1,0],"OAI-SearchBot":["partial","named",1,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["blocked","named",1,0],"Claude-SearchBot":["partial","named",1,2],"anthropic-ai":["partial","named",1,2],"PerplexityBot":["partial","named",1,2],"Perplexity-User":["blocked","named",1,0],"Google-Extended":["partial","named",1,2],"Applebot-Extended":["blocked","named",1,0],"CCBot":["partial","named",1,2],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["partial","named",1,2],"cohere-ai":["partial","named",1,2],"Diffbot":["partial","*",2,0],"ContextFetchBot":["partial","*",2,0]},"llms":{"status":"http_error","http":402,"bytes":612},"home":{"status":"http_error","http":402,"url":"https://www.verywellfit.com/"}},
{"rank":5580,"domain":"uproxx.com","origin":"https://uproxx.com","robots":{"status":"ok","http":200,"url":"https://uproxx.com/robots.txt","bytes":308,"groups":2},"bots":{"GPTBot":["partial","*",8,1],"ChatGPT-User":["partial","*",8,1],"OAI-SearchBot":["partial","*",8,1],"ClaudeBot":["partial","*",8,1],"Claude-User":["partial","*",8,1],"Claude-SearchBot":["partial","*",8,1],"anthropic-ai":["partial","*",8,1],"PerplexityBot":["partial","*",8,1],"Perplexity-User":["partial","*",8,1],"Google-Extended":["partial","*",8,1],"Applebot-Extended":["partial","*",8,1],"CCBot":["partial","*",8,1],"Bytespider":["partial","*",8,1],"Amazonbot":["partial","*",8,1],"meta-externalagent":["partial","*",8,1],"cohere-ai":["partial","*",8,1],"Diffbot":["partial","*",8,1],"ContextFetchBot":["partial","*",8,1]},"llms":{"status":"html","http":200,"bytes":282049},"home":{"status":"ok","http":200,"url":"https://uproxx.com/","html_tokens":70452,"markdown_tokens":3707,"pruned_tokens":466}},
{"rank":5581,"domain":"nikon.com","origin":"https://nikon.com","robots":{"status":"missing","http":404,"url":"https://www.nikon.com/robots.txt","bytes":10792,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"present","http":200,"bytes":51733},"home":{"status":"ok","http":200,"url":"https://www.nikon.com/","html_tokens":9487,"markdown_tokens":44,"pruned_tokens":44}},
{"rank":5582,"domain":"tianqi.com","origin":"https://tianqi.com","robots":{"status":"ok","http":200,"url":"https://www.tianqi.com/robots.txt","bytes":520,"groups":3},"bots":{"GPTBot":["partial","*",19,0],"ChatGPT-User":["partial","*",19,0],"OAI-SearchBot":["partial","*",19,0],"ClaudeBot":["partial","*",19,0],"Claude-User":["partial","*",19,0],"Claude-SearchBot":["partial","*",19,0],"anthropic-ai":["partial","*",19,0],"PerplexityBot":["partial","*",19,0],"Perplexity-User":["partial","*",19,0],"Google-Extended":["partial","*",19,0],"Applebot-Extended":["partial","*",19,0],"CCBot":["partial","*",19,0],"Bytespider":["partial","*",19,0],"Amazonbot":["partial","*",19,0],"meta-externalagent":["partial","*",19,0],"cohere-ai":["partial","*",19,0],"Diffbot":["partial","*",19,0],"ContextFetchBot":["partial","*",19,0]},"llms":{"status":"html","http":200,"bytes":298},"home":{"status":"ok","http":200,"url":"https://www.tianqi.com/","html_tokens":23140,"markdown_tokens":27,"pruned_tokens":27}},
{"rank":5583,"domain":"sncf-connect.com","origin":"https://sncf-connect.com","robots":{"status":"refused","http":403,"url":"https://www.sncf-connect.com/robots.txt","bytes":776,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":5584,"domain":"tinkoff.ru","origin":"https://tinkoff.ru","robots":{"status":"unreachable","http":null,"url":"https://tinkoff.ru/robots.txt","bytes":0,"groups":0,"error":"[SSL: CERTIFICATE_VERIFY_FAILED] certificate verify failed: self-signed certificate in certificate chain (_ssl.c:1028)"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":5585,"domain":"royal.uk","origin":"https://royal.uk","robots":{"status":"ok","http":200,"url":"https://www.royal.uk/robots.txt","bytes":2141,"groups":1},"bots":{"GPTBot":["partial","*",35,22],"ChatGPT-User":["partial","*",35,22],"OAI-SearchBot":["partial","*",35,22],"ClaudeBot":["partial","*",35,22],"Claude-User":["partial","*",35,22],"Claude-SearchBot":["partial","*",35,22],"anthropic-ai":["partial","*",35,22],"PerplexityBot":["partial","*",35,22],"Perplexity-User":["partial","*",35,22],"Google-Extended":["partial","*",35,22],"Applebot-Extended":["partial","*",35,22],"CCBot":["partial","*",35,22],"Bytespider":["partial","*",35,22],"Amazonbot":["partial","*",35,22],"meta-externalagent":["partial","*",35,22],"cohere-ai":["partial","*",35,22],"Diffbot":["partial","*",35,22],"ContextFetchBot":["partial","*",35,22]},"llms":{"status":"missing","http":404,"bytes":186},"home":{"status":"ok","http":200,"url":"https://www.royal.uk/","html_tokens":15771,"markdown_tokens":73,"pruned_tokens":73}},
{"rank":5586,"domain":"kempinski.com","origin":"https://kempinski.com","robots":{"status":"ok","http":200,"url":"https://www.kempinski.com/robots.txt","bytes":186,"groups":1},"bots":{"GPTBot":["partial","*",5,1],"ChatGPT-User":["partial","*",5,1],"OAI-SearchBot":["partial","*",5,1],"ClaudeBot":["partial","*",5,1],"Claude-User":["partial","*",5,1],"Claude-SearchBot":["partial","*",5,1],"anthropic-ai":["partial","*",5,1],"PerplexityBot":["partial","*",5,1],"Perplexity-User":["partial","*",5,1],"Google-Extended":["partial","*",5,1],"Applebot-Extended":["partial","*",5,1],"CCBot":["partial","*",5,1],"Bytespider":["partial","*",5,1],"Amazonbot":["partial","*",5,1],"meta-externalagent":["partial","*",5,1],"cohere-ai":["partial","*",5,1],"Diffbot":["partial","*",5,1],"ContextFetchBot":["partial","*",5,1]},"llms":{"status":"present","http":200,"bytes":33066},"home":{"status":"ok","http":200,"url":"https://www.kempinski.com/en/","html_tokens":51404,"markdown_tokens":253,"pruned_tokens":252}},
{"rank":5587,"domain":"typo3.org","origin":"https://typo3.org","robots":{"status":"ok","http":200,"url":"https://typo3.org/robots.txt","bytes":714,"groups":2},"bots":{"GPTBot":["partial","named",4,1],"ChatGPT-User":["partial","named",4,1],"OAI-SearchBot":["partial","named",4,1],"ClaudeBot":["partial","named",4,1],"Claude-User":["partial","named",4,1],"Claude-SearchBot":["partial","named",4,1],"anthropic-ai":["partial","*",4,1],"PerplexityBot":["partial","named",4,1],"Perplexity-User":["partial","named",4,1],"Google-Extended":["partial","named",4,1],"Applebot-Extended":["partial","named",4,1],"CCBot":["partial","named",4,1],"Bytespider":["partial","*",4,1],"Amazonbot":["partial","*",4,1],"meta-externalagent":["partial","named",4,1],"cohere-ai":["partial","*",4,1],"Diffbot":["partial","*",4,1],"ContextFetchBot":["partial","*",4,1]},"llms":{"status":"missing","http":404,"bytes":91833},"home":{"status":"ok","http":200,"url":"https://typo3.org/","html_tokens":31779,"markdown_tokens":543,"pruned_tokens":404}},
{"rank":5588,"domain":"duoduocdn.com","origin":"https://duoduocdn.com","robots":{"status":"unreachable","http":null,"url":"https://duoduocdn.com/robots.txt","bytes":0,"groups":0,"error":"cannot resolve duoduocdn.com"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":5589,"domain":"pinshape.com","origin":"https://pinshape.com","robots":{"status":"ok","http":200,"url":"https://pinshape.com/robots.txt","bytes":1248,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":447},"home":{"status":"ok","http":200,"url":"https://pinshape.com/","html_tokens":20416,"markdown_tokens":53,"pruned_tokens":53}},
{"rank":5590,"domain":"ccdi.gov.cn","origin":"https://www.ccdi.gov.cn","robots":{"status":"refused","http":420,"url":"https://www.ccdi.gov.cn/robots.txt","bytes":10379,"groups":0,"error":"HTTP 420"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":5591,"domain":"eventsair.com","origin":"https://eventsair.com","robots":{"status":"ok","http":200,"url":"https://www.eventsair.com/robots.txt","bytes":46,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"present","http":200,"bytes":3926},"home":{"status":"ok","http":200,"url":"https://www.eventsair.com/","html_tokens":79499,"markdown_tokens":594,"pruned_tokens":586}},
{"rank":5592,"domain":"vestacp.com","origin":"https://vestacp.com","robots":{"status":"ok","http":200,"url":"https://vestacp.com/robots.txt","bytes":39,"groups":1},"bots":{"GPTBot":["partial","*",1,0],"ChatGPT-User":["partial","*",1,0],"OAI-SearchBot":["partial","*",1,0],"ClaudeBot":["partial","*",1,0],"Claude-User":["partial","*",1,0],"Claude-SearchBot":["partial","*",1,0],"anthropic-ai":["partial","*",1,0],"PerplexityBot":["partial","*",1,0],"Perplexity-User":["partial","*",1,0],"Google-Extended":["partial","*",1,0],"Applebot-Extended":["partial","*",1,0],"CCBot":["partial","*",1,0],"Bytespider":["partial","*",1,0],"Amazonbot":["partial","*",1,0],"meta-externalagent":["partial","*",1,0],"cohere-ai":["partial","*",1,0],"Diffbot":["partial","*",1,0],"ContextFetchBot":["partial","*",1,0]},"llms":{"status":"missing","http":404,"bytes":58351},"home":{"status":"ok","http":200,"url":"https://vestacp.com/","html_tokens":23558,"markdown_tokens":358,"pruned_tokens":83}},
{"rank":5593,"domain":"guardian.ng","origin":"https://guardian.ng","robots":{"status":"refused","http":403,"url":"https://guardian.ng/robots.txt","bytes":5527,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":5594,"domain":"usajobs.gov","origin":"https://usajobs.gov","robots":{"status":"ok","http":200,"url":"https://www.usajobs.gov/robots.txt","bytes":150,"groups":1},"bots":{"GPTBot":["partial","*",4,0],"ChatGPT-User":["partial","*",4,0],"OAI-SearchBot":["partial","*",4,0],"ClaudeBot":["partial","*",4,0],"Claude-User":["partial","*",4,0],"Claude-SearchBot":["partial","*",4,0],"anthropic-ai":["partial","*",4,0],"PerplexityBot":["partial","*",4,0],"Perplexity-User":["partial","*",4,0],"Google-Extended":["partial","*",4,0],"Applebot-Extended":["partial","*",4,0],"CCBot":["partial","*",4,0],"Bytespider":["partial","*",4,0],"Amazonbot":["partial","*",4,0],"meta-externalagent":["partial","*",4,0],"cohere-ai":["partial","*",4,0],"Diffbot":["partial","*",4,0],"ContextFetchBot":["partial","*",4,0]},"llms":{"status":"missing","http":404,"bytes":27950},"home":{"status":"ok","http":200,"url":"https://www.usajobs.gov/","html_tokens":15752,"markdown_tokens":819,"pruned_tokens":75}},
{"rank":5595,"domain":"fishki.net","origin":"https://fishki.net","robots":{"status":"ok","http":200,"url":"https://fishki.net/robots.txt","bytes":2157,"groups":16},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["blocked","named",1,0],"OAI-SearchBot":["partial","*",69,2],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",69,2],"Claude-SearchBot":["partial","*",69,2],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["partial","*",69,2],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["partial","*",69,2],"meta-externalagent":["partial","*",69,2],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",69,2]},"llms":{"status":"missing","http":404,"bytes":196},"home":{"status":"ok","http":200,"url":"https://fishki.net/","html_tokens":186083,"markdown_tokens":1270,"pruned_tokens":165}},
{"rank":5596,"domain":"toyokeizai.net","origin":"https://toyokeizai.net","robots":{"status":"ok","http":200,"url":"https://toyokeizai.net/robots.txt","bytes":191,"groups":1},"bots":{"GPTBot":["partial","*",6,0],"ChatGPT-User":["partial","*",6,0],"OAI-SearchBot":["partial","*",6,0],"ClaudeBot":["partial","*",6,0],"Claude-User":["partial","*",6,0],"Claude-SearchBot":["partial","*",6,0],"anthropic-ai":["partial","*",6,0],"PerplexityBot":["partial","*",6,0],"Perplexity-User":["partial","*",6,0],"Google-Extended":["partial","*",6,0],"Applebot-Extended":["partial","*",6,0],"CCBot":["partial","*",6,0],"Bytespider":["partial","*",6,0],"Amazonbot":["partial","*",6,0],"meta-externalagent":["partial","*",6,0],"cohere-ai":["partial","*",6,0],"Diffbot":["partial","*",6,0],"ContextFetchBot":["partial","*",6,0]},"llms":{"status":"missing","http":404,"bytes":48656},"home":{"status":"ok","http":200,"url":"https://toyokeizai.net/","html_tokens":38546,"markdown_tokens":8,"pruned_tokens":8}},
{"rank":5597,"domain":"smartasset.com","origin":"https://smartasset.com","robots":{"status":"ok","http":200,"url":"https://smartasset.com/robots.txt","bytes":900,"groups":1},"bots":{"GPTBot":["partial","*",28,0],"ChatGPT-User":["partial","*",28,0],"OAI-SearchBot":["partial","*",28,0],"ClaudeBot":["partial","*",28,0],"Claude-User":["partial","*",28,0],"Claude-SearchBot":["partial","*",28,0],"anthropic-ai":["partial","*",28,0],"PerplexityBot":["partial","*",28,0],"Perplexity-User":["partial","*",28,0],"Google-Extended":["partial","*",28,0],"Applebot-Extended":["partial","*",28,0],"CCBot":["partial","*",28,0],"Bytespider":["partial","*",28,0],"Amazonbot":["partial","*",28,0],"meta-externalagent":["partial","*",28,0],"cohere-ai":["partial","*",28,0],"Diffbot":["partial","*",28,0],"ContextFetchBot":["partial","*",28,0]},"llms":{"status":"present","http":200,"bytes":7736},"home":{"status":"ok","http":200,"url":"https://smartasset.com/","html_tokens":38314,"markdown_tokens":212,"pruned_tokens":68}},
{"rank":5598,"domain":"litres.ru","origin":"https://litres.ru","robots":{"status":"ok","http":200,"url":"https://www.litres.ru/robots.txt","bytes":2420,"groups":2},"bots":{"GPTBot":["partial","*",22,28],"ChatGPT-User":["partial","*",22,28],"OAI-SearchBot":["partial","*",22,28],"ClaudeBot":["partial","*",22,28],"Claude-User":["partial","*",22,28],"Claude-SearchBot":["partial","*",22,28],"anthropic-ai":["partial","*",22,28],"PerplexityBot":["partial","*",22,28],"Perplexity-User":["partial","*",22,28],"Google-Extended":["partial","*",22,28],"Applebot-Extended":["partial","*",22,28],"CCBot":["partial","*",22,28],"Bytespider":["partial","*",22,28],"Amazonbot":["partial","*",22,28],"meta-externalagent":["partial","*",22,28],"cohere-ai":["partial","*",22,28],"Diffbot":["partial","*",22,28],"ContextFetchBot":["partial","*",22,28]},"llms":{"status":"missing","http":404,"bytes":42239},"home":{"status":"ok","http":200,"url":"https://www.litres.ru/","html_tokens":303373,"markdown_tokens":145,"pruned_tokens":145}},
{"rank":5599,"domain":"dreamwidth.org","origin":"https://dreamwidth.org","robots":{"status":"ok","http":200,"url":"https://www.dreamwidth.org/robots.txt","bytes":456,"groups":1},"bots":{"GPTBot":["partial","*",4,0],"ChatGPT-User":["partial","*",4,0],"OAI-SearchBot":["partial","*",4,0],"ClaudeBot":["partial","*",4,0],"Claude-User":["partial","*",4,0],"Claude-SearchBot":["partial","*",4,0],"anthropic-ai":["partial","*",4,0],"PerplexityBot":["partial","*",4,0],"Perplexity-User":["partial","*",4,0],"Google-Extended":["partial","*",4,0],"Applebot-Extended":["partial","*",4,0],"CCBot":["partial","*",4,0],"Bytespider":["partial","*",4,0],"Amazonbot":["partial","*",4,0],"meta-externalagent":["partial","*",4,0],"cohere-ai":["partial","*",4,0],"Diffbot":["partial","*",4,0],"ContextFetchBot":["partial","*",4,0]},"llms":{"status":"missing","http":404,"bytes":15026},"home":{"status":"ok","http":200,"url":"https://www.dreamwidth.org/","html_tokens":3605,"markdown_tokens":227,"pruned_tokens":120}},
{"rank":5600,"domain":"netgate.com","origin":"https://netgate.com","robots":{"status":"ok","http":200,"url":"https://www.netgate.com/robots.txt","bytes":210,"groups":1},"bots":{"GPTBot":["partial","*",7,0],"ChatGPT-User":["partial","*",7,0],"OAI-SearchBot":["partial","*",7,0],"ClaudeBot":["partial","*",7,0],"Claude-User":["partial","*",7,0],"Claude-SearchBot":["partial","*",7,0],"anthropic-ai":["partial","*",7,0],"PerplexityBot":["partial","*",7,0],"Perplexity-User":["partial","*",7,0],"Google-Extended":["partial","*",7,0],"Applebot-Extended":["partial","*",7,0],"CCBot":["partial","*",7,0],"Bytespider":["partial","*",7,0],"Amazonbot":["partial","*",7,0],"meta-externalagent":["partial","*",7,0],"cohere-ai":["partial","*",7,0],"Diffbot":["partial","*",7,0],"ContextFetchBot":["partial","*",7,0]},"llms":{"status":"missing","http":404,"bytes":99843},"home":{"status":"ok","http":200,"url":"https://www.netgate.com/","html_tokens":51686,"markdown_tokens":2306,"pruned_tokens":374}},
{"rank":5601,"domain":"king5.com","origin":"https://king5.com","robots":{"status":"ok","http":200,"url":"https://www.king5.com/robots.txt","bytes":447,"groups":2},"bots":{"GPTBot":["partial","*",10,0],"ChatGPT-User":["partial","*",10,0],"OAI-SearchBot":["partial","*",10,0],"ClaudeBot":["partial","*",10,0],"Claude-User":["partial","*",10,0],"Claude-SearchBot":["partial","*",10,0],"anthropic-ai":["partial","*",10,0],"PerplexityBot":["partial","*",10,0],"Perplexity-User":["partial","*",10,0],"Google-Extended":["partial","*",10,0],"Applebot-Extended":["partial","*",10,0],"CCBot":["partial","*",10,0],"Bytespider":["partial","*",10,0],"Amazonbot":["partial","*",10,0],"meta-externalagent":["partial","*",10,0],"cohere-ai":["partial","*",10,0],"Diffbot":["partial","*",10,0],"ContextFetchBot":["partial","*",10,0]},"llms":{"status":"missing","http":404,"bytes":6836},"home":{"status":"ok","http":200,"url":"https://www.king5.com/","html_tokens":103888,"markdown_tokens":1638,"pruned_tokens":168}},
{"rank":5602,"domain":"dailyherald.com","origin":"https://dailyherald.com","robots":{"status":"ok","http":200,"url":"https://www.dailyherald.com/robots.txt","bytes":14,"groups":1},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"missing","http":404,"bytes":88324},"home":{"status":"ok","http":200,"url":"https://www.dailyherald.com/","html_tokens":53,"markdown_tokens":0,"pruned_tokens":0}},
{"rank":5603,"domain":"visitscotland.com","origin":"https://visitscotland.com","robots":{"status":"ok","http":200,"url":"https://www.visitscotland.com/robots.txt","bytes":715,"groups":1},"bots":{"GPTBot":["partial","*",12,0],"ChatGPT-User":["partial","*",12,0],"OAI-SearchBot":["partial","*",12,0],"ClaudeBot":["partial","*",12,0],"Claude-User":["partial","*",12,0],"Claude-SearchBot":["partial","*",12,0],"anthropic-ai":["partial","*",12,0],"PerplexityBot":["partial","*",12,0],"Perplexity-User":["partial","*",12,0],"Google-Extended":["partial","*",12,0],"Applebot-Extended":["partial","*",12,0],"CCBot":["partial","*",12,0],"Bytespider":["partial","*",12,0],"Amazonbot":["partial","*",12,0],"meta-externalagent":["partial","*",12,0],"cohere-ai":["partial","*",12,0],"Diffbot":["partial","*",12,0],"ContextFetchBot":["partial","*",12,0]},"llms":{"status":"missing","http":404,"bytes":150310},"home":{"status":"ok","http":200,"url":"https://www.visitscotland.com/","html_tokens":57858,"markdown_tokens":357,"pruned_tokens":28}},
{"rank":5604,"domain":"bps.org.uk","origin":"https://bps.org.uk","robots":{"status":"ok","http":200,"url":"https://bps.org.uk/robots.txt","bytes":2676,"groups":32},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["allowed","*",0,1],"OAI-SearchBot":["allowed","*",0,1],"ClaudeBot":["blocked","named",1,0],"Claude-User":["allowed","*",0,1],"Claude-SearchBot":["allowed","*",0,1],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["allowed","*",0,1],"Perplexity-User":["allowed","*",0,1],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["allowed","*",0,1]},"llms":{"status":"missing","http":404,"bytes":135913},"home":{"status":"ok","http":200,"url":"https://www.bps.org.uk/","html_tokens":55263,"markdown_tokens":578,"pruned_tokens":124}},
{"rank":5605,"domain":"firefox.com.cn","origin":"http://firefox.com.cn","robots":{"status":"html","http":200,"url":"http://firefox.com.cn/robots.txt","bytes":812,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":5606,"domain":"legislation.gov.au","origin":"https://legislation.gov.au","robots":{"status":"ok","http":200,"url":"https://www.legislation.gov.au/robots.txt","bytes":210,"groups":1},"bots":{"GPTBot":["partial","*",1,0],"ChatGPT-User":["partial","*",1,0],"OAI-SearchBot":["partial","*",1,0],"ClaudeBot":["partial","*",1,0],"Claude-User":["partial","*",1,0],"Claude-SearchBot":["partial","*",1,0],"anthropic-ai":["partial","*",1,0],"PerplexityBot":["partial","*",1,0],"Perplexity-User":["partial","*",1,0],"Google-Extended":["partial","*",1,0],"Applebot-Extended":["partial","*",1,0],"CCBot":["partial","*",1,0],"Bytespider":["partial","*",1,0],"Amazonbot":["partial","*",1,0],"meta-externalagent":["partial","*",1,0],"cohere-ai":["partial","*",1,0],"Diffbot":["partial","*",1,0],"ContextFetchBot":["partial","*",1,0]},"llms":{"status":"html","http":200,"bytes":20392},"home":{"status":"ok","http":200,"url":"https://www.legislation.gov.au/","html_tokens":14931,"markdown_tokens":216,"pruned_tokens":104}},
{"rank":5607,"domain":"foxitsoftware.com","origin":"https://foxitsoftware.com","robots":{"status":"ok","http":200,"url":"https://www.foxit.com/robots.txt","bytes":555,"groups":7},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["partial","*",11,0],"OAI-SearchBot":["partial","*",11,0],"ClaudeBot":["partial","*",11,0],"Claude-User":["partial","*",11,0],"Claude-SearchBot":["partial","*",11,0],"anthropic-ai":["partial","*",11,0],"PerplexityBot":["partial","*",11,0],"Perplexity-User":["partial","*",11,0],"Google-Extended":["partial","*",11,0],"Applebot-Extended":["partial","*",11,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["partial","*",11,0],"cohere-ai":["partial","*",11,0],"Diffbot":["partial","*",11,0],"ContextFetchBot":["partial","*",11,0]},"llms":{"status":"present","http":200,"bytes":6170},"home":{"status":"ok","http":200,"url":"https://www.foxit.com/","html_tokens":360829,"markdown_tokens":261,"pruned_tokens":59}},
{"rank":5608,"domain":"postandcourier.com","origin":"https://postandcourier.com","robots":{"status":"ok","http":200,"url":"https://postandcourier.com/robots.txt","bytes":27080,"groups":30},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["blocked","named",1,0],"OAI-SearchBot":["blocked","named",1,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",600,0],"Claude-SearchBot":["partial","*",600,0],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["partial","*",600,0],"Google-Extended":["partial","*",600,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["partial","*",600,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",600,0]},"llms":{"status":"http_error","http":429,"bytes":64},"home":{"status":"ok","http":200,"url":"https://www.postandcourier.com/","html_tokens":124533,"markdown_tokens":98,"pruned_tokens":98}},
{"rank":5609,"domain":"wnd.com","origin":"https://wnd.com","robots":{"status":"ok","http":200,"url":"https://www.wnd.com/robots.txt","bytes":3885,"groups":59},"bots":{"GPTBot":["partial","*",45,4],"ChatGPT-User":["partial","*",45,4],"OAI-SearchBot":["partial","*",45,4],"ClaudeBot":["partial","*",45,4],"Claude-User":["partial","*",45,4],"Claude-SearchBot":["partial","*",45,4],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["partial","*",45,4],"Perplexity-User":["partial","*",45,4],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["partial","*",45,4],"CCBot":["blocked","named",1,0],"Bytespider":["partial","*",45,4],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["partial","*",45,4],"cohere-ai":["partial","*",45,4],"Diffbot":["partial","*",45,4],"ContextFetchBot":["partial","*",45,4]},"llms":{"status":"crawl_delay","error":"Crawl-delay 120 s"},"home":{"status":"crawl_delay","error":"Crawl-delay 120 s"}},
{"rank":5610,"domain":"bytedance.com","origin":"https://bytedance.com","robots":{"status":"ok","http":200,"url":"https://bytedance.com/robots.txt","bytes":37,"groups":1},"bots":{"GPTBot":["partial","*",1,0],"ChatGPT-User":["partial","*",1,0],"OAI-SearchBot":["partial","*",1,0],"ClaudeBot":["partial","*",1,0],"Claude-User":["partial","*",1,0],"Claude-SearchBot":["partial","*",1,0],"anthropic-ai":["partial","*",1,0],"PerplexityBot":["partial","*",1,0],"Perplexity-User":["partial","*",1,0],"Google-Extended":["partial","*",1,0],"Applebot-Extended":["partial","*",1,0],"CCBot":["partial","*",1,0],"Bytespider":["partial","*",1,0],"Amazonbot":["partial","*",1,0],"meta-externalagent":["partial","*",1,0],"cohere-ai":["partial","*",1,0],"Diffbot":["partial","*",1,0],"ContextFetchBot":["partial","*",1,0]},"llms":{"status":"html","http":200,"bytes":201307},"home":{"status":"ok","http":200,"url":"https://www.bytedance.com/","html_tokens":50317,"markdown_tokens":493,"pruned_tokens":444}},
{"rank":5611,"domain":"oebb.at","origin":"https://oebb.at","robots":{"status":"missing","http":404,"url":"https://www.oebb.at/de/fehlerseiten/404.html","bytes":138735,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":138735},"home":{"status":"ok","http":200,"url":"https://www.oebb.at/","html_tokens":51587,"markdown_tokens":42,"pruned_tokens":42}},
{"rank":5612,"domain":"texasmonthly.com","origin":"https://texasmonthly.com","robots":{"status":"ok","http":200,"url":"https://www.texasmonthly.com/robots.txt","bytes":1021,"groups":2},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["blocked","named",1,0],"OAI-SearchBot":["blocked","named",1,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",5,1],"Claude-SearchBot":["partial","*",5,1],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["partial","*",5,1],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",5,1]},"llms":{"status":"http_error","http":403,"bytes":5569},"home":{"status":"http_error","http":403,"url":"https://texasmonthly.com/"}},
{"rank":5613,"domain":"purevolume.com","origin":"https://purevolume.com","robots":{"status":"ok","http":200,"url":"https://www.purevolume.com/robots.txt","bytes":525,"groups":6},"bots":{"GPTBot":["partial","*",13,1],"ChatGPT-User":["partial","*",13,1],"OAI-SearchBot":["partial","*",13,1],"ClaudeBot":["partial","*",13,1],"Claude-User":["partial","*",13,1],"Claude-SearchBot":["partial","*",13,1],"anthropic-ai":["partial","*",13,1],"PerplexityBot":["partial","*",13,1],"Perplexity-User":["partial","*",13,1],"Google-Extended":["partial","*",13,1],"Applebot-Extended":["partial","*",13,1],"CCBot":["partial","*",13,1],"Bytespider":["partial","*",13,1],"Amazonbot":["partial","*",13,1],"meta-externalagent":["partial","*",13,1],"cohere-ai":["partial","*",13,1],"Diffbot":["partial","*",13,1],"ContextFetchBot":["partial","*",13,1]},"llms":{"status":"missing","http":404,"bytes":60},"home":{"status":"ok","http":200,"url":"https://www.purevolume.com/","html_tokens":44306,"markdown_tokens":281,"pruned_tokens":173}},
{"rank":5614,"domain":"depositfiles.com","origin":"https://depositfiles.com","robots":{"status":"ok","http":200,"url":"https://depositfiles.com/robots.txt","bytes":26,"groups":1},"bots":{"GPTBot":["blocked","*",1,0],"ChatGPT-User":["blocked","*",1,0],"OAI-SearchBot":["blocked","*",1,0],"ClaudeBot":["blocked","*",1,0],"Claude-User":["blocked","*",1,0],"Claude-SearchBot":["blocked","*",1,0],"anthropic-ai":["blocked","*",1,0],"PerplexityBot":["blocked","*",1,0],"Perplexity-User":["blocked","*",1,0],"Google-Extended":["blocked","*",1,0],"Applebot-Extended":["blocked","*",1,0],"CCBot":["blocked","*",1,0],"Bytespider":["blocked","*",1,0],"Amazonbot":["blocked","*",1,0],"meta-externalagent":["blocked","*",1,0],"cohere-ai":["blocked","*",1,0],"Diffbot":["blocked","*",1,0],"ContextFetchBot":["blocked","*",1,0]},"llms":{"status":"disallowed"},"home":{"status":"disallowed"}},
{"rank":5615,"domain":"vietnamnet.vn","origin":"https://vietnamnet.vn","robots":{"status":"ok","http":200,"url":"https://vietnamnet.vn/robots.txt","bytes":329,"groups":1},"bots":{"GPTBot":["partial","*",3,1],"ChatGPT-User":["partial","*",3,1],"OAI-SearchBot":["partial","*",3,1],"ClaudeBot":["partial","*",3,1],"Claude-User":["partial","*",3,1],"Claude-SearchBot":["partial","*",3,1],"anthropic-ai":["partial","*",3,1],"PerplexityBot":["partial","*",3,1],"Perplexity-User":["partial","*",3,1],"Google-Extended":["partial","*",3,1],"Applebot-Extended":["partial","*",3,1],"CCBot":["partial","*",3,1],"Bytespider":["partial","*",3,1],"Amazonbot":["partial","*",3,1],"meta-externalagent":["partial","*",3,1],"cohere-ai":["partial","*",3,1],"Diffbot":["partial","*",3,1],"ContextFetchBot":["partial","*",3,1]},"llms":{"status":"present","http":200,"bytes":25298},"home":{"status":"ok","http":200,"url":"https://vietnamnet.vn/","html_tokens":60213,"markdown_tokens":158,"pruned_tokens":126}},
{"rank":5616,"domain":"simon.com","origin":"https://simon.com","robots":{"status":"ok","http":200,"url":"https://www.simon.com/robots.txt","bytes":7267,"groups":26},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["allowed","named",0,2],"OAI-SearchBot":["allowed","named",0,1],"ClaudeBot":["blocked","named",1,0],"Claude-User":["allowed","named",0,1],"Claude-SearchBot":["allowed","named",0,1],"anthropic-ai":["allowed","named",0,1],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["blocked","named",1,0],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["allowed","named",0,1],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["allowed","named",1,1],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",12,0]},"llms":{"status":"http_error","http":307,"bytes":5712},"home":{"status":"ok","http":307,"url":"https://www.simon.com/","html_tokens":1428,"markdown_tokens":0,"pruned_tokens":0}},
{"rank":5617,"domain":"publons.com","origin":"https://publons.com","robots":{"status":"ok","http":200,"url":"https://publons.com/robots.txt","bytes":171,"groups":2},"bots":{"GPTBot":["partial","*",1,1],"ChatGPT-User":["partial","*",1,1],"OAI-SearchBot":["partial","*",1,1],"ClaudeBot":["partial","*",1,1],"Claude-User":["partial","*",1,1],"Claude-SearchBot":["partial","*",1,1],"anthropic-ai":["partial","*",1,1],"PerplexityBot":["partial","*",1,1],"Perplexity-User":["partial","*",1,1],"Google-Extended":["partial","*",1,1],"Applebot-Extended":["partial","*",1,1],"CCBot":["partial","*",1,1],"Bytespider":["partial","*",1,1],"Amazonbot":["partial","*",1,1],"meta-externalagent":["partial","*",1,1],"cohere-ai":["partial","*",1,1],"Diffbot":["partial","*",1,1],"ContextFetchBot":["partial","*",1,1]},"llms":{"status":"missing","http":404,"bytes":2277},"home":{"status":"disallowed","error":"robots.txt of access.clarivate.com not readable (html)"}},
{"rank":5618,"domain":"framer.com","origin":"https://framer.com","robots":{"status":"ok","http":200,"url":"https://framer.com/robots.txt","bytes":29,"groups":1},"bots":{"GPTBot":["partial","*",1,0],"ChatGPT-User":["partial","*",1,0],"OAI-SearchBot":["partial","*",1,0],"ClaudeBot":["partial","*",1,0],"Claude-User":["partial","*",1,0],"Claude-SearchBot":["partial","*",1,0],"anthropic-ai":["partial","*",1,0],"PerplexityBot":["partial","*",1,0],"Perplexity-User":["partial","*",1,0],"Google-Extended":["partial","*",1,0],"Applebot-Extended":["partial","*",1,0],"CCBot":["partial","*",1,0],"Bytespider":["partial","*",1,0],"Amazonbot":["partial","*",1,0],"meta-externalagent":["partial","*",1,0],"cohere-ai":["partial","*",1,0],"Diffbot":["partial","*",1,0],"ContextFetchBot":["partial","*",1,0]},"llms":{"status":"present","http":200,"bytes":9636},"home":{"status":"ok","http":200,"url":"https://www.framer.com/","html_tokens":477320,"markdown_tokens":563,"pruned_tokens":172}},
{"rank":5619,"domain":"lendingtree.com","origin":"https://lendingtree.com","robots":{"status":"ok","http":200,"url":"https://www.lendingtree.com/robots.txt","bytes":606,"groups":2},"bots":{"GPTBot":["partial","named",6,0],"ChatGPT-User":["partial","named",6,0],"OAI-SearchBot":["partial","named",6,0],"ClaudeBot":["partial","named",6,0],"Claude-User":["partial","named",6,0],"Claude-SearchBot":["partial","named",6,0],"anthropic-ai":["partial","*",5,1],"PerplexityBot":["partial","named",6,0],"Perplexity-User":["partial","*",5,1],"Google-Extended":["partial","named",6,0],"Applebot-Extended":["partial","*",5,1],"CCBot":["partial","named",6,0],"Bytespider":["partial","*",5,1],"Amazonbot":["partial","*",5,1],"meta-externalagent":["partial","*",5,1],"cohere-ai":["partial","*",5,1],"Diffbot":["partial","*",5,1],"ContextFetchBot":["partial","*",5,1]},"llms":{"status":"present","http":200,"bytes":12189},"home":{"status":"ok","http":200,"url":"https://www.lendingtree.com/","html_tokens":153247,"markdown_tokens":623,"pruned_tokens":253}},
{"rank":5620,"domain":"ghacks.net","origin":"https://ghacks.net","robots":{"status":"ok","http":200,"url":"https://www.ghacks.net/robots.txt","bytes":393,"groups":1},"bots":{"GPTBot":["partial","*",15,1],"ChatGPT-User":["partial","*",15,1],"OAI-SearchBot":["partial","*",15,1],"ClaudeBot":["partial","*",15,1],"Claude-User":["partial","*",15,1],"Claude-SearchBot":["partial","*",15,1],"anthropic-ai":["partial","*",15,1],"PerplexityBot":["partial","*",15,1],"Perplexity-User":["partial","*",15,1],"Google-Extended":["partial","*",15,1],"Applebot-Extended":["partial","*",15,1],"CCBot":["partial","*",15,1],"Bytespider":["partial","*",15,1],"Amazonbot":["partial","*",15,1],"meta-externalagent":["partial","*",15,1],"cohere-ai":["partial","*",15,1],"Diffbot":["partial","*",15,1],"ContextFetchBot":["partial","*",15,1]},"llms":{"status":"missing","http":404,"bytes":29842},"home":{"status":"http_error","http":403,"url":"https://ghacks.net/"}},
{"rank":5621,"domain":"starwoodhotels.com","origin":"https://starwoodhotels.com","robots":{"status":"ok","http":200,"url":"https://www.starwoodhotels.com/robots.txt","bytes":2181,"groups":1},"bots":{"GPTBot":["partial","*",38,18],"ChatGPT-User":["partial","*",38,18],"OAI-SearchBot":["partial","*",38,18],"ClaudeBot":["partial","*",38,18],"Claude-User":["partial","*",38,18],"Claude-SearchBot":["partial","*",38,18],"anthropic-ai":["partial","*",38,18],"PerplexityBot":["partial","*",38,18],"Perplexity-User":["partial","*",38,18],"Google-Extended":["partial","*",38,18],"Applebot-Extended":["partial","*",38,18],"CCBot":["partial","*",38,18],"Bytespider":["partial","*",38,18],"Amazonbot":["partial","*",38,18],"meta-externalagent":["partial","*",38,18],"cohere-ai":["partial","*",38,18],"Diffbot":["partial","*",38,18],"ContextFetchBot":["partial","*",38,18]},"llms":{"status":"present","http":200,"bytes":1902},"home":{"status":"ok","http":200,"url":"https://www.starwoodhotels.com/","html_tokens":145768,"markdown_tokens":2373,"pruned_tokens":120}},
{"rank":5622,"domain":"mca.gov.cn","origin":"https://mca.gov.cn","robots":{"status":"timeout","http":null,"url":"https://mca.gov.cn/robots.txt","bytes":0,"groups":0,"error":"timed out after 10 s"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":5623,"domain":"habrahabr.ru","origin":"https://habrahabr.ru","robots":{"status":"ok","http":200,"url":"https://habr.com/robots.txt","bytes":1624,"groups":4},"bots":{"GPTBot":["partial","*",12,0],"ChatGPT-User":["partial","*",12,0],"OAI-SearchBot":["partial","*",12,0],"ClaudeBot":["partial","*",12,0],"Claude-User":["partial","*",12,0],"Claude-SearchBot":["partial","*",12,0],"anthropic-ai":["partial","*",12,0],"PerplexityBot":["partial","*",12,0],"Perplexity-User":["partial","*",12,0],"Google-Extended":["partial","*",12,0],"Applebot-Extended":["partial","*",12,0],"CCBot":["partial","*",12,0],"Bytespider":["partial","*",12,0],"Amazonbot":["partial","*",12,0],"meta-externalagent":["partial","*",12,0],"cohere-ai":["partial","*",12,0],"Diffbot":["partial","*",12,0],"ContextFetchBot":["partial","*",12,0]},"llms":{"status":"timeout","error":"timed out after 10 s"},"home":{"status":"timeout","error":"timed out after 15 s"}},
{"rank":5624,"domain":"trulia.com","origin":"https://trulia.com","robots":{"status":"ok","http":200,"url":"https://www.trulia.com/robots.txt","bytes":10006,"groups":2},"bots":{"GPTBot":["partial","*",138,6],"ChatGPT-User":["partial","*",138,6],"OAI-SearchBot":["partial","*",138,6],"ClaudeBot":["partial","*",138,6],"Claude-User":["partial","*",138,6],"Claude-SearchBot":["partial","*",138,6],"anthropic-ai":["partial","*",138,6],"PerplexityBot":["partial","*",138,6],"Perplexity-User":["partial","*",138,6],"Google-Extended":["partial","*",138,6],"Applebot-Extended":["partial","*",138,6],"CCBot":["partial","*",138,6],"Bytespider":["partial","*",138,6],"Amazonbot":["partial","*",138,6],"meta-externalagent":["partial","*",138,6],"cohere-ai":["partial","*",138,6],"Diffbot":["partial","*",138,6],"ContextFetchBot":["partial","*",138,6]},"llms":{"status":"present","http":200,"bytes":6461},"home":{"status":"ok","http":200,"url":"https://www.trulia.com/","html_tokens":121406,"markdown_tokens":86,"pruned_tokens":15}},
{"rank":5625,"domain":"aninews.in","origin":"https://aninews.in","robots":{"status":"ok","http":200,"url":"https://aninews.in/robots.txt","bytes":212,"groups":2},"bots":{"GPTBot":["partial","*",4,0],"ChatGPT-User":["partial","*",4,0],"OAI-SearchBot":["partial","*",4,0],"ClaudeBot":["partial","*",4,0],"Claude-User":["partial","*",4,0],"Claude-SearchBot":["partial","*",4,0],"anthropic-ai":["partial","*",4,0],"PerplexityBot":["partial","*",4,0],"Perplexity-User":["partial","*",4,0],"Google-Extended":["partial","*",4,0],"Applebot-Extended":["partial","*",4,0],"CCBot":["partial","*",4,0],"Bytespider":["partial","*",4,0],"Amazonbot":["partial","*",4,0],"meta-externalagent":["partial","*",4,0],"cohere-ai":["partial","*",4,0],"Diffbot":["partial","*",4,0],"ContextFetchBot":["partial","*",4,0]},"llms":{"status":"http_error","http":403,"bytes":479},"home":{"status":"http_error","http":403,"url":"https://aninews.in/"}},
{"rank":5626,"domain":"trust.org","origin":"https://www.trust.org","robots":{"status":"ok","http":200,"url":"https://www.trust.org/robots.txt","bytes":111,"groups":1},"bots":{"GPTBot":["partial","*",1,1],"ChatGPT-User":["partial","*",1,1],"OAI-SearchBot":["partial","*",1,1],"ClaudeBot":["partial","*",1,1],"Claude-User":["partial","*",1,1],"Claude-SearchBot":["partial","*",1,1],"anthropic-ai":["partial","*",1,1],"PerplexityBot":["partial","*",1,1],"Perplexity-User":["partial","*",1,1],"Google-Extended":["partial","*",1,1],"Applebot-Extended":["partial","*",1,1],"CCBot":["partial","*",1,1],"Bytespider":["partial","*",1,1],"Amazonbot":["partial","*",1,1],"meta-externalagent":["partial","*",1,1],"cohere-ai":["partial","*",1,1],"Diffbot":["partial","*",1,1],"ContextFetchBot":["partial","*",1,1]},"llms":{"status":"http_error","http":403,"bytes":5566},"home":{"status":"http_error","http":403,"url":"https://www.trust.org/"}},
{"rank":5627,"domain":"moddb.com","origin":"https://moddb.com","robots":{"status":"ok","http":200,"url":"https://www.moddb.com/robots.txt","bytes":3130,"groups":3},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["partial","*",53,0],"OAI-SearchBot":["partial","*",53,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",53,0],"Claude-SearchBot":["partial","*",53,0],"anthropic-ai":["partial","*",53,0],"PerplexityBot":["partial","*",53,0],"Perplexity-User":["partial","*",53,0],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["partial","*",53,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["partial","*",53,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",53,0]},"llms":{"status":"missing","http":404,"bytes":4288},"home":{"status":"http_error","http":403,"url":"https://moddb.com/"}},
{"rank":5628,"domain":"middleeastmonitor.com","origin":"https://middleeastmonitor.com","robots":{"status":"ok","http":200,"url":"https://www.middleeastmonitor.com/robots.txt","bytes":182,"groups":1},"bots":{"GPTBot":["partial","*",1,1],"ChatGPT-User":["partial","*",1,1],"OAI-SearchBot":["partial","*",1,1],"ClaudeBot":["partial","*",1,1],"Claude-User":["partial","*",1,1],"Claude-SearchBot":["partial","*",1,1],"anthropic-ai":["partial","*",1,1],"PerplexityBot":["partial","*",1,1],"Perplexity-User":["partial","*",1,1],"Google-Extended":["partial","*",1,1],"Applebot-Extended":["partial","*",1,1],"CCBot":["partial","*",1,1],"Bytespider":["partial","*",1,1],"Amazonbot":["partial","*",1,1],"meta-externalagent":["partial","*",1,1],"cohere-ai":["partial","*",1,1],"Diffbot":["partial","*",1,1],"ContextFetchBot":["partial","*",1,1]},"llms":{"status":"missing","http":404,"bytes":89296},"home":{"status":"ok","http":200,"url":"https://www.middleeastmonitor.com/","html_tokens":55808,"markdown_tokens":591,"pruned_tokens":171}},
{"rank":5629,"domain":"smore.com","origin":"https://smore.com","robots":{"status":"ok","http":200,"url":"https://www.smore.com/robots.txt","bytes":301,"groups":1},"bots":{"GPTBot":["partial","*",6,1],"ChatGPT-User":["partial","*",6,1],"OAI-SearchBot":["partial","*",6,1],"ClaudeBot":["partial","*",6,1],"Claude-User":["partial","*",6,1],"Claude-SearchBot":["partial","*",6,1],"anthropic-ai":["partial","*",6,1],"PerplexityBot":["partial","*",6,1],"Perplexity-User":["partial","*",6,1],"Google-Extended":["partial","*",6,1],"Applebot-Extended":["partial","*",6,1],"CCBot":["partial","*",6,1],"Bytespider":["partial","*",6,1],"Amazonbot":["partial","*",6,1],"meta-externalagent":["partial","*",6,1],"cohere-ai":["partial","*",6,1],"Diffbot":["partial","*",6,1],"ContextFetchBot":["partial","*",6,1]},"llms":{"status":"http_error","http":403,"bytes":5540},"home":{"status":"http_error","http":403,"url":"https://smore.com/"}},
{"rank":5630,"domain":"mwcbarcelona.com","origin":"https://mwcbarcelona.com","robots":{"status":"ok","http":200,"url":"https://www.mwcbarcelona.com/robots.txt","bytes":287,"groups":1},"bots":{"GPTBot":["partial","*",5,0],"ChatGPT-User":["partial","*",5,0],"OAI-SearchBot":["partial","*",5,0],"ClaudeBot":["partial","*",5,0],"Claude-User":["partial","*",5,0],"Claude-SearchBot":["partial","*",5,0],"anthropic-ai":["partial","*",5,0],"PerplexityBot":["partial","*",5,0],"Perplexity-User":["partial","*",5,0],"Google-Extended":["partial","*",5,0],"Applebot-Extended":["partial","*",5,0],"CCBot":["partial","*",5,0],"Bytespider":["partial","*",5,0],"Amazonbot":["partial","*",5,0],"meta-externalagent":["partial","*",5,0],"cohere-ai":["partial","*",5,0],"Diffbot":["partial","*",5,0],"ContextFetchBot":["partial","*",5,0]},"llms":{"status":"missing","http":404,"bytes":97226},"home":{"status":"ok","http":200,"url":"https://www.mwcbarcelona.com/","html_tokens":44554,"markdown_tokens":67,"pruned_tokens":67}},
{"rank":5631,"domain":"snyk.io","origin":"https://snyk.io","robots":{"status":"ok","http":200,"url":"https://snyk.io/robots.txt","bytes":968,"groups":4},"bots":{"GPTBot":["partial","*",8,1],"ChatGPT-User":["partial","*",8,1],"OAI-SearchBot":["partial","*",8,1],"ClaudeBot":["partial","*",8,1],"Claude-User":["partial","*",8,1],"Claude-SearchBot":["partial","*",8,1],"anthropic-ai":["partial","*",8,1],"PerplexityBot":["partial","*",8,1],"Perplexity-User":["partial","*",8,1],"Google-Extended":["partial","*",8,1],"Applebot-Extended":["partial","*",8,1],"CCBot":["partial","*",8,1],"Bytespider":["partial","*",8,1],"Amazonbot":["partial","*",8,1],"meta-externalagent":["partial","*",8,1],"cohere-ai":["partial","*",8,1],"Diffbot":["partial","*",8,1],"ContextFetchBot":["partial","*",8,1]},"llms":{"status":"present","http":200,"bytes":12767},"home":{"status":"ok","http":200,"url":"https://snyk.io/","html_tokens":90378,"markdown_tokens":772,"pruned_tokens":175}},
{"rank":5632,"domain":"ringcentral.com","origin":"https://ringcentral.com","robots":{"status":"ok","http":200,"url":"https://www.ringcentral.com/robots.txt","bytes":3573,"groups":9},"bots":{"GPTBot":["partial","*",68,3],"ChatGPT-User":["allowed","named",0,1],"OAI-SearchBot":["allowed","named",0,1],"ClaudeBot":["partial","*",68,3],"Claude-User":["allowed","named",0,1],"Claude-SearchBot":["allowed","named",0,1],"anthropic-ai":["partial","*",68,3],"PerplexityBot":["allowed","named",0,1],"Perplexity-User":["allowed","named",0,1],"Google-Extended":["allowed","named",0,1],"Applebot-Extended":["allowed","named",0,1],"CCBot":["partial","*",68,3],"Bytespider":["partial","*",68,3],"Amazonbot":["partial","*",68,3],"meta-externalagent":["partial","*",68,3],"cohere-ai":["partial","*",68,3],"Diffbot":["partial","*",68,3],"ContextFetchBot":["partial","*",68,3]},"llms":{"status":"present","http":200,"bytes":2874},"home":{"status":"ok","http":200,"url":"https://www.ringcentral.com/","html_tokens":306754,"markdown_tokens":726,"pruned_tokens":70}},
{"rank":5633,"domain":"opensocietyfoundations.org","origin":"https://opensocietyfoundations.org","robots":{"status":"ok","http":200,"url":"https://www.opensocietyfoundations.org/robots.txt","bytes":433,"groups":3},"bots":{"GPTBot":["partial","*",10,0],"ChatGPT-User":["partial","*",10,0],"OAI-SearchBot":["partial","*",10,0],"ClaudeBot":["partial","*",10,0],"Claude-User":["partial","*",10,0],"Claude-SearchBot":["partial","*",10,0],"anthropic-ai":["partial","*",10,0],"PerplexityBot":["partial","*",10,0],"Perplexity-User":["partial","*",10,0],"Google-Extended":["partial","*",10,0],"Applebot-Extended":["partial","*",10,0],"CCBot":["partial","*",10,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["partial","*",10,0],"meta-externalagent":["partial","*",10,0],"cohere-ai":["partial","*",10,0],"Diffbot":["partial","*",10,0],"ContextFetchBot":["partial","*",10,0]},"llms":{"status":"missing","http":404,"bytes":26406},"home":{"status":"ok","http":200,"url":"https://www.opensocietyfoundations.org/","html_tokens":44508,"markdown_tokens":335,"pruned_tokens":159}},
{"rank":5634,"domain":"duden.de","origin":"https://duden.de","robots":{"status":"ok","http":200,"url":"https://www.duden.de/robots.txt","bytes":4479,"groups":51},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["partial","*",65,0],"OAI-SearchBot":["partial","*",65,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",65,0],"Claude-SearchBot":["partial","*",65,0],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["partial","*",65,0],"Perplexity-User":["partial","*",65,0],"Google-Extended":["partial","*",65,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["partial","*",65,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",65,0]},"llms":{"status":"http_error","http":403,"bytes":32003},"home":{"status":"ok","http":200,"url":"https://www.duden.de/","html_tokens":28771,"markdown_tokens":49,"pruned_tokens":49}},
{"rank":5635,"domain":"liebherr.com","origin":"https://liebherr.com","robots":{"status":"ok","http":200,"url":"https://www.liebherr.com/robots.txt","bytes":321,"groups":1},"bots":{"GPTBot":["partial","*",8,0],"ChatGPT-User":["partial","*",8,0],"OAI-SearchBot":["partial","*",8,0],"ClaudeBot":["partial","*",8,0],"Claude-User":["partial","*",8,0],"Claude-SearchBot":["partial","*",8,0],"anthropic-ai":["partial","*",8,0],"PerplexityBot":["partial","*",8,0],"Perplexity-User":["partial","*",8,0],"Google-Extended":["partial","*",8,0],"Applebot-Extended":["partial","*",8,0],"CCBot":["partial","*",8,0],"Bytespider":["partial","*",8,0],"Amazonbot":["partial","*",8,0],"meta-externalagent":["partial","*",8,0],"cohere-ai":["partial","*",8,0],"Diffbot":["partial","*",8,0],"ContextFetchBot":["partial","*",8,0]},"llms":{"status":"missing","http":404,"bytes":0},"home":{"status":"ok","http":200,"url":"https://www.liebherr.com/en-us/group/start-page-3782471","html_tokens":79738,"markdown_tokens":123,"pruned_tokens":123}},
{"rank":5636,"domain":"kuronekoyamato.co.jp","origin":"https://www.kuronekoyamato.co.jp","robots":{"status":"missing","http":404,"url":"https://www.kuronekoyamato.co.jp/robots.txt","bytes":25061,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":25061},"home":{"status":"ok","http":200,"url":"https://www.kuronekoyamato.co.jp/","html_tokens":1906,"markdown_tokens":23,"pruned_tokens":23}},
{"rank":5637,"domain":"insideclimatenews.org","origin":"https://insideclimatenews.org","robots":{"status":"ok","http":200,"url":"https://insideclimatenews.org/robots.txt","bytes":190,"groups":1},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"http_error","http":403,"bytes":103},"home":{"status":"http_error","http":403,"url":"https://insideclimatenews.org/"}},
{"rank":5638,"domain":"6abc.com","origin":"https://6abc.com","robots":{"status":"ok","http":200,"url":"https://6abc.com/robots.txt","bytes":1181,"groups":26},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["blocked","named",1,0],"OAI-SearchBot":["partial","*",8,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",8,0],"Claude-SearchBot":["partial","*",8,0],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["partial","*",8,0],"Perplexity-User":["partial","*",8,0],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["partial","*",8,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",8,0]},"llms":{"status":"html","http":200,"bytes":73812},"home":{"status":"ok","http":200,"url":"https://6abc.com/","html_tokens":125302,"markdown_tokens":108,"pruned_tokens":36}},
{"rank":5639,"domain":"nahb.org","origin":"https://nahb.org","robots":{"status":"ok","http":200,"url":"https://www.nahb.org/robots","bytes":77,"groups":1},"bots":{"GPTBot":["partial","*",1,0],"ChatGPT-User":["partial","*",1,0],"OAI-SearchBot":["partial","*",1,0],"ClaudeBot":["partial","*",1,0],"Claude-User":["partial","*",1,0],"Claude-SearchBot":["partial","*",1,0],"anthropic-ai":["partial","*",1,0],"PerplexityBot":["partial","*",1,0],"Perplexity-User":["partial","*",1,0],"Google-Extended":["partial","*",1,0],"Applebot-Extended":["partial","*",1,0],"CCBot":["partial","*",1,0],"Bytespider":["partial","*",1,0],"Amazonbot":["partial","*",1,0],"meta-externalagent":["partial","*",1,0],"cohere-ai":["partial","*",1,0],"Diffbot":["partial","*",1,0],"ContextFetchBot":["partial","*",1,0]},"llms":{"status":"missing","http":404,"bytes":1245},"home":{"status":"ok","http":200,"url":"https://www.nahb.org/","html_tokens":22180,"markdown_tokens":69,"pruned_tokens":69}},
{"rank":5640,"domain":"abc.xyz","origin":"https://abc.xyz","robots":{"status":"ok","http":200,"url":"https://abc.xyz/robots.txt","bytes":207,"groups":4},"bots":{"GPTBot":["allowed","*",0,1],"ChatGPT-User":["allowed","*",0,1],"OAI-SearchBot":["allowed","*",0,1],"ClaudeBot":["allowed","*",0,1],"Claude-User":["allowed","*",0,1],"Claude-SearchBot":["allowed","*",0,1],"anthropic-ai":["allowed","*",0,1],"PerplexityBot":["allowed","*",0,1],"Perplexity-User":["allowed","*",0,1],"Google-Extended":["allowed","*",0,1],"Applebot-Extended":["allowed","*",0,1],"CCBot":["allowed","*",0,1],"Bytespider":["allowed","*",0,1],"Amazonbot":["allowed","*",0,1],"meta-externalagent":["allowed","*",0,1],"cohere-ai":["allowed","*",0,1],"Diffbot":["allowed","*",0,1],"ContextFetchBot":["allowed","*",0,1]},"llms":{"status":"missing","http":404,"bytes":1286},"home":{"status":"http_error","http":403,"url":"https://abc.xyz/"}},
{"rank":5641,"domain":"azlyrics.com","origin":"https://azlyrics.com","robots":{"status":"ok","http":200,"url":"https://www.azlyrics.com/robots.txt","bytes":308,"groups":3},"bots":{"GPTBot":["partial","*",4,1],"ChatGPT-User":["partial","*",4,1],"OAI-SearchBot":["partial","*",4,1],"ClaudeBot":["partial","*",4,1],"Claude-User":["partial","*",4,1],"Claude-SearchBot":["partial","*",4,1],"anthropic-ai":["partial","*",4,1],"PerplexityBot":["partial","*",4,1],"Perplexity-User":["partial","*",4,1],"Google-Extended":["partial","*",4,1],"Applebot-Extended":["partial","*",4,1],"CCBot":["partial","*",4,1],"Bytespider":["partial","*",4,1],"Amazonbot":["partial","*",4,1],"meta-externalagent":["partial","*",4,1],"cohere-ai":["partial","*",4,1],"Diffbot":["partial","*",4,1],"ContextFetchBot":["partial","*",4,1]},"llms":{"status":"missing","http":404,"bytes":78846},"home":{"status":"ok","http":200,"url":"https://www.azlyrics.com/","html_tokens":19712,"markdown_tokens":50,"pruned_tokens":50}},
{"rank":5642,"domain":"jyu.fi","origin":"https://jyu.fi","robots":{"status":"ok","http":200,"url":"https://www.jyu.fi/robots.txt","bytes":2866,"groups":14},"bots":{"GPTBot":["partial","*",22,18],"ChatGPT-User":["partial","*",22,18],"OAI-SearchBot":["partial","*",22,18],"ClaudeBot":["partial","*",22,18],"Claude-User":["partial","*",22,18],"Claude-SearchBot":["partial","*",22,18],"anthropic-ai":["partial","*",22,18],"PerplexityBot":["partial","*",22,18],"Perplexity-User":["partial","*",22,18],"Google-Extended":["partial","*",22,18],"Applebot-Extended":["partial","*",22,18],"CCBot":["blocked","named",1,0],"Bytespider":["partial","*",22,18],"Amazonbot":["partial","*",22,18],"meta-externalagent":["partial","*",22,18],"cohere-ai":["partial","*",22,18],"Diffbot":["partial","*",22,18],"ContextFetchBot":["partial","*",22,18]},"llms":{"status":"http_error","http":403,"bytes":382},"home":{"status":"ok","http":200,"url":"https://www.jyu.fi/fi","html_tokens":38110,"markdown_tokens":150,"pruned_tokens":150}},
{"rank":5643,"domain":"amzn.asia","origin":"https://amzn.asia","robots":{"status":"missing","http":404,"url":"https://amzn.asia/robots.txt","bytes":28,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":28},"home":{"status":"http_error","http":404,"url":"https://amzn.asia/"}},
{"rank":5644,"domain":"thecanadianencyclopedia.ca","origin":"https://thecanadianencyclopedia.ca","robots":{"status":"refused","http":403,"url":"https://thecanadianencyclopedia.ca/robots.txt","bytes":919,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":5645,"domain":"philpapers.org","origin":"https://philpapers.org","robots":{"status":"ok","http":200,"url":"https://philpapers.org/robots.txt","bytes":1250,"groups":9},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["partial","*",17,0],"OAI-SearchBot":["partial","*",17,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",17,0],"Claude-SearchBot":["partial","*",17,0],"anthropic-ai":["partial","*",17,0],"PerplexityBot":["partial","*",17,0],"Perplexity-User":["partial","*",17,0],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["partial","*",17,0],"Diffbot":["partial","*",17,0],"ContextFetchBot":["partial","*",17,0]},"llms":{"status":"missing","http":404,"bytes":77},"home":{"status":"http_error","http":403,"url":"https://philpapers.org/"}},
{"rank":5646,"domain":"bioone.org","origin":"https://bioone.org","robots":{"status":"html","http":200,"url":"https://bioone.org/robots.txt","bytes":1161,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":5647,"domain":"elibrary.ru","origin":"https://www.elibrary.ru","robots":{"status":"ok","http":200,"url":"https://www.elibrary.ru/robots.txt","bytes":941,"groups":1},"bots":{"GPTBot":["partial","*",32,1],"ChatGPT-User":["partial","*",32,1],"OAI-SearchBot":["partial","*",32,1],"ClaudeBot":["partial","*",32,1],"Claude-User":["partial","*",32,1],"Claude-SearchBot":["partial","*",32,1],"anthropic-ai":["partial","*",32,1],"PerplexityBot":["partial","*",32,1],"Perplexity-User":["partial","*",32,1],"Google-Extended":["partial","*",32,1],"Applebot-Extended":["partial","*",32,1],"CCBot":["partial","*",32,1],"Bytespider":["partial","*",32,1],"Amazonbot":["partial","*",32,1],"meta-externalagent":["partial","*",32,1],"cohere-ai":["partial","*",32,1],"Diffbot":["partial","*",32,1],"ContextFetchBot":["partial","*",32,1]},"llms":{"status":"html","http":200,"bytes":691},"home":{"status":"http_error","http":403,"url":"https://www.elibrary.ru/"}},
{"rank":5648,"domain":"uta.edu","origin":"https://uta.edu","robots":{"status":"ok","http":200,"url":"https://www.uta.edu/robots.txt","bytes":474,"groups":1},"bots":{"GPTBot":["partial","*",15,0],"ChatGPT-User":["partial","*",15,0],"OAI-SearchBot":["partial","*",15,0],"ClaudeBot":["partial","*",15,0],"Claude-User":["partial","*",15,0],"Claude-SearchBot":["partial","*",15,0],"anthropic-ai":["partial","*",15,0],"PerplexityBot":["partial","*",15,0],"Perplexity-User":["partial","*",15,0],"Google-Extended":["partial","*",15,0],"Applebot-Extended":["partial","*",15,0],"CCBot":["partial","*",15,0],"Bytespider":["partial","*",15,0],"Amazonbot":["partial","*",15,0],"meta-externalagent":["partial","*",15,0],"cohere-ai":["partial","*",15,0],"Diffbot":["partial","*",15,0],"ContextFetchBot":["partial","*",15,0]},"llms":{"status":"missing","http":404,"bytes":21739},"home":{"status":"ok","http":200,"url":"https://www.uta.edu/","html_tokens":26052,"markdown_tokens":627,"pruned_tokens":191}},
{"rank":5649,"domain":"kuaidi100.com","origin":"https://kuaidi100.com","robots":{"status":"html","http":200,"url":"https://www.kuaidi100.com/","bytes":64265,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":5650,"domain":"amsterdam.nl","origin":"https://amsterdam.nl","robots":{"status":"ok","http":200,"url":"https://amsterdam.nl/robots.txt","bytes":3767,"groups":49},"bots":{"GPTBot":["blocked","*",1,0],"ChatGPT-User":["blocked","*",1,0],"OAI-SearchBot":["blocked","*",1,0],"ClaudeBot":["blocked","*",1,0],"Claude-User":["blocked","*",1,0],"Claude-SearchBot":["blocked","*",1,0],"anthropic-ai":["blocked","*",1,0],"PerplexityBot":["blocked","*",1,0],"Perplexity-User":["blocked","*",1,0],"Google-Extended":["blocked","*",1,0],"Applebot-Extended":["blocked","*",1,0],"CCBot":["blocked","*",1,0],"Bytespider":["blocked","*",1,0],"Amazonbot":["blocked","*",1,0],"meta-externalagent":["blocked","*",1,0],"cohere-ai":["blocked","*",1,0],"Diffbot":["blocked","*",1,0],"ContextFetchBot":["blocked","*",1,0]},"llms":{"status":"disallowed"},"home":{"status":"disallowed"}},
{"rank":5651,"domain":"fgv.br","origin":"https://fgv.br","robots":{"status":"unreachable","http":null,"url":"https://fgv.br/robots.txt","bytes":0,"groups":0,"error":"[SSL: SSLV3_ALERT_HANDSHAKE_FAILURE] sslv3 alert handshake failure (_ssl.c:1028)"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":5652,"domain":"thepointsguy.com","origin":"https://thepointsguy.com","robots":{"status":"ok","http":200,"url":"https://thepointsguy.com/robots.txt","bytes":478,"groups":1},"bots":{"GPTBot":["partial","*",12,0],"ChatGPT-User":["partial","*",12,0],"OAI-SearchBot":["partial","*",12,0],"ClaudeBot":["partial","*",12,0],"Claude-User":["partial","*",12,0],"Claude-SearchBot":["partial","*",12,0],"anthropic-ai":["partial","*",12,0],"PerplexityBot":["partial","*",12,0],"Perplexity-User":["partial","*",12,0],"Google-Extended":["partial","*",12,0],"Applebot-Extended":["partial","*",12,0],"CCBot":["partial","*",12,0],"Bytespider":["partial","*",12,0],"Amazonbot":["partial","*",12,0],"meta-externalagent":["partial","*",12,0],"cohere-ai":["partial","*",12,0],"Diffbot":["partial","*",12,0],"ContextFetchBot":["partial","*",12,0]},"llms":{"status":"missing","http":404,"bytes":73680},"home":{"status":"ok","http":200,"url":"https://thepointsguy.com/","html_tokens":163012,"markdown_tokens":224,"pruned_tokens":116}},
{"rank":5653,"domain":"nbcmiami.com","origin":"https://nbcmiami.com","robots":{"status":"ok","http":200,"url":"https://www.nbcmiami.com/robots.txt","bytes":11418,"groups":27},"bots":{"GPTBot":["partial","named",1,6],"ChatGPT-User":["partial","named",1,6],"OAI-SearchBot":["partial","*",13,3],"ClaudeBot":["partial","named",1,6],"Claude-User":["partial","*",13,3],"Claude-SearchBot":["partial","*",13,3],"anthropic-ai":["partial","named",1,6],"PerplexityBot":["partial","named",1,6],"Perplexity-User":["partial","*",13,3],"Google-Extended":["partial","named",1,6],"Applebot-Extended":["partial","*",13,3],"CCBot":["partial","named",1,6],"Bytespider":["partial","named",1,6],"Amazonbot":["partial","named",1,8],"meta-externalagent":["partial","*",13,3],"cohere-ai":["partial","*",13,3],"Diffbot":["partial","named",1,6],"ContextFetchBot":["partial","*",13,3]},"llms":{"status":"html","http":200,"bytes":619301},"home":{"status":"ok","http":200,"url":"https://www.nbcmiami.com/","html_tokens":154818,"markdown_tokens":3,"pruned_tokens":3}},
{"rank":5654,"domain":"pepsico.com","origin":"https://pepsico.com","robots":{"status":"ok","http":200,"url":"https://www.pepsico.com/robots.txt","bytes":68,"groups":1},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"missing","http":404,"bytes":8160},"home":{"status":"ok","http":200,"url":"https://www.pepsico.com/","html_tokens":71670,"markdown_tokens":78,"pruned_tokens":78}},
{"rank":5655,"domain":"sciencemuseum.org.uk","origin":"https://sciencemuseum.org.uk","robots":{"status":"ok","http":200,"url":"https://www.sciencemuseum.org.uk/robots.txt","bytes":2113,"groups":2},"bots":{"GPTBot":["partial","*",32,18],"ChatGPT-User":["partial","*",32,18],"OAI-SearchBot":["partial","*",32,18],"ClaudeBot":["partial","*",32,18],"Claude-User":["partial","*",32,18],"Claude-SearchBot":["partial","*",32,18],"anthropic-ai":["partial","*",32,18],"PerplexityBot":["partial","*",32,18],"Perplexity-User":["partial","*",32,18],"Google-Extended":["partial","*",32,18],"Applebot-Extended":["partial","*",32,18],"CCBot":["partial","*",32,18],"Bytespider":["partial","*",32,18],"Amazonbot":["partial","*",32,18],"meta-externalagent":["partial","*",32,18],"cohere-ai":["partial","*",32,18],"Diffbot":["partial","*",32,18],"ContextFetchBot":["partial","*",32,18]},"llms":{"status":"missing","http":404,"bytes":565},"home":{"status":"ok","http":200,"url":"https://www.sciencemuseum.org.uk/","html_tokens":20117,"markdown_tokens":61,"pruned_tokens":61}},
{"rank":5656,"domain":"beincrypto.com","origin":"https://beincrypto.com","robots":{"status":"refused","http":403,"url":"https://beincrypto.com/robots.txt","bytes":5531,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":5657,"domain":"ynet.co.il","origin":"https://ynet.co.il","robots":{"status":"ok","http":200,"url":"https://www.ynet.co.il/robots.txt","bytes":1762,"groups":1},"bots":{"GPTBot":["partial","*",33,0],"ChatGPT-User":["partial","*",33,0],"OAI-SearchBot":["partial","*",33,0],"ClaudeBot":["partial","*",33,0],"Claude-User":["partial","*",33,0],"Claude-SearchBot":["partial","*",33,0],"anthropic-ai":["partial","*",33,0],"PerplexityBot":["partial","*",33,0],"Perplexity-User":["partial","*",33,0],"Google-Extended":["partial","*",33,0],"Applebot-Extended":["partial","*",33,0],"CCBot":["partial","*",33,0],"Bytespider":["partial","*",33,0],"Amazonbot":["partial","*",33,0],"meta-externalagent":["partial","*",33,0],"cohere-ai":["partial","*",33,0],"Diffbot":["partial","*",33,0],"ContextFetchBot":["partial","*",33,0]},"llms":{"status":"missing","http":404,"bytes":83685},"home":{"status":"ok","http":200,"url":"https://www.ynet.co.il/home/0,7340,L-8,00.html","html_tokens":125102,"markdown_tokens":42,"pruned_tokens":42}},
{"rank":5658,"domain":"fsa.go.jp","origin":"https://www.fsa.go.jp","robots":{"status":"missing","http":404,"url":"https://www.fsa.go.jp/robots.txt","bytes":36020,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":36020},"home":{"status":"ok","http":200,"url":"https://www.fsa.go.jp/","html_tokens":8102,"markdown_tokens":278,"pruned_tokens":174}},
{"rank":5659,"domain":"svd.se","origin":"https://svd.se","robots":{"status":"ok","http":200,"url":"https://www.svd.se/robots.txt","bytes":1696,"groups":2},"bots":{"GPTBot":["partial","*",5,1],"ChatGPT-User":["partial","*",5,1],"OAI-SearchBot":["partial","named",5,1],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",5,1],"Claude-SearchBot":["blocked","named",1,0],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["partial","*",5,1],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",5,1]},"llms":{"status":"missing","http":404,"bytes":191049},"home":{"status":"ok","http":200,"url":"https://www.svd.se","html_tokens":90247,"markdown_tokens":34,"pruned_tokens":34}},
{"rank":5660,"domain":"mts.ru","origin":"https://mts.ru","robots":{"status":"ok","http":200,"url":"https://moskva.mts.ru/robotstxt","bytes":6839,"groups":2},"bots":{"GPTBot":["partial","*",72,4],"ChatGPT-User":["partial","*",72,4],"OAI-SearchBot":["partial","*",72,4],"ClaudeBot":["partial","*",72,4],"Claude-User":["partial","*",72,4],"Claude-SearchBot":["partial","*",72,4],"anthropic-ai":["partial","*",72,4],"PerplexityBot":["partial","*",72,4],"Perplexity-User":["partial","*",72,4],"Google-Extended":["partial","*",72,4],"Applebot-Extended":["partial","*",72,4],"CCBot":["partial","*",72,4],"Bytespider":["partial","*",72,4],"Amazonbot":["partial","*",72,4],"meta-externalagent":["partial","*",72,4],"cohere-ai":["partial","*",72,4],"Diffbot":["partial","*",72,4],"ContextFetchBot":["partial","*",72,4]},"llms":{"status":"missing","http":404,"bytes":170931},"home":{"status":"ok","http":200,"url":"https://moskva.mts.ru/personal","html_tokens":234968,"markdown_tokens":20,"pruned_tokens":20}},
{"rank":5661,"domain":"kdocs.cn","origin":"https://kdocs.cn","robots":{"status":"ok","http":200,"url":"https://kdocs.cn/robots.txt","bytes":406,"groups":1},"bots":{"GPTBot":["partial","*",22,0],"ChatGPT-User":["partial","*",22,0],"OAI-SearchBot":["partial","*",22,0],"ClaudeBot":["partial","*",22,0],"Claude-User":["partial","*",22,0],"Claude-SearchBot":["partial","*",22,0],"anthropic-ai":["partial","*",22,0],"PerplexityBot":["partial","*",22,0],"Perplexity-User":["partial","*",22,0],"Google-Extended":["partial","*",22,0],"Applebot-Extended":["partial","*",22,0],"CCBot":["partial","*",22,0],"Bytespider":["partial","*",22,0],"Amazonbot":["partial","*",22,0],"meta-externalagent":["partial","*",22,0],"cohere-ai":["partial","*",22,0],"Diffbot":["partial","*",22,0],"ContextFetchBot":["partial","*",22,0]},"llms":{"status":"missing","http":404,"bytes":13320},"home":{"status":"ok","http":200,"url":"https://account.kdocs.cn/passport/singlesign?cb=https%3A%2F%2Fwww.kdocs.cn%2F&form=kdocs","html_tokens":5873,"markdown_tokens":0,"pruned_tokens":0}},
{"rank":5662,"domain":"vodafone.com","origin":"https://vodafone.com","robots":{"status":"refused","http":403,"url":"https://www.vodafone.com/robots.txt","bytes":4545,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":5663,"domain":"zawya.com","origin":"https://zawya.com","robots":{"status":"ok","http":200,"url":"https://www.zawya.com/robots.txt","bytes":12714,"groups":126},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["blocked","named",1,0],"OAI-SearchBot":["blocked","named",1,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",20,3],"Claude-SearchBot":["partial","*",20,3],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["partial","*",20,3],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",20,3]},"llms":{"status":"missing","http":404,"bytes":82978},"home":{"status":"ok","http":200,"url":"https://www.zawya.com/en/","html_tokens":60967,"markdown_tokens":6,"pruned_tokens":6}},
{"rank":5664,"domain":"cyberchimps.com","origin":"https://cyberchimps.com","robots":{"status":"ok","http":200,"url":"https://cyberchimps.com/robots.txt","bytes":1890,"groups":6},"bots":{"GPTBot":["partial","*",21,0],"ChatGPT-User":["partial","*",21,0],"OAI-SearchBot":["partial","*",21,0],"ClaudeBot":["partial","*",21,0],"Claude-User":["partial","*",21,0],"Claude-SearchBot":["partial","*",21,0],"anthropic-ai":["partial","*",21,0],"PerplexityBot":["partial","*",21,0],"Perplexity-User":["partial","*",21,0],"Google-Extended":["partial","*",21,0],"Applebot-Extended":["partial","*",21,0],"CCBot":["partial","*",21,0],"Bytespider":["partial","*",21,0],"Amazonbot":["partial","*",21,0],"meta-externalagent":["partial","*",21,0],"cohere-ai":["partial","*",21,0],"Diffbot":["partial","*",21,0],"ContextFetchBot":["partial","*",21,0]},"llms":{"status":"missing","http":404,"bytes":968024},"home":{"status":"ok","http":200,"url":"https://cyberchimps.com/","html_tokens":282161,"markdown_tokens":685,"pruned_tokens":93}},
{"rank":5665,"domain":"mhthemes.com","origin":"https://mhthemes.com","robots":{"status":"ok","http":200,"url":"https://mhthemes.com/robots.txt","bytes":1367,"groups":3},"bots":{"GPTBot":["partial","named",4,5],"ChatGPT-User":["partial","named",4,5],"OAI-SearchBot":["partial","named",4,5],"ClaudeBot":["partial","named",4,5],"Claude-User":["partial","*",4,1],"Claude-SearchBot":["partial","*",4,1],"anthropic-ai":["partial","named",4,5],"PerplexityBot":["partial","named",4,5],"Perplexity-User":["partial","*",4,1],"Google-Extended":["partial","named",4,5],"Applebot-Extended":["partial","named",4,5],"CCBot":["partial","named",4,5],"Bytespider":["partial","named",4,5],"Amazonbot":["partial","named",4,5],"meta-externalagent":["partial","named",4,5],"cohere-ai":["partial","named",4,5],"Diffbot":["partial","named",4,5],"ContextFetchBot":["partial","*",4,1]},"llms":{"status":"present","http":200,"bytes":159152},"home":{"status":"ok","http":200,"url":"https://mhthemes.com/","html_tokens":43392,"markdown_tokens":900,"pruned_tokens":175}},
{"rank":5666,"domain":"kidney.org","origin":"https://kidney.org","robots":{"status":"refused","http":403,"url":"https://kidney.org/robots.txt","bytes":560,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":5667,"domain":"brandsly.com","origin":"https://brandsly.com","robots":{"status":"ok","http":200,"url":"https://brandsly.com/robots.txt","bytes":372,"groups":7},"bots":{"GPTBot":["partial","*",6,0],"ChatGPT-User":["partial","*",6,0],"OAI-SearchBot":["partial","*",6,0],"ClaudeBot":["partial","*",6,0],"Claude-User":["partial","*",6,0],"Claude-SearchBot":["partial","*",6,0],"anthropic-ai":["partial","*",6,0],"PerplexityBot":["partial","*",6,0],"Perplexity-User":["partial","*",6,0],"Google-Extended":["partial","*",6,0],"Applebot-Extended":["partial","*",6,0],"CCBot":["partial","*",6,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["partial","*",6,0],"meta-externalagent":["partial","*",6,0],"cohere-ai":["partial","*",6,0],"Diffbot":["partial","*",6,0],"ContextFetchBot":["partial","*",6,0]},"llms":{"status":"missing","http":404,"bytes":333507},"home":{"status":"http_error","http":403,"url":"https://brandsly.com/"}},
{"rank":5668,"domain":"marines.mil","origin":"https://www.marines.mil","robots":{"status":"refused","http":403,"url":"https://www.marines.mil/robots.txt","bytes":381,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":5669,"domain":"cdp.net","origin":"https://cdp.net","robots":{"status":"ok","http":200,"url":"https://www.cdp.net/robots.txt","bytes":238,"groups":2},"bots":{"GPTBot":["partial","*",8,0],"ChatGPT-User":["partial","*",8,0],"OAI-SearchBot":["partial","*",8,0],"ClaudeBot":["partial","*",8,0],"Claude-User":["partial","*",8,0],"Claude-SearchBot":["partial","*",8,0],"anthropic-ai":["partial","*",8,0],"PerplexityBot":["partial","*",8,0],"Perplexity-User":["partial","*",8,0],"Google-Extended":["partial","*",8,0],"Applebot-Extended":["partial","*",8,0],"CCBot":["partial","*",8,0],"Bytespider":["partial","*",8,0],"Amazonbot":["partial","*",8,0],"meta-externalagent":["partial","*",8,0],"cohere-ai":["partial","*",8,0],"Diffbot":["partial","*",8,0],"ContextFetchBot":["partial","*",8,0]},"llms":{"status":"present","http":200,"bytes":2627},"home":{"status":"ok","http":200,"url":"https://www.cdp.net/en","html_tokens":219421,"markdown_tokens":654,"pruned_tokens":117}},
{"rank":5670,"domain":"otto.de","origin":"https://otto.de","robots":{"status":"ok","http":200,"url":"https://www.otto.de/robots.txt","bytes":2148,"groups":3},"bots":{"GPTBot":["partial","named",5,0],"ChatGPT-User":["partial","named",5,0],"OAI-SearchBot":["partial","named",5,0],"ClaudeBot":["partial","*",4,0],"Claude-User":["partial","*",4,0],"Claude-SearchBot":["partial","*",4,0],"anthropic-ai":["partial","*",4,0],"PerplexityBot":["partial","named",5,0],"Perplexity-User":["partial","named",5,0],"Google-Extended":["partial","*",4,0],"Applebot-Extended":["partial","*",4,0],"CCBot":["partial","*",4,0],"Bytespider":["partial","*",4,0],"Amazonbot":["partial","*",4,0],"meta-externalagent":["partial","*",4,0],"cohere-ai":["partial","*",4,0],"Diffbot":["partial","*",4,0],"ContextFetchBot":["partial","*",4,0]},"llms":{"status":"present","http":200,"bytes":15185},"home":{"status":"ok","http":200,"url":"https://www.otto.de/","html_tokens":81918,"markdown_tokens":591,"pruned_tokens":174}},
{"rank":5671,"domain":"nltimes.nl","origin":"https://nltimes.nl","robots":{"status":"ok","http":200,"url":"https://nltimes.nl/robots.txt","bytes":3033,"groups":21},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["blocked","named",1,0],"OAI-SearchBot":["partial","*",42,18],"ClaudeBot":["partial","*",42,18],"Claude-User":["partial","*",42,18],"Claude-SearchBot":["partial","*",42,18],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["partial","*",42,18],"Perplexity-User":["partial","*",42,18],"Google-Extended":["partial","*",42,18],"Applebot-Extended":["partial","*",42,18],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["partial","*",42,18],"meta-externalagent":["partial","*",42,18],"cohere-ai":["blocked","named",1,0],"Diffbot":["partial","*",42,18],"ContextFetchBot":["partial","*",42,18]},"llms":{"status":"missing","http":404,"bytes":184},"home":{"status":"ok","http":200,"url":"https://nltimes.nl/","html_tokens":25629,"markdown_tokens":5,"pruned_tokens":5}},
{"rank":5672,"domain":"paddle.com","origin":"https://paddle.com","robots":{"status":"ok","http":200,"url":"https://www.paddle.com/robots.txt","bytes":95,"groups":1},"bots":{"GPTBot":["partial","*",1,1],"ChatGPT-User":["partial","*",1,1],"OAI-SearchBot":["partial","*",1,1],"ClaudeBot":["partial","*",1,1],"Claude-User":["partial","*",1,1],"Claude-SearchBot":["partial","*",1,1],"anthropic-ai":["partial","*",1,1],"PerplexityBot":["partial","*",1,1],"Perplexity-User":["partial","*",1,1],"Google-Extended":["partial","*",1,1],"Applebot-Extended":["partial","*",1,1],"CCBot":["partial","*",1,1],"Bytespider":["partial","*",1,1],"Amazonbot":["partial","*",1,1],"meta-externalagent":["partial","*",1,1],"cohere-ai":["partial","*",1,1],"Diffbot":["partial","*",1,1],"ContextFetchBot":["partial","*",1,1]},"llms":{"status":"missing","http":404,"bytes":634732},"home":{"status":"ok","http":200,"url":"https://www.paddle.com/","html_tokens":279671,"markdown_tokens":255,"pruned_tokens":74}},
{"rank":5673,"domain":"hashnode.com","origin":"https://hashnode.com","robots":{"status":"ok","http":200,"url":"https://hashnode.com/robots.txt","bytes":282,"groups":1},"bots":{"GPTBot":["partial","*",11,1],"ChatGPT-User":["partial","*",11,1],"OAI-SearchBot":["partial","*",11,1],"ClaudeBot":["partial","*",11,1],"Claude-User":["partial","*",11,1],"Claude-SearchBot":["partial","*",11,1],"anthropic-ai":["partial","*",11,1],"PerplexityBot":["partial","*",11,1],"Perplexity-User":["partial","*",11,1],"Google-Extended":["partial","*",11,1],"Applebot-Extended":["partial","*",11,1],"CCBot":["partial","*",11,1],"Bytespider":["partial","*",11,1],"Amazonbot":["partial","*",11,1],"meta-externalagent":["partial","*",11,1],"cohere-ai":["partial","*",11,1],"Diffbot":["partial","*",11,1],"ContextFetchBot":["partial","*",11,1]},"llms":{"status":"html","http":200,"bytes":71245},"home":{"status":"ok","http":200,"url":"https://hashnode.com/","html_tokens":95247,"markdown_tokens":252,"pruned_tokens":166}},
{"rank":5674,"domain":"timeslive.co.za","origin":"https://timeslive.co.za","robots":{"status":"ok","http":200,"url":"https://www.timeslive.co.za/robots.txt","bytes":1367,"groups":25},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["blocked","named",1,0],"OAI-SearchBot":["allowed","*",0,1],"ClaudeBot":["blocked","named",1,0],"Claude-User":["allowed","*",0,1],"Claude-SearchBot":["allowed","*",0,1],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["allowed","*",0,1],"Google-Extended":["partial","named",2,1],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["allowed","*",0,1]},"llms":{"status":"missing","http":404,"bytes":186764},"home":{"status":"ok","http":200,"url":"https://www.timeslive.co.za/","html_tokens":90074,"markdown_tokens":83,"pruned_tokens":83}},
{"rank":5675,"domain":"unistra.fr","origin":"https://unistra.fr","robots":{"status":"ok","http":200,"url":"https://www.unistra.fr/robots.txt","bytes":2665,"groups":9},"bots":{"GPTBot":["partial","named",10,0],"ChatGPT-User":["partial","named",10,0],"OAI-SearchBot":["partial","*",23,18],"ClaudeBot":["partial","named",10,0],"Claude-User":["partial","*",23,18],"Claude-SearchBot":["partial","*",23,18],"anthropic-ai":["partial","named",10,0],"PerplexityBot":["partial","*",23,18],"Perplexity-User":["partial","*",23,18],"Google-Extended":["partial","*",23,18],"Applebot-Extended":["partial","*",23,18],"CCBot":["partial","*",23,18],"Bytespider":["blocked","named",1,0],"Amazonbot":["partial","named",10,0],"meta-externalagent":["partial","*",23,18],"cohere-ai":["partial","*",23,18],"Diffbot":["partial","*",23,18],"ContextFetchBot":["partial","*",23,18]},"llms":{"status":"missing","http":404,"bytes":360383},"home":{"status":"ok","http":200,"url":"https://www.unistra.fr/fr","html_tokens":141247,"markdown_tokens":225,"pruned_tokens":172}},
{"rank":5676,"domain":"szczecin.pl","origin":"https://www.szczecin.pl","robots":{"status":"html","http":200,"url":"https://szczecin.eu/pl","bytes":103938,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":5677,"domain":"uc3m.es","origin":"https://uc3m.es","robots":{"status":"ok","http":200,"url":"https://uc3m.es/robots.txt","bytes":5526,"groups":1},"bots":{"GPTBot":["partial","*",63,1],"ChatGPT-User":["partial","*",63,1],"OAI-SearchBot":["partial","*",63,1],"ClaudeBot":["partial","*",63,1],"Claude-User":["partial","*",63,1],"Claude-SearchBot":["partial","*",63,1],"anthropic-ai":["partial","*",63,1],"PerplexityBot":["partial","*",63,1],"Perplexity-User":["partial","*",63,1],"Google-Extended":["partial","*",63,1],"Applebot-Extended":["partial","*",63,1],"CCBot":["partial","*",63,1],"Bytespider":["partial","*",63,1],"Amazonbot":["partial","*",63,1],"meta-externalagent":["partial","*",63,1],"cohere-ai":["partial","*",63,1],"Diffbot":["partial","*",63,1],"ContextFetchBot":["partial","*",63,1]},"llms":{"status":"missing","http":404,"bytes":3271},"home":{"status":"ok","http":200,"url":"https://uc3m.es/","html_tokens":237,"markdown_tokens":0,"pruned_tokens":0}},
{"rank":5678,"domain":"apc.org","origin":"https://apc.org","robots":{"status":"ok","http":200,"url":"https://www.apc.org/robots.txt","bytes":2093,"groups":1},"bots":{"GPTBot":["partial","*",35,18],"ChatGPT-User":["partial","*",35,18],"OAI-SearchBot":["partial","*",35,18],"ClaudeBot":["partial","*",35,18],"Claude-User":["partial","*",35,18],"Claude-SearchBot":["partial","*",35,18],"anthropic-ai":["partial","*",35,18],"PerplexityBot":["partial","*",35,18],"Perplexity-User":["partial","*",35,18],"Google-Extended":["partial","*",35,18],"Applebot-Extended":["partial","*",35,18],"CCBot":["partial","*",35,18],"Bytespider":["partial","*",35,18],"Amazonbot":["partial","*",35,18],"meta-externalagent":["partial","*",35,18],"cohere-ai":["partial","*",35,18],"Diffbot":["partial","*",35,18],"ContextFetchBot":["partial","*",35,18]},"llms":{"status":"http_error","http":403,"bytes":146},"home":{"status":"ok","http":200,"url":"https://www.apc.org/en","html_tokens":13482,"markdown_tokens":47,"pruned_tokens":36}},
{"rank":5679,"domain":"silabs.com","origin":"https://silabs.com","robots":{"status":"refused","http":403,"url":"https://www.silabs.com/robots.txt","bytes":382,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":5680,"domain":"firebaseapp.com","origin":"https://firebaseapp.com","robots":{"status":"html","http":200,"url":"https://firebase.google.com/products/hosting/","bytes":510718,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":5681,"domain":"credit-suisse.com","origin":"https://credit-suisse.com","robots":{"status":"html","http":200,"url":"https://www.ubs.com/us/en.html","bytes":141552,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":5682,"domain":"mnhn.fr","origin":"https://mnhn.fr","robots":{"status":"refused","http":403,"url":"https://mnhn.fr/robots.txt","bytes":5488,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":5683,"domain":"soup.io","origin":"https://soup.io","robots":{"status":"ok","http":200,"url":"https://www.soup.io/robots.txt","bytes":169,"groups":1},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"html","http":200,"bytes":99114},"home":{"status":"ok","http":200,"url":"https://www.soup.io/","html_tokens":24777,"markdown_tokens":40,"pruned_tokens":40}},
{"rank":5684,"domain":"maps.google.nl","origin":"https://maps.google.nl","robots":{"status":"ok","http":200,"url":"https://maps.google.nl/robots.txt","bytes":6624,"groups":4},"bots":{"GPTBot":["partial","*",169,64],"ChatGPT-User":["partial","*",169,64],"OAI-SearchBot":["partial","*",169,64],"ClaudeBot":["partial","*",169,64],"Claude-User":["partial","*",169,64],"Claude-SearchBot":["partial","*",169,64],"anthropic-ai":["partial","*",169,64],"PerplexityBot":["partial","*",169,64],"Perplexity-User":["partial","*",169,64],"Google-Extended":["partial","*",169,64],"Applebot-Extended":["partial","*",169,64],"CCBot":["partial","*",169,64],"Bytespider":["partial","*",169,64],"Amazonbot":["partial","*",169,64],"meta-externalagent":["partial","*",169,64],"cohere-ai":["partial","*",169,64],"Diffbot":["partial","*",169,64],"ContextFetchBot":["partial","*",169,64]},"llms":{"status":"missing","http":404,"bytes":1569},"home":{"status":"ok","http":200,"url":"https://www.google.nl/maps","html_tokens":54148,"markdown_tokens":30,"pruned_tokens":30}},
{"rank":5685,"domain":"world-nuclear.org","origin":"https://world-nuclear.org","robots":{"status":"missing","http":404,"url":"https://world-nuclear.org/robots.txt","bytes":42132,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":42132},"home":{"status":"ok","http":200,"url":"https://world-nuclear.org/","html_tokens":23284,"markdown_tokens":249,"pruned_tokens":122}},
{"rank":5686,"domain":"flourish.studio","origin":"https://flourish.studio","robots":{"status":"ok","http":200,"url":"https://flourish.studio/robots.txt","bytes":1206,"groups":4},"bots":{"GPTBot":["partial","named",1,8],"ChatGPT-User":["partial","*",10,0],"OAI-SearchBot":["partial","*",10,0],"ClaudeBot":["partial","named",1,8],"Claude-User":["partial","*",10,0],"Claude-SearchBot":["partial","*",10,0],"anthropic-ai":["partial","named",1,8],"PerplexityBot":["allowed","named",0,1],"Perplexity-User":["allowed","named",0,1],"Google-Extended":["partial","*",10,0],"Applebot-Extended":["partial","named",1,8],"CCBot":["partial","named",1,8],"Bytespider":["blocked","named",1,0],"Amazonbot":["partial","*",10,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["partial","*",10,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",10,0]},"llms":{"status":"present","http":200,"bytes":4027},"home":{"status":"ok","http":200,"url":"https://flourish.studio/","html_tokens":18998,"markdown_tokens":585,"pruned_tokens":162}},
{"rank":5687,"domain":"axs.com","origin":"https://axs.com","robots":{"status":"ok","http":200,"url":"https://www.axs.com/robots.txt","bytes":354,"groups":4},"bots":{"GPTBot":["partial","*",5,0],"ChatGPT-User":["partial","*",5,0],"OAI-SearchBot":["allowed","named",0,1],"ClaudeBot":["partial","*",5,0],"Claude-User":["partial","*",5,0],"Claude-SearchBot":["partial","*",5,0],"anthropic-ai":["partial","*",5,0],"PerplexityBot":["partial","*",5,0],"Perplexity-User":["partial","*",5,0],"Google-Extended":["partial","*",5,0],"Applebot-Extended":["partial","*",5,0],"CCBot":["partial","*",5,0],"Bytespider":["partial","*",5,0],"Amazonbot":["partial","*",5,0],"meta-externalagent":["partial","*",5,0],"cohere-ai":["partial","*",5,0],"Diffbot":["partial","*",5,0],"ContextFetchBot":["partial","*",5,0]},"llms":{"status":"http_error","http":403,"bytes":152314},"home":{"status":"http_error","http":403,"url":"https://www.axs.com/"}},
{"rank":5688,"domain":"gta5-mods.com","origin":"https://gta5-mods.com","robots":{"status":"ok","http":200,"url":"https://www.gta5-mods.com/robots.txt","bytes":331,"groups":1},"bots":{"GPTBot":["partial","*",9,0],"ChatGPT-User":["partial","*",9,0],"OAI-SearchBot":["partial","*",9,0],"ClaudeBot":["partial","*",9,0],"Claude-User":["partial","*",9,0],"Claude-SearchBot":["partial","*",9,0],"anthropic-ai":["partial","*",9,0],"PerplexityBot":["partial","*",9,0],"Perplexity-User":["partial","*",9,0],"Google-Extended":["partial","*",9,0],"Applebot-Extended":["partial","*",9,0],"CCBot":["partial","*",9,0],"Bytespider":["partial","*",9,0],"Amazonbot":["partial","*",9,0],"meta-externalagent":["partial","*",9,0],"cohere-ai":["partial","*",9,0],"Diffbot":["partial","*",9,0],"ContextFetchBot":["partial","*",9,0]},"llms":{"status":"missing","http":404,"bytes":7094},"home":{"status":"ok","http":200,"url":"https://www.gta5-mods.com/","html_tokens":13457,"markdown_tokens":19,"pruned_tokens":19}},
{"rank":5689,"domain":"mango.com","origin":"https://mango.com","robots":{"status":"refused","http":403,"url":"https://www.mango.com/robots.txt","bytes":381,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":5690,"domain":"kiwislot.co.nz","origin":"https://kiwislot.co.nz","robots":{"status":"ok","http":200,"url":"https://kiwislot.co.nz/robots.txt","bytes":532,"groups":1},"bots":{"GPTBot":["partial","*",19,0],"ChatGPT-User":["partial","*",19,0],"OAI-SearchBot":["partial","*",19,0],"ClaudeBot":["partial","*",19,0],"Claude-User":["partial","*",19,0],"Claude-SearchBot":["partial","*",19,0],"anthropic-ai":["partial","*",19,0],"PerplexityBot":["partial","*",19,0],"Perplexity-User":["partial","*",19,0],"Google-Extended":["partial","*",19,0],"Applebot-Extended":["partial","*",19,0],"CCBot":["partial","*",19,0],"Bytespider":["partial","*",19,0],"Amazonbot":["partial","*",19,0],"meta-externalagent":["partial","*",19,0],"cohere-ai":["partial","*",19,0],"Diffbot":["partial","*",19,0],"ContextFetchBot":["partial","*",19,0]},"llms":{"status":"missing","http":404,"bytes":24602},"home":{"status":"ok","http":200,"url":"https://kiwislot.co.nz/","html_tokens":20030,"markdown_tokens":286,"pruned_tokens":169}},
{"rank":5691,"domain":"empireonline.com","origin":"https://empireonline.com","robots":{"status":"ok","http":200,"url":"https://www.empireonline.com/robots.txt","bytes":2163,"groups":56},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["blocked","named",1,0],"OAI-SearchBot":["blocked","named",1,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["blocked","named",1,0],"Claude-SearchBot":["blocked","named",1,0],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["blocked","named",1,0],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",2,0]},"llms":{"status":"missing","http":404,"bytes":24160},"home":{"status":"ok","http":200,"url":"https://www.empireonline.com/","html_tokens":104503,"markdown_tokens":215,"pruned_tokens":173}},
{"rank":5692,"domain":"nea.org","origin":"https://nea.org","robots":{"status":"ok","http":200,"url":"https://www.nea.org/robots.txt","bytes":4492,"groups":2},"bots":{"GPTBot":["partial","*",38,22],"ChatGPT-User":["partial","*",38,22],"OAI-SearchBot":["partial","*",38,22],"ClaudeBot":["partial","*",38,22],"Claude-User":["partial","*",38,22],"Claude-SearchBot":["partial","*",38,22],"anthropic-ai":["partial","*",38,22],"PerplexityBot":["partial","*",38,22],"Perplexity-User":["partial","*",38,22],"Google-Extended":["partial","*",38,22],"Applebot-Extended":["partial","*",38,22],"CCBot":["partial","*",38,22],"Bytespider":["partial","*",38,22],"Amazonbot":["partial","*",38,22],"meta-externalagent":["partial","*",38,22],"cohere-ai":["partial","*",38,22],"Diffbot":["partial","*",38,22],"ContextFetchBot":["partial","*",38,22]},"llms":{"status":"missing","http":404,"bytes":546},"home":{"status":"ok","http":200,"url":"https://www.nea.org/","html_tokens":30441,"markdown_tokens":900,"pruned_tokens":191}},
{"rank":5693,"domain":"freerepublic.com","origin":"https://freerepublic.com","robots":{"status":"ok","http":200,"url":"https://freerepublic.com/robots.txt","bytes":107,"groups":1},"bots":{"GPTBot":["partial","*",4,0],"ChatGPT-User":["partial","*",4,0],"OAI-SearchBot":["partial","*",4,0],"ClaudeBot":["partial","*",4,0],"Claude-User":["partial","*",4,0],"Claude-SearchBot":["partial","*",4,0],"anthropic-ai":["partial","*",4,0],"PerplexityBot":["partial","*",4,0],"Perplexity-User":["partial","*",4,0],"Google-Extended":["partial","*",4,0],"Applebot-Extended":["partial","*",4,0],"CCBot":["partial","*",4,0],"Bytespider":["partial","*",4,0],"Amazonbot":["partial","*",4,0],"meta-externalagent":["partial","*",4,0],"cohere-ai":["partial","*",4,0],"Diffbot":["partial","*",4,0],"ContextFetchBot":["partial","*",4,0]},"llms":{"status":"missing","http":404,"bytes":162},"home":{"status":"ok","http":200,"url":"https://freerepublic.com/tag/*/index","html_tokens":10523,"markdown_tokens":2739,"pruned_tokens":172}},
{"rank":5694,"domain":"iab.com","origin":"https://iab.com","robots":{"status":"ok","http":200,"url":"https://www.iab.com/robots.txt","bytes":169,"groups":1},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"other_text","http":200,"bytes":6158},"home":{"status":"timeout","error":"timed out after 15 s"}},
{"rank":5695,"domain":"nationalrail.co.uk","origin":"https://nationalrail.co.uk","robots":{"status":"ok","http":200,"url":"https://www.nationalrail.co.uk/robots.txt","bytes":1912,"groups":4},"bots":{"GPTBot":["partial","*",3,1],"ChatGPT-User":["partial","*",3,1],"OAI-SearchBot":["partial","*",3,1],"ClaudeBot":["partial","*",3,1],"Claude-User":["partial","*",3,1],"Claude-SearchBot":["partial","*",3,1],"anthropic-ai":["partial","*",3,1],"PerplexityBot":["partial","*",3,1],"Perplexity-User":["partial","*",3,1],"Google-Extended":["allowed","named",0,1],"Applebot-Extended":["partial","*",3,1],"CCBot":["partial","*",3,1],"Bytespider":["partial","*",3,1],"Amazonbot":["partial","*",3,1],"meta-externalagent":["partial","*",3,1],"cohere-ai":["partial","*",3,1],"Diffbot":["partial","*",3,1],"ContextFetchBot":["partial","*",3,1]},"llms":{"status":"missing","http":404,"bytes":163095},"home":{"status":"ok","http":200,"url":"https://www.nationalrail.co.uk/","html_tokens":64276,"markdown_tokens":645,"pruned_tokens":174}},
{"rank":5696,"domain":"sparknotes.com","origin":"https://sparknotes.com","robots":{"status":"refused","http":403,"url":"https://www.sparknotes.com/robots.txt","bytes":386,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":5697,"domain":"nsf.org","origin":"https://nsf.org","robots":{"status":"ok","http":200,"url":"https://www.nsf.org/robots.txt","bytes":278,"groups":1},"bots":{"GPTBot":["partial","*",5,0],"ChatGPT-User":["partial","*",5,0],"OAI-SearchBot":["partial","*",5,0],"ClaudeBot":["partial","*",5,0],"Claude-User":["partial","*",5,0],"Claude-SearchBot":["partial","*",5,0],"anthropic-ai":["partial","*",5,0],"PerplexityBot":["partial","*",5,0],"Perplexity-User":["partial","*",5,0],"Google-Extended":["partial","*",5,0],"Applebot-Extended":["partial","*",5,0],"CCBot":["partial","*",5,0],"Bytespider":["partial","*",5,0],"Amazonbot":["partial","*",5,0],"meta-externalagent":["partial","*",5,0],"cohere-ai":["partial","*",5,0],"Diffbot":["partial","*",5,0],"ContextFetchBot":["partial","*",5,0]},"llms":{"status":"missing","http":404,"bytes":107287},"home":{"status":"http_error","http":403,"url":"https://www.nsf.org"}},
{"rank":5698,"domain":"transfermarkt.com","origin":"https://transfermarkt.com","robots":{"status":"ok","http":200,"url":"https://www.transfermarkt.com/robots.txt","bytes":131,"groups":3},"bots":{"GPTBot":["allowed","*",0,1],"ChatGPT-User":["allowed","*",0,1],"OAI-SearchBot":["allowed","*",0,1],"ClaudeBot":["allowed","*",0,1],"Claude-User":["allowed","*",0,1],"Claude-SearchBot":["allowed","*",0,1],"anthropic-ai":["allowed","*",0,1],"PerplexityBot":["allowed","*",0,1],"Perplexity-User":["allowed","*",0,1],"Google-Extended":["allowed","*",0,1],"Applebot-Extended":["allowed","*",0,1],"CCBot":["allowed","*",0,1],"Bytespider":["allowed","*",0,1],"Amazonbot":["allowed","*",0,1],"meta-externalagent":["allowed","*",0,1],"cohere-ai":["allowed","*",0,1],"Diffbot":["allowed","*",0,1],"ContextFetchBot":["allowed","*",0,1]},"llms":{"status":"missing","http":404,"bytes":80302},"home":{"status":"ok","http":200,"url":"https://www.transfermarkt.com/","html_tokens":73401,"markdown_tokens":6830,"pruned_tokens":168}},
{"rank":5699,"domain":"deere.com","origin":"https://deere.com","robots":{"status":"ok","http":200,"url":"https://www.deere.com/robots.txt","bytes":104,"groups":1},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"missing","http":404,"bytes":102768},"home":{"status":"ok","http":200,"url":"https://www.deere.com/en-us/","html_tokens":35084,"markdown_tokens":12,"pruned_tokens":12}},
{"rank":5700,"domain":"hypermart.net","origin":"https://www.hypermart.net","robots":{"status":"refused","http":403,"url":"https://www.networksolutions.com/ipage?brand=ipage&siteID=100&channelID=P99C100S653N0B5A1D0E0000V113","bytes":12504,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":5701,"domain":"wizzair.com","origin":"https://wizzair.com","robots":{"status":"ok","http":200,"url":"https://www.wizzair.com/robots.txt","bytes":10862,"groups":1},"bots":{"GPTBot":["partial","*",377,0],"ChatGPT-User":["partial","*",377,0],"OAI-SearchBot":["partial","*",377,0],"ClaudeBot":["partial","*",377,0],"Claude-User":["partial","*",377,0],"Claude-SearchBot":["partial","*",377,0],"anthropic-ai":["partial","*",377,0],"PerplexityBot":["partial","*",377,0],"Perplexity-User":["partial","*",377,0],"Google-Extended":["partial","*",377,0],"Applebot-Extended":["partial","*",377,0],"CCBot":["partial","*",377,0],"Bytespider":["partial","*",377,0],"Amazonbot":["partial","*",377,0],"meta-externalagent":["partial","*",377,0],"cohere-ai":["partial","*",377,0],"Diffbot":["partial","*",377,0],"ContextFetchBot":["partial","*",377,0]},"llms":{"status":"http_error","http":405,"bytes":2519},"home":{"status":"http_error","http":405,"url":"https://www.wizzair.com/"}},
{"rank":5702,"domain":"eyeofhorusslot.com","origin":"https://eyeofhorusslot.com","robots":{"status":"ok","http":200,"url":"https://eyeofhorusslot.com/robots.txt","bytes":541,"groups":1},"bots":{"GPTBot":["partial","*",19,0],"ChatGPT-User":["partial","*",19,0],"OAI-SearchBot":["partial","*",19,0],"ClaudeBot":["partial","*",19,0],"Claude-User":["partial","*",19,0],"Claude-SearchBot":["partial","*",19,0],"anthropic-ai":["partial","*",19,0],"PerplexityBot":["partial","*",19,0],"Perplexity-User":["partial","*",19,0],"Google-Extended":["partial","*",19,0],"Applebot-Extended":["partial","*",19,0],"CCBot":["partial","*",19,0],"Bytespider":["partial","*",19,0],"Amazonbot":["partial","*",19,0],"meta-externalagent":["partial","*",19,0],"cohere-ai":["partial","*",19,0],"Diffbot":["partial","*",19,0],"ContextFetchBot":["partial","*",19,0]},"llms":{"status":"missing","http":404,"bytes":24998},"home":{"status":"ok","http":200,"url":"https://eyeofhorusslot.com/","html_tokens":22150,"markdown_tokens":2017,"pruned_tokens":138}},
{"rank":5703,"domain":"zf.com","origin":"https://zf.com","robots":{"status":"refused","http":403,"url":"https://www.zf.com/robots.txt","bytes":15777,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":5704,"domain":"bedbathandbeyond.com","origin":"https://bedbathandbeyond.com","robots":{"status":"ok","http":200,"url":"https://www.bedbathandbeyond.com/robots.txt","bytes":4065,"groups":15},"bots":{"GPTBot":["partial","named",19,0],"ChatGPT-User":["partial","*",41,0],"OAI-SearchBot":["partial","*",41,0],"ClaudeBot":["partial","*",41,0],"Claude-User":["partial","*",41,0],"Claude-SearchBot":["partial","*",41,0],"anthropic-ai":["partial","*",41,0],"PerplexityBot":["partial","*",41,0],"Perplexity-User":["partial","*",41,0],"Google-Extended":["partial","*",41,0],"Applebot-Extended":["partial","*",41,0],"CCBot":["partial","*",41,0],"Bytespider":["partial","*",41,0],"Amazonbot":["partial","named",11,0],"meta-externalagent":["partial","named",2,0],"cohere-ai":["partial","*",41,0],"Diffbot":["partial","*",41,0],"ContextFetchBot":["partial","*",41,0]},"llms":{"status":"html","http":200,"bytes":1454927},"home":{"status":"ok","http":200,"url":"https://www.bedbathandbeyond.com/","html_tokens":363722,"markdown_tokens":8,"pruned_tokens":8}},
{"rank":5705,"domain":"overcast.fm","origin":"https://overcast.fm","robots":{"status":"missing","http":404,"url":"https://overcast.fm/robots.txt","bytes":350,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":350},"home":{"status":"ok","http":200,"url":"https://overcast.fm/","html_tokens":23252,"markdown_tokens":178,"pruned_tokens":20}},
{"rank":5706,"domain":"milano.it","origin":"https://milano.it","robots":{"status":"unreachable","http":null,"url":"https://milano.it/robots.txt","bytes":0,"groups":0,"error":"cannot resolve milano.it"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":5707,"domain":"startbootstrap.com","origin":"https://startbootstrap.com","robots":{"status":"ok","http":200,"url":"https://startbootstrap.com/robots.txt","bytes":62,"groups":1},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"html","http":200,"bytes":12460},"home":{"status":"ok","http":200,"url":"https://startbootstrap.com/","html_tokens":3350,"markdown_tokens":0,"pruned_tokens":0}},
{"rank":5708,"domain":"plasticsurgery.org","origin":"https://plasticsurgery.org","robots":{"status":"ok","http":200,"url":"https://www.plasticsurgery.org/robots.txt","bytes":368,"groups":1},"bots":{"GPTBot":["partial","*",6,0],"ChatGPT-User":["partial","*",6,0],"OAI-SearchBot":["partial","*",6,0],"ClaudeBot":["partial","*",6,0],"Claude-User":["partial","*",6,0],"Claude-SearchBot":["partial","*",6,0],"anthropic-ai":["partial","*",6,0],"PerplexityBot":["partial","*",6,0],"Perplexity-User":["partial","*",6,0],"Google-Extended":["partial","*",6,0],"Applebot-Extended":["partial","*",6,0],"CCBot":["partial","*",6,0],"Bytespider":["partial","*",6,0],"Amazonbot":["partial","*",6,0],"meta-externalagent":["partial","*",6,0],"cohere-ai":["partial","*",6,0],"Diffbot":["partial","*",6,0],"ContextFetchBot":["partial","*",6,0]},"llms":{"status":"missing","http":404,"bytes":93917},"home":{"status":"ok","http":200,"url":"https://www.plasticsurgery.org/","html_tokens":26511,"markdown_tokens":253,"pruned_tokens":118}},
{"rank":5709,"domain":"orst.edu","origin":"https://www.orst.edu","robots":{"status":"missing","http":404,"url":"https://oregonstate.edu/404","bytes":29075,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":29075},"home":{"status":"http_error","http":404,"url":"https://oregonstate.edu/404"}},
{"rank":5710,"domain":"unstoppabledomains.com","origin":"https://unstoppabledomains.com","robots":{"status":"ok","http":200,"url":"https://unstoppabledomains.com/robots.txt","bytes":8930,"groups":1},"bots":{"GPTBot":["partial","*",316,0],"ChatGPT-User":["partial","*",316,0],"OAI-SearchBot":["partial","*",316,0],"ClaudeBot":["partial","*",316,0],"Claude-User":["partial","*",316,0],"Claude-SearchBot":["partial","*",316,0],"anthropic-ai":["partial","*",316,0],"PerplexityBot":["partial","*",316,0],"Perplexity-User":["partial","*",316,0],"Google-Extended":["partial","*",316,0],"Applebot-Extended":["partial","*",316,0],"CCBot":["partial","*",316,0],"Bytespider":["partial","*",316,0],"Amazonbot":["partial","*",316,0],"meta-externalagent":["partial","*",316,0],"cohere-ai":["partial","*",316,0],"Diffbot":["partial","*",316,0],"ContextFetchBot":["partial","*",316,0]},"llms":{"status":"present","http":200,"bytes":26799},"home":{"status":"ok","http":200,"url":"https://unstoppabledomains.com/","html_tokens":61355,"markdown_tokens":220,"pruned_tokens":117}},
{"rank":5711,"domain":"aicpa.org","origin":"https://aicpa.org","robots":{"status":"ok","http":200,"url":"https://www.aicpa-cima.com/robots.txt","bytes":299,"groups":5},"bots":{"GPTBot":["partial","*",4,0],"ChatGPT-User":["partial","*",4,0],"OAI-SearchBot":["partial","*",4,0],"ClaudeBot":["partial","*",4,0],"Claude-User":["partial","*",4,0],"Claude-SearchBot":["partial","*",4,0],"anthropic-ai":["partial","*",4,0],"PerplexityBot":["partial","*",4,0],"Perplexity-User":["partial","*",4,0],"Google-Extended":["partial","*",4,0],"Applebot-Extended":["partial","*",4,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["partial","*",4,0],"meta-externalagent":["partial","*",4,0],"cohere-ai":["partial","*",4,0],"Diffbot":["partial","*",4,0],"ContextFetchBot":["partial","*",4,0]},"llms":{"status":"missing","http":404,"bytes":233432},"home":{"status":"ok","http":200,"url":"https://www.aicpa-cima.com/","html_tokens":55348,"markdown_tokens":14,"pruned_tokens":14}},
{"rank":5712,"domain":"angular.io","origin":"https://angular.io","robots":{"status":"ok","http":200,"url":"https://angular.io/robots.txt","bytes":92,"groups":1},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"missing","http":404,"bytes":5899},"home":{"status":"ok","http":200,"url":"https://angular.dev/","html_tokens":25634,"markdown_tokens":296,"pruned_tokens":166}},
{"rank":5713,"domain":"livenation.com","origin":"https://livenation.com","robots":{"status":"ok","http":200,"url":"https://www.livenation.com/robots.txt","bytes":300,"groups":1},"bots":{"GPTBot":["partial","*",9,1],"ChatGPT-User":["partial","*",9,1],"OAI-SearchBot":["partial","*",9,1],"ClaudeBot":["partial","*",9,1],"Claude-User":["partial","*",9,1],"Claude-SearchBot":["partial","*",9,1],"anthropic-ai":["partial","*",9,1],"PerplexityBot":["partial","*",9,1],"Perplexity-User":["partial","*",9,1],"Google-Extended":["partial","*",9,1],"Applebot-Extended":["partial","*",9,1],"CCBot":["partial","*",9,1],"Bytespider":["partial","*",9,1],"Amazonbot":["partial","*",9,1],"meta-externalagent":["partial","*",9,1],"cohere-ai":["partial","*",9,1],"Diffbot":["partial","*",9,1],"ContextFetchBot":["partial","*",9,1]},"llms":{"status":"html","http":200,"bytes":247351},"home":{"status":"ok","http":200,"url":"https://www.livenation.com/","html_tokens":145063,"markdown_tokens":32,"pruned_tokens":32}},
{"rank":5714,"domain":"shizuoka.jp","origin":"https://shizuoka.jp","robots":{"status":"unreachable","http":null,"url":"https://shizuoka.jp/robots.txt","bytes":0,"groups":0,"error":"cannot resolve shizuoka.jp"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":5715,"domain":"santepubliquefrance.fr","origin":"https://santepubliquefrance.fr","robots":{"status":"ok","http":200,"url":"https://www.santepubliquefrance.fr/robots.txt","bytes":2027,"groups":1},"bots":{"GPTBot":["partial","*",34,18],"ChatGPT-User":["partial","*",34,18],"OAI-SearchBot":["partial","*",34,18],"ClaudeBot":["partial","*",34,18],"Claude-User":["partial","*",34,18],"Claude-SearchBot":["partial","*",34,18],"anthropic-ai":["partial","*",34,18],"PerplexityBot":["partial","*",34,18],"Perplexity-User":["partial","*",34,18],"Google-Extended":["partial","*",34,18],"Applebot-Extended":["partial","*",34,18],"CCBot":["partial","*",34,18],"Bytespider":["partial","*",34,18],"Amazonbot":["partial","*",34,18],"meta-externalagent":["partial","*",34,18],"cohere-ai":["partial","*",34,18],"Diffbot":["partial","*",34,18],"ContextFetchBot":["partial","*",34,18]},"llms":{"status":"missing","http":404,"bytes":200},"home":{"status":"ok","http":200,"url":"https://www.santepubliquefrance.fr/","html_tokens":55752,"markdown_tokens":172,"pruned_tokens":172}},
{"rank":5716,"domain":"myheritage.com","origin":"https://myheritage.com","robots":{"status":"ok","http":200,"url":"https://www.myheritage.com/robots.txt","bytes":38723,"groups":6},"bots":{"GPTBot":["partial","named",7,0],"ChatGPT-User":["partial","named",7,0],"OAI-SearchBot":["partial","named",7,0],"ClaudeBot":["partial","named",7,0],"Claude-User":["partial","*",273,3],"Claude-SearchBot":["partial","*",273,3],"anthropic-ai":["partial","named",7,0],"PerplexityBot":["partial","named",7,0],"Perplexity-User":["partial","*",273,3],"Google-Extended":["partial","named",7,0],"Applebot-Extended":["partial","named",7,0],"CCBot":["partial","*",273,3],"Bytespider":["blocked","named",1,0],"Amazonbot":["partial","named",14,0],"meta-externalagent":["partial","named",7,0],"cohere-ai":["partial","named",7,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",273,3]},"llms":{"status":"missing","http":404,"bytes":112369},"home":{"status":"ok","http":200,"url":"https://www.myheritage.com/","html_tokens":49203,"markdown_tokens":376,"pruned_tokens":51}},
{"rank":5717,"domain":"baymard.com","origin":"https://baymard.com","robots":{"status":"ok","http":200,"url":"https://baymard.com/robots.txt","bytes":768,"groups":1},"bots":{"GPTBot":["partial","*",25,3],"ChatGPT-User":["partial","*",25,3],"OAI-SearchBot":["partial","*",25,3],"ClaudeBot":["partial","*",25,3],"Claude-User":["partial","*",25,3],"Claude-SearchBot":["partial","*",25,3],"anthropic-ai":["partial","*",25,3],"PerplexityBot":["partial","*",25,3],"Perplexity-User":["partial","*",25,3],"Google-Extended":["partial","*",25,3],"Applebot-Extended":["partial","*",25,3],"CCBot":["partial","*",25,3],"Bytespider":["partial","*",25,3],"Amazonbot":["partial","*",25,3],"meta-externalagent":["partial","*",25,3],"cohere-ai":["partial","*",25,3],"Diffbot":["partial","*",25,3],"ContextFetchBot":["partial","*",25,3]},"llms":{"status":"present","http":200,"bytes":89992},"home":{"status":"ok","http":200,"url":"https://baymard.com/","html_tokens":148862,"markdown_tokens":696,"pruned_tokens":327}},
{"rank":5718,"domain":"msci.com","origin":"https://msci.com","robots":{"status":"ok","http":200,"url":"https://www.msci.com/robots.txt","bytes":1586,"groups":1},"bots":{"GPTBot":["partial","*",54,0],"ChatGPT-User":["partial","*",54,0],"OAI-SearchBot":["partial","*",54,0],"ClaudeBot":["partial","*",54,0],"Claude-User":["partial","*",54,0],"Claude-SearchBot":["partial","*",54,0],"anthropic-ai":["partial","*",54,0],"PerplexityBot":["partial","*",54,0],"Perplexity-User":["partial","*",54,0],"Google-Extended":["partial","*",54,0],"Applebot-Extended":["partial","*",54,0],"CCBot":["partial","*",54,0],"Bytespider":["partial","*",54,0],"Amazonbot":["partial","*",54,0],"meta-externalagent":["partial","*",54,0],"cohere-ai":["partial","*",54,0],"Diffbot":["partial","*",54,0],"ContextFetchBot":["partial","*",54,0]},"llms":{"status":"missing","http":404,"bytes":184021},"home":{"status":"ok","http":200,"url":"https://www.msci.com/","html_tokens":128687,"markdown_tokens":94,"pruned_tokens":94}},
{"rank":5719,"domain":"dreamhosters.com","origin":"https://dreamhosters.com","robots":{"status":"timeout","http":null,"url":"https://dreamhosters.com/robots.txt","bytes":0,"groups":0,"error":"ConnectTimeout"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":5720,"domain":"kahoot.com","origin":"https://kahoot.com","robots":{"status":"ok","http":200,"url":"https://kahoot.com/robots.txt","bytes":521,"groups":1},"bots":{"GPTBot":["partial","*",1,1],"ChatGPT-User":["partial","*",1,1],"OAI-SearchBot":["partial","*",1,1],"ClaudeBot":["partial","*",1,1],"Claude-User":["partial","*",1,1],"Claude-SearchBot":["partial","*",1,1],"anthropic-ai":["partial","*",1,1],"PerplexityBot":["partial","*",1,1],"Perplexity-User":["partial","*",1,1],"Google-Extended":["partial","*",1,1],"Applebot-Extended":["partial","*",1,1],"CCBot":["partial","*",1,1],"Bytespider":["partial","*",1,1],"Amazonbot":["partial","*",1,1],"meta-externalagent":["partial","*",1,1],"cohere-ai":["partial","*",1,1],"Diffbot":["partial","*",1,1],"ContextFetchBot":["partial","*",1,1]},"llms":{"status":"missing","http":404,"bytes":74767},"home":{"status":"ok","http":200,"url":"https://kahoot.com/","html_tokens":115275,"markdown_tokens":172,"pruned_tokens":53}},
{"rank":5721,"domain":"emerson.com","origin":"https://emerson.com","robots":{"status":"refused","http":403,"url":"https://www.emerson.com/robots.txt","bytes":4548,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":5722,"domain":"sciencebasedtargets.org","origin":"https://sciencebasedtargets.org","robots":{"status":"ok","http":200,"url":"https://sciencebasedtargets.org/robots.txt","bytes":94,"groups":1},"bots":{"GPTBot":["partial","*",1,0],"ChatGPT-User":["partial","*",1,0],"OAI-SearchBot":["partial","*",1,0],"ClaudeBot":["partial","*",1,0],"Claude-User":["partial","*",1,0],"Claude-SearchBot":["partial","*",1,0],"anthropic-ai":["partial","*",1,0],"PerplexityBot":["partial","*",1,0],"Perplexity-User":["partial","*",1,0],"Google-Extended":["partial","*",1,0],"Applebot-Extended":["partial","*",1,0],"CCBot":["partial","*",1,0],"Bytespider":["partial","*",1,0],"Amazonbot":["partial","*",1,0],"meta-externalagent":["partial","*",1,0],"cohere-ai":["partial","*",1,0],"Diffbot":["partial","*",1,0],"ContextFetchBot":["partial","*",1,0]},"llms":{"status":"missing","http":404,"bytes":211368},"home":{"status":"ok","http":200,"url":"https://sciencebasedtargets.org/","html_tokens":62054,"markdown_tokens":523,"pruned_tokens":174}},
{"rank":5723,"domain":"fairtrade.net","origin":"https://fairtrade.net","robots":{"status":"ok","http":200,"url":"https://fairtrade.net/robots.txt","bytes":22,"groups":1},"bots":{"GPTBot":["allowed","*",0,1],"ChatGPT-User":["allowed","*",0,1],"OAI-SearchBot":["allowed","*",0,1],"ClaudeBot":["allowed","*",0,1],"Claude-User":["allowed","*",0,1],"Claude-SearchBot":["allowed","*",0,1],"anthropic-ai":["allowed","*",0,1],"PerplexityBot":["allowed","*",0,1],"Perplexity-User":["allowed","*",0,1],"Google-Extended":["allowed","*",0,1],"Applebot-Extended":["allowed","*",0,1],"CCBot":["allowed","*",0,1],"Bytespider":["allowed","*",0,1],"Amazonbot":["allowed","*",0,1],"meta-externalagent":["allowed","*",0,1],"cohere-ai":["allowed","*",0,1],"Diffbot":["allowed","*",0,1],"ContextFetchBot":["allowed","*",0,1]},"llms":{"status":"missing","http":404,"bytes":180912},"home":{"status":"ok","http":200,"url":"https://www.fairtrade.net/us-en.html","html_tokens":53413,"markdown_tokens":366,"pruned_tokens":277}},
{"rank":5724,"domain":"scimagojr.com","origin":"https://scimagojr.com","robots":{"status":"ok","http":200,"url":"https://www.scimagojr.com/robots.txt","bytes":556,"groups":1},"bots":{"GPTBot":["partial","*",12,2],"ChatGPT-User":["partial","*",12,2],"OAI-SearchBot":["partial","*",12,2],"ClaudeBot":["partial","*",12,2],"Claude-User":["partial","*",12,2],"Claude-SearchBot":["partial","*",12,2],"anthropic-ai":["partial","*",12,2],"PerplexityBot":["partial","*",12,2],"Perplexity-User":["partial","*",12,2],"Google-Extended":["partial","*",12,2],"Applebot-Extended":["partial","*",12,2],"CCBot":["partial","*",12,2],"Bytespider":["partial","*",12,2],"Amazonbot":["partial","*",12,2],"meta-externalagent":["partial","*",12,2],"cohere-ai":["partial","*",12,2],"Diffbot":["partial","*",12,2],"ContextFetchBot":["partial","*",12,2]},"llms":{"status":"missing","http":404,"bytes":9756},"home":{"status":"http_error","http":403,"url":"https://scimagojr.com/"}},
{"rank":5725,"domain":"eventim.de","origin":"https://eventim.de","robots":{"status":"timeout","http":null,"url":"https://eventim.de/robots.txt","bytes":0,"groups":0,"error":"timed out after 10 s"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":5726,"domain":"rover.ebay.com","origin":"https://rover.ebay.com","robots":{"status":"ok","http":200,"url":"https://rover.ebay.com/robots.txt","bytes":11534,"groups":6},"bots":{"GPTBot":["partial","named",1,4],"ChatGPT-User":["partial","*",208,4],"OAI-SearchBot":["partial","*",208,4],"ClaudeBot":["partial","named",1,4],"Claude-User":["partial","*",208,4],"Claude-SearchBot":["partial","*",208,4],"anthropic-ai":["partial","named",1,4],"PerplexityBot":["partial","named",1,4],"Perplexity-User":["partial","*",208,4],"Google-Extended":["partial","*",208,4],"Applebot-Extended":["partial","named",1,4],"CCBot":["partial","named",1,4],"Bytespider":["partial","named",1,4],"Amazonbot":["partial","named",1,4],"meta-externalagent":["partial","named",1,4],"cohere-ai":["partial","*",208,4],"Diffbot":["partial","*",208,4],"ContextFetchBot":["partial","*",208,4]},"llms":{"status":"missing","http":404,"bytes":42},"home":{"status":"not_html","error":"unsupported content type image/gif"}},
{"rank":5727,"domain":"irna.ir","origin":"https://irna.ir","robots":{"status":"ok","http":200,"url":"https://www.irna.ir/robots.txt","bytes":2550,"groups":3},"bots":{"GPTBot":["partial","*",7,0],"ChatGPT-User":["partial","*",7,0],"OAI-SearchBot":["partial","*",7,0],"ClaudeBot":["partial","*",7,0],"Claude-User":["partial","*",7,0],"Claude-SearchBot":["partial","*",7,0],"anthropic-ai":["partial","*",7,0],"PerplexityBot":["partial","*",7,0],"Perplexity-User":["partial","*",7,0],"Google-Extended":["partial","*",7,0],"Applebot-Extended":["partial","*",7,0],"CCBot":["partial","*",7,0],"Bytespider":["partial","*",7,0],"Amazonbot":["partial","*",7,0],"meta-externalagent":["partial","*",7,0],"cohere-ai":["partial","*",7,0],"Diffbot":["partial","*",7,0],"ContextFetchBot":["partial","*",7,0]},"llms":{"status":"html","http":200,"bytes":6218},"home":{"status":"ok","http":200,"url":"https://www.irna.ir/","html_tokens":1522,"markdown_tokens":11,"pruned_tokens":11}},
{"rank":5728,"domain":"sony.co.jp","origin":"https://sony.co.jp","robots":{"status":"ok","http":200,"url":"https://www.sony.co.jp/robots.txt","bytes":74,"groups":1},"bots":{"GPTBot":["partial","*",1,0],"ChatGPT-User":["partial","*",1,0],"OAI-SearchBot":["partial","*",1,0],"ClaudeBot":["partial","*",1,0],"Claude-User":["partial","*",1,0],"Claude-SearchBot":["partial","*",1,0],"anthropic-ai":["partial","*",1,0],"PerplexityBot":["partial","*",1,0],"Perplexity-User":["partial","*",1,0],"Google-Extended":["partial","*",1,0],"Applebot-Extended":["partial","*",1,0],"CCBot":["partial","*",1,0],"Bytespider":["partial","*",1,0],"Amazonbot":["partial","*",1,0],"meta-externalagent":["partial","*",1,0],"cohere-ai":["partial","*",1,0],"Diffbot":["partial","*",1,0],"ContextFetchBot":["partial","*",1,0]},"llms":{"status":"missing","http":404,"bytes":158025},"home":{"status":"ok","http":200,"url":"https://www.sony.co.jp/","html_tokens":60266,"markdown_tokens":22,"pruned_tokens":18}},
{"rank":5729,"domain":"bitsdujour.com","origin":"https://bitsdujour.com","robots":{"status":"ok","http":200,"url":"https://www.bitsdujour.com/robots.txt","bytes":294,"groups":1},"bots":{"GPTBot":["partial","*",13,0],"ChatGPT-User":["partial","*",13,0],"OAI-SearchBot":["partial","*",13,0],"ClaudeBot":["partial","*",13,0],"Claude-User":["partial","*",13,0],"Claude-SearchBot":["partial","*",13,0],"anthropic-ai":["partial","*",13,0],"PerplexityBot":["partial","*",13,0],"Perplexity-User":["partial","*",13,0],"Google-Extended":["partial","*",13,0],"Applebot-Extended":["partial","*",13,0],"CCBot":["partial","*",13,0],"Bytespider":["partial","*",13,0],"Amazonbot":["partial","*",13,0],"meta-externalagent":["partial","*",13,0],"cohere-ai":["partial","*",13,0],"Diffbot":["partial","*",13,0],"ContextFetchBot":["partial","*",13,0]},"llms":{"status":"missing","http":404,"bytes":5036},"home":{"status":"ok","http":200,"url":"https://www.bitsdujour.com/","html_tokens":20763,"markdown_tokens":93,"pruned_tokens":93}},
{"rank":5730,"domain":"synxis.com","origin":"https://synxis.com","robots":{"status":"html","http":200,"url":"https://www.avenhospitality.com/","bytes":376211,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":5731,"domain":"pgatour.com","origin":"https://pgatour.com","robots":{"status":"ok","http":200,"url":"https://www.pgatour.com/robots.txt","bytes":840,"groups":1},"bots":{"GPTBot":["partial","*",15,0],"ChatGPT-User":["partial","*",15,0],"OAI-SearchBot":["partial","*",15,0],"ClaudeBot":["partial","*",15,0],"Claude-User":["partial","*",15,0],"Claude-SearchBot":["partial","*",15,0],"anthropic-ai":["partial","*",15,0],"PerplexityBot":["partial","*",15,0],"Perplexity-User":["partial","*",15,0],"Google-Extended":["partial","*",15,0],"Applebot-Extended":["partial","*",15,0],"CCBot":["partial","*",15,0],"Bytespider":["partial","*",15,0],"Amazonbot":["partial","*",15,0],"meta-externalagent":["partial","*",15,0],"cohere-ai":["partial","*",15,0],"Diffbot":["partial","*",15,0],"ContextFetchBot":["partial","*",15,0]},"llms":{"status":"present","http":200,"bytes":9093},"home":{"status":"ok","http":200,"url":"https://www.pgatour.com/","html_tokens":494307,"markdown_tokens":84,"pruned_tokens":84}},
{"rank":5732,"domain":"euskadi.eus","origin":"https://euskadi.eus","robots":{"status":"ok","http":200,"url":"https://www.euskadi.eus/robots.txt","bytes":33250,"groups":1},"bots":{"GPTBot":["partial","*",958,21],"ChatGPT-User":["partial","*",958,21],"OAI-SearchBot":["partial","*",958,21],"ClaudeBot":["partial","*",958,21],"Claude-User":["partial","*",958,21],"Claude-SearchBot":["partial","*",958,21],"anthropic-ai":["partial","*",958,21],"PerplexityBot":["partial","*",958,21],"Perplexity-User":["partial","*",958,21],"Google-Extended":["partial","*",958,21],"Applebot-Extended":["partial","*",958,21],"CCBot":["partial","*",958,21],"Bytespider":["partial","*",958,21],"Amazonbot":["partial","*",958,21],"meta-externalagent":["partial","*",958,21],"cohere-ai":["partial","*",958,21],"Diffbot":["partial","*",958,21],"ContextFetchBot":["partial","*",958,21]},"llms":{"status":"missing","http":404,"bytes":40},"home":{"status":"ok","http":200,"url":"https://www.euskadi.eus/hasiera/","html_tokens":8698,"markdown_tokens":566,"pruned_tokens":144}},
{"rank":5733,"domain":"franceculture.fr","origin":"https://franceculture.fr","robots":{"status":"ok","http":200,"url":"https://www.radiofrance.fr/robots.txt","bytes":4390,"groups":73},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["blocked","named",1,0],"OAI-SearchBot":["blocked","named",1,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",10,0],"Claude-SearchBot":["partial","*",10,0],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["partial","*",10,0],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["partial","*",10,0],"ContextFetchBot":["partial","*",10,0]},"llms":{"status":"missing","http":404,"bytes":153724},"home":{"status":"ok","http":200,"url":"https://www.radiofrance.fr/franceculture","html_tokens":478456,"markdown_tokens":1349,"pruned_tokens":135}},
{"rank":5734,"domain":"qwant.com","origin":"https://qwant.com","robots":{"status":"ok","http":200,"url":"https://www.qwant.com/robots.txt","bytes":2376,"groups":5},"bots":{"GPTBot":["partial","named",4,0],"ChatGPT-User":["partial","named",4,0],"OAI-SearchBot":["partial","*",4,0],"ClaudeBot":["partial","named",4,0],"Claude-User":["partial","named",4,0],"Claude-SearchBot":["partial","*",4,0],"anthropic-ai":["partial","named",4,0],"PerplexityBot":["partial","*",4,0],"Perplexity-User":["partial","named",4,0],"Google-Extended":["partial","named",4,0],"Applebot-Extended":["partial","named",4,0],"CCBot":["partial","named",4,0],"Bytespider":["partial","named",4,0],"Amazonbot":["partial","named",4,0],"meta-externalagent":["partial","named",4,0],"cohere-ai":["partial","named",4,0],"Diffbot":["partial","named",4,0],"ContextFetchBot":["partial","*",4,0]},"llms":{"status":"missing","http":404,"bytes":23278},"home":{"status":"ok","http":200,"url":"https://www.qwant.com/","html_tokens":36150,"markdown_tokens":26,"pruned_tokens":26}},
{"rank":5735,"domain":"hubei.gov.cn","origin":"https://www.hubei.gov.cn","robots":{"status":"missing","http":404,"url":"https://www.hubei.gov.cn/robots.txt","bytes":16525,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":16525},"home":{"status":"http_error","http":412,"url":"https://www.hubei.gov.cn/"}},
{"rank":5736,"domain":"ssense.com","origin":"https://ssense.com","robots":{"status":"ok","http":200,"url":"https://www.ssense.com/robots.txt","bytes":19071,"groups":1},"bots":{"GPTBot":["partial","*",1089,0],"ChatGPT-User":["partial","*",1089,0],"OAI-SearchBot":["partial","*",1089,0],"ClaudeBot":["partial","*",1089,0],"Claude-User":["partial","*",1089,0],"Claude-SearchBot":["partial","*",1089,0],"anthropic-ai":["partial","*",1089,0],"PerplexityBot":["partial","*",1089,0],"Perplexity-User":["partial","*",1089,0],"Google-Extended":["partial","*",1089,0],"Applebot-Extended":["partial","*",1089,0],"CCBot":["partial","*",1089,0],"Bytespider":["partial","*",1089,0],"Amazonbot":["partial","*",1089,0],"meta-externalagent":["partial","*",1089,0],"cohere-ai":["partial","*",1089,0],"Diffbot":["partial","*",1089,0],"ContextFetchBot":["partial","*",1089,0]},"llms":{"status":"missing","http":404,"bytes":402034},"home":{"status":"http_error","http":403,"url":"https://www.ssense.com/"}},
{"rank":5737,"domain":"asuscomm.com","origin":"http://asuscomm.com","robots":{"status":"html","http":200,"url":"http://asuscomm.com/robots.txt","bytes":2967,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":5738,"domain":"ecowatch.com","origin":"https://ecowatch.com","robots":{"status":"ok","http":200,"url":"https://ecowatch.com/robots.txt","bytes":644,"groups":10},"bots":{"GPTBot":["partial","*",6,1],"ChatGPT-User":["partial","*",6,1],"OAI-SearchBot":["partial","*",6,1],"ClaudeBot":["partial","*",6,1],"Claude-User":["partial","*",6,1],"Claude-SearchBot":["partial","*",6,1],"anthropic-ai":["partial","*",6,1],"PerplexityBot":["partial","*",6,1],"Perplexity-User":["partial","*",6,1],"Google-Extended":["partial","*",6,1],"Applebot-Extended":["partial","*",6,1],"CCBot":["partial","*",6,1],"Bytespider":["partial","*",6,1],"Amazonbot":["partial","*",6,1],"meta-externalagent":["partial","*",6,1],"cohere-ai":["partial","*",6,1],"Diffbot":["partial","*",6,1],"ContextFetchBot":["partial","*",6,1]},"llms":{"status":"missing","http":404,"bytes":180804},"home":{"status":"ok","http":200,"url":"https://www.ecowatch.com/","html_tokens":64405,"markdown_tokens":263,"pruned_tokens":107}},
{"rank":5739,"domain":"wmich.edu","origin":"https://wmich.edu","robots":{"status":"refused","http":403,"url":"https://wmich.edu/robots.txt","bytes":560,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":5740,"domain":"wada-ama.org","origin":"https://wada-ama.org","robots":{"status":"html","http":202,"url":"https://wada-ama.org/robots.txt","bytes":2427,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":5741,"domain":"paint.net","origin":"https://paint.net","robots":{"status":"missing","http":404,"url":"https://paint.net/robots.txt","bytes":355,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":355},"home":{"status":"ok","http":200,"url":"https://paint.net/","html_tokens":3138,"markdown_tokens":599,"pruned_tokens":251}},
{"rank":5742,"domain":"uma.es","origin":"https://www.uma.es","robots":{"status":"missing","http":404,"url":"https://www.uma.es/robots.txt","bytes":208,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":46828},"home":{"status":"ok","http":200,"url":"https://www.uma.es/","html_tokens":24278,"markdown_tokens":136,"pruned_tokens":136}},
{"rank":5743,"domain":"gcs-web.com","origin":"https://gcs-web.com","robots":{"status":"timeout","http":null,"url":"https://gcs-web.com/robots.txt","bytes":0,"groups":0,"error":"ConnectTimeout"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":5744,"domain":"kaist.ac.kr","origin":"https://kaist.ac.kr","robots":{"status":"ok","http":200,"url":"https://kaist.ac.kr/robots.txt","bytes":23,"groups":1},"bots":{"GPTBot":["allowed","*",0,1],"ChatGPT-User":["allowed","*",0,1],"OAI-SearchBot":["allowed","*",0,1],"ClaudeBot":["allowed","*",0,1],"Claude-User":["allowed","*",0,1],"Claude-SearchBot":["allowed","*",0,1],"anthropic-ai":["allowed","*",0,1],"PerplexityBot":["allowed","*",0,1],"Perplexity-User":["allowed","*",0,1],"Google-Extended":["allowed","*",0,1],"Applebot-Extended":["allowed","*",0,1],"CCBot":["allowed","*",0,1],"Bytespider":["allowed","*",0,1],"Amazonbot":["allowed","*",0,1],"meta-externalagent":["allowed","*",0,1],"cohere-ai":["allowed","*",0,1],"Diffbot":["allowed","*",0,1],"ContextFetchBot":["allowed","*",0,1]},"llms":{"status":"missing","http":404,"bytes":3253},"home":{"status":"ok","http":200,"url":"https://kaist.ac.kr/","html_tokens":115,"markdown_tokens":10,"pruned_tokens":10}},
{"rank":5745,"domain":"napster.com","origin":"https://napster.com","robots":{"status":"ok","http":200,"url":"https://www.napster.com/robots.txt","bytes":745,"groups":5},"bots":{"GPTBot":["partial","named",4,1],"ChatGPT-User":["partial","*",14,1],"OAI-SearchBot":["allowed","named",0,1],"ClaudeBot":["partial","*",14,1],"Claude-User":["partial","*",14,1],"Claude-SearchBot":["partial","*",14,1],"anthropic-ai":["partial","*",14,1],"PerplexityBot":["partial","named",4,1],"Perplexity-User":["partial","*",14,1],"Google-Extended":["partial","*",14,1],"Applebot-Extended":["partial","*",14,1],"CCBot":["partial","*",14,1],"Bytespider":["partial","*",14,1],"Amazonbot":["partial","*",14,1],"meta-externalagent":["partial","*",14,1],"cohere-ai":["partial","*",14,1],"Diffbot":["partial","*",14,1],"ContextFetchBot":["partial","*",14,1]},"llms":{"status":"present","http":200,"bytes":5453},"home":{"status":"ok","http":200,"url":"https://www.napster.com/","html_tokens":44000,"markdown_tokens":167,"pruned_tokens":67}},
{"rank":5746,"domain":"rcn.com","origin":"https://rcn.com","robots":{"status":"ok","http":200,"url":"https://www.rcn.com/robots.txt","bytes":0,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"html","http":200,"bytes":0},"home":{"status":"ok","http":200,"url":"https://www.astound.com/","html_tokens":129918,"markdown_tokens":1740,"pruned_tokens":159}},
{"rank":5747,"domain":"casinotest2.com","origin":"https://casinotest2.com","robots":{"status":"ok","http":200,"url":"https://casinotest2.com/robots.txt","bytes":107,"groups":3},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":66553},"home":{"status":"ok","http":200,"url":"https://casinotest2.com/","html_tokens":211513,"markdown_tokens":5311,"pruned_tokens":117}},
{"rank":5748,"domain":"instacart.com","origin":"https://instacart.com","robots":{"status":"ok","http":200,"url":"https://www.instacart.com/robots.txt","bytes":13851,"groups":23},"bots":{"GPTBot":["blocked","*",1,0],"ChatGPT-User":["partial","named",30,2],"OAI-SearchBot":["partial","named",30,1],"ClaudeBot":["blocked","*",1,0],"Claude-User":["blocked","*",1,0],"Claude-SearchBot":["blocked","*",1,0],"anthropic-ai":["blocked","*",1,0],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["blocked","named",1,0],"Google-Extended":["blocked","*",1,0],"Applebot-Extended":["blocked","*",1,0],"CCBot":["blocked","*",1,0],"Bytespider":["blocked","*",1,0],"Amazonbot":["blocked","*",1,0],"meta-externalagent":["blocked","named",2,0],"cohere-ai":["blocked","*",1,0],"Diffbot":["blocked","*",1,0],"ContextFetchBot":["blocked","*",1,0]},"llms":{"status":"disallowed"},"home":{"status":"disallowed"}},
{"rank":5749,"domain":"businessinsider.com.au","origin":"http://businessinsider.com.au","robots":{"status":"missing","http":404,"url":"http://businessinsider.com.au/robots.txt","bytes":146,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":146},"home":{"status":"http_error","http":404,"url":"http://businessinsider.com.au/"}},
{"rank":5750,"domain":"rescue.org","origin":"https://rescue.org","robots":{"status":"refused","http":403,"url":"https://rescue.org/robots.txt","bytes":560,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":5751,"domain":"google.com.bd","origin":"https://google.com.bd","robots":{"status":"ok","http":200,"url":"https://www.google.com.bd/robots.txt","bytes":6624,"groups":4},"bots":{"GPTBot":["partial","*",169,64],"ChatGPT-User":["partial","*",169,64],"OAI-SearchBot":["partial","*",169,64],"ClaudeBot":["partial","*",169,64],"Claude-User":["partial","*",169,64],"Claude-SearchBot":["partial","*",169,64],"anthropic-ai":["partial","*",169,64],"PerplexityBot":["partial","*",169,64],"Perplexity-User":["partial","*",169,64],"Google-Extended":["partial","*",169,64],"Applebot-Extended":["partial","*",169,64],"CCBot":["partial","*",169,64],"Bytespider":["partial","*",169,64],"Amazonbot":["partial","*",169,64],"meta-externalagent":["partial","*",169,64],"cohere-ai":["partial","*",169,64],"Diffbot":["partial","*",169,64],"ContextFetchBot":["partial","*",169,64]},"llms":{"status":"missing","http":404,"bytes":1569},"home":{"status":"ok","http":200,"url":"https://www.google.com/","html_tokens":21310,"markdown_tokens":30,"pruned_tokens":30}},
{"rank":5752,"domain":"elcomercio.pe","origin":"https://elcomercio.pe","robots":{"status":"ok","http":200,"url":"https://elcomercio.pe/robots.txt","bytes":4231,"groups":2},"bots":{"GPTBot":["partial","*",4,0],"ChatGPT-User":["partial","*",4,0],"OAI-SearchBot":["partial","*",4,0],"ClaudeBot":["partial","*",4,0],"Claude-User":["partial","*",4,0],"Claude-SearchBot":["partial","*",4,0],"anthropic-ai":["partial","*",4,0],"PerplexityBot":["partial","*",4,0],"Perplexity-User":["partial","*",4,0],"Google-Extended":["partial","*",4,0],"Applebot-Extended":["partial","*",4,0],"CCBot":["partial","*",4,0],"Bytespider":["partial","*",4,0],"Amazonbot":["partial","*",4,0],"meta-externalagent":["partial","*",4,0],"cohere-ai":["partial","*",4,0],"Diffbot":["partial","*",4,0],"ContextFetchBot":["partial","*",4,0]},"llms":{"status":"missing","http":404,"bytes":2357535},"home":{"status":"ok","http":200,"url":"https://elcomercio.pe/","html_tokens":186004,"markdown_tokens":347,"pruned_tokens":174}},
{"rank":5753,"domain":"ef.com","origin":"https://ef.com","robots":{"status":"ok","http":200,"url":"https://www.ef.com/robots.txt","bytes":406,"groups":2},"bots":{"GPTBot":["partial","*",13,0],"ChatGPT-User":["partial","*",13,0],"OAI-SearchBot":["partial","*",13,0],"ClaudeBot":["partial","*",13,0],"Claude-User":["partial","*",13,0],"Claude-SearchBot":["partial","*",13,0],"anthropic-ai":["partial","*",13,0],"PerplexityBot":["partial","*",13,0],"Perplexity-User":["partial","*",13,0],"Google-Extended":["partial","*",13,0],"Applebot-Extended":["partial","*",13,0],"CCBot":["partial","*",13,0],"Bytespider":["partial","*",13,0],"Amazonbot":["partial","*",13,0],"meta-externalagent":["partial","*",13,0],"cohere-ai":["partial","*",13,0],"Diffbot":["partial","*",13,0],"ContextFetchBot":["partial","*",13,0]},"llms":{"status":"missing","http":404,"bytes":1039879},"home":{"status":"ok","http":200,"url":"https://www.ef.com/","html_tokens":19974,"markdown_tokens":367,"pruned_tokens":175}},
{"rank":5754,"domain":"jta.org","origin":"https://jta.org","robots":{"status":"ok","http":200,"url":"https://jta.org/robots.txt","bytes":340,"groups":7},"bots":{"GPTBot":["partial","*",2,0],"ChatGPT-User":["partial","*",2,0],"OAI-SearchBot":["partial","*",2,0],"ClaudeBot":["partial","*",2,0],"Claude-User":["partial","*",2,0],"Claude-SearchBot":["partial","*",2,0],"anthropic-ai":["partial","*",2,0],"PerplexityBot":["partial","*",2,0],"Perplexity-User":["partial","*",2,0],"Google-Extended":["partial","*",2,0],"Applebot-Extended":["partial","*",2,0],"CCBot":["partial","*",2,0],"Bytespider":["partial","*",2,0],"Amazonbot":["partial","*",2,0],"meta-externalagent":["partial","*",2,0],"cohere-ai":["partial","*",2,0],"Diffbot":["partial","*",2,0],"ContextFetchBot":["partial","*",2,0]},"llms":{"status":"missing","http":404,"bytes":127380},"home":{"status":"ok","http":200,"url":"https://www.jta.org/","html_tokens":45738,"markdown_tokens":12,"pruned_tokens":12}},
{"rank":5755,"domain":"edf.org","origin":"https://edf.org","robots":{"status":"ok","http":200,"url":"https://www.edf.org/robots.txt","bytes":5068,"groups":1},"bots":{"GPTBot":["partial","*",100,18],"ChatGPT-User":["partial","*",100,18],"OAI-SearchBot":["partial","*",100,18],"ClaudeBot":["partial","*",100,18],"Claude-User":["partial","*",100,18],"Claude-SearchBot":["partial","*",100,18],"anthropic-ai":["partial","*",100,18],"PerplexityBot":["partial","*",100,18],"Perplexity-User":["partial","*",100,18],"Google-Extended":["partial","*",100,18],"Applebot-Extended":["partial","*",100,18],"CCBot":["partial","*",100,18],"Bytespider":["partial","*",100,18],"Amazonbot":["partial","*",100,18],"meta-externalagent":["partial","*",100,18],"cohere-ai":["partial","*",100,18],"Diffbot":["partial","*",100,18],"ContextFetchBot":["partial","*",100,18]},"llms":{"status":"missing","http":404,"bytes":568},"home":{"status":"ok","http":200,"url":"https://www.edf.org/","html_tokens":53610,"markdown_tokens":962,"pruned_tokens":305}},
{"rank":5756,"domain":"louisville.edu","origin":"https://louisville.edu","robots":{"status":"ok","http":200,"url":"https://louisville.edu/robots.txt","bytes":1691,"groups":3},"bots":{"GPTBot":["partial","*",35,18],"ChatGPT-User":["partial","*",35,18],"OAI-SearchBot":["partial","*",35,18],"ClaudeBot":["partial","*",35,18],"Claude-User":["partial","*",35,18],"Claude-SearchBot":["partial","*",35,18],"anthropic-ai":["partial","*",35,18],"PerplexityBot":["partial","*",35,18],"Perplexity-User":["partial","*",35,18],"Google-Extended":["partial","*",35,18],"Applebot-Extended":["partial","*",35,18],"CCBot":["partial","*",35,18],"Bytespider":["partial","*",35,18],"Amazonbot":["partial","*",35,18],"meta-externalagent":["partial","*",35,18],"cohere-ai":["partial","*",35,18],"Diffbot":["partial","*",35,18],"ContextFetchBot":["partial","*",35,18]},"llms":{"status":"missing","http":404,"bytes":32924},"home":{"status":"ok","http":200,"url":"https://louisville.edu/","html_tokens":63485,"markdown_tokens":42,"pruned_tokens":42}},
{"rank":5757,"domain":"musee-orsay.fr","origin":"https://musee-orsay.fr","robots":{"status":"ok","http":200,"url":"https://www.musee-orsay.fr/robots.txt","bytes":1838,"groups":1},"bots":{"GPTBot":["partial","*",32,18],"ChatGPT-User":["partial","*",32,18],"OAI-SearchBot":["partial","*",32,18],"ClaudeBot":["partial","*",32,18],"Claude-User":["partial","*",32,18],"Claude-SearchBot":["partial","*",32,18],"anthropic-ai":["partial","*",32,18],"PerplexityBot":["partial","*",32,18],"Perplexity-User":["partial","*",32,18],"Google-Extended":["partial","*",32,18],"Applebot-Extended":["partial","*",32,18],"CCBot":["partial","*",32,18],"Bytespider":["partial","*",32,18],"Amazonbot":["partial","*",32,18],"meta-externalagent":["partial","*",32,18],"cohere-ai":["partial","*",32,18],"Diffbot":["partial","*",32,18],"ContextFetchBot":["partial","*",32,18]},"llms":{"status":"missing","http":404,"bytes":192},"home":{"status":"http_error","http":403,"url":"https://www.musee-orsay.fr/"}},
{"rank":5758,"domain":"infn.it","origin":"https://www.infn.it","robots":{"status":"ok","http":200,"url":"https://www.infn.it/robots.txt","bytes":396,"groups":1},"bots":{"GPTBot":["partial","*",1,1],"ChatGPT-User":["partial","*",1,1],"OAI-SearchBot":["partial","*",1,1],"ClaudeBot":["partial","*",1,1],"Claude-User":["partial","*",1,1],"Claude-SearchBot":["partial","*",1,1],"anthropic-ai":["partial","*",1,1],"PerplexityBot":["partial","*",1,1],"Perplexity-User":["partial","*",1,1],"Google-Extended":["partial","*",1,1],"Applebot-Extended":["partial","*",1,1],"CCBot":["partial","*",1,1],"Bytespider":["partial","*",1,1],"Amazonbot":["partial","*",1,1],"meta-externalagent":["partial","*",1,1],"cohere-ai":["partial","*",1,1],"Diffbot":["partial","*",1,1],"ContextFetchBot":["partial","*",1,1]},"llms":{"status":"missing","http":404,"bytes":125636},"home":{"status":"ok","http":200,"url":"https://www.infn.it/","html_tokens":52095,"markdown_tokens":85,"pruned_tokens":85}},
{"rank":5759,"domain":"londonstockexchange.com","origin":"https://londonstockexchange.com","robots":{"status":"ok","http":200,"url":"https://www.londonstockexchange.com/robots.txt","bytes":254,"groups":1},"bots":{"GPTBot":["partial","*",5,0],"ChatGPT-User":["partial","*",5,0],"OAI-SearchBot":["partial","*",5,0],"ClaudeBot":["partial","*",5,0],"Claude-User":["partial","*",5,0],"Claude-SearchBot":["partial","*",5,0],"anthropic-ai":["partial","*",5,0],"PerplexityBot":["partial","*",5,0],"Perplexity-User":["partial","*",5,0],"Google-Extended":["partial","*",5,0],"Applebot-Extended":["partial","*",5,0],"CCBot":["partial","*",5,0],"Bytespider":["partial","*",5,0],"Amazonbot":["partial","*",5,0],"meta-externalagent":["partial","*",5,0],"cohere-ai":["partial","*",5,0],"Diffbot":["partial","*",5,0],"ContextFetchBot":["partial","*",5,0]},"llms":{"status":"html","http":200,"bytes":61809},"home":{"status":"ok","http":200,"url":"https://www.londonstockexchange.com/","html_tokens":13749,"markdown_tokens":0,"pruned_tokens":0}},
{"rank":5760,"domain":"noblogs.org","origin":"https://noblogs.org","robots":{"status":"unreachable","http":null,"url":"https://noblogs.org/robots.txt","bytes":0,"groups":0,"error":"cannot resolve noblogs.org"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":5761,"domain":"trouw.nl","origin":"https://trouw.nl","robots":{"status":"ok","http":200,"url":"https://www.trouw.nl/robots.txt","bytes":2984,"groups":41},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["blocked","named",1,0],"OAI-SearchBot":["blocked","named",1,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",14,2],"Claude-SearchBot":["blocked","named",1,0],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["blocked","named",1,0],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["partial","*",14,2],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",14,2]},"llms":{"status":"missing","http":404,"bytes":75940},"home":{"status":"ok","http":200,"url":"https://www.trouw.nl/","html_tokens":279772,"markdown_tokens":1221,"pruned_tokens":154}},
{"rank":5762,"domain":"radio.cz","origin":"https://radio.cz","robots":{"status":"ok","http":200,"url":"https://www.radio.cz/robots.txt","bytes":888,"groups":1},"bots":{"GPTBot":["partial","*",32,0],"ChatGPT-User":["partial","*",32,0],"OAI-SearchBot":["partial","*",32,0],"ClaudeBot":["partial","*",32,0],"Claude-User":["partial","*",32,0],"Claude-SearchBot":["partial","*",32,0],"anthropic-ai":["partial","*",32,0],"PerplexityBot":["partial","*",32,0],"Perplexity-User":["partial","*",32,0],"Google-Extended":["partial","*",32,0],"Applebot-Extended":["partial","*",32,0],"CCBot":["partial","*",32,0],"Bytespider":["partial","*",32,0],"Amazonbot":["partial","*",32,0],"meta-externalagent":["partial","*",32,0],"cohere-ai":["partial","*",32,0],"Diffbot":["partial","*",32,0],"ContextFetchBot":["partial","*",32,0]},"llms":{"status":"missing","http":404,"bytes":25603},"home":{"status":"disallowed","error":"robots.txt of english.radio.cz not readable (refused)"}},
{"rank":5763,"domain":"technion.ac.il","origin":"https://technion.ac.il","robots":{"status":"html","http":200,"url":"https://www.technion.ac.il/","bytes":1426394,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":5764,"domain":"mondaq.com","origin":"https://mondaq.com","robots":{"status":"ok","http":200,"url":"https://www.mondaq.com/robots.txt","bytes":3497,"groups":36},"bots":{"GPTBot":["partial","*",45,0],"ChatGPT-User":["partial","*",45,0],"OAI-SearchBot":["partial","*",45,0],"ClaudeBot":["partial","*",45,0],"Claude-User":["partial","*",45,0],"Claude-SearchBot":["partial","*",45,0],"anthropic-ai":["partial","*",45,0],"PerplexityBot":["partial","*",45,0],"Perplexity-User":["partial","*",45,0],"Google-Extended":["partial","*",45,0],"Applebot-Extended":["partial","*",45,0],"CCBot":["partial","*",45,0],"Bytespider":["partial","*",45,0],"Amazonbot":["partial","*",45,0],"meta-externalagent":["partial","*",45,0],"cohere-ai":["partial","*",45,0],"Diffbot":["partial","*",45,0],"ContextFetchBot":["partial","*",45,0]},"llms":{"status":"missing","http":404,"bytes":19542},"home":{"status":"ok","http":200,"url":"https://www.mondaq.com/","html_tokens":36113,"markdown_tokens":214,"pruned_tokens":145}},
{"rank":5765,"domain":"biospace.com","origin":"https://biospace.com","robots":{"status":"ok","http":200,"url":"https://www.biospace.com/robots.txt","bytes":304,"groups":2},"bots":{"GPTBot":["partial","*",3,0],"ChatGPT-User":["partial","*",3,0],"OAI-SearchBot":["partial","*",3,0],"ClaudeBot":["partial","*",3,0],"Claude-User":["partial","*",3,0],"Claude-SearchBot":["partial","*",3,0],"anthropic-ai":["partial","*",3,0],"PerplexityBot":["partial","*",3,0],"Perplexity-User":["partial","*",3,0],"Google-Extended":["partial","*",3,0],"Applebot-Extended":["partial","*",3,0],"CCBot":["partial","*",3,0],"Bytespider":["partial","*",3,0],"Amazonbot":["partial","*",3,0],"meta-externalagent":["partial","named",1,0],"cohere-ai":["partial","*",3,0],"Diffbot":["partial","*",3,0],"ContextFetchBot":["partial","*",3,0]},"llms":{"status":"missing","http":404,"bytes":107033},"home":{"status":"ok","http":200,"url":"https://www.biospace.com/","html_tokens":151866,"markdown_tokens":3217,"pruned_tokens":71}},
{"rank":5766,"domain":"passkey.com","origin":"https://passkey.com","robots":{"status":"html","http":200,"url":"https://www.cvent.com/en/event-marketing-management/passkey-room-block-management/log-in","bytes":38929,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":5767,"domain":"telegraphindia.com","origin":"https://telegraphindia.com","robots":{"status":"refused","http":403,"url":"https://www.telegraphindia.com/robots.txt","bytes":390,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":5768,"domain":"axis.com","origin":"https://axis.com","robots":{"status":"ok","http":200,"url":"https://www.axis.com/robots.txt","bytes":1656,"groups":1},"bots":{"GPTBot":["partial","*",23,18],"ChatGPT-User":["partial","*",23,18],"OAI-SearchBot":["partial","*",23,18],"ClaudeBot":["partial","*",23,18],"Claude-User":["partial","*",23,18],"Claude-SearchBot":["partial","*",23,18],"anthropic-ai":["partial","*",23,18],"PerplexityBot":["partial","*",23,18],"Perplexity-User":["partial","*",23,18],"Google-Extended":["partial","*",23,18],"Applebot-Extended":["partial","*",23,18],"CCBot":["partial","*",23,18],"Bytespider":["partial","*",23,18],"Amazonbot":["partial","*",23,18],"meta-externalagent":["partial","*",23,18],"cohere-ai":["partial","*",23,18],"Diffbot":["partial","*",23,18],"ContextFetchBot":["partial","*",23,18]},"llms":{"status":"missing","http":404,"bytes":8351},"home":{"status":"ok","http":200,"url":"https://www.axis.com/en-us","html_tokens":72700,"markdown_tokens":735,"pruned_tokens":212}},
{"rank":5769,"domain":"enamad.ir","origin":"https://enamad.ir","robots":{"status":"timeout","http":null,"url":"https://enamad.ir/robots.txt","bytes":0,"groups":0,"error":"ConnectTimeout"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":5770,"domain":"planetary.org","origin":"https://planetary.org","robots":{"status":"ok","http":200,"url":"https://www.planetary.org/robots.txt","bytes":317,"groups":1},"bots":{"GPTBot":["partial","*",6,0],"ChatGPT-User":["partial","*",6,0],"OAI-SearchBot":["partial","*",6,0],"ClaudeBot":["partial","*",6,0],"Claude-User":["partial","*",6,0],"Claude-SearchBot":["partial","*",6,0],"anthropic-ai":["partial","*",6,0],"PerplexityBot":["partial","*",6,0],"Perplexity-User":["partial","*",6,0],"Google-Extended":["partial","*",6,0],"Applebot-Extended":["partial","*",6,0],"CCBot":["partial","*",6,0],"Bytespider":["partial","*",6,0],"Amazonbot":["partial","*",6,0],"meta-externalagent":["partial","*",6,0],"cohere-ai":["partial","*",6,0],"Diffbot":["partial","*",6,0],"ContextFetchBot":["partial","*",6,0]},"llms":{"status":"missing","http":404,"bytes":60248},"home":{"status":"ok","http":200,"url":"https://www.planetary.org/","html_tokens":21756,"markdown_tokens":138,"pruned_tokens":139}},
{"rank":5771,"domain":"bme.hu","origin":"https://bme.hu","robots":{"status":"ok","http":200,"url":"https://www.bme.hu/robots.txt","bytes":2027,"groups":1},"bots":{"GPTBot":["partial","*",34,18],"ChatGPT-User":["partial","*",34,18],"OAI-SearchBot":["partial","*",34,18],"ClaudeBot":["partial","*",34,18],"Claude-User":["partial","*",34,18],"Claude-SearchBot":["partial","*",34,18],"anthropic-ai":["partial","*",34,18],"PerplexityBot":["partial","*",34,18],"Perplexity-User":["partial","*",34,18],"Google-Extended":["partial","*",34,18],"Applebot-Extended":["partial","*",34,18],"CCBot":["partial","*",34,18],"Bytespider":["partial","*",34,18],"Amazonbot":["partial","*",34,18],"meta-externalagent":["partial","*",34,18],"cohere-ai":["partial","*",34,18],"Diffbot":["partial","*",34,18],"ContextFetchBot":["partial","*",34,18]},"llms":{"status":"missing","http":404,"bytes":184},"home":{"status":"ok","http":200,"url":"https://www.bme.hu/","html_tokens":65834,"markdown_tokens":157,"pruned_tokens":92}},
{"rank":5772,"domain":"twz.com","origin":"https://twz.com","robots":{"status":"ok","http":200,"url":"https://www.twz.com/robots.txt","bytes":358,"groups":2},"bots":{"GPTBot":["partial","*",6,1],"ChatGPT-User":["partial","*",6,1],"OAI-SearchBot":["partial","*",6,1],"ClaudeBot":["partial","*",6,1],"Claude-User":["partial","*",6,1],"Claude-SearchBot":["partial","*",6,1],"anthropic-ai":["partial","*",6,1],"PerplexityBot":["partial","*",6,1],"Perplexity-User":["partial","*",6,1],"Google-Extended":["partial","*",6,1],"Applebot-Extended":["partial","*",6,1],"CCBot":["partial","*",6,1],"Bytespider":["partial","*",6,1],"Amazonbot":["partial","*",6,1],"meta-externalagent":["partial","*",6,1],"cohere-ai":["partial","*",6,1],"Diffbot":["partial","*",6,1],"ContextFetchBot":["partial","*",6,1]},"llms":{"status":"missing","http":404,"bytes":118454},"home":{"status":"ok","http":200,"url":"https://www.twz.com/","html_tokens":66938,"markdown_tokens":713,"pruned_tokens":93}},
{"rank":5773,"domain":"google.co.ve","origin":"https://google.co.ve","robots":{"status":"ok","http":200,"url":"https://www.google.co.ve/robots.txt","bytes":6624,"groups":4},"bots":{"GPTBot":["partial","*",169,64],"ChatGPT-User":["partial","*",169,64],"OAI-SearchBot":["partial","*",169,64],"ClaudeBot":["partial","*",169,64],"Claude-User":["partial","*",169,64],"Claude-SearchBot":["partial","*",169,64],"anthropic-ai":["partial","*",169,64],"PerplexityBot":["partial","*",169,64],"Perplexity-User":["partial","*",169,64],"Google-Extended":["partial","*",169,64],"Applebot-Extended":["partial","*",169,64],"CCBot":["partial","*",169,64],"Bytespider":["partial","*",169,64],"Amazonbot":["partial","*",169,64],"meta-externalagent":["partial","*",169,64],"cohere-ai":["partial","*",169,64],"Diffbot":["partial","*",169,64],"ContextFetchBot":["partial","*",169,64]},"llms":{"status":"missing","http":404,"bytes":1569},"home":{"status":"ok","http":200,"url":"https://www.google.com/","html_tokens":21324,"markdown_tokens":30,"pruned_tokens":30}},
{"rank":5774,"domain":"yam.com","origin":"https://yam.com","robots":{"status":"html","http":200,"url":"https://www.yam.com/","bytes":132521,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":5775,"domain":"limesurvey.org","origin":"https://limesurvey.org","robots":{"status":"ok","http":200,"url":"https://www.limesurvey.org/robots.txt","bytes":1477,"groups":6},"bots":{"GPTBot":["partial","*",19,5],"ChatGPT-User":["partial","*",19,5],"OAI-SearchBot":["partial","*",19,5],"ClaudeBot":["partial","*",19,5],"Claude-User":["partial","*",19,5],"Claude-SearchBot":["partial","*",19,5],"anthropic-ai":["partial","*",19,5],"PerplexityBot":["partial","*",19,5],"Perplexity-User":["partial","*",19,5],"Google-Extended":["partial","*",19,5],"Applebot-Extended":["partial","*",19,5],"CCBot":["partial","*",19,5],"Bytespider":["partial","*",19,5],"Amazonbot":["partial","*",19,5],"meta-externalagent":["partial","*",19,5],"cohere-ai":["partial","*",19,5],"Diffbot":["partial","*",19,5],"ContextFetchBot":["partial","*",19,5]},"llms":{"status":"other_text","http":200,"bytes":38981},"home":{"status":"ok","http":200,"url":"https://www.limesurvey.org/","html_tokens":95374,"markdown_tokens":1020,"pruned_tokens":429}},
{"rank":5776,"domain":"google.si","origin":"https://google.si","robots":{"status":"ok","http":200,"url":"https://www.google.si/robots.txt","bytes":6624,"groups":4},"bots":{"GPTBot":["partial","*",169,64],"ChatGPT-User":["partial","*",169,64],"OAI-SearchBot":["partial","*",169,64],"ClaudeBot":["partial","*",169,64],"Claude-User":["partial","*",169,64],"Claude-SearchBot":["partial","*",169,64],"anthropic-ai":["partial","*",169,64],"PerplexityBot":["partial","*",169,64],"Perplexity-User":["partial","*",169,64],"Google-Extended":["partial","*",169,64],"Applebot-Extended":["partial","*",169,64],"CCBot":["partial","*",169,64],"Bytespider":["partial","*",169,64],"Amazonbot":["partial","*",169,64],"meta-externalagent":["partial","*",169,64],"cohere-ai":["partial","*",169,64],"Diffbot":["partial","*",169,64],"ContextFetchBot":["partial","*",169,64]},"llms":{"status":"missing","http":404,"bytes":1569},"home":{"status":"ok","http":200,"url":"https://www.google.com/","html_tokens":21313,"markdown_tokens":30,"pruned_tokens":30}},
{"rank":5777,"domain":"cppreference.com","origin":"https://cppreference.com","robots":{"status":"missing","http":404,"url":"https://cppreference.com/robots.txt","bytes":10780,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":10748},"home":{"status":"ok","http":200,"url":"https://cppreference.com/","html_tokens":11794,"markdown_tokens":16,"pruned_tokens":15}},
{"rank":5778,"domain":"thebase.in","origin":"https://thebase.in","robots":{"status":"ok","http":200,"url":"https://thebase.com/robots.txt","bytes":105,"groups":1},"bots":{"GPTBot":["partial","*",2,1],"ChatGPT-User":["partial","*",2,1],"OAI-SearchBot":["partial","*",2,1],"ClaudeBot":["partial","*",2,1],"Claude-User":["partial","*",2,1],"Claude-SearchBot":["partial","*",2,1],"anthropic-ai":["partial","*",2,1],"PerplexityBot":["partial","*",2,1],"Perplexity-User":["partial","*",2,1],"Google-Extended":["partial","*",2,1],"Applebot-Extended":["partial","*",2,1],"CCBot":["partial","*",2,1],"Bytespider":["partial","*",2,1],"Amazonbot":["partial","*",2,1],"meta-externalagent":["partial","*",2,1],"cohere-ai":["partial","*",2,1],"Diffbot":["partial","*",2,1],"ContextFetchBot":["partial","*",2,1]},"llms":{"status":"html","http":200,"bytes":149241},"home":{"status":"ok","http":200,"url":"https://thebase.com/","html_tokens":35899,"markdown_tokens":151,"pruned_tokens":151}},
{"rank":5779,"domain":"cheezburger.com","origin":"https://cheezburger.com","robots":{"status":"ok","http":200,"url":"https://cheezburger.com/robots.txt","bytes":745,"groups":1},"bots":{"GPTBot":["partial","*",18,0],"ChatGPT-User":["partial","*",18,0],"OAI-SearchBot":["partial","*",18,0],"ClaudeBot":["partial","*",18,0],"Claude-User":["partial","*",18,0],"Claude-SearchBot":["partial","*",18,0],"anthropic-ai":["partial","*",18,0],"PerplexityBot":["partial","*",18,0],"Perplexity-User":["partial","*",18,0],"Google-Extended":["partial","*",18,0],"Applebot-Extended":["partial","*",18,0],"CCBot":["partial","*",18,0],"Bytespider":["partial","*",18,0],"Amazonbot":["partial","*",18,0],"meta-externalagent":["partial","*",18,0],"cohere-ai":["partial","*",18,0],"Diffbot":["partial","*",18,0],"ContextFetchBot":["partial","*",18,0]},"llms":{"status":"missing","http":404,"bytes":108868},"home":{"status":"ok","http":200,"url":"https://cheezburger.com/","html_tokens":48354,"markdown_tokens":56,"pruned_tokens":56}},
{"rank":5780,"domain":"wallpaper.com","origin":"https://wallpaper.com","robots":{"status":"ok","http":200,"url":"https://www.wallpaper.com/robots.txt","bytes":1825,"groups":5},"bots":{"GPTBot":["partial","*",24,0],"ChatGPT-User":["partial","*",24,0],"OAI-SearchBot":["partial","*",24,0],"ClaudeBot":["partial","*",24,0],"Claude-User":["partial","*",24,0],"Claude-SearchBot":["partial","*",24,0],"anthropic-ai":["partial","*",24,0],"PerplexityBot":["partial","*",24,0],"Perplexity-User":["partial","*",24,0],"Google-Extended":["partial","*",24,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["partial","*",24,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["partial","*",24,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",24,0]},"llms":{"status":"missing","http":404,"bytes":581780},"home":{"status":"ok","http":200,"url":"https://www.wallpaper.com/","html_tokens":203369,"markdown_tokens":510,"pruned_tokens":60}},
{"rank":5781,"domain":"mfa.gov.cn","origin":"https://www.mfa.gov.cn","robots":{"status":"html","http":200,"url":"https://www.mfa.gov.cn/web/system/index_17321.shtml","bytes":54156,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":5782,"domain":"nyt.com","origin":"https://nyt.com","robots":{"status":"ok","http":200,"url":"https://www.nytimes.com/robots.txt","bytes":8547,"groups":63},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["blocked","named",1,0],"OAI-SearchBot":["blocked","named",1,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["blocked","named",1,0],"Claude-SearchBot":["blocked","named",1,0],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["blocked","named",1,0],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["partial","named",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",86,23]},"llms":{"status":"missing","http":404,"bytes":68678},"home":{"status":"http_error","http":403,"url":"https://www.nytimes.com/"}},
{"rank":5783,"domain":"eurosport.com","origin":"https://eurosport.com","robots":{"status":"ok","http":200,"url":"https://www.tntsports.co.uk/robots.txt","bytes":4076,"groups":5},"bots":{"GPTBot":["partial","*",5,0],"ChatGPT-User":["partial","*",5,0],"OAI-SearchBot":["partial","*",5,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",5,0],"Claude-SearchBot":["partial","*",5,0],"anthropic-ai":["partial","*",5,0],"PerplexityBot":["partial","*",5,0],"Perplexity-User":["partial","*",5,0],"Google-Extended":["partial","*",5,0],"Applebot-Extended":["partial","*",5,0],"CCBot":["partial","*",5,0],"Bytespider":["partial","*",5,0],"Amazonbot":["partial","*",5,0],"meta-externalagent":["partial","*",5,0],"cohere-ai":["partial","*",5,0],"Diffbot":["partial","*",5,0],"ContextFetchBot":["partial","*",5,0]},"llms":{"status":"missing","http":404,"bytes":197107},"home":{"status":"ok","http":200,"url":"https://www.tntsports.co.uk/","html_tokens":236457,"markdown_tokens":35,"pruned_tokens":35}},
{"rank":5784,"domain":"nationaltoday.com","origin":"https://nationaltoday.com","robots":{"status":"ok","http":200,"url":"https://nationaltoday.com/robots.txt","bytes":181,"groups":1},"bots":{"GPTBot":["partial","*",2,1],"ChatGPT-User":["partial","*",2,1],"OAI-SearchBot":["partial","*",2,1],"ClaudeBot":["partial","*",2,1],"Claude-User":["partial","*",2,1],"Claude-SearchBot":["partial","*",2,1],"anthropic-ai":["partial","*",2,1],"PerplexityBot":["partial","*",2,1],"Perplexity-User":["partial","*",2,1],"Google-Extended":["partial","*",2,1],"Applebot-Extended":["partial","*",2,1],"CCBot":["partial","*",2,1],"Bytespider":["partial","*",2,1],"Amazonbot":["partial","*",2,1],"meta-externalagent":["partial","*",2,1],"cohere-ai":["partial","*",2,1],"Diffbot":["partial","*",2,1],"ContextFetchBot":["partial","*",2,1]},"llms":{"status":"missing","http":404,"bytes":69977},"home":{"status":"ok","http":200,"url":"https://nationaltoday.com/","html_tokens":48354,"markdown_tokens":54,"pruned_tokens":54}},
{"rank":5785,"domain":"canoe.ca","origin":"https://canoe.ca","robots":{"status":"ok","http":200,"url":"https://www.canoe.ca/robots.txt","bytes":934,"groups":23},"bots":{"GPTBot":["partial","*",5,1],"ChatGPT-User":["partial","*",5,1],"OAI-SearchBot":["partial","*",5,1],"ClaudeBot":["partial","*",5,1],"Claude-User":["partial","*",5,1],"Claude-SearchBot":["partial","*",5,1],"anthropic-ai":["partial","*",5,1],"PerplexityBot":["partial","*",5,1],"Perplexity-User":["partial","*",5,1],"Google-Extended":["partial","*",5,1],"Applebot-Extended":["partial","*",5,1],"CCBot":["partial","*",5,1],"Bytespider":["partial","*",5,1],"Amazonbot":["partial","*",5,1],"meta-externalagent":["partial","*",5,1],"cohere-ai":["partial","*",5,1],"Diffbot":["partial","*",5,1],"ContextFetchBot":["partial","*",5,1]},"llms":{"status":"html","http":200,"bytes":46802},"home":{"status":"ok","http":200,"url":"https://www.canoe.ca/","html_tokens":11690,"markdown_tokens":942,"pruned_tokens":134}},
{"rank":5786,"domain":"franceinter.fr","origin":"https://franceinter.fr","robots":{"status":"ok","http":200,"url":"https://www.radiofrance.fr/robots.txt","bytes":4390,"groups":73},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["blocked","named",1,0],"OAI-SearchBot":["blocked","named",1,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",10,0],"Claude-SearchBot":["partial","*",10,0],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["partial","*",10,0],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["partial","*",10,0],"ContextFetchBot":["partial","*",10,0]},"llms":{"status":"missing","http":404,"bytes":153724},"home":{"status":"ok","http":200,"url":"https://www.radiofrance.fr/franceinter","html_tokens":660998,"markdown_tokens":1080,"pruned_tokens":146}},
{"rank":5787,"domain":"folkd.com","origin":"https://folkd.com","robots":{"status":"refused","http":403,"url":"https://folkd.com/robots.txt","bytes":5525,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":5788,"domain":"kucoin.com","origin":"https://kucoin.com","robots":{"status":"ok","http":200,"url":"https://www.kucoin.com/robots.txt","bytes":252,"groups":1},"bots":{"GPTBot":["partial","*",4,0],"ChatGPT-User":["partial","*",4,0],"OAI-SearchBot":["partial","*",4,0],"ClaudeBot":["partial","*",4,0],"Claude-User":["partial","*",4,0],"Claude-SearchBot":["partial","*",4,0],"anthropic-ai":["partial","*",4,0],"PerplexityBot":["partial","*",4,0],"Perplexity-User":["partial","*",4,0],"Google-Extended":["partial","*",4,0],"Applebot-Extended":["partial","*",4,0],"CCBot":["partial","*",4,0],"Bytespider":["partial","*",4,0],"Amazonbot":["partial","*",4,0],"meta-externalagent":["partial","*",4,0],"cohere-ai":["partial","*",4,0],"Diffbot":["partial","*",4,0],"ContextFetchBot":["partial","*",4,0]},"llms":{"status":"present","http":200,"bytes":3816},"home":{"status":"ok","http":200,"url":"https://www.kucoin.com/","html_tokens":102059,"markdown_tokens":24,"pruned_tokens":24}},
{"rank":5789,"domain":"ionos.co.uk","origin":"https://ionos.co.uk","robots":{"status":"ok","http":200,"url":"https://www.ionos.co.uk/robots.txt","bytes":452,"groups":2},"bots":{"GPTBot":["partial","*",8,1],"ChatGPT-User":["partial","*",8,1],"OAI-SearchBot":["partial","*",8,1],"ClaudeBot":["partial","*",8,1],"Claude-User":["partial","*",8,1],"Claude-SearchBot":["partial","*",8,1],"anthropic-ai":["partial","*",8,1],"PerplexityBot":["partial","*",8,1],"Perplexity-User":["partial","*",8,1],"Google-Extended":["partial","*",8,1],"Applebot-Extended":["partial","*",8,1],"CCBot":["partial","*",8,1],"Bytespider":["partial","*",8,1],"Amazonbot":["partial","*",8,1],"meta-externalagent":["partial","*",8,1],"cohere-ai":["partial","*",8,1],"Diffbot":["partial","*",8,1],"ContextFetchBot":["partial","*",8,1]},"llms":{"status":"missing","http":404,"bytes":648879},"home":{"status":"ok","http":200,"url":"https://www.ionos.co.uk/","html_tokens":209337,"markdown_tokens":95,"pruned_tokens":72}},
{"rank":5790,"domain":"bbva.com","origin":"https://bbva.com","robots":{"status":"ok","http":200,"url":"https://www.bbva.com/robots.txt","bytes":911,"groups":1},"bots":{"GPTBot":["partial","*",23,0],"ChatGPT-User":["partial","*",23,0],"OAI-SearchBot":["partial","*",23,0],"ClaudeBot":["partial","*",23,0],"Claude-User":["partial","*",23,0],"Claude-SearchBot":["partial","*",23,0],"anthropic-ai":["partial","*",23,0],"PerplexityBot":["partial","*",23,0],"Perplexity-User":["partial","*",23,0],"Google-Extended":["partial","*",23,0],"Applebot-Extended":["partial","*",23,0],"CCBot":["partial","*",23,0],"Bytespider":["partial","*",23,0],"Amazonbot":["partial","*",23,0],"meta-externalagent":["partial","*",23,0],"cohere-ai":["partial","*",23,0],"Diffbot":["partial","*",23,0],"ContextFetchBot":["partial","*",23,0]},"llms":{"status":"missing","http":404,"bytes":98621},"home":{"status":"ok","http":200,"url":"https://www.bbva.com/en/","html_tokens":41208,"markdown_tokens":6,"pruned_tokens":6}},
{"rank":5791,"domain":"dynatrace.com","origin":"https://dynatrace.com","robots":{"status":"ok","http":200,"url":"https://www.dynatrace.com/robots.txt","bytes":319,"groups":1},"bots":{"GPTBot":["partial","*",9,0],"ChatGPT-User":["partial","*",9,0],"OAI-SearchBot":["partial","*",9,0],"ClaudeBot":["partial","*",9,0],"Claude-User":["partial","*",9,0],"Claude-SearchBot":["partial","*",9,0],"anthropic-ai":["partial","*",9,0],"PerplexityBot":["partial","*",9,0],"Perplexity-User":["partial","*",9,0],"Google-Extended":["partial","*",9,0],"Applebot-Extended":["partial","*",9,0],"CCBot":["partial","*",9,0],"Bytespider":["partial","*",9,0],"Amazonbot":["partial","*",9,0],"meta-externalagent":["partial","*",9,0],"cohere-ai":["partial","*",9,0],"Diffbot":["partial","*",9,0],"ContextFetchBot":["partial","*",9,0]},"llms":{"status":"present","http":200,"bytes":182111},"home":{"status":"ok","http":200,"url":"https://www.dynatrace.com/","html_tokens":36791,"markdown_tokens":301,"pruned_tokens":151}},
{"rank":5792,"domain":"mewe.com","origin":"https://mewe.com","robots":{"status":"html","http":200,"url":"https://mewe.com/robots.txt","bytes":15465,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":5793,"domain":"flock.com","origin":"https://flock.com","robots":{"status":"ok","http":200,"url":"https://flock.com/robots.txt","bytes":118,"groups":1},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"missing","http":404,"bytes":43946},"home":{"status":"ok","http":200,"url":"https://www.flock.com/","html_tokens":15933,"markdown_tokens":806,"pruned_tokens":259}},
{"rank":5794,"domain":"newsnationnow.com","origin":"https://newsnationnow.com","robots":{"status":"ok","http":200,"url":"https://www.newsnationnow.com/robots.txt","bytes":2332,"groups":40},"bots":{"GPTBot":["blocked","named",2,0],"ChatGPT-User":["blocked","named",1,0],"OAI-SearchBot":["blocked","named",1,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",12,1],"Claude-SearchBot":["partial","*",12,1],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["blocked","named",2,0],"Perplexity-User":["partial","*",12,1],"Google-Extended":["blocked","named",2,0],"Applebot-Extended":["blocked","named",2,0],"CCBot":["blocked","named",2,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["partial","*",12,1],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",12,1]},"llms":{"status":"http_error","http":403,"bytes":6514},"home":{"status":"http_error","http":403,"url":"https://newsnationnow.com/"}},
{"rank":5795,"domain":"colgate.com","origin":"https://colgate.com","robots":{"status":"ok","http":200,"url":"https://www.colgate.com/robots.txt","bytes":2121,"groups":1},"bots":{"GPTBot":["partial","*",9,0],"ChatGPT-User":["partial","*",9,0],"OAI-SearchBot":["partial","*",9,0],"ClaudeBot":["partial","*",9,0],"Claude-User":["partial","*",9,0],"Claude-SearchBot":["partial","*",9,0],"anthropic-ai":["partial","*",9,0],"PerplexityBot":["partial","*",9,0],"Perplexity-User":["partial","*",9,0],"Google-Extended":["partial","*",9,0],"Applebot-Extended":["partial","*",9,0],"CCBot":["partial","*",9,0],"Bytespider":["partial","*",9,0],"Amazonbot":["partial","*",9,0],"meta-externalagent":["partial","*",9,0],"cohere-ai":["partial","*",9,0],"Diffbot":["partial","*",9,0],"ContextFetchBot":["partial","*",9,0]},"llms":{"status":"html","http":200,"bytes":591119},"home":{"status":"ok","http":200,"url":"https://www.colgate.com/en-us","html_tokens":147746,"markdown_tokens":287,"pruned_tokens":55}},
{"rank":5796,"domain":"ricoh.com","origin":"https://ricoh.com","robots":{"status":"ok","http":200,"url":"https://www.ricoh.com/-/media/Ricoh/Sites/com/robots.txt","bytes":326,"groups":1},"bots":{"GPTBot":["partial","*",11,0],"ChatGPT-User":["partial","*",11,0],"OAI-SearchBot":["partial","*",11,0],"ClaudeBot":["partial","*",11,0],"Claude-User":["partial","*",11,0],"Claude-SearchBot":["partial","*",11,0],"anthropic-ai":["partial","*",11,0],"PerplexityBot":["partial","*",11,0],"Perplexity-User":["partial","*",11,0],"Google-Extended":["partial","*",11,0],"Applebot-Extended":["partial","*",11,0],"CCBot":["partial","*",11,0],"Bytespider":["partial","*",11,0],"Amazonbot":["partial","*",11,0],"meta-externalagent":["partial","*",11,0],"cohere-ai":["partial","*",11,0],"Diffbot":["partial","*",11,0],"ContextFetchBot":["partial","*",11,0]},"llms":{"status":"present","http":200,"bytes":3338},"home":{"status":"ok","http":200,"url":"https://www.ricoh.com/","html_tokens":16324,"markdown_tokens":164,"pruned_tokens":164}},
{"rank":5797,"domain":"byd.com","origin":"https://byd.com","robots":{"status":"missing","http":404,"url":"https://www.byd.com/robots.txt","bytes":0,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":0},"home":{"status":"ok","http":200,"url":"https://www.byd.com/us","html_tokens":15560,"markdown_tokens":7,"pruned_tokens":7}},
{"rank":5798,"domain":"beget.tech","origin":"https://beget.tech","robots":{"status":"html","http":200,"url":"https://beget.tech/robots.txt","bytes":10934,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":5799,"domain":"polskieradio.pl","origin":"https://polskieradio.pl","robots":{"status":"missing","http":404,"url":"https://www.polskieradio.pl/robots.txt","bytes":506534,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":0},"home":{"status":"ok","http":200,"url":"https://www.polskieradio.pl/","html_tokens":706408,"markdown_tokens":173,"pruned_tokens":173}},
{"rank":5800,"domain":"rspb.org.uk","origin":"https://rspb.org.uk","robots":{"status":"ok","http":200,"url":"https://www.rspb.org.uk/robots.txt","bytes":237,"groups":1},"bots":{"GPTBot":["partial","*",3,2],"ChatGPT-User":["partial","*",3,2],"OAI-SearchBot":["partial","*",3,2],"ClaudeBot":["partial","*",3,2],"Claude-User":["partial","*",3,2],"Claude-SearchBot":["partial","*",3,2],"anthropic-ai":["partial","*",3,2],"PerplexityBot":["partial","*",3,2],"Perplexity-User":["partial","*",3,2],"Google-Extended":["partial","*",3,2],"Applebot-Extended":["partial","*",3,2],"CCBot":["partial","*",3,2],"Bytespider":["partial","*",3,2],"Amazonbot":["partial","*",3,2],"meta-externalagent":["partial","*",3,2],"cohere-ai":["partial","*",3,2],"Diffbot":["partial","*",3,2],"ContextFetchBot":["partial","*",3,2]},"llms":{"status":"html","http":200,"bytes":221489},"home":{"status":"ok","http":200,"url":"https://www.rspb.org.uk/","html_tokens":105820,"markdown_tokens":73,"pruned_tokens":54}},
{"rank":5801,"domain":"wenming.cn","origin":"https://wenming.cn","robots":{"status":"missing","http":404,"url":"http://wenming.cn/robots.txt","bytes":1920,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":1920},"home":{"status":"ok","http":200,"url":"http://wenming.cn/","html_tokens":14726,"markdown_tokens":38,"pruned_tokens":38}},
{"rank":5802,"domain":"eggsinc.se","origin":"https://eggsinc.se","robots":{"status":"ok","http":200,"url":"https://eggsinc.se/robots.txt","bytes":125,"groups":1},"bots":{"GPTBot":["partial","*",2,1],"ChatGPT-User":["partial","*",2,1],"OAI-SearchBot":["partial","*",2,1],"ClaudeBot":["partial","*",2,1],"Claude-User":["partial","*",2,1],"Claude-SearchBot":["partial","*",2,1],"anthropic-ai":["partial","*",2,1],"PerplexityBot":["partial","*",2,1],"Perplexity-User":["partial","*",2,1],"Google-Extended":["partial","*",2,1],"Applebot-Extended":["partial","*",2,1],"CCBot":["partial","*",2,1],"Bytespider":["partial","*",2,1],"Amazonbot":["partial","*",2,1],"meta-externalagent":["partial","*",2,1],"cohere-ai":["partial","*",2,1],"Diffbot":["partial","*",2,1],"ContextFetchBot":["partial","*",2,1]},"llms":{"status":"missing","http":404,"bytes":2020},"home":{"status":"ok","http":200,"url":"https://eggsinc.se/","html_tokens":11625,"markdown_tokens":5737,"pruned_tokens":120}},
{"rank":5803,"domain":"theinquirer.net","origin":"https://theinquirer.net","robots":{"status":"timeout","http":null,"url":"https://theinquirer.net/robots.txt","bytes":0,"groups":0,"error":"timed out after 10 s"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":5804,"domain":"cnbctv18.com","origin":"https://cnbctv18.com","robots":{"status":"ok","http":200,"url":"https://www.cnbctv18.com/robots.txt","bytes":3113,"groups":22},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["blocked","named",1,0],"OAI-SearchBot":["blocked","named",1,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",73,1],"Claude-SearchBot":["partial","*",73,1],"anthropic-ai":["partial","*",73,1],"PerplexityBot":["partial","*",73,1],"Perplexity-User":["partial","*",73,1],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["partial","*",73,1],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["partial","*",73,1],"meta-externalagent":["partial","*",73,1],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",73,1]},"llms":{"status":"missing","http":404,"bytes":19714},"home":{"status":"ok","http":200,"url":"https://www.cnbctv18.com/","html_tokens":137435,"markdown_tokens":13,"pruned_tokens":13}},
{"rank":5805,"domain":"google.co.ke","origin":"https://google.co.ke","robots":{"status":"ok","http":200,"url":"https://www.google.co.ke/robots.txt","bytes":6624,"groups":4},"bots":{"GPTBot":["partial","*",169,64],"ChatGPT-User":["partial","*",169,64],"OAI-SearchBot":["partial","*",169,64],"ClaudeBot":["partial","*",169,64],"Claude-User":["partial","*",169,64],"Claude-SearchBot":["partial","*",169,64],"anthropic-ai":["partial","*",169,64],"PerplexityBot":["partial","*",169,64],"Perplexity-User":["partial","*",169,64],"Google-Extended":["partial","*",169,64],"Applebot-Extended":["partial","*",169,64],"CCBot":["partial","*",169,64],"Bytespider":["partial","*",169,64],"Amazonbot":["partial","*",169,64],"meta-externalagent":["partial","*",169,64],"cohere-ai":["partial","*",169,64],"Diffbot":["partial","*",169,64],"ContextFetchBot":["partial","*",169,64]},"llms":{"status":"missing","http":404,"bytes":1569},"home":{"status":"ok","http":200,"url":"https://www.google.com/","html_tokens":21292,"markdown_tokens":30,"pruned_tokens":30}},
{"rank":5806,"domain":"hinet.net","origin":"https://hinet.net","robots":{"status":"html","http":200,"url":"https://www.cht.com.tw/home/consumer/broadband/preferential/hinetupgrade","bytes":232479,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":5807,"domain":"autoevolution.com","origin":"https://autoevolution.com","robots":{"status":"ok","http":200,"url":"https://www.autoevolution.com/robots.txt","bytes":2046,"groups":25},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["partial","*",39,0],"OAI-SearchBot":["partial","*",39,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",39,0],"Claude-SearchBot":["partial","*",39,0],"anthropic-ai":["partial","*",39,0],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["partial","*",39,0],"Google-Extended":["partial","*",39,0],"Applebot-Extended":["partial","*",39,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["partial","*",39,0],"cohere-ai":["partial","*",39,0],"Diffbot":["partial","*",39,0],"ContextFetchBot":["partial","*",39,0]},"llms":{"status":"missing","http":404,"bytes":23768},"home":{"status":"http_error","http":403,"url":"https://www.autoevolution.com/"}},
{"rank":5808,"domain":"domainseller.site","origin":"https://domainseller.site","robots":{"status":"unreachable","http":null,"url":"https://domainseller.site/robots.txt","bytes":0,"groups":0,"error":"cannot resolve domainseller.site"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":5809,"domain":"schneider-electric.com","origin":"https://schneider-electric.com","robots":{"status":"ok","http":200,"url":"https://www.se.com/robots.txt","bytes":46313,"groups":1},"bots":{"GPTBot":["partial","*",69,86],"ChatGPT-User":["partial","*",69,86],"OAI-SearchBot":["partial","*",69,86],"ClaudeBot":["partial","*",69,86],"Claude-User":["partial","*",69,86],"Claude-SearchBot":["partial","*",69,86],"anthropic-ai":["partial","*",69,86],"PerplexityBot":["partial","*",69,86],"Perplexity-User":["partial","*",69,86],"Google-Extended":["partial","*",69,86],"Applebot-Extended":["partial","*",69,86],"CCBot":["partial","*",69,86],"Bytespider":["partial","*",69,86],"Amazonbot":["partial","*",69,86],"meta-externalagent":["partial","*",69,86],"cohere-ai":["partial","*",69,86],"Diffbot":["partial","*",69,86],"ContextFetchBot":["partial","*",69,86]},"llms":{"status":"missing","http":404,"bytes":355},"home":{"status":"ok","http":200,"url":"https://www.se.com/ww/en/","html_tokens":495767,"markdown_tokens":427,"pruned_tokens":44}},
{"rank":5810,"domain":"terraform.io","origin":"https://terraform.io","robots":{"status":"html","http":200,"url":"https://developer.hashicorp.com/terraform","bytes":146690,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":5811,"domain":"catholicnewsagency.com","origin":"https://catholicnewsagency.com","robots":{"status":"ok","http":200,"url":"https://www.catholicnewsagency.com/robots.txt","bytes":27,"groups":1},"bots":{"GPTBot":["blocked","*",1,0],"ChatGPT-User":["blocked","*",1,0],"OAI-SearchBot":["blocked","*",1,0],"ClaudeBot":["blocked","*",1,0],"Claude-User":["blocked","*",1,0],"Claude-SearchBot":["blocked","*",1,0],"anthropic-ai":["blocked","*",1,0],"PerplexityBot":["blocked","*",1,0],"Perplexity-User":["blocked","*",1,0],"Google-Extended":["blocked","*",1,0],"Applebot-Extended":["blocked","*",1,0],"CCBot":["blocked","*",1,0],"Bytespider":["blocked","*",1,0],"Amazonbot":["blocked","*",1,0],"meta-externalagent":["blocked","*",1,0],"cohere-ai":["blocked","*",1,0],"Diffbot":["blocked","*",1,0],"ContextFetchBot":["blocked","*",1,0]},"llms":{"status":"disallowed"},"home":{"status":"disallowed"}},
{"rank":5812,"domain":"thelocal.se","origin":"https://thelocal.se","robots":{"status":"ok","http":200,"url":"https://www.thelocal.se/robots.txt","bytes":1370,"groups":32},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["blocked","named",1,0],"OAI-SearchBot":["partial","*",6,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",6,0],"Claude-SearchBot":["partial","*",6,0],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["partial","*",6,0],"Google-Extended":["partial","*",6,0],"Applebot-Extended":["partial","*",6,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["partial","*",6,0],"cohere-ai":["partial","*",6,0],"Diffbot":["partial","*",6,0],"ContextFetchBot":["partial","*",6,0]},"llms":{"status":"missing","http":404,"bytes":109644},"home":{"status":"ok","http":200,"url":"https://www.thelocal.se/","html_tokens":57648,"markdown_tokens":61,"pruned_tokens":61}},
{"rank":5813,"domain":"symbaloo.com","origin":"https://symbaloo.com","robots":{"status":"ok","http":200,"url":"https://www.symbaloo.com/robots.txt","bytes":36,"groups":1},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"present","http":200,"bytes":7813},"home":{"status":"ok","http":200,"url":"https://www.symbaloo.com/","html_tokens":3721,"markdown_tokens":8,"pruned_tokens":8}},
{"rank":5814,"domain":"manilatimes.net","origin":"https://manilatimes.net","robots":{"status":"ok","http":200,"url":"https://www.manilatimes.net/robots.txt","bytes":206,"groups":2},"bots":{"GPTBot":["partial","*",5,0],"ChatGPT-User":["partial","*",5,0],"OAI-SearchBot":["partial","*",5,0],"ClaudeBot":["partial","*",5,0],"Claude-User":["partial","*",5,0],"Claude-SearchBot":["partial","*",5,0],"anthropic-ai":["partial","*",5,0],"PerplexityBot":["partial","*",5,0],"Perplexity-User":["partial","*",5,0],"Google-Extended":["partial","*",5,0],"Applebot-Extended":["partial","*",5,0],"CCBot":["partial","*",5,0],"Bytespider":["partial","*",5,0],"Amazonbot":["partial","*",5,0],"meta-externalagent":["partial","*",5,0],"cohere-ai":["partial","*",5,0],"Diffbot":["partial","*",5,0],"ContextFetchBot":["partial","*",5,0]},"llms":{"status":"crawl_delay","error":"Crawl-delay 600 s"},"home":{"status":"crawl_delay","error":"Crawl-delay 600 s"}},
{"rank":5815,"domain":"sachsen-anhalt.de","origin":"https://sachsen-anhalt.de","robots":{"status":"ok","http":200,"url":"https://sachsen-anhalt.de/robots.txt","bytes":237,"groups":1},"bots":{"GPTBot":["partial","*",4,1],"ChatGPT-User":["partial","*",4,1],"OAI-SearchBot":["partial","*",4,1],"ClaudeBot":["partial","*",4,1],"Claude-User":["partial","*",4,1],"Claude-SearchBot":["partial","*",4,1],"anthropic-ai":["partial","*",4,1],"PerplexityBot":["partial","*",4,1],"Perplexity-User":["partial","*",4,1],"Google-Extended":["partial","*",4,1],"Applebot-Extended":["partial","*",4,1],"CCBot":["partial","*",4,1],"Bytespider":["partial","*",4,1],"Amazonbot":["partial","*",4,1],"meta-externalagent":["partial","*",4,1],"cohere-ai":["partial","*",4,1],"Diffbot":["partial","*",4,1],"ContextFetchBot":["partial","*",4,1]},"llms":{"status":"missing","http":404,"bytes":38957},"home":{"status":"ok","http":200,"url":"https://sachsen-anhalt.de/","html_tokens":18775,"markdown_tokens":135,"pruned_tokens":135}},
{"rank":5816,"domain":"sdu.dk","origin":"https://www.sdu.dk","robots":{"status":"ok","http":200,"url":"https://www.sdu.dk/robots.txt","bytes":175,"groups":1},"bots":{"GPTBot":["partial","*",5,0],"ChatGPT-User":["partial","*",5,0],"OAI-SearchBot":["partial","*",5,0],"ClaudeBot":["partial","*",5,0],"Claude-User":["partial","*",5,0],"Claude-SearchBot":["partial","*",5,0],"anthropic-ai":["partial","*",5,0],"PerplexityBot":["partial","*",5,0],"Perplexity-User":["partial","*",5,0],"Google-Extended":["partial","*",5,0],"Applebot-Extended":["partial","*",5,0],"CCBot":["partial","*",5,0],"Bytespider":["partial","*",5,0],"Amazonbot":["partial","*",5,0],"meta-externalagent":["partial","*",5,0],"cohere-ai":["partial","*",5,0],"Diffbot":["partial","*",5,0],"ContextFetchBot":["partial","*",5,0]},"llms":{"status":"other_text","http":200,"bytes":240},"home":{"status":"ok","http":200,"url":"https://www.sdu.dk/da","html_tokens":52328,"markdown_tokens":889,"pruned_tokens":161}},
{"rank":5817,"domain":"freepatentsonline.com","origin":"https://freepatentsonline.com","robots":{"status":"ok","http":200,"url":"https://www.freepatentsonline.com/robots.txt","bytes":886,"groups":2},"bots":{"GPTBot":["blocked","*",1,0],"ChatGPT-User":["blocked","*",1,0],"OAI-SearchBot":["blocked","*",1,0],"ClaudeBot":["blocked","*",1,0],"Claude-User":["blocked","*",1,0],"Claude-SearchBot":["blocked","*",1,0],"anthropic-ai":["blocked","*",1,0],"PerplexityBot":["blocked","*",1,0],"Perplexity-User":["blocked","*",1,0],"Google-Extended":["blocked","*",1,0],"Applebot-Extended":["blocked","*",1,0],"CCBot":["blocked","*",1,0],"Bytespider":["blocked","*",1,0],"Amazonbot":["blocked","*",1,0],"meta-externalagent":["blocked","*",1,0],"cohere-ai":["blocked","*",1,0],"Diffbot":["blocked","*",1,0],"ContextFetchBot":["blocked","*",1,0]},"llms":{"status":"disallowed"},"home":{"status":"disallowed"}},
{"rank":5818,"domain":"aha.org","origin":"https://aha.org","robots":{"status":"refused","http":403,"url":"https://aha.org/robots.txt","bytes":560,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":5819,"domain":"indigo.ca","origin":"https://indigo.ca","robots":{"status":"ok","http":200,"url":"https://www.indigo.ca/robots.txt","bytes":7559,"groups":7},"bots":{"GPTBot":["partial","*",46,0],"ChatGPT-User":["partial","*",46,0],"OAI-SearchBot":["partial","*",46,0],"ClaudeBot":["partial","*",46,0],"Claude-User":["partial","*",46,0],"Claude-SearchBot":["partial","*",46,0],"anthropic-ai":["partial","*",46,0],"PerplexityBot":["partial","*",46,0],"Perplexity-User":["partial","*",46,0],"Google-Extended":["partial","*",46,0],"Applebot-Extended":["partial","*",46,0],"CCBot":["partial","*",46,0],"Bytespider":["partial","*",46,0],"Amazonbot":["partial","*",46,0],"meta-externalagent":["partial","*",46,0],"cohere-ai":["partial","*",46,0],"Diffbot":["partial","*",46,0],"ContextFetchBot":["partial","*",46,0]},"llms":{"status":"present","http":200,"bytes":5768},"home":{"status":"ok","http":200,"url":"https://www.indigo.ca/","html_tokens":133857,"markdown_tokens":118,"pruned_tokens":118}},
{"rank":5820,"domain":"ntsb.gov","origin":"https://ntsb.gov","robots":{"status":"ok","http":200,"url":"https://www.ntsb.gov/robots.txt","bytes":408,"groups":1},"bots":{"GPTBot":["partial","*",16,0],"ChatGPT-User":["partial","*",16,0],"OAI-SearchBot":["partial","*",16,0],"ClaudeBot":["partial","*",16,0],"Claude-User":["partial","*",16,0],"Claude-SearchBot":["partial","*",16,0],"anthropic-ai":["partial","*",16,0],"PerplexityBot":["partial","*",16,0],"Perplexity-User":["partial","*",16,0],"Google-Extended":["partial","*",16,0],"Applebot-Extended":["partial","*",16,0],"CCBot":["partial","*",16,0],"Bytespider":["partial","*",16,0],"Amazonbot":["partial","*",16,0],"meta-externalagent":["partial","*",16,0],"cohere-ai":["partial","*",16,0],"Diffbot":["partial","*",16,0],"ContextFetchBot":["partial","*",16,0]},"llms":{"status":"html","http":200,"bytes":49031},"home":{"status":"ok","http":200,"url":"https://www.ntsb.gov/Pages/home.aspx","html_tokens":16194,"markdown_tokens":392,"pruned_tokens":35}},
{"rank":5821,"domain":"uxdesign.cc","origin":"https://uxdesign.cc","robots":{"status":"ok","http":200,"url":"https://uxdesign.cc/robots.txt","bytes":885,"groups":2},"bots":{"GPTBot":["partial","named",1,7],"ChatGPT-User":["partial","*",15,5],"OAI-SearchBot":["partial","*",15,5],"ClaudeBot":["partial","named",1,7],"Claude-User":["partial","*",15,5],"Claude-SearchBot":["partial","*",15,5],"anthropic-ai":["partial","*",15,5],"PerplexityBot":["partial","*",15,5],"Perplexity-User":["partial","*",15,5],"Google-Extended":["partial","*",15,5],"Applebot-Extended":["partial","named",1,7],"CCBot":["partial","*",15,5],"Bytespider":["partial","named",1,7],"Amazonbot":["partial","named",1,7],"meta-externalagent":["partial","named",1,7],"cohere-ai":["partial","*",15,5],"Diffbot":["partial","*",15,5],"ContextFetchBot":["partial","*",15,5]},"llms":{"status":"http_error","http":403,"bytes":4904},"home":{"status":"http_error","http":403,"url":"https://uxdesign.cc/"}},
{"rank":5822,"domain":"amazon.nl","origin":"https://amazon.nl","robots":{"status":"ok","http":200,"url":"https://www.amazon.nl/robots.txt","bytes":7059,"groups":101},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["blocked","named",1,0],"OAI-SearchBot":["blocked","named",1,0],"ClaudeBot":["blocked","named",2,0],"Claude-User":["blocked","named",1,0],"Claude-SearchBot":["blocked","named",1,0],"anthropic-ai":["partial","*",91,8],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["blocked","named",1,0],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["partial","*",91,8],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["partial","*",91,8],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",91,8]},"llms":{"status":"missing","http":404,"bytes":2165},"home":{"status":"ok","http":200,"url":"https://www.amazon.nl/-/en/","html_tokens":202992,"markdown_tokens":849,"pruned_tokens":175}},
{"rank":5823,"domain":"wrike.com","origin":"https://wrike.com","robots":{"status":"ok","http":200,"url":"https://www.wrike.com/robots.txt","bytes":856,"groups":2},"bots":{"GPTBot":["partial","*",26,0],"ChatGPT-User":["partial","*",26,0],"OAI-SearchBot":["partial","*",26,0],"ClaudeBot":["partial","*",26,0],"Claude-User":["partial","*",26,0],"Claude-SearchBot":["partial","*",26,0],"anthropic-ai":["partial","*",26,0],"PerplexityBot":["partial","*",26,0],"Perplexity-User":["partial","*",26,0],"Google-Extended":["partial","*",26,0],"Applebot-Extended":["partial","*",26,0],"CCBot":["partial","*",26,0],"Bytespider":["partial","*",26,0],"Amazonbot":["partial","*",26,0],"meta-externalagent":["partial","*",26,0],"cohere-ai":["partial","*",26,0],"Diffbot":["partial","*",26,0],"ContextFetchBot":["partial","*",26,0]},"llms":{"status":"present","http":200,"bytes":3579},"home":{"status":"ok","http":200,"url":"https://www.wrike.com/","html_tokens":363848,"markdown_tokens":1045,"pruned_tokens":280}},
{"rank":5824,"domain":"ut.ee","origin":"https://ut.ee","robots":{"status":"ok","http":200,"url":"https://ut.ee/robots.txt","bytes":1809,"groups":1},"bots":{"GPTBot":["partial","*",29,18],"ChatGPT-User":["partial","*",29,18],"OAI-SearchBot":["partial","*",29,18],"ClaudeBot":["partial","*",29,18],"Claude-User":["partial","*",29,18],"Claude-SearchBot":["partial","*",29,18],"anthropic-ai":["partial","*",29,18],"PerplexityBot":["partial","*",29,18],"Perplexity-User":["partial","*",29,18],"Google-Extended":["partial","*",29,18],"Applebot-Extended":["partial","*",29,18],"CCBot":["partial","*",29,18],"Bytespider":["partial","*",29,18],"Amazonbot":["partial","*",29,18],"meta-externalagent":["partial","*",29,18],"cohere-ai":["partial","*",29,18],"Diffbot":["partial","*",29,18],"ContextFetchBot":["partial","*",29,18]},"llms":{"status":"missing","http":404,"bytes":540},"home":{"status":"ok","http":200,"url":"https://ut.ee/et","html_tokens":150425,"markdown_tokens":326,"pruned_tokens":152}},
{"rank":5825,"domain":"oklahoman.com","origin":"https://oklahoman.com","robots":{"status":"ok","http":200,"url":"https://www.oklahoman.com/robots.txt","bytes":13609,"groups":289},"bots":{"GPTBot":["partial","named",1,4],"ChatGPT-User":["partial","named",1,4],"OAI-SearchBot":["partial","named",1,4],"ClaudeBot":["blocked","named",1,0],"Claude-User":["blocked","named",1,0],"Claude-SearchBot":["blocked","named",1,0],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["partial","named",1,4],"Perplexity-User":["blocked","named",1,0],"Google-Extended":["partial","named",1,4],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",2,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",32,0]},"llms":{"status":"http_error","http":402,"bytes":254},"home":{"status":"http_error","http":402,"url":"https://www.oklahoman.com/"}},
{"rank":5826,"domain":"gold.org","origin":"https://gold.org","robots":{"status":"ok","http":200,"url":"https://www.gold.org/robots.txt","bytes":2005,"groups":1},"bots":{"GPTBot":["partial","*",33,18],"ChatGPT-User":["partial","*",33,18],"OAI-SearchBot":["partial","*",33,18],"ClaudeBot":["partial","*",33,18],"Claude-User":["partial","*",33,18],"Claude-SearchBot":["partial","*",33,18],"anthropic-ai":["partial","*",33,18],"PerplexityBot":["partial","*",33,18],"Perplexity-User":["partial","*",33,18],"Google-Extended":["partial","*",33,18],"Applebot-Extended":["partial","*",33,18],"CCBot":["partial","*",33,18],"Bytespider":["partial","*",33,18],"Amazonbot":["partial","*",33,18],"meta-externalagent":["partial","*",33,18],"cohere-ai":["partial","*",33,18],"Diffbot":["partial","*",33,18],"ContextFetchBot":["partial","*",33,18]},"llms":{"status":"missing","http":404,"bytes":1511},"home":{"status":"ok","http":200,"url":"https://www.gold.org/","html_tokens":20896,"markdown_tokens":139,"pruned_tokens":139}},
{"rank":5827,"domain":"cultofmac.com","origin":"https://cultofmac.com","robots":{"status":"ok","http":200,"url":"https://www.cultofmac.com/robots.txt","bytes":279,"groups":4},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["blocked","named",1,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"other_text","http":200,"bytes":3188},"home":{"status":"ok","http":200,"url":"https://www.cultofmac.com/","html_tokens":74105,"markdown_tokens":69,"pruned_tokens":69}},
{"rank":5828,"domain":"cihar.com","origin":"https://cihar.com","robots":{"status":"ok","http":200,"url":"https://cihar.com/robots.txt","bytes":76,"groups":1},"bots":{"GPTBot":["partial","*",1,0],"ChatGPT-User":["partial","*",1,0],"OAI-SearchBot":["partial","*",1,0],"ClaudeBot":["partial","*",1,0],"Claude-User":["partial","*",1,0],"Claude-SearchBot":["partial","*",1,0],"anthropic-ai":["partial","*",1,0],"PerplexityBot":["partial","*",1,0],"Perplexity-User":["partial","*",1,0],"Google-Extended":["partial","*",1,0],"Applebot-Extended":["partial","*",1,0],"CCBot":["partial","*",1,0],"Bytespider":["partial","*",1,0],"Amazonbot":["partial","*",1,0],"meta-externalagent":["partial","*",1,0],"cohere-ai":["partial","*",1,0],"Diffbot":["partial","*",1,0],"ContextFetchBot":["partial","*",1,0]},"llms":{"status":"missing","http":404,"bytes":10112},"home":{"status":"ok","http":200,"url":"https://cihar.com/","html_tokens":3807,"markdown_tokens":542,"pruned_tokens":147}},
{"rank":5829,"domain":"gotquestions.org","origin":"https://gotquestions.org","robots":{"status":"ok","http":200,"url":"https://www.gotquestions.org/robots.txt","bytes":54,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":8779},"home":{"status":"ok","http":200,"url":"https://www.gotquestions.org/","html_tokens":2530,"markdown_tokens":154,"pruned_tokens":154}},
{"rank":5830,"domain":"ebay.it","origin":"https://ebay.it","robots":{"status":"ok","http":200,"url":"https://www.ebay.it/robots.txt","bytes":16652,"groups":7},"bots":{"GPTBot":["partial","named",1,3],"ChatGPT-User":["partial","*",205,7],"OAI-SearchBot":["partial","*",205,7],"ClaudeBot":["partial","named",1,3],"Claude-User":["partial","*",205,7],"Claude-SearchBot":["partial","*",205,7],"anthropic-ai":["partial","named",1,3],"PerplexityBot":["partial","named",1,3],"Perplexity-User":["partial","*",205,7],"Google-Extended":["partial","*",205,7],"Applebot-Extended":["partial","named",1,3],"CCBot":["partial","named",1,3],"Bytespider":["partial","named",1,3],"Amazonbot":["partial","named",1,3],"meta-externalagent":["partial","named",1,3],"cohere-ai":["partial","*",205,7],"Diffbot":["partial","*",205,7],"ContextFetchBot":["partial","*",205,7]},"llms":{"status":"missing","http":404,"bytes":79464},"home":{"status":"http_error","http":403,"url":"https://www.ebay.it/"}},
{"rank":5831,"domain":"savethechildren.org","origin":"https://savethechildren.org","robots":{"status":"refused","http":403,"url":"https://www.savethechildren.org/robots.txt","bytes":4913,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":5832,"domain":"learningapps.org","origin":"https://learningapps.org","robots":{"status":"ok","http":200,"url":"https://learningapps.org/robots.txt","bytes":465,"groups":1},"bots":{"GPTBot":["partial","*",19,0],"ChatGPT-User":["partial","*",19,0],"OAI-SearchBot":["partial","*",19,0],"ClaudeBot":["partial","*",19,0],"Claude-User":["partial","*",19,0],"Claude-SearchBot":["partial","*",19,0],"anthropic-ai":["partial","*",19,0],"PerplexityBot":["partial","*",19,0],"Perplexity-User":["partial","*",19,0],"Google-Extended":["partial","*",19,0],"Applebot-Extended":["partial","*",19,0],"CCBot":["partial","*",19,0],"Bytespider":["partial","*",19,0],"Amazonbot":["partial","*",19,0],"meta-externalagent":["partial","*",19,0],"cohere-ai":["partial","*",19,0],"Diffbot":["partial","*",19,0],"ContextFetchBot":["partial","*",19,0]},"llms":{"status":"missing","http":404,"bytes":424},"home":{"status":"ok","http":200,"url":"https://learningapps.org/","html_tokens":5459,"markdown_tokens":34,"pruned_tokens":34}},
{"rank":5833,"domain":"unl.pt","origin":"https://unl.pt","robots":{"status":"unreachable","http":null,"url":"https://unl.pt/robots.txt","bytes":0,"groups":0,"error":"[SSL: CERTIFICATE_VERIFY_FAILED] certificate verify failed: unable to get local issuer certificate (_ssl.c:1028)"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":5834,"domain":"matplotlib.org","origin":"https://matplotlib.org","robots":{"status":"ok","http":200,"url":"https://matplotlib.org/robots.txt","bytes":1154,"groups":1},"bots":{"GPTBot":["partial","*",33,1],"ChatGPT-User":["partial","*",33,1],"OAI-SearchBot":["partial","*",33,1],"ClaudeBot":["partial","*",33,1],"Claude-User":["partial","*",33,1],"Claude-SearchBot":["partial","*",33,1],"anthropic-ai":["partial","*",33,1],"PerplexityBot":["partial","*",33,1],"Perplexity-User":["partial","*",33,1],"Google-Extended":["partial","*",33,1],"Applebot-Extended":["partial","*",33,1],"CCBot":["partial","*",33,1],"Bytespider":["partial","*",33,1],"Amazonbot":["partial","*",33,1],"meta-externalagent":["partial","*",33,1],"cohere-ai":["partial","*",33,1],"Diffbot":["partial","*",33,1],"ContextFetchBot":["partial","*",33,1]},"llms":{"status":"missing","http":404,"bytes":17903},"home":{"status":"ok","http":200,"url":"https://matplotlib.org/","html_tokens":8719,"markdown_tokens":590,"pruned_tokens":289}},
{"rank":5835,"domain":"osaka.lg.jp","origin":"https://osaka.lg.jp","robots":{"status":"unreachable","http":null,"url":"https://osaka.lg.jp/robots.txt","bytes":0,"groups":0,"error":"cannot resolve osaka.lg.jp"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":5836,"domain":"thisamericanlife.org","origin":"https://thisamericanlife.org","robots":{"status":"ok","http":200,"url":"https://www.thisamericanlife.org/robots.txt","bytes":2189,"groups":1},"bots":{"GPTBot":["partial","*",36,32],"ChatGPT-User":["partial","*",36,32],"OAI-SearchBot":["partial","*",36,32],"ClaudeBot":["partial","*",36,32],"Claude-User":["partial","*",36,32],"Claude-SearchBot":["partial","*",36,32],"anthropic-ai":["partial","*",36,32],"PerplexityBot":["partial","*",36,32],"Perplexity-User":["partial","*",36,32],"Google-Extended":["partial","*",36,32],"Applebot-Extended":["partial","*",36,32],"CCBot":["partial","*",36,32],"Bytespider":["partial","*",36,32],"Amazonbot":["partial","*",36,32],"meta-externalagent":["partial","*",36,32],"cohere-ai":["partial","*",36,32],"Diffbot":["partial","*",36,32],"ContextFetchBot":["partial","*",36,32]},"llms":{"status":"missing","http":404,"bytes":81710},"home":{"status":"ok","http":200,"url":"https://www.thisamericanlife.org/","html_tokens":30812,"markdown_tokens":248,"pruned_tokens":143}},
{"rank":5837,"domain":"epic.org","origin":"https://epic.org","robots":{"status":"ok","http":200,"url":"https://epic.org/robots.txt","bytes":115,"groups":1},"bots":{"GPTBot":["partial","*",1,1],"ChatGPT-User":["partial","*",1,1],"OAI-SearchBot":["partial","*",1,1],"ClaudeBot":["partial","*",1,1],"Claude-User":["partial","*",1,1],"Claude-SearchBot":["partial","*",1,1],"anthropic-ai":["partial","*",1,1],"PerplexityBot":["partial","*",1,1],"Perplexity-User":["partial","*",1,1],"Google-Extended":["partial","*",1,1],"Applebot-Extended":["partial","*",1,1],"CCBot":["partial","*",1,1],"Bytespider":["partial","*",1,1],"Amazonbot":["partial","*",1,1],"meta-externalagent":["partial","*",1,1],"cohere-ai":["partial","*",1,1],"Diffbot":["partial","*",1,1],"ContextFetchBot":["partial","*",1,1]},"llms":{"status":"missing","http":404,"bytes":83166},"home":{"status":"http_error","http":403,"url":"https://epic.org/"}},
{"rank":5838,"domain":"carbonmade.com","origin":"https://carbonmade.com","robots":{"status":"ok","http":200,"url":"https://carbonmade.com/robots.txt","bytes":22,"groups":1},"bots":{"GPTBot":["allowed","*",0,1],"ChatGPT-User":["allowed","*",0,1],"OAI-SearchBot":["allowed","*",0,1],"ClaudeBot":["allowed","*",0,1],"Claude-User":["allowed","*",0,1],"Claude-SearchBot":["allowed","*",0,1],"anthropic-ai":["allowed","*",0,1],"PerplexityBot":["allowed","*",0,1],"Perplexity-User":["allowed","*",0,1],"Google-Extended":["allowed","*",0,1],"Applebot-Extended":["allowed","*",0,1],"CCBot":["allowed","*",0,1],"Bytespider":["allowed","*",0,1],"Amazonbot":["allowed","*",0,1],"meta-externalagent":["allowed","*",0,1],"cohere-ai":["allowed","*",0,1],"Diffbot":["allowed","*",0,1],"ContextFetchBot":["allowed","*",0,1]},"llms":{"status":"html","http":200,"bytes":96634},"home":{"status":"ok","http":200,"url":"https://carbonmade.com/","html_tokens":274653,"markdown_tokens":1212,"pruned_tokens":245}},
{"rank":5839,"domain":"wacom.com","origin":"https://wacom.com","robots":{"status":"ok","http":200,"url":"https://wacom.com/robots.txt","bytes":27,"groups":1},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"missing","http":404,"bytes":190},"home":{"status":"ok","http":200,"url":"https://www.wacom.com/en-us","html_tokens":34381,"markdown_tokens":106,"pruned_tokens":106}},
{"rank":5840,"domain":"pypa.io","origin":"https://pypa.io","robots":{"status":"ok","http":200,"url":"https://www.pypa.io/robots.txt","bytes":661,"groups":1},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"missing","http":404,"bytes":2346},"home":{"status":"ok","http":200,"url":"https://www.pypa.io/en/latest/","html_tokens":3032,"markdown_tokens":301,"pruned_tokens":113}},
{"rank":5841,"domain":"uliege.be","origin":"https://uliege.be","robots":{"status":"server_error","http":500,"url":"https://uliege.be/robots.txt","bytes":73692,"groups":0,"error":"HTTP 500"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":5842,"domain":"jigsy.com","origin":"https://jigsy.com","robots":{"status":"ok","http":200,"url":"https://jigsy.com/robots.txt","bytes":38,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"html","http":200,"bytes":5342},"home":{"status":"ok","http":200,"url":"https://jigsy.com/","html_tokens":3716,"markdown_tokens":65,"pruned_tokens":65}},
{"rank":5843,"domain":"blogspot.se","origin":"https://blogspot.se","robots":{"status":"ok","http":200,"url":"https://blogspot.se/robots.txt","bytes":6624,"groups":4},"bots":{"GPTBot":["partial","*",169,64],"ChatGPT-User":["partial","*",169,64],"OAI-SearchBot":["partial","*",169,64],"ClaudeBot":["partial","*",169,64],"Claude-User":["partial","*",169,64],"Claude-SearchBot":["partial","*",169,64],"anthropic-ai":["partial","*",169,64],"PerplexityBot":["partial","*",169,64],"Perplexity-User":["partial","*",169,64],"Google-Extended":["partial","*",169,64],"Applebot-Extended":["partial","*",169,64],"CCBot":["partial","*",169,64],"Bytespider":["partial","*",169,64],"Amazonbot":["partial","*",169,64],"meta-externalagent":["partial","*",169,64],"cohere-ai":["partial","*",169,64],"Diffbot":["partial","*",169,64],"ContextFetchBot":["partial","*",169,64]},"llms":{"status":"missing","http":404,"bytes":1569},"home":{"status":"ok","http":200,"url":"https://www.google.com/","html_tokens":21460,"markdown_tokens":30,"pruned_tokens":30}},
{"rank":5844,"domain":"stsci.edu","origin":"https://stsci.edu","robots":{"status":"refused","http":403,"url":"https://stsci.edu/robots.txt","bytes":919,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":5845,"domain":"fosstodon.org","origin":"https://fosstodon.org","robots":{"status":"ok","http":200,"url":"https://fosstodon.org/robots.txt","bytes":231,"groups":2},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["partial","*",3,0],"OAI-SearchBot":["partial","*",3,0],"ClaudeBot":["partial","*",3,0],"Claude-User":["partial","*",3,0],"Claude-SearchBot":["partial","*",3,0],"anthropic-ai":["partial","*",3,0],"PerplexityBot":["partial","*",3,0],"Perplexity-User":["partial","*",3,0],"Google-Extended":["partial","*",3,0],"Applebot-Extended":["partial","*",3,0],"CCBot":["partial","*",3,0],"Bytespider":["partial","*",3,0],"Amazonbot":["partial","*",3,0],"meta-externalagent":["partial","*",3,0],"cohere-ai":["partial","*",3,0],"Diffbot":["partial","*",3,0],"ContextFetchBot":["partial","*",3,0]},"llms":{"status":"missing","http":404,"bytes":1539},"home":{"status":"ok","http":200,"url":"https://fosstodon.org/","html_tokens":13651,"markdown_tokens":34,"pruned_tokens":34}},
{"rank":5846,"domain":"springsource.org","origin":"https://springsource.org","robots":{"status":"html","http":200,"url":"https://spring.io/","bytes":405167,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":5847,"domain":"ashrae.org","origin":"https://ashrae.org","robots":{"status":"ok","http":200,"url":"https://ashrae.org/robots.txt","bytes":175485,"groups":1},"bots":{"GPTBot":["partial","*",1653,0],"ChatGPT-User":["partial","*",1653,0],"OAI-SearchBot":["partial","*",1653,0],"ClaudeBot":["partial","*",1653,0],"Claude-User":["partial","*",1653,0],"Claude-SearchBot":["partial","*",1653,0],"anthropic-ai":["partial","*",1653,0],"PerplexityBot":["partial","*",1653,0],"Perplexity-User":["partial","*",1653,0],"Google-Extended":["partial","*",1653,0],"Applebot-Extended":["partial","*",1653,0],"CCBot":["partial","*",1653,0],"Bytespider":["partial","*",1653,0],"Amazonbot":["partial","*",1653,0],"meta-externalagent":["partial","*",1653,0],"cohere-ai":["partial","*",1653,0],"Diffbot":["partial","*",1653,0],"ContextFetchBot":["partial","*",1653,0]},"llms":{"status":"missing","http":404,"bytes":43726},"home":{"status":"ok","http":200,"url":"https://ashrae.org/","html_tokens":37753,"markdown_tokens":214,"pruned_tokens":108}},
{"rank":5848,"domain":"lazada.com.my","origin":"https://lazada.com.my","robots":{"status":"ok","http":200,"url":"https://www.lazada.com.my/robots.txt","bytes":222,"groups":4},"bots":{"GPTBot":["allowed","named",0,1],"ChatGPT-User":["allowed","named",0,1],"OAI-SearchBot":["allowed","named",0,1],"ClaudeBot":["partial","*",4,0],"Claude-User":["partial","*",4,0],"Claude-SearchBot":["partial","*",4,0],"anthropic-ai":["partial","*",4,0],"PerplexityBot":["partial","*",4,0],"Perplexity-User":["partial","*",4,0],"Google-Extended":["partial","*",4,0],"Applebot-Extended":["partial","*",4,0],"CCBot":["partial","*",4,0],"Bytespider":["partial","*",4,0],"Amazonbot":["partial","*",4,0],"meta-externalagent":["partial","*",4,0],"cohere-ai":["partial","*",4,0],"Diffbot":["partial","*",4,0],"ContextFetchBot":["partial","*",4,0]},"llms":{"status":"present","http":200,"bytes":2669},"home":{"status":"ok","http":200,"url":"https://www.lazada.com.my/","html_tokens":115218,"markdown_tokens":571,"pruned_tokens":197}},
{"rank":5849,"domain":"airbnb.co.uk","origin":"https://airbnb.co.uk","robots":{"status":"ok","http":200,"url":"https://www.airbnb.co.uk/robots.txt","bytes":20932,"groups":18},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["partial","named",61,2],"OAI-SearchBot":["partial","named",61,2],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",61,2],"Claude-SearchBot":["partial","*",61,2],"anthropic-ai":["partial","named",63,2],"PerplexityBot":["partial","named",63,2],"Perplexity-User":["partial","*",61,2],"Google-Extended":["partial","*",61,2],"Applebot-Extended":["blocked","named",1,0],"CCBot":["partial","*",61,2],"Bytespider":["partial","*",61,2],"Amazonbot":["partial","*",61,2],"meta-externalagent":["partial","named",61,2],"cohere-ai":["partial","named",66,2],"Diffbot":["partial","*",61,2],"ContextFetchBot":["partial","*",61,2]},"llms":{"status":"missing","http":404,"bytes":2947},"home":{"status":"ok","http":200,"url":"https://www.airbnb.co.uk/","html_tokens":168427,"markdown_tokens":789,"pruned_tokens":174}},
{"rank":5850,"domain":"cloudways.com","origin":"https://cloudways.com","robots":{"status":"refused","http":403,"url":"https://www.cloudways.com/robots.txt","bytes":5576,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":5851,"domain":"care2.com","origin":"https://care2.com","robots":{"status":"html","http":200,"url":"https://www.care2.com","bytes":96877,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":5852,"domain":"kohls.com","origin":"https://kohls.com","robots":{"status":"refused","http":403,"url":"https://kohls.com/robots.txt","bytes":373,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":5853,"domain":"alaskaair.com","origin":"https://alaskaair.com","robots":{"status":"ok","http":200,"url":"https://www.alaskaair.com/robots.txt","bytes":1965,"groups":1},"bots":{"GPTBot":["partial","*",58,6],"ChatGPT-User":["partial","*",58,6],"OAI-SearchBot":["partial","*",58,6],"ClaudeBot":["partial","*",58,6],"Claude-User":["partial","*",58,6],"Claude-SearchBot":["partial","*",58,6],"anthropic-ai":["partial","*",58,6],"PerplexityBot":["partial","*",58,6],"Perplexity-User":["partial","*",58,6],"Google-Extended":["partial","*",58,6],"Applebot-Extended":["partial","*",58,6],"CCBot":["partial","*",58,6],"Bytespider":["partial","*",58,6],"Amazonbot":["partial","*",58,6],"meta-externalagent":["partial","*",58,6],"cohere-ai":["partial","*",58,6],"Diffbot":["partial","*",58,6],"ContextFetchBot":["partial","*",58,6]},"llms":{"status":"html","http":200,"bytes":3017496},"home":{"status":"ok","http":200,"url":"https://www.alaskaair.com/","html_tokens":795243,"markdown_tokens":0,"pruned_tokens":0}},
{"rank":5854,"domain":"pagina12.com.ar","origin":"https://pagina12.com.ar","robots":{"status":"ok","http":200,"url":"https://www.pagina12.com.ar/robots.txt","bytes":181,"groups":1},"bots":{"GPTBot":["allowed","*",0,1],"ChatGPT-User":["allowed","*",0,1],"OAI-SearchBot":["allowed","*",0,1],"ClaudeBot":["allowed","*",0,1],"Claude-User":["allowed","*",0,1],"Claude-SearchBot":["allowed","*",0,1],"anthropic-ai":["allowed","*",0,1],"PerplexityBot":["allowed","*",0,1],"Perplexity-User":["allowed","*",0,1],"Google-Extended":["allowed","*",0,1],"Applebot-Extended":["allowed","*",0,1],"CCBot":["allowed","*",0,1],"Bytespider":["allowed","*",0,1],"Amazonbot":["allowed","*",0,1],"meta-externalagent":["allowed","*",0,1],"cohere-ai":["allowed","*",0,1],"Diffbot":["allowed","*",0,1],"ContextFetchBot":["allowed","*",0,1]},"llms":{"status":"missing","http":404,"bytes":41754},"home":{"status":"ok","http":200,"url":"https://www.pagina12.com.ar/","html_tokens":231667,"markdown_tokens":138,"pruned_tokens":138}},
{"rank":5855,"domain":"marquette.edu","origin":"https://marquette.edu","robots":{"status":"ok","http":200,"url":"https://www.marquette.edu/robots.txt","bytes":1688,"groups":2},"bots":{"GPTBot":["partial","*",53,0],"ChatGPT-User":["partial","*",53,0],"OAI-SearchBot":["partial","*",53,0],"ClaudeBot":["partial","*",53,0],"Claude-User":["partial","*",53,0],"Claude-SearchBot":["partial","*",53,0],"anthropic-ai":["partial","*",53,0],"PerplexityBot":["partial","*",53,0],"Perplexity-User":["partial","*",53,0],"Google-Extended":["partial","*",53,0],"Applebot-Extended":["partial","*",53,0],"CCBot":["partial","*",53,0],"Bytespider":["partial","*",53,0],"Amazonbot":["partial","*",53,0],"meta-externalagent":["partial","*",53,0],"cohere-ai":["partial","*",53,0],"Diffbot":["partial","*",53,0],"ContextFetchBot":["partial","*",53,0]},"llms":{"status":"missing","http":404,"bytes":12440},"home":{"status":"ok","http":200,"url":"https://www.marquette.edu/","html_tokens":11667,"markdown_tokens":606,"pruned_tokens":281}},
{"rank":5856,"domain":"baike.com","origin":"https://baike.com","robots":{"status":"ok","http":200,"url":"https://www.baike.com/robots.txt","bytes":1919,"groups":29},"bots":{"GPTBot":["partial","*",9,1],"ChatGPT-User":["partial","*",9,1],"OAI-SearchBot":["partial","*",9,1],"ClaudeBot":["partial","*",9,1],"Claude-User":["partial","*",9,1],"Claude-SearchBot":["partial","*",9,1],"anthropic-ai":["partial","*",9,1],"PerplexityBot":["partial","*",9,1],"Perplexity-User":["partial","*",9,1],"Google-Extended":["partial","*",9,1],"Applebot-Extended":["partial","*",9,1],"CCBot":["partial","*",9,1],"Bytespider":["partial","*",9,1],"Amazonbot":["partial","*",9,1],"meta-externalagent":["partial","*",9,1],"cohere-ai":["partial","*",9,1],"Diffbot":["partial","*",9,1],"ContextFetchBot":["partial","*",9,1]},"llms":{"status":"html","http":200,"bytes":79343},"home":{"status":"ok","http":200,"url":"https://www.baike.com/","html_tokens":15180,"markdown_tokens":65,"pruned_tokens":65}},
{"rank":5857,"domain":"demandsage.com","origin":"https://demandsage.com","robots":{"status":"ok","http":200,"url":"https://www.demandsage.com/robots.txt","bytes":756,"groups":18},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"missing","http":404,"bytes":306046},"home":{"status":"ok","http":200,"url":"https://www.demandsage.com/","html_tokens":85312,"markdown_tokens":276,"pruned_tokens":120}},
{"rank":5858,"domain":"google.com.uy","origin":"https://google.com.uy","robots":{"status":"ok","http":200,"url":"https://www.google.com.uy/robots.txt","bytes":6624,"groups":4},"bots":{"GPTBot":["partial","*",169,64],"ChatGPT-User":["partial","*",169,64],"OAI-SearchBot":["partial","*",169,64],"ClaudeBot":["partial","*",169,64],"Claude-User":["partial","*",169,64],"Claude-SearchBot":["partial","*",169,64],"anthropic-ai":["partial","*",169,64],"PerplexityBot":["partial","*",169,64],"Perplexity-User":["partial","*",169,64],"Google-Extended":["partial","*",169,64],"Applebot-Extended":["partial","*",169,64],"CCBot":["partial","*",169,64],"Bytespider":["partial","*",169,64],"Amazonbot":["partial","*",169,64],"meta-externalagent":["partial","*",169,64],"cohere-ai":["partial","*",169,64],"Diffbot":["partial","*",169,64],"ContextFetchBot":["partial","*",169,64]},"llms":{"status":"missing","http":404,"bytes":1569},"home":{"status":"ok","http":200,"url":"https://www.google.com/","html_tokens":21429,"markdown_tokens":30,"pruned_tokens":30}},
{"rank":5859,"domain":"nyti.ms","origin":"https://nyti.ms","robots":{"status":"ok","http":200,"url":"https://nyti.ms/robots.txt","bytes":123,"groups":1},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"http_error","http":403,"bytes":771},"home":{"status":"http_error","http":403,"url":"https://www.nytimes.com/"}},
{"rank":5860,"domain":"casino.com","origin":"https://casino.com","robots":{"status":"ok","http":200,"url":"https://www.casino.com/robots.txt","bytes":407,"groups":2},"bots":{"GPTBot":["partial","named",1,1],"ChatGPT-User":["partial","named",1,1],"OAI-SearchBot":["partial","*",1,1],"ClaudeBot":["partial","named",1,1],"Claude-User":["partial","*",1,1],"Claude-SearchBot":["partial","*",1,1],"anthropic-ai":["partial","named",1,1],"PerplexityBot":["partial","named",1,1],"Perplexity-User":["partial","*",1,1],"Google-Extended":["partial","named",1,1],"Applebot-Extended":["partial","*",1,1],"CCBot":["partial","*",1,1],"Bytespider":["partial","*",1,1],"Amazonbot":["partial","*",1,1],"meta-externalagent":["partial","named",1,1],"cohere-ai":["partial","*",1,1],"Diffbot":["partial","*",1,1],"ContextFetchBot":["partial","*",1,1]},"llms":{"status":"present","http":200,"bytes":4289},"home":{"status":"ok","http":200,"url":"https://www.casino.com/","html_tokens":272075,"markdown_tokens":1442,"pruned_tokens":441}},
{"rank":5861,"domain":"israelnationalnews.com","origin":"https://israelnationalnews.com","robots":{"status":"ok","http":200,"url":"https://www.israelnationalnews.com/robots.txt","bytes":1062,"groups":1},"bots":{"GPTBot":["partial","*",37,0],"ChatGPT-User":["partial","*",37,0],"OAI-SearchBot":["partial","*",37,0],"ClaudeBot":["partial","*",37,0],"Claude-User":["partial","*",37,0],"Claude-SearchBot":["partial","*",37,0],"anthropic-ai":["partial","*",37,0],"PerplexityBot":["partial","*",37,0],"Perplexity-User":["partial","*",37,0],"Google-Extended":["partial","*",37,0],"Applebot-Extended":["partial","*",37,0],"CCBot":["partial","*",37,0],"Bytespider":["partial","*",37,0],"Amazonbot":["partial","*",37,0],"meta-externalagent":["partial","*",37,0],"cohere-ai":["partial","*",37,0],"Diffbot":["partial","*",37,0],"ContextFetchBot":["partial","*",37,0]},"llms":{"status":"missing","http":404,"bytes":42829},"home":{"status":"ok","http":200,"url":"https://www.israelnationalnews.com/","html_tokens":50169,"markdown_tokens":56,"pruned_tokens":56}},
{"rank":5862,"domain":"divx.com","origin":"https://divx.com","robots":{"status":"ok","http":200,"url":"https://divx.com/robots.txt","bytes":301,"groups":1},"bots":{"GPTBot":["partial","*",5,1],"ChatGPT-User":["partial","*",5,1],"OAI-SearchBot":["partial","*",5,1],"ClaudeBot":["partial","*",5,1],"Claude-User":["partial","*",5,1],"Claude-SearchBot":["partial","*",5,1],"anthropic-ai":["partial","*",5,1],"PerplexityBot":["partial","*",5,1],"Perplexity-User":["partial","*",5,1],"Google-Extended":["partial","*",5,1],"Applebot-Extended":["partial","*",5,1],"CCBot":["partial","*",5,1],"Bytespider":["partial","*",5,1],"Amazonbot":["partial","*",5,1],"meta-externalagent":["partial","*",5,1],"cohere-ai":["partial","*",5,1],"Diffbot":["partial","*",5,1],"ContextFetchBot":["partial","*",5,1]},"llms":{"status":"present","http":200,"bytes":4788},"home":{"status":"ok","http":200,"url":"https://www.divx.com/","html_tokens":54842,"markdown_tokens":537,"pruned_tokens":251}},
{"rank":5863,"domain":"postfix.org","origin":"https://postfix.org","robots":{"status":"ok","http":200,"url":"https://www.postfix.org/robots.txt","bytes":24,"groups":1},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"missing","http":404,"bytes":146},"home":{"status":"ok","http":200,"url":"https://www.postfix.org/","html_tokens":1432,"markdown_tokens":238,"pruned_tokens":215}},
{"rank":5864,"domain":"nhk.jp","origin":"https://nhk.jp","robots":{"status":"ok","http":200,"url":"https://nhk.jp/robots.txt","bytes":331,"groups":1},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["blocked","named",1,0],"OAI-SearchBot":["blocked","named",1,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":38549},"home":{"status":"unreachable","error":"<StreamReset stream_id:1, error_code:2, remote_reset:True>"}},
{"rank":5865,"domain":"abc13.com","origin":"https://abc13.com","robots":{"status":"ok","http":200,"url":"https://abc13.com/robots.txt","bytes":1124,"groups":26},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["blocked","named",1,0],"OAI-SearchBot":["partial","*",7,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",7,0],"Claude-SearchBot":["partial","*",7,0],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["partial","*",7,0],"Perplexity-User":["partial","*",7,0],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["partial","*",7,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",7,0]},"llms":{"status":"html","http":200,"bytes":73757},"home":{"status":"ok","http":200,"url":"https://abc13.com/","html_tokens":124917,"markdown_tokens":99,"pruned_tokens":51}},
{"rank":5866,"domain":"unb.ca","origin":"https://unb.ca","robots":{"status":"ok","http":200,"url":"https://www.unb.ca/robots.txt","bytes":900,"groups":1},"bots":{"GPTBot":["partial","*",29,0],"ChatGPT-User":["partial","*",29,0],"OAI-SearchBot":["partial","*",29,0],"ClaudeBot":["partial","*",29,0],"Claude-User":["partial","*",29,0],"Claude-SearchBot":["partial","*",29,0],"anthropic-ai":["partial","*",29,0],"PerplexityBot":["partial","*",29,0],"Perplexity-User":["partial","*",29,0],"Google-Extended":["partial","*",29,0],"Applebot-Extended":["partial","*",29,0],"CCBot":["partial","*",29,0],"Bytespider":["partial","*",29,0],"Amazonbot":["partial","*",29,0],"meta-externalagent":["partial","*",29,0],"cohere-ai":["partial","*",29,0],"Diffbot":["partial","*",29,0],"ContextFetchBot":["partial","*",29,0]},"llms":{"status":"missing","http":404,"bytes":37582},"home":{"status":"ok","http":200,"url":"https://www.unb.ca/","html_tokens":23101,"markdown_tokens":92,"pruned_tokens":92}},
{"rank":5867,"domain":"kirkusreviews.com","origin":"https://kirkusreviews.com","robots":{"status":"ok","http":200,"url":"https://www.kirkusreviews.com/robots.txt","bytes":1880,"groups":3},"bots":{"GPTBot":["partial","*",20,0],"ChatGPT-User":["partial","*",20,0],"OAI-SearchBot":["partial","*",20,0],"ClaudeBot":["partial","*",20,0],"Claude-User":["partial","*",20,0],"Claude-SearchBot":["partial","*",20,0],"anthropic-ai":["partial","*",20,0],"PerplexityBot":["partial","*",20,0],"Perplexity-User":["partial","*",20,0],"Google-Extended":["partial","*",20,0],"Applebot-Extended":["partial","*",20,0],"CCBot":["partial","*",20,0],"Bytespider":["partial","*",20,0],"Amazonbot":["partial","*",20,0],"meta-externalagent":["partial","*",20,0],"cohere-ai":["partial","*",20,0],"Diffbot":["partial","*",20,0],"ContextFetchBot":["partial","*",20,0]},"llms":{"status":"timeout","error":"timed out after 10 s"},"home":{"status":"ok","http":200,"url":"https://www.kirkusreviews.com/","html_tokens":39716,"markdown_tokens":147,"pruned_tokens":147}},
{"rank":5868,"domain":"wfaa.com","origin":"https://wfaa.com","robots":{"status":"ok","http":200,"url":"https://www.wfaa.com/robots.txt","bytes":444,"groups":2},"bots":{"GPTBot":["partial","*",10,0],"ChatGPT-User":["partial","*",10,0],"OAI-SearchBot":["partial","*",10,0],"ClaudeBot":["partial","*",10,0],"Claude-User":["partial","*",10,0],"Claude-SearchBot":["partial","*",10,0],"anthropic-ai":["partial","*",10,0],"PerplexityBot":["partial","*",10,0],"Perplexity-User":["partial","*",10,0],"Google-Extended":["partial","*",10,0],"Applebot-Extended":["partial","*",10,0],"CCBot":["partial","*",10,0],"Bytespider":["partial","*",10,0],"Amazonbot":["partial","*",10,0],"meta-externalagent":["partial","*",10,0],"cohere-ai":["partial","*",10,0],"Diffbot":["partial","*",10,0],"ContextFetchBot":["partial","*",10,0]},"llms":{"status":"missing","http":404,"bytes":6843},"home":{"status":"ok","http":200,"url":"https://www.wfaa.com/","html_tokens":109101,"markdown_tokens":1622,"pruned_tokens":44}},
{"rank":5869,"domain":"capital.fr","origin":"https://capital.fr","robots":{"status":"ok","http":200,"url":"https://www.capital.fr/robots.txt","bytes":3291,"groups":9},"bots":{"GPTBot":["partial","*",80,2],"ChatGPT-User":["partial","*",80,2],"OAI-SearchBot":["partial","*",80,2],"ClaudeBot":["partial","*",80,2],"Claude-User":["partial","*",80,2],"Claude-SearchBot":["partial","*",80,2],"anthropic-ai":["partial","*",80,2],"PerplexityBot":["partial","*",80,2],"Perplexity-User":["partial","*",80,2],"Google-Extended":["partial","*",80,2],"Applebot-Extended":["partial","*",80,2],"CCBot":["blocked","named",1,0],"Bytespider":["partial","*",80,2],"Amazonbot":["partial","*",80,2],"meta-externalagent":["partial","*",80,2],"cohere-ai":["partial","*",80,2],"Diffbot":["partial","*",80,2],"ContextFetchBot":["partial","*",80,2]},"llms":{"status":"present","http":200,"bytes":3885},"home":{"status":"ok","http":200,"url":"https://www.capital.fr/","html_tokens":73740,"markdown_tokens":28,"pruned_tokens":28}},
{"rank":5870,"domain":"larousse.fr","origin":"https://larousse.fr","robots":{"status":"ok","http":200,"url":"https://www.larousse.fr/robots.txt","bytes":760,"groups":1},"bots":{"GPTBot":["partial","*",9,0],"ChatGPT-User":["partial","*",9,0],"OAI-SearchBot":["partial","*",9,0],"ClaudeBot":["partial","*",9,0],"Claude-User":["partial","*",9,0],"Claude-SearchBot":["partial","*",9,0],"anthropic-ai":["partial","*",9,0],"PerplexityBot":["partial","*",9,0],"Perplexity-User":["partial","*",9,0],"Google-Extended":["partial","*",9,0],"Applebot-Extended":["partial","*",9,0],"CCBot":["partial","*",9,0],"Bytespider":["partial","*",9,0],"Amazonbot":["partial","*",9,0],"meta-externalagent":["partial","*",9,0],"cohere-ai":["partial","*",9,0],"Diffbot":["partial","*",9,0],"ContextFetchBot":["partial","*",9,0]},"llms":{"status":"html","http":200,"bytes":37352},"home":{"status":"ok","http":200,"url":"https://www.larousse.fr/","html_tokens":14127,"markdown_tokens":81,"pruned_tokens":81}},
{"rank":5871,"domain":"idealo.de","origin":"https://idealo.de","robots":{"status":"refused","http":403,"url":"https://www.idealo.de/robots.txt","bytes":3888,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":5872,"domain":"orthoinfo.org","origin":"https://www.orthoinfo.org","robots":{"status":"ok","http":200,"url":"https://www.orthoinfo.org/robots.txt","bytes":320,"groups":1},"bots":{"GPTBot":["partial","*",10,0],"ChatGPT-User":["partial","*",10,0],"OAI-SearchBot":["partial","*",10,0],"ClaudeBot":["partial","*",10,0],"Claude-User":["partial","*",10,0],"Claude-SearchBot":["partial","*",10,0],"anthropic-ai":["partial","*",10,0],"PerplexityBot":["partial","*",10,0],"Perplexity-User":["partial","*",10,0],"Google-Extended":["partial","*",10,0],"Applebot-Extended":["partial","*",10,0],"CCBot":["partial","*",10,0],"Bytespider":["partial","*",10,0],"Amazonbot":["partial","*",10,0],"meta-externalagent":["partial","*",10,0],"cohere-ai":["partial","*",10,0],"Diffbot":["partial","*",10,0],"ContextFetchBot":["partial","*",10,0]},"llms":{"status":"missing","http":404,"bytes":46259},"home":{"status":"ok","http":200,"url":"https://www.orthoinfo.org/","html_tokens":27725,"markdown_tokens":1121,"pruned_tokens":135}},
{"rank":5873,"domain":"imore.com","origin":"https://imore.com","robots":{"status":"ok","http":200,"url":"https://www.imore.com/robots.txt","bytes":1695,"groups":5},"bots":{"GPTBot":["partial","*",20,0],"ChatGPT-User":["partial","*",20,0],"OAI-SearchBot":["partial","*",20,0],"ClaudeBot":["partial","*",20,0],"Claude-User":["partial","*",20,0],"Claude-SearchBot":["partial","*",20,0],"anthropic-ai":["partial","*",20,0],"PerplexityBot":["partial","*",20,0],"Perplexity-User":["partial","*",20,0],"Google-Extended":["partial","*",20,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["partial","*",20,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["partial","*",20,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",20,0]},"llms":{"status":"missing","http":404,"bytes":498917},"home":{"status":"ok","http":200,"url":"https://www.imore.com/","html_tokens":196140,"markdown_tokens":494,"pruned_tokens":153}},
{"rank":5874,"domain":"funnyordie.com","origin":"https://funnyordie.com","robots":{"status":"refused","http":403,"url":"https://funnyordie.com/robots.txt","bytes":7722,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":5875,"domain":"realmoneygaming.ca","origin":"https://realmoneygaming.ca","robots":{"status":"ok","http":200,"url":"https://realmoneygaming.ca/robots.txt","bytes":541,"groups":1},"bots":{"GPTBot":["partial","*",19,0],"ChatGPT-User":["partial","*",19,0],"OAI-SearchBot":["partial","*",19,0],"ClaudeBot":["partial","*",19,0],"Claude-User":["partial","*",19,0],"Claude-SearchBot":["partial","*",19,0],"anthropic-ai":["partial","*",19,0],"PerplexityBot":["partial","*",19,0],"Perplexity-User":["partial","*",19,0],"Google-Extended":["partial","*",19,0],"Applebot-Extended":["partial","*",19,0],"CCBot":["partial","*",19,0],"Bytespider":["partial","*",19,0],"Amazonbot":["partial","*",19,0],"meta-externalagent":["partial","*",19,0],"cohere-ai":["partial","*",19,0],"Diffbot":["partial","*",19,0],"ContextFetchBot":["partial","*",19,0]},"llms":{"status":"missing","http":404,"bytes":21816},"home":{"status":"ok","http":200,"url":"https://realmoneygaming.ca/","html_tokens":29410,"markdown_tokens":1321,"pruned_tokens":149}},
{"rank":5876,"domain":"rome2rio.com","origin":"https://rome2rio.com","robots":{"status":"ok","http":200,"url":"https://www.rome2rio.com/robots.txt","bytes":10454,"groups":2},"bots":{"GPTBot":["partial","*",26,1],"ChatGPT-User":["partial","*",26,1],"OAI-SearchBot":["partial","*",26,1],"ClaudeBot":["partial","*",26,1],"Claude-User":["partial","*",26,1],"Claude-SearchBot":["partial","*",26,1],"anthropic-ai":["partial","*",26,1],"PerplexityBot":["partial","*",26,1],"Perplexity-User":["partial","*",26,1],"Google-Extended":["partial","*",26,1],"Applebot-Extended":["partial","*",26,1],"CCBot":["partial","*",26,1],"Bytespider":["partial","*",26,1],"Amazonbot":["partial","*",26,1],"meta-externalagent":["partial","*",26,1],"cohere-ai":["partial","*",26,1],"Diffbot":["partial","*",26,1],"ContextFetchBot":["partial","*",26,1]},"llms":{"status":"present","http":200,"bytes":4251},"home":{"status":"http_error","http":403,"url":"https://www.rome2rio.com/"}},
{"rank":5877,"domain":"usfca.edu","origin":"https://usfca.edu","robots":{"status":"refused","http":403,"url":"https://usfca.edu/robots.txt","bytes":560,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":5878,"domain":"tripadvisor.ca","origin":"https://tripadvisor.ca","robots":{"status":"ok","http":200,"url":"https://www.tripadvisor.ca/robots.txt","bytes":22907,"groups":9},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["partial","named",623,8],"OAI-SearchBot":["partial","named",623,8],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",621,8],"Claude-SearchBot":["partial","*",621,8],"anthropic-ai":["partial","*",621,8],"PerplexityBot":["partial","named",625,8],"Perplexity-User":["partial","*",621,8],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["partial","*",621,8],"ContextFetchBot":["partial","*",621,8]},"llms":{"status":"http_error","http":403,"bytes":774},"home":{"status":"http_error","http":403,"url":"https://www.tripadvisor.ca/"}},
{"rank":5879,"domain":"dynamics.com","origin":"https://dynamics.com","robots":{"status":"html","http":200,"url":"https://www.microsoft.com/en-us/dynamics-365/","bytes":430413,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":5880,"domain":"getpaint.net","origin":"https://getpaint.net","robots":{"status":"missing","http":404,"url":"https://paint.net/robots.txt","bytes":355,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":355},"home":{"status":"ok","http":200,"url":"https://paint.net/","html_tokens":3138,"markdown_tokens":599,"pruned_tokens":174}},
{"rank":5881,"domain":"uit.no","origin":"https://uit.no","robots":{"status":"ok","http":200,"url":"https://uit.no/robots.txt","bytes":832,"groups":2},"bots":{"GPTBot":["partial","*",23,8],"ChatGPT-User":["partial","*",23,8],"OAI-SearchBot":["partial","*",23,8],"ClaudeBot":["partial","*",23,8],"Claude-User":["partial","*",23,8],"Claude-SearchBot":["partial","*",23,8],"anthropic-ai":["partial","*",23,8],"PerplexityBot":["partial","*",23,8],"Perplexity-User":["partial","*",23,8],"Google-Extended":["partial","*",23,8],"Applebot-Extended":["partial","*",23,8],"CCBot":["partial","*",23,8],"Bytespider":["partial","*",23,8],"Amazonbot":["partial","*",23,8],"meta-externalagent":["partial","*",23,8],"cohere-ai":["partial","*",23,8],"Diffbot":["partial","*",23,8],"ContextFetchBot":["partial","*",23,8]},"llms":{"status":"missing","http":404,"bytes":381},"home":{"status":"ok","http":200,"url":"https://uit.no/","html_tokens":168,"markdown_tokens":0,"pruned_tokens":0}},
{"rank":5882,"domain":"hw.ac.uk","origin":"https://hw.ac.uk","robots":{"status":"ok","http":200,"url":"https://hw.ac.uk/robots.txt","bytes":359,"groups":1},"bots":{"GPTBot":["partial","*",11,0],"ChatGPT-User":["partial","*",11,0],"OAI-SearchBot":["partial","*",11,0],"ClaudeBot":["partial","*",11,0],"Claude-User":["partial","*",11,0],"Claude-SearchBot":["partial","*",11,0],"anthropic-ai":["partial","*",11,0],"PerplexityBot":["partial","*",11,0],"Perplexity-User":["partial","*",11,0],"Google-Extended":["partial","*",11,0],"Applebot-Extended":["partial","*",11,0],"CCBot":["partial","*",11,0],"Bytespider":["partial","*",11,0],"Amazonbot":["partial","*",11,0],"meta-externalagent":["partial","*",11,0],"cohere-ai":["partial","*",11,0],"Diffbot":["partial","*",11,0],"ContextFetchBot":["partial","*",11,0]},"llms":{"status":"missing","http":404,"bytes":14},"home":{"status":"ok","http":200,"url":"https://hw.ac.uk/","html_tokens":125517,"markdown_tokens":715,"pruned_tokens":51}},
{"rank":5883,"domain":"mof.go.jp","origin":"https://www.mof.go.jp","robots":{"status":"missing","http":404,"url":"https://www.mof.go.jp/robots.txt","bytes":20480,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":20480},"home":{"status":"ok","http":200,"url":"https://www.mof.go.jp/","html_tokens":10697,"markdown_tokens":69,"pruned_tokens":69}},
{"rank":5884,"domain":"gob.gt","origin":"https://gob.gt","robots":{"status":"unreachable","http":null,"url":"https://gob.gt/robots.txt","bytes":0,"groups":0,"error":"cannot resolve gob.gt"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":5885,"domain":"google.ee","origin":"https://google.ee","robots":{"status":"ok","http":200,"url":"https://www.google.ee/robots.txt","bytes":6624,"groups":4},"bots":{"GPTBot":["partial","*",169,64],"ChatGPT-User":["partial","*",169,64],"OAI-SearchBot":["partial","*",169,64],"ClaudeBot":["partial","*",169,64],"Claude-User":["partial","*",169,64],"Claude-SearchBot":["partial","*",169,64],"anthropic-ai":["partial","*",169,64],"PerplexityBot":["partial","*",169,64],"Perplexity-User":["partial","*",169,64],"Google-Extended":["partial","*",169,64],"Applebot-Extended":["partial","*",169,64],"CCBot":["partial","*",169,64],"Bytespider":["partial","*",169,64],"Amazonbot":["partial","*",169,64],"meta-externalagent":["partial","*",169,64],"cohere-ai":["partial","*",169,64],"Diffbot":["partial","*",169,64],"ContextFetchBot":["partial","*",169,64]},"llms":{"status":"missing","http":404,"bytes":1569},"home":{"status":"ok","http":200,"url":"https://www.google.com/","html_tokens":21312,"markdown_tokens":30,"pruned_tokens":30}},
{"rank":5886,"domain":"in.us","origin":"https://in.us","robots":{"status":"unreachable","http":null,"url":"https://in.us/robots.txt","bytes":0,"groups":0,"error":"cannot resolve in.us"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":5887,"domain":"kugou.com","origin":"https://kugou.com","robots":{"status":"missing","http":404,"url":"https://www.kugou.com/robots.txt","bytes":16643,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":16643},"home":{"status":"ok","http":200,"url":"https://www.kugou.com/","html_tokens":179501,"markdown_tokens":18,"pruned_tokens":18}},
{"rank":5888,"domain":"mercadolivre.com.br","origin":"https://mercadolivre.com.br","robots":{"status":"refused","http":403,"url":"https://mercadolivre.com.br/robots.txt","bytes":2585,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":5889,"domain":"airasia.com","origin":"https://airasia.com","robots":{"status":"ok","http":200,"url":"https://www.airasia.com/robots.txt","bytes":956,"groups":1},"bots":{"GPTBot":["partial","*",26,0],"ChatGPT-User":["partial","*",26,0],"OAI-SearchBot":["partial","*",26,0],"ClaudeBot":["partial","*",26,0],"Claude-User":["partial","*",26,0],"Claude-SearchBot":["partial","*",26,0],"anthropic-ai":["partial","*",26,0],"PerplexityBot":["partial","*",26,0],"Perplexity-User":["partial","*",26,0],"Google-Extended":["partial","*",26,0],"Applebot-Extended":["partial","*",26,0],"CCBot":["partial","*",26,0],"Bytespider":["partial","*",26,0],"Amazonbot":["partial","*",26,0],"meta-externalagent":["partial","*",26,0],"cohere-ai":["partial","*",26,0],"Diffbot":["partial","*",26,0],"ContextFetchBot":["partial","*",26,0]},"llms":{"status":"html","http":200,"bytes":246477},"home":{"status":"ok","http":200,"url":"https://www.airasia.com/en/gb/","html_tokens":61598,"markdown_tokens":77,"pruned_tokens":77}},
{"rank":5890,"domain":"patient.info","origin":"https://patient.info","robots":{"status":"ok","http":200,"url":"https://patient.info/robots.txt","bytes":616,"groups":2},"bots":{"GPTBot":["partial","*",21,0],"ChatGPT-User":["partial","*",21,0],"OAI-SearchBot":["partial","*",21,0],"ClaudeBot":["partial","*",21,0],"Claude-User":["partial","*",21,0],"Claude-SearchBot":["partial","*",21,0],"anthropic-ai":["partial","*",21,0],"PerplexityBot":["partial","*",21,0],"Perplexity-User":["partial","*",21,0],"Google-Extended":["partial","*",21,0],"Applebot-Extended":["partial","*",21,0],"CCBot":["partial","*",21,0],"Bytespider":["partial","*",21,0],"Amazonbot":["partial","*",21,0],"meta-externalagent":["partial","*",21,0],"cohere-ai":["partial","*",21,0],"Diffbot":["partial","*",21,0],"ContextFetchBot":["partial","*",21,0]},"llms":{"status":"missing","http":404,"bytes":278394},"home":{"status":"ok","http":200,"url":"https://patient.info/","html_tokens":129905,"markdown_tokens":330,"pruned_tokens":96}},
{"rank":5891,"domain":"joomshaper.com","origin":"https://joomshaper.com","robots":{"status":"ok","http":200,"url":"https://www.joomshaper.com/robots.txt","bytes":761,"groups":1},"bots":{"GPTBot":["partial","*",10,0],"ChatGPT-User":["partial","*",10,0],"OAI-SearchBot":["partial","*",10,0],"ClaudeBot":["partial","*",10,0],"Claude-User":["partial","*",10,0],"Claude-SearchBot":["partial","*",10,0],"anthropic-ai":["partial","*",10,0],"PerplexityBot":["partial","*",10,0],"Perplexity-User":["partial","*",10,0],"Google-Extended":["partial","*",10,0],"Applebot-Extended":["partial","*",10,0],"CCBot":["partial","*",10,0],"Bytespider":["partial","*",10,0],"Amazonbot":["partial","*",10,0],"meta-externalagent":["partial","*",10,0],"cohere-ai":["partial","*",10,0],"Diffbot":["partial","*",10,0],"ContextFetchBot":["partial","*",10,0]},"llms":{"status":"missing","http":404,"bytes":8661},"home":{"status":"ok","http":200,"url":"https://www.joomshaper.com/","html_tokens":57419,"markdown_tokens":496,"pruned_tokens":197}},
{"rank":5892,"domain":"abchina.com","origin":"https://www.abchina.com","robots":{"status":"error","http":302,"url":"https://www.abchina.com/robots.txt","bytes":2564,"groups":0,"error":"HTTP 302"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":5893,"domain":"jica.go.jp","origin":"https://www.jica.go.jp","robots":{"status":"missing","http":404,"url":"https://www.jica.go.jp/Resource/robots.txt","bytes":110586,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":110586},"home":{"status":"ok","http":200,"url":"https://www.jica.go.jp/","html_tokens":47110,"markdown_tokens":2145,"pruned_tokens":172}},
{"rank":5894,"domain":"oxfordreference.com","origin":"https://oxfordreference.com","robots":{"status":"ok","http":200,"url":"https://oxfordreference.com/robots.txt","bytes":810,"groups":2},"bots":{"GPTBot":["partial","*",28,0],"ChatGPT-User":["partial","*",28,0],"OAI-SearchBot":["partial","*",28,0],"ClaudeBot":["partial","*",28,0],"Claude-User":["partial","*",28,0],"Claude-SearchBot":["partial","*",28,0],"anthropic-ai":["partial","*",28,0],"PerplexityBot":["partial","*",28,0],"Perplexity-User":["partial","*",28,0],"Google-Extended":["partial","*",28,0],"Applebot-Extended":["partial","*",28,0],"CCBot":["partial","*",28,0],"Bytespider":["partial","*",28,0],"Amazonbot":["partial","*",28,0],"meta-externalagent":["partial","*",28,0],"cohere-ai":["partial","*",28,0],"Diffbot":["partial","*",28,0],"ContextFetchBot":["partial","*",28,0]},"llms":{"status":"timeout","error":"timed out after 10 s"},"home":{"status":"timeout","error":"timed out after 15 s"}},
{"rank":5895,"domain":"lavozdegalicia.es","origin":"https://lavozdegalicia.es","robots":{"status":"ok","http":200,"url":"https://www.lavozdegalicia.es/robots.txt","bytes":4077,"groups":2},"bots":{"GPTBot":["partial","*",10,0],"ChatGPT-User":["partial","*",10,0],"OAI-SearchBot":["partial","*",10,0],"ClaudeBot":["partial","*",10,0],"Claude-User":["partial","*",10,0],"Claude-SearchBot":["partial","*",10,0],"anthropic-ai":["partial","*",10,0],"PerplexityBot":["partial","*",10,0],"Perplexity-User":["partial","*",10,0],"Google-Extended":["partial","*",10,0],"Applebot-Extended":["partial","*",10,0],"CCBot":["partial","*",10,0],"Bytespider":["partial","*",10,0],"Amazonbot":["partial","*",10,0],"meta-externalagent":["partial","*",10,0],"cohere-ai":["partial","*",10,0],"Diffbot":["partial","*",10,0],"ContextFetchBot":["partial","*",10,0]},"llms":{"status":"timeout","error":"timed out after 10 s"},"home":{"status":"disallowed","error":"robots.txt of www.lavozdegalicia.es not readable (timeout)"}},
{"rank":5896,"domain":"google.org","origin":"https://google.org","robots":{"status":"ok","http":200,"url":"https://www.google.org/robots.txt","bytes":67,"groups":1},"bots":{"GPTBot":["allowed","*",0,1],"ChatGPT-User":["allowed","*",0,1],"OAI-SearchBot":["allowed","*",0,1],"ClaudeBot":["allowed","*",0,1],"Claude-User":["allowed","*",0,1],"Claude-SearchBot":["allowed","*",0,1],"anthropic-ai":["allowed","*",0,1],"PerplexityBot":["allowed","*",0,1],"Perplexity-User":["allowed","*",0,1],"Google-Extended":["allowed","*",0,1],"Applebot-Extended":["allowed","*",0,1],"CCBot":["allowed","*",0,1],"Bytespider":["allowed","*",0,1],"Amazonbot":["allowed","*",0,1],"meta-externalagent":["allowed","*",0,1],"cohere-ai":["allowed","*",0,1],"Diffbot":["allowed","*",0,1],"ContextFetchBot":["allowed","*",0,1]},"llms":{"status":"missing","http":404,"bytes":1569},"home":{"status":"ok","http":200,"url":"https://www.google.org/","html_tokens":30358,"markdown_tokens":191,"pruned_tokens":115}},
{"rank":5897,"domain":"glaad.org","origin":"https://glaad.org","robots":{"status":"refused","http":403,"url":"https://glaad.org/robots.txt","bytes":118,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":5898,"domain":"fastcodesign.com","origin":"https://fastcodesign.com","robots":{"status":"refused","http":403,"url":"https://www.fastcompany.com/co-design","bytes":775,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":5899,"domain":"oohcams.com","origin":"https://oohcams.com","robots":{"status":"ok","http":200,"url":"https://pwa.oohcams.com/robots.txt","bytes":175,"groups":1},"bots":{"GPTBot":["partial","*",3,1],"ChatGPT-User":["partial","*",3,1],"OAI-SearchBot":["partial","*",3,1],"ClaudeBot":["partial","*",3,1],"Claude-User":["partial","*",3,1],"Claude-SearchBot":["partial","*",3,1],"anthropic-ai":["partial","*",3,1],"PerplexityBot":["partial","*",3,1],"Perplexity-User":["partial","*",3,1],"Google-Extended":["partial","*",3,1],"Applebot-Extended":["partial","*",3,1],"CCBot":["partial","*",3,1],"Bytespider":["partial","*",3,1],"Amazonbot":["partial","*",3,1],"meta-externalagent":["partial","*",3,1],"cohere-ai":["partial","*",3,1],"Diffbot":["partial","*",3,1],"ContextFetchBot":["partial","*",3,1]},"llms":{"status":"disallowed","error":"robots.txt of www.oohcams.com not readable (html)"},"home":{"status":"ok","http":200,"url":"https://pwa.oohcams.com/","html_tokens":35311,"markdown_tokens":28,"pruned_tokens":28}},
{"rank":5900,"domain":"ghost.io","origin":"https://ghost.io","robots":{"status":"html","http":200,"url":"https://ghost.org/","bytes":123112,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":5901,"domain":"jbl.com","origin":"https://www.jbl.com","robots":{"status":"ok","http":200,"url":"https://www.jbl.com/robots.txt","bytes":644,"groups":1},"bots":{"GPTBot":["partial","*",25,0],"ChatGPT-User":["partial","*",25,0],"OAI-SearchBot":["partial","*",25,0],"ClaudeBot":["partial","*",25,0],"Claude-User":["partial","*",25,0],"Claude-SearchBot":["partial","*",25,0],"anthropic-ai":["partial","*",25,0],"PerplexityBot":["partial","*",25,0],"Perplexity-User":["partial","*",25,0],"Google-Extended":["partial","*",25,0],"Applebot-Extended":["partial","*",25,0],"CCBot":["partial","*",25,0],"Bytespider":["partial","*",25,0],"Amazonbot":["partial","*",25,0],"meta-externalagent":["partial","*",25,0],"cohere-ai":["partial","*",25,0],"Diffbot":["partial","*",25,0],"ContextFetchBot":["partial","*",25,0]},"llms":{"status":"http_error","http":403,"bytes":1128},"home":{"status":"http_error","http":403,"url":"https://www.jbl.com/"}},
{"rank":5902,"domain":"cbpp.org","origin":"https://cbpp.org","robots":{"status":"ok","http":200,"url":"https://cbpp.org/robots.txt","bytes":3117,"groups":21},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["partial","*",41,22],"OAI-SearchBot":["partial","*",41,22],"ClaudeBot":["partial","*",41,22],"Claude-User":["partial","*",41,22],"Claude-SearchBot":["partial","*",41,22],"anthropic-ai":["partial","*",41,22],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["partial","*",41,22],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",2,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["partial","*",41,22],"Diffbot":["partial","*",41,22],"ContextFetchBot":["partial","*",41,22]},"llms":{"status":"present","http":200,"bytes":2167},"home":{"status":"http_error","http":403,"url":"https://cbpp.org/"}},
{"rank":5903,"domain":"iospress.com","origin":"https://iospress.com","robots":{"status":"ok","http":200,"url":"https://www.iospress.com/robots.txt","bytes":1942,"groups":1},"bots":{"GPTBot":["partial","*",49,0],"ChatGPT-User":["partial","*",49,0],"OAI-SearchBot":["partial","*",49,0],"ClaudeBot":["partial","*",49,0],"Claude-User":["partial","*",49,0],"Claude-SearchBot":["partial","*",49,0],"anthropic-ai":["partial","*",49,0],"PerplexityBot":["partial","*",49,0],"Perplexity-User":["partial","*",49,0],"Google-Extended":["partial","*",49,0],"Applebot-Extended":["partial","*",49,0],"CCBot":["partial","*",49,0],"Bytespider":["partial","*",49,0],"Amazonbot":["partial","*",49,0],"meta-externalagent":["partial","*",49,0],"cohere-ai":["partial","*",49,0],"Diffbot":["partial","*",49,0],"ContextFetchBot":["partial","*",49,0]},"llms":{"status":"missing","http":404,"bytes":190},"home":{"status":"ok","http":200,"url":"https://www.iospress.com/","html_tokens":20961,"markdown_tokens":72,"pruned_tokens":72}},
{"rank":5904,"domain":"mozilla-europe.org","origin":"http://mozilla-europe.org","robots":{"status":"html","http":200,"url":"http://mozilla-europe.org/robots.txt","bytes":812,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":5905,"domain":"getfirefox.com","origin":"https://getfirefox.com","robots":{"status":"html","http":200,"url":"https://www.firefox.com/en-US/?redirect_source=getfirefox-com","bytes":141865,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":5906,"domain":"geocities.ws","origin":"https://geocities.ws","robots":{"status":"missing","http":404,"url":"https://geocities.ws/robots.txt","bytes":692,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":690},"home":{"status":"http_error","http":403,"url":"https://geocities.ws/"}},
{"rank":5907,"domain":"bitcoinist.com","origin":"https://bitcoinist.com","robots":{"status":"ok","http":200,"url":"https://bitcoinist.com/robots.txt","bytes":137,"groups":1},"bots":{"GPTBot":["partial","*",2,1],"ChatGPT-User":["partial","*",2,1],"OAI-SearchBot":["partial","*",2,1],"ClaudeBot":["partial","*",2,1],"Claude-User":["partial","*",2,1],"Claude-SearchBot":["partial","*",2,1],"anthropic-ai":["partial","*",2,1],"PerplexityBot":["partial","*",2,1],"Perplexity-User":["partial","*",2,1],"Google-Extended":["partial","*",2,1],"Applebot-Extended":["partial","*",2,1],"CCBot":["partial","*",2,1],"Bytespider":["partial","*",2,1],"Amazonbot":["partial","*",2,1],"meta-externalagent":["partial","*",2,1],"cohere-ai":["partial","*",2,1],"Diffbot":["partial","*",2,1],"ContextFetchBot":["partial","*",2,1]},"llms":{"status":"html","http":200,"bytes":221467},"home":{"status":"ok","http":200,"url":"https://bitcoinist.com/","html_tokens":55359,"markdown_tokens":378,"pruned_tokens":85}},
{"rank":5908,"domain":"sitemeter.com","origin":"https://sitemeter.com","robots":{"status":"ok","http":200,"url":"https://sitemeter.com/robots.txt","bytes":66,"groups":1},"bots":{"GPTBot":["allowed","*",0,1],"ChatGPT-User":["allowed","*",0,1],"OAI-SearchBot":["allowed","*",0,1],"ClaudeBot":["allowed","*",0,1],"Claude-User":["allowed","*",0,1],"Claude-SearchBot":["allowed","*",0,1],"anthropic-ai":["allowed","*",0,1],"PerplexityBot":["allowed","*",0,1],"Perplexity-User":["allowed","*",0,1],"Google-Extended":["allowed","*",0,1],"Applebot-Extended":["allowed","*",0,1],"CCBot":["allowed","*",0,1],"Bytespider":["allowed","*",0,1],"Amazonbot":["allowed","*",0,1],"meta-externalagent":["allowed","*",0,1],"cohere-ai":["allowed","*",0,1],"Diffbot":["allowed","*",0,1],"ContextFetchBot":["allowed","*",0,1]},"llms":{"status":"other_text","http":200,"bytes":65},"home":{"status":"ok","http":200,"url":"https://sitemeter.com/","html_tokens":29,"markdown_tokens":0,"pruned_tokens":0}},
{"rank":5909,"domain":"tr.ee","origin":"https://tr.ee","robots":{"status":"ok","http":200,"url":"https://tr.ee/robots.txt","bytes":150,"groups":1},"bots":{"GPTBot":["partial","*",4,1],"ChatGPT-User":["partial","*",4,1],"OAI-SearchBot":["partial","*",4,1],"ClaudeBot":["partial","*",4,1],"Claude-User":["partial","*",4,1],"Claude-SearchBot":["partial","*",4,1],"anthropic-ai":["partial","*",4,1],"PerplexityBot":["partial","*",4,1],"Perplexity-User":["partial","*",4,1],"Google-Extended":["partial","*",4,1],"Applebot-Extended":["partial","*",4,1],"CCBot":["partial","*",4,1],"Bytespider":["partial","*",4,1],"Amazonbot":["partial","*",4,1],"meta-externalagent":["partial","*",4,1],"cohere-ai":["partial","*",4,1],"Diffbot":["partial","*",4,1],"ContextFetchBot":["partial","*",4,1]},"llms":{"status":"missing","http":404,"bytes":1415},"home":{"status":"ok","http":200,"url":"https://tr.ee/","html_tokens":55992,"markdown_tokens":262,"pruned_tokens":106}},
{"rank":5910,"domain":"opole.pl","origin":"https://www.opole.pl","robots":{"status":"ok","http":200,"url":"https://www.opole.pl/robots.txt","bytes":2141,"groups":1},"bots":{"GPTBot":["partial","*",35,22],"ChatGPT-User":["partial","*",35,22],"OAI-SearchBot":["partial","*",35,22],"ClaudeBot":["partial","*",35,22],"Claude-User":["partial","*",35,22],"Claude-SearchBot":["partial","*",35,22],"anthropic-ai":["partial","*",35,22],"PerplexityBot":["partial","*",35,22],"Perplexity-User":["partial","*",35,22],"Google-Extended":["partial","*",35,22],"Applebot-Extended":["partial","*",35,22],"CCBot":["partial","*",35,22],"Bytespider":["partial","*",35,22],"Amazonbot":["partial","*",35,22],"meta-externalagent":["partial","*",35,22],"cohere-ai":["partial","*",35,22],"Diffbot":["partial","*",35,22],"ContextFetchBot":["partial","*",35,22]},"llms":{"status":"missing","http":404,"bytes":186},"home":{"status":"ok","http":200,"url":"https://www.opole.pl/dla-mieszkanca","html_tokens":35724,"markdown_tokens":432,"pruned_tokens":105}},
{"rank":5911,"domain":"posit.co","origin":"https://posit.co","robots":{"status":"ok","http":200,"url":"https://posit.co/robots.txt","bytes":2937,"groups":1},"bots":{"GPTBot":["partial","*",59,18],"ChatGPT-User":["partial","*",59,18],"OAI-SearchBot":["partial","*",59,18],"ClaudeBot":["partial","*",59,18],"Claude-User":["partial","*",59,18],"Claude-SearchBot":["partial","*",59,18],"anthropic-ai":["partial","*",59,18],"PerplexityBot":["partial","*",59,18],"Perplexity-User":["partial","*",59,18],"Google-Extended":["partial","*",59,18],"Applebot-Extended":["partial","*",59,18],"CCBot":["partial","*",59,18],"Bytespider":["partial","*",59,18],"Amazonbot":["partial","*",59,18],"meta-externalagent":["partial","*",59,18],"cohere-ai":["partial","*",59,18],"Diffbot":["partial","*",59,18],"ContextFetchBot":["partial","*",59,18]},"llms":{"status":"present","http":200,"bytes":9113},"home":{"status":"ok","http":200,"url":"https://posit.co/","html_tokens":76050,"markdown_tokens":315,"pruned_tokens":111}},
{"rank":5912,"domain":"pennmedicine.org","origin":"https://pennmedicine.org","robots":{"status":"ok","http":200,"url":"https://www.pennmedicine.org/robots.txt","bytes":117,"groups":1},"bots":{"GPTBot":["partial","*",2,0],"ChatGPT-User":["partial","*",2,0],"OAI-SearchBot":["partial","*",2,0],"ClaudeBot":["partial","*",2,0],"Claude-User":["partial","*",2,0],"Claude-SearchBot":["partial","*",2,0],"anthropic-ai":["partial","*",2,0],"PerplexityBot":["partial","*",2,0],"Perplexity-User":["partial","*",2,0],"Google-Extended":["partial","*",2,0],"Applebot-Extended":["partial","*",2,0],"CCBot":["partial","*",2,0],"Bytespider":["partial","*",2,0],"Amazonbot":["partial","*",2,0],"meta-externalagent":["partial","*",2,0],"cohere-ai":["partial","*",2,0],"Diffbot":["partial","*",2,0],"ContextFetchBot":["partial","*",2,0]},"llms":{"status":"present","http":200,"bytes":4641},"home":{"status":"ok","http":200,"url":"https://www.pennmedicine.org/","html_tokens":53,"markdown_tokens":0,"pruned_tokens":0}},
{"rank":5913,"domain":"portableapps.com","origin":"https://portableapps.com","robots":{"status":"ok","http":200,"url":"https://portableapps.com/robots.txt","bytes":7170,"groups":3},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["blocked","named",1,0],"OAI-SearchBot":["blocked","named",1,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["blocked","named",1,0],"Claude-SearchBot":["blocked","named",1,0],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["blocked","named",1,0],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",43,32]},"llms":{"status":"missing","http":404,"bytes":291},"home":{"status":"ok","http":200,"url":"https://portableapps.com/","html_tokens":5546,"markdown_tokens":139,"pruned_tokens":40}},
{"rank":5914,"domain":"jamieoliver.com","origin":"https://jamieoliver.com","robots":{"status":"ok","http":200,"url":"https://www.jamieoliver.com/robots.txt","bytes":2336,"groups":9},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["partial","*",21,6],"OAI-SearchBot":["partial","*",21,6],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",21,6],"Claude-SearchBot":["partial","*",21,6],"anthropic-ai":["partial","*",21,6],"PerplexityBot":["partial","*",21,6],"Perplexity-User":["partial","*",21,6],"Google-Extended":["partial","*",21,6],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["partial","*",21,6],"Diffbot":["partial","*",21,6],"ContextFetchBot":["partial","*",21,6]},"llms":{"status":"missing","http":404,"bytes":402735},"home":{"status":"ok","http":200,"url":"https://www.jamieoliver.com/","html_tokens":200159,"markdown_tokens":40,"pruned_tokens":40}},
{"rank":5915,"domain":"mtv.de","origin":"https://mtv.de","robots":{"status":"ok","http":200,"url":"https://www.mtv.de/robots.txt","bytes":66,"groups":2},"bots":{"GPTBot":["partial","*",1,0],"ChatGPT-User":["partial","*",1,0],"OAI-SearchBot":["partial","*",1,0],"ClaudeBot":["partial","*",1,0],"Claude-User":["partial","*",1,0],"Claude-SearchBot":["partial","*",1,0],"anthropic-ai":["partial","*",1,0],"PerplexityBot":["partial","*",1,0],"Perplexity-User":["partial","*",1,0],"Google-Extended":["partial","*",1,0],"Applebot-Extended":["partial","*",1,0],"CCBot":["partial","*",1,0],"Bytespider":["partial","*",1,0],"Amazonbot":["partial","*",1,0],"meta-externalagent":["partial","*",1,0],"cohere-ai":["partial","*",1,0],"Diffbot":["partial","*",1,0],"ContextFetchBot":["partial","*",1,0]},"llms":{"status":"missing","http":404,"bytes":159890},"home":{"status":"ok","http":200,"url":"https://www.mtv.de","html_tokens":58735,"markdown_tokens":83,"pruned_tokens":26}},
{"rank":5916,"domain":"bgsu.edu","origin":"https://bgsu.edu","robots":{"status":"ok","http":200,"url":"https://www.bgsu.edu/robots.txt","bytes":730,"groups":2},"bots":{"GPTBot":["partial","*",7,1],"ChatGPT-User":["partial","*",7,1],"OAI-SearchBot":["partial","*",7,1],"ClaudeBot":["partial","*",7,1],"Claude-User":["partial","*",7,1],"Claude-SearchBot":["partial","*",7,1],"anthropic-ai":["partial","*",7,1],"PerplexityBot":["partial","*",7,1],"Perplexity-User":["partial","*",7,1],"Google-Extended":["partial","*",7,1],"Applebot-Extended":["partial","*",7,1],"CCBot":["partial","*",7,1],"Bytespider":["partial","*",7,1],"Amazonbot":["partial","*",7,1],"meta-externalagent":["partial","*",7,1],"cohere-ai":["partial","*",7,1],"Diffbot":["partial","*",7,1],"ContextFetchBot":["partial","*",7,1]},"llms":{"status":"missing","http":404,"bytes":123032},"home":{"status":"ok","http":200,"url":"https://www.bgsu.edu/","html_tokens":30197,"markdown_tokens":1669,"pruned_tokens":891}},
{"rank":5917,"domain":"bluestacks.com","origin":"https://bluestacks.com","robots":{"status":"ok","http":200,"url":"https://www.bluestacks.com/robots.txt","bytes":1208,"groups":5},"bots":{"GPTBot":["partial","*",25,1],"ChatGPT-User":["partial","*",25,1],"OAI-SearchBot":["partial","*",25,1],"ClaudeBot":["partial","*",25,1],"Claude-User":["partial","*",25,1],"Claude-SearchBot":["partial","*",25,1],"anthropic-ai":["partial","*",25,1],"PerplexityBot":["partial","*",25,1],"Perplexity-User":["partial","*",25,1],"Google-Extended":["partial","*",25,1],"Applebot-Extended":["partial","*",25,1],"CCBot":["partial","*",25,1],"Bytespider":["partial","*",25,1],"Amazonbot":["partial","*",25,1],"meta-externalagent":["partial","*",25,1],"cohere-ai":["partial","*",25,1],"Diffbot":["partial","*",25,1],"ContextFetchBot":["partial","*",25,1]},"llms":{"status":"missing","http":404,"bytes":53966},"home":{"status":"ok","http":200,"url":"https://www.bluestacks.com/","html_tokens":151864,"markdown_tokens":472,"pruned_tokens":218}},
{"rank":5918,"domain":"chabad.org","origin":"https://chabad.org","robots":{"status":"ok","http":200,"url":"https://www.chabad.org/robots.txt","bytes":65900,"groups":2},"bots":{"GPTBot":["partial","*",1985,3],"ChatGPT-User":["partial","*",1985,3],"OAI-SearchBot":["partial","*",1985,3],"ClaudeBot":["partial","*",1985,3],"Claude-User":["partial","*",1985,3],"Claude-SearchBot":["partial","*",1985,3],"anthropic-ai":["partial","*",1985,3],"PerplexityBot":["partial","*",1985,3],"Perplexity-User":["partial","*",1985,3],"Google-Extended":["partial","*",1985,3],"Applebot-Extended":["partial","*",1985,3],"CCBot":["partial","*",1985,3],"Bytespider":["partial","*",1985,3],"Amazonbot":["partial","*",1985,3],"meta-externalagent":["partial","*",1985,3],"cohere-ai":["partial","*",1985,3],"Diffbot":["partial","*",1985,3],"ContextFetchBot":["partial","*",1985,3]},"llms":{"status":"missing","http":404,"bytes":0},"home":{"status":"http_error","http":403,"url":"https://chabad.org/"}},
{"rank":5919,"domain":"stuttgarter-zeitung.de","origin":"https://stuttgarter-zeitung.de","robots":{"status":"ok","http":200,"url":"https://www.stuttgarter-zeitung.de/robots.txt","bytes":1171,"groups":6},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["blocked","named",1,0],"OAI-SearchBot":["partial","*",8,0],"ClaudeBot":["partial","*",8,0],"Claude-User":["partial","*",8,0],"Claude-SearchBot":["partial","*",8,0],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["partial","*",8,0],"Google-Extended":["partial","*",8,0],"Applebot-Extended":["partial","*",8,0],"CCBot":["blocked","named",1,0],"Bytespider":["partial","*",8,0],"Amazonbot":["partial","*",8,0],"meta-externalagent":["partial","*",8,0],"cohere-ai":["partial","*",8,0],"Diffbot":["partial","*",8,0],"ContextFetchBot":["partial","*",8,0]},"llms":{"status":"missing","http":404,"bytes":114156},"home":{"status":"ok","http":200,"url":"https://www.stuttgarter-zeitung.de/","html_tokens":261029,"markdown_tokens":135,"pruned_tokens":66}},
{"rank":5920,"domain":"australia.com","origin":"https://australia.com","robots":{"status":"ok","http":200,"url":"https://www.australia.com/robots.txt","bytes":3224,"groups":1},"bots":{"GPTBot":["partial","*",25,0],"ChatGPT-User":["partial","*",25,0],"OAI-SearchBot":["partial","*",25,0],"ClaudeBot":["partial","*",25,0],"Claude-User":["partial","*",25,0],"Claude-SearchBot":["partial","*",25,0],"anthropic-ai":["partial","*",25,0],"PerplexityBot":["partial","*",25,0],"Perplexity-User":["partial","*",25,0],"Google-Extended":["partial","*",25,0],"Applebot-Extended":["partial","*",25,0],"CCBot":["partial","*",25,0],"Bytespider":["partial","*",25,0],"Amazonbot":["partial","*",25,0],"meta-externalagent":["partial","*",25,0],"cohere-ai":["partial","*",25,0],"Diffbot":["partial","*",25,0],"ContextFetchBot":["partial","*",25,0]},"llms":{"status":"present","http":200,"bytes":224676},"home":{"status":"ok","http":200,"url":"https://www.australia.com/en-us","html_tokens":141706,"markdown_tokens":65,"pruned_tokens":65}},
{"rank":5921,"domain":"google.ge","origin":"https://google.ge","robots":{"status":"ok","http":200,"url":"https://www.google.ge/robots.txt","bytes":6624,"groups":4},"bots":{"GPTBot":["partial","*",169,64],"ChatGPT-User":["partial","*",169,64],"OAI-SearchBot":["partial","*",169,64],"ClaudeBot":["partial","*",169,64],"Claude-User":["partial","*",169,64],"Claude-SearchBot":["partial","*",169,64],"anthropic-ai":["partial","*",169,64],"PerplexityBot":["partial","*",169,64],"Perplexity-User":["partial","*",169,64],"Google-Extended":["partial","*",169,64],"Applebot-Extended":["partial","*",169,64],"CCBot":["partial","*",169,64],"Bytespider":["partial","*",169,64],"Amazonbot":["partial","*",169,64],"meta-externalagent":["partial","*",169,64],"cohere-ai":["partial","*",169,64],"Diffbot":["partial","*",169,64],"ContextFetchBot":["partial","*",169,64]},"llms":{"status":"missing","http":404,"bytes":1569},"home":{"status":"ok","http":200,"url":"https://www.google.com/","html_tokens":21474,"markdown_tokens":30,"pruned_tokens":30}},
{"rank":5922,"domain":"totalenergies.com","origin":"https://totalenergies.com","robots":{"status":"ok","http":200,"url":"https://totalenergies.com/robots.txt","bytes":4513,"groups":1},"bots":{"GPTBot":["partial","*",30,48],"ChatGPT-User":["partial","*",30,48],"OAI-SearchBot":["partial","*",30,48],"ClaudeBot":["partial","*",30,48],"Claude-User":["partial","*",30,48],"Claude-SearchBot":["partial","*",30,48],"anthropic-ai":["partial","*",30,48],"PerplexityBot":["partial","*",30,48],"Perplexity-User":["partial","*",30,48],"Google-Extended":["partial","*",30,48],"Applebot-Extended":["partial","*",30,48],"CCBot":["partial","*",30,48],"Bytespider":["partial","*",30,48],"Amazonbot":["partial","*",30,48],"meta-externalagent":["partial","*",30,48],"cohere-ai":["partial","*",30,48],"Diffbot":["partial","*",30,48],"ContextFetchBot":["partial","*",30,48]},"llms":{"status":"missing","http":404,"bytes":552},"home":{"status":"ok","http":200,"url":"https://totalenergies.com/","html_tokens":28492,"markdown_tokens":172,"pruned_tokens":170}},
{"rank":5923,"domain":"tweakers.net","origin":"https://tweakers.net","robots":{"status":"ok","http":200,"url":"https://tweakers.net/robots.txt","bytes":2505,"groups":5},"bots":{"GPTBot":["partial","*",62,0],"ChatGPT-User":["partial","*",62,0],"OAI-SearchBot":["partial","*",62,0],"ClaudeBot":["partial","*",62,0],"Claude-User":["partial","*",62,0],"Claude-SearchBot":["partial","*",62,0],"anthropic-ai":["partial","*",62,0],"PerplexityBot":["partial","*",62,0],"Perplexity-User":["partial","*",62,0],"Google-Extended":["partial","*",62,0],"Applebot-Extended":["partial","*",62,0],"CCBot":["partial","*",62,0],"Bytespider":["partial","*",62,0],"Amazonbot":["partial","*",62,0],"meta-externalagent":["partial","*",62,0],"cohere-ai":["partial","*",62,0],"Diffbot":["partial","*",62,0],"ContextFetchBot":["partial","*",62,0]},"llms":{"status":"disallowed","error":"robots.txt of myprivacy.dpgmedia.nl disallows /consent?siteKey=kzz6o4y8ybwvpkea&callbackUrl=https%3A%2F%2Ftweakers.net%2Fprivacy-gate%2Fstore%2F%3"},"home":{"status":"http_error","http":403,"url":"https://tweakers.net/"}},
{"rank":5924,"domain":"iza.org","origin":"https://iza.org","robots":{"status":"ok","http":200,"url":"https://www.iza.org/robots.txt","bytes":24,"groups":1},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"missing","http":404,"bytes":85498},"home":{"status":"ok","http":200,"url":"https://www.iza.org/","html_tokens":45329,"markdown_tokens":422,"pruned_tokens":229}},
{"rank":5925,"domain":"marketresearchfuture.com","origin":"https://marketresearchfuture.com","robots":{"status":"ok","http":200,"url":"https://www.marketresearchfuture.com/robots.txt","bytes":1380,"groups":9},"bots":{"GPTBot":["allowed","named",0,1],"ChatGPT-User":["allowed","named",0,1],"OAI-SearchBot":["allowed","named",0,1],"ClaudeBot":["allowed","named",0,1],"Claude-User":["partial","*",38,1],"Claude-SearchBot":["partial","*",38,1],"anthropic-ai":["allowed","named",0,1],"PerplexityBot":["allowed","named",0,1],"Perplexity-User":["partial","*",38,1],"Google-Extended":["allowed","named",0,1],"Applebot-Extended":["partial","*",38,1],"CCBot":["partial","*",38,1],"Bytespider":["partial","*",38,1],"Amazonbot":["partial","*",38,1],"meta-externalagent":["partial","*",38,1],"cohere-ai":["partial","*",38,1],"Diffbot":["allowed","named",0,1],"ContextFetchBot":["partial","*",38,1]},"llms":{"status":"present","http":200,"bytes":225454},"home":{"status":"ok","http":200,"url":"https://www.marketresearchfuture.com/","html_tokens":53727,"markdown_tokens":740,"pruned_tokens":143}},
{"rank":5926,"domain":"eqxiu.com","origin":"https://eqxiu.com","robots":{"status":"ok","http":200,"url":"https://eqxiu.com/robots.txt","bytes":0,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"empty","http":200,"bytes":0},"home":{"status":"ok","http":200,"url":"https://www.eqxiu.com/","html_tokens":136655,"markdown_tokens":671,"pruned_tokens":150}},
{"rank":5927,"domain":"jiji.com","origin":"https://jiji.com","robots":{"status":"ok","http":200,"url":"https://www.jiji.com/robots.txt","bytes":5411,"groups":65},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["blocked","named",1,0],"OAI-SearchBot":["blocked","named",1,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["blocked","named",1,0],"Claude-SearchBot":["blocked","named",1,0],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["blocked","named",1,0],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",5,4]},"llms":{"status":"missing","http":404,"bytes":1151},"home":{"status":"ok","http":200,"url":"https://www.jiji.com/","html_tokens":16633,"markdown_tokens":11,"pruned_tokens":11}},
{"rank":5928,"domain":"doctorswithoutborders.org","origin":"https://doctorswithoutborders.org","robots":{"status":"refused","http":403,"url":"https://doctorswithoutborders.org/robots.txt","bytes":560,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":5929,"domain":"doctissimo.fr","origin":"https://doctissimo.fr","robots":{"status":"ok","http":200,"url":"https://www.doctissimo.fr/robots.txt","bytes":4431,"groups":2},"bots":{"GPTBot":["partial","*",115,10],"ChatGPT-User":["partial","*",115,10],"OAI-SearchBot":["partial","*",115,10],"ClaudeBot":["partial","*",115,10],"Claude-User":["partial","*",115,10],"Claude-SearchBot":["partial","*",115,10],"anthropic-ai":["partial","*",115,10],"PerplexityBot":["partial","*",115,10],"Perplexity-User":["partial","*",115,10],"Google-Extended":["partial","*",115,10],"Applebot-Extended":["partial","*",115,10],"CCBot":["partial","named",5,0],"Bytespider":["partial","*",115,10],"Amazonbot":["partial","*",115,10],"meta-externalagent":["partial","*",115,10],"cohere-ai":["partial","*",115,10],"Diffbot":["partial","*",115,10],"ContextFetchBot":["partial","*",115,10]},"llms":{"status":"missing","http":404,"bytes":148921},"home":{"status":"ok","http":200,"url":"https://www.doctissimo.fr/","html_tokens":113442,"markdown_tokens":218,"pruned_tokens":112}},
{"rank":5930,"domain":"adsabs.harvard.edu","origin":"https://adsabs.harvard.edu","robots":{"status":"ok","http":200,"url":"https://adsabs.harvard.edu/robots.txt","bytes":757,"groups":5},"bots":{"GPTBot":["partial","*",3,0],"ChatGPT-User":["partial","*",3,0],"OAI-SearchBot":["partial","*",3,0],"ClaudeBot":["partial","*",3,0],"Claude-User":["partial","*",3,0],"Claude-SearchBot":["partial","*",3,0],"anthropic-ai":["partial","*",3,0],"PerplexityBot":["partial","*",3,0],"Perplexity-User":["partial","*",3,0],"Google-Extended":["partial","*",3,0],"Applebot-Extended":["partial","*",3,0],"CCBot":["partial","*",3,0],"Bytespider":["partial","*",3,0],"Amazonbot":["partial","*",3,0],"meta-externalagent":["partial","*",3,0],"cohere-ai":["partial","*",3,0],"Diffbot":["partial","*",3,0],"ContextFetchBot":["partial","*",3,0]},"llms":{"status":"missing","http":404,"bytes":196},"home":{"status":"disallowed","error":"robots.txt of ui.adsabs.harvard.edu not readable (html)"}},
{"rank":5931,"domain":"ivermedx.com","origin":"https://ivermedx.com","robots":{"status":"ok","http":200,"url":"https://ivermedx.com/robots.txt","bytes":144,"groups":1},"bots":{"GPTBot":["partial","*",5,0],"ChatGPT-User":["partial","*",5,0],"OAI-SearchBot":["partial","*",5,0],"ClaudeBot":["partial","*",5,0],"Claude-User":["partial","*",5,0],"Claude-SearchBot":["partial","*",5,0],"anthropic-ai":["partial","*",5,0],"PerplexityBot":["partial","*",5,0],"Perplexity-User":["partial","*",5,0],"Google-Extended":["partial","*",5,0],"Applebot-Extended":["partial","*",5,0],"CCBot":["partial","*",5,0],"Bytespider":["partial","*",5,0],"Amazonbot":["partial","*",5,0],"meta-externalagent":["partial","*",5,0],"cohere-ai":["partial","*",5,0],"Diffbot":["partial","*",5,0],"ContextFetchBot":["partial","*",5,0]},"llms":{"status":"missing","http":404,"bytes":641},"home":{"status":"ok","http":200,"url":"https://ivermedx.com/","html_tokens":11832,"markdown_tokens":3867,"pruned_tokens":151}},
{"rank":5932,"domain":"tm.fr","origin":"https://tm.fr","robots":{"status":"unreachable","http":null,"url":"https://tm.fr/robots.txt","bytes":0,"groups":0,"error":"cannot resolve tm.fr"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":5933,"domain":"fafafaplaypokie.com","origin":"https://fafafaplaypokie.com","robots":{"status":"ok","http":200,"url":"https://fafafaplaypokie.com/robots.txt","bytes":542,"groups":1},"bots":{"GPTBot":["partial","*",19,0],"ChatGPT-User":["partial","*",19,0],"OAI-SearchBot":["partial","*",19,0],"ClaudeBot":["partial","*",19,0],"Claude-User":["partial","*",19,0],"Claude-SearchBot":["partial","*",19,0],"anthropic-ai":["partial","*",19,0],"PerplexityBot":["partial","*",19,0],"Perplexity-User":["partial","*",19,0],"Google-Extended":["partial","*",19,0],"Applebot-Extended":["partial","*",19,0],"CCBot":["partial","*",19,0],"Bytespider":["partial","*",19,0],"Amazonbot":["partial","*",19,0],"meta-externalagent":["partial","*",19,0],"cohere-ai":["partial","*",19,0],"Diffbot":["partial","*",19,0],"ContextFetchBot":["partial","*",19,0]},"llms":{"status":"missing","http":404,"bytes":15},"home":{"status":"http_error","http":404,"url":"https://fafafaplaypokie.com/"}},
{"rank":5934,"domain":"mayoclinichealthsystem.org","origin":"https://mayoclinichealthsystem.org","robots":{"status":"refused","http":403,"url":"https://www.mayoclinichealthsystem.org/robots.txt","bytes":396,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":5935,"domain":"thebalancecareers.com","origin":"https://thebalancecareers.com","robots":{"status":"ok","http":200,"url":"https://www.thebalancemoney.com/robots.txt","bytes":3483,"groups":6},"bots":{"GPTBot":["partial","named",1,0],"ChatGPT-User":["partial","named",1,0],"OAI-SearchBot":["partial","named",1,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["blocked","named",1,0],"Claude-SearchBot":["partial","named",1,2],"anthropic-ai":["partial","named",1,2],"PerplexityBot":["partial","named",1,2],"Perplexity-User":["blocked","named",1,0],"Google-Extended":["partial","named",1,2],"Applebot-Extended":["blocked","named",1,0],"CCBot":["partial","named",1,2],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["partial","named",1,2],"cohere-ai":["partial","named",1,2],"Diffbot":["partial","*",2,0],"ContextFetchBot":["partial","*",2,0]},"llms":{"status":"http_error","http":402,"bytes":612},"home":{"status":"http_error","http":402,"url":"https://www.thebalancemoney.com/career-planning-6265513"}},
{"rank":5936,"domain":"techaro.lol","origin":"https://techaro.lol","robots":{"status":"ok","http":200,"url":"https://techaro.lol/robots.txt","bytes":4190,"groups":2},"bots":{"GPTBot":["partial","named",1,1],"ChatGPT-User":["partial","named",1,1],"OAI-SearchBot":["partial","named",1,1],"ClaudeBot":["partial","named",1,1],"Claude-User":["partial","named",1,1],"Claude-SearchBot":["partial","named",1,1],"anthropic-ai":["partial","named",1,1],"PerplexityBot":["partial","named",1,1],"Perplexity-User":["partial","named",1,1],"Google-Extended":["partial","named",1,1],"Applebot-Extended":["partial","named",1,1],"CCBot":["partial","*",1,1],"Bytespider":["partial","named",1,1],"Amazonbot":["partial","named",1,1],"meta-externalagent":["partial","named",1,1],"cohere-ai":["partial","named",1,1],"Diffbot":["partial","named",1,1],"ContextFetchBot":["partial","*",1,1]},"llms":{"status":"present","http":200,"bytes":3724},"home":{"status":"ok","http":200,"url":"https://techaro.lol/","html_tokens":7656,"markdown_tokens":347,"pruned_tokens":137}},
{"rank":5937,"domain":"e2ma.net","origin":"https://www.e2ma.net","robots":{"status":"missing","http":404,"url":"https://www.e2ma.net/robots.txt","bytes":2298,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":2298},"home":{"status":"ok","http":200,"url":"https://main.e2ma.net/app/auth/login/","html_tokens":1658,"markdown_tokens":15,"pruned_tokens":15}},
{"rank":5938,"domain":"qnap.com","origin":"https://qnap.com","robots":{"status":"ok","http":200,"url":"https://www.qnap.com/robots.txt","bytes":984,"groups":3},"bots":{"GPTBot":["partial","*",13,29],"ChatGPT-User":["partial","*",13,29],"OAI-SearchBot":["partial","*",13,29],"ClaudeBot":["partial","*",13,29],"Claude-User":["partial","*",13,29],"Claude-SearchBot":["partial","*",13,29],"anthropic-ai":["partial","*",13,29],"PerplexityBot":["partial","*",13,29],"Perplexity-User":["partial","*",13,29],"Google-Extended":["partial","*",13,29],"Applebot-Extended":["partial","*",13,29],"CCBot":["partial","*",13,29],"Bytespider":["partial","*",13,29],"Amazonbot":["partial","*",13,29],"meta-externalagent":["partial","*",13,29],"cohere-ai":["partial","*",13,29],"Diffbot":["partial","*",13,29],"ContextFetchBot":["partial","*",13,29]},"llms":{"status":"present","http":200,"bytes":1918},"home":{"status":"ok","http":200,"url":"https://www.qnap.com/en-us","html_tokens":41089,"markdown_tokens":235,"pruned_tokens":191}},
{"rank":5939,"domain":"wgbh.org","origin":"https://wgbh.org","robots":{"status":"ok","http":200,"url":"https://wgbh.org/robots.txt","bytes":154,"groups":2},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["blocked","named",1,0],"OAI-SearchBot":["partial","*",3,0],"ClaudeBot":["partial","*",3,0],"Claude-User":["partial","*",3,0],"Claude-SearchBot":["partial","*",3,0],"anthropic-ai":["partial","*",3,0],"PerplexityBot":["partial","*",3,0],"Perplexity-User":["partial","*",3,0],"Google-Extended":["partial","*",3,0],"Applebot-Extended":["partial","*",3,0],"CCBot":["partial","*",3,0],"Bytespider":["partial","*",3,0],"Amazonbot":["partial","*",3,0],"meta-externalagent":["partial","*",3,0],"cohere-ai":["partial","*",3,0],"Diffbot":["partial","*",3,0],"ContextFetchBot":["partial","*",3,0]},"llms":{"status":"missing","http":404,"bytes":164488},"home":{"status":"ok","http":200,"url":"https://wgbh.org/","html_tokens":82723,"markdown_tokens":120,"pruned_tokens":120}},
{"rank":5940,"domain":"gizmag.com","origin":"https://gizmag.com","robots":{"status":"ok","http":200,"url":"https://newatlas.com/robots.txt","bytes":357,"groups":3},"bots":{"GPTBot":["partial","*",3,0],"ChatGPT-User":["partial","*",3,0],"OAI-SearchBot":["partial","*",3,0],"ClaudeBot":["partial","*",3,0],"Claude-User":["partial","*",3,0],"Claude-SearchBot":["partial","*",3,0],"anthropic-ai":["partial","*",3,0],"PerplexityBot":["partial","*",3,0],"Perplexity-User":["partial","*",3,0],"Google-Extended":["partial","*",3,0],"Applebot-Extended":["partial","*",3,0],"CCBot":["partial","*",3,0],"Bytespider":["partial","*",3,0],"Amazonbot":["partial","*",3,0],"meta-externalagent":["partial","*",3,0],"cohere-ai":["partial","*",3,0],"Diffbot":["partial","*",3,0],"ContextFetchBot":["partial","*",3,0]},"llms":{"status":"missing","http":404,"bytes":97328},"home":{"status":"ok","http":200,"url":"https://newatlas.com/","html_tokens":53726,"markdown_tokens":835,"pruned_tokens":151}},
{"rank":5941,"domain":"thalia.de","origin":"https://thalia.de","robots":{"status":"refused","http":403,"url":"https://thalia.de/robots.txt","bytes":9191,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":5942,"domain":"maastrichtuniversity.nl","origin":"https://maastrichtuniversity.nl","robots":{"status":"ok","http":200,"url":"https://www.maastrichtuniversity.nl/robots.txt","bytes":2141,"groups":1},"bots":{"GPTBot":["partial","*",35,22],"ChatGPT-User":["partial","*",35,22],"OAI-SearchBot":["partial","*",35,22],"ClaudeBot":["partial","*",35,22],"Claude-User":["partial","*",35,22],"Claude-SearchBot":["partial","*",35,22],"anthropic-ai":["partial","*",35,22],"PerplexityBot":["partial","*",35,22],"Perplexity-User":["partial","*",35,22],"Google-Extended":["partial","*",35,22],"Applebot-Extended":["partial","*",35,22],"CCBot":["partial","*",35,22],"Bytespider":["partial","*",35,22],"Amazonbot":["partial","*",35,22],"meta-externalagent":["partial","*",35,22],"cohere-ai":["partial","*",35,22],"Diffbot":["partial","*",35,22],"ContextFetchBot":["partial","*",35,22]},"llms":{"status":"http_error","http":403,"bytes":146},"home":{"status":"ok","http":200,"url":"https://www.maastrichtuniversity.nl/","html_tokens":159767,"markdown_tokens":418,"pruned_tokens":136}},
{"rank":5943,"domain":"zimbio.com","origin":"https://zimbio.com","robots":{"status":"ok","http":200,"url":"https://www.zimbio.com/robots.txt","bytes":118,"groups":1},"bots":{"GPTBot":["partial","*",1,1],"ChatGPT-User":["partial","*",1,1],"OAI-SearchBot":["partial","*",1,1],"ClaudeBot":["partial","*",1,1],"Claude-User":["partial","*",1,1],"Claude-SearchBot":["partial","*",1,1],"anthropic-ai":["partial","*",1,1],"PerplexityBot":["partial","*",1,1],"Perplexity-User":["partial","*",1,1],"Google-Extended":["partial","*",1,1],"Applebot-Extended":["partial","*",1,1],"CCBot":["partial","*",1,1],"Bytespider":["partial","*",1,1],"Amazonbot":["partial","*",1,1],"meta-externalagent":["partial","*",1,1],"cohere-ai":["partial","*",1,1],"Diffbot":["partial","*",1,1],"ContextFetchBot":["partial","*",1,1]},"llms":{"status":"html","http":200,"bytes":150860},"home":{"status":"ok","http":200,"url":"https://www.zimbio.com/","html_tokens":37701,"markdown_tokens":217,"pruned_tokens":150}},
{"rank":5944,"domain":"saatchiart.com","origin":"https://saatchiart.com","robots":{"status":"refused","http":403,"url":"https://www.saatchiart.com/robots.txt","bytes":386,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":5945,"domain":"uminho.pt","origin":"https://www.uminho.pt","robots":{"status":"server_error","http":500,"url":"https://www.uminho.pt/robots.txt","bytes":45259,"groups":0,"error":"HTTP 500"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":5946,"domain":"podchaser.com","origin":"https://podchaser.com","robots":{"status":"missing","http":404,"url":"https://podchaser.com/robots.txt","bytes":9,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"http_error","http":403,"bytes":919},"home":{"status":"http_error","http":403,"url":"https://podchaser.com/"}},
{"rank":5947,"domain":"nt.gov.au","origin":"https://nt.gov.au","robots":{"status":"refused","http":403,"url":"https://nt.gov.au/robots.txt","bytes":5504,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":5948,"domain":"childmind.org","origin":"https://childmind.org","robots":{"status":"ok","http":200,"url":"https://childmind.org/robots.txt","bytes":654,"groups":1},"bots":{"GPTBot":["partial","*",16,1],"ChatGPT-User":["partial","*",16,1],"OAI-SearchBot":["partial","*",16,1],"ClaudeBot":["partial","*",16,1],"Claude-User":["partial","*",16,1],"Claude-SearchBot":["partial","*",16,1],"anthropic-ai":["partial","*",16,1],"PerplexityBot":["partial","*",16,1],"Perplexity-User":["partial","*",16,1],"Google-Extended":["partial","*",16,1],"Applebot-Extended":["partial","*",16,1],"CCBot":["partial","*",16,1],"Bytespider":["partial","*",16,1],"Amazonbot":["partial","*",16,1],"meta-externalagent":["partial","*",16,1],"cohere-ai":["partial","*",16,1],"Diffbot":["partial","*",16,1],"ContextFetchBot":["partial","*",16,1]},"llms":{"status":"http_error","http":403,"bytes":6448},"home":{"status":"http_error","http":403,"url":"https://childmind.org/"}},
{"rank":5949,"domain":"metafilter.com","origin":"https://metafilter.com","robots":{"status":"ok","http":200,"url":"https://www.metafilter.com/robots.txt","bytes":2061,"groups":15},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["partial","*",13,0],"OAI-SearchBot":["partial","*",13,0],"ClaudeBot":["partial","*",13,0],"Claude-User":["partial","*",13,0],"Claude-SearchBot":["partial","*",13,0],"anthropic-ai":["partial","*",13,0],"PerplexityBot":["partial","*",13,0],"Perplexity-User":["partial","*",13,0],"Google-Extended":["partial","*",13,0],"Applebot-Extended":["partial","*",13,0],"CCBot":["partial","*",13,0],"Bytespider":["partial","*",13,0],"Amazonbot":["partial","*",13,0],"meta-externalagent":["partial","*",13,0],"cohere-ai":["partial","*",13,0],"Diffbot":["partial","*",13,0],"ContextFetchBot":["partial","*",13,0]},"llms":{"status":"missing","http":404,"bytes":17169},"home":{"status":"http_error","http":403,"url":"https://www.metafilter.com/"}},
{"rank":5950,"domain":"henan.gov.cn","origin":"https://henan.gov.cn","robots":{"status":"refused","http":403,"url":"https://henan.gov.cn/robots.txt","bytes":1626,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":5951,"domain":"istudio.sellto.carousell.sg","origin":"https://istudio.sellto.carousell.sg","robots":{"status":"ok","http":200,"url":"https://istudio.sellto.carousell.sg/robots.txt","bytes":245,"groups":1},"bots":{"GPTBot":["partial","*",10,0],"ChatGPT-User":["partial","*",10,0],"OAI-SearchBot":["partial","*",10,0],"ClaudeBot":["partial","*",10,0],"Claude-User":["partial","*",10,0],"Claude-SearchBot":["partial","*",10,0],"anthropic-ai":["partial","*",10,0],"PerplexityBot":["partial","*",10,0],"Perplexity-User":["partial","*",10,0],"Google-Extended":["partial","*",10,0],"Applebot-Extended":["partial","*",10,0],"CCBot":["partial","*",10,0],"Bytespider":["partial","*",10,0],"Amazonbot":["partial","*",10,0],"meta-externalagent":["partial","*",10,0],"cohere-ai":["partial","*",10,0],"Diffbot":["partial","*",10,0],"ContextFetchBot":["partial","*",10,0]},"llms":{"status":"missing","http":404,"bytes":4487},"home":{"status":"http_error","http":403,"url":"https://istudio.sellto.carousell.sg/"}},
{"rank":5952,"domain":"ebird.org","origin":"https://ebird.org","robots":{"status":"ok","http":200,"url":"https://ebird.org/robots.txt","bytes":2409,"groups":2},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["blocked","named",1,0],"OAI-SearchBot":["blocked","named",1,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["blocked","named",1,0],"Claude-SearchBot":["blocked","named",1,0],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["blocked","named",1,0],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["blocked","*",1,0]},"llms":{"status":"disallowed"},"home":{"status":"disallowed"}},
{"rank":5953,"domain":"orcd.co","origin":"https://orcd.co","robots":{"status":"ok","http":200,"url":"https://orcd.co/robots.txt","bytes":60,"groups":1},"bots":{"GPTBot":["allowed","*",0,1],"ChatGPT-User":["allowed","*",0,1],"OAI-SearchBot":["allowed","*",0,1],"ClaudeBot":["allowed","*",0,1],"Claude-User":["allowed","*",0,1],"Claude-SearchBot":["allowed","*",0,1],"anthropic-ai":["allowed","*",0,1],"PerplexityBot":["allowed","*",0,1],"Perplexity-User":["allowed","*",0,1],"Google-Extended":["allowed","*",0,1],"Applebot-Extended":["allowed","*",0,1],"CCBot":["allowed","*",0,1],"Bytespider":["allowed","*",0,1],"Amazonbot":["allowed","*",0,1],"meta-externalagent":["allowed","*",0,1],"cohere-ai":["allowed","*",0,1],"Diffbot":["allowed","*",0,1],"ContextFetchBot":["allowed","*",0,1]},"llms":{"status":"missing","http":404,"bytes":0},"home":{"status":"ok","http":200,"url":"https://orcd.co/","html_tokens":12157,"markdown_tokens":23,"pruned_tokens":23}},
{"rank":5954,"domain":"cites.org","origin":"https://cites.org","robots":{"status":"ok","http":200,"url":"https://cites.org/robots.txt","bytes":2676,"groups":32},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["allowed","*",0,1],"OAI-SearchBot":["allowed","*",0,1],"ClaudeBot":["blocked","named",1,0],"Claude-User":["allowed","*",0,1],"Claude-SearchBot":["allowed","*",0,1],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["allowed","*",0,1],"Perplexity-User":["allowed","*",0,1],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["allowed","*",0,1]},"llms":{"status":"http_error","http":403,"bytes":5498},"home":{"status":"http_error","http":403,"url":"https://cites.org/"}},
{"rank":5955,"domain":"canadiantire.ca","origin":"https://canadiantire.ca","robots":{"status":"ok","http":200,"url":"https://www.canadiantire.ca/robots.txt","bytes":364,"groups":1},"bots":{"GPTBot":["partial","*",1,4],"ChatGPT-User":["partial","*",1,4],"OAI-SearchBot":["partial","*",1,4],"ClaudeBot":["partial","*",1,4],"Claude-User":["partial","*",1,4],"Claude-SearchBot":["partial","*",1,4],"anthropic-ai":["partial","*",1,4],"PerplexityBot":["partial","*",1,4],"Perplexity-User":["partial","*",1,4],"Google-Extended":["partial","*",1,4],"Applebot-Extended":["partial","*",1,4],"CCBot":["partial","*",1,4],"Bytespider":["partial","*",1,4],"Amazonbot":["partial","*",1,4],"meta-externalagent":["partial","*",1,4],"cohere-ai":["partial","*",1,4],"Diffbot":["partial","*",1,4],"ContextFetchBot":["partial","*",1,4]},"llms":{"status":"too_large","error":"response larger than 5 MB"},"home":{"status":"ok","http":200,"url":"https://www.canadiantire.ca/en.html","html_tokens":108194,"markdown_tokens":8,"pruned_tokens":8}},
{"rank":5956,"domain":"pri.org","origin":"https://pri.org","robots":{"status":"ok","http":200,"url":"https://theworld.org/robots.txt","bytes":323,"groups":4},"bots":{"GPTBot":["partial","*",5,0],"ChatGPT-User":["partial","*",5,0],"OAI-SearchBot":["partial","*",5,0],"ClaudeBot":["partial","*",5,0],"Claude-User":["partial","*",5,0],"Claude-SearchBot":["partial","*",5,0],"anthropic-ai":["partial","*",5,0],"PerplexityBot":["partial","*",5,0],"Perplexity-User":["partial","*",5,0],"Google-Extended":["partial","*",5,0],"Applebot-Extended":["partial","*",5,0],"CCBot":["partial","*",5,0],"Bytespider":["partial","*",5,0],"Amazonbot":["partial","*",5,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["partial","*",5,0],"Diffbot":["partial","*",5,0],"ContextFetchBot":["partial","*",5,0]},"llms":{"status":"missing","http":404,"bytes":31181},"home":{"status":"ok","http":200,"url":"https://theworld.org/","html_tokens":1066756,"markdown_tokens":94,"pruned_tokens":94}},
{"rank":5957,"domain":"asics.com","origin":"https://asics.com","robots":{"status":"ok","http":200,"url":"https://www.asics.com/robots.txt","bytes":1978,"groups":1},"bots":{"GPTBot":["partial","*",67,0],"ChatGPT-User":["partial","*",67,0],"OAI-SearchBot":["partial","*",67,0],"ClaudeBot":["partial","*",67,0],"Claude-User":["partial","*",67,0],"Claude-SearchBot":["partial","*",67,0],"anthropic-ai":["partial","*",67,0],"PerplexityBot":["partial","*",67,0],"Perplexity-User":["partial","*",67,0],"Google-Extended":["partial","*",67,0],"Applebot-Extended":["partial","*",67,0],"CCBot":["partial","*",67,0],"Bytespider":["partial","*",67,0],"Amazonbot":["partial","*",67,0],"meta-externalagent":["partial","*",67,0],"cohere-ai":["partial","*",67,0],"Diffbot":["partial","*",67,0],"ContextFetchBot":["partial","*",67,0]},"llms":{"status":"error","error":"more than 5 redirects"},"home":{"status":"http_error","http":403,"url":"https://www.asics.com/us/en-us/"}},
{"rank":5958,"domain":"zabbix.com","origin":"https://zabbix.com","robots":{"status":"ok","http":200,"url":"https://www.zabbix.com/robots.txt","bytes":198,"groups":1},"bots":{"GPTBot":["partial","*",7,0],"ChatGPT-User":["partial","*",7,0],"OAI-SearchBot":["partial","*",7,0],"ClaudeBot":["partial","*",7,0],"Claude-User":["partial","*",7,0],"Claude-SearchBot":["partial","*",7,0],"anthropic-ai":["partial","*",7,0],"PerplexityBot":["partial","*",7,0],"Perplexity-User":["partial","*",7,0],"Google-Extended":["partial","*",7,0],"Applebot-Extended":["partial","*",7,0],"CCBot":["partial","*",7,0],"Bytespider":["partial","*",7,0],"Amazonbot":["partial","*",7,0],"meta-externalagent":["partial","*",7,0],"cohere-ai":["partial","*",7,0],"Diffbot":["partial","*",7,0],"ContextFetchBot":["partial","*",7,0]},"llms":{"status":"html","http":200,"bytes":104386},"home":{"status":"ok","http":200,"url":"https://www.zabbix.com/","html_tokens":65742,"markdown_tokens":280,"pruned_tokens":217}},
{"rank":5959,"domain":"nttdata.com","origin":"https://nttdata.com","robots":{"status":"ok","http":200,"url":"https://www.nttdata.com/robots.txt","bytes":362,"groups":2},"bots":{"GPTBot":["partial","*",3,0],"ChatGPT-User":["partial","*",3,0],"OAI-SearchBot":["partial","*",3,0],"ClaudeBot":["partial","*",3,0],"Claude-User":["partial","*",3,0],"Claude-SearchBot":["partial","*",3,0],"anthropic-ai":["partial","*",3,0],"PerplexityBot":["partial","*",3,0],"Perplexity-User":["partial","*",3,0],"Google-Extended":["partial","*",3,0],"Applebot-Extended":["partial","*",3,0],"CCBot":["partial","*",3,0],"Bytespider":["partial","*",3,0],"Amazonbot":["partial","*",3,0],"meta-externalagent":["partial","*",3,0],"cohere-ai":["partial","*",3,0],"Diffbot":["partial","*",3,0],"ContextFetchBot":["partial","*",3,0]},"llms":{"status":"missing","http":404,"bytes":206439},"home":{"status":"ok","http":200,"url":"https://www.nttdata.com/en-us","html_tokens":95187,"markdown_tokens":118,"pruned_tokens":118}},
{"rank":5960,"domain":"chubb.com","origin":"https://chubb.com","robots":{"status":"ok","http":200,"url":"https://www.chubb.com/robots.txt","bytes":527,"groups":1},"bots":{"GPTBot":["partial","*",9,0],"ChatGPT-User":["partial","*",9,0],"OAI-SearchBot":["partial","*",9,0],"ClaudeBot":["partial","*",9,0],"Claude-User":["partial","*",9,0],"Claude-SearchBot":["partial","*",9,0],"anthropic-ai":["partial","*",9,0],"PerplexityBot":["partial","*",9,0],"Perplexity-User":["partial","*",9,0],"Google-Extended":["partial","*",9,0],"Applebot-Extended":["partial","*",9,0],"CCBot":["partial","*",9,0],"Bytespider":["partial","*",9,0],"Amazonbot":["partial","*",9,0],"meta-externalagent":["partial","*",9,0],"cohere-ai":["partial","*",9,0],"Diffbot":["partial","*",9,0],"ContextFetchBot":["partial","*",9,0]},"llms":{"status":"missing","http":404,"bytes":288164},"home":{"status":"ok","http":200,"url":"https://www.chubb.com/us-en/","html_tokens":78488,"markdown_tokens":207,"pruned_tokens":207}},
{"rank":5961,"domain":"myflorida.com","origin":"https://myflorida.com","robots":{"status":"missing","http":404,"url":"https://www.myflorida.gov/robots.txt","bytes":19090,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":19090},"home":{"status":"ok","http":200,"url":"https://www.myflorida.gov/","html_tokens":4773,"markdown_tokens":13,"pruned_tokens":13}},
{"rank":5962,"domain":"expressjs.com","origin":"https://expressjs.com","robots":{"status":"ok","http":200,"url":"https://expressjs.com/robots.txt","bytes":73,"groups":1},"bots":{"GPTBot":["allowed","*",0,1],"ChatGPT-User":["allowed","*",0,1],"OAI-SearchBot":["allowed","*",0,1],"ClaudeBot":["allowed","*",0,1],"Claude-User":["allowed","*",0,1],"Claude-SearchBot":["allowed","*",0,1],"anthropic-ai":["allowed","*",0,1],"PerplexityBot":["allowed","*",0,1],"Perplexity-User":["allowed","*",0,1],"Google-Extended":["allowed","*",0,1],"Applebot-Extended":["allowed","*",0,1],"CCBot":["allowed","*",0,1],"Bytespider":["allowed","*",0,1],"Amazonbot":["allowed","*",0,1],"meta-externalagent":["allowed","*",0,1],"cohere-ai":["allowed","*",0,1],"Diffbot":["allowed","*",0,1],"ContextFetchBot":["allowed","*",0,1]},"llms":{"status":"present","http":200,"bytes":1552},"home":{"status":"ok","http":200,"url":"https://expressjs.com/","html_tokens":22093,"markdown_tokens":163,"pruned_tokens":132}},
{"rank":5963,"domain":"lazada.vn","origin":"https://lazada.vn","robots":{"status":"ok","http":200,"url":"https://www.lazada.vn/robots.txt","bytes":221,"groups":4},"bots":{"GPTBot":["allowed","named",0,1],"ChatGPT-User":["allowed","named",0,1],"OAI-SearchBot":["allowed","named",0,1],"ClaudeBot":["partial","*",4,0],"Claude-User":["partial","*",4,0],"Claude-SearchBot":["partial","*",4,0],"anthropic-ai":["partial","*",4,0],"PerplexityBot":["partial","*",4,0],"Perplexity-User":["partial","*",4,0],"Google-Extended":["partial","*",4,0],"Applebot-Extended":["partial","*",4,0],"CCBot":["partial","*",4,0],"Bytespider":["partial","*",4,0],"Amazonbot":["partial","*",4,0],"meta-externalagent":["partial","*",4,0],"cohere-ai":["partial","*",4,0],"Diffbot":["partial","*",4,0],"ContextFetchBot":["partial","*",4,0]},"llms":{"status":"present","http":200,"bytes":3661},"home":{"status":"ok","http":200,"url":"https://www.lazada.vn/","html_tokens":116662,"markdown_tokens":336,"pruned_tokens":289}},
{"rank":5964,"domain":"wionews.com","origin":"https://wionews.com","robots":{"status":"ok","http":200,"url":"https://www.wionews.com/robots.txt","bytes":1059,"groups":8},"bots":{"GPTBot":["allowed","named",0,1],"ChatGPT-User":["allowed","named",0,1],"OAI-SearchBot":["blocked","named",1,0],"ClaudeBot":["allowed","named",0,1],"Claude-User":["partial","*",1,1],"Claude-SearchBot":["partial","*",1,1],"anthropic-ai":["partial","*",1,1],"PerplexityBot":["allowed","named",0,1],"Perplexity-User":["allowed","named",0,1],"Google-Extended":["allowed","named",0,1],"Applebot-Extended":["partial","*",1,1],"CCBot":["partial","*",1,1],"Bytespider":["blocked","named",1,0],"Amazonbot":["partial","*",1,1],"meta-externalagent":["partial","*",1,1],"cohere-ai":["partial","*",1,1],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",1,1]},"llms":{"status":"missing","http":404,"bytes":28791},"home":{"status":"ok","http":200,"url":"https://www.wionews.com/","html_tokens":81335,"markdown_tokens":20,"pruned_tokens":20}},
{"rank":5965,"domain":"comunidad.madrid","origin":"https://comunidad.madrid","robots":{"status":"missing","http":404,"url":"https://www.comunidad.madrid/robots.txt","bytes":16517,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":16517},"home":{"status":"http_error","http":404,"url":"https://www.comunidad.madrid/"}},
{"rank":5966,"domain":"latrobe.edu.au","origin":"https://latrobe.edu.au","robots":{"status":"refused","http":403,"url":"https://www.latrobe.edu.au/robots.txt","bytes":5577,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":5967,"domain":"dexerto.com","origin":"https://dexerto.com","robots":{"status":"ok","http":200,"url":"https://www.dexerto.com/robots.txt","bytes":654,"groups":2},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["blocked","named",1,0],"OAI-SearchBot":["partial","*",2,1],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",2,1],"Claude-SearchBot":["partial","*",2,1],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["partial","*",2,1],"Perplexity-User":["partial","*",2,1],"Google-Extended":["partial","*",2,1],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["partial","*",2,1],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",2,1]},"llms":{"status":"present","http":200,"bytes":5240},"home":{"status":"ok","http":200,"url":"https://www.dexerto.com/","html_tokens":128518,"markdown_tokens":1853,"pruned_tokens":175}},
{"rank":5968,"domain":"arsenal.com","origin":"https://arsenal.com","robots":{"status":"ok","http":200,"url":"https://www.arsenal.com/robots.txt","bytes":258,"groups":1},"bots":{"GPTBot":["partial","*",5,1],"ChatGPT-User":["partial","*",5,1],"OAI-SearchBot":["partial","*",5,1],"ClaudeBot":["partial","*",5,1],"Claude-User":["partial","*",5,1],"Claude-SearchBot":["partial","*",5,1],"anthropic-ai":["partial","*",5,1],"PerplexityBot":["partial","*",5,1],"Perplexity-User":["partial","*",5,1],"Google-Extended":["partial","*",5,1],"Applebot-Extended":["partial","*",5,1],"CCBot":["partial","*",5,1],"Bytespider":["partial","*",5,1],"Amazonbot":["partial","*",5,1],"meta-externalagent":["partial","*",5,1],"cohere-ai":["partial","*",5,1],"Diffbot":["partial","*",5,1],"ContextFetchBot":["partial","*",5,1]},"llms":{"status":"http_error","http":503,"bytes":282},"home":{"status":"ok","http":200,"url":"https://www.arsenal.com/","html_tokens":29086,"markdown_tokens":2,"pruned_tokens":2}},
{"rank":5969,"domain":"skincancer.org","origin":"https://skincancer.org","robots":{"status":"ok","http":200,"url":"https://skincancer.org/robots.txt","bytes":220,"groups":2},"bots":{"GPTBot":["partial","*",2,0],"ChatGPT-User":["partial","*",2,0],"OAI-SearchBot":["partial","*",2,0],"ClaudeBot":["partial","*",2,0],"Claude-User":["partial","*",2,0],"Claude-SearchBot":["partial","*",2,0],"anthropic-ai":["partial","*",2,0],"PerplexityBot":["partial","*",2,0],"Perplexity-User":["partial","*",2,0],"Google-Extended":["partial","*",2,0],"Applebot-Extended":["partial","*",2,0],"CCBot":["partial","*",2,0],"Bytespider":["partial","*",2,0],"Amazonbot":["partial","*",2,0],"meta-externalagent":["partial","*",2,0],"cohere-ai":["partial","*",2,0],"Diffbot":["partial","*",2,0],"ContextFetchBot":["partial","*",2,0]},"llms":{"status":"http_error","http":403,"bytes":5780},"home":{"status":"ok","http":200,"url":"https://www.skincancer.org/","html_tokens":60433,"markdown_tokens":156,"pruned_tokens":156}},
{"rank":5970,"domain":"kyoto.jp","origin":"https://kyoto.jp","robots":{"status":"unreachable","http":null,"url":"https://kyoto.jp/robots.txt","bytes":0,"groups":0,"error":"cannot resolve kyoto.jp"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":5971,"domain":"husqvarna.com","origin":"https://husqvarna.com","robots":{"status":"ok","http":200,"url":"https://www.husqvarna.com/robots.txt","bytes":196,"groups":1},"bots":{"GPTBot":["allowed","*",0,1],"ChatGPT-User":["allowed","*",0,1],"OAI-SearchBot":["allowed","*",0,1],"ClaudeBot":["allowed","*",0,1],"Claude-User":["allowed","*",0,1],"Claude-SearchBot":["allowed","*",0,1],"anthropic-ai":["allowed","*",0,1],"PerplexityBot":["allowed","*",0,1],"Perplexity-User":["allowed","*",0,1],"Google-Extended":["allowed","*",0,1],"Applebot-Extended":["allowed","*",0,1],"CCBot":["allowed","*",0,1],"Bytespider":["allowed","*",0,1],"Amazonbot":["allowed","*",0,1],"meta-externalagent":["allowed","*",0,1],"cohere-ai":["allowed","*",0,1],"Diffbot":["allowed","*",0,1],"ContextFetchBot":["allowed","*",0,1]},"llms":{"status":"present","http":200,"bytes":134},"home":{"status":"ok","http":200,"url":"https://www.husqvarna.com/","html_tokens":13758,"markdown_tokens":43,"pruned_tokens":43}},
{"rank":5972,"domain":"tuoitre.vn","origin":"https://tuoitre.vn","robots":{"status":"ok","http":200,"url":"https://tuoitre.vn/robots.txt","bytes":374,"groups":1},"bots":{"GPTBot":["partial","*",8,1],"ChatGPT-User":["partial","*",8,1],"OAI-SearchBot":["partial","*",8,1],"ClaudeBot":["partial","*",8,1],"Claude-User":["partial","*",8,1],"Claude-SearchBot":["partial","*",8,1],"anthropic-ai":["partial","*",8,1],"PerplexityBot":["partial","*",8,1],"Perplexity-User":["partial","*",8,1],"Google-Extended":["partial","*",8,1],"Applebot-Extended":["partial","*",8,1],"CCBot":["partial","*",8,1],"Bytespider":["partial","*",8,1],"Amazonbot":["partial","*",8,1],"meta-externalagent":["partial","*",8,1],"cohere-ai":["partial","*",8,1],"Diffbot":["partial","*",8,1],"ContextFetchBot":["partial","*",8,1]},"llms":{"status":"present","http":200,"bytes":3009},"home":{"status":"ok","http":200,"url":"https://tuoitre.vn/","html_tokens":68615,"markdown_tokens":155,"pruned_tokens":155}},
{"rank":5973,"domain":"nv.ua","origin":"https://nv.ua","robots":{"status":"ok","http":200,"url":"https://nv.ua/robots.txt","bytes":1527,"groups":1},"bots":{"GPTBot":["partial","*",45,0],"ChatGPT-User":["partial","*",45,0],"OAI-SearchBot":["partial","*",45,0],"ClaudeBot":["partial","*",45,0],"Claude-User":["partial","*",45,0],"Claude-SearchBot":["partial","*",45,0],"anthropic-ai":["partial","*",45,0],"PerplexityBot":["partial","*",45,0],"Perplexity-User":["partial","*",45,0],"Google-Extended":["partial","*",45,0],"Applebot-Extended":["partial","*",45,0],"CCBot":["partial","*",45,0],"Bytespider":["partial","*",45,0],"Amazonbot":["partial","*",45,0],"meta-externalagent":["partial","*",45,0],"cohere-ai":["partial","*",45,0],"Diffbot":["partial","*",45,0],"ContextFetchBot":["partial","*",45,0]},"llms":{"status":"other_text","http":200,"bytes":2157},"home":{"status":"http_error","http":403,"url":"https://nv.ua/"}},
{"rank":5974,"domain":"rxlist.com","origin":"https://rxlist.com","robots":{"status":"ok","http":200,"url":"https://www.rxlist.com/robots.txt","bytes":74885,"groups":8},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["partial","*",2,0],"OAI-SearchBot":["partial","*",2,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",2,0],"Claude-SearchBot":["partial","*",2,0],"anthropic-ai":["partial","*",2,0],"PerplexityBot":["partial","*",2,0],"Perplexity-User":["partial","*",2,0],"Google-Extended":["partial","*",2,0],"Applebot-Extended":["partial","*",2,0],"CCBot":["blocked","named",1,0],"Bytespider":["partial","*",2,0],"Amazonbot":["partial","*",2,0],"meta-externalagent":["partial","*",2,0],"cohere-ai":["partial","*",2,0],"Diffbot":["partial","*",2,0],"ContextFetchBot":["partial","*",2,0]},"llms":{"status":"missing","http":404,"bytes":143005},"home":{"status":"ok","http":200,"url":"https://www.rxlist.com/","html_tokens":39747,"markdown_tokens":97,"pruned_tokens":97}},
{"rank":5975,"domain":"riken.jp","origin":"https://www.riken.jp","robots":{"status":"missing","http":404,"url":"https://www.riken.jp/robots.txt","bytes":13619,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":13619},"home":{"status":"ok","http":200,"url":"https://www.riken.jp/","html_tokens":5230,"markdown_tokens":30,"pruned_tokens":30}},
{"rank":5976,"domain":"martindale.com","origin":"https://martindale.com","robots":{"status":"ok","http":200,"url":"https://www.martindale.com/robots.txt","bytes":933,"groups":4},"bots":{"GPTBot":["blocked","named",2,0],"ChatGPT-User":["partial","*",11,0],"OAI-SearchBot":["partial","*",11,0],"ClaudeBot":["partial","*",11,0],"Claude-User":["partial","*",11,0],"Claude-SearchBot":["partial","*",11,0],"anthropic-ai":["partial","*",11,0],"PerplexityBot":["partial","*",11,0],"Perplexity-User":["partial","*",11,0],"Google-Extended":["partial","*",11,0],"Applebot-Extended":["partial","*",11,0],"CCBot":["partial","*",11,0],"Bytespider":["partial","*",11,0],"Amazonbot":["partial","*",11,0],"meta-externalagent":["partial","*",11,0],"cohere-ai":["partial","*",11,0],"Diffbot":["partial","*",11,0],"ContextFetchBot":["partial","*",11,0]},"llms":{"status":"missing","http":404,"bytes":300512},"home":{"status":"ok","http":200,"url":"https://www.martindale.com/","html_tokens":100699,"markdown_tokens":54,"pruned_tokens":54}},
{"rank":5977,"domain":"oulu.fi","origin":"https://oulu.fi","robots":{"status":"ok","http":200,"url":"https://www.oulu.fi/robots.txt","bytes":1858,"groups":1},"bots":{"GPTBot":["partial","*",26,18],"ChatGPT-User":["partial","*",26,18],"OAI-SearchBot":["partial","*",26,18],"ClaudeBot":["partial","*",26,18],"Claude-User":["partial","*",26,18],"Claude-SearchBot":["partial","*",26,18],"anthropic-ai":["partial","*",26,18],"PerplexityBot":["partial","*",26,18],"Perplexity-User":["partial","*",26,18],"Google-Extended":["partial","*",26,18],"Applebot-Extended":["partial","*",26,18],"CCBot":["partial","*",26,18],"Bytespider":["partial","*",26,18],"Amazonbot":["partial","*",26,18],"meta-externalagent":["partial","*",26,18],"cohere-ai":["partial","*",26,18],"Diffbot":["partial","*",26,18],"ContextFetchBot":["partial","*",26,18]},"llms":{"status":"present","http":200,"bytes":14760},"home":{"status":"ok","http":200,"url":"https://www.oulu.fi/fi","html_tokens":23439,"markdown_tokens":203,"pruned_tokens":162}},
{"rank":5978,"domain":"thefederalist.com","origin":"https://thefederalist.com","robots":{"status":"ok","http":200,"url":"https://thefederalist.com/robots.txt","bytes":140,"groups":1},"bots":{"GPTBot":["partial","*",3,2],"ChatGPT-User":["partial","*",3,2],"OAI-SearchBot":["partial","*",3,2],"ClaudeBot":["partial","*",3,2],"Claude-User":["partial","*",3,2],"Claude-SearchBot":["partial","*",3,2],"anthropic-ai":["partial","*",3,2],"PerplexityBot":["partial","*",3,2],"Perplexity-User":["partial","*",3,2],"Google-Extended":["partial","*",3,2],"Applebot-Extended":["partial","*",3,2],"CCBot":["partial","*",3,2],"Bytespider":["partial","*",3,2],"Amazonbot":["partial","*",3,2],"meta-externalagent":["partial","*",3,2],"cohere-ai":["partial","*",3,2],"Diffbot":["partial","*",3,2],"ContextFetchBot":["partial","*",3,2]},"llms":{"status":"http_error","http":403,"bytes":5570},"home":{"status":"http_error","http":403,"url":"https://thefederalist.com/"}},
{"rank":5979,"domain":"orlilab.com","origin":"https://orlilab.com","robots":{"status":"ok","http":200,"url":"https://orlilab.com/robots.txt","bytes":143,"groups":1},"bots":{"GPTBot":["partial","*",5,0],"ChatGPT-User":["partial","*",5,0],"OAI-SearchBot":["partial","*",5,0],"ClaudeBot":["partial","*",5,0],"Claude-User":["partial","*",5,0],"Claude-SearchBot":["partial","*",5,0],"anthropic-ai":["partial","*",5,0],"PerplexityBot":["partial","*",5,0],"Perplexity-User":["partial","*",5,0],"Google-Extended":["partial","*",5,0],"Applebot-Extended":["partial","*",5,0],"CCBot":["partial","*",5,0],"Bytespider":["partial","*",5,0],"Amazonbot":["partial","*",5,0],"meta-externalagent":["partial","*",5,0],"cohere-ai":["partial","*",5,0],"Diffbot":["partial","*",5,0],"ContextFetchBot":["partial","*",5,0]},"llms":{"status":"missing","http":404,"bytes":640},"home":{"status":"ok","http":200,"url":"https://orlilab.com/","html_tokens":14254,"markdown_tokens":4345,"pruned_tokens":155}},
{"rank":5980,"domain":"mtsu.edu","origin":"https://mtsu.edu","robots":{"status":"ok","http":200,"url":"https://www.mtsu.edu/robots.txt","bytes":350,"groups":1},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"http_error","http":403,"bytes":5539},"home":{"status":"http_error","http":403,"url":"https://mtsu.edu/"}},
{"rank":5981,"domain":"noscript.net","origin":"https://noscript.net","robots":{"status":"missing","http":404,"url":"https://noscript.net/robots.txt","bytes":1,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":1},"home":{"status":"ok","http":200,"url":"https://noscript.net/","html_tokens":1742,"markdown_tokens":384,"pruned_tokens":295}},
{"rank":5982,"domain":"pressbooks.pub","origin":"https://pressbooks.pub","robots":{"status":"ok","http":200,"url":"https://pressbooks.pub/robots.txt","bytes":347,"groups":1},"bots":{"GPTBot":["partial","*",11,1],"ChatGPT-User":["partial","*",11,1],"OAI-SearchBot":["partial","*",11,1],"ClaudeBot":["partial","*",11,1],"Claude-User":["partial","*",11,1],"Claude-SearchBot":["partial","*",11,1],"anthropic-ai":["partial","*",11,1],"PerplexityBot":["partial","*",11,1],"Perplexity-User":["partial","*",11,1],"Google-Extended":["partial","*",11,1],"Applebot-Extended":["partial","*",11,1],"CCBot":["partial","*",11,1],"Bytespider":["partial","*",11,1],"Amazonbot":["partial","*",11,1],"meta-externalagent":["partial","*",11,1],"cohere-ai":["partial","*",11,1],"Diffbot":["partial","*",11,1],"ContextFetchBot":["partial","*",11,1]},"llms":{"status":"missing","http":404,"bytes":42378},"home":{"status":"http_error","http":403,"url":"https://pressbooks.pub/"}},
{"rank":5983,"domain":"uni-erlangen.de","origin":"https://uni-erlangen.de","robots":{"status":"ok","http":200,"url":"https://www.fau.de/robots.txt","bytes":108,"groups":1},"bots":{"GPTBot":["partial","*",1,1],"ChatGPT-User":["partial","*",1,1],"OAI-SearchBot":["partial","*",1,1],"ClaudeBot":["partial","*",1,1],"Claude-User":["partial","*",1,1],"Claude-SearchBot":["partial","*",1,1],"anthropic-ai":["partial","*",1,1],"PerplexityBot":["partial","*",1,1],"Perplexity-User":["partial","*",1,1],"Google-Extended":["partial","*",1,1],"Applebot-Extended":["partial","*",1,1],"CCBot":["partial","*",1,1],"Bytespider":["partial","*",1,1],"Amazonbot":["partial","*",1,1],"meta-externalagent":["partial","*",1,1],"cohere-ai":["partial","*",1,1],"Diffbot":["partial","*",1,1],"ContextFetchBot":["partial","*",1,1]},"llms":{"status":"missing","http":404,"bytes":404669},"home":{"status":"ok","http":200,"url":"https://www.fau.de/","html_tokens":112699,"markdown_tokens":122,"pruned_tokens":66}},
{"rank":5984,"domain":"denofgeek.com","origin":"https://denofgeek.com","robots":{"status":"ok","http":200,"url":"https://www.denofgeek.com/robots.txt","bytes":287,"groups":2},"bots":{"GPTBot":["partial","*",3,1],"ChatGPT-User":["partial","*",3,1],"OAI-SearchBot":["partial","*",3,1],"ClaudeBot":["partial","*",3,1],"Claude-User":["partial","*",3,1],"Claude-SearchBot":["partial","*",3,1],"anthropic-ai":["partial","*",3,1],"PerplexityBot":["partial","*",3,1],"Perplexity-User":["partial","*",3,1],"Google-Extended":["partial","*",3,1],"Applebot-Extended":["partial","*",3,1],"CCBot":["partial","*",3,1],"Bytespider":["partial","*",3,1],"Amazonbot":["partial","*",3,1],"meta-externalagent":["partial","*",3,1],"cohere-ai":["partial","*",3,1],"Diffbot":["partial","*",3,1],"ContextFetchBot":["partial","*",3,1]},"llms":{"status":"missing","http":404,"bytes":178279},"home":{"status":"ok","http":200,"url":"https://www.denofgeek.com/","html_tokens":59181,"markdown_tokens":599,"pruned_tokens":172}},
{"rank":5985,"domain":"essence.com","origin":"https://essence.com","robots":{"status":"ok","http":200,"url":"https://www.essence.com/robots.txt","bytes":1369,"groups":1},"bots":{"GPTBot":["partial","*",23,4],"ChatGPT-User":["partial","*",23,4],"OAI-SearchBot":["partial","*",23,4],"ClaudeBot":["partial","*",23,4],"Claude-User":["partial","*",23,4],"Claude-SearchBot":["partial","*",23,4],"anthropic-ai":["partial","*",23,4],"PerplexityBot":["partial","*",23,4],"Perplexity-User":["partial","*",23,4],"Google-Extended":["partial","*",23,4],"Applebot-Extended":["partial","*",23,4],"CCBot":["partial","*",23,4],"Bytespider":["partial","*",23,4],"Amazonbot":["partial","*",23,4],"meta-externalagent":["partial","*",23,4],"cohere-ai":["partial","*",23,4],"Diffbot":["partial","*",23,4],"ContextFetchBot":["partial","*",23,4]},"llms":{"status":"http_error","http":403,"bytes":4905},"home":{"status":"http_error","http":403,"url":"https://www.essence.com/"}},
{"rank":5986,"domain":"ncoa.org","origin":"https://ncoa.org","robots":{"status":"ok","http":200,"url":"https://www.ncoa.org/robots.txt","bytes":72,"groups":1},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"missing","http":404,"bytes":30086},"home":{"status":"ok","http":200,"url":"https://www.ncoa.org/","html_tokens":34845,"markdown_tokens":690,"pruned_tokens":302}},
{"rank":5987,"domain":"aleteia.org","origin":"https://aleteia.org","robots":{"status":"ok","http":200,"url":"https://aleteia.org/robots.txt","bytes":673,"groups":7},"bots":{"GPTBot":["partial","*",10,0],"ChatGPT-User":["partial","*",10,0],"OAI-SearchBot":["partial","*",10,0],"ClaudeBot":["partial","*",10,0],"Claude-User":["partial","*",10,0],"Claude-SearchBot":["partial","*",10,0],"anthropic-ai":["partial","*",10,0],"PerplexityBot":["partial","*",10,0],"Perplexity-User":["partial","*",10,0],"Google-Extended":["partial","*",10,0],"Applebot-Extended":["partial","*",10,0],"CCBot":["partial","*",10,0],"Bytespider":["partial","*",10,0],"Amazonbot":["partial","*",10,0],"meta-externalagent":["partial","*",10,0],"cohere-ai":["partial","*",10,0],"Diffbot":["partial","*",10,0],"ContextFetchBot":["partial","*",10,0]},"llms":{"status":"missing","http":404,"bytes":173474},"home":{"status":"ok","http":200,"url":"https://aleteia.org/","html_tokens":138915,"markdown_tokens":53,"pruned_tokens":53}},
{"rank":5988,"domain":"charite.de","origin":"https://charite.de","robots":{"status":"ok","http":200,"url":"https://www.charite.de/robots.txt","bytes":2311,"groups":1},"bots":{"GPTBot":["partial","*",73,12],"ChatGPT-User":["partial","*",73,12],"OAI-SearchBot":["partial","*",73,12],"ClaudeBot":["partial","*",73,12],"Claude-User":["partial","*",73,12],"Claude-SearchBot":["partial","*",73,12],"anthropic-ai":["partial","*",73,12],"PerplexityBot":["partial","*",73,12],"Perplexity-User":["partial","*",73,12],"Google-Extended":["partial","*",73,12],"Applebot-Extended":["partial","*",73,12],"CCBot":["partial","*",73,12],"Bytespider":["partial","*",73,12],"Amazonbot":["partial","*",73,12],"meta-externalagent":["partial","*",73,12],"cohere-ai":["partial","*",73,12],"Diffbot":["partial","*",73,12],"ContextFetchBot":["partial","*",73,12]},"llms":{"status":"missing","http":404,"bytes":31768},"home":{"status":"ok","http":200,"url":"https://www.charite.de/","html_tokens":16906,"markdown_tokens":256,"pruned_tokens":153}},
{"rank":5989,"domain":"nascar.com","origin":"https://nascar.com","robots":{"status":"refused","http":403,"url":"https://www.nascar.com/robots.txt","bytes":4904,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":5990,"domain":"ecma-international.org","origin":"https://ecma-international.org","robots":{"status":"ok","http":200,"url":"https://ecma-international.org/robots.txt","bytes":30,"groups":1},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"missing","http":404,"bytes":78785},"home":{"status":"ok","http":200,"url":"https://ecma-international.org/","html_tokens":22048,"markdown_tokens":94,"pruned_tokens":94}},
{"rank":5991,"domain":"roll20.net","origin":"https://roll20.net","robots":{"status":"ok","http":200,"url":"https://roll20.net/robots.txt","bytes":546,"groups":2},"bots":{"GPTBot":["partial","*",16,0],"ChatGPT-User":["partial","*",16,0],"OAI-SearchBot":["partial","*",16,0],"ClaudeBot":["partial","*",16,0],"Claude-User":["partial","*",16,0],"Claude-SearchBot":["partial","*",16,0],"anthropic-ai":["partial","*",16,0],"PerplexityBot":["partial","*",16,0],"Perplexity-User":["partial","*",16,0],"Google-Extended":["partial","*",16,0],"Applebot-Extended":["partial","*",16,0],"CCBot":["partial","*",16,0],"Bytespider":["partial","*",16,0],"Amazonbot":["partial","*",16,0],"meta-externalagent":["partial","*",16,0],"cohere-ai":["partial","*",16,0],"Diffbot":["partial","*",16,0],"ContextFetchBot":["partial","*",16,0]},"llms":{"status":"missing","http":404,"bytes":44291},"home":{"status":"ok","http":200,"url":"https://roll20.net/","html_tokens":18917,"markdown_tokens":489,"pruned_tokens":151}},
{"rank":5992,"domain":"xinhuaxmt.com","origin":"https://www.xinhuaxmt.com","robots":{"status":"missing","http":404,"url":"https://www.xinhuaxmt.com/robots.txt","bytes":247,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":245},"home":{"status":"ok","http":200,"url":"https://www.xinhuaxmt.com/","html_tokens":227,"markdown_tokens":27,"pruned_tokens":27}},
{"rank":5993,"domain":"renesas.com","origin":"https://renesas.com","robots":{"status":"ok","http":200,"url":"https://www.renesas.com/robots.txt","bytes":3157,"groups":14},"bots":{"GPTBot":["allowed","named",0,1],"ChatGPT-User":["allowed","named",0,1],"OAI-SearchBot":["allowed","named",0,1],"ClaudeBot":["allowed","named",0,1],"Claude-User":["allowed","named",0,1],"Claude-SearchBot":["allowed","named",0,1],"anthropic-ai":["allowed","named",0,1],"PerplexityBot":["allowed","named",0,1],"Perplexity-User":["allowed","named",0,1],"Google-Extended":["allowed","named",0,1],"Applebot-Extended":["allowed","named",0,1],"CCBot":["allowed","named",0,1],"Bytespider":["blocked","named",1,0],"Amazonbot":["partial","*",40,18],"meta-externalagent":["partial","*",40,18],"cohere-ai":["partial","*",40,18],"Diffbot":["partial","*",40,18],"ContextFetchBot":["partial","*",40,18]},"llms":{"status":"present","http":200,"bytes":6778},"home":{"status":"ok","http":200,"url":"https://www.renesas.com/en","html_tokens":62535,"markdown_tokens":91,"pruned_tokens":54}},
{"rank":5994,"domain":"awstrack.me","origin":"https://awstrack.me","robots":{"status":"unreachable","http":null,"url":"https://awstrack.me/robots.txt","bytes":0,"groups":0,"error":"cannot resolve awstrack.me"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":5995,"domain":"pixar.com","origin":"https://pixar.com","robots":{"status":"ok","http":200,"url":"https://pixar.com/robots.txt","bytes":1504,"groups":1},"bots":{"GPTBot":["partial","named",27,1],"ChatGPT-User":["partial","*",27,1],"OAI-SearchBot":["partial","*",27,1],"ClaudeBot":["partial","named",27,1],"Claude-User":["partial","*",27,1],"Claude-SearchBot":["partial","*",27,1],"anthropic-ai":["partial","named",27,1],"PerplexityBot":["partial","*",27,1],"Perplexity-User":["partial","*",27,1],"Google-Extended":["partial","named",27,1],"Applebot-Extended":["partial","named",27,1],"CCBot":["partial","named",27,1],"Bytespider":["partial","named",27,1],"Amazonbot":["partial","named",27,1],"meta-externalagent":["partial","named",27,1],"cohere-ai":["partial","named",27,1],"Diffbot":["partial","*",27,1],"ContextFetchBot":["partial","*",27,1]},"llms":{"status":"missing","http":404,"bytes":0},"home":{"status":"ok","http":200,"url":"https://www.pixar.com/","html_tokens":129724,"markdown_tokens":42,"pruned_tokens":42}},
{"rank":5996,"domain":"komonews.com","origin":"https://komonews.com","robots":{"status":"ok","http":200,"url":"https://komonews.com/robots.txt","bytes":565,"groups":12},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["blocked","named",1,0],"OAI-SearchBot":["partial","*",1,1],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",1,1],"Claude-SearchBot":["partial","*",1,1],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["partial","*",1,1],"Perplexity-User":["partial","*",1,1],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["partial","*",1,1],"Amazonbot":["partial","*",1,1],"meta-externalagent":["partial","*",1,1],"cohere-ai":["blocked","named",1,0],"Diffbot":["partial","*",1,1],"ContextFetchBot":["partial","*",1,1]},"llms":{"status":"http_error","http":400,"bytes":45},"home":{"status":"ok","http":200,"url":"https://komonews.com/","html_tokens":132945,"markdown_tokens":62,"pruned_tokens":62}},
{"rank":5997,"domain":"dothome.co.kr","origin":"https://dothome.co.kr","robots":{"status":"ok","http":200,"url":"https://www.dothome.co.kr/robots.txt","bytes":239,"groups":1},"bots":{"GPTBot":["partial","*",7,1],"ChatGPT-User":["partial","*",7,1],"OAI-SearchBot":["partial","*",7,1],"ClaudeBot":["partial","*",7,1],"Claude-User":["partial","*",7,1],"Claude-SearchBot":["partial","*",7,1],"anthropic-ai":["partial","*",7,1],"PerplexityBot":["partial","*",7,1],"Perplexity-User":["partial","*",7,1],"Google-Extended":["partial","*",7,1],"Applebot-Extended":["partial","*",7,1],"CCBot":["partial","*",7,1],"Bytespider":["partial","*",7,1],"Amazonbot":["partial","*",7,1],"meta-externalagent":["partial","*",7,1],"cohere-ai":["partial","*",7,1],"Diffbot":["partial","*",7,1],"ContextFetchBot":["partial","*",7,1]},"llms":{"status":"disallowed","error":"robots.txt of www.dothome.co.kr disallows /expirationinfo/404.php"},"home":{"status":"ok","http":200,"url":"https://www.dothome.co.kr/","html_tokens":24621,"markdown_tokens":138,"pruned_tokens":138}},
{"rank":5998,"domain":"nyecasino.eu","origin":"https://nyecasino.eu","robots":{"status":"ok","http":200,"url":"https://nyecasino.eu/robots.txt","bytes":529,"groups":1},"bots":{"GPTBot":["partial","*",19,0],"ChatGPT-User":["partial","*",19,0],"OAI-SearchBot":["partial","*",19,0],"ClaudeBot":["partial","*",19,0],"Claude-User":["partial","*",19,0],"Claude-SearchBot":["partial","*",19,0],"anthropic-ai":["partial","*",19,0],"PerplexityBot":["partial","*",19,0],"Perplexity-User":["partial","*",19,0],"Google-Extended":["partial","*",19,0],"Applebot-Extended":["partial","*",19,0],"CCBot":["partial","*",19,0],"Bytespider":["partial","*",19,0],"Amazonbot":["partial","*",19,0],"meta-externalagent":["partial","*",19,0],"cohere-ai":["partial","*",19,0],"Diffbot":["partial","*",19,0],"ContextFetchBot":["partial","*",19,0]},"llms":{"status":"missing","http":404,"bytes":16200},"home":{"status":"ok","http":200,"url":"https://nyecasino.eu/","html_tokens":39828,"markdown_tokens":2865,"pruned_tokens":163}},
{"rank":5999,"domain":"revues.org","origin":"https://revues.org","robots":{"status":"html","http":200,"url":"https://journals.openedition.org/","bytes":104045,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":6000,"domain":"outbrain.com","origin":"https://outbrain.com","robots":{"status":"ok","http":200,"url":"https://www.outbrain.com/robots.txt","bytes":2736,"groups":8},"bots":{"GPTBot":["partial","*",37,1],"ChatGPT-User":["partial","*",37,1],"OAI-SearchBot":["partial","*",37,1],"ClaudeBot":["partial","*",37,1],"Claude-User":["partial","*",37,1],"Claude-SearchBot":["partial","*",37,1],"anthropic-ai":["partial","*",37,1],"PerplexityBot":["partial","*",37,1],"Perplexity-User":["partial","*",37,1],"Google-Extended":["partial","*",37,1],"Applebot-Extended":["partial","*",37,1],"CCBot":["partial","*",37,1],"Bytespider":["partial","*",37,1],"Amazonbot":["partial","*",37,1],"meta-externalagent":["partial","*",37,1],"cohere-ai":["partial","*",37,1],"Diffbot":["partial","*",37,1],"ContextFetchBot":["partial","*",37,1]},"llms":{"status":"http_error","http":406,"bytes":0},"home":{"status":"ok","http":200,"url":"https://www.outbrain.com/","html_tokens":6380,"markdown_tokens":167,"pruned_tokens":123}},
{"rank":6001,"domain":"worldhappiness.report","origin":"https://worldhappiness.report","robots":{"status":"ok","http":200,"url":"https://www.worldhappiness.report/robots.txt","bytes":75,"groups":1},"bots":{"GPTBot":["allowed","*",0,1],"ChatGPT-User":["allowed","*",0,1],"OAI-SearchBot":["allowed","*",0,1],"ClaudeBot":["allowed","*",0,1],"Claude-User":["allowed","*",0,1],"Claude-SearchBot":["allowed","*",0,1],"anthropic-ai":["allowed","*",0,1],"PerplexityBot":["allowed","*",0,1],"Perplexity-User":["allowed","*",0,1],"Google-Extended":["allowed","*",0,1],"Applebot-Extended":["allowed","*",0,1],"CCBot":["allowed","*",0,1],"Bytespider":["allowed","*",0,1],"Amazonbot":["allowed","*",0,1],"meta-externalagent":["allowed","*",0,1],"cohere-ai":["allowed","*",0,1],"Diffbot":["allowed","*",0,1],"ContextFetchBot":["allowed","*",0,1]},"llms":{"status":"missing","http":404,"bytes":29646},"home":{"status":"ok","http":200,"url":"https://www.worldhappiness.report/","html_tokens":16437,"markdown_tokens":187,"pruned_tokens":65}},
{"rank":6002,"domain":"bloglines.com","origin":"https://bloglines.com","robots":{"status":"missing","http":404,"url":"https://www.bloglines.com/robots.txt","bytes":2353,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":2353},"home":{"status":"ok","http":200,"url":"https://www.bloglines.com/","html_tokens":837,"markdown_tokens":159,"pruned_tokens":64}},
{"rank":6003,"domain":"salvationarmyusa.org","origin":"https://salvationarmyusa.org","robots":{"status":"ok","http":200,"url":"https://www.salvationarmyusa.org/robots.txt","bytes":473,"groups":1},"bots":{"GPTBot":["allowed","*",0,1],"ChatGPT-User":["allowed","*",0,1],"OAI-SearchBot":["allowed","*",0,1],"ClaudeBot":["allowed","*",0,1],"Claude-User":["allowed","*",0,1],"Claude-SearchBot":["allowed","*",0,1],"anthropic-ai":["allowed","*",0,1],"PerplexityBot":["allowed","*",0,1],"Perplexity-User":["allowed","*",0,1],"Google-Extended":["allowed","*",0,1],"Applebot-Extended":["allowed","*",0,1],"CCBot":["allowed","*",0,1],"Bytespider":["allowed","*",0,1],"Amazonbot":["allowed","*",0,1],"meta-externalagent":["allowed","*",0,1],"cohere-ai":["allowed","*",0,1],"Diffbot":["allowed","*",0,1],"ContextFetchBot":["allowed","*",0,1]},"llms":{"status":"missing","http":404,"bytes":388387},"home":{"status":"ok","http":200,"url":"https://www.salvationarmyusa.org/","html_tokens":149637,"markdown_tokens":112,"pruned_tokens":112}},
{"rank":6004,"domain":"invite.viber.com","origin":"https://invite.viber.com","robots":{"status":"html","http":200,"url":"https://invite.viber.com/robots.txt","bytes":49761,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":6005,"domain":"gob.pa","origin":"https://gob.pa","robots":{"status":"unreachable","http":null,"url":"https://gob.pa/robots.txt","bytes":0,"groups":0,"error":"cannot resolve gob.pa"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":6006,"domain":"vidyard.com","origin":"https://vidyard.com","robots":{"status":"ok","http":200,"url":"https://vidyard.com/robots.txt","bytes":899,"groups":4},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["allowed","named",0,1],"OAI-SearchBot":["partial","*",1,1],"ClaudeBot":["allowed","named",0,1],"Claude-User":["allowed","named",0,1],"Claude-SearchBot":["partial","*",1,1],"anthropic-ai":["partial","*",1,1],"PerplexityBot":["allowed","named",0,1],"Perplexity-User":["partial","*",1,1],"Google-Extended":["allowed","named",0,1],"Applebot-Extended":["partial","*",1,1],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["partial","*",1,1],"meta-externalagent":["partial","*",1,1],"cohere-ai":["partial","*",1,1],"Diffbot":["partial","*",1,1],"ContextFetchBot":["partial","*",1,1]},"llms":{"status":"http_error","http":403,"bytes":5542},"home":{"status":"http_error","http":403,"url":"https://vidyard.com/"}},
{"rank":6007,"domain":"nextgov.com","origin":"https://nextgov.com","robots":{"status":"ok","http":200,"url":"https://www.nextgov.com/robots.txt","bytes":204,"groups":3},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["partial","*",5,0],"OAI-SearchBot":["partial","*",5,0],"ClaudeBot":["partial","*",5,0],"Claude-User":["partial","*",5,0],"Claude-SearchBot":["partial","*",5,0],"anthropic-ai":["partial","*",5,0],"PerplexityBot":["partial","*",5,0],"Perplexity-User":["partial","*",5,0],"Google-Extended":["partial","*",5,0],"Applebot-Extended":["partial","*",5,0],"CCBot":["partial","*",5,0],"Bytespider":["partial","*",5,0],"Amazonbot":["partial","*",5,0],"meta-externalagent":["partial","*",5,0],"cohere-ai":["partial","*",5,0],"Diffbot":["partial","*",5,0],"ContextFetchBot":["partial","*",5,0]},"llms":{"status":"missing","http":404,"bytes":153},"home":{"status":"ok","http":200,"url":"https://www.nextgov.com/","html_tokens":45291,"markdown_tokens":494,"pruned_tokens":150}},
{"rank":6008,"domain":"jguru.com","origin":"https://jguru.com","robots":{"status":"ok","http":200,"url":"https://jguru.com/robots.txt","bytes":4725,"groups":3},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":106880},"home":{"status":"ok","http":200,"url":"https://www.jguru.com/","html_tokens":57969,"markdown_tokens":42,"pruned_tokens":42}},
{"rank":6009,"domain":"wwf.org.uk","origin":"https://wwf.org.uk","robots":{"status":"ok","http":200,"url":"https://www.wwf.org.uk/robots.txt","bytes":1073,"groups":1},"bots":{"GPTBot":["partial","*",5,18],"ChatGPT-User":["partial","*",5,18],"OAI-SearchBot":["partial","*",5,18],"ClaudeBot":["partial","*",5,18],"Claude-User":["partial","*",5,18],"Claude-SearchBot":["partial","*",5,18],"anthropic-ai":["partial","*",5,18],"PerplexityBot":["partial","*",5,18],"Perplexity-User":["partial","*",5,18],"Google-Extended":["partial","*",5,18],"Applebot-Extended":["partial","*",5,18],"CCBot":["partial","*",5,18],"Bytespider":["partial","*",5,18],"Amazonbot":["partial","*",5,18],"meta-externalagent":["partial","*",5,18],"cohere-ai":["partial","*",5,18],"Diffbot":["partial","*",5,18],"ContextFetchBot":["partial","*",5,18]},"llms":{"status":"missing","http":404,"bytes":188},"home":{"status":"ok","http":200,"url":"https://www.wwf.org.uk/","html_tokens":19396,"markdown_tokens":628,"pruned_tokens":149}},
{"rank":6010,"domain":"privacymark.jp","origin":"https://privacymark.jp","robots":{"status":"missing","http":404,"url":"https://privacymark.jp/robots.txt","bytes":52896,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":52896},"home":{"status":"ok","http":200,"url":"https://privacymark.jp/","html_tokens":17554,"markdown_tokens":60,"pruned_tokens":60}},
{"rank":6011,"domain":"bravejournal.net","origin":"https://bravejournal.net","robots":{"status":"ok","http":200,"url":"https://bravejournal.net/robots.txt","bytes":1248,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"http_error","http":530,"bytes":7637},"home":{"status":"http_error","http":530,"url":"https://bravejournal.net/"}},
{"rank":6012,"domain":"asiaone.com","origin":"https://asiaone.com","robots":{"status":"ok","http":200,"url":"https://asiaone.com/robots.txt","bytes":25,"groups":1},"bots":{"GPTBot":["blocked","*",1,0],"ChatGPT-User":["blocked","*",1,0],"OAI-SearchBot":["blocked","*",1,0],"ClaudeBot":["blocked","*",1,0],"Claude-User":["blocked","*",1,0],"Claude-SearchBot":["blocked","*",1,0],"anthropic-ai":["blocked","*",1,0],"PerplexityBot":["blocked","*",1,0],"Perplexity-User":["blocked","*",1,0],"Google-Extended":["blocked","*",1,0],"Applebot-Extended":["blocked","*",1,0],"CCBot":["blocked","*",1,0],"Bytespider":["blocked","*",1,0],"Amazonbot":["blocked","*",1,0],"meta-externalagent":["blocked","*",1,0],"cohere-ai":["blocked","*",1,0],"Diffbot":["blocked","*",1,0],"ContextFetchBot":["blocked","*",1,0]},"llms":{"status":"disallowed"},"home":{"status":"disallowed"}},
{"rank":6013,"domain":"wpcomstaging.com","origin":"https://wpcomstaging.com","robots":{"status":"unreachable","http":null,"url":"https://wpcomstaging.com/robots.txt","bytes":0,"groups":0,"error":"cannot resolve wpcomstaging.com"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":6014,"domain":"lalibre.be","origin":"https://lalibre.be","robots":{"status":"ok","http":200,"url":"https://www.lalibre.be/robots.txt","bytes":9206,"groups":125},"bots":{"GPTBot":["partial","named",1,11],"ChatGPT-User":["partial","named",1,11],"OAI-SearchBot":["blocked","named",1,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",18,0],"Claude-SearchBot":["partial","*",18,0],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["partial","*",18,0],"Google-Extended":["partial","named",1,11],"Applebot-Extended":["blocked","named",1,0],"CCBot":["partial","named",1,11],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["partial","*",18,0],"ContextFetchBot":["partial","*",18,0]},"llms":{"status":"missing","http":404,"bytes":327051},"home":{"status":"ok","http":200,"url":"https://www.lalibre.be/","html_tokens":764569,"markdown_tokens":25,"pruned_tokens":25}},
{"rank":6015,"domain":"iberia.com","origin":"https://iberia.com","robots":{"status":"refused","http":403,"url":"https://www.iberia.com/robots.txt","bytes":16560,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":6016,"domain":"freemusicarchive.org","origin":"https://freemusicarchive.org","robots":{"status":"ok","http":200,"url":"https://freemusicarchive.org/robots.txt","bytes":340,"groups":7},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["blocked","named",1,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["blocked","named",1,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":25613},"home":{"status":"ok","http":200,"url":"https://freemusicarchive.org/","html_tokens":4458,"markdown_tokens":483,"pruned_tokens":124}},
{"rank":6017,"domain":"optinmonster.com","origin":"https://optinmonster.com","robots":{"status":"ok","http":200,"url":"https://optinmonster.com/robots.txt","bytes":556,"groups":1},"bots":{"GPTBot":["partial","*",16,1],"ChatGPT-User":["partial","*",16,1],"OAI-SearchBot":["partial","*",16,1],"ClaudeBot":["partial","*",16,1],"Claude-User":["partial","*",16,1],"Claude-SearchBot":["partial","*",16,1],"anthropic-ai":["partial","*",16,1],"PerplexityBot":["partial","*",16,1],"Perplexity-User":["partial","*",16,1],"Google-Extended":["partial","*",16,1],"Applebot-Extended":["partial","*",16,1],"CCBot":["partial","*",16,1],"Bytespider":["partial","*",16,1],"Amazonbot":["partial","*",16,1],"meta-externalagent":["partial","*",16,1],"cohere-ai":["partial","*",16,1],"Diffbot":["partial","*",16,1],"ContextFetchBot":["partial","*",16,1]},"llms":{"status":"other_text","http":200,"bytes":454986},"home":{"status":"ok","http":200,"url":"https://optinmonster.com/","html_tokens":105742,"markdown_tokens":601,"pruned_tokens":385}},
{"rank":6018,"domain":"ravelry.com","origin":"https://ravelry.com","robots":{"status":"ok","http":200,"url":"https://www.ravelry.com/robots.txt","bytes":1366,"groups":25},"bots":{"GPTBot":["partial","*",6,0],"ChatGPT-User":["partial","*",6,0],"OAI-SearchBot":["partial","*",6,0],"ClaudeBot":["partial","*",6,0],"Claude-User":["partial","*",6,0],"Claude-SearchBot":["partial","*",6,0],"anthropic-ai":["partial","*",6,0],"PerplexityBot":["partial","*",6,0],"Perplexity-User":["partial","*",6,0],"Google-Extended":["partial","*",6,0],"Applebot-Extended":["partial","*",6,0],"CCBot":["partial","*",6,0],"Bytespider":["partial","*",6,0],"Amazonbot":["partial","*",6,0],"meta-externalagent":["partial","*",6,0],"cohere-ai":["partial","*",6,0],"Diffbot":["partial","*",6,0],"ContextFetchBot":["partial","*",6,0]},"llms":{"status":"present","http":200,"bytes":465},"home":{"status":"ok","http":200,"url":"https://www.ravelry.com/account/login","html_tokens":1661,"markdown_tokens":18,"pruned_tokens":18}},
{"rank":6019,"domain":"yandexcloud.net","origin":"https://yandexcloud.net","robots":{"status":"unreachable","http":null,"url":"https://yandexcloud.net/robots.txt","bytes":0,"groups":0,"error":"cannot resolve yandexcloud.net"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":6020,"domain":"cbre.com","origin":"https://cbre.com","robots":{"status":"refused","http":403,"url":"https://www.cbre.com/robots.txt","bytes":5533,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":6021,"domain":"scroll.in","origin":"https://scroll.in","robots":{"status":"ok","http":200,"url":"https://scroll.in/robots.txt","bytes":195,"groups":2},"bots":{"GPTBot":["partial","*",2,0],"ChatGPT-User":["partial","*",2,0],"OAI-SearchBot":["partial","*",2,0],"ClaudeBot":["partial","*",2,0],"Claude-User":["partial","*",2,0],"Claude-SearchBot":["partial","*",2,0],"anthropic-ai":["partial","*",2,0],"PerplexityBot":["partial","*",2,0],"Perplexity-User":["partial","*",2,0],"Google-Extended":["partial","*",2,0],"Applebot-Extended":["partial","*",2,0],"CCBot":["partial","*",2,0],"Bytespider":["partial","*",2,0],"Amazonbot":["partial","*",2,0],"meta-externalagent":["partial","*",2,0],"cohere-ai":["partial","*",2,0],"Diffbot":["partial","*",2,0],"ContextFetchBot":["partial","*",2,0]},"llms":{"status":"missing","http":404,"bytes":2681},"home":{"status":"ok","http":200,"url":"https://scroll.in/","html_tokens":28707,"markdown_tokens":34,"pruned_tokens":34}},
{"rank":6022,"domain":"tyc.edu.tw","origin":"https://tyc.edu.tw","robots":{"status":"unreachable","http":null,"url":"https://tyc.edu.tw/robots.txt","bytes":0,"groups":0,"error":"cannot resolve tyc.edu.tw"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":6023,"domain":"cq.gov.cn","origin":"https://cq.gov.cn","robots":{"status":"timeout","http":null,"url":"https://cq.gov.cn/robots.txt","bytes":0,"groups":0,"error":"ConnectTimeout"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":6024,"domain":"rhizome.org","origin":"https://rhizome.org","robots":{"status":"timeout","http":null,"url":"https://rhizome.org/robots.txt","bytes":0,"groups":0,"error":"timed out after 10 s"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":6025,"domain":"cncf.io","origin":"https://cncf.io","robots":{"status":"refused","http":403,"url":"https://cncf.io/robots.txt","bytes":560,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":6026,"domain":"gmanetwork.com","origin":"https://gmanetwork.com","robots":{"status":"ok","http":200,"url":"https://www.gmanetwork.com/robots.txt","bytes":3384,"groups":20},"bots":{"GPTBot":["allowed","named",0,1],"ChatGPT-User":["allowed","named",0,1],"OAI-SearchBot":["partial","*",30,20],"ClaudeBot":["allowed","named",0,1],"Claude-User":["allowed","named",0,1],"Claude-SearchBot":["allowed","named",0,1],"anthropic-ai":["allowed","named",0,1],"PerplexityBot":["allowed","named",0,1],"Perplexity-User":["allowed","named",0,1],"Google-Extended":["allowed","named",0,1],"Applebot-Extended":["partial","*",30,20],"CCBot":["partial","*",30,20],"Bytespider":["partial","*",30,20],"Amazonbot":["partial","*",30,20],"meta-externalagent":["partial","*",30,20],"cohere-ai":["allowed","named",0,1],"Diffbot":["partial","*",30,20],"ContextFetchBot":["partial","*",30,20]},"llms":{"status":"present","http":200,"bytes":27627},"home":{"status":"ok","http":200,"url":"https://www.gmanetwork.com/","html_tokens":133882,"markdown_tokens":104,"pruned_tokens":104}},
{"rank":6027,"domain":"maxon.net","origin":"https://maxon.net","robots":{"status":"ok","http":200,"url":"https://www.maxon.net/robots.txt","bytes":717,"groups":2},"bots":{"GPTBot":["partial","*",1,2],"ChatGPT-User":["partial","*",1,2],"OAI-SearchBot":["partial","*",1,2],"ClaudeBot":["partial","*",1,2],"Claude-User":["partial","*",1,2],"Claude-SearchBot":["partial","*",1,2],"anthropic-ai":["partial","*",1,2],"PerplexityBot":["partial","*",1,2],"Perplexity-User":["partial","*",1,2],"Google-Extended":["partial","*",1,2],"Applebot-Extended":["partial","*",1,2],"CCBot":["partial","*",1,2],"Bytespider":["partial","*",1,2],"Amazonbot":["partial","*",1,2],"meta-externalagent":["partial","*",1,2],"cohere-ai":["partial","*",1,2],"Diffbot":["partial","*",1,2],"ContextFetchBot":["partial","*",1,2]},"llms":{"status":"missing","http":404,"bytes":21897},"home":{"status":"ok","http":200,"url":"https://www.maxon.net/en/","html_tokens":104171,"markdown_tokens":284,"pruned_tokens":80}},
{"rank":6028,"domain":"ghanaweb.com","origin":"https://ghanaweb.com","robots":{"status":"ok","http":200,"url":"https://www.ghanaweb.com/robots.txt","bytes":621,"groups":3},"bots":{"GPTBot":["partial","*",6,0],"ChatGPT-User":["partial","*",6,0],"OAI-SearchBot":["partial","*",6,0],"ClaudeBot":["partial","*",6,0],"Claude-User":["partial","*",6,0],"Claude-SearchBot":["partial","*",6,0],"anthropic-ai":["partial","*",6,0],"PerplexityBot":["partial","*",6,0],"Perplexity-User":["partial","*",6,0],"Google-Extended":["partial","*",6,0],"Applebot-Extended":["partial","*",6,0],"CCBot":["partial","*",6,0],"Bytespider":["partial","*",6,0],"Amazonbot":["partial","*",6,0],"meta-externalagent":["partial","*",6,0],"cohere-ai":["partial","*",6,0],"Diffbot":["partial","*",6,0],"ContextFetchBot":["partial","*",6,0]},"llms":{"status":"missing","http":404,"bytes":1861},"home":{"status":"ok","http":200,"url":"https://ghanaweb.com/","html_tokens":457,"markdown_tokens":0,"pruned_tokens":0}},
{"rank":6029,"domain":"sciencespo.fr","origin":"https://sciencespo.fr","robots":{"status":"ok","http":200,"url":"https://www.sciencespo.fr/robots.txt","bytes":3387,"groups":1},"bots":{"GPTBot":["partial","*",74,0],"ChatGPT-User":["partial","*",74,0],"OAI-SearchBot":["partial","*",74,0],"ClaudeBot":["partial","*",74,0],"Claude-User":["partial","*",74,0],"Claude-SearchBot":["partial","*",74,0],"anthropic-ai":["partial","*",74,0],"PerplexityBot":["partial","*",74,0],"Perplexity-User":["partial","*",74,0],"Google-Extended":["partial","*",74,0],"Applebot-Extended":["partial","*",74,0],"CCBot":["partial","*",74,0],"Bytespider":["partial","*",74,0],"Amazonbot":["partial","*",74,0],"meta-externalagent":["partial","*",74,0],"cohere-ai":["partial","*",74,0],"Diffbot":["partial","*",74,0],"ContextFetchBot":["partial","*",74,0]},"llms":{"status":"timeout","error":"timed out after 10 s"},"home":{"status":"ok","http":200,"url":"https://www.sciencespo.fr/","html_tokens":80,"markdown_tokens":9,"pruned_tokens":9}},
{"rank":6030,"domain":"unocha.org","origin":"https://unocha.org","robots":{"status":"ok","http":200,"url":"https://www.unocha.org/robots.txt","bytes":2152,"groups":1},"bots":{"GPTBot":["partial","*",35,22],"ChatGPT-User":["partial","*",35,22],"OAI-SearchBot":["partial","*",35,22],"ClaudeBot":["partial","*",35,22],"Claude-User":["partial","*",35,22],"Claude-SearchBot":["partial","*",35,22],"anthropic-ai":["partial","*",35,22],"PerplexityBot":["partial","*",35,22],"Perplexity-User":["partial","*",35,22],"Google-Extended":["partial","*",35,22],"Applebot-Extended":["partial","*",35,22],"CCBot":["partial","*",35,22],"Bytespider":["partial","*",35,22],"Amazonbot":["partial","*",35,22],"meta-externalagent":["partial","*",35,22],"cohere-ai":["partial","*",35,22],"Diffbot":["partial","*",35,22],"ContextFetchBot":["partial","*",35,22]},"llms":{"status":"missing","http":404,"bytes":146},"home":{"status":"ok","http":200,"url":"https://www.unocha.org/","html_tokens":67373,"markdown_tokens":1007,"pruned_tokens":156}},
{"rank":6031,"domain":"publitas.com","origin":"https://publitas.com","robots":{"status":"refused","http":403,"url":"https://publitas.com/robots.txt","bytes":103,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":6032,"domain":"ladbible.com","origin":"https://ladbible.com","robots":{"status":"ok","http":200,"url":"https://www.ladbible.com/robots.txt","bytes":214,"groups":2},"bots":{"GPTBot":["partial","*",2,1],"ChatGPT-User":["partial","*",2,1],"OAI-SearchBot":["partial","*",2,1],"ClaudeBot":["partial","*",2,1],"Claude-User":["partial","*",2,1],"Claude-SearchBot":["partial","*",2,1],"anthropic-ai":["partial","*",2,1],"PerplexityBot":["partial","*",2,1],"Perplexity-User":["partial","*",2,1],"Google-Extended":["partial","*",2,1],"Applebot-Extended":["partial","*",2,1],"CCBot":["partial","*",2,1],"Bytespider":["partial","*",2,1],"Amazonbot":["partial","*",2,1],"meta-externalagent":["partial","*",2,1],"cohere-ai":["partial","*",2,1],"Diffbot":["partial","*",2,1],"ContextFetchBot":["partial","*",2,1]},"llms":{"status":"missing","http":404,"bytes":94190},"home":{"status":"ok","http":200,"url":"https://www.ladbible.com/","html_tokens":152614,"markdown_tokens":216,"pruned_tokens":26}},
{"rank":6033,"domain":"brusselstimes.com","origin":"https://brusselstimes.com","robots":{"status":"ok","http":200,"url":"https://www.brusselstimes.com/robots.txt","bytes":182,"groups":1},"bots":{"GPTBot":["partial","*",2,1],"ChatGPT-User":["partial","*",2,1],"OAI-SearchBot":["partial","*",2,1],"ClaudeBot":["partial","*",2,1],"Claude-User":["partial","*",2,1],"Claude-SearchBot":["partial","*",2,1],"anthropic-ai":["partial","*",2,1],"PerplexityBot":["partial","*",2,1],"Perplexity-User":["partial","*",2,1],"Google-Extended":["partial","*",2,1],"Applebot-Extended":["partial","*",2,1],"CCBot":["partial","*",2,1],"Bytespider":["partial","*",2,1],"Amazonbot":["partial","*",2,1],"meta-externalagent":["partial","*",2,1],"cohere-ai":["partial","*",2,1],"Diffbot":["partial","*",2,1],"ContextFetchBot":["partial","*",2,1]},"llms":{"status":"missing","http":404,"bytes":9},"home":{"status":"ok","http":200,"url":"https://www.brusselstimes.com/","html_tokens":77911,"markdown_tokens":15,"pruned_tokens":15}},
{"rank":6034,"domain":"openstax.org","origin":"https://openstax.org","robots":{"status":"ok","http":200,"url":"https://openstax.org/robots.txt","bytes":453,"groups":2},"bots":{"GPTBot":["partial","named",1,0],"ChatGPT-User":["partial","*",14,0],"OAI-SearchBot":["partial","*",14,0],"ClaudeBot":["partial","*",14,0],"Claude-User":["partial","*",14,0],"Claude-SearchBot":["partial","*",14,0],"anthropic-ai":["partial","*",14,0],"PerplexityBot":["partial","*",14,0],"Perplexity-User":["partial","*",14,0],"Google-Extended":["partial","*",14,0],"Applebot-Extended":["partial","*",14,0],"CCBot":["partial","*",14,0],"Bytespider":["partial","*",14,0],"Amazonbot":["partial","*",14,0],"meta-externalagent":["partial","*",14,0],"cohere-ai":["partial","*",14,0],"Diffbot":["partial","*",14,0],"ContextFetchBot":["partial","*",14,0]},"llms":{"status":"html","http":200,"bytes":12343},"home":{"status":"ok","http":200,"url":"https://openstax.org/","html_tokens":3086,"markdown_tokens":14,"pruned_tokens":14}},
{"rank":6035,"domain":"mct.gov.cn","origin":"https://mct.gov.cn","robots":{"status":"missing","http":404,"url":"https://mct.gov.cn/robots.txt","bytes":1163,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":1163},"home":{"status":"ok","http":200,"url":"https://mct.gov.cn/","html_tokens":32467,"markdown_tokens":565,"pruned_tokens":114}},
{"rank":6036,"domain":"giveawayoftheday.com","origin":"https://giveawayoftheday.com","robots":{"status":"ok","http":200,"url":"https://giveawayoftheday.com/robots.txt","bytes":390,"groups":2},"bots":{"GPTBot":["partial","*",14,0],"ChatGPT-User":["partial","*",14,0],"OAI-SearchBot":["partial","*",14,0],"ClaudeBot":["partial","*",14,0],"Claude-User":["partial","*",14,0],"Claude-SearchBot":["partial","*",14,0],"anthropic-ai":["partial","*",14,0],"PerplexityBot":["partial","*",14,0],"Perplexity-User":["partial","*",14,0],"Google-Extended":["partial","*",14,0],"Applebot-Extended":["partial","*",14,0],"CCBot":["partial","*",14,0],"Bytespider":["partial","*",14,0],"Amazonbot":["partial","*",14,0],"meta-externalagent":["partial","*",14,0],"cohere-ai":["partial","*",14,0],"Diffbot":["partial","*",14,0],"ContextFetchBot":["partial","*",14,0]},"llms":{"status":"html","http":200,"bytes":55158},"home":{"status":"ok","http":200,"url":"https://www.giveawayoftheday.com","html_tokens":13774,"markdown_tokens":399,"pruned_tokens":166}},
{"rank":6037,"domain":"blubrry.com","origin":"https://blubrry.com","robots":{"status":"ok","http":200,"url":"https://blubrry.com/robots.txt","bytes":126,"groups":1},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"http_error","http":403,"bytes":263},"home":{"status":"ok","http":200,"url":"https://blubrry.com/","html_tokens":33724,"markdown_tokens":145,"pruned_tokens":145}},
{"rank":6038,"domain":"annals.org","origin":"https://annals.org","robots":{"status":"ok","http":200,"url":"https://www.acpjournals.org/robots.txt","bytes":964,"groups":6},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["partial","*",17,7],"OAI-SearchBot":["partial","*",17,7],"ClaudeBot":["partial","*",17,7],"Claude-User":["partial","*",17,7],"Claude-SearchBot":["partial","*",17,7],"anthropic-ai":["partial","*",17,7],"PerplexityBot":["partial","*",17,7],"Perplexity-User":["partial","*",17,7],"Google-Extended":["partial","*",17,7],"Applebot-Extended":["blocked","named",1,0],"CCBot":["partial","*",17,7],"Bytespider":["partial","*",17,7],"Amazonbot":["partial","*",17,7],"meta-externalagent":["partial","*",17,7],"cohere-ai":["partial","*",17,7],"Diffbot":["partial","*",17,7],"ContextFetchBot":["partial","*",17,7]},"llms":{"status":"http_error","http":403,"bytes":5520},"home":{"status":"http_error","http":403,"url":"https://annals.org/"}},
{"rank":6039,"domain":"theblock.co","origin":"https://theblock.co","robots":{"status":"ok","http":200,"url":"https://www.theblock.co/robots.txt","bytes":890,"groups":2},"bots":{"GPTBot":["partial","named",8,0],"ChatGPT-User":["partial","*",8,0],"OAI-SearchBot":["partial","*",8,0],"ClaudeBot":["partial","named",8,0],"Claude-User":["partial","*",8,0],"Claude-SearchBot":["partial","*",8,0],"anthropic-ai":["partial","*",8,0],"PerplexityBot":["partial","named",8,0],"Perplexity-User":["partial","*",8,0],"Google-Extended":["partial","*",8,0],"Applebot-Extended":["partial","named",8,0],"CCBot":["partial","named",8,0],"Bytespider":["partial","named",8,0],"Amazonbot":["partial","named",8,0],"meta-externalagent":["partial","named",8,0],"cohere-ai":["partial","*",8,0],"Diffbot":["partial","*",8,0],"ContextFetchBot":["partial","*",8,0]},"llms":{"status":"present","http":200,"bytes":10659},"home":{"status":"http_error","http":403,"url":"https://theblock.co/"}},
{"rank":6040,"domain":"codeigniter.com","origin":"https://codeigniter.com","robots":{"status":"ok","http":200,"url":"https://codeigniter.com/robots.txt","bytes":2701,"groups":33},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["allowed","*",0,1],"OAI-SearchBot":["allowed","*",0,1],"ClaudeBot":["blocked","named",1,0],"Claude-User":["allowed","*",0,1],"Claude-SearchBot":["allowed","*",0,1],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["allowed","*",0,1],"Perplexity-User":["allowed","*",0,1],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["allowed","*",0,1]},"llms":{"status":"missing","http":404,"bytes":2278},"home":{"status":"ok","http":200,"url":"https://codeigniter.com/","html_tokens":10536,"markdown_tokens":214,"pruned_tokens":128}},
{"rank":6041,"domain":"elespectador.com","origin":"https://elespectador.com","robots":{"status":"ok","http":200,"url":"https://www.elespectador.com/robots.txt","bytes":7681,"groups":33},"bots":{"GPTBot":["partial","*",73,1],"ChatGPT-User":["partial","*",73,1],"OAI-SearchBot":["partial","*",73,1],"ClaudeBot":["partial","*",73,1],"Claude-User":["partial","*",73,1],"Claude-SearchBot":["partial","*",73,1],"anthropic-ai":["partial","*",73,1],"PerplexityBot":["partial","*",73,1],"Perplexity-User":["partial","*",73,1],"Google-Extended":["partial","*",73,1],"Applebot-Extended":["partial","*",73,1],"CCBot":["partial","*",73,1],"Bytespider":["partial","*",73,1],"Amazonbot":["partial","*",73,1],"meta-externalagent":["partial","*",73,1],"cohere-ai":["partial","*",73,1],"Diffbot":["partial","*",73,1],"ContextFetchBot":["partial","*",73,1]},"llms":{"status":"missing","http":404,"bytes":247274},"home":{"status":"ok","http":200,"url":"https://www.elespectador.com/","html_tokens":139073,"markdown_tokens":100,"pruned_tokens":100}},
{"rank":6042,"domain":"playboy.com","origin":"https://playboy.com","robots":{"status":"ok","http":200,"url":"https://www.playboy.com/robots.txt","bytes":191,"groups":1},"bots":{"GPTBot":["partial","*",1,0],"ChatGPT-User":["partial","*",1,0],"OAI-SearchBot":["partial","*",1,0],"ClaudeBot":["partial","*",1,0],"Claude-User":["partial","*",1,0],"Claude-SearchBot":["partial","*",1,0],"anthropic-ai":["partial","*",1,0],"PerplexityBot":["partial","*",1,0],"Perplexity-User":["partial","*",1,0],"Google-Extended":["partial","*",1,0],"Applebot-Extended":["partial","*",1,0],"CCBot":["partial","*",1,0],"Bytespider":["partial","*",1,0],"Amazonbot":["partial","*",1,0],"meta-externalagent":["partial","*",1,0],"cohere-ai":["partial","*",1,0],"Diffbot":["partial","*",1,0],"ContextFetchBot":["partial","*",1,0]},"llms":{"status":"http_error","http":403,"bytes":6165},"home":{"status":"ok","http":200,"url":"https://www.playboy.com/","html_tokens":40702,"markdown_tokens":101,"pruned_tokens":101}},
{"rank":6043,"domain":"notes.io","origin":"https://notes.io","robots":{"status":"ok","http":200,"url":"https://notes.io/robots.txt","bytes":87,"groups":1},"bots":{"GPTBot":["allowed","*",0,1],"ChatGPT-User":["allowed","*",0,1],"OAI-SearchBot":["allowed","*",0,1],"ClaudeBot":["allowed","*",0,1],"Claude-User":["allowed","*",0,1],"Claude-SearchBot":["allowed","*",0,1],"anthropic-ai":["allowed","*",0,1],"PerplexityBot":["allowed","*",0,1],"Perplexity-User":["allowed","*",0,1],"Google-Extended":["allowed","*",0,1],"Applebot-Extended":["allowed","*",0,1],"CCBot":["allowed","*",0,1],"Bytespider":["allowed","*",0,1],"Amazonbot":["allowed","*",0,1],"meta-externalagent":["allowed","*",0,1],"cohere-ai":["allowed","*",0,1],"Diffbot":["allowed","*",0,1],"ContextFetchBot":["allowed","*",0,1]},"llms":{"status":"html","http":200,"bytes":12850},"home":{"status":"ok","http":200,"url":"https://notes.io/","html_tokens":3331,"markdown_tokens":403,"pruned_tokens":312}},
{"rank":6044,"domain":"ifrc.org","origin":"https://ifrc.org","robots":{"status":"ok","http":200,"url":"https://www.ifrc.org/robots.txt","bytes":1765,"groups":1},"bots":{"GPTBot":["partial","*",22,23],"ChatGPT-User":["partial","*",22,23],"OAI-SearchBot":["partial","*",22,23],"ClaudeBot":["partial","*",22,23],"Claude-User":["partial","*",22,23],"Claude-SearchBot":["partial","*",22,23],"anthropic-ai":["partial","*",22,23],"PerplexityBot":["partial","*",22,23],"Perplexity-User":["partial","*",22,23],"Google-Extended":["partial","*",22,23],"Applebot-Extended":["partial","*",22,23],"CCBot":["partial","*",22,23],"Bytespider":["partial","*",22,23],"Amazonbot":["partial","*",22,23],"meta-externalagent":["partial","*",22,23],"cohere-ai":["partial","*",22,23],"Diffbot":["partial","*",22,23],"ContextFetchBot":["partial","*",22,23]},"llms":{"status":"missing","http":404,"bytes":507},"home":{"status":"http_error","http":403,"url":"https://www.ifrc.org/"}},
{"rank":6045,"domain":"trino-casino.com","origin":"https://trino-casino.com","robots":{"status":"ok","http":200,"url":"https://trino-casino.com/robots.txt","bytes":67,"groups":1},"bots":{"GPTBot":["partial","*",1,1],"ChatGPT-User":["partial","*",1,1],"OAI-SearchBot":["partial","*",1,1],"ClaudeBot":["partial","*",1,1],"Claude-User":["partial","*",1,1],"Claude-SearchBot":["partial","*",1,1],"anthropic-ai":["partial","*",1,1],"PerplexityBot":["partial","*",1,1],"Perplexity-User":["partial","*",1,1],"Google-Extended":["partial","*",1,1],"Applebot-Extended":["partial","*",1,1],"CCBot":["partial","*",1,1],"Bytespider":["partial","*",1,1],"Amazonbot":["partial","*",1,1],"meta-externalagent":["partial","*",1,1],"cohere-ai":["partial","*",1,1],"Diffbot":["partial","*",1,1],"ContextFetchBot":["partial","*",1,1]},"llms":{"status":"missing","http":404,"bytes":50613},"home":{"status":"ok","http":200,"url":"https://trino-casino.com/","html_tokens":17927,"markdown_tokens":1789,"pruned_tokens":131}},
{"rank":6046,"domain":"oapen.org","origin":"https://oapen.org","robots":{"status":"missing","http":404,"url":"https://oapen.org/robots.txt","bytes":44109,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":44109},"home":{"status":"ok","http":200,"url":"https://oapen.org/","html_tokens":19516,"markdown_tokens":373,"pruned_tokens":65}},
{"rank":6047,"domain":"zhipin.com","origin":"https://zhipin.com","robots":{"status":"ok","http":200,"url":"https://www.zhipin.com/robots.txt","bytes":1040,"groups":3},"bots":{"GPTBot":["partial","*",40,0],"ChatGPT-User":["partial","*",40,0],"OAI-SearchBot":["partial","*",40,0],"ClaudeBot":["partial","*",40,0],"Claude-User":["partial","*",40,0],"Claude-SearchBot":["partial","*",40,0],"anthropic-ai":["partial","*",40,0],"PerplexityBot":["partial","*",40,0],"Perplexity-User":["partial","*",40,0],"Google-Extended":["partial","*",40,0],"Applebot-Extended":["partial","*",40,0],"CCBot":["partial","*",40,0],"Bytespider":["partial","*",40,0],"Amazonbot":["partial","*",40,0],"meta-externalagent":["partial","*",40,0],"cohere-ai":["partial","*",40,0],"Diffbot":["partial","*",40,0],"ContextFetchBot":["partial","*",40,0]},"llms":{"status":"missing","http":404,"bytes":3797},"home":{"status":"ok","http":200,"url":"https://www.zhipin.com/","html_tokens":127882,"markdown_tokens":40,"pruned_tokens":40}},
{"rank":6048,"domain":"bremen.de","origin":"https://bremen.de","robots":{"status":"ok","http":200,"url":"https://www.bremen.de/robots.txt","bytes":576,"groups":1},"bots":{"GPTBot":["partial","*",13,0],"ChatGPT-User":["partial","*",13,0],"OAI-SearchBot":["partial","*",13,0],"ClaudeBot":["partial","*",13,0],"Claude-User":["partial","*",13,0],"Claude-SearchBot":["partial","*",13,0],"anthropic-ai":["partial","*",13,0],"PerplexityBot":["partial","*",13,0],"Perplexity-User":["partial","*",13,0],"Google-Extended":["partial","*",13,0],"Applebot-Extended":["partial","*",13,0],"CCBot":["partial","*",13,0],"Bytespider":["partial","*",13,0],"Amazonbot":["partial","*",13,0],"meta-externalagent":["partial","*",13,0],"cohere-ai":["partial","*",13,0],"Diffbot":["partial","*",13,0],"ContextFetchBot":["partial","*",13,0]},"llms":{"status":"present","http":200,"bytes":1366},"home":{"status":"http_error","http":403,"url":"https://www.bremen.de/"}},
{"rank":6049,"domain":"travis-ci.org","origin":"https://travis-ci.org","robots":{"status":"ok","http":200,"url":"https://app.travis-ci.com/robots.txt","bytes":51,"groups":1},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"html","http":200,"bytes":59655},"home":{"status":"ok","http":200,"url":"https://www.travis-ci.com/","html_tokens":41949,"markdown_tokens":3939,"pruned_tokens":143}},
{"rank":6050,"domain":"umaryland.edu","origin":"https://umaryland.edu","robots":{"status":"ok","http":200,"url":"https://www.umaryland.edu/robots.txt","bytes":23,"groups":1},"bots":{"GPTBot":["allowed","*",0,1],"ChatGPT-User":["allowed","*",0,1],"OAI-SearchBot":["allowed","*",0,1],"ClaudeBot":["allowed","*",0,1],"Claude-User":["allowed","*",0,1],"Claude-SearchBot":["allowed","*",0,1],"anthropic-ai":["allowed","*",0,1],"PerplexityBot":["allowed","*",0,1],"Perplexity-User":["allowed","*",0,1],"Google-Extended":["allowed","*",0,1],"Applebot-Extended":["allowed","*",0,1],"CCBot":["allowed","*",0,1],"Bytespider":["allowed","*",0,1],"Amazonbot":["allowed","*",0,1],"meta-externalagent":["allowed","*",0,1],"cohere-ai":["allowed","*",0,1],"Diffbot":["allowed","*",0,1],"ContextFetchBot":["allowed","*",0,1]},"llms":{"status":"missing","http":404,"bytes":147281},"home":{"status":"ok","http":200,"url":"https://www.umaryland.edu/","html_tokens":41753,"markdown_tokens":703,"pruned_tokens":158}},
{"rank":6051,"domain":"thriveglobal.com","origin":"https://thriveglobal.com","robots":{"status":"ok","http":200,"url":"https://thriveglobal.com/robots.txt","bytes":45,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":54086},"home":{"status":"ok","http":200,"url":"https://thriveglobal.com/","html_tokens":37360,"markdown_tokens":499,"pruned_tokens":103}},
{"rank":6052,"domain":"bitget.com","origin":"https://bitget.com","robots":{"status":"ok","http":200,"url":"https://www.bitget.com/robots.txt","bytes":1761,"groups":1},"bots":{"GPTBot":["partial","*",53,2],"ChatGPT-User":["partial","*",53,2],"OAI-SearchBot":["partial","*",53,2],"ClaudeBot":["partial","*",53,2],"Claude-User":["partial","*",53,2],"Claude-SearchBot":["partial","*",53,2],"anthropic-ai":["partial","*",53,2],"PerplexityBot":["partial","*",53,2],"Perplexity-User":["partial","*",53,2],"Google-Extended":["partial","*",53,2],"Applebot-Extended":["partial","*",53,2],"CCBot":["partial","*",53,2],"Bytespider":["partial","*",53,2],"Amazonbot":["partial","*",53,2],"meta-externalagent":["partial","*",53,2],"cohere-ai":["partial","*",53,2],"Diffbot":["partial","*",53,2],"ContextFetchBot":["partial","*",53,2]},"llms":{"status":"other_text","http":200,"bytes":55134},"home":{"status":"ok","http":200,"url":"https://www.bitget.com/","html_tokens":79599,"markdown_tokens":66,"pruned_tokens":66}},
{"rank":6053,"domain":"aphapublications.org","origin":"https://aphapublications.org","robots":{"status":"ok","http":200,"url":"https://apha-prod.literatumonline.com/robots.txt","bytes":4,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"http_error","http":403,"bytes":5573},"home":{"status":"http_error","http":403,"url":"https://aphapublications.org/"}},
{"rank":6054,"domain":"untappd.com","origin":"https://untappd.com","robots":{"status":"ok","http":200,"url":"https://untappd.com/robots.txt","bytes":691,"groups":3},"bots":{"GPTBot":["partial","*",1,0],"ChatGPT-User":["partial","*",1,0],"OAI-SearchBot":["partial","*",1,0],"ClaudeBot":["partial","*",1,0],"Claude-User":["partial","*",1,0],"Claude-SearchBot":["partial","*",1,0],"anthropic-ai":["partial","*",1,0],"PerplexityBot":["partial","*",1,0],"Perplexity-User":["partial","*",1,0],"Google-Extended":["partial","*",1,0],"Applebot-Extended":["partial","*",1,0],"CCBot":["partial","*",1,0],"Bytespider":["partial","*",1,0],"Amazonbot":["partial","*",1,0],"meta-externalagent":["partial","*",1,0],"cohere-ai":["partial","*",1,0],"Diffbot":["partial","*",1,0],"ContextFetchBot":["partial","*",1,0]},"llms":{"status":"missing","http":404,"bytes":6515},"home":{"status":"ok","http":200,"url":"https://untappd.com/","html_tokens":32544,"markdown_tokens":281,"pruned_tokens":40}},
{"rank":6055,"domain":"bepress.com","origin":"https://bepress.com","robots":{"status":"html","http":200,"url":"https://www.elsevier.com/products/digital-commons","bytes":299139,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":6056,"domain":"it168.com","origin":"https://it168.com","robots":{"status":"html","http":200,"url":"https://it168.com/robots.txt","bytes":39,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":6057,"domain":"ufc.com","origin":"https://ufc.com","robots":{"status":"ok","http":200,"url":"https://www.ufc.com/robots.txt","bytes":2038,"groups":1},"bots":{"GPTBot":["partial","*",32,18],"ChatGPT-User":["partial","*",32,18],"OAI-SearchBot":["partial","*",32,18],"ClaudeBot":["partial","*",32,18],"Claude-User":["partial","*",32,18],"Claude-SearchBot":["partial","*",32,18],"anthropic-ai":["partial","*",32,18],"PerplexityBot":["partial","*",32,18],"Perplexity-User":["partial","*",32,18],"Google-Extended":["partial","*",32,18],"Applebot-Extended":["partial","*",32,18],"CCBot":["partial","*",32,18],"Bytespider":["partial","*",32,18],"Amazonbot":["partial","*",32,18],"meta-externalagent":["partial","*",32,18],"cohere-ai":["partial","*",32,18],"Diffbot":["partial","*",32,18],"ContextFetchBot":["partial","*",32,18]},"llms":{"status":"missing","http":404,"bytes":185},"home":{"status":"ok","http":200,"url":"https://www.ufc.com/","html_tokens":54569,"markdown_tokens":51,"pruned_tokens":51}},
{"rank":6058,"domain":"uni-augsburg.de","origin":"https://uni-augsburg.de","robots":{"status":"ok","http":200,"url":"https://www.uni-augsburg.de/robots.txt","bytes":307,"groups":5},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["partial","*",2,1],"OAI-SearchBot":["partial","*",2,1],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",2,1],"Claude-SearchBot":["partial","*",2,1],"anthropic-ai":["partial","*",2,1],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["partial","*",2,1],"Google-Extended":["partial","*",2,1],"Applebot-Extended":["partial","*",2,1],"CCBot":["partial","*",2,1],"Bytespider":["partial","*",2,1],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["partial","*",2,1],"cohere-ai":["partial","*",2,1],"Diffbot":["partial","*",2,1],"ContextFetchBot":["partial","*",2,1]},"llms":{"status":"missing","http":404,"bytes":101895},"home":{"status":"ok","http":200,"url":"https://www.uni-augsburg.de/en/","html_tokens":40972,"markdown_tokens":848,"pruned_tokens":310}},
{"rank":6059,"domain":"sfr.fr","origin":"https://sfr.fr","robots":{"status":"unreachable","http":null,"url":"https://sfr.fr/robots.txt","bytes":0,"groups":0,"error":"ReadError"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":6060,"domain":"niche.com","origin":"https://niche.com","robots":{"status":"ok","http":200,"url":"https://www.niche.com/robots.txt","bytes":4602,"groups":30},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["partial","*",3,0],"OAI-SearchBot":["partial","*",3,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",3,0],"Claude-SearchBot":["partial","*",3,0],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["partial","*",3,0],"Perplexity-User":["partial","*",3,0],"Google-Extended":["partial","*",3,0],"Applebot-Extended":["partial","*",3,0],"CCBot":["partial","*",3,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["partial","*",3,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",3,0]},"llms":{"status":"missing","http":404,"bytes":104161},"home":{"status":"ok","http":200,"url":"https://www.niche.com/","html_tokens":40410,"markdown_tokens":208,"pruned_tokens":23}},
{"rank":6061,"domain":"rolex.com","origin":"https://rolex.com","robots":{"status":"missing","http":404,"url":"https://rolex.com/robots.txt","bytes":13,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"http_error","http":403,"bytes":371},"home":{"status":"http_error","http":403,"url":"https://rolex.com/"}},
{"rank":6062,"domain":"rbc.ua","origin":"https://rbc.ua","robots":{"status":"ok","http":200,"url":"https://www.rbc.ua/robots.txt","bytes":312,"groups":1},"bots":{"GPTBot":["partial","*",11,0],"ChatGPT-User":["partial","*",11,0],"OAI-SearchBot":["partial","*",11,0],"ClaudeBot":["partial","*",11,0],"Claude-User":["partial","*",11,0],"Claude-SearchBot":["partial","*",11,0],"anthropic-ai":["partial","*",11,0],"PerplexityBot":["partial","*",11,0],"Perplexity-User":["partial","*",11,0],"Google-Extended":["partial","*",11,0],"Applebot-Extended":["partial","*",11,0],"CCBot":["partial","*",11,0],"Bytespider":["partial","*",11,0],"Amazonbot":["partial","*",11,0],"meta-externalagent":["partial","*",11,0],"cohere-ai":["partial","*",11,0],"Diffbot":["partial","*",11,0],"ContextFetchBot":["partial","*",11,0]},"llms":{"status":"missing","http":404,"bytes":507},"home":{"status":"ok","http":200,"url":"https://www.rbc.ua/","html_tokens":63496,"markdown_tokens":178,"pruned_tokens":151}},
{"rank":6063,"domain":"theweathernetwork.com","origin":"https://theweathernetwork.com","robots":{"status":"ok","http":200,"url":"https://www.theweathernetwork.com/robots.txt","bytes":327,"groups":7},"bots":{"GPTBot":["partial","*",3,2],"ChatGPT-User":["partial","*",3,2],"OAI-SearchBot":["partial","*",3,2],"ClaudeBot":["partial","*",3,2],"Claude-User":["partial","*",3,2],"Claude-SearchBot":["partial","*",3,2],"anthropic-ai":["partial","*",3,2],"PerplexityBot":["partial","*",3,2],"Perplexity-User":["partial","*",3,2],"Google-Extended":["partial","*",3,2],"Applebot-Extended":["partial","*",3,2],"CCBot":["partial","*",3,2],"Bytespider":["partial","*",3,2],"Amazonbot":["partial","*",3,2],"meta-externalagent":["partial","*",3,2],"cohere-ai":["partial","*",3,2],"Diffbot":["partial","*",3,2],"ContextFetchBot":["partial","*",3,2]},"llms":{"status":"present","http":200,"bytes":8316},"home":{"status":"ok","http":200,"url":"https://www.theweathernetwork.com/en","html_tokens":95446,"markdown_tokens":24,"pruned_tokens":24}},
{"rank":6064,"domain":"kyodonews.net","origin":"https://www.kyodonews.net","robots":{"status":"ok","http":200,"url":"https://english.kyodonews.net/robots.txt","bytes":1464,"groups":34},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["blocked","named",1,0],"OAI-SearchBot":["blocked","named",1,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["blocked","named",1,0],"Claude-SearchBot":["blocked","named",1,0],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["blocked","named",1,0],"Google-Extended":["partial","named",1,2],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["partial","named",1,1],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",4,0]},"llms":{"status":"missing","http":404,"bytes":15295},"home":{"status":"ok","http":200,"url":"https://english.kyodonews.net/","html_tokens":18732,"markdown_tokens":217,"pruned_tokens":123}},
{"rank":6065,"domain":"dailywire.com","origin":"https://dailywire.com","robots":{"status":"html","http":200,"url":"https://dailywire.com/robots.txt","bytes":693,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":6066,"domain":"ltn.com.tw","origin":"https://ltn.com.tw","robots":{"status":"ok","http":200,"url":"https://ltn.com.tw/robots.txt","bytes":74,"groups":2},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"html","http":200,"bytes":0},"home":{"status":"error","http":200,"url":"https://ltn.com.tw/","error":"unparseable HTML"}},
{"rank":6067,"domain":"vivaldi.com","origin":"https://vivaldi.com","robots":{"status":"ok","http":200,"url":"https://vivaldi.com/robots.txt","bytes":195,"groups":2},"bots":{"GPTBot":["partial","*",3,0],"ChatGPT-User":["partial","*",3,0],"OAI-SearchBot":["partial","*",3,0],"ClaudeBot":["partial","*",3,0],"Claude-User":["partial","*",3,0],"Claude-SearchBot":["partial","*",3,0],"anthropic-ai":["partial","*",3,0],"PerplexityBot":["partial","*",3,0],"Perplexity-User":["partial","*",3,0],"Google-Extended":["partial","*",3,0],"Applebot-Extended":["partial","*",3,0],"CCBot":["partial","*",3,0],"Bytespider":["partial","*",3,0],"Amazonbot":["partial","*",3,0],"meta-externalagent":["partial","*",3,0],"cohere-ai":["partial","*",3,0],"Diffbot":["partial","*",3,0],"ContextFetchBot":["partial","*",3,0]},"llms":{"status":"missing","http":404,"bytes":52679},"home":{"status":"ok","http":200,"url":"https://vivaldi.com/","html_tokens":23635,"markdown_tokens":422,"pruned_tokens":288}},
{"rank":6068,"domain":"uni-mannheim.de","origin":"https://www.uni-mannheim.de","robots":{"status":"ok","http":200,"url":"https://www.uni-mannheim.de/robots.txt","bytes":284,"groups":1},"bots":{"GPTBot":["partial","*",5,0],"ChatGPT-User":["partial","*",5,0],"OAI-SearchBot":["partial","*",5,0],"ClaudeBot":["partial","*",5,0],"Claude-User":["partial","*",5,0],"Claude-SearchBot":["partial","*",5,0],"anthropic-ai":["partial","*",5,0],"PerplexityBot":["partial","*",5,0],"Perplexity-User":["partial","*",5,0],"Google-Extended":["partial","*",5,0],"Applebot-Extended":["partial","*",5,0],"CCBot":["partial","*",5,0],"Bytespider":["partial","*",5,0],"Amazonbot":["partial","*",5,0],"meta-externalagent":["partial","*",5,0],"cohere-ai":["partial","*",5,0],"Diffbot":["partial","*",5,0],"ContextFetchBot":["partial","*",5,0]},"llms":{"status":"missing","http":404,"bytes":108324},"home":{"status":"ok","http":200,"url":"https://www.uni-mannheim.de/","html_tokens":33749,"markdown_tokens":105,"pruned_tokens":48}},
{"rank":6069,"domain":"rightmove.co.uk","origin":"https://rightmove.co.uk","robots":{"status":"ok","http":200,"url":"https://www.rightmove.co.uk/robots.txt","bytes":4823,"groups":11},"bots":{"GPTBot":["partial","named",2,4],"ChatGPT-User":["partial","*",93,0],"OAI-SearchBot":["partial","*",93,0],"ClaudeBot":["partial","*",93,0],"Claude-User":["partial","*",93,0],"Claude-SearchBot":["partial","*",93,0],"anthropic-ai":["partial","*",93,0],"PerplexityBot":["partial","*",93,0],"Perplexity-User":["partial","*",93,0],"Google-Extended":["partial","*",93,0],"Applebot-Extended":["partial","*",93,0],"CCBot":["blocked","named",1,0],"Bytespider":["partial","*",93,0],"Amazonbot":["partial","*",93,0],"meta-externalagent":["partial","*",93,0],"cohere-ai":["partial","*",93,0],"Diffbot":["partial","*",93,0],"ContextFetchBot":["partial","*",93,0]},"llms":{"status":"html","http":200,"bytes":158505},"home":{"status":"ok","http":200,"url":"https://www.rightmove.co.uk/","html_tokens":85060,"markdown_tokens":18,"pruned_tokens":18}},
{"rank":6070,"domain":"i24slot.org","origin":"https://i24slot.org","robots":{"status":"ok","http":200,"url":"https://i24slot.org/robots.txt","bytes":67,"groups":1},"bots":{"GPTBot":["partial","*",1,1],"ChatGPT-User":["partial","*",1,1],"OAI-SearchBot":["partial","*",1,1],"ClaudeBot":["partial","*",1,1],"Claude-User":["partial","*",1,1],"Claude-SearchBot":["partial","*",1,1],"anthropic-ai":["partial","*",1,1],"PerplexityBot":["partial","*",1,1],"Perplexity-User":["partial","*",1,1],"Google-Extended":["partial","*",1,1],"Applebot-Extended":["partial","*",1,1],"CCBot":["partial","*",1,1],"Bytespider":["partial","*",1,1],"Amazonbot":["partial","*",1,1],"meta-externalagent":["partial","*",1,1],"cohere-ai":["partial","*",1,1],"Diffbot":["partial","*",1,1],"ContextFetchBot":["partial","*",1,1]},"llms":{"status":"missing","http":404,"bytes":49399},"home":{"status":"ok","http":200,"url":"https://i24slot.org/","html_tokens":16823,"markdown_tokens":1894,"pruned_tokens":1387}},
{"rank":6071,"domain":"hometalk.com","origin":"https://hometalk.com","robots":{"status":"refused","http":406,"url":"https://www.hometalk.com/robots.txt","bytes":425,"groups":0,"error":"HTTP 406"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":6072,"domain":"al.us","origin":"https://al.us","robots":{"status":"unreachable","http":null,"url":"https://al.us/robots.txt","bytes":0,"groups":0,"error":"cannot resolve al.us"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":6073,"domain":"us-cert.gov","origin":"https://us-cert.gov","robots":{"status":"refused","http":403,"url":"https://www.cisa.gov/robots.txt","bytes":380,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":6074,"domain":"geonames.org","origin":"https://geonames.org","robots":{"status":"ok","http":200,"url":"http://www.geonames.org/robots.txt","bytes":198,"groups":2},"bots":{"GPTBot":["partial","*",6,0],"ChatGPT-User":["partial","*",6,0],"OAI-SearchBot":["partial","*",6,0],"ClaudeBot":["partial","*",6,0],"Claude-User":["partial","*",6,0],"Claude-SearchBot":["partial","*",6,0],"anthropic-ai":["partial","*",6,0],"PerplexityBot":["partial","*",6,0],"Perplexity-User":["partial","*",6,0],"Google-Extended":["partial","*",6,0],"Applebot-Extended":["partial","*",6,0],"CCBot":["partial","*",6,0],"Bytespider":["partial","*",6,0],"Amazonbot":["partial","*",6,0],"meta-externalagent":["partial","*",6,0],"cohere-ai":["partial","*",6,0],"Diffbot":["partial","*",6,0],"ContextFetchBot":["partial","*",6,0]},"llms":{"status":"unreachable","error":"Server disconnected without sending a response."},"home":{"status":"ok","http":200,"url":"http://www.geonames.org/","html_tokens":3707,"markdown_tokens":56,"pruned_tokens":56}},
{"rank":6075,"domain":"therealdeal.com","origin":"https://therealdeal.com","robots":{"status":"refused","http":403,"url":"https://therealdeal.com/robots.txt","bytes":14911,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":6076,"domain":"chaoxing.com","origin":"https://chaoxing.com","robots":{"status":"missing","http":404,"url":"https://www.chaoxing.com/robots.txt","bytes":200,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":200},"home":{"status":"ok","http":200,"url":"https://www.chaoxing.com/","html_tokens":143735,"markdown_tokens":27,"pruned_tokens":27}},
{"rank":6077,"domain":"mfa.gov.tr","origin":"https://mfa.gov.tr","robots":{"status":"ok","http":200,"url":"https://www.mfa.gov.tr/robots.txt","bytes":107,"groups":1},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"html","http":200,"bytes":34566},"home":{"status":"ok","http":200,"url":"https://www.mfa.gov.tr/default.en.mfa","html_tokens":16201,"markdown_tokens":85,"pruned_tokens":85}},
{"rank":6078,"domain":"amu.edu.pl","origin":"https://amu.edu.pl","robots":{"status":"ok","http":200,"url":"https://amu.edu.pl/robots.txt","bytes":723,"groups":4},"bots":{"GPTBot":["partial","*",22,0],"ChatGPT-User":["partial","*",22,0],"OAI-SearchBot":["partial","*",22,0],"ClaudeBot":["partial","*",22,0],"Claude-User":["partial","*",22,0],"Claude-SearchBot":["partial","*",22,0],"anthropic-ai":["partial","*",22,0],"PerplexityBot":["partial","*",22,0],"Perplexity-User":["partial","*",22,0],"Google-Extended":["partial","*",22,0],"Applebot-Extended":["partial","*",22,0],"CCBot":["partial","*",22,0],"Bytespider":["partial","*",22,0],"Amazonbot":["partial","*",22,0],"meta-externalagent":["partial","*",22,0],"cohere-ai":["partial","*",22,0],"Diffbot":["partial","*",22,0],"ContextFetchBot":["partial","*",22,0]},"llms":{"status":"missing","http":404,"bytes":8966},"home":{"status":"ok","http":200,"url":"https://amu.edu.pl/","html_tokens":57897,"markdown_tokens":1108,"pruned_tokens":151}},
{"rank":6079,"domain":"humankinetics.com","origin":"https://humankinetics.com","robots":{"status":"html","http":200,"url":"https://us.humankinetics.com/","bytes":284308,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":6080,"domain":"www.gov.za","origin":"https://www.gov.za","robots":{"status":"ok","http":200,"url":"https://www.gov.za/robots.txt","bytes":1706,"groups":1},"bots":{"GPTBot":["partial","*",26,18],"ChatGPT-User":["partial","*",26,18],"OAI-SearchBot":["partial","*",26,18],"ClaudeBot":["partial","*",26,18],"Claude-User":["partial","*",26,18],"Claude-SearchBot":["partial","*",26,18],"anthropic-ai":["partial","*",26,18],"PerplexityBot":["partial","*",26,18],"Perplexity-User":["partial","*",26,18],"Google-Extended":["partial","*",26,18],"Applebot-Extended":["partial","*",26,18],"CCBot":["partial","*",26,18],"Bytespider":["partial","*",26,18],"Amazonbot":["partial","*",26,18],"meta-externalagent":["partial","*",26,18],"cohere-ai":["partial","*",26,18],"Diffbot":["partial","*",26,18],"ContextFetchBot":["partial","*",26,18]},"llms":{"status":"http_error","http":403,"bytes":146},"home":{"status":"ok","http":200,"url":"https://www.gov.za/","html_tokens":13341,"markdown_tokens":684,"pruned_tokens":149}},
{"rank":6081,"domain":"base.shop","origin":"https://www.base.shop","robots":{"status":"ok","http":200,"url":"https://www.base.shop/robots.txt","bytes":807,"groups":3},"bots":{"GPTBot":["partial","*",6,1],"ChatGPT-User":["partial","*",6,1],"OAI-SearchBot":["partial","*",6,1],"ClaudeBot":["partial","*",6,1],"Claude-User":["partial","*",6,1],"Claude-SearchBot":["partial","*",6,1],"anthropic-ai":["partial","*",6,1],"PerplexityBot":["partial","*",6,1],"Perplexity-User":["partial","*",6,1],"Google-Extended":["partial","*",6,1],"Applebot-Extended":["partial","*",6,1],"CCBot":["partial","*",6,1],"Bytespider":["partial","*",6,1],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["partial","*",6,1],"cohere-ai":["partial","*",6,1],"Diffbot":["partial","*",6,1],"ContextFetchBot":["partial","*",6,1]},"llms":{"status":"missing","http":404,"bytes":59473},"home":{"status":"http_error","http":404,"url":"https://www.base.shop/"}},
{"rank":6082,"domain":"rstudio.com","origin":"https://rstudio.com","robots":{"status":"html","http":200,"url":"https://posit.co/","bytes":304209,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":6083,"domain":"globes.co.il","origin":"https://globes.co.il","robots":{"status":"ok","http":200,"url":"https://www.globes.co.il/robots.txt","bytes":631,"groups":2},"bots":{"GPTBot":["partial","*",9,1],"ChatGPT-User":["partial","*",9,1],"OAI-SearchBot":["partial","*",9,1],"ClaudeBot":["partial","*",9,1],"Claude-User":["partial","*",9,1],"Claude-SearchBot":["partial","*",9,1],"anthropic-ai":["partial","*",9,1],"PerplexityBot":["partial","*",9,1],"Perplexity-User":["partial","*",9,1],"Google-Extended":["partial","*",9,1],"Applebot-Extended":["partial","*",9,1],"CCBot":["partial","*",9,1],"Bytespider":["partial","*",9,1],"Amazonbot":["partial","*",9,1],"meta-externalagent":["partial","*",9,1],"cohere-ai":["partial","*",9,1],"Diffbot":["partial","*",9,1],"ContextFetchBot":["partial","*",9,1]},"llms":{"status":"missing","http":404,"bytes":9138},"home":{"status":"ok","http":200,"url":"https://www.globes.co.il/","html_tokens":45956,"markdown_tokens":2023,"pruned_tokens":152}},
{"rank":6084,"domain":"welivesecurity.com","origin":"https://welivesecurity.com","robots":{"status":"refused","http":403,"url":"https://www.welivesecurity.com/robots.txt","bytes":266,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":6085,"domain":"portswigger.net","origin":"https://portswigger.net","robots":{"status":"ok","http":200,"url":"https://portswigger.net/robots.txt","bytes":149,"groups":1},"bots":{"GPTBot":["partial","*",1,0],"ChatGPT-User":["partial","*",1,0],"OAI-SearchBot":["partial","*",1,0],"ClaudeBot":["partial","*",1,0],"Claude-User":["partial","*",1,0],"Claude-SearchBot":["partial","*",1,0],"anthropic-ai":["partial","*",1,0],"PerplexityBot":["partial","*",1,0],"Perplexity-User":["partial","*",1,0],"Google-Extended":["partial","*",1,0],"Applebot-Extended":["partial","*",1,0],"CCBot":["partial","*",1,0],"Bytespider":["partial","*",1,0],"Amazonbot":["partial","*",1,0],"meta-externalagent":["partial","*",1,0],"cohere-ai":["partial","*",1,0],"Diffbot":["partial","*",1,0],"ContextFetchBot":["partial","*",1,0]},"llms":{"status":"missing","http":404,"bytes":23542},"home":{"status":"ok","http":200,"url":"https://portswigger.net/","html_tokens":20660,"markdown_tokens":224,"pruned_tokens":45}},
{"rank":6086,"domain":"sc.gov.cn","origin":"https://sc.gov.cn","robots":{"status":"unreachable","http":null,"url":"https://sc.gov.cn/robots.txt","bytes":0,"groups":0,"error":"cannot resolve sc.gov.cn"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":6087,"domain":"jiemian.com","origin":"https://jiemian.com","robots":{"status":"ok","http":200,"url":"https://www.jiemian.com/robots.txt","bytes":52,"groups":1},"bots":{"GPTBot":["partial","*",2,0],"ChatGPT-User":["partial","*",2,0],"OAI-SearchBot":["partial","*",2,0],"ClaudeBot":["partial","*",2,0],"Claude-User":["partial","*",2,0],"Claude-SearchBot":["partial","*",2,0],"anthropic-ai":["partial","*",2,0],"PerplexityBot":["partial","*",2,0],"Perplexity-User":["partial","*",2,0],"Google-Extended":["partial","*",2,0],"Applebot-Extended":["partial","*",2,0],"CCBot":["partial","*",2,0],"Bytespider":["partial","*",2,0],"Amazonbot":["partial","*",2,0],"meta-externalagent":["partial","*",2,0],"cohere-ai":["partial","*",2,0],"Diffbot":["partial","*",2,0],"ContextFetchBot":["partial","*",2,0]},"llms":{"status":"missing","http":404,"bytes":1170},"home":{"status":"ok","http":200,"url":"https://www.jiemian.com/","html_tokens":34653,"markdown_tokens":26,"pruned_tokens":26}},
{"rank":6088,"domain":"couriermail.com.au","origin":"https://couriermail.com.au","robots":{"status":"ok","http":200,"url":"https://www.couriermail.com.au/robots.txt","bytes":4204,"groups":4},"bots":{"GPTBot":["partial","*",1,2],"ChatGPT-User":["partial","*",1,2],"OAI-SearchBot":["partial","*",1,2],"ClaudeBot":["partial","*",1,2],"Claude-User":["partial","*",1,2],"Claude-SearchBot":["partial","*",1,2],"anthropic-ai":["partial","*",1,2],"PerplexityBot":["partial","*",1,2],"Perplexity-User":["partial","*",1,2],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["partial","*",1,2],"CCBot":["partial","*",1,2],"Bytespider":["partial","*",1,2],"Amazonbot":["partial","*",1,2],"meta-externalagent":["partial","*",1,2],"cohere-ai":["partial","*",1,2],"Diffbot":["partial","*",1,2],"ContextFetchBot":["partial","*",1,2]},"llms":{"status":"disallowed"},"home":{"status":"disallowed"}},
{"rank":6089,"domain":"leonardo.ai","origin":"https://leonardo.ai","robots":{"status":"ok","http":200,"url":"https://leonardo.ai/robots.txt","bytes":270,"groups":1},"bots":{"GPTBot":["partial","*",9,1],"ChatGPT-User":["partial","*",9,1],"OAI-SearchBot":["partial","*",9,1],"ClaudeBot":["partial","*",9,1],"Claude-User":["partial","*",9,1],"Claude-SearchBot":["partial","*",9,1],"anthropic-ai":["partial","*",9,1],"PerplexityBot":["partial","*",9,1],"Perplexity-User":["partial","*",9,1],"Google-Extended":["partial","*",9,1],"Applebot-Extended":["partial","*",9,1],"CCBot":["partial","*",9,1],"Bytespider":["partial","*",9,1],"Amazonbot":["partial","*",9,1],"meta-externalagent":["partial","*",9,1],"cohere-ai":["partial","*",9,1],"Diffbot":["partial","*",9,1],"ContextFetchBot":["partial","*",9,1]},"llms":{"status":"missing","http":404,"bytes":125934},"home":{"status":"http_error","http":403,"url":"https://leonardo.ai/"}},
{"rank":6090,"domain":"askmen.com","origin":"https://askmen.com","robots":{"status":"ok","http":200,"url":"https://www.askmen.com/robots.txt","bytes":6175,"groups":3},"bots":{"GPTBot":["blocked","named",2,0],"ChatGPT-User":["blocked","named",2,0],"OAI-SearchBot":["blocked","named",2,0],"ClaudeBot":["blocked","named",2,0],"Claude-User":["blocked","named",1,0],"Claude-SearchBot":["blocked","named",1,0],"anthropic-ai":["blocked","named",2,0],"PerplexityBot":["blocked","named",2,0],"Perplexity-User":["blocked","named",2,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["blocked","named",2,0],"CCBot":["blocked","named",2,0],"Bytespider":["blocked","named",2,0],"Amazonbot":["blocked","named",2,0],"meta-externalagent":["blocked","named",2,0],"cohere-ai":["blocked","named",2,0],"Diffbot":["blocked","named",2,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"missing","http":410,"bytes":55610},"home":{"status":"ok","http":200,"url":"https://www.askmen.com/","html_tokens":20079,"markdown_tokens":18,"pruned_tokens":18}},
{"rank":6091,"domain":"slu.edu","origin":"https://slu.edu","robots":{"status":"ok","http":200,"url":"https://www.slu.edu/robots.txt","bytes":351,"groups":2},"bots":{"GPTBot":["partial","*",8,0],"ChatGPT-User":["partial","*",8,0],"OAI-SearchBot":["partial","*",8,0],"ClaudeBot":["partial","*",8,0],"Claude-User":["partial","*",8,0],"Claude-SearchBot":["partial","*",8,0],"anthropic-ai":["partial","*",8,0],"PerplexityBot":["partial","*",8,0],"Perplexity-User":["partial","*",8,0],"Google-Extended":["partial","*",8,0],"Applebot-Extended":["partial","*",8,0],"CCBot":["partial","*",8,0],"Bytespider":["partial","*",8,0],"Amazonbot":["partial","*",8,0],"meta-externalagent":["partial","*",8,0],"cohere-ai":["partial","*",8,0],"Diffbot":["partial","*",8,0],"ContextFetchBot":["partial","*",8,0]},"llms":{"status":"missing","http":404,"bytes":121510},"home":{"status":"ok","http":200,"url":"https://www.slu.edu/","html_tokens":36828,"markdown_tokens":247,"pruned_tokens":199}},
{"rank":6092,"domain":"siriusxm.com","origin":"https://siriusxm.com","robots":{"status":"ok","http":200,"url":"https://www.siriusxm.com/robots.txt","bytes":222,"groups":2},"bots":{"GPTBot":["partial","*",1,3],"ChatGPT-User":["partial","*",1,3],"OAI-SearchBot":["partial","*",1,3],"ClaudeBot":["partial","*",1,3],"Claude-User":["partial","*",1,3],"Claude-SearchBot":["partial","*",1,3],"anthropic-ai":["partial","*",1,3],"PerplexityBot":["partial","*",1,3],"Perplexity-User":["partial","*",1,3],"Google-Extended":["partial","*",1,3],"Applebot-Extended":["partial","*",1,3],"CCBot":["partial","*",1,3],"Bytespider":["partial","*",1,3],"Amazonbot":["partial","*",1,3],"meta-externalagent":["partial","*",1,3],"cohere-ai":["partial","*",1,3],"Diffbot":["partial","*",1,3],"ContextFetchBot":["partial","*",1,3]},"llms":{"status":"missing","http":404,"bytes":13746},"home":{"status":"ok","http":200,"url":"https://www.siriusxm.com/","html_tokens":96331,"markdown_tokens":628,"pruned_tokens":77}},
{"rank":6093,"domain":"songkick.com","origin":"https://songkick.com","robots":{"status":"ok","http":200,"url":"https://songkick.com/robots.txt","bytes":5665,"groups":30},"bots":{"GPTBot":["partial","named",126,1],"ChatGPT-User":["partial","named",126,1],"OAI-SearchBot":["partial","named",126,1],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",126,1],"Claude-SearchBot":["partial","*",126,1],"anthropic-ai":["partial","*",126,1],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["partial","*",126,1],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["partial","*",126,1],"CCBot":["blocked","named",2,0],"Bytespider":["partial","*",126,1],"Amazonbot":["partial","named",126,1],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["partial","*",126,1],"Diffbot":["partial","*",126,1],"ContextFetchBot":["partial","*",126,1]},"llms":{"status":"missing","http":404,"bytes":40319},"home":{"status":"ok","http":200,"url":"https://songkick.com/","html_tokens":12210,"markdown_tokens":22,"pruned_tokens":22}},
{"rank":6094,"domain":"digitimes.com","origin":"https://digitimes.com","robots":{"status":"ok","http":200,"url":"https://www.digitimes.com/robots.txt","bytes":470,"groups":3},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["partial","*",15,0],"OAI-SearchBot":["partial","*",15,0],"ClaudeBot":["partial","*",15,0],"Claude-User":["partial","*",15,0],"Claude-SearchBot":["partial","*",15,0],"anthropic-ai":["partial","*",15,0],"PerplexityBot":["partial","*",15,0],"Perplexity-User":["partial","*",15,0],"Google-Extended":["partial","*",15,0],"Applebot-Extended":["partial","*",15,0],"CCBot":["partial","*",15,0],"Bytespider":["partial","*",15,0],"Amazonbot":["partial","*",15,0],"meta-externalagent":["partial","*",15,0],"cohere-ai":["partial","*",15,0],"Diffbot":["partial","*",15,0],"ContextFetchBot":["partial","*",15,0]},"llms":{"status":"html","http":200,"bytes":29249},"home":{"status":"ok","http":200,"url":"https://www.digitimes.com/","html_tokens":28971,"markdown_tokens":58,"pruned_tokens":58}},
{"rank":6095,"domain":"merckgroup.com","origin":"https://merckgroup.com","robots":{"status":"refused","http":403,"url":"https://www.merckgroup.com/robots.txt","bytes":386,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":6096,"domain":"precedenceresearch.com","origin":"https://precedenceresearch.com","robots":{"status":"ok","http":200,"url":"https://www.precedenceresearch.com/robots.txt","bytes":325,"groups":1},"bots":{"GPTBot":["partial","*",7,0],"ChatGPT-User":["partial","*",7,0],"OAI-SearchBot":["partial","*",7,0],"ClaudeBot":["partial","*",7,0],"Claude-User":["partial","*",7,0],"Claude-SearchBot":["partial","*",7,0],"anthropic-ai":["partial","*",7,0],"PerplexityBot":["partial","*",7,0],"Perplexity-User":["partial","*",7,0],"Google-Extended":["partial","*",7,0],"Applebot-Extended":["partial","*",7,0],"CCBot":["partial","*",7,0],"Bytespider":["partial","*",7,0],"Amazonbot":["partial","*",7,0],"meta-externalagent":["partial","*",7,0],"cohere-ai":["partial","*",7,0],"Diffbot":["partial","*",7,0],"ContextFetchBot":["partial","*",7,0]},"llms":{"status":"present","http":200,"bytes":12286},"home":{"status":"ok","http":200,"url":"https://www.precedenceresearch.com/","html_tokens":29477,"markdown_tokens":1876,"pruned_tokens":116}},
{"rank":6097,"domain":"google.com.ng","origin":"https://google.com.ng","robots":{"status":"ok","http":200,"url":"https://www.google.com.ng/robots.txt","bytes":6624,"groups":4},"bots":{"GPTBot":["partial","*",169,64],"ChatGPT-User":["partial","*",169,64],"OAI-SearchBot":["partial","*",169,64],"ClaudeBot":["partial","*",169,64],"Claude-User":["partial","*",169,64],"Claude-SearchBot":["partial","*",169,64],"anthropic-ai":["partial","*",169,64],"PerplexityBot":["partial","*",169,64],"Perplexity-User":["partial","*",169,64],"Google-Extended":["partial","*",169,64],"Applebot-Extended":["partial","*",169,64],"CCBot":["partial","*",169,64],"Bytespider":["partial","*",169,64],"Amazonbot":["partial","*",169,64],"meta-externalagent":["partial","*",169,64],"cohere-ai":["partial","*",169,64],"Diffbot":["partial","*",169,64],"ContextFetchBot":["partial","*",169,64]},"llms":{"status":"missing","http":404,"bytes":1569},"home":{"status":"ok","http":200,"url":"https://www.google.com/","html_tokens":21263,"markdown_tokens":30,"pruned_tokens":30}},
{"rank":6098,"domain":"madison.com","origin":"https://madison.com","robots":{"status":"ok","http":200,"url":"https://madison.com/robots.txt","bytes":20266,"groups":31},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["blocked","named",1,0],"OAI-SearchBot":["blocked","named",1,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",271,0],"Claude-SearchBot":["partial","*",271,0],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["partial","*",271,0],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["partial","*",271,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",271,0]},"llms":{"status":"http_error","http":429,"bytes":64},"home":{"status":"ok","http":200,"url":"https://madison.com/","html_tokens":138329,"markdown_tokens":1081,"pruned_tokens":112}},
{"rank":6099,"domain":"csrc.gov.cn","origin":"https://www.csrc.gov.cn","robots":{"status":"html","http":200,"url":"https://www.csrc.gov.cn/","bytes":222019,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":6100,"domain":"sedaily.com","origin":"https://sedaily.com","robots":{"status":"ok","http":200,"url":"https://www.sedaily.com/robots.txt","bytes":897,"groups":3},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["partial","named",4,1],"OAI-SearchBot":["partial","named",4,1],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","named",4,1],"Claude-SearchBot":["partial","named",4,1],"anthropic-ai":["partial","*",4,0],"PerplexityBot":["partial","named",4,1],"Perplexity-User":["partial","named",4,1],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["partial","*",4,0],"Diffbot":["partial","*",4,0],"ContextFetchBot":["partial","*",4,0]},"llms":{"status":"missing","http":404,"bytes":41922},"home":{"status":"ok","http":200,"url":"https://www.sedaily.com/","html_tokens":44348,"markdown_tokens":442,"pruned_tokens":167}},
{"rank":6101,"domain":"gamer.com.tw","origin":"https://gamer.com.tw","robots":{"status":"ok","http":200,"url":"https://www.gamer.com.tw/robots.txt","bytes":25,"groups":1},"bots":{"GPTBot":["allowed","*",0,1],"ChatGPT-User":["allowed","*",0,1],"OAI-SearchBot":["allowed","*",0,1],"ClaudeBot":["allowed","*",0,1],"Claude-User":["allowed","*",0,1],"Claude-SearchBot":["allowed","*",0,1],"anthropic-ai":["allowed","*",0,1],"PerplexityBot":["allowed","*",0,1],"Perplexity-User":["allowed","*",0,1],"Google-Extended":["allowed","*",0,1],"Applebot-Extended":["allowed","*",0,1],"CCBot":["allowed","*",0,1],"Bytespider":["allowed","*",0,1],"Amazonbot":["allowed","*",0,1],"meta-externalagent":["allowed","*",0,1],"cohere-ai":["allowed","*",0,1],"Diffbot":["allowed","*",0,1],"ContextFetchBot":["allowed","*",0,1]},"llms":{"status":"html","http":200,"bytes":2130},"home":{"status":"ok","http":200,"url":"https://www.gamer.com.tw/","html_tokens":133499,"markdown_tokens":25,"pruned_tokens":25}},
{"rank":6102,"domain":"akamaized.net","origin":"https://akamaized.net","robots":{"status":"unreachable","http":null,"url":"https://akamaized.net/robots.txt","bytes":0,"groups":0,"error":"cannot resolve akamaized.net"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":6103,"domain":"sk.ca","origin":"https://sk.ca","robots":{"status":"unreachable","http":null,"url":"https://sk.ca/robots.txt","bytes":0,"groups":0,"error":"cannot resolve sk.ca"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":6104,"domain":"ucanr.edu","origin":"https://ucanr.edu","robots":{"status":"refused","http":403,"url":"https://ucanr.edu/robots.txt","bytes":560,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":6105,"domain":"publicradio.org","origin":"https://publicradio.org","robots":{"status":"unreachable","http":null,"url":"https://publicradio.org/robots.txt","bytes":0,"groups":0,"error":"cannot resolve www.publicradio.org"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":6106,"domain":"crisisgroup.org","origin":"https://crisisgroup.org","robots":{"status":"refused","http":403,"url":"https://crisisgroup.org/robots.txt","bytes":4909,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":6107,"domain":"grants.gov","origin":"https://grants.gov","robots":{"status":"ok","http":200,"url":"https://grants.gov/robots.txt","bytes":22,"groups":1},"bots":{"GPTBot":["allowed","*",0,1],"ChatGPT-User":["allowed","*",0,1],"OAI-SearchBot":["allowed","*",0,1],"ClaudeBot":["allowed","*",0,1],"Claude-User":["allowed","*",0,1],"Claude-SearchBot":["allowed","*",0,1],"anthropic-ai":["allowed","*",0,1],"PerplexityBot":["allowed","*",0,1],"Perplexity-User":["allowed","*",0,1],"Google-Extended":["allowed","*",0,1],"Applebot-Extended":["allowed","*",0,1],"CCBot":["allowed","*",0,1],"Bytespider":["allowed","*",0,1],"Amazonbot":["allowed","*",0,1],"meta-externalagent":["allowed","*",0,1],"cohere-ai":["allowed","*",0,1],"Diffbot":["allowed","*",0,1],"ContextFetchBot":["allowed","*",0,1]},"llms":{"status":"html","http":200,"bytes":27116},"home":{"status":"ok","http":200,"url":"https://grants.gov/","html_tokens":15562,"markdown_tokens":353,"pruned_tokens":212}},
{"rank":6108,"domain":"box.net","origin":"https://box.net","robots":{"status":"ok","http":200,"url":"https://box.net/robots.txt","bytes":2187,"groups":2},"bots":{"GPTBot":["partial","*",2,20],"ChatGPT-User":["partial","*",2,20],"OAI-SearchBot":["partial","*",2,20],"ClaudeBot":["partial","*",2,20],"Claude-User":["partial","*",2,20],"Claude-SearchBot":["partial","*",2,20],"anthropic-ai":["partial","*",2,20],"PerplexityBot":["partial","*",2,20],"Perplexity-User":["partial","*",2,20],"Google-Extended":["partial","*",2,20],"Applebot-Extended":["partial","*",2,20],"CCBot":["partial","*",2,20],"Bytespider":["partial","*",2,20],"Amazonbot":["partial","*",2,20],"meta-externalagent":["partial","*",2,20],"cohere-ai":["partial","*",2,20],"Diffbot":["partial","*",2,20],"ContextFetchBot":["partial","*",2,20]},"llms":{"status":"disallowed"},"home":{"status":"ok","http":200,"url":"https://www.box.com/","html_tokens":126998,"markdown_tokens":649,"pruned_tokens":88}},
{"rank":6109,"domain":"alzheimers.org.uk","origin":"https://alzheimers.org.uk","robots":{"status":"ok","http":200,"url":"https://www.alzheimers.org.uk/robots.txt","bytes":2445,"groups":1},"bots":{"GPTBot":["partial","*",48,18],"ChatGPT-User":["partial","*",48,18],"OAI-SearchBot":["partial","*",48,18],"ClaudeBot":["partial","*",48,18],"Claude-User":["partial","*",48,18],"Claude-SearchBot":["partial","*",48,18],"anthropic-ai":["partial","*",48,18],"PerplexityBot":["partial","*",48,18],"Perplexity-User":["partial","*",48,18],"Google-Extended":["partial","*",48,18],"Applebot-Extended":["partial","*",48,18],"CCBot":["partial","*",48,18],"Bytespider":["partial","*",48,18],"Amazonbot":["partial","*",48,18],"meta-externalagent":["partial","*",48,18],"cohere-ai":["partial","*",48,18],"Diffbot":["partial","*",48,18],"ContextFetchBot":["partial","*",48,18]},"llms":{"status":"http_error","http":403,"bytes":17},"home":{"status":"ok","http":200,"url":"https://www.alzheimers.org.uk/","html_tokens":54858,"markdown_tokens":55,"pruned_tokens":55}},
{"rank":6110,"domain":"sundance.org","origin":"https://sundance.org","robots":{"status":"refused","http":403,"url":"https://www.sundance.org/robots.txt","bytes":560,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":6111,"domain":"fiocruz.br","origin":"https://fiocruz.br","robots":{"status":"ok","http":200,"url":"https://fiocruz.br/robots.txt","bytes":2141,"groups":1},"bots":{"GPTBot":["partial","*",35,22],"ChatGPT-User":["partial","*",35,22],"OAI-SearchBot":["partial","*",35,22],"ClaudeBot":["partial","*",35,22],"Claude-User":["partial","*",35,22],"Claude-SearchBot":["partial","*",35,22],"anthropic-ai":["partial","*",35,22],"PerplexityBot":["partial","*",35,22],"Perplexity-User":["partial","*",35,22],"Google-Extended":["partial","*",35,22],"Applebot-Extended":["partial","*",35,22],"CCBot":["partial","*",35,22],"Bytespider":["partial","*",35,22],"Amazonbot":["partial","*",35,22],"meta-externalagent":["partial","*",35,22],"cohere-ai":["partial","*",35,22],"Diffbot":["partial","*",35,22],"ContextFetchBot":["partial","*",35,22]},"llms":{"status":"http_error","http":403,"bytes":146},"home":{"status":"ok","http":200,"url":"https://fiocruz.br/","html_tokens":55581,"markdown_tokens":159,"pruned_tokens":105}},
{"rank":6112,"domain":"parker.com","origin":"https://parker.com","robots":{"status":"refused","http":403,"url":"https://www.parker.com/robots.txt","bytes":382,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":6113,"domain":"manager-magazin.de","origin":"https://manager-magazin.de","robots":{"status":"ok","http":200,"url":"https://www.manager-magazin.de/robots.txt","bytes":5581,"groups":111},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["blocked","named",1,0],"OAI-SearchBot":["blocked","named",1,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["blocked","named",1,0],"Claude-SearchBot":["blocked","named",1,0],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["partial","*",1,1],"Perplexity-User":["partial","*",1,1],"Google-Extended":["partial","*",1,1],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["partial","*",1,1],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",1,1]},"llms":{"status":"missing","http":404,"bytes":20909},"home":{"status":"ok","http":200,"url":"https://www.manager-magazin.de/","html_tokens":273711,"markdown_tokens":305,"pruned_tokens":49}},
{"rank":6114,"domain":"calgaryherald.com","origin":"https://calgaryherald.com","robots":{"status":"ok","http":200,"url":"https://calgaryherald.com/robots.txt","bytes":1233,"groups":15},"bots":{"GPTBot":["partial","*",3,6],"ChatGPT-User":["partial","*",3,6],"OAI-SearchBot":["partial","*",3,6],"ClaudeBot":["partial","*",3,6],"Claude-User":["partial","*",3,6],"Claude-SearchBot":["partial","*",3,6],"anthropic-ai":["partial","*",3,6],"PerplexityBot":["partial","*",3,6],"Perplexity-User":["partial","*",3,6],"Google-Extended":["partial","*",3,6],"Applebot-Extended":["partial","*",3,6],"CCBot":["partial","*",3,6],"Bytespider":["partial","*",3,6],"Amazonbot":["partial","*",3,6],"meta-externalagent":["partial","*",3,6],"cohere-ai":["partial","*",3,6],"Diffbot":["partial","*",3,6],"ContextFetchBot":["partial","*",3,6]},"llms":{"status":"missing","http":404,"bytes":307781},"home":{"status":"ok","http":200,"url":"https://calgaryherald.com/","html_tokens":141198,"markdown_tokens":122,"pruned_tokens":122}},
{"rank":6115,"domain":"groupe-sncf.com","origin":"https://groupe-sncf.com","robots":{"status":"refused","http":403,"url":"https://www.groupe-sncf.com/robots.txt","bytes":775,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":6116,"domain":"bafin.de","origin":"https://bafin.de","robots":{"status":"ok","http":200,"url":"https://bafin.de/robots.txt","bytes":281,"groups":1},"bots":{"GPTBot":["partial","*",6,3],"ChatGPT-User":["partial","*",6,3],"OAI-SearchBot":["partial","*",6,3],"ClaudeBot":["partial","*",6,3],"Claude-User":["partial","*",6,3],"Claude-SearchBot":["partial","*",6,3],"anthropic-ai":["partial","*",6,3],"PerplexityBot":["partial","*",6,3],"Perplexity-User":["partial","*",6,3],"Google-Extended":["partial","*",6,3],"Applebot-Extended":["partial","*",6,3],"CCBot":["partial","*",6,3],"Bytespider":["partial","*",6,3],"Amazonbot":["partial","*",6,3],"meta-externalagent":["partial","*",6,3],"cohere-ai":["partial","*",6,3],"Diffbot":["partial","*",6,3],"ContextFetchBot":["partial","*",6,3]},"llms":{"status":"missing","http":404,"bytes":62055},"home":{"status":"ok","http":200,"url":"https://www.bafin.de/DE/home_node.html","html_tokens":38071,"markdown_tokens":269,"pruned_tokens":149}},
{"rank":6117,"domain":"openpr.com","origin":"https://openpr.com","robots":{"status":"ok","http":200,"url":"https://www.openpr.com/robots.txt","bytes":1049,"groups":24},"bots":{"GPTBot":["partial","*",3,0],"ChatGPT-User":["partial","*",3,0],"OAI-SearchBot":["partial","*",3,0],"ClaudeBot":["allowed","named",0,0],"Claude-User":["partial","*",3,0],"Claude-SearchBot":["partial","*",3,0],"anthropic-ai":["partial","*",3,0],"PerplexityBot":["partial","*",3,0],"Perplexity-User":["partial","*",3,0],"Google-Extended":["partial","*",3,0],"Applebot-Extended":["partial","*",3,0],"CCBot":["partial","*",3,0],"Bytespider":["partial","*",3,0],"Amazonbot":["partial","*",3,0],"meta-externalagent":["partial","*",3,0],"cohere-ai":["partial","*",3,0],"Diffbot":["partial","*",3,0],"ContextFetchBot":["partial","*",3,0]},"llms":{"status":"missing","http":404,"bytes":301},"home":{"status":"ok","http":200,"url":"https://www.openpr.com/","html_tokens":68864,"markdown_tokens":10258,"pruned_tokens":131}},
{"rank":6118,"domain":"dundee.ac.uk","origin":"https://dundee.ac.uk","robots":{"status":"refused","http":403,"url":"https://www.dundee.ac.uk/robots.txt","bytes":5575,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":6119,"domain":"abs-cbn.com","origin":"https://abs-cbn.com","robots":{"status":"refused","http":403,"url":"https://abs-cbn.com/robots.txt","bytes":379,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":6120,"domain":"neurips.cc","origin":"https://neurips.cc","robots":{"status":"ok","http":200,"url":"https://neurips.cc/robots.txt","bytes":1038,"groups":10},"bots":{"GPTBot":["partial","named",1,0],"ChatGPT-User":["partial","named",1,0],"OAI-SearchBot":["partial","named",1,0],"ClaudeBot":["partial","*",8,0],"Claude-User":["partial","*",8,0],"Claude-SearchBot":["partial","*",8,0],"anthropic-ai":["partial","*",8,0],"PerplexityBot":["allowed","named",0,0],"Perplexity-User":["partial","*",8,0],"Google-Extended":["partial","*",8,0],"Applebot-Extended":["partial","*",8,0],"CCBot":["partial","*",8,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["allowed","named",0,0],"meta-externalagent":["blocked","named",2,0],"cohere-ai":["partial","*",8,0],"Diffbot":["partial","*",8,0],"ContextFetchBot":["partial","*",8,0]},"llms":{"status":"missing","http":404,"bytes":47680},"home":{"status":"ok","http":200,"url":"https://neurips.cc/","html_tokens":43320,"markdown_tokens":1053,"pruned_tokens":130}},
{"rank":6121,"domain":"rbth.com","origin":"https://rbth.com","robots":{"status":"ok","http":200,"url":"https://www.rbth.com/robots.txt","bytes":384,"groups":4},"bots":{"GPTBot":["partial","*",2,2],"ChatGPT-User":["partial","*",2,2],"OAI-SearchBot":["partial","*",2,2],"ClaudeBot":["partial","*",2,2],"Claude-User":["partial","*",2,2],"Claude-SearchBot":["partial","*",2,2],"anthropic-ai":["partial","*",2,2],"PerplexityBot":["partial","*",2,2],"Perplexity-User":["partial","*",2,2],"Google-Extended":["partial","*",2,2],"Applebot-Extended":["partial","*",2,2],"CCBot":["partial","*",2,2],"Bytespider":["partial","*",2,2],"Amazonbot":["partial","*",2,2],"meta-externalagent":["partial","*",2,2],"cohere-ai":["partial","*",2,2],"Diffbot":["partial","*",2,2],"ContextFetchBot":["partial","*",2,2]},"llms":{"status":"missing","http":404,"bytes":20544},"home":{"status":"ok","http":200,"url":"https://www.rbth.com/","html_tokens":8884,"markdown_tokens":157,"pruned_tokens":157}},
{"rank":6122,"domain":"350.org","origin":"https://350.org","robots":{"status":"ok","http":200,"url":"https://350.org/robots.txt","bytes":116,"groups":1},"bots":{"GPTBot":["partial","*",3,0],"ChatGPT-User":["partial","*",3,0],"OAI-SearchBot":["partial","*",3,0],"ClaudeBot":["partial","*",3,0],"Claude-User":["partial","*",3,0],"Claude-SearchBot":["partial","*",3,0],"anthropic-ai":["partial","*",3,0],"PerplexityBot":["partial","*",3,0],"Perplexity-User":["partial","*",3,0],"Google-Extended":["partial","*",3,0],"Applebot-Extended":["partial","*",3,0],"CCBot":["partial","*",3,0],"Bytespider":["partial","*",3,0],"Amazonbot":["partial","*",3,0],"meta-externalagent":["partial","*",3,0],"cohere-ai":["partial","*",3,0],"Diffbot":["partial","*",3,0],"ContextFetchBot":["partial","*",3,0]},"llms":{"status":"missing","http":404,"bytes":160698},"home":{"status":"ok","http":200,"url":"https://350.org/","html_tokens":88032,"markdown_tokens":289,"pruned_tokens":169}},
{"rank":6123,"domain":"collegehumor.com","origin":"http://collegehumor.com","robots":{"status":"html","http":200,"url":"https://www.dropout.tv/plans","bytes":219898,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":6124,"domain":"waseda.jp","origin":"https://www.waseda.jp","robots":{"status":"refused","http":403,"url":"https://www.waseda.jp/robots.txt","bytes":4542,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":6125,"domain":"goldbett.org","origin":"https://goldbett.org","robots":{"status":"ok","http":200,"url":"https://goldbett.org/robots.txt","bytes":67,"groups":1},"bots":{"GPTBot":["partial","*",1,1],"ChatGPT-User":["partial","*",1,1],"OAI-SearchBot":["partial","*",1,1],"ClaudeBot":["partial","*",1,1],"Claude-User":["partial","*",1,1],"Claude-SearchBot":["partial","*",1,1],"anthropic-ai":["partial","*",1,1],"PerplexityBot":["partial","*",1,1],"Perplexity-User":["partial","*",1,1],"Google-Extended":["partial","*",1,1],"Applebot-Extended":["partial","*",1,1],"CCBot":["partial","*",1,1],"Bytespider":["partial","*",1,1],"Amazonbot":["partial","*",1,1],"meta-externalagent":["partial","*",1,1],"cohere-ai":["partial","*",1,1],"Diffbot":["partial","*",1,1],"ContextFetchBot":["partial","*",1,1]},"llms":{"status":"missing","http":404,"bytes":58688},"home":{"status":"ok","http":200,"url":"https://goldbett.org/","html_tokens":20675,"markdown_tokens":2092,"pruned_tokens":146}},
{"rank":6126,"domain":"bitcoinmagazine.com","origin":"https://bitcoinmagazine.com","robots":{"status":"ok","http":200,"url":"https://bitcoinmagazine.com/robots.txt","bytes":865,"groups":9},"bots":{"GPTBot":["partial","*",6,2],"ChatGPT-User":["partial","*",6,2],"OAI-SearchBot":["partial","*",6,2],"ClaudeBot":["partial","*",6,2],"Claude-User":["partial","*",6,2],"Claude-SearchBot":["partial","*",6,2],"anthropic-ai":["partial","*",6,2],"PerplexityBot":["partial","*",6,2],"Perplexity-User":["partial","*",6,2],"Google-Extended":["partial","*",6,2],"Applebot-Extended":["partial","*",6,2],"CCBot":["partial","*",6,2],"Bytespider":["partial","*",6,2],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["partial","*",6,2],"cohere-ai":["partial","*",6,2],"Diffbot":["partial","*",6,2],"ContextFetchBot":["partial","*",6,2]},"llms":{"status":"http_error","http":403,"bytes":5572},"home":{"status":"http_error","http":403,"url":"https://bitcoinmagazine.com/"}},
{"rank":6127,"domain":"republicworld.com","origin":"https://republicworld.com","robots":{"status":"ok","http":200,"url":"https://www.republicworld.com/robots.txt","bytes":1746,"groups":12},"bots":{"GPTBot":["allowed","named",0,1],"ChatGPT-User":["allowed","named",0,1],"OAI-SearchBot":["allowed","*",0,1],"ClaudeBot":["allowed","named",0,1],"Claude-User":["allowed","*",0,1],"Claude-SearchBot":["allowed","*",0,1],"anthropic-ai":["allowed","named",0,1],"PerplexityBot":["allowed","named",0,1],"Perplexity-User":["allowed","*",0,1],"Google-Extended":["allowed","named",0,1],"Applebot-Extended":["allowed","*",0,1],"CCBot":["allowed","named",0,1],"Bytespider":["allowed","*",0,1],"Amazonbot":["allowed","*",0,1],"meta-externalagent":["allowed","*",0,1],"cohere-ai":["allowed","*",0,1],"Diffbot":["allowed","*",0,1],"ContextFetchBot":["allowed","*",0,1]},"llms":{"status":"missing","http":404,"bytes":231578},"home":{"status":"ok","http":200,"url":"https://www.republicworld.com/","html_tokens":461547,"markdown_tokens":37,"pruned_tokens":37}},
{"rank":6128,"domain":"fang.com","origin":"https://fang.com","robots":{"status":"ok","http":200,"url":"https://fang.com/robots.txt","bytes":906,"groups":1},"bots":{"GPTBot":["partial","*",30,0],"ChatGPT-User":["partial","*",30,0],"OAI-SearchBot":["partial","*",30,0],"ClaudeBot":["partial","*",30,0],"Claude-User":["partial","*",30,0],"Claude-SearchBot":["partial","*",30,0],"anthropic-ai":["partial","*",30,0],"PerplexityBot":["partial","*",30,0],"Perplexity-User":["partial","*",30,0],"Google-Extended":["partial","*",30,0],"Applebot-Extended":["partial","*",30,0],"CCBot":["partial","*",30,0],"Bytespider":["partial","*",30,0],"Amazonbot":["partial","*",30,0],"meta-externalagent":["partial","*",30,0],"cohere-ai":["partial","*",30,0],"Diffbot":["partial","*",30,0],"ContextFetchBot":["partial","*",30,0]},"llms":{"status":"missing","http":404,"bytes":22},"home":{"status":"ok","http":200,"url":"https://www1.fang.com/","html_tokens":39310,"markdown_tokens":14,"pruned_tokens":14}},
{"rank":6129,"domain":"mgtv.com","origin":"https://mgtv.com","robots":{"status":"ok","http":200,"url":"https://www.mgtv.com/robots.txt","bytes":66,"groups":1},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"html","http":200,"bytes":21650},"home":{"status":"ok","http":200,"url":"https://www.mgtv.com/","html_tokens":5336,"markdown_tokens":19,"pruned_tokens":19}},
{"rank":6130,"domain":"jouwweb.nl","origin":"https://jouwweb.nl","robots":{"status":"ok","http":200,"url":"https://www.jouwweb.nl/robots.txt","bytes":87,"groups":1},"bots":{"GPTBot":["partial","*",1,0],"ChatGPT-User":["partial","*",1,0],"OAI-SearchBot":["partial","*",1,0],"ClaudeBot":["partial","*",1,0],"Claude-User":["partial","*",1,0],"Claude-SearchBot":["partial","*",1,0],"anthropic-ai":["partial","*",1,0],"PerplexityBot":["partial","*",1,0],"Perplexity-User":["partial","*",1,0],"Google-Extended":["partial","*",1,0],"Applebot-Extended":["partial","*",1,0],"CCBot":["partial","*",1,0],"Bytespider":["partial","*",1,0],"Amazonbot":["partial","*",1,0],"meta-externalagent":["partial","*",1,0],"cohere-ai":["partial","*",1,0],"Diffbot":["partial","*",1,0],"ContextFetchBot":["partial","*",1,0]},"llms":{"status":"present","http":200,"bytes":1669},"home":{"status":"ok","http":200,"url":"https://www.jouwweb.nl/","html_tokens":35291,"markdown_tokens":480,"pruned_tokens":146}},
{"rank":6131,"domain":"nintendolife.com","origin":"https://nintendolife.com","robots":{"status":"ok","http":200,"url":"https://www.nintendolife.com/robots.txt","bytes":415,"groups":3},"bots":{"GPTBot":["partial","*",13,0],"ChatGPT-User":["partial","*",13,0],"OAI-SearchBot":["partial","*",13,0],"ClaudeBot":["partial","*",13,0],"Claude-User":["partial","*",13,0],"Claude-SearchBot":["partial","*",13,0],"anthropic-ai":["partial","*",13,0],"PerplexityBot":["partial","*",13,0],"Perplexity-User":["partial","*",13,0],"Google-Extended":["partial","*",13,0],"Applebot-Extended":["partial","*",13,0],"CCBot":["partial","*",13,0],"Bytespider":["partial","*",13,0],"Amazonbot":["partial","*",13,0],"meta-externalagent":["partial","*",13,0],"cohere-ai":["partial","*",13,0],"Diffbot":["partial","*",13,0],"ContextFetchBot":["partial","*",13,0]},"llms":{"status":"missing","http":404,"bytes":62051},"home":{"status":"ok","http":200,"url":"https://www.nintendolife.com/","html_tokens":57557,"markdown_tokens":5193,"pruned_tokens":167}},
{"rank":6132,"domain":"4399.com","origin":"https://4399.com","robots":{"status":"ok","http":200,"url":"http://www.4399.com/robots.txt","bytes":722,"groups":1},"bots":{"GPTBot":["partial","*",27,0],"ChatGPT-User":["partial","*",27,0],"OAI-SearchBot":["partial","*",27,0],"ClaudeBot":["partial","*",27,0],"Claude-User":["partial","*",27,0],"Claude-SearchBot":["partial","*",27,0],"anthropic-ai":["partial","*",27,0],"PerplexityBot":["partial","*",27,0],"Perplexity-User":["partial","*",27,0],"Google-Extended":["partial","*",27,0],"Applebot-Extended":["partial","*",27,0],"CCBot":["partial","*",27,0],"Bytespider":["partial","*",27,0],"Amazonbot":["partial","*",27,0],"meta-externalagent":["partial","*",27,0],"cohere-ai":["partial","*",27,0],"Diffbot":["partial","*",27,0],"ContextFetchBot":["partial","*",27,0]},"llms":{"status":"missing","http":404,"bytes":13110},"home":{"status":"ok","http":200,"url":"https://www.4399.com/","html_tokens":58479,"markdown_tokens":343,"pruned_tokens":60}},
{"rank":6133,"domain":"bluetooth.com","origin":"https://bluetooth.com","robots":{"status":"ok","http":200,"url":"https://www.bluetooth.com/robots.txt","bytes":137,"groups":1},"bots":{"GPTBot":["partial","*",1,1],"ChatGPT-User":["partial","*",1,1],"OAI-SearchBot":["partial","*",1,1],"ClaudeBot":["partial","*",1,1],"Claude-User":["partial","*",1,1],"Claude-SearchBot":["partial","*",1,1],"anthropic-ai":["partial","*",1,1],"PerplexityBot":["partial","*",1,1],"Perplexity-User":["partial","*",1,1],"Google-Extended":["partial","*",1,1],"Applebot-Extended":["partial","*",1,1],"CCBot":["partial","*",1,1],"Bytespider":["partial","*",1,1],"Amazonbot":["partial","*",1,1],"meta-externalagent":["partial","*",1,1],"cohere-ai":["partial","*",1,1],"Diffbot":["partial","*",1,1],"ContextFetchBot":["partial","*",1,1]},"llms":{"status":"missing","http":404,"bytes":60},"home":{"status":"ok","http":200,"url":"https://www.bluetooth.com/","html_tokens":120730,"markdown_tokens":326,"pruned_tokens":185}},
{"rank":6134,"domain":"pewforum.org","origin":"https://pewforum.org","robots":{"status":"missing","http":404,"url":"https://www.pewresearch.org/religion/robots.txt","bytes":287679,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":287673},"home":{"status":"ok","http":200,"url":"https://www.pewresearch.org/topic/religion/","html_tokens":140375,"markdown_tokens":237,"pruned_tokens":115}},
{"rank":6135,"domain":"business.gov.au","origin":"https://business.gov.au","robots":{"status":"ok","http":200,"url":"https://business.gov.au/robots.txt","bytes":122,"groups":1},"bots":{"GPTBot":["partial","*",2,0],"ChatGPT-User":["partial","*",2,0],"OAI-SearchBot":["partial","*",2,0],"ClaudeBot":["partial","*",2,0],"Claude-User":["partial","*",2,0],"Claude-SearchBot":["partial","*",2,0],"anthropic-ai":["partial","*",2,0],"PerplexityBot":["partial","*",2,0],"Perplexity-User":["partial","*",2,0],"Google-Extended":["partial","*",2,0],"Applebot-Extended":["partial","*",2,0],"CCBot":["partial","*",2,0],"Bytespider":["partial","*",2,0],"Amazonbot":["partial","*",2,0],"meta-externalagent":["partial","*",2,0],"cohere-ai":["partial","*",2,0],"Diffbot":["partial","*",2,0],"ContextFetchBot":["partial","*",2,0]},"llms":{"status":"html","http":200,"bytes":321466},"home":{"status":"ok","http":200,"url":"https://business.gov.au/","html_tokens":89660,"markdown_tokens":39,"pruned_tokens":39}},
{"rank":6136,"domain":"boj.or.jp","origin":"https://www.boj.or.jp","robots":{"status":"missing","http":404,"url":"https://www.boj.or.jp/robots.txt","bytes":3297,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":3297},"home":{"status":"ok","http":200,"url":"https://www.boj.or.jp/","html_tokens":15339,"markdown_tokens":536,"pruned_tokens":103}},
{"rank":6137,"domain":"comingsoon.net","origin":"https://comingsoon.net","robots":{"status":"ok","http":200,"url":"https://www.comingsoon.net/robots.txt","bytes":451,"groups":3},"bots":{"GPTBot":["partial","*",2,3],"ChatGPT-User":["partial","*",2,3],"OAI-SearchBot":["partial","*",2,3],"ClaudeBot":["partial","*",2,3],"Claude-User":["partial","*",2,3],"Claude-SearchBot":["partial","*",2,3],"anthropic-ai":["partial","*",2,3],"PerplexityBot":["partial","*",2,3],"Perplexity-User":["partial","*",2,3],"Google-Extended":["partial","*",2,3],"Applebot-Extended":["partial","*",2,3],"CCBot":["partial","*",2,3],"Bytespider":["partial","*",2,3],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["partial","*",2,3],"cohere-ai":["partial","*",2,3],"Diffbot":["partial","*",2,3],"ContextFetchBot":["partial","*",2,3]},"llms":{"status":"missing","http":404,"bytes":198104},"home":{"status":"http_error","http":403,"url":"https://comingsoon.net/"}},
{"rank":6138,"domain":"apkmirror.com","origin":"https://apkmirror.com","robots":{"status":"ok","http":200,"url":"https://www.apkmirror.com/robots.txt","bytes":648,"groups":3},"bots":{"GPTBot":["partial","named",2,1],"ChatGPT-User":["partial","named",2,1],"OAI-SearchBot":["partial","named",2,1],"ClaudeBot":["partial","*",2,1],"Claude-User":["partial","*",2,1],"Claude-SearchBot":["partial","*",2,1],"anthropic-ai":["partial","*",2,1],"PerplexityBot":["partial","*",2,1],"Perplexity-User":["partial","*",2,1],"Google-Extended":["partial","*",2,1],"Applebot-Extended":["partial","*",2,1],"CCBot":["partial","*",2,1],"Bytespider":["partial","named",2,1],"Amazonbot":["partial","*",2,1],"meta-externalagent":["partial","named",2,1],"cohere-ai":["partial","*",2,1],"Diffbot":["partial","*",2,1],"ContextFetchBot":["partial","*",2,1]},"llms":{"status":"http_error","http":403,"bytes":7},"home":{"status":"http_error","http":403,"url":"https://apkmirror.com/"}},
{"rank":6139,"domain":"recruitee.com","origin":"https://recruitee.com","robots":{"status":"ok","http":200,"url":"https://recruitee.com/robots.txt","bytes":2674,"groups":8},"bots":{"GPTBot":["partial","named",12,1],"ChatGPT-User":["partial","named",12,1],"OAI-SearchBot":["partial","*",12,0],"ClaudeBot":["partial","named",12,1],"Claude-User":["partial","*",12,0],"Claude-SearchBot":["partial","*",12,0],"anthropic-ai":["partial","*",12,0],"PerplexityBot":["partial","named",12,1],"Perplexity-User":["partial","*",12,0],"Google-Extended":["partial","named",12,1],"Applebot-Extended":["partial","*",12,0],"CCBot":["partial","named",12,1],"Bytespider":["partial","*",12,0],"Amazonbot":["partial","*",12,0],"meta-externalagent":["partial","*",12,0],"cohere-ai":["partial","*",12,0],"Diffbot":["partial","*",12,0],"ContextFetchBot":["partial","*",12,0]},"llms":{"status":"missing","http":404,"bytes":7064},"home":{"status":"ok","http":200,"url":"https://recruitee.com/","html_tokens":59040,"markdown_tokens":3496,"pruned_tokens":259}},
{"rank":6140,"domain":"serif.com","origin":"https://serif.com","robots":{"status":"ok","http":200,"url":"https://serif.com/robots.txt","bytes":23,"groups":1},"bots":{"GPTBot":["allowed","*",0,1],"ChatGPT-User":["allowed","*",0,1],"OAI-SearchBot":["allowed","*",0,1],"ClaudeBot":["allowed","*",0,1],"Claude-User":["allowed","*",0,1],"Claude-SearchBot":["allowed","*",0,1],"anthropic-ai":["allowed","*",0,1],"PerplexityBot":["allowed","*",0,1],"Perplexity-User":["allowed","*",0,1],"Google-Extended":["allowed","*",0,1],"Applebot-Extended":["allowed","*",0,1],"CCBot":["allowed","*",0,1],"Bytespider":["allowed","*",0,1],"Amazonbot":["allowed","*",0,1],"meta-externalagent":["allowed","*",0,1],"cohere-ai":["allowed","*",0,1],"Diffbot":["allowed","*",0,1],"ContextFetchBot":["allowed","*",0,1]},"llms":{"status":"html","http":200,"bytes":9277},"home":{"status":"ok","http":200,"url":"https://www.affinity.studio/","html_tokens":2320,"markdown_tokens":39,"pruned_tokens":39}},
{"rank":6141,"domain":"whatismyip.com","origin":"https://whatismyip.com","robots":{"status":"ok","http":200,"url":"https://whatismyip.com/robots.txt","bytes":2676,"groups":32},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["allowed","*",0,1],"OAI-SearchBot":["allowed","*",0,1],"ClaudeBot":["blocked","named",1,0],"Claude-User":["allowed","*",0,1],"Claude-SearchBot":["allowed","*",0,1],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["allowed","*",0,1],"Perplexity-User":["allowed","*",0,1],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["allowed","*",0,1]},"llms":{"status":"present","http":200,"bytes":3265},"home":{"status":"http_error","http":403,"url":"https://www.whatismyip.com/"}},
{"rank":6142,"domain":"fan-slot.com","origin":"https://fan-slot.com","robots":{"status":"ok","http":200,"url":"https://fan-slot.com/robots.txt","bytes":529,"groups":1},"bots":{"GPTBot":["partial","*",19,0],"ChatGPT-User":["partial","*",19,0],"OAI-SearchBot":["partial","*",19,0],"ClaudeBot":["partial","*",19,0],"Claude-User":["partial","*",19,0],"Claude-SearchBot":["partial","*",19,0],"anthropic-ai":["partial","*",19,0],"PerplexityBot":["partial","*",19,0],"Perplexity-User":["partial","*",19,0],"Google-Extended":["partial","*",19,0],"Applebot-Extended":["partial","*",19,0],"CCBot":["partial","*",19,0],"Bytespider":["partial","*",19,0],"Amazonbot":["partial","*",19,0],"meta-externalagent":["partial","*",19,0],"cohere-ai":["partial","*",19,0],"Diffbot":["partial","*",19,0],"ContextFetchBot":["partial","*",19,0]},"llms":{"status":"missing","http":404,"bytes":17236},"home":{"status":"ok","http":200,"url":"https://fan-slot.com/","html_tokens":79747,"markdown_tokens":1153,"pruned_tokens":108}},
{"rank":6143,"domain":"telenet.be","origin":"https://telenet.be","robots":{"status":"ok","http":200,"url":"https://www2.telenet.be/robots.txt","bytes":314,"groups":1},"bots":{"GPTBot":["partial","*",2,3],"ChatGPT-User":["partial","*",2,3],"OAI-SearchBot":["partial","*",2,3],"ClaudeBot":["partial","*",2,3],"Claude-User":["partial","*",2,3],"Claude-SearchBot":["partial","*",2,3],"anthropic-ai":["partial","*",2,3],"PerplexityBot":["partial","*",2,3],"Perplexity-User":["partial","*",2,3],"Google-Extended":["partial","*",2,3],"Applebot-Extended":["partial","*",2,3],"CCBot":["partial","*",2,3],"Bytespider":["partial","*",2,3],"Amazonbot":["partial","*",2,3],"meta-externalagent":["partial","*",2,3],"cohere-ai":["partial","*",2,3],"Diffbot":["partial","*",2,3],"ContextFetchBot":["partial","*",2,3]},"llms":{"status":"missing","http":404,"bytes":223147},"home":{"status":"ok","http":200,"url":"https://www2.telenet.be/residential/nl","html_tokens":112372,"markdown_tokens":365,"pruned_tokens":168}},
{"rank":6144,"domain":"exame.com","origin":"https://exame.com","robots":{"status":"ok","http":200,"url":"https://exame.com/robots.txt","bytes":234,"groups":2},"bots":{"GPTBot":["partial","*",5,0],"ChatGPT-User":["partial","*",5,0],"OAI-SearchBot":["partial","*",5,0],"ClaudeBot":["partial","*",5,0],"Claude-User":["partial","*",5,0],"Claude-SearchBot":["partial","*",5,0],"anthropic-ai":["partial","*",5,0],"PerplexityBot":["partial","*",5,0],"Perplexity-User":["partial","*",5,0],"Google-Extended":["allowed","named",0,1],"Applebot-Extended":["partial","*",5,0],"CCBot":["partial","*",5,0],"Bytespider":["partial","*",5,0],"Amazonbot":["partial","*",5,0],"meta-externalagent":["partial","*",5,0],"cohere-ai":["partial","*",5,0],"Diffbot":["partial","*",5,0],"ContextFetchBot":["partial","*",5,0]},"llms":{"status":"missing","http":404,"bytes":67149},"home":{"status":"ok","http":200,"url":"https://exame.com/","html_tokens":125575,"markdown_tokens":80,"pruned_tokens":80}},
{"rank":6145,"domain":"swiss.com","origin":"https://swiss.com","robots":{"status":"ok","http":200,"url":"https://www.swiss.com/robots.txt","bytes":231,"groups":1},"bots":{"GPTBot":["partial","*",2,1],"ChatGPT-User":["partial","*",2,1],"OAI-SearchBot":["partial","*",2,1],"ClaudeBot":["partial","*",2,1],"Claude-User":["partial","*",2,1],"Claude-SearchBot":["partial","*",2,1],"anthropic-ai":["partial","*",2,1],"PerplexityBot":["partial","*",2,1],"Perplexity-User":["partial","*",2,1],"Google-Extended":["partial","*",2,1],"Applebot-Extended":["partial","*",2,1],"CCBot":["partial","*",2,1],"Bytespider":["partial","*",2,1],"Amazonbot":["partial","*",2,1],"meta-externalagent":["partial","*",2,1],"cohere-ai":["partial","*",2,1],"Diffbot":["partial","*",2,1],"ContextFetchBot":["partial","*",2,1]},"llms":{"status":"missing","http":404,"bytes":27150},"home":{"status":"http_error","http":403,"url":"https://www.swiss.com/"}},
{"rank":6146,"domain":"shure.com","origin":"https://shure.com","robots":{"status":"ok","http":200,"url":"https://www.shure.com/robots.txt","bytes":263,"groups":2},"bots":{"GPTBot":["partial","*",6,1],"ChatGPT-User":["partial","*",6,1],"OAI-SearchBot":["partial","*",6,1],"ClaudeBot":["partial","*",6,1],"Claude-User":["partial","*",6,1],"Claude-SearchBot":["partial","*",6,1],"anthropic-ai":["partial","*",6,1],"PerplexityBot":["partial","*",6,1],"Perplexity-User":["partial","*",6,1],"Google-Extended":["partial","*",6,1],"Applebot-Extended":["partial","*",6,1],"CCBot":["partial","*",6,1],"Bytespider":["partial","*",6,1],"Amazonbot":["partial","*",6,1],"meta-externalagent":["partial","*",6,1],"cohere-ai":["partial","*",6,1],"Diffbot":["partial","*",6,1],"ContextFetchBot":["partial","*",6,1]},"llms":{"status":"present","http":200,"bytes":12641},"home":{"status":"ok","http":200,"url":"https://www.shure.com/en-US","html_tokens":208309,"markdown_tokens":1478,"pruned_tokens":700}},
{"rank":6147,"domain":"google.kz","origin":"https://google.kz","robots":{"status":"ok","http":200,"url":"https://www.google.kz/robots.txt","bytes":6624,"groups":4},"bots":{"GPTBot":["partial","*",169,64],"ChatGPT-User":["partial","*",169,64],"OAI-SearchBot":["partial","*",169,64],"ClaudeBot":["partial","*",169,64],"Claude-User":["partial","*",169,64],"Claude-SearchBot":["partial","*",169,64],"anthropic-ai":["partial","*",169,64],"PerplexityBot":["partial","*",169,64],"Perplexity-User":["partial","*",169,64],"Google-Extended":["partial","*",169,64],"Applebot-Extended":["partial","*",169,64],"CCBot":["partial","*",169,64],"Bytespider":["partial","*",169,64],"Amazonbot":["partial","*",169,64],"meta-externalagent":["partial","*",169,64],"cohere-ai":["partial","*",169,64],"Diffbot":["partial","*",169,64],"ContextFetchBot":["partial","*",169,64]},"llms":{"status":"missing","http":404,"bytes":1569},"home":{"status":"ok","http":200,"url":"https://www.google.com/","html_tokens":21351,"markdown_tokens":30,"pruned_tokens":30}},
{"rank":6148,"domain":"ust.hk","origin":"https://www.ust.hk","robots":{"status":"ok","http":200,"url":"https://hkust.edu.hk/robots.txt","bytes":2027,"groups":1},"bots":{"GPTBot":["partial","*",34,18],"ChatGPT-User":["partial","*",34,18],"OAI-SearchBot":["partial","*",34,18],"ClaudeBot":["partial","*",34,18],"Claude-User":["partial","*",34,18],"Claude-SearchBot":["partial","*",34,18],"anthropic-ai":["partial","*",34,18],"PerplexityBot":["partial","*",34,18],"Perplexity-User":["partial","*",34,18],"Google-Extended":["partial","*",34,18],"Applebot-Extended":["partial","*",34,18],"CCBot":["partial","*",34,18],"Bytespider":["partial","*",34,18],"Amazonbot":["partial","*",34,18],"meta-externalagent":["partial","*",34,18],"cohere-ai":["partial","*",34,18],"Diffbot":["partial","*",34,18],"ContextFetchBot":["partial","*",34,18]},"llms":{"status":"missing","http":404,"bytes":146},"home":{"status":"ok","http":200,"url":"https://hkust.edu.hk/","html_tokens":37657,"markdown_tokens":666,"pruned_tokens":146}},
{"rank":6149,"domain":"wearesocial.com","origin":"https://wearesocial.com","robots":{"status":"ok","http":200,"url":"https://wearesocial.com/robots.txt","bytes":894,"groups":1},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"http_error","http":403,"bytes":5568},"home":{"status":"http_error","http":403,"url":"https://wearesocial.com/"}},
{"rank":6150,"domain":"titech.ac.jp","origin":"https://www.titech.ac.jp","robots":{"status":"ok","http":200,"url":"https://www.titech.ac.jp/robots.txt","bytes":261,"groups":2},"bots":{"GPTBot":["partial","*",5,0],"ChatGPT-User":["partial","*",5,0],"OAI-SearchBot":["partial","*",5,0],"ClaudeBot":["partial","*",5,0],"Claude-User":["partial","*",5,0],"Claude-SearchBot":["partial","*",5,0],"anthropic-ai":["partial","*",5,0],"PerplexityBot":["partial","*",5,0],"Perplexity-User":["partial","*",5,0],"Google-Extended":["partial","*",5,0],"Applebot-Extended":["partial","*",5,0],"CCBot":["partial","*",5,0],"Bytespider":["partial","*",5,0],"Amazonbot":["partial","*",5,0],"meta-externalagent":["partial","*",5,0],"cohere-ai":["partial","*",5,0],"Diffbot":["partial","*",5,0],"ContextFetchBot":["partial","*",5,0]},"llms":{"status":"html","http":200,"bytes":71886},"home":{"status":"ok","http":200,"url":"https://www.titech.ac.jp/","html_tokens":16811,"markdown_tokens":11,"pruned_tokens":11}},
{"rank":6151,"domain":"bunka.go.jp","origin":"https://www.bunka.go.jp","robots":{"status":"missing","http":404,"url":"https://www.bunka.go.jp/robots.txt","bytes":16207,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":16207},"home":{"status":"ok","http":200,"url":"https://www.bunka.go.jp/","html_tokens":10214,"markdown_tokens":230,"pruned_tokens":166}},
{"rank":6152,"domain":"search.com","origin":"https://search.com","robots":{"status":"ok","http":200,"url":"https://search.com/robots.txt","bytes":1214,"groups":16},"bots":{"GPTBot":["partial","*",6,8],"ChatGPT-User":["allowed","named",0,1],"OAI-SearchBot":["partial","*",6,8],"ClaudeBot":["partial","*",6,8],"Claude-User":["partial","*",6,8],"Claude-SearchBot":["partial","*",6,8],"anthropic-ai":["partial","*",6,8],"PerplexityBot":["allowed","named",0,1],"Perplexity-User":["partial","*",6,8],"Google-Extended":["allowed","named",0,1],"Applebot-Extended":["partial","*",6,8],"CCBot":["partial","*",6,8],"Bytespider":["partial","*",6,8],"Amazonbot":["partial","*",6,8],"meta-externalagent":["partial","*",6,8],"cohere-ai":["partial","*",6,8],"Diffbot":["partial","*",6,8],"ContextFetchBot":["partial","*",6,8]},"llms":{"status":"html","http":200,"bytes":1098568},"home":{"status":"ok","http":200,"url":"https://search.com/","html_tokens":274560,"markdown_tokens":172,"pruned_tokens":13}},
{"rank":6153,"domain":"kali.org","origin":"https://kali.org","robots":{"status":"ok","http":200,"url":"https://www.kali.org/robots.txt","bytes":2260,"groups":1},"bots":{"GPTBot":["partial","*",60,0],"ChatGPT-User":["partial","*",60,0],"OAI-SearchBot":["partial","*",60,0],"ClaudeBot":["partial","*",60,0],"Claude-User":["partial","*",60,0],"Claude-SearchBot":["partial","*",60,0],"anthropic-ai":["partial","*",60,0],"PerplexityBot":["partial","*",60,0],"Perplexity-User":["partial","*",60,0],"Google-Extended":["partial","*",60,0],"Applebot-Extended":["partial","*",60,0],"CCBot":["partial","*",60,0],"Bytespider":["partial","*",60,0],"Amazonbot":["partial","*",60,0],"meta-externalagent":["partial","*",60,0],"cohere-ai":["partial","*",60,0],"Diffbot":["partial","*",60,0],"ContextFetchBot":["partial","*",60,0]},"llms":{"status":"missing","http":404,"bytes":34456},"home":{"status":"ok","http":200,"url":"https://www.kali.org/","html_tokens":12580,"markdown_tokens":1133,"pruned_tokens":418}},
{"rank":6154,"domain":"thegamer.com","origin":"https://thegamer.com","robots":{"status":"ok","http":200,"url":"https://www.thegamer.com/robots.txt","bytes":2037,"groups":2},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["partial","*",22,0],"OAI-SearchBot":["partial","*",22,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",22,0],"Claude-SearchBot":["partial","*",22,0],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["partial","*",22,0],"Perplexity-User":["partial","*",22,0],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",22,0]},"llms":{"status":"missing","http":404,"bytes":105498},"home":{"status":"ok","http":200,"url":"https://www.thegamer.com/","html_tokens":234087,"markdown_tokens":179,"pruned_tokens":33}},
{"rank":6155,"domain":"milliyet.com.tr","origin":"https://milliyet.com.tr","robots":{"status":"ok","http":200,"url":"https://www.milliyet.com.tr/robots.txt","bytes":475,"groups":12},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["blocked","named",1,0],"OAI-SearchBot":["blocked","named",1,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",3,0],"Claude-SearchBot":["partial","*",3,0],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["partial","*",3,0],"Google-Extended":["partial","*",3,0],"Applebot-Extended":["partial","*",3,0],"CCBot":["partial","*",3,0],"Bytespider":["partial","*",3,0],"Amazonbot":["partial","*",3,0],"meta-externalagent":["partial","*",3,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["partial","*",3,0],"ContextFetchBot":["partial","*",3,0]},"llms":{"status":"missing","http":404,"bytes":2798},"home":{"status":"ok","http":200,"url":"https://www.milliyet.com.tr/","html_tokens":56476,"markdown_tokens":72,"pruned_tokens":72}},
{"rank":6156,"domain":"dsgvo-gesetz.de","origin":"https://dsgvo-gesetz.de","robots":{"status":"ok","http":200,"url":"https://dsgvo-gesetz.de/robots.txt","bytes":268,"groups":1},"bots":{"GPTBot":["partial","*",7,1],"ChatGPT-User":["partial","*",7,1],"OAI-SearchBot":["partial","*",7,1],"ClaudeBot":["partial","*",7,1],"Claude-User":["partial","*",7,1],"Claude-SearchBot":["partial","*",7,1],"anthropic-ai":["partial","*",7,1],"PerplexityBot":["partial","*",7,1],"Perplexity-User":["partial","*",7,1],"Google-Extended":["partial","*",7,1],"Applebot-Extended":["partial","*",7,1],"CCBot":["partial","*",7,1],"Bytespider":["partial","*",7,1],"Amazonbot":["partial","*",7,1],"meta-externalagent":["partial","*",7,1],"cohere-ai":["partial","*",7,1],"Diffbot":["partial","*",7,1],"ContextFetchBot":["partial","*",7,1]},"llms":{"status":"missing","http":404,"bytes":81440},"home":{"status":"ok","http":200,"url":"https://dsgvo-gesetz.de/","html_tokens":31757,"markdown_tokens":321,"pruned_tokens":161}},
{"rank":6157,"domain":"jmu.edu","origin":"https://jmu.edu","robots":{"status":"ok","http":200,"url":"https://www.jmu.edu/robots.txt","bytes":3412,"groups":1},"bots":{"GPTBot":["partial","*",94,0],"ChatGPT-User":["partial","*",94,0],"OAI-SearchBot":["partial","*",94,0],"ClaudeBot":["partial","*",94,0],"Claude-User":["partial","*",94,0],"Claude-SearchBot":["partial","*",94,0],"anthropic-ai":["partial","*",94,0],"PerplexityBot":["partial","*",94,0],"Perplexity-User":["partial","*",94,0],"Google-Extended":["partial","*",94,0],"Applebot-Extended":["partial","*",94,0],"CCBot":["partial","*",94,0],"Bytespider":["partial","*",94,0],"Amazonbot":["partial","*",94,0],"meta-externalagent":["partial","*",94,0],"cohere-ai":["partial","*",94,0],"Diffbot":["partial","*",94,0],"ContextFetchBot":["partial","*",94,0]},"llms":{"status":"missing","http":404,"bytes":86005},"home":{"status":"ok","http":200,"url":"https://www.jmu.edu/index.shtml","html_tokens":41238,"markdown_tokens":177,"pruned_tokens":93}},
{"rank":6158,"domain":"copyscape.com","origin":"https://copyscape.com","robots":{"status":"ok","http":200,"url":"https://copyscape.com/robots.txt","bytes":516,"groups":1},"bots":{"GPTBot":["partial","*",22,0],"ChatGPT-User":["partial","*",22,0],"OAI-SearchBot":["partial","*",22,0],"ClaudeBot":["partial","*",22,0],"Claude-User":["partial","*",22,0],"Claude-SearchBot":["partial","*",22,0],"anthropic-ai":["partial","*",22,0],"PerplexityBot":["partial","*",22,0],"Perplexity-User":["partial","*",22,0],"Google-Extended":["partial","*",22,0],"Applebot-Extended":["partial","*",22,0],"CCBot":["partial","*",22,0],"Bytespider":["partial","*",22,0],"Amazonbot":["partial","*",22,0],"meta-externalagent":["partial","*",22,0],"cohere-ai":["partial","*",22,0],"Diffbot":["partial","*",22,0],"ContextFetchBot":["partial","*",22,0]},"llms":{"status":"missing","http":404,"bytes":286},"home":{"status":"ok","http":200,"url":"https://www.copyscape.com/","html_tokens":7419,"markdown_tokens":77,"pruned_tokens":77}},
{"rank":6159,"domain":"ircam.fr","origin":"https://ircam.fr","robots":{"status":"html","http":200,"url":"https://www.ircam.fr/en","bytes":635647,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":6160,"domain":"aist.go.jp","origin":"https://www.aist.go.jp","robots":{"status":"ok","http":200,"url":"https://www.aist.go.jp/robots.txt","bytes":99,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":20340},"home":{"status":"ok","http":200,"url":"https://www.aist.go.jp/","html_tokens":6184,"markdown_tokens":71,"pruned_tokens":71}},
{"rank":6161,"domain":"prosieben.de","origin":"https://prosieben.de","robots":{"status":"html","http":200,"url":"https://www.joyn.de/mediatheken/prosieben","bytes":284852,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":6162,"domain":"port.ac.uk","origin":"https://port.ac.uk","robots":{"status":"ok","http":200,"url":"https://www.port.ac.uk/robots.txt","bytes":2171,"groups":7},"bots":{"GPTBot":["allowed","*",0,18],"ChatGPT-User":["allowed","*",0,18],"OAI-SearchBot":["allowed","*",0,18],"ClaudeBot":["allowed","*",0,18],"Claude-User":["allowed","*",0,18],"Claude-SearchBot":["allowed","*",0,18],"anthropic-ai":["allowed","*",0,18],"PerplexityBot":["allowed","*",0,18],"Perplexity-User":["allowed","*",0,18],"Google-Extended":["allowed","*",0,18],"Applebot-Extended":["allowed","*",0,18],"CCBot":["allowed","*",0,18],"Bytespider":["blocked","named",1,0],"Amazonbot":["allowed","*",0,18],"meta-externalagent":["allowed","*",0,18],"cohere-ai":["allowed","*",0,18],"Diffbot":["allowed","*",0,18],"ContextFetchBot":["allowed","*",0,18]},"llms":{"status":"present","http":200,"bytes":4783},"home":{"status":"ok","http":200,"url":"https://www.port.ac.uk/","html_tokens":42884,"markdown_tokens":406,"pruned_tokens":32}},
{"rank":6163,"domain":"google.fm","origin":"https://google.fm","robots":{"status":"ok","http":200,"url":"https://www.google.fm/robots.txt","bytes":6624,"groups":4},"bots":{"GPTBot":["partial","*",169,64],"ChatGPT-User":["partial","*",169,64],"OAI-SearchBot":["partial","*",169,64],"ClaudeBot":["partial","*",169,64],"Claude-User":["partial","*",169,64],"Claude-SearchBot":["partial","*",169,64],"anthropic-ai":["partial","*",169,64],"PerplexityBot":["partial","*",169,64],"Perplexity-User":["partial","*",169,64],"Google-Extended":["partial","*",169,64],"Applebot-Extended":["partial","*",169,64],"CCBot":["partial","*",169,64],"Bytespider":["partial","*",169,64],"Amazonbot":["partial","*",169,64],"meta-externalagent":["partial","*",169,64],"cohere-ai":["partial","*",169,64],"Diffbot":["partial","*",169,64],"ContextFetchBot":["partial","*",169,64]},"llms":{"status":"missing","http":404,"bytes":1569},"home":{"status":"ok","http":200,"url":"https://www.google.com/","html_tokens":21458,"markdown_tokens":30,"pruned_tokens":30}},
{"rank":6164,"domain":"maine.edu","origin":"https://maine.edu","robots":{"status":"ok","http":200,"url":"https://www.maine.edu/robots.txt","bytes":68,"groups":1},"bots":{"GPTBot":["partial","*",1,1],"ChatGPT-User":["partial","*",1,1],"OAI-SearchBot":["partial","*",1,1],"ClaudeBot":["partial","*",1,1],"Claude-User":["partial","*",1,1],"Claude-SearchBot":["partial","*",1,1],"anthropic-ai":["partial","*",1,1],"PerplexityBot":["partial","*",1,1],"Perplexity-User":["partial","*",1,1],"Google-Extended":["partial","*",1,1],"Applebot-Extended":["partial","*",1,1],"CCBot":["partial","*",1,1],"Bytespider":["partial","*",1,1],"Amazonbot":["partial","*",1,1],"meta-externalagent":["partial","*",1,1],"cohere-ai":["partial","*",1,1],"Diffbot":["partial","*",1,1],"ContextFetchBot":["partial","*",1,1]},"llms":{"status":"missing","http":404,"bytes":296480},"home":{"status":"ok","http":200,"url":"https://www.maine.edu/","html_tokens":86470,"markdown_tokens":437,"pruned_tokens":92}},
{"rank":6165,"domain":"my3w.com","origin":"https://my3w.com","robots":{"status":"unreachable","http":null,"url":"https://my3w.com/robots.txt","bytes":0,"groups":0,"error":"cannot resolve my3w.com"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":6166,"domain":"pinterest.it","origin":"https://pinterest.it","robots":{"status":"ok","http":200,"url":"https://it.pinterest.com/robots.txt","bytes":35080,"groups":8},"bots":{"GPTBot":["blocked","*",1,0],"ChatGPT-User":["blocked","*",1,0],"OAI-SearchBot":["blocked","*",1,0],"ClaudeBot":["blocked","*",1,0],"Claude-User":["blocked","*",1,0],"Claude-SearchBot":["blocked","*",1,0],"anthropic-ai":["blocked","*",1,0],"PerplexityBot":["blocked","*",1,0],"Perplexity-User":["blocked","*",1,0],"Google-Extended":["blocked","*",1,0],"Applebot-Extended":["blocked","*",1,0],"CCBot":["blocked","*",1,0],"Bytespider":["blocked","*",1,0],"Amazonbot":["blocked","*",1,0],"meta-externalagent":["blocked","*",1,0],"cohere-ai":["blocked","*",1,0],"Diffbot":["blocked","*",1,0],"ContextFetchBot":["blocked","*",1,0]},"llms":{"status":"disallowed"},"home":{"status":"disallowed"}},
{"rank":6167,"domain":"officedepot.com","origin":"https://officedepot.com","robots":{"status":"unreachable","http":null,"url":"https://officedepot.com/robots.txt","bytes":0,"groups":0,"error":"<StreamReset stream_id:1, error_code:2, remote_reset:True>"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":6168,"domain":"sandiego.edu","origin":"https://sandiego.edu","robots":{"status":"ok","http":200,"url":"https://www.sandiego.edu/robots.txt","bytes":2896,"groups":2},"bots":{"GPTBot":["partial","*",44,6],"ChatGPT-User":["partial","*",44,6],"OAI-SearchBot":["partial","*",44,6],"ClaudeBot":["partial","*",44,6],"Claude-User":["partial","*",44,6],"Claude-SearchBot":["partial","*",44,6],"anthropic-ai":["partial","*",44,6],"PerplexityBot":["partial","*",44,6],"Perplexity-User":["partial","*",44,6],"Google-Extended":["partial","*",44,6],"Applebot-Extended":["partial","*",44,6],"CCBot":["partial","*",44,6],"Bytespider":["partial","*",44,6],"Amazonbot":["partial","*",44,6],"meta-externalagent":["partial","*",44,6],"cohere-ai":["partial","*",44,6],"Diffbot":["partial","*",44,6],"ContextFetchBot":["partial","*",44,6]},"llms":{"status":"missing","http":404,"bytes":42303},"home":{"status":"ok","http":200,"url":"https://www.sandiego.edu/","html_tokens":26725,"markdown_tokens":222,"pruned_tokens":126}},
{"rank":6169,"domain":"globalsources.com","origin":"https://globalsources.com","robots":{"status":"refused","http":403,"url":"https://www.globalsources.com/robots.txt","bytes":919,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":6170,"domain":"majestic.com","origin":"https://majestic.com","robots":{"status":"ok","http":200,"url":"https://majestic.com/robots.txt","bytes":517,"groups":5},"bots":{"GPTBot":["partial","*",3,0],"ChatGPT-User":["partial","*",3,0],"OAI-SearchBot":["partial","*",3,0],"ClaudeBot":["partial","*",3,0],"Claude-User":["partial","*",3,0],"Claude-SearchBot":["partial","*",3,0],"anthropic-ai":["partial","*",3,0],"PerplexityBot":["partial","*",3,0],"Perplexity-User":["partial","*",3,0],"Google-Extended":["partial","*",3,0],"Applebot-Extended":["partial","*",3,0],"CCBot":["partial","*",3,0],"Bytespider":["partial","*",3,0],"Amazonbot":["partial","*",3,0],"meta-externalagent":["partial","*",3,0],"cohere-ai":["partial","*",3,0],"Diffbot":["partial","*",3,0],"ContextFetchBot":["partial","*",3,0]},"llms":{"status":"present","http":200,"bytes":30281},"home":{"status":"ok","http":200,"url":"https://majestic.com/","html_tokens":27365,"markdown_tokens":3548,"pruned_tokens":3196}},
{"rank":6171,"domain":"middlebury.edu","origin":"https://middlebury.edu","robots":{"status":"refused","http":403,"url":"https://middlebury.edu/robots.txt","bytes":560,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":6172,"domain":"sumupstore.com","origin":"https://sumupstore.com","robots":{"status":"missing","http":404,"url":"https://sumupstore.com/robots.txt","bytes":0,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":0},"home":{"status":"http_error","http":403,"url":"https://sumupstore.com/"}},
{"rank":6173,"domain":"sumup.com","origin":"https://sumup.com","robots":{"status":"ok","http":200,"url":"https://sumup.com/robots.txt","bytes":0,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":153283},"home":{"status":"ok","http":200,"url":"https://www.sumup.com/en-us/","html_tokens":248069,"markdown_tokens":357,"pruned_tokens":217}},
{"rank":6174,"domain":"marketingdive.com","origin":"https://marketingdive.com","robots":{"status":"ok","http":200,"url":"https://www.marketingdive.com/robots.txt","bytes":4954,"groups":13},"bots":{"GPTBot":["partial","*",15,2],"ChatGPT-User":["partial","*",15,2],"OAI-SearchBot":["partial","*",15,2],"ClaudeBot":["partial","*",15,2],"Claude-User":["partial","*",15,2],"Claude-SearchBot":["partial","*",15,2],"anthropic-ai":["partial","*",15,2],"PerplexityBot":["partial","*",15,2],"Perplexity-User":["partial","*",15,2],"Google-Extended":["partial","*",15,2],"Applebot-Extended":["partial","*",15,2],"CCBot":["partial","*",15,2],"Bytespider":["partial","*",15,2],"Amazonbot":["partial","named",17,2],"meta-externalagent":["partial","*",15,2],"cohere-ai":["partial","*",15,2],"Diffbot":["partial","*",15,2],"ContextFetchBot":["partial","*",15,2]},"llms":{"status":"missing","http":404,"bytes":132799},"home":{"status":"ok","http":200,"url":"https://www.marketingdive.com/","html_tokens":67870,"markdown_tokens":826,"pruned_tokens":153}},
{"rank":6175,"domain":"society6.com","origin":"https://society6.com","robots":{"status":"ok","http":200,"url":"https://society6.com/robots.txt","bytes":8642,"groups":6},"bots":{"GPTBot":["partial","*",227,0],"ChatGPT-User":["partial","*",227,0],"OAI-SearchBot":["partial","*",227,0],"ClaudeBot":["partial","*",227,0],"Claude-User":["partial","*",227,0],"Claude-SearchBot":["partial","*",227,0],"anthropic-ai":["partial","*",227,0],"PerplexityBot":["partial","*",227,0],"Perplexity-User":["partial","*",227,0],"Google-Extended":["partial","*",227,0],"Applebot-Extended":["partial","*",227,0],"CCBot":["partial","*",227,0],"Bytespider":["partial","*",227,0],"Amazonbot":["partial","*",227,0],"meta-externalagent":["partial","*",227,0],"cohere-ai":["partial","*",227,0],"Diffbot":["partial","*",227,0],"ContextFetchBot":["partial","*",227,0]},"llms":{"status":"present","http":200,"bytes":5765},"home":{"status":"ok","http":200,"url":"https://society6.com/","html_tokens":238026,"markdown_tokens":290,"pruned_tokens":279}},
{"rank":6176,"domain":"olx.ua","origin":"https://olx.ua","robots":{"status":"refused","http":403,"url":"https://olx.ua/robots.txt","bytes":919,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":6177,"domain":"gamespy.com","origin":"https://gamespy.com","robots":{"status":"unreachable","http":null,"url":"https://gamespy.com/robots.txt","bytes":0,"groups":0,"error":"cannot resolve gamespy.com"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":6178,"domain":"soso.com","origin":"https://soso.com","robots":{"status":"ok","http":200,"url":"https://soso.com/robots.txt","bytes":10474,"groups":15},"bots":{"GPTBot":["blocked","*",1,0],"ChatGPT-User":["blocked","*",1,0],"OAI-SearchBot":["blocked","*",1,0],"ClaudeBot":["blocked","*",1,0],"Claude-User":["blocked","*",1,0],"Claude-SearchBot":["blocked","*",1,0],"anthropic-ai":["blocked","*",1,0],"PerplexityBot":["blocked","*",1,0],"Perplexity-User":["blocked","*",1,0],"Google-Extended":["blocked","*",1,0],"Applebot-Extended":["blocked","*",1,0],"CCBot":["blocked","*",1,0],"Bytespider":["blocked","*",1,0],"Amazonbot":["blocked","*",1,0],"meta-externalagent":["blocked","*",1,0],"cohere-ai":["blocked","*",1,0],"Diffbot":["blocked","*",1,0],"ContextFetchBot":["blocked","*",1,0]},"llms":{"status":"disallowed"},"home":{"status":"disallowed"}},
{"rank":6179,"domain":"espn.co.uk","origin":"https://espn.co.uk","robots":{"status":"ok","http":200,"url":"http://www.espn.co.uk/robots.txt","bytes":3457,"groups":11},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["blocked","named",1,0],"OAI-SearchBot":["partial","*",118,3],"ClaudeBot":["partial","*",118,3],"Claude-User":["partial","*",118,3],"Claude-SearchBot":["partial","*",118,3],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["partial","*",118,3],"Perplexity-User":["partial","*",118,3],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["partial","*",118,3],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["partial","*",118,3],"meta-externalagent":["partial","*",118,3],"cohere-ai":["partial","*",118,3],"Diffbot":["partial","*",118,3],"ContextFetchBot":["partial","*",118,3]},"llms":{"status":"missing","http":404,"bytes":1},"home":{"status":"ok","http":200,"url":"https://www.espn.co.uk/","html_tokens":64325,"markdown_tokens":887,"pruned_tokens":122}},
{"rank":6180,"domain":"kxan.com","origin":"https://kxan.com","robots":{"status":"ok","http":200,"url":"https://www.kxan.com/robots.txt","bytes":2305,"groups":40},"bots":{"GPTBot":["blocked","named",2,0],"ChatGPT-User":["blocked","named",1,0],"OAI-SearchBot":["blocked","named",1,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",12,1],"Claude-SearchBot":["partial","*",12,1],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["blocked","named",2,0],"Perplexity-User":["partial","*",12,1],"Google-Extended":["blocked","named",2,0],"Applebot-Extended":["blocked","named",2,0],"CCBot":["blocked","named",2,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["partial","*",12,1],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",12,1]},"llms":{"status":"http_error","http":403,"bytes":6514},"home":{"status":"http_error","http":403,"url":"https://kxan.com/"}},
{"rank":6181,"domain":"rakuten.ne.jp","origin":"https://www.rakuten.ne.jp","robots":{"status":"ok","http":200,"url":"https://www.rakuten.ne.jp/robots.txt","bytes":1649,"groups":1},"bots":{"GPTBot":["partial","*",2,106],"ChatGPT-User":["partial","*",2,106],"OAI-SearchBot":["partial","*",2,106],"ClaudeBot":["partial","*",2,106],"Claude-User":["partial","*",2,106],"Claude-SearchBot":["partial","*",2,106],"anthropic-ai":["partial","*",2,106],"PerplexityBot":["partial","*",2,106],"Perplexity-User":["partial","*",2,106],"Google-Extended":["partial","*",2,106],"Applebot-Extended":["partial","*",2,106],"CCBot":["partial","*",2,106],"Bytespider":["partial","*",2,106],"Amazonbot":["partial","*",2,106],"meta-externalagent":["partial","*",2,106],"cohere-ai":["partial","*",2,106],"Diffbot":["partial","*",2,106],"ContextFetchBot":["partial","*",2,106]},"llms":{"status":"disallowed"},"home":{"status":"ok","http":200,"url":"https://www.rakuten.co.jp/","html_tokens":42101,"markdown_tokens":20,"pruned_tokens":20}},
{"rank":6182,"domain":"cartier.com","origin":"https://cartier.com","robots":{"status":"ok","http":200,"url":"https://www.cartier.com/robots.txt","bytes":5874,"groups":1},"bots":{"GPTBot":["partial","*",210,0],"ChatGPT-User":["partial","*",210,0],"OAI-SearchBot":["partial","*",210,0],"ClaudeBot":["partial","*",210,0],"Claude-User":["partial","*",210,0],"Claude-SearchBot":["partial","*",210,0],"anthropic-ai":["partial","*",210,0],"PerplexityBot":["partial","*",210,0],"Perplexity-User":["partial","*",210,0],"Google-Extended":["partial","*",210,0],"Applebot-Extended":["partial","*",210,0],"CCBot":["partial","*",210,0],"Bytespider":["partial","*",210,0],"Amazonbot":["partial","*",210,0],"meta-externalagent":["partial","*",210,0],"cohere-ai":["partial","*",210,0],"Diffbot":["partial","*",210,0],"ContextFetchBot":["partial","*",210,0]},"llms":{"status":"timeout","error":"timed out after 10 s"},"home":{"status":"ok","http":200,"url":"https://www.cartier.com/en-us/home","html_tokens":232489,"markdown_tokens":181,"pruned_tokens":29}},
{"rank":6183,"domain":"mitsubishielectric.com","origin":"https://mitsubishielectric.com","robots":{"status":"ok","http":200,"url":"https://www.mitsubishielectric.com/robots.txt","bytes":169,"groups":1},"bots":{"GPTBot":["partial","*",2,0],"ChatGPT-User":["partial","*",2,0],"OAI-SearchBot":["partial","*",2,0],"ClaudeBot":["partial","*",2,0],"Claude-User":["partial","*",2,0],"Claude-SearchBot":["partial","*",2,0],"anthropic-ai":["partial","*",2,0],"PerplexityBot":["partial","*",2,0],"Perplexity-User":["partial","*",2,0],"Google-Extended":["partial","*",2,0],"Applebot-Extended":["partial","*",2,0],"CCBot":["partial","*",2,0],"Bytespider":["partial","*",2,0],"Amazonbot":["partial","*",2,0],"meta-externalagent":["partial","*",2,0],"cohere-ai":["partial","*",2,0],"Diffbot":["partial","*",2,0],"ContextFetchBot":["partial","*",2,0]},"llms":{"status":"missing","http":404,"bytes":43218},"home":{"status":"ok","http":200,"url":"https://www.mitsubishielectric.com/en/","html_tokens":23318,"markdown_tokens":24,"pruned_tokens":24}},
{"rank":6184,"domain":"oreillynet.com","origin":"https://oreillynet.com","robots":{"status":"ok","http":200,"url":"https://www.oreilly.com/robots.txt","bytes":3450,"groups":7},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["partial","named",1,1],"OAI-SearchBot":["partial","named",1,1],"ClaudeBot":["partial","named",1,1],"Claude-User":["partial","*",19,0],"Claude-SearchBot":["partial","named",1,1],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["partial","named",1,1],"Perplexity-User":["partial","named",1,1],"Google-Extended":["partial","named",1,1],"Applebot-Extended":["partial","named",1,1],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",19,0]},"llms":{"status":"present","http":200,"bytes":18762},"home":{"status":"ok","http":200,"url":"https://www.oreilly.com/","html_tokens":79011,"markdown_tokens":458,"pruned_tokens":148}},
{"rank":6185,"domain":"booksamillion.com","origin":"https://booksamillion.com","robots":{"status":"refused","http":403,"url":"https://booksamillion.com/robots.txt","bytes":5534,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":6186,"domain":"ga.us","origin":"https://ga.us","robots":{"status":"unreachable","http":null,"url":"https://ga.us/robots.txt","bytes":0,"groups":0,"error":"cannot resolve ga.us"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":6187,"domain":"cvshome.org","origin":"https://cvshome.org","robots":{"status":"ok","http":200,"url":"https://cvshome.org/robots.txt","bytes":1248,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"html","http":200,"bytes":6547},"home":{"status":"ok","http":200,"url":"https://cvshome.org/","html_tokens":1637,"markdown_tokens":22,"pruned_tokens":22}},
{"rank":6188,"domain":"werite.net","origin":"https://werite.net","robots":{"status":"ok","http":200,"url":"https://werite.net/robots.txt","bytes":1248,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"http_error","http":530,"bytes":7619},"home":{"status":"http_error","http":530,"url":"https://werite.net/"}},
{"rank":6189,"domain":"ulta.com","origin":"https://ulta.com","robots":{"status":"ok","http":200,"url":"https://www.ulta.com/robots.txt","bytes":2483,"groups":10},"bots":{"GPTBot":["partial","named",8,0],"ChatGPT-User":["partial","named",6,0],"OAI-SearchBot":["partial","named",6,0],"ClaudeBot":["partial","named",8,1],"Claude-User":["partial","named",8,1],"Claude-SearchBot":["partial","named",8,1],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["partial","named",8,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["partial","named",8,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["partial","named",8,0],"Bytespider":["partial","named",8,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["partial","named",8,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"timeout","error":"timed out after 10 s"},"home":{"status":"ok","http":200,"url":"https://www.ulta.com/","html_tokens":407372,"markdown_tokens":148,"pruned_tokens":148}},
{"rank":6190,"domain":"google.mn","origin":"https://google.mn","robots":{"status":"ok","http":200,"url":"https://www.google.mn/robots.txt","bytes":6624,"groups":4},"bots":{"GPTBot":["partial","*",169,64],"ChatGPT-User":["partial","*",169,64],"OAI-SearchBot":["partial","*",169,64],"ClaudeBot":["partial","*",169,64],"Claude-User":["partial","*",169,64],"Claude-SearchBot":["partial","*",169,64],"anthropic-ai":["partial","*",169,64],"PerplexityBot":["partial","*",169,64],"Perplexity-User":["partial","*",169,64],"Google-Extended":["partial","*",169,64],"Applebot-Extended":["partial","*",169,64],"CCBot":["partial","*",169,64],"Bytespider":["partial","*",169,64],"Amazonbot":["partial","*",169,64],"meta-externalagent":["partial","*",169,64],"cohere-ai":["partial","*",169,64],"Diffbot":["partial","*",169,64],"ContextFetchBot":["partial","*",169,64]},"llms":{"status":"missing","http":404,"bytes":1569},"home":{"status":"ok","http":200,"url":"https://www.google.com/","html_tokens":21461,"markdown_tokens":30,"pruned_tokens":30}},
{"rank":6191,"domain":"nrc.gov","origin":"https://nrc.gov","robots":{"status":"html","http":200,"url":"https://www.nrc.gov/","bytes":130701,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":6192,"domain":"uni-giessen.de","origin":"https://uni-giessen.de","robots":{"status":"ok","http":200,"url":"https://www.uni-giessen.de/robots.txt","bytes":880,"groups":3},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"missing","http":404,"bytes":32292},"home":{"status":"ok","http":200,"url":"https://www.uni-giessen.de/de","html_tokens":21419,"markdown_tokens":274,"pruned_tokens":168}},
{"rank":6193,"domain":"digital.gov.ru","origin":"https://digital.gov.ru","robots":{"status":"timeout","http":null,"url":"https://digital.gov.ru/robots.txt","bytes":0,"groups":0,"error":"ConnectTimeout"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":6194,"domain":"openweathermap.org","origin":"https://openweathermap.org","robots":{"status":"ok","http":200,"url":"https://openweathermap.org/robots.txt","bytes":72,"groups":1},"bots":{"GPTBot":["allowed","*",0,1],"ChatGPT-User":["allowed","*",0,1],"OAI-SearchBot":["allowed","*",0,1],"ClaudeBot":["allowed","*",0,1],"Claude-User":["allowed","*",0,1],"Claude-SearchBot":["allowed","*",0,1],"anthropic-ai":["allowed","*",0,1],"PerplexityBot":["allowed","*",0,1],"Perplexity-User":["allowed","*",0,1],"Google-Extended":["allowed","*",0,1],"Applebot-Extended":["allowed","*",0,1],"CCBot":["allowed","*",0,1],"Bytespider":["allowed","*",0,1],"Amazonbot":["allowed","*",0,1],"meta-externalagent":["allowed","*",0,1],"cohere-ai":["allowed","*",0,1],"Diffbot":["allowed","*",0,1],"ContextFetchBot":["allowed","*",0,1]},"llms":{"status":"present","http":200,"bytes":3018},"home":{"status":"ok","http":200,"url":"https://openweathermap.org/","html_tokens":28581,"markdown_tokens":185,"pruned_tokens":48}},
{"rank":6195,"domain":"spie.org","origin":"https://spie.org","robots":{"status":"ok","http":200,"url":"https://spie.org/robots.txt","bytes":394,"groups":1},"bots":{"GPTBot":["partial","*",11,4],"ChatGPT-User":["partial","*",11,4],"OAI-SearchBot":["partial","*",11,4],"ClaudeBot":["partial","*",11,4],"Claude-User":["partial","*",11,4],"Claude-SearchBot":["partial","*",11,4],"anthropic-ai":["partial","*",11,4],"PerplexityBot":["partial","*",11,4],"Perplexity-User":["partial","*",11,4],"Google-Extended":["partial","*",11,4],"Applebot-Extended":["partial","*",11,4],"CCBot":["partial","*",11,4],"Bytespider":["partial","*",11,4],"Amazonbot":["partial","*",11,4],"meta-externalagent":["partial","*",11,4],"cohere-ai":["partial","*",11,4],"Diffbot":["partial","*",11,4],"ContextFetchBot":["partial","*",11,4]},"llms":{"status":"missing","http":404,"bytes":198887},"home":{"status":"ok","http":200,"url":"https://spie.org/","html_tokens":289,"markdown_tokens":21,"pruned_tokens":21}},
{"rank":6196,"domain":"uni-graz.at","origin":"https://uni-graz.at","robots":{"status":"ok","http":200,"url":"https://uni-graz.at/robots.txt","bytes":500,"groups":1},"bots":{"GPTBot":["partial","*",22,0],"ChatGPT-User":["partial","*",22,0],"OAI-SearchBot":["partial","*",22,0],"ClaudeBot":["partial","*",22,0],"Claude-User":["partial","*",22,0],"Claude-SearchBot":["partial","*",22,0],"anthropic-ai":["partial","*",22,0],"PerplexityBot":["partial","*",22,0],"Perplexity-User":["partial","*",22,0],"Google-Extended":["partial","*",22,0],"Applebot-Extended":["partial","*",22,0],"CCBot":["partial","*",22,0],"Bytespider":["partial","*",22,0],"Amazonbot":["partial","*",22,0],"meta-externalagent":["partial","*",22,0],"cohere-ai":["partial","*",22,0],"Diffbot":["partial","*",22,0],"ContextFetchBot":["partial","*",22,0]},"llms":{"status":"missing","http":404,"bytes":87698},"home":{"status":"ok","http":200,"url":"https://www.uni-graz.at/de/","html_tokens":30687,"markdown_tokens":137,"pruned_tokens":85}},
{"rank":6197,"domain":"writesonic.com","origin":"https://writesonic.com","robots":{"status":"ok","http":200,"url":"https://writesonic.com/robots.txt","bytes":164,"groups":1},"bots":{"GPTBot":["allowed","*",0,1],"ChatGPT-User":["allowed","*",0,1],"OAI-SearchBot":["allowed","*",0,1],"ClaudeBot":["allowed","*",0,1],"Claude-User":["allowed","*",0,1],"Claude-SearchBot":["allowed","*",0,1],"anthropic-ai":["allowed","*",0,1],"PerplexityBot":["allowed","*",0,1],"Perplexity-User":["allowed","*",0,1],"Google-Extended":["allowed","*",0,1],"Applebot-Extended":["allowed","*",0,1],"CCBot":["allowed","*",0,1],"Bytespider":["allowed","*",0,1],"Amazonbot":["allowed","*",0,1],"meta-externalagent":["allowed","*",0,1],"cohere-ai":["allowed","*",0,1],"Diffbot":["allowed","*",0,1],"ContextFetchBot":["allowed","*",0,1]},"llms":{"status":"present","http":200,"bytes":11720},"home":{"status":"ok","http":200,"url":"https://writesonic.com/","html_tokens":184571,"markdown_tokens":973,"pruned_tokens":99}},
{"rank":6198,"domain":"nv.us","origin":"https://nv.us","robots":{"status":"unreachable","http":null,"url":"https://nv.us/robots.txt","bytes":0,"groups":0,"error":"cannot resolve nv.us"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":6199,"domain":"constitutioncenter.org","origin":"https://constitutioncenter.org","robots":{"status":"ok","http":200,"url":"https://constitutioncenter.org/robots.txt","bytes":113,"groups":1},"bots":{"GPTBot":["partial","*",4,1],"ChatGPT-User":["partial","*",4,1],"OAI-SearchBot":["partial","*",4,1],"ClaudeBot":["partial","*",4,1],"Claude-User":["partial","*",4,1],"Claude-SearchBot":["partial","*",4,1],"anthropic-ai":["partial","*",4,1],"PerplexityBot":["partial","*",4,1],"Perplexity-User":["partial","*",4,1],"Google-Extended":["partial","*",4,1],"Applebot-Extended":["partial","*",4,1],"CCBot":["partial","*",4,1],"Bytespider":["partial","*",4,1],"Amazonbot":["partial","*",4,1],"meta-externalagent":["partial","*",4,1],"cohere-ai":["partial","*",4,1],"Diffbot":["partial","*",4,1],"ContextFetchBot":["partial","*",4,1]},"llms":{"status":"present","http":200,"bytes":12901},"home":{"status":"ok","http":200,"url":"https://constitutioncenter.org/","html_tokens":27388,"markdown_tokens":872,"pruned_tokens":67}},
{"rank":6200,"domain":"sprinklr.com","origin":"https://sprinklr.com","robots":{"status":"ok","http":200,"url":"https://www.sprinklr.com/robots.txt","bytes":483,"groups":1},"bots":{"GPTBot":["partial","*",16,1],"ChatGPT-User":["partial","*",16,1],"OAI-SearchBot":["partial","*",16,1],"ClaudeBot":["partial","*",16,1],"Claude-User":["partial","*",16,1],"Claude-SearchBot":["partial","*",16,1],"anthropic-ai":["partial","*",16,1],"PerplexityBot":["partial","*",16,1],"Perplexity-User":["partial","*",16,1],"Google-Extended":["partial","*",16,1],"Applebot-Extended":["partial","*",16,1],"CCBot":["partial","*",16,1],"Bytespider":["partial","*",16,1],"Amazonbot":["partial","*",16,1],"meta-externalagent":["partial","*",16,1],"cohere-ai":["partial","*",16,1],"Diffbot":["partial","*",16,1],"ContextFetchBot":["partial","*",16,1]},"llms":{"status":"present","http":200,"bytes":54672},"home":{"status":"ok","http":200,"url":"https://www.sprinklr.com/","html_tokens":175678,"markdown_tokens":950,"pruned_tokens":371}},
{"rank":6201,"domain":"talksport.com","origin":"https://talksport.com","robots":{"status":"ok","http":200,"url":"https://talksport.com/robots.txt","bytes":2443,"groups":2},"bots":{"GPTBot":["partial","named",40,3],"ChatGPT-User":["partial","named",40,3],"OAI-SearchBot":["blocked","*",1,0],"ClaudeBot":["blocked","*",1,0],"Claude-User":["blocked","*",1,0],"Claude-SearchBot":["blocked","*",1,0],"anthropic-ai":["blocked","*",1,0],"PerplexityBot":["blocked","*",1,0],"Perplexity-User":["blocked","*",1,0],"Google-Extended":["blocked","*",1,0],"Applebot-Extended":["blocked","*",1,0],"CCBot":["blocked","*",1,0],"Bytespider":["blocked","*",1,0],"Amazonbot":["blocked","*",1,0],"meta-externalagent":["blocked","*",1,0],"cohere-ai":["blocked","*",1,0],"Diffbot":["blocked","*",1,0],"ContextFetchBot":["blocked","*",1,0]},"llms":{"status":"disallowed"},"home":{"status":"disallowed"}},
{"rank":6202,"domain":"ohio.edu","origin":"https://ohio.edu","robots":{"status":"ok","http":200,"url":"https://www.ohio.edu/robots.txt","bytes":1346,"groups":1},"bots":{"GPTBot":["partial","*",32,22],"ChatGPT-User":["partial","*",32,22],"OAI-SearchBot":["partial","*",32,22],"ClaudeBot":["partial","*",32,22],"Claude-User":["partial","*",32,22],"Claude-SearchBot":["partial","*",32,22],"anthropic-ai":["partial","*",32,22],"PerplexityBot":["partial","*",32,22],"Perplexity-User":["partial","*",32,22],"Google-Extended":["partial","*",32,22],"Applebot-Extended":["partial","*",32,22],"CCBot":["partial","*",32,22],"Bytespider":["partial","*",32,22],"Amazonbot":["partial","*",32,22],"meta-externalagent":["partial","*",32,22],"cohere-ai":["partial","*",32,22],"Diffbot":["partial","*",32,22],"ContextFetchBot":["partial","*",32,22]},"llms":{"status":"missing","http":404,"bytes":186},"home":{"status":"ok","http":200,"url":"https://www.ohio.edu/","html_tokens":39775,"markdown_tokens":428,"pruned_tokens":136}},
{"rank":6203,"domain":"tcl.com","origin":"https://tcl.com","robots":{"status":"ok","http":200,"url":"https://www.tcl.com/robots.txt","bytes":5730,"groups":14},"bots":{"GPTBot":["allowed","named",0,1],"ChatGPT-User":["allowed","named",0,1],"OAI-SearchBot":["allowed","named",0,1],"ClaudeBot":["allowed","named",0,1],"Claude-User":["allowed","named",0,1],"Claude-SearchBot":["allowed","named",0,1],"anthropic-ai":["partial","*",35,5],"PerplexityBot":["allowed","named",0,1],"Perplexity-User":["allowed","named",0,1],"Google-Extended":["allowed","named",0,1],"Applebot-Extended":["allowed","named",0,1],"CCBot":["partial","*",35,5],"Bytespider":["partial","*",35,5],"Amazonbot":["allowed","named",0,1],"meta-externalagent":["allowed","named",0,1],"cohere-ai":["partial","*",35,5],"Diffbot":["partial","*",35,5],"ContextFetchBot":["partial","*",35,5]},"llms":{"status":"missing","http":404,"bytes":487},"home":{"status":"ok","http":200,"url":"https://us.tcl.com/","html_tokens":576045,"markdown_tokens":1693,"pruned_tokens":175}},
{"rank":6204,"domain":"openstreetmap.de","origin":"https://openstreetmap.de","robots":{"status":"missing","http":404,"url":"https://openstreetmap.de/robots.txt","bytes":236,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":236},"home":{"status":"ok","http":200,"url":"https://openstreetmap.de/","html_tokens":3159,"markdown_tokens":407,"pruned_tokens":397}},
{"rank":6205,"domain":"defenseone.com","origin":"https://defenseone.com","robots":{"status":"ok","http":200,"url":"https://www.defenseone.com/robots.txt","bytes":204,"groups":3},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["partial","*",5,0],"OAI-SearchBot":["partial","*",5,0],"ClaudeBot":["partial","*",5,0],"Claude-User":["partial","*",5,0],"Claude-SearchBot":["partial","*",5,0],"anthropic-ai":["partial","*",5,0],"PerplexityBot":["partial","*",5,0],"Perplexity-User":["partial","*",5,0],"Google-Extended":["partial","*",5,0],"Applebot-Extended":["partial","*",5,0],"CCBot":["partial","*",5,0],"Bytespider":["partial","*",5,0],"Amazonbot":["partial","*",5,0],"meta-externalagent":["partial","*",5,0],"cohere-ai":["partial","*",5,0],"Diffbot":["partial","*",5,0],"ContextFetchBot":["partial","*",5,0]},"llms":{"status":"missing","http":404,"bytes":153},"home":{"status":"ok","http":200,"url":"https://www.defenseone.com/","html_tokens":49519,"markdown_tokens":95,"pruned_tokens":50}},
{"rank":6206,"domain":"convio.net","origin":"http://convio.net","robots":{"status":"missing","http":404,"url":"https://www.blackbaud.com/site-not-found","bytes":166781,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":166781},"home":{"status":"http_error","http":404,"url":"https://www.blackbaud.com/site-not-found"}},
{"rank":6207,"domain":"progressive.com","origin":"https://progressive.com","robots":{"status":"ok","http":200,"url":"https://www.progressive.com/robots.txt","bytes":1919,"groups":3},"bots":{"GPTBot":["partial","*",25,1],"ChatGPT-User":["partial","*",25,1],"OAI-SearchBot":["partial","*",25,1],"ClaudeBot":["partial","*",25,1],"Claude-User":["partial","*",25,1],"Claude-SearchBot":["partial","*",25,1],"anthropic-ai":["partial","*",25,1],"PerplexityBot":["partial","*",25,1],"Perplexity-User":["partial","*",25,1],"Google-Extended":["partial","*",25,1],"Applebot-Extended":["partial","*",25,1],"CCBot":["partial","*",25,1],"Bytespider":["partial","*",25,1],"Amazonbot":["partial","*",25,1],"meta-externalagent":["partial","*",25,1],"cohere-ai":["partial","*",25,1],"Diffbot":["partial","*",25,1],"ContextFetchBot":["partial","*",25,1]},"llms":{"status":"missing","http":404,"bytes":282120},"home":{"status":"ok","http":200,"url":"https://www.progressive.com/","html_tokens":86442,"markdown_tokens":322,"pruned_tokens":218}},
{"rank":6208,"domain":"nationwide.com","origin":"https://nationwide.com","robots":{"status":"ok","http":200,"url":"https://www.nationwide.com/robots.txt","bytes":3548,"groups":1},"bots":{"GPTBot":["partial","*",93,0],"ChatGPT-User":["partial","*",93,0],"OAI-SearchBot":["partial","*",93,0],"ClaudeBot":["partial","*",93,0],"Claude-User":["partial","*",93,0],"Claude-SearchBot":["partial","*",93,0],"anthropic-ai":["partial","*",93,0],"PerplexityBot":["partial","*",93,0],"Perplexity-User":["partial","*",93,0],"Google-Extended":["partial","*",93,0],"Applebot-Extended":["partial","*",93,0],"CCBot":["partial","*",93,0],"Bytespider":["partial","*",93,0],"Amazonbot":["partial","*",93,0],"meta-externalagent":["partial","*",93,0],"cohere-ai":["partial","*",93,0],"Diffbot":["partial","*",93,0],"ContextFetchBot":["partial","*",93,0]},"llms":{"status":"missing","http":404,"bytes":9},"home":{"status":"ok","http":200,"url":"https://www.nationwide.com/","html_tokens":64519,"markdown_tokens":173,"pruned_tokens":173}},
{"rank":6209,"domain":"pearsonvue.com","origin":"https://www.pearsonvue.com","robots":{"status":"ok","http":200,"url":"https://www.pearsonvue.com/robots.txt","bytes":1789,"groups":2},"bots":{"GPTBot":["partial","named",19,0],"ChatGPT-User":["partial","*",19,0],"OAI-SearchBot":["partial","*",19,0],"ClaudeBot":["partial","named",19,0],"Claude-User":["partial","*",19,0],"Claude-SearchBot":["partial","*",19,0],"anthropic-ai":["partial","*",19,0],"PerplexityBot":["partial","named",19,0],"Perplexity-User":["partial","*",19,0],"Google-Extended":["partial","named",19,0],"Applebot-Extended":["partial","*",19,0],"CCBot":["partial","named",19,0],"Bytespider":["partial","*",19,0],"Amazonbot":["partial","*",19,0],"meta-externalagent":["partial","*",19,0],"cohere-ai":["partial","*",19,0],"Diffbot":["partial","*",19,0],"ContextFetchBot":["partial","*",19,0]},"llms":{"status":"missing","http":404,"bytes":120800},"home":{"status":"ok","http":200,"url":"https://www.pearsonvue.com/","html_tokens":36934,"markdown_tokens":487,"pruned_tokens":138}},
{"rank":6210,"domain":"csulb.edu","origin":"https://csulb.edu","robots":{"status":"ok","http":200,"url":"https://www.csulb.edu/robots.txt","bytes":1628,"groups":1},"bots":{"GPTBot":["partial","*",23,18],"ChatGPT-User":["partial","*",23,18],"OAI-SearchBot":["partial","*",23,18],"ClaudeBot":["partial","*",23,18],"Claude-User":["partial","*",23,18],"Claude-SearchBot":["partial","*",23,18],"anthropic-ai":["partial","*",23,18],"PerplexityBot":["partial","*",23,18],"Perplexity-User":["partial","*",23,18],"Google-Extended":["partial","*",23,18],"Applebot-Extended":["partial","*",23,18],"CCBot":["partial","*",23,18],"Bytespider":["partial","*",23,18],"Amazonbot":["partial","*",23,18],"meta-externalagent":["partial","*",23,18],"cohere-ai":["partial","*",23,18],"Diffbot":["partial","*",23,18],"ContextFetchBot":["partial","*",23,18]},"llms":{"status":"missing","http":404,"bytes":187},"home":{"status":"ok","http":200,"url":"https://www.csulb.edu","html_tokens":23733,"markdown_tokens":104,"pruned_tokens":60}},
{"rank":6211,"domain":"occrp.org","origin":"https://occrp.org","robots":{"status":"ok","http":200,"url":"https://www.occrp.org/robots.txt","bytes":681,"groups":2},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"missing","http":404,"bytes":82092},"home":{"status":"ok","http":200,"url":"https://www.occrp.org/en","html_tokens":144468,"markdown_tokens":445,"pruned_tokens":204}},
{"rank":6212,"domain":"rfa.org","origin":"https://rfa.org","robots":{"status":"ok","http":200,"url":"https://www.rfa.org/arc/outboundfeeds/english/robots.txt","bytes":600,"groups":1},"bots":{"GPTBot":["partial","*",11,0],"ChatGPT-User":["partial","*",11,0],"OAI-SearchBot":["partial","*",11,0],"ClaudeBot":["partial","*",11,0],"Claude-User":["partial","*",11,0],"Claude-SearchBot":["partial","*",11,0],"anthropic-ai":["partial","*",11,0],"PerplexityBot":["partial","*",11,0],"Perplexity-User":["partial","*",11,0],"Google-Extended":["partial","*",11,0],"Applebot-Extended":["partial","*",11,0],"CCBot":["partial","*",11,0],"Bytespider":["partial","*",11,0],"Amazonbot":["partial","*",11,0],"meta-externalagent":["partial","*",11,0],"cohere-ai":["partial","*",11,0],"Diffbot":["partial","*",11,0],"ContextFetchBot":["partial","*",11,0]},"llms":{"status":"missing","http":404,"bytes":43653},"home":{"status":"ok","http":200,"url":"https://www.rfa.org/english/","html_tokens":44733,"markdown_tokens":191,"pruned_tokens":25}},
{"rank":6213,"domain":"hokkaido.jp","origin":"https://hokkaido.jp","robots":{"status":"unreachable","http":null,"url":"https://hokkaido.jp/robots.txt","bytes":0,"groups":0,"error":"cannot resolve hokkaido.jp"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":6214,"domain":"google.is","origin":"https://google.is","robots":{"status":"ok","http":200,"url":"https://www.google.is/robots.txt","bytes":6624,"groups":4},"bots":{"GPTBot":["partial","*",169,64],"ChatGPT-User":["partial","*",169,64],"OAI-SearchBot":["partial","*",169,64],"ClaudeBot":["partial","*",169,64],"Claude-User":["partial","*",169,64],"Claude-SearchBot":["partial","*",169,64],"anthropic-ai":["partial","*",169,64],"PerplexityBot":["partial","*",169,64],"Perplexity-User":["partial","*",169,64],"Google-Extended":["partial","*",169,64],"Applebot-Extended":["partial","*",169,64],"CCBot":["partial","*",169,64],"Bytespider":["partial","*",169,64],"Amazonbot":["partial","*",169,64],"meta-externalagent":["partial","*",169,64],"cohere-ai":["partial","*",169,64],"Diffbot":["partial","*",169,64],"ContextFetchBot":["partial","*",169,64]},"llms":{"status":"missing","http":404,"bytes":1569},"home":{"status":"ok","http":200,"url":"https://www.google.com/","html_tokens":21354,"markdown_tokens":30,"pruned_tokens":30}},
{"rank":6215,"domain":"hao123.com","origin":"https://hao123.com","robots":{"status":"ok","http":200,"url":"https://hao123.com/robots.txt","bytes":569,"groups":16},"bots":{"GPTBot":["blocked","*",1,0],"ChatGPT-User":["blocked","*",1,0],"OAI-SearchBot":["blocked","*",1,0],"ClaudeBot":["blocked","*",1,0],"Claude-User":["blocked","*",1,0],"Claude-SearchBot":["blocked","*",1,0],"anthropic-ai":["blocked","*",1,0],"PerplexityBot":["blocked","*",1,0],"Perplexity-User":["blocked","*",1,0],"Google-Extended":["blocked","*",1,0],"Applebot-Extended":["blocked","*",1,0],"CCBot":["blocked","*",1,0],"Bytespider":["blocked","*",1,0],"Amazonbot":["blocked","*",1,0],"meta-externalagent":["blocked","*",1,0],"cohere-ai":["blocked","*",1,0],"Diffbot":["blocked","*",1,0],"ContextFetchBot":["blocked","*",1,0]},"llms":{"status":"disallowed"},"home":{"status":"disallowed"}},
{"rank":6216,"domain":"asiatimes.com","origin":"https://asiatimes.com","robots":{"status":"ok","http":200,"url":"https://asiatimes.com/robots.txt","bytes":514,"groups":2},"bots":{"GPTBot":["partial","*",5,1],"ChatGPT-User":["partial","*",5,1],"OAI-SearchBot":["partial","*",5,1],"ClaudeBot":["partial","*",5,1],"Claude-User":["partial","*",5,1],"Claude-SearchBot":["partial","*",5,1],"anthropic-ai":["partial","*",5,1],"PerplexityBot":["partial","*",5,1],"Perplexity-User":["partial","*",5,1],"Google-Extended":["partial","*",5,1],"Applebot-Extended":["partial","*",5,1],"CCBot":["partial","*",5,1],"Bytespider":["partial","*",5,1],"Amazonbot":["partial","*",5,1],"meta-externalagent":["partial","*",5,1],"cohere-ai":["partial","*",5,1],"Diffbot":["partial","*",5,1],"ContextFetchBot":["partial","*",5,1]},"llms":{"status":"missing","http":404,"bytes":213132},"home":{"status":"ok","http":200,"url":"https://asiatimes.com/","html_tokens":121708,"markdown_tokens":148,"pruned_tokens":148}},
{"rank":6217,"domain":"doxyatlas.com","origin":"https://doxyatlas.com","robots":{"status":"ok","http":200,"url":"https://doxyatlas.com/robots.txt","bytes":73,"groups":1},"bots":{"GPTBot":["partial","*",1,0],"ChatGPT-User":["partial","*",1,0],"OAI-SearchBot":["partial","*",1,0],"ClaudeBot":["partial","*",1,0],"Claude-User":["partial","*",1,0],"Claude-SearchBot":["partial","*",1,0],"anthropic-ai":["partial","*",1,0],"PerplexityBot":["partial","*",1,0],"Perplexity-User":["partial","*",1,0],"Google-Extended":["partial","*",1,0],"Applebot-Extended":["partial","*",1,0],"CCBot":["partial","*",1,0],"Bytespider":["partial","*",1,0],"Amazonbot":["partial","*",1,0],"meta-externalagent":["partial","*",1,0],"cohere-ai":["partial","*",1,0],"Diffbot":["partial","*",1,0],"ContextFetchBot":["partial","*",1,0]},"llms":{"status":"missing","http":404,"bytes":642},"home":{"status":"ok","http":200,"url":"https://doxyatlas.com/","html_tokens":3013,"markdown_tokens":98,"pruned_tokens":98}},
{"rank":6218,"domain":"camera.it","origin":"https://camera.it","robots":{"status":"html","http":200,"url":"https://www.camera.it","bytes":269,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":6219,"domain":"wsbtv.com","origin":"https://wsbtv.com","robots":{"status":"ok","http":200,"url":"https://www.wsbtv.com/robots.txt","bytes":244,"groups":2},"bots":{"GPTBot":["partial","*",1,0],"ChatGPT-User":["partial","*",1,0],"OAI-SearchBot":["partial","*",1,0],"ClaudeBot":["partial","*",1,0],"Claude-User":["partial","*",1,0],"Claude-SearchBot":["partial","*",1,0],"anthropic-ai":["partial","*",1,0],"PerplexityBot":["partial","*",1,0],"Perplexity-User":["partial","*",1,0],"Google-Extended":["partial","*",1,0],"Applebot-Extended":["partial","*",1,0],"CCBot":["partial","*",1,0],"Bytespider":["partial","*",1,0],"Amazonbot":["partial","*",1,0],"meta-externalagent":["partial","*",1,0],"cohere-ai":["partial","*",1,0],"Diffbot":["partial","*",1,0],"ContextFetchBot":["partial","*",1,0]},"llms":{"status":"missing","http":404,"bytes":318019},"home":{"status":"ok","http":200,"url":"https://www.wsbtv.com/","html_tokens":113019,"markdown_tokens":68,"pruned_tokens":68}},
{"rank":6220,"domain":"invisionapp.com","origin":"https://invisionapp.com","robots":{"status":"html","http":200,"url":"https://miro.com/","bytes":1028900,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":6221,"domain":"pv-magazine.com","origin":"https://pv-magazine.com","robots":{"status":"ok","http":200,"url":"https://www.pv-magazine.com/robots.txt","bytes":193,"groups":1},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"http_error","http":403,"bytes":5568},"home":{"status":"http_error","http":403,"url":"https://pv-magazine.com/"}},
{"rank":6222,"domain":"steamdb.info","origin":"https://steamdb.info","robots":{"status":"ok","http":200,"url":"https://steamdb.info/robots.txt","bytes":1709,"groups":23},"bots":{"GPTBot":["partial","*",23,5],"ChatGPT-User":["partial","*",23,5],"OAI-SearchBot":["partial","*",23,5],"ClaudeBot":["partial","*",23,5],"Claude-User":["partial","*",23,5],"Claude-SearchBot":["partial","*",23,5],"anthropic-ai":["partial","*",23,5],"PerplexityBot":["partial","*",23,5],"Perplexity-User":["partial","*",23,5],"Google-Extended":["partial","*",23,5],"Applebot-Extended":["partial","*",23,5],"CCBot":["partial","*",23,5],"Bytespider":["partial","*",23,5],"Amazonbot":["partial","*",23,5],"meta-externalagent":["partial","*",23,5],"cohere-ai":["partial","*",23,5],"Diffbot":["partial","*",23,5],"ContextFetchBot":["partial","*",23,5]},"llms":{"status":"missing","http":404,"bytes":31236},"home":{"status":"http_error","http":403,"url":"https://steamdb.info/"}},
{"rank":6223,"domain":"streetinsider.com","origin":"https://streetinsider.com","robots":{"status":"ok","http":200,"url":"https://www.streetinsider.com/robots.txt","bytes":693,"groups":8},"bots":{"GPTBot":["partial","*",7,0],"ChatGPT-User":["partial","*",7,0],"OAI-SearchBot":["partial","*",7,0],"ClaudeBot":["partial","*",7,0],"Claude-User":["partial","*",7,0],"Claude-SearchBot":["partial","*",7,0],"anthropic-ai":["partial","*",7,0],"PerplexityBot":["partial","*",7,0],"Perplexity-User":["partial","*",7,0],"Google-Extended":["partial","*",7,0],"Applebot-Extended":["partial","*",7,0],"CCBot":["partial","*",7,0],"Bytespider":["partial","*",7,0],"Amazonbot":["partial","*",7,0],"meta-externalagent":["partial","*",7,0],"cohere-ai":["partial","*",7,0],"Diffbot":["partial","*",7,0],"ContextFetchBot":["partial","*",7,0]},"llms":{"status":"missing","http":404,"bytes":25622},"home":{"status":"http_error","http":403,"url":"https://streetinsider.com/"}},
{"rank":6224,"domain":"bnef.com","origin":"https://bnef.com","robots":{"status":"refused","http":403,"url":"https://www.bnef.com/robots.txt","bytes":380,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":6225,"domain":"caringbridge.org","origin":"https://caringbridge.org","robots":{"status":"ok","http":200,"url":"https://www.caringbridge.org/robots.txt","bytes":209,"groups":2},"bots":{"GPTBot":["partial","*",5,0],"ChatGPT-User":["partial","*",5,0],"OAI-SearchBot":["partial","*",5,0],"ClaudeBot":["partial","*",5,0],"Claude-User":["partial","*",5,0],"Claude-SearchBot":["partial","*",5,0],"anthropic-ai":["partial","*",5,0],"PerplexityBot":["partial","*",5,0],"Perplexity-User":["partial","*",5,0],"Google-Extended":["partial","*",5,0],"Applebot-Extended":["partial","*",5,0],"CCBot":["partial","*",5,0],"Bytespider":["partial","*",5,0],"Amazonbot":["partial","*",5,0],"meta-externalagent":["partial","*",5,0],"cohere-ai":["partial","*",5,0],"Diffbot":["partial","*",5,0],"ContextFetchBot":["partial","*",5,0]},"llms":{"status":"html","http":200,"bytes":13259},"home":{"status":"ok","http":200,"url":"https://www.caringbridge.org/","html_tokens":11624,"markdown_tokens":0,"pruned_tokens":0}},
{"rank":6226,"domain":"gold-bets.org","origin":"https://gold-bets.org","robots":{"status":"unreachable","http":null,"url":"https://gold-bets.org/robots.txt","bytes":0,"groups":0,"error":"cannot resolve gold-bets.org"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":6227,"domain":"ny1.com","origin":"https://ny1.com","robots":{"status":"ok","http":200,"url":"https://ny1.com/robots.txt","bytes":1484,"groups":24},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["partial","*",10,7],"OAI-SearchBot":["partial","*",10,7],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",10,7],"Claude-SearchBot":["partial","*",10,7],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["partial","*",10,7],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",10,7]},"llms":{"status":"disallowed"},"home":{"status":"ok","http":200,"url":"https://ny1.com/nyc/all-boroughs","html_tokens":35645,"markdown_tokens":44,"pruned_tokens":44}},
{"rank":6228,"domain":"live.net","origin":"https://live.net","robots":{"status":"unreachable","http":null,"url":"https://live.net/robots.txt","bytes":0,"groups":0,"error":"cannot resolve live.net"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":6229,"domain":"washingtonian.com","origin":"https://washingtonian.com","robots":{"status":"ok","http":200,"url":"https://washingtonian.com/robots.txt","bytes":167,"groups":1},"bots":{"GPTBot":["partial","*",1,1],"ChatGPT-User":["partial","*",1,1],"OAI-SearchBot":["partial","*",1,1],"ClaudeBot":["partial","*",1,1],"Claude-User":["partial","*",1,1],"Claude-SearchBot":["partial","*",1,1],"anthropic-ai":["partial","*",1,1],"PerplexityBot":["partial","*",1,1],"Perplexity-User":["partial","*",1,1],"Google-Extended":["partial","*",1,1],"Applebot-Extended":["partial","*",1,1],"CCBot":["partial","*",1,1],"Bytespider":["partial","*",1,1],"Amazonbot":["partial","*",1,1],"meta-externalagent":["partial","*",1,1],"cohere-ai":["partial","*",1,1],"Diffbot":["partial","*",1,1],"ContextFetchBot":["partial","*",1,1]},"llms":{"status":"other_text","http":200,"bytes":705423},"home":{"status":"ok","http":200,"url":"https://washingtonian.com/","html_tokens":65571,"markdown_tokens":35,"pruned_tokens":35}},
{"rank":6230,"domain":"ebrd.com","origin":"https://ebrd.com","robots":{"status":"ok","http":200,"url":"https://www.ebrd.com/robots.txt","bytes":68,"groups":1},"bots":{"GPTBot":["allowed","*",0,1],"ChatGPT-User":["allowed","*",0,1],"OAI-SearchBot":["allowed","*",0,1],"ClaudeBot":["allowed","*",0,1],"Claude-User":["allowed","*",0,1],"Claude-SearchBot":["allowed","*",0,1],"anthropic-ai":["allowed","*",0,1],"PerplexityBot":["allowed","*",0,1],"Perplexity-User":["allowed","*",0,1],"Google-Extended":["allowed","*",0,1],"Applebot-Extended":["allowed","*",0,1],"CCBot":["allowed","*",0,1],"Bytespider":["allowed","*",0,1],"Amazonbot":["allowed","*",0,1],"meta-externalagent":["allowed","*",0,1],"cohere-ai":["allowed","*",0,1],"Diffbot":["allowed","*",0,1],"ContextFetchBot":["allowed","*",0,1]},"llms":{"status":"present","http":200,"bytes":3004},"home":{"status":"ok","http":200,"url":"https://www.ebrd.com/","html_tokens":36129,"markdown_tokens":258,"pruned_tokens":176}},
{"rank":6231,"domain":"ctv.ca","origin":"https://ctv.ca","robots":{"status":"ok","http":200,"url":"https://www.ctv.ca/robots.txt","bytes":254,"groups":2},"bots":{"GPTBot":["partial","*",3,0],"ChatGPT-User":["partial","*",3,0],"OAI-SearchBot":["partial","*",3,0],"ClaudeBot":["partial","*",3,0],"Claude-User":["partial","*",3,0],"Claude-SearchBot":["partial","*",3,0],"anthropic-ai":["partial","*",3,0],"PerplexityBot":["partial","*",3,0],"Perplexity-User":["partial","*",3,0],"Google-Extended":["partial","*",3,0],"Applebot-Extended":["partial","*",3,0],"CCBot":["partial","*",3,0],"Bytespider":["partial","*",3,0],"Amazonbot":["partial","*",3,0],"meta-externalagent":["partial","*",3,0],"cohere-ai":["partial","*",3,0],"Diffbot":["partial","*",3,0],"ContextFetchBot":["partial","*",3,0]},"llms":{"status":"html","http":200,"bytes":1811},"home":{"status":"ok","http":200,"url":"https://www.crave.ca/en/ctv","html_tokens":453,"markdown_tokens":0,"pruned_tokens":0}},
{"rank":6232,"domain":"shinobi.jp","origin":"https://shinobi.jp","robots":{"status":"ok","http":200,"url":"https://www.shinobi.jp/robots.txt","bytes":22,"groups":1},"bots":{"GPTBot":["allowed","*",0,1],"ChatGPT-User":["allowed","*",0,1],"OAI-SearchBot":["allowed","*",0,1],"ClaudeBot":["allowed","*",0,1],"Claude-User":["allowed","*",0,1],"Claude-SearchBot":["allowed","*",0,1],"anthropic-ai":["allowed","*",0,1],"PerplexityBot":["allowed","*",0,1],"Perplexity-User":["allowed","*",0,1],"Google-Extended":["allowed","*",0,1],"Applebot-Extended":["allowed","*",0,1],"CCBot":["allowed","*",0,1],"Bytespider":["allowed","*",0,1],"Amazonbot":["allowed","*",0,1],"meta-externalagent":["allowed","*",0,1],"cohere-ai":["allowed","*",0,1],"Diffbot":["allowed","*",0,1],"ContextFetchBot":["allowed","*",0,1]},"llms":{"status":"missing","http":404,"bytes":159},"home":{"status":"ok","http":200,"url":"https://www.shinobi.jp/","html_tokens":1833,"markdown_tokens":97,"pruned_tokens":97}},
{"rank":6233,"domain":"musicbrainz.org","origin":"https://musicbrainz.org","robots":{"status":"ok","http":200,"url":"https://musicbrainz.org/robots.txt","bytes":1009,"groups":7},"bots":{"GPTBot":["partial","*",35,0],"ChatGPT-User":["partial","*",35,0],"OAI-SearchBot":["partial","*",35,0],"ClaudeBot":["partial","*",35,0],"Claude-User":["partial","*",35,0],"Claude-SearchBot":["partial","*",35,0],"anthropic-ai":["partial","*",35,0],"PerplexityBot":["partial","*",35,0],"Perplexity-User":["partial","*",35,0],"Google-Extended":["partial","*",35,0],"Applebot-Extended":["partial","*",35,0],"CCBot":["partial","*",35,0],"Bytespider":["partial","*",35,0],"Amazonbot":["partial","*",35,0],"meta-externalagent":["partial","*",35,0],"cohere-ai":["partial","*",35,0],"Diffbot":["partial","*",35,0],"ContextFetchBot":["partial","*",35,0]},"llms":{"status":"missing","http":404,"bytes":12341},"home":{"status":"ok","http":200,"url":"https://musicbrainz.org/","html_tokens":8916,"markdown_tokens":200,"pruned_tokens":200}},
{"rank":6234,"domain":"moa.gov.cn","origin":"https://www.moa.gov.cn","robots":{"status":"ok","http":200,"url":"https://www.moa.gov.cn/robots.txt","bytes":24,"groups":1},"bots":{"GPTBot":["blocked","*",1,0],"ChatGPT-User":["blocked","*",1,0],"OAI-SearchBot":["blocked","*",1,0],"ClaudeBot":["blocked","*",1,0],"Claude-User":["blocked","*",1,0],"Claude-SearchBot":["blocked","*",1,0],"anthropic-ai":["blocked","*",1,0],"PerplexityBot":["blocked","*",1,0],"Perplexity-User":["blocked","*",1,0],"Google-Extended":["blocked","*",1,0],"Applebot-Extended":["blocked","*",1,0],"CCBot":["blocked","*",1,0],"Bytespider":["blocked","*",1,0],"Amazonbot":["blocked","*",1,0],"meta-externalagent":["blocked","*",1,0],"cohere-ai":["blocked","*",1,0],"Diffbot":["blocked","*",1,0],"ContextFetchBot":["blocked","*",1,0]},"llms":{"status":"disallowed"},"home":{"status":"disallowed"}},
{"rank":6235,"domain":"gob.sv","origin":"https://gob.sv","robots":{"status":"unreachable","http":null,"url":"https://gob.sv/robots.txt","bytes":0,"groups":0,"error":"cannot resolve gob.sv"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":6236,"domain":"williams.edu","origin":"https://williams.edu","robots":{"status":"refused","http":403,"url":"https://williams.edu/robots.txt","bytes":5528,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":6237,"domain":"medicaid.gov","origin":"https://medicaid.gov","robots":{"status":"ok","http":200,"url":"https://www.medicaid.gov/robots.txt","bytes":2687,"groups":1},"bots":{"GPTBot":["partial","*",56,18],"ChatGPT-User":["partial","*",56,18],"OAI-SearchBot":["partial","*",56,18],"ClaudeBot":["partial","*",56,18],"Claude-User":["partial","*",56,18],"Claude-SearchBot":["partial","*",56,18],"anthropic-ai":["partial","*",56,18],"PerplexityBot":["partial","*",56,18],"Perplexity-User":["partial","*",56,18],"Google-Extended":["partial","*",56,18],"Applebot-Extended":["partial","*",56,18],"CCBot":["partial","*",56,18],"Bytespider":["partial","*",56,18],"Amazonbot":["partial","*",56,18],"meta-externalagent":["partial","*",56,18],"cohere-ai":["partial","*",56,18],"Diffbot":["partial","*",56,18],"ContextFetchBot":["partial","*",56,18]},"llms":{"status":"missing","http":404,"bytes":30487},"home":{"status":"ok","http":200,"url":"https://www.medicaid.gov/","html_tokens":38384,"markdown_tokens":89,"pruned_tokens":70}},
{"rank":6238,"domain":"palgrave.com","origin":"https://palgrave.com","robots":{"status":"ok","http":200,"url":"https://www.palgrave.com/robots.txt","bytes":286,"groups":1},"bots":{"GPTBot":["partial","*",6,2],"ChatGPT-User":["partial","*",6,2],"OAI-SearchBot":["partial","*",6,2],"ClaudeBot":["partial","*",6,2],"Claude-User":["partial","*",6,2],"Claude-SearchBot":["partial","*",6,2],"anthropic-ai":["partial","*",6,2],"PerplexityBot":["partial","*",6,2],"Perplexity-User":["partial","*",6,2],"Google-Extended":["partial","*",6,2],"Applebot-Extended":["partial","*",6,2],"CCBot":["partial","*",6,2],"Bytespider":["partial","*",6,2],"Amazonbot":["partial","*",6,2],"meta-externalagent":["partial","*",6,2],"cohere-ai":["partial","*",6,2],"Diffbot":["partial","*",6,2],"ContextFetchBot":["partial","*",6,2]},"llms":{"status":"html","http":200,"bytes":251243},"home":{"status":"ok","http":200,"url":"https://link.springer.com/brands/palgrave","html_tokens":62808,"markdown_tokens":655,"pruned_tokens":138}},
{"rank":6239,"domain":"allmylinks.com","origin":"https://allmylinks.com","robots":{"status":"ok","http":200,"url":"https://allmylinks.com/robots.txt","bytes":474,"groups":1},"bots":{"GPTBot":["partial","*",19,0],"ChatGPT-User":["partial","*",19,0],"OAI-SearchBot":["partial","*",19,0],"ClaudeBot":["partial","*",19,0],"Claude-User":["partial","*",19,0],"Claude-SearchBot":["partial","*",19,0],"anthropic-ai":["partial","*",19,0],"PerplexityBot":["partial","*",19,0],"Perplexity-User":["partial","*",19,0],"Google-Extended":["partial","*",19,0],"Applebot-Extended":["partial","*",19,0],"CCBot":["partial","*",19,0],"Bytespider":["partial","*",19,0],"Amazonbot":["partial","*",19,0],"meta-externalagent":["partial","*",19,0],"cohere-ai":["partial","*",19,0],"Diffbot":["partial","*",19,0],"ContextFetchBot":["partial","*",19,0]},"llms":{"status":"missing","http":404,"bytes":46363},"home":{"status":"ok","http":200,"url":"https://allmylinks.com/","html_tokens":20659,"markdown_tokens":9,"pruned_tokens":9}},
{"rank":6240,"domain":"google.com.pr","origin":"https://google.com.pr","robots":{"status":"ok","http":200,"url":"https://www.google.com.pr/robots.txt","bytes":6624,"groups":4},"bots":{"GPTBot":["partial","*",169,64],"ChatGPT-User":["partial","*",169,64],"OAI-SearchBot":["partial","*",169,64],"ClaudeBot":["partial","*",169,64],"Claude-User":["partial","*",169,64],"Claude-SearchBot":["partial","*",169,64],"anthropic-ai":["partial","*",169,64],"PerplexityBot":["partial","*",169,64],"Perplexity-User":["partial","*",169,64],"Google-Extended":["partial","*",169,64],"Applebot-Extended":["partial","*",169,64],"CCBot":["partial","*",169,64],"Bytespider":["partial","*",169,64],"Amazonbot":["partial","*",169,64],"meta-externalagent":["partial","*",169,64],"cohere-ai":["partial","*",169,64],"Diffbot":["partial","*",169,64],"ContextFetchBot":["partial","*",169,64]},"llms":{"status":"missing","http":404,"bytes":1569},"home":{"status":"ok","http":200,"url":"https://www.google.com/","html_tokens":21287,"markdown_tokens":30,"pruned_tokens":30}},
{"rank":6241,"domain":"uoc.edu","origin":"https://uoc.edu","robots":{"status":"ok","http":200,"url":"https://www.uoc.edu/robots.txt","bytes":2436,"groups":1},"bots":{"GPTBot":["partial","*",52,16],"ChatGPT-User":["partial","*",52,16],"OAI-SearchBot":["partial","*",52,16],"ClaudeBot":["partial","*",52,16],"Claude-User":["partial","*",52,16],"Claude-SearchBot":["partial","*",52,16],"anthropic-ai":["partial","*",52,16],"PerplexityBot":["partial","*",52,16],"Perplexity-User":["partial","*",52,16],"Google-Extended":["partial","*",52,16],"Applebot-Extended":["partial","*",52,16],"CCBot":["partial","*",52,16],"Bytespider":["partial","*",52,16],"Amazonbot":["partial","*",52,16],"meta-externalagent":["partial","*",52,16],"cohere-ai":["partial","*",52,16],"Diffbot":["partial","*",52,16],"ContextFetchBot":["partial","*",52,16]},"llms":{"status":"present","http":200,"bytes":25691},"home":{"status":"ok","http":200,"url":"https://www.uoc.edu/","html_tokens":1392,"markdown_tokens":191,"pruned_tokens":56}},
{"rank":6242,"domain":"themewagon.com","origin":"https://themewagon.com","robots":{"status":"ok","http":200,"url":"https://themewagon.com/robots.txt","bytes":385,"groups":1},"bots":{"GPTBot":["partial","*",8,1],"ChatGPT-User":["partial","*",8,1],"OAI-SearchBot":["partial","*",8,1],"ClaudeBot":["partial","*",8,1],"Claude-User":["partial","*",8,1],"Claude-SearchBot":["partial","*",8,1],"anthropic-ai":["partial","*",8,1],"PerplexityBot":["partial","*",8,1],"Perplexity-User":["partial","*",8,1],"Google-Extended":["partial","*",8,1],"Applebot-Extended":["partial","*",8,1],"CCBot":["partial","*",8,1],"Bytespider":["partial","*",8,1],"Amazonbot":["partial","*",8,1],"meta-externalagent":["partial","*",8,1],"cohere-ai":["partial","*",8,1],"Diffbot":["partial","*",8,1],"ContextFetchBot":["partial","*",8,1]},"llms":{"status":"missing","http":404,"bytes":55989},"home":{"status":"ok","http":200,"url":"https://themewagon.com/","html_tokens":80382,"markdown_tokens":125,"pruned_tokens":125}},
{"rank":6243,"domain":"osaka.jp","origin":"https://osaka.jp","robots":{"status":"unreachable","http":null,"url":"https://osaka.jp/robots.txt","bytes":0,"groups":0,"error":"cannot resolve osaka.jp"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":6244,"domain":"graphicriver.net","origin":"https://graphicriver.net","robots":{"status":"refused","http":403,"url":"https://graphicriver.net/robots.txt","bytes":5575,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":6245,"domain":"ra.co","origin":"https://ra.co","robots":{"status":"ok","http":200,"url":"https://ra.co/robots.txt","bytes":1164,"groups":26},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["blocked","named",1,0],"OAI-SearchBot":["partial","*",10,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",10,0],"Claude-SearchBot":["partial","*",10,0],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["partial","*",10,0],"Google-Extended":["partial","*",10,0],"Applebot-Extended":["partial","*",10,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["partial","*",10,0],"cohere-ai":["partial","*",10,0],"Diffbot":["partial","*",10,0],"ContextFetchBot":["partial","*",10,0]},"llms":{"status":"missing","http":404,"bytes":101866},"home":{"status":"http_error","http":403,"url":"https://ra.co/"}},
{"rank":6246,"domain":"letour.fr","origin":"https://letour.fr","robots":{"status":"ok","http":200,"url":"https://www.letour.fr/robots.txt","bytes":590,"groups":4},"bots":{"GPTBot":["partial","*",9,0],"ChatGPT-User":["partial","*",9,0],"OAI-SearchBot":["partial","*",9,0],"ClaudeBot":["partial","*",9,0],"Claude-User":["partial","*",9,0],"Claude-SearchBot":["partial","*",9,0],"anthropic-ai":["partial","*",9,0],"PerplexityBot":["partial","*",9,0],"Perplexity-User":["partial","*",9,0],"Google-Extended":["partial","*",9,0],"Applebot-Extended":["partial","*",9,0],"CCBot":["partial","*",9,0],"Bytespider":["partial","*",9,0],"Amazonbot":["partial","*",9,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["partial","*",9,0],"Diffbot":["partial","*",9,0],"ContextFetchBot":["partial","*",9,0]},"llms":{"status":"html","http":200,"bytes":228788},"home":{"status":"ok","http":200,"url":"https://www.letour.fr/en/","html_tokens":57189,"markdown_tokens":173,"pruned_tokens":173}},
{"rank":6247,"domain":"rubygems.org","origin":"https://rubygems.org","robots":{"status":"ok","http":200,"url":"https://rubygems.org/robots.txt","bytes":213,"groups":1},"bots":{"GPTBot":["partial","*",5,0],"ChatGPT-User":["partial","*",5,0],"OAI-SearchBot":["partial","*",5,0],"ClaudeBot":["partial","*",5,0],"Claude-User":["partial","*",5,0],"Claude-SearchBot":["partial","*",5,0],"anthropic-ai":["partial","*",5,0],"PerplexityBot":["partial","*",5,0],"Perplexity-User":["partial","*",5,0],"Google-Extended":["partial","*",5,0],"Applebot-Extended":["partial","*",5,0],"CCBot":["partial","*",5,0],"Bytespider":["partial","*",5,0],"Amazonbot":["partial","*",5,0],"meta-externalagent":["partial","*",5,0],"cohere-ai":["partial","*",5,0],"Diffbot":["partial","*",5,0],"ContextFetchBot":["partial","*",5,0]},"llms":{"status":"missing","http":404,"bytes":794},"home":{"status":"ok","http":200,"url":"https://rubygems.org/","html_tokens":9069,"markdown_tokens":133,"pruned_tokens":133}},
{"rank":6248,"domain":"futuremarketinsights.com","origin":"https://futuremarketinsights.com","robots":{"status":"ok","http":200,"url":"https://www.futuremarketinsights.com/robots.txt","bytes":3085,"groups":4},"bots":{"GPTBot":["partial","named",13,0],"ChatGPT-User":["partial","named",13,0],"OAI-SearchBot":["partial","named",13,0],"ClaudeBot":["partial","named",13,0],"Claude-User":["partial","named",13,0],"Claude-SearchBot":["partial","named",13,0],"anthropic-ai":["partial","*",11,0],"PerplexityBot":["partial","named",13,0],"Perplexity-User":["partial","named",13,0],"Google-Extended":["allowed","named",0,1],"Applebot-Extended":["partial","*",11,0],"CCBot":["partial","named",13,0],"Bytespider":["partial","*",11,0],"Amazonbot":["partial","*",11,0],"meta-externalagent":["partial","*",11,0],"cohere-ai":["partial","*",11,0],"Diffbot":["partial","*",11,0],"ContextFetchBot":["partial","*",11,0]},"llms":{"status":"missing","http":404,"bytes":24980},"home":{"status":"ok","http":200,"url":"https://www.futuremarketinsights.com/","html_tokens":17099,"markdown_tokens":728,"pruned_tokens":86}},
{"rank":6249,"domain":"sputnikglobe.com","origin":"https://sputnikglobe.com","robots":{"status":"timeout","http":null,"url":"https://sputnikglobe.com/robots.txt","bytes":0,"groups":0,"error":"ConnectTimeout"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":6250,"domain":"creditkarma.com","origin":"https://creditkarma.com","robots":{"status":"ok","http":200,"url":"https://www.creditkarma.com/robots.txt","bytes":1059,"groups":4},"bots":{"GPTBot":["partial","*",23,0],"ChatGPT-User":["partial","*",23,0],"OAI-SearchBot":["partial","*",23,0],"ClaudeBot":["partial","*",23,0],"Claude-User":["partial","*",23,0],"Claude-SearchBot":["partial","*",23,0],"anthropic-ai":["partial","*",23,0],"PerplexityBot":["partial","*",23,0],"Perplexity-User":["partial","*",23,0],"Google-Extended":["partial","*",23,0],"Applebot-Extended":["partial","*",23,0],"CCBot":["partial","*",23,0],"Bytespider":["partial","*",23,0],"Amazonbot":["partial","*",23,0],"meta-externalagent":["partial","*",23,0],"cohere-ai":["partial","*",23,0],"Diffbot":["partial","*",23,0],"ContextFetchBot":["partial","*",23,0]},"llms":{"status":"http_error","http":500,"bytes":389817},"home":{"status":"ok","http":200,"url":"https://www.creditkarma.com/","html_tokens":72011,"markdown_tokens":592,"pruned_tokens":114}},
{"rank":6251,"domain":"dickssportinggoods.com","origin":"https://dickssportinggoods.com","robots":{"status":"refused","http":403,"url":"https://www.dickssportinggoods.com/robots.txt","bytes":1438,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":6252,"domain":"stackoverflow.co","origin":"https://stackoverflow.co","robots":{"status":"ok","http":200,"url":"https://stackoverflow.co/robots.txt","bytes":69,"groups":1},"bots":{"GPTBot":["allowed","*",0,1],"ChatGPT-User":["allowed","*",0,1],"OAI-SearchBot":["allowed","*",0,1],"ClaudeBot":["allowed","*",0,1],"Claude-User":["allowed","*",0,1],"Claude-SearchBot":["allowed","*",0,1],"anthropic-ai":["allowed","*",0,1],"PerplexityBot":["allowed","*",0,1],"Perplexity-User":["allowed","*",0,1],"Google-Extended":["allowed","*",0,1],"Applebot-Extended":["allowed","*",0,1],"CCBot":["allowed","*",0,1],"Bytespider":["allowed","*",0,1],"Amazonbot":["allowed","*",0,1],"meta-externalagent":["allowed","*",0,1],"cohere-ai":["allowed","*",0,1],"Diffbot":["allowed","*",0,1],"ContextFetchBot":["allowed","*",0,1]},"llms":{"status":"missing","http":404,"bytes":10503},"home":{"status":"ok","http":200,"url":"https://stackoverflow.co/","html_tokens":19667,"markdown_tokens":471,"pruned_tokens":22}},
{"rank":6253,"domain":"fender.com","origin":"https://fender.com","robots":{"status":"ok","http":200,"url":"https://www.fender.com/robots.txt","bytes":3648,"groups":2},"bots":{"GPTBot":["partial","*",40,23],"ChatGPT-User":["partial","*",40,23],"OAI-SearchBot":["partial","*",40,23],"ClaudeBot":["partial","*",40,23],"Claude-User":["partial","*",40,23],"Claude-SearchBot":["partial","*",40,23],"anthropic-ai":["partial","*",40,23],"PerplexityBot":["partial","*",40,23],"Perplexity-User":["partial","*",40,23],"Google-Extended":["partial","*",40,23],"Applebot-Extended":["partial","*",40,23],"CCBot":["partial","*",40,23],"Bytespider":["partial","*",40,23],"Amazonbot":["partial","*",40,23],"meta-externalagent":["partial","*",40,23],"cohere-ai":["partial","*",40,23],"Diffbot":["partial","*",40,23],"ContextFetchBot":["partial","*",40,23]},"llms":{"status":"present","http":200,"bytes":5704},"home":{"status":"http_error","http":403,"url":"https://www.fender.com/"}},
{"rank":6254,"domain":"physicsworld.com","origin":"https://physicsworld.com","robots":{"status":"ok","http":200,"url":"https://physicsworld.com/robots.txt","bytes":76,"groups":1},"bots":{"GPTBot":["partial","*",2,0],"ChatGPT-User":["partial","*",2,0],"OAI-SearchBot":["partial","*",2,0],"ClaudeBot":["partial","*",2,0],"Claude-User":["partial","*",2,0],"Claude-SearchBot":["partial","*",2,0],"anthropic-ai":["partial","*",2,0],"PerplexityBot":["partial","*",2,0],"Perplexity-User":["partial","*",2,0],"Google-Extended":["partial","*",2,0],"Applebot-Extended":["partial","*",2,0],"CCBot":["partial","*",2,0],"Bytespider":["partial","*",2,0],"Amazonbot":["partial","*",2,0],"meta-externalagent":["partial","*",2,0],"cohere-ai":["partial","*",2,0],"Diffbot":["partial","*",2,0],"ContextFetchBot":["partial","*",2,0]},"llms":{"status":"crawl_delay","error":"Crawl-delay 600 s"},"home":{"status":"crawl_delay","error":"Crawl-delay 600 s"}},
{"rank":6255,"domain":"qianlong.com","origin":"https://qianlong.com","robots":{"status":"refused","http":400,"url":"https://qianlong.com/robots.txt","bytes":22,"groups":0,"error":"HTTP 400"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":6256,"domain":"elephantjournal.com","origin":"https://elephantjournal.com","robots":{"status":"ok","http":200,"url":"https://www.elephantjournal.com/robots.txt","bytes":69,"groups":1},"bots":{"GPTBot":["partial","*",1,1],"ChatGPT-User":["partial","*",1,1],"OAI-SearchBot":["partial","*",1,1],"ClaudeBot":["partial","*",1,1],"Claude-User":["partial","*",1,1],"Claude-SearchBot":["partial","*",1,1],"anthropic-ai":["partial","*",1,1],"PerplexityBot":["partial","*",1,1],"Perplexity-User":["partial","*",1,1],"Google-Extended":["partial","*",1,1],"Applebot-Extended":["partial","*",1,1],"CCBot":["partial","*",1,1],"Bytespider":["partial","*",1,1],"Amazonbot":["partial","*",1,1],"meta-externalagent":["partial","*",1,1],"cohere-ai":["partial","*",1,1],"Diffbot":["partial","*",1,1],"ContextFetchBot":["partial","*",1,1]},"llms":{"status":"missing","http":404,"bytes":1971},"home":{"status":"ok","http":200,"url":"https://www.elephantjournal.com/","html_tokens":29431,"markdown_tokens":270,"pruned_tokens":99}},
{"rank":6257,"domain":"usra.edu","origin":"https://www.usra.edu","robots":{"status":"ok","http":200,"url":"https://www.usra.edu/robots.txt","bytes":142,"groups":1},"bots":{"GPTBot":["partial","*",6,0],"ChatGPT-User":["partial","*",6,0],"OAI-SearchBot":["partial","*",6,0],"ClaudeBot":["partial","*",6,0],"Claude-User":["partial","*",6,0],"Claude-SearchBot":["partial","*",6,0],"anthropic-ai":["partial","*",6,0],"PerplexityBot":["partial","*",6,0],"Perplexity-User":["partial","*",6,0],"Google-Extended":["partial","*",6,0],"Applebot-Extended":["partial","*",6,0],"CCBot":["partial","*",6,0],"Bytespider":["partial","*",6,0],"Amazonbot":["partial","*",6,0],"meta-externalagent":["partial","*",6,0],"cohere-ai":["partial","*",6,0],"Diffbot":["partial","*",6,0],"ContextFetchBot":["partial","*",6,0]},"llms":{"status":"missing","http":404,"bytes":38404},"home":{"status":"http_error","http":403,"url":"https://www.usra.edu/"}},
{"rank":6258,"domain":"vz.ru","origin":"https://vz.ru","robots":{"status":"ok","http":200,"url":"https://vz.ru/robots.txt","bytes":305,"groups":1},"bots":{"GPTBot":["partial","*",11,0],"ChatGPT-User":["partial","*",11,0],"OAI-SearchBot":["partial","*",11,0],"ClaudeBot":["partial","*",11,0],"Claude-User":["partial","*",11,0],"Claude-SearchBot":["partial","*",11,0],"anthropic-ai":["partial","*",11,0],"PerplexityBot":["partial","*",11,0],"Perplexity-User":["partial","*",11,0],"Google-Extended":["partial","*",11,0],"Applebot-Extended":["partial","*",11,0],"CCBot":["partial","*",11,0],"Bytespider":["partial","*",11,0],"Amazonbot":["partial","*",11,0],"meta-externalagent":["partial","*",11,0],"cohere-ai":["partial","*",11,0],"Diffbot":["partial","*",11,0],"ContextFetchBot":["partial","*",11,0]},"llms":{"status":"present","http":200,"bytes":2117},"home":{"status":"ok","http":200,"url":"https://vz.ru/","html_tokens":62049,"markdown_tokens":2470,"pruned_tokens":128}},
{"rank":6259,"domain":"ageuk.org.uk","origin":"https://ageuk.org.uk","robots":{"status":"ok","http":200,"url":"https://www.ageuk.org.uk/robots.txt","bytes":602,"groups":1},"bots":{"GPTBot":["partial","*",19,0],"ChatGPT-User":["partial","*",19,0],"OAI-SearchBot":["partial","*",19,0],"ClaudeBot":["partial","*",19,0],"Claude-User":["partial","*",19,0],"Claude-SearchBot":["partial","*",19,0],"anthropic-ai":["partial","*",19,0],"PerplexityBot":["partial","*",19,0],"Perplexity-User":["partial","*",19,0],"Google-Extended":["partial","*",19,0],"Applebot-Extended":["partial","*",19,0],"CCBot":["partial","*",19,0],"Bytespider":["partial","*",19,0],"Amazonbot":["partial","*",19,0],"meta-externalagent":["partial","*",19,0],"cohere-ai":["partial","*",19,0],"Diffbot":["partial","*",19,0],"ContextFetchBot":["partial","*",19,0]},"llms":{"status":"missing","http":404,"bytes":97042},"home":{"status":"ok","http":200,"url":"https://www.ageuk.org.uk/","html_tokens":29525,"markdown_tokens":80,"pruned_tokens":40}},
{"rank":6260,"domain":"khou.com","origin":"https://khou.com","robots":{"status":"ok","http":200,"url":"https://www.khou.com/robots.txt","bytes":444,"groups":2},"bots":{"GPTBot":["partial","*",10,0],"ChatGPT-User":["partial","*",10,0],"OAI-SearchBot":["partial","*",10,0],"ClaudeBot":["partial","*",10,0],"Claude-User":["partial","*",10,0],"Claude-SearchBot":["partial","*",10,0],"anthropic-ai":["partial","*",10,0],"PerplexityBot":["partial","*",10,0],"Perplexity-User":["partial","*",10,0],"Google-Extended":["partial","*",10,0],"Applebot-Extended":["partial","*",10,0],"CCBot":["partial","*",10,0],"Bytespider":["partial","*",10,0],"Amazonbot":["partial","*",10,0],"meta-externalagent":["partial","*",10,0],"cohere-ai":["partial","*",10,0],"Diffbot":["partial","*",10,0],"ContextFetchBot":["partial","*",10,0]},"llms":{"status":"missing","http":404,"bytes":6844},"home":{"status":"ok","http":200,"url":"https://www.khou.com/","html_tokens":100507,"markdown_tokens":1648,"pruned_tokens":168}},
{"rank":6261,"domain":"universe.com","origin":"https://universe.com","robots":{"status":"ok","http":200,"url":"https://www.universe.com/robots.txt","bytes":272,"groups":1},"bots":{"GPTBot":["partial","*",8,0],"ChatGPT-User":["partial","*",8,0],"OAI-SearchBot":["partial","*",8,0],"ClaudeBot":["partial","*",8,0],"Claude-User":["partial","*",8,0],"Claude-SearchBot":["partial","*",8,0],"anthropic-ai":["partial","*",8,0],"PerplexityBot":["partial","*",8,0],"Perplexity-User":["partial","*",8,0],"Google-Extended":["partial","*",8,0],"Applebot-Extended":["partial","*",8,0],"CCBot":["partial","*",8,0],"Bytespider":["partial","*",8,0],"Amazonbot":["partial","*",8,0],"meta-externalagent":["partial","*",8,0],"cohere-ai":["partial","*",8,0],"Diffbot":["partial","*",8,0],"ContextFetchBot":["partial","*",8,0]},"llms":{"status":"missing","http":404,"bytes":0},"home":{"status":"ok","http":200,"url":"https://www.universe.com/","html_tokens":21217,"markdown_tokens":340,"pruned_tokens":110}},
{"rank":6262,"domain":"tubitv.com","origin":"https://tubitv.com","robots":{"status":"ok","http":200,"url":"https://tubitv.com/robots.txt","bytes":315,"groups":2},"bots":{"GPTBot":["partial","*",3,0],"ChatGPT-User":["partial","*",3,0],"OAI-SearchBot":["partial","*",3,0],"ClaudeBot":["partial","*",3,0],"Claude-User":["partial","*",3,0],"Claude-SearchBot":["partial","*",3,0],"anthropic-ai":["partial","*",3,0],"PerplexityBot":["partial","*",3,0],"Perplexity-User":["partial","*",3,0],"Google-Extended":["partial","*",3,0],"Applebot-Extended":["partial","*",3,0],"CCBot":["partial","*",3,0],"Bytespider":["partial","*",3,0],"Amazonbot":["partial","*",3,0],"meta-externalagent":["partial","*",3,0],"cohere-ai":["partial","*",3,0],"Diffbot":["partial","*",3,0],"ContextFetchBot":["partial","*",3,0]},"llms":{"status":"missing","http":404,"bytes":387646},"home":{"status":"ok","http":200,"url":"https://tubitv.com/","html_tokens":155718,"markdown_tokens":308,"pruned_tokens":143}},
{"rank":6263,"domain":"clickondetroit.com","origin":"https://clickondetroit.com","robots":{"status":"ok","http":200,"url":"https://www.clickondetroit.com/robots.txt","bytes":1189,"groups":9},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["partial","*",5,0],"OAI-SearchBot":["partial","*",5,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",5,0],"Claude-SearchBot":["partial","*",5,0],"anthropic-ai":["partial","*",5,0],"PerplexityBot":["partial","*",5,0],"Perplexity-User":["partial","*",5,0],"Google-Extended":["partial","*",5,0],"Applebot-Extended":["partial","*",5,0],"CCBot":["partial","*",5,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["partial","*",5,0],"cohere-ai":["partial","*",5,0],"Diffbot":["partial","*",5,0],"ContextFetchBot":["partial","*",5,0]},"llms":{"status":"missing","http":404,"bytes":157085},"home":{"status":"ok","http":200,"url":"https://www.clickondetroit.com/","html_tokens":217775,"markdown_tokens":94,"pruned_tokens":94}},
{"rank":6264,"domain":"ledauphine.com","origin":"https://ledauphine.com","robots":{"status":"ok","http":200,"url":"https://www.ledauphine.com/robots.txt","bytes":4918,"groups":11},"bots":{"GPTBot":["partial","named",1,2],"ChatGPT-User":["partial","named",1,2],"OAI-SearchBot":["partial","*",39,1],"ClaudeBot":["partial","*",39,1],"Claude-User":["partial","*",39,1],"Claude-SearchBot":["partial","*",39,1],"anthropic-ai":["partial","named",1,2],"PerplexityBot":["partial","*",39,1],"Perplexity-User":["partial","*",39,1],"Google-Extended":["partial","named",1,2],"Applebot-Extended":["partial","*",39,1],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["partial","named",1,2],"meta-externalagent":["partial","*",39,1],"cohere-ai":["partial","*",39,1],"Diffbot":["partial","*",39,1],"ContextFetchBot":["partial","*",39,1]},"llms":{"status":"missing","http":404,"bytes":5016},"home":{"status":"ok","http":200,"url":"https://www.ledauphine.com/","html_tokens":281902,"markdown_tokens":255,"pruned_tokens":119}},
{"rank":6265,"domain":"passportintime.com","origin":"https://passportintime.com","robots":{"status":"ok","http":200,"url":"https://passportintime.com/robots.txt","bytes":141,"groups":1},"bots":{"GPTBot":["partial","*",2,1],"ChatGPT-User":["partial","*",2,1],"OAI-SearchBot":["partial","*",2,1],"ClaudeBot":["partial","*",2,1],"Claude-User":["partial","*",2,1],"Claude-SearchBot":["partial","*",2,1],"anthropic-ai":["partial","*",2,1],"PerplexityBot":["partial","*",2,1],"Perplexity-User":["partial","*",2,1],"Google-Extended":["partial","*",2,1],"Applebot-Extended":["partial","*",2,1],"CCBot":["partial","*",2,1],"Bytespider":["partial","*",2,1],"Amazonbot":["partial","*",2,1],"meta-externalagent":["partial","*",2,1],"cohere-ai":["partial","*",2,1],"Diffbot":["partial","*",2,1],"ContextFetchBot":["partial","*",2,1]},"llms":{"status":"html","http":200,"bytes":54389},"home":{"status":"ok","http":200,"url":"https://passportintime.com/","html_tokens":13336,"markdown_tokens":7188,"pruned_tokens":173}},
{"rank":6266,"domain":"uniswap.org","origin":"https://uniswap.org","robots":{"status":"ok","http":200,"url":"https://uniswap.org/robots.txt","bytes":114,"groups":1},"bots":{"GPTBot":["allowed","*",0,1],"ChatGPT-User":["allowed","*",0,1],"OAI-SearchBot":["allowed","*",0,1],"ClaudeBot":["allowed","*",0,1],"Claude-User":["allowed","*",0,1],"Claude-SearchBot":["allowed","*",0,1],"anthropic-ai":["allowed","*",0,1],"PerplexityBot":["allowed","*",0,1],"Perplexity-User":["allowed","*",0,1],"Google-Extended":["allowed","*",0,1],"Applebot-Extended":["allowed","*",0,1],"CCBot":["allowed","*",0,1],"Bytespider":["allowed","*",0,1],"Amazonbot":["allowed","*",0,1],"meta-externalagent":["allowed","*",0,1],"cohere-ai":["allowed","*",0,1],"Diffbot":["allowed","*",0,1],"ContextFetchBot":["allowed","*",0,1]},"llms":{"status":"html","http":200,"bytes":87397},"home":{"status":"ok","http":200,"url":"https://app.uniswap.org/","html_tokens":21850,"markdown_tokens":12,"pruned_tokens":12}},
{"rank":6267,"domain":"nbcsandiego.com","origin":"https://nbcsandiego.com","robots":{"status":"ok","http":200,"url":"https://www.nbcsandiego.com/robots.txt","bytes":10422,"groups":27},"bots":{"GPTBot":["partial","named",1,6],"ChatGPT-User":["partial","named",1,6],"OAI-SearchBot":["partial","*",13,3],"ClaudeBot":["partial","named",1,6],"Claude-User":["partial","*",13,3],"Claude-SearchBot":["partial","*",13,3],"anthropic-ai":["partial","named",1,6],"PerplexityBot":["partial","named",1,6],"Perplexity-User":["partial","*",13,3],"Google-Extended":["partial","named",1,6],"Applebot-Extended":["partial","*",13,3],"CCBot":["partial","named",1,6],"Bytespider":["partial","named",1,6],"Amazonbot":["partial","named",1,8],"meta-externalagent":["partial","*",13,3],"cohere-ai":["partial","*",13,3],"Diffbot":["partial","named",1,6],"ContextFetchBot":["partial","*",13,3]},"llms":{"status":"html","http":200,"bytes":536658},"home":{"status":"ok","http":200,"url":"https://www.nbcsandiego.com/","html_tokens":134139,"markdown_tokens":73,"pruned_tokens":73}},
{"rank":6268,"domain":"meinbezirk.at","origin":"https://meinbezirk.at","robots":{"status":"ok","http":200,"url":"https://www.meinbezirk.at/robots.txt","bytes":252,"groups":1},"bots":{"GPTBot":["partial","*",3,1],"ChatGPT-User":["partial","*",3,1],"OAI-SearchBot":["partial","*",3,1],"ClaudeBot":["partial","*",3,1],"Claude-User":["partial","*",3,1],"Claude-SearchBot":["partial","*",3,1],"anthropic-ai":["partial","*",3,1],"PerplexityBot":["partial","*",3,1],"Perplexity-User":["partial","*",3,1],"Google-Extended":["partial","*",3,1],"Applebot-Extended":["partial","*",3,1],"CCBot":["partial","*",3,1],"Bytespider":["partial","*",3,1],"Amazonbot":["partial","*",3,1],"meta-externalagent":["partial","*",3,1],"cohere-ai":["partial","*",3,1],"Diffbot":["partial","*",3,1],"ContextFetchBot":["partial","*",3,1]},"llms":{"status":"missing","http":404,"bytes":56447},"home":{"status":"ok","http":200,"url":"https://www.meinbezirk.at/","html_tokens":78318,"markdown_tokens":126,"pruned_tokens":126}},
{"rank":6269,"domain":"wireguard.com","origin":"https://wireguard.com","robots":{"status":"missing","http":404,"url":"https://www.wireguard.com/robots.txt","bytes":146,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":146},"home":{"status":"ok","http":200,"url":"https://www.wireguard.com/","html_tokens":5517,"markdown_tokens":2596,"pruned_tokens":195}},
{"rank":6270,"domain":"tu-dortmund.de","origin":"https://tu-dortmund.de","robots":{"status":"ok","http":200,"url":"https://www.tu-dortmund.de/robots.txt","bytes":59,"groups":1},"bots":{"GPTBot":["allowed","*",0,1],"ChatGPT-User":["allowed","*",0,1],"OAI-SearchBot":["allowed","*",0,1],"ClaudeBot":["allowed","*",0,1],"Claude-User":["allowed","*",0,1],"Claude-SearchBot":["allowed","*",0,1],"anthropic-ai":["allowed","*",0,1],"PerplexityBot":["allowed","*",0,1],"Perplexity-User":["allowed","*",0,1],"Google-Extended":["allowed","*",0,1],"Applebot-Extended":["allowed","*",0,1],"CCBot":["allowed","*",0,1],"Bytespider":["allowed","*",0,1],"Amazonbot":["allowed","*",0,1],"meta-externalagent":["allowed","*",0,1],"cohere-ai":["allowed","*",0,1],"Diffbot":["allowed","*",0,1],"ContextFetchBot":["allowed","*",0,1]},"llms":{"status":"missing","http":404,"bytes":83446},"home":{"status":"ok","http":200,"url":"https://www.tu-dortmund.de/","html_tokens":373710,"markdown_tokens":495,"pruned_tokens":239}},
{"rank":6271,"domain":"nationaleatingdisorders.org","origin":"https://nationaleatingdisorders.org","robots":{"status":"ok","http":200,"url":"https://nationaleatingdisorders.org/robots.txt","bytes":333,"groups":1},"bots":{"GPTBot":["partial","*",4,1],"ChatGPT-User":["partial","*",4,1],"OAI-SearchBot":["partial","*",4,1],"ClaudeBot":["partial","*",4,1],"Claude-User":["partial","*",4,1],"Claude-SearchBot":["partial","*",4,1],"anthropic-ai":["partial","*",4,1],"PerplexityBot":["partial","*",4,1],"Perplexity-User":["partial","*",4,1],"Google-Extended":["partial","*",4,1],"Applebot-Extended":["partial","*",4,1],"CCBot":["partial","*",4,1],"Bytespider":["partial","*",4,1],"Amazonbot":["partial","*",4,1],"meta-externalagent":["partial","*",4,1],"cohere-ai":["partial","*",4,1],"Diffbot":["partial","*",4,1],"ContextFetchBot":["partial","*",4,1]},"llms":{"status":"other_text","http":200,"bytes":4612},"home":{"status":"http_error","http":403,"url":"https://nationaleatingdisorders.org/"}},
{"rank":6272,"domain":"amnestyusa.org","origin":"https://amnestyusa.org","robots":{"status":"ok","http":200,"url":"https://www.amnestyusa.org/robots.txt","bytes":252,"groups":2},"bots":{"GPTBot":["partial","*",2,1],"ChatGPT-User":["partial","*",2,1],"OAI-SearchBot":["partial","*",2,1],"ClaudeBot":["partial","*",2,1],"Claude-User":["partial","*",2,1],"Claude-SearchBot":["partial","*",2,1],"anthropic-ai":["partial","*",2,1],"PerplexityBot":["partial","*",2,1],"Perplexity-User":["partial","*",2,1],"Google-Extended":["partial","*",2,1],"Applebot-Extended":["partial","*",2,1],"CCBot":["partial","*",2,1],"Bytespider":["partial","*",2,1],"Amazonbot":["partial","*",2,1],"meta-externalagent":["partial","*",2,1],"cohere-ai":["partial","*",2,1],"Diffbot":["partial","*",2,1],"ContextFetchBot":["partial","*",2,1]},"llms":{"status":"present","http":200,"bytes":4997},"home":{"status":"http_error","http":403,"url":"https://amnestyusa.org/"}},
{"rank":6273,"domain":"spamhaus.org","origin":"https://spamhaus.org","robots":{"status":"html","http":200,"url":"https://www.spamhaus.org","bytes":641228,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":6274,"domain":"calmatters.org","origin":"https://calmatters.org","robots":{"status":"ok","http":200,"url":"https://calmatters.org/robots.txt","bytes":585,"groups":1},"bots":{"GPTBot":["partial","*",4,0],"ChatGPT-User":["partial","*",4,0],"OAI-SearchBot":["partial","*",4,0],"ClaudeBot":["partial","*",4,0],"Claude-User":["partial","*",4,0],"Claude-SearchBot":["partial","*",4,0],"anthropic-ai":["partial","*",4,0],"PerplexityBot":["partial","*",4,0],"Perplexity-User":["partial","*",4,0],"Google-Extended":["partial","*",4,0],"Applebot-Extended":["partial","*",4,0],"CCBot":["partial","*",4,0],"Bytespider":["partial","*",4,0],"Amazonbot":["partial","*",4,0],"meta-externalagent":["partial","*",4,0],"cohere-ai":["partial","*",4,0],"Diffbot":["partial","*",4,0],"ContextFetchBot":["partial","*",4,0]},"llms":{"status":"missing","http":404,"bytes":154689},"home":{"status":"ok","http":200,"url":"https://calmatters.org/","html_tokens":85395,"markdown_tokens":468,"pruned_tokens":94}},
{"rank":6275,"domain":"betfair.com","origin":"https://betfair.com","robots":{"status":"refused","http":403,"url":"https://www.betfair.com/robots.txt","bytes":2705,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":6276,"domain":"fiba.basketball","origin":"https://fiba.basketball","robots":{"status":"html","http":200,"url":"https://www.fiba.basketball/en","bytes":433874,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":6277,"domain":"lucidchart.com","origin":"https://lucidchart.com","robots":{"status":"ok","http":200,"url":"https://www.lucidchart.com/robots.txt","bytes":2503,"groups":1},"bots":{"GPTBot":["partial","*",71,1],"ChatGPT-User":["partial","*",71,1],"OAI-SearchBot":["partial","*",71,1],"ClaudeBot":["partial","*",71,1],"Claude-User":["partial","*",71,1],"Claude-SearchBot":["partial","*",71,1],"anthropic-ai":["partial","*",71,1],"PerplexityBot":["partial","*",71,1],"Perplexity-User":["partial","*",71,1],"Google-Extended":["partial","*",71,1],"Applebot-Extended":["partial","*",71,1],"CCBot":["partial","*",71,1],"Bytespider":["partial","*",71,1],"Amazonbot":["partial","*",71,1],"meta-externalagent":["partial","*",71,1],"cohere-ai":["partial","*",71,1],"Diffbot":["partial","*",71,1],"ContextFetchBot":["partial","*",71,1]},"llms":{"status":"disallowed","error":"robots.txt of lucid.app disallows /llms.txt"},"home":{"status":"ok","http":200,"url":"https://lucid.co/lucidchart","html_tokens":207249,"markdown_tokens":469,"pruned_tokens":185}},
{"rank":6278,"domain":"waikato.ac.nz","origin":"https://waikato.ac.nz","robots":{"status":"ok","http":200,"url":"https://www.waikato.ac.nz/robots.txt","bytes":6964,"groups":62},"bots":{"GPTBot":["partial","*",47,0],"ChatGPT-User":["partial","*",47,0],"OAI-SearchBot":["partial","*",47,0],"ClaudeBot":["partial","*",47,0],"Claude-User":["partial","*",47,0],"Claude-SearchBot":["partial","*",47,0],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["partial","*",47,0],"Perplexity-User":["partial","*",47,0],"Google-Extended":["partial","*",47,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["partial","*",47,0],"ContextFetchBot":["partial","*",47,0]},"llms":{"status":"present","http":200,"bytes":9500},"home":{"status":"ok","http":200,"url":"https://www.waikato.ac.nz/","html_tokens":75508,"markdown_tokens":237,"pruned_tokens":134}},
{"rank":6279,"domain":"wefunder.com","origin":"https://wefunder.com","robots":{"status":"ok","http":200,"url":"https://wefunder.com/robots.txt","bytes":590,"groups":1},"bots":{"GPTBot":["partial","*",9,0],"ChatGPT-User":["partial","*",9,0],"OAI-SearchBot":["partial","*",9,0],"ClaudeBot":["partial","*",9,0],"Claude-User":["partial","*",9,0],"Claude-SearchBot":["partial","*",9,0],"anthropic-ai":["partial","*",9,0],"PerplexityBot":["partial","*",9,0],"Perplexity-User":["partial","*",9,0],"Google-Extended":["partial","*",9,0],"Applebot-Extended":["partial","*",9,0],"CCBot":["partial","*",9,0],"Bytespider":["partial","*",9,0],"Amazonbot":["partial","*",9,0],"meta-externalagent":["partial","*",9,0],"cohere-ai":["partial","*",9,0],"Diffbot":["partial","*",9,0],"ContextFetchBot":["partial","*",9,0]},"llms":{"status":"present","http":200,"bytes":12983},"home":{"status":"http_error","http":403,"url":"https://wefunder.com/"}},
{"rank":6280,"domain":"gov-online.go.jp","origin":"https://gov-online.go.jp","robots":{"status":"refused","http":403,"url":"https://gov-online.go.jp/robots.txt","bytes":0,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":6281,"domain":"correios.com.br","origin":"https://correios.com.br","robots":{"status":"html","http":200,"url":"https://www.correios.com.br","bytes":150348,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":6282,"domain":"pokerstars.com","origin":"https://pokerstars.com","robots":{"status":"refused","http":403,"url":"https://www.pokerstars.com/robots.txt","bytes":382,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":6283,"domain":"iss.it","origin":"https://www.iss.it","robots":{"status":"html","http":200,"url":"https://www.iss.it/robots.txt","bytes":244,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":6284,"domain":"u-bordeaux.fr","origin":"https://u-bordeaux.fr","robots":{"status":"ok","http":200,"url":"https://www.u-bordeaux.fr/robots.txt","bytes":520,"groups":1},"bots":{"GPTBot":["partial","*",16,0],"ChatGPT-User":["partial","*",16,0],"OAI-SearchBot":["partial","*",16,0],"ClaudeBot":["partial","*",16,0],"Claude-User":["partial","*",16,0],"Claude-SearchBot":["partial","*",16,0],"anthropic-ai":["partial","*",16,0],"PerplexityBot":["partial","*",16,0],"Perplexity-User":["partial","*",16,0],"Google-Extended":["partial","*",16,0],"Applebot-Extended":["partial","*",16,0],"CCBot":["partial","*",16,0],"Bytespider":["partial","*",16,0],"Amazonbot":["partial","*",16,0],"meta-externalagent":["partial","*",16,0],"cohere-ai":["partial","*",16,0],"Diffbot":["partial","*",16,0],"ContextFetchBot":["partial","*",16,0]},"llms":{"status":"missing","http":404,"bytes":307145},"home":{"status":"ok","http":200,"url":"https://www.u-bordeaux.fr/","html_tokens":115853,"markdown_tokens":397,"pruned_tokens":138}},
{"rank":6285,"domain":"trekbikes.com","origin":"https://trekbikes.com","robots":{"status":"ok","http":200,"url":"https://www.trekbikes.com/robots.txt","bytes":307,"groups":1},"bots":{"GPTBot":["partial","*",6,0],"ChatGPT-User":["partial","*",6,0],"OAI-SearchBot":["partial","*",6,0],"ClaudeBot":["partial","*",6,0],"Claude-User":["partial","*",6,0],"Claude-SearchBot":["partial","*",6,0],"anthropic-ai":["partial","*",6,0],"PerplexityBot":["partial","*",6,0],"Perplexity-User":["partial","*",6,0],"Google-Extended":["partial","*",6,0],"Applebot-Extended":["partial","*",6,0],"CCBot":["partial","*",6,0],"Bytespider":["partial","*",6,0],"Amazonbot":["partial","*",6,0],"meta-externalagent":["partial","*",6,0],"cohere-ai":["partial","*",6,0],"Diffbot":["partial","*",6,0],"ContextFetchBot":["partial","*",6,0]},"llms":{"status":"missing","http":404,"bytes":120069},"home":{"status":"ok","http":200,"url":"https://www.trekbikes.com/us/en_US/","html_tokens":62164,"markdown_tokens":70,"pruned_tokens":70}},
{"rank":6286,"domain":"chula.ac.th","origin":"https://www.chula.ac.th","robots":{"status":"ok","http":200,"url":"https://www.chula.ac.th/robots.txt","bytes":705,"groups":7},"bots":{"GPTBot":["partial","*",15,2],"ChatGPT-User":["partial","*",15,2],"OAI-SearchBot":["partial","*",15,2],"ClaudeBot":["partial","*",15,2],"Claude-User":["partial","*",15,2],"Claude-SearchBot":["partial","*",15,2],"anthropic-ai":["partial","*",15,2],"PerplexityBot":["partial","*",15,2],"Perplexity-User":["partial","*",15,2],"Google-Extended":["partial","*",15,2],"Applebot-Extended":["partial","*",15,2],"CCBot":["partial","*",15,2],"Bytespider":["blocked","named",1,0],"Amazonbot":["partial","*",15,2],"meta-externalagent":["partial","*",15,2],"cohere-ai":["partial","*",15,2],"Diffbot":["partial","*",15,2],"ContextFetchBot":["partial","*",15,2]},"llms":{"status":"missing","http":404,"bytes":202353},"home":{"status":"ok","http":200,"url":"https://www.chula.ac.th/","html_tokens":179823,"markdown_tokens":771,"pruned_tokens":102}},
{"rank":6287,"domain":"sega.com","origin":"https://sega.com","robots":{"status":"refused","http":429,"url":"https://sega.com/robots.txt","bytes":34222,"groups":0,"error":"HTTP 429"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":6288,"domain":"mega.co.nz","origin":"https://mega.co.nz","robots":{"status":"ok","http":200,"url":"https://mega.co.nz/robots.txt","bytes":0,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":0},"home":{"status":"ok","http":200,"url":"https://mega.co.nz/","html_tokens":139,"markdown_tokens":0,"pruned_tokens":0}},
{"rank":6289,"domain":"canberratimes.com.au","origin":"https://canberratimes.com.au","robots":{"status":"ok","http":200,"url":"https://www.canberratimes.com.au/robots.txt","bytes":3526,"groups":42},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["blocked","named",1,0],"OAI-SearchBot":["blocked","named",1,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",9,0],"Claude-SearchBot":["partial","*",9,0],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["blocked","named",1,0],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["allowed","named",0,2],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",9,0]},"llms":{"status":"missing","http":404,"bytes":208896},"home":{"status":"ok","http":200,"url":"https://www.canberratimes.com.au/","html_tokens":204629,"markdown_tokens":102,"pruned_tokens":102}},
{"rank":6290,"domain":"denver7.com","origin":"https://denver7.com","robots":{"status":"ok","http":200,"url":"https://www.denver7.com/robots.txt","bytes":389,"groups":4},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["partial","*",5,0],"OAI-SearchBot":["partial","*",5,0],"ClaudeBot":["partial","*",5,0],"Claude-User":["partial","*",5,0],"Claude-SearchBot":["partial","*",5,0],"anthropic-ai":["partial","*",5,0],"PerplexityBot":["partial","*",5,0],"Perplexity-User":["partial","*",5,0],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["partial","*",5,0],"CCBot":["partial","*",5,0],"Bytespider":["partial","*",5,0],"Amazonbot":["partial","*",5,0],"meta-externalagent":["partial","*",5,0],"cohere-ai":["partial","*",5,0],"Diffbot":["partial","*",5,0],"ContextFetchBot":["partial","*",5,0]},"llms":{"status":"missing","http":404,"bytes":78095},"home":{"status":"ok","http":200,"url":"https://www.denver7.com/","html_tokens":110401,"markdown_tokens":68,"pruned_tokens":66}},
{"rank":6291,"domain":"fruits.co","origin":"https://fruits.co","robots":{"status":"ok","http":200,"url":"https://www.fruits.co/robots.txt","bytes":142,"groups":1},"bots":{"GPTBot":["partial","*",7,0],"ChatGPT-User":["partial","*",7,0],"OAI-SearchBot":["partial","*",7,0],"ClaudeBot":["partial","*",7,0],"Claude-User":["partial","*",7,0],"Claude-SearchBot":["partial","*",7,0],"anthropic-ai":["partial","*",7,0],"PerplexityBot":["partial","*",7,0],"Perplexity-User":["partial","*",7,0],"Google-Extended":["partial","*",7,0],"Applebot-Extended":["partial","*",7,0],"CCBot":["partial","*",7,0],"Bytespider":["partial","*",7,0],"Amazonbot":["partial","*",7,0],"meta-externalagent":["partial","*",7,0],"cohere-ai":["partial","*",7,0],"Diffbot":["partial","*",7,0],"ContextFetchBot":["partial","*",7,0]},"llms":{"status":"missing","http":404,"bytes":51111},"home":{"status":"ok","http":200,"url":"https://www.fruits.co/","html_tokens":18675,"markdown_tokens":110,"pruned_tokens":24}},
{"rank":6292,"domain":"wingify.com","origin":"https://wingify.com","robots":{"status":"ok","http":200,"url":"https://wingify.com/robots.txt","bytes":1856,"groups":34},"bots":{"GPTBot":["allowed","named",0,1],"ChatGPT-User":["allowed","named",0,1],"OAI-SearchBot":["allowed","named",0,1],"ClaudeBot":["allowed","named",0,1],"Claude-User":["allowed","named",0,1],"Claude-SearchBot":["allowed","named",0,1],"anthropic-ai":["partial","*",2,0],"PerplexityBot":["allowed","named",0,1],"Perplexity-User":["allowed","named",0,1],"Google-Extended":["allowed","named",0,1],"Applebot-Extended":["allowed","named",0,1],"CCBot":["partial","named",15,1],"Bytespider":["allowed","named",0,1],"Amazonbot":["allowed","named",0,1],"meta-externalagent":["allowed","named",0,2],"cohere-ai":["allowed","named",0,1],"Diffbot":["allowed","named",0,1],"ContextFetchBot":["partial","*",2,0]},"llms":{"status":"present","http":200,"bytes":87650},"home":{"status":"ok","http":200,"url":"https://wingify.com/","html_tokens":163926,"markdown_tokens":1670,"pruned_tokens":624}},
{"rank":6293,"domain":"browsehappy.com","origin":"https://browsehappy.com","robots":{"status":"html","http":200,"url":"https://browsehappy.com/robots.txt/","bytes":26889,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":6294,"domain":"commonwealthfund.org","origin":"https://commonwealthfund.org","robots":{"status":"ok","http":200,"url":"https://commonwealthfund.org/robots.txt","bytes":2222,"groups":1},"bots":{"GPTBot":["partial","*",34,18],"ChatGPT-User":["partial","*",34,18],"OAI-SearchBot":["partial","*",34,18],"ClaudeBot":["partial","*",34,18],"Claude-User":["partial","*",34,18],"Claude-SearchBot":["partial","*",34,18],"anthropic-ai":["partial","*",34,18],"PerplexityBot":["partial","*",34,18],"Perplexity-User":["partial","*",34,18],"Google-Extended":["partial","*",34,18],"Applebot-Extended":["partial","*",34,18],"CCBot":["partial","*",34,18],"Bytespider":["partial","*",34,18],"Amazonbot":["partial","*",34,18],"meta-externalagent":["partial","*",34,18],"cohere-ai":["partial","*",34,18],"Diffbot":["partial","*",34,18],"ContextFetchBot":["partial","*",34,18]},"llms":{"status":"missing","http":404,"bytes":198},"home":{"status":"http_error","http":403,"url":"https://commonwealthfund.org/"}},
{"rank":6295,"domain":"rvo.nl","origin":"https://rvo.nl","robots":{"status":"ok","http":200,"url":"https://www.rvo.nl/robots.txt","bytes":583,"groups":1},"bots":{"GPTBot":["partial","*",9,0],"ChatGPT-User":["partial","*",9,0],"OAI-SearchBot":["partial","*",9,0],"ClaudeBot":["partial","*",9,0],"Claude-User":["partial","*",9,0],"Claude-SearchBot":["partial","*",9,0],"anthropic-ai":["partial","*",9,0],"PerplexityBot":["partial","*",9,0],"Perplexity-User":["partial","*",9,0],"Google-Extended":["partial","*",9,0],"Applebot-Extended":["partial","*",9,0],"CCBot":["partial","*",9,0],"Bytespider":["partial","*",9,0],"Amazonbot":["partial","*",9,0],"meta-externalagent":["partial","*",9,0],"cohere-ai":["partial","*",9,0],"Diffbot":["partial","*",9,0],"ContextFetchBot":["partial","*",9,0]},"llms":{"status":"missing","http":404,"bytes":8943},"home":{"status":"ok","http":200,"url":"https://www.rvo.nl/","html_tokens":84963,"markdown_tokens":171,"pruned_tokens":172}},
{"rank":6296,"domain":"trtworld.com","origin":"https://trtworld.com","robots":{"status":"ok","http":200,"url":"https://www.trtworld.com/robots.txt","bytes":612,"groups":2},"bots":{"GPTBot":["partial","*",20,1],"ChatGPT-User":["partial","*",20,1],"OAI-SearchBot":["partial","*",20,1],"ClaudeBot":["partial","*",20,1],"Claude-User":["partial","*",20,1],"Claude-SearchBot":["partial","*",20,1],"anthropic-ai":["partial","*",20,1],"PerplexityBot":["partial","*",20,1],"Perplexity-User":["partial","*",20,1],"Google-Extended":["partial","*",20,1],"Applebot-Extended":["partial","*",20,1],"CCBot":["partial","*",20,1],"Bytespider":["partial","*",20,1],"Amazonbot":["partial","*",20,1],"meta-externalagent":["partial","*",20,1],"cohere-ai":["partial","*",20,1],"Diffbot":["partial","*",20,1],"ContextFetchBot":["partial","*",20,1]},"llms":{"status":"present","http":200,"bytes":6786},"home":{"status":"ok","http":200,"url":"https://www.trtworld.com/","html_tokens":329551,"markdown_tokens":71,"pruned_tokens":71}},
{"rank":6297,"domain":"publico.pt","origin":"https://publico.pt","robots":{"status":"refused","http":403,"url":"https://publico.pt/robots.txt","bytes":919,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":6298,"domain":"cmaj.ca","origin":"https://cmaj.ca","robots":{"status":"refused","http":403,"url":"https://cmaj.ca/robots.txt","bytes":5187,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":6299,"domain":"ibo.org","origin":"https://ibo.org","robots":{"status":"refused","http":403,"url":"https://ibo.org/robots.txt","bytes":5502,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":6300,"domain":"tidycal.com","origin":"https://tidycal.com","robots":{"status":"ok","http":200,"url":"https://tidycal.com/robots.txt","bytes":113,"groups":1},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"present","http":200,"bytes":3778},"home":{"status":"ok","http":200,"url":"https://tidycal.com/","html_tokens":16942,"markdown_tokens":298,"pruned_tokens":108}},
{"rank":6301,"domain":"audiojungle.net","origin":"https://audiojungle.net","robots":{"status":"ok","http":200,"url":"https://audiojungle.net/robots.txt","bytes":833,"groups":2},"bots":{"GPTBot":["partial","*",27,0],"ChatGPT-User":["partial","*",27,0],"OAI-SearchBot":["partial","*",27,0],"ClaudeBot":["partial","*",27,0],"Claude-User":["partial","*",27,0],"Claude-SearchBot":["partial","*",27,0],"anthropic-ai":["partial","*",27,0],"PerplexityBot":["partial","*",27,0],"Perplexity-User":["partial","*",27,0],"Google-Extended":["partial","*",27,0],"Applebot-Extended":["partial","*",27,0],"CCBot":["blocked","named",1,0],"Bytespider":["partial","*",27,0],"Amazonbot":["partial","*",27,0],"meta-externalagent":["partial","*",27,0],"cohere-ai":["partial","*",27,0],"Diffbot":["partial","*",27,0],"ContextFetchBot":["partial","*",27,0]},"llms":{"status":"missing","http":404,"bytes":21},"home":{"status":"ok","http":200,"url":"https://audiojungle.net/","html_tokens":143930,"markdown_tokens":63,"pruned_tokens":63}},
{"rank":6302,"domain":"asean.org","origin":"https://asean.org","robots":{"status":"ok","http":200,"url":"https://asean.org/robots.txt","bytes":0,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":495340},"home":{"status":"ok","http":307,"url":"https://asean.org/","html_tokens":326,"markdown_tokens":23,"pruned_tokens":23}},
{"rank":6303,"domain":"psmag.com","origin":"https://psmag.com","robots":{"status":"ok","http":200,"url":"https://psmag.com/robots.txt","bytes":167,"groups":1},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"missing","http":404,"bytes":59100},"home":{"status":"ok","http":200,"url":"https://psmag.com/","html_tokens":46937,"markdown_tokens":440,"pruned_tokens":160}},
{"rank":6304,"domain":"whoi.edu","origin":"https://whoi.edu","robots":{"status":"ok","http":200,"url":"https://www.whoi.edu/robots.txt","bytes":375,"groups":1},"bots":{"GPTBot":["partial","*",5,1],"ChatGPT-User":["partial","*",5,1],"OAI-SearchBot":["partial","*",5,1],"ClaudeBot":["partial","*",5,1],"Claude-User":["partial","*",5,1],"Claude-SearchBot":["partial","*",5,1],"anthropic-ai":["partial","*",5,1],"PerplexityBot":["partial","*",5,1],"Perplexity-User":["partial","*",5,1],"Google-Extended":["partial","*",5,1],"Applebot-Extended":["partial","*",5,1],"CCBot":["partial","*",5,1],"Bytespider":["partial","*",5,1],"Amazonbot":["partial","*",5,1],"meta-externalagent":["partial","*",5,1],"cohere-ai":["partial","*",5,1],"Diffbot":["partial","*",5,1],"ContextFetchBot":["partial","*",5,1]},"llms":{"status":"other_text","http":200,"bytes":330789},"home":{"status":"ok","http":200,"url":"https://www.whoi.edu/","html_tokens":61783,"markdown_tokens":579,"pruned_tokens":145}},
{"rank":6305,"domain":"tmcnet.com","origin":"https://tmcnet.com","robots":{"status":"ok","http":200,"url":"https://tmcnet.com/robots.txt","bytes":688,"groups":7},"bots":{"GPTBot":["partial","named",3,0],"ChatGPT-User":["partial","named",3,0],"OAI-SearchBot":["partial","*",2,0],"ClaudeBot":["partial","named",3,0],"Claude-User":["partial","*",2,0],"Claude-SearchBot":["partial","*",2,0],"anthropic-ai":["partial","*",2,0],"PerplexityBot":["partial","*",2,0],"Perplexity-User":["partial","*",2,0],"Google-Extended":["partial","*",2,0],"Applebot-Extended":["partial","*",2,0],"CCBot":["partial","*",2,0],"Bytespider":["partial","*",2,0],"Amazonbot":["partial","named",3,0],"meta-externalagent":["partial","*",2,0],"cohere-ai":["partial","*",2,0],"Diffbot":["partial","*",2,0],"ContextFetchBot":["partial","*",2,0]},"llms":{"status":"missing","http":404,"bytes":6681},"home":{"status":"http_error","http":403,"url":"https://tmcnet.com/"}},
{"rank":6306,"domain":"corporate-ir.net","origin":"https://corporate-ir.net","robots":{"status":"unreachable","http":null,"url":"https://corporate-ir.net/robots.txt","bytes":0,"groups":0,"error":"cannot resolve corporate-ir.net"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":6307,"domain":"ausha.co","origin":"https://ausha.co","robots":{"status":"ok","http":200,"url":"https://www.ausha.co/robots.txt","bytes":119,"groups":1},"bots":{"GPTBot":["partial","*",1,0],"ChatGPT-User":["partial","*",1,0],"OAI-SearchBot":["partial","*",1,0],"ClaudeBot":["partial","*",1,0],"Claude-User":["partial","*",1,0],"Claude-SearchBot":["partial","*",1,0],"anthropic-ai":["partial","*",1,0],"PerplexityBot":["partial","*",1,0],"Perplexity-User":["partial","*",1,0],"Google-Extended":["partial","*",1,0],"Applebot-Extended":["partial","*",1,0],"CCBot":["partial","*",1,0],"Bytespider":["partial","*",1,0],"Amazonbot":["partial","*",1,0],"meta-externalagent":["partial","*",1,0],"cohere-ai":["partial","*",1,0],"Diffbot":["partial","*",1,0],"ContextFetchBot":["partial","*",1,0]},"llms":{"status":"present","http":200,"bytes":4374},"home":{"status":"ok","http":200,"url":"https://www.ausha.co/","html_tokens":151770,"markdown_tokens":2127,"pruned_tokens":278}},
{"rank":6308,"domain":"ckuvjtwp2.top","origin":"https://ckuvjtwp2.top","robots":{"status":"ok","http":200,"url":"https://celmamagazine.com/robots.txt","bytes":121,"groups":1},"bots":{"GPTBot":["partial","*",1,1],"ChatGPT-User":["partial","*",1,1],"OAI-SearchBot":["partial","*",1,1],"ClaudeBot":["partial","*",1,1],"Claude-User":["partial","*",1,1],"Claude-SearchBot":["partial","*",1,1],"anthropic-ai":["partial","*",1,1],"PerplexityBot":["partial","*",1,1],"Perplexity-User":["partial","*",1,1],"Google-Extended":["partial","*",1,1],"Applebot-Extended":["partial","*",1,1],"CCBot":["partial","*",1,1],"Bytespider":["partial","*",1,1],"Amazonbot":["partial","*",1,1],"meta-externalagent":["partial","*",1,1],"cohere-ai":["partial","*",1,1],"Diffbot":["partial","*",1,1],"ContextFetchBot":["partial","*",1,1]},"llms":{"status":"missing","http":404,"bytes":72232},"home":{"status":"ok","http":200,"url":"https://celmamagazine.com/","html_tokens":28402,"markdown_tokens":60,"pruned_tokens":52}},
{"rank":6309,"domain":"multiply.com","origin":"https://multiply.com","robots":{"status":"html","http":200,"url":"https://system1.com/","bytes":25579,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":6310,"domain":"hstspreload.org","origin":"https://hstspreload.org","robots":{"status":"missing","http":404,"url":"https://hstspreload.org/robots.txt","bytes":19,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":19},"home":{"status":"ok","http":200,"url":"https://hstspreload.org/","html_tokens":3953,"markdown_tokens":1946,"pruned_tokens":403}},
{"rank":6311,"domain":"imd.org","origin":"https://imd.org","robots":{"status":"ok","http":200,"url":"https://www.imd.org/robots.txt","bytes":490,"groups":7},"bots":{"GPTBot":["partial","*",3,1],"ChatGPT-User":["partial","*",3,1],"OAI-SearchBot":["partial","*",3,1],"ClaudeBot":["partial","*",3,1],"Claude-User":["partial","*",3,1],"Claude-SearchBot":["partial","*",3,1],"anthropic-ai":["partial","*",3,1],"PerplexityBot":["partial","*",3,1],"Perplexity-User":["partial","*",3,1],"Google-Extended":["partial","*",3,1],"Applebot-Extended":["partial","*",3,1],"CCBot":["partial","*",3,1],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["partial","*",3,1],"cohere-ai":["partial","*",3,1],"Diffbot":["partial","*",3,1],"ContextFetchBot":["partial","*",3,1]},"llms":{"status":"other_text","http":200,"bytes":3072163},"home":{"status":"ok","http":200,"url":"https://www.imd.org/","html_tokens":93871,"markdown_tokens":92,"pruned_tokens":92}},
{"rank":6312,"domain":"web.nhk","origin":"https://www.web.nhk","robots":{"status":"ok","http":200,"url":"https://www.web.nhk/robots.txt","bytes":427,"groups":1},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["blocked","named",1,0],"OAI-SearchBot":["blocked","named",1,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["blocked","named",1,0],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"http_error","http":401,"bytes":133},"home":{"status":"too_large","error":"response larger than 5 MB"}},
{"rank":6313,"domain":"cea.fr","origin":"https://cea.fr","robots":{"status":"ok","http":200,"url":"https://www.cea.fr/robots.txt","bytes":3700,"groups":1},"bots":{"GPTBot":["partial","*",86,0],"ChatGPT-User":["partial","*",86,0],"OAI-SearchBot":["partial","*",86,0],"ClaudeBot":["partial","*",86,0],"Claude-User":["partial","*",86,0],"Claude-SearchBot":["partial","*",86,0],"anthropic-ai":["partial","*",86,0],"PerplexityBot":["partial","*",86,0],"Perplexity-User":["partial","*",86,0],"Google-Extended":["partial","*",86,0],"Applebot-Extended":["partial","*",86,0],"CCBot":["partial","*",86,0],"Bytespider":["partial","*",86,0],"Amazonbot":["partial","*",86,0],"meta-externalagent":["partial","*",86,0],"cohere-ai":["partial","*",86,0],"Diffbot":["partial","*",86,0],"ContextFetchBot":["partial","*",86,0]},"llms":{"status":"missing","http":404,"bytes":643},"home":{"status":"ok","http":200,"url":"https://www.cea.fr/","html_tokens":29155,"markdown_tokens":403,"pruned_tokens":109}},
{"rank":6314,"domain":"civitai.com","origin":"https://civitai.com","robots":{"status":"ok","http":200,"url":"https://civitai.com/robots.txt","bytes":1304,"groups":1},"bots":{"GPTBot":["partial","*",32,1],"ChatGPT-User":["partial","*",32,1],"OAI-SearchBot":["partial","*",32,1],"ClaudeBot":["partial","*",32,1],"Claude-User":["partial","*",32,1],"Claude-SearchBot":["partial","*",32,1],"anthropic-ai":["partial","*",32,1],"PerplexityBot":["partial","*",32,1],"Perplexity-User":["partial","*",32,1],"Google-Extended":["partial","*",32,1],"Applebot-Extended":["partial","*",32,1],"CCBot":["partial","*",32,1],"Bytespider":["partial","*",32,1],"Amazonbot":["partial","*",32,1],"meta-externalagent":["partial","*",32,1],"cohere-ai":["partial","*",32,1],"Diffbot":["partial","*",32,1],"ContextFetchBot":["partial","*",32,1]},"llms":{"status":"missing","http":404,"bytes":88250},"home":{"status":"http_error","http":403,"url":"https://civitai.com/"}},
{"rank":6315,"domain":"suhosin.org","origin":"https://suhosin.org","robots":{"status":"missing","http":404,"url":"https://suhosin.org/robots.txt","bytes":9379,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":9379},"home":{"status":"ok","http":200,"url":"https://suhosin.org/","html_tokens":620,"markdown_tokens":236,"pruned_tokens":79}},
{"rank":6316,"domain":"tribunnews.com","origin":"https://tribunnews.com","robots":{"status":"ok","http":200,"url":"https://www.tribunnews.com/robots.txt","bytes":1154,"groups":11},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["blocked","named",1,0],"OAI-SearchBot":["partial","*",10,6],"ClaudeBot":["partial","*",10,6],"Claude-User":["partial","*",10,6],"Claude-SearchBot":["partial","*",10,6],"anthropic-ai":["partial","*",10,6],"PerplexityBot":["partial","*",10,6],"Perplexity-User":["partial","*",10,6],"Google-Extended":["allowed","named",0,1],"Applebot-Extended":["partial","*",10,6],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["partial","*",10,6],"meta-externalagent":["partial","*",10,6],"cohere-ai":["partial","*",10,6],"Diffbot":["partial","*",10,6],"ContextFetchBot":["partial","*",10,6]},"llms":{"status":"http_error","http":403,"bytes":7431},"home":{"status":"http_error","http":403,"url":"https://tribunnews.com/"}},
{"rank":6317,"domain":"tsn.ca","origin":"https://tsn.ca","robots":{"status":"ok","http":200,"url":"https://www.tsn.ca/robots.txt","bytes":17583,"groups":12},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["blocked","named",1,0],"OAI-SearchBot":["blocked","named",1,0],"ClaudeBot":["partial","*",400,1],"Claude-User":["partial","*",400,1],"Claude-SearchBot":["partial","*",400,1],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["partial","*",400,1],"Google-Extended":["partial","*",400,1],"Applebot-Extended":["partial","*",400,1],"CCBot":["blocked","named",1,0],"Bytespider":["partial","*",400,1],"Amazonbot":["partial","*",400,1],"meta-externalagent":["partial","*",400,1],"cohere-ai":["partial","*",400,1],"Diffbot":["partial","*",400,1],"ContextFetchBot":["partial","*",400,1]},"llms":{"status":"missing","http":404,"bytes":147067},"home":{"status":"ok","http":200,"url":"https://www.tsn.ca/","html_tokens":121964,"markdown_tokens":358,"pruned_tokens":132}},
{"rank":6318,"domain":"deccanchronicle.com","origin":"https://deccanchronicle.com","robots":{"status":"refused","http":403,"url":"https://www.deccanchronicle.com/robots.txt","bytes":153,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":6319,"domain":"education.fr","origin":"http://education.fr","robots":{"status":"ok","http":200,"url":"https://www.education.gouv.fr/robots.txt","bytes":2408,"groups":3},"bots":{"GPTBot":["partial","*",20,12],"ChatGPT-User":["partial","*",20,12],"OAI-SearchBot":["partial","*",20,12],"ClaudeBot":["partial","*",20,12],"Claude-User":["partial","*",20,12],"Claude-SearchBot":["partial","*",20,12],"anthropic-ai":["partial","*",20,12],"PerplexityBot":["partial","*",20,12],"Perplexity-User":["partial","*",20,12],"Google-Extended":["partial","*",20,12],"Applebot-Extended":["partial","*",20,12],"CCBot":["partial","*",20,12],"Bytespider":["blocked","named",1,0],"Amazonbot":["partial","*",20,12],"meta-externalagent":["partial","*",20,12],"cohere-ai":["partial","*",20,12],"Diffbot":["partial","*",20,12],"ContextFetchBot":["partial","*",20,12]},"llms":{"status":"http_error","http":403,"bytes":146},"home":{"status":"http_error","http":403,"url":"https://www.education.gouv.fr/"}},
{"rank":6320,"domain":"or.tz","origin":"https://or.tz","robots":{"status":"unreachable","http":null,"url":"https://or.tz/robots.txt","bytes":0,"groups":0,"error":"cannot resolve or.tz"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":6321,"domain":"link.amazon","origin":"https://link.amazon","robots":{"status":"refused","http":403,"url":"https://link.amazon/robots.txt","bytes":118,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":6322,"domain":"usitc.gov","origin":"https://usitc.gov","robots":{"status":"refused","http":403,"url":"https://usitc.gov/robots.txt","bytes":373,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":6323,"domain":"kemdikbud.go.id","origin":"https://kemdikbud.go.id","robots":{"status":"unreachable","http":null,"url":"https://kemdikbud.go.id/robots.txt","bytes":0,"groups":0,"error":"cannot resolve kemdikbud.go.id"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":6324,"domain":"hackerrank.com","origin":"https://hackerrank.com","robots":{"status":"ok","http":200,"url":"https://www.hackerrank.com/robots.txt","bytes":680,"groups":5},"bots":{"GPTBot":["partial","*",13,2],"ChatGPT-User":["partial","*",13,2],"OAI-SearchBot":["partial","*",13,2],"ClaudeBot":["partial","*",13,2],"Claude-User":["partial","*",13,2],"Claude-SearchBot":["partial","*",13,2],"anthropic-ai":["partial","*",13,2],"PerplexityBot":["partial","*",13,2],"Perplexity-User":["partial","*",13,2],"Google-Extended":["partial","*",13,2],"Applebot-Extended":["partial","*",13,2],"CCBot":["partial","*",13,2],"Bytespider":["partial","*",13,2],"Amazonbot":["partial","*",13,2],"meta-externalagent":["partial","*",13,2],"cohere-ai":["partial","*",13,2],"Diffbot":["partial","*",13,2],"ContextFetchBot":["partial","*",13,2]},"llms":{"status":"present","http":200,"bytes":5456},"home":{"status":"ok","http":200,"url":"https://www.hackerrank.com/","html_tokens":26163,"markdown_tokens":3918,"pruned_tokens":98}},
{"rank":6325,"domain":"bafa.de","origin":"https://bafa.de","robots":{"status":"missing","http":404,"url":"https://bafa.de/robots.txt","bytes":46418,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":46418},"home":{"status":"ok","http":200,"url":"https://www.bafa.de/DE/Home/home_node.html","html_tokens":17544,"markdown_tokens":71,"pruned_tokens":71}},
{"rank":6326,"domain":"stereogum.com","origin":"https://stereogum.com","robots":{"status":"ok","http":200,"url":"https://stereogum.com/robots.txt","bytes":263,"groups":2},"bots":{"GPTBot":["partial","*",7,0],"ChatGPT-User":["partial","*",7,0],"OAI-SearchBot":["partial","*",7,0],"ClaudeBot":["partial","*",7,0],"Claude-User":["partial","*",7,0],"Claude-SearchBot":["partial","*",7,0],"anthropic-ai":["partial","*",7,0],"PerplexityBot":["partial","*",7,0],"Perplexity-User":["partial","*",7,0],"Google-Extended":["partial","*",7,0],"Applebot-Extended":["partial","*",7,0],"CCBot":["partial","*",7,0],"Bytespider":["partial","*",7,0],"Amazonbot":["partial","*",7,0],"meta-externalagent":["partial","*",7,0],"cohere-ai":["partial","*",7,0],"Diffbot":["partial","*",7,0],"ContextFetchBot":["partial","*",7,0]},"llms":{"status":"missing","http":404,"bytes":9},"home":{"status":"ok","http":200,"url":"https://stereogum.com/","html_tokens":70300,"markdown_tokens":25,"pruned_tokens":25}},
{"rank":6327,"domain":"azurefd.net","origin":"https://www.azurefd.net","robots":{"status":"missing","http":404,"url":"https://www.azurefd.net/robots.txt","bytes":266389,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"error","error":"Invalid input ConnectionInputs.RECV_HEADERS in state ConnectionState.CLOSED"},"home":{"status":"http_error","http":404,"url":"https://www.azurefd.net/"}},
{"rank":6328,"domain":"boell.de","origin":"https://boell.de","robots":{"status":"ok","http":200,"url":"https://www.boell.de/robots.txt","bytes":2627,"groups":1},"bots":{"GPTBot":["partial","*",45,20],"ChatGPT-User":["partial","*",45,20],"OAI-SearchBot":["partial","*",45,20],"ClaudeBot":["partial","*",45,20],"Claude-User":["partial","*",45,20],"Claude-SearchBot":["partial","*",45,20],"anthropic-ai":["partial","*",45,20],"PerplexityBot":["partial","*",45,20],"Perplexity-User":["partial","*",45,20],"Google-Extended":["partial","*",45,20],"Applebot-Extended":["partial","*",45,20],"CCBot":["partial","*",45,20],"Bytespider":["partial","*",45,20],"Amazonbot":["partial","*",45,20],"meta-externalagent":["partial","*",45,20],"cohere-ai":["partial","*",45,20],"Diffbot":["partial","*",45,20],"ContextFetchBot":["partial","*",45,20]},"llms":{"status":"missing","http":404,"bytes":186},"home":{"status":"timeout","error":"timed out after 15 s"}},
{"rank":6329,"domain":"groups.io","origin":"https://groups.io","robots":{"status":"ok","http":200,"url":"https://groups.io/robots.txt","bytes":3186,"groups":2},"bots":{"GPTBot":["partial","*",4,0],"ChatGPT-User":["partial","*",4,0],"OAI-SearchBot":["partial","*",4,0],"ClaudeBot":["partial","*",4,0],"Claude-User":["partial","*",4,0],"Claude-SearchBot":["partial","*",4,0],"anthropic-ai":["partial","*",4,0],"PerplexityBot":["partial","*",4,0],"Perplexity-User":["partial","*",4,0],"Google-Extended":["partial","*",4,0],"Applebot-Extended":["partial","named",5,0],"CCBot":["partial","*",4,0],"Bytespider":["partial","*",4,0],"Amazonbot":["partial","*",4,0],"meta-externalagent":["partial","*",4,0],"cohere-ai":["partial","*",4,0],"Diffbot":["partial","*",4,0],"ContextFetchBot":["partial","*",4,0]},"llms":{"status":"missing","http":404,"bytes":222},"home":{"status":"ok","http":200,"url":"https://groups.io/","html_tokens":22820,"markdown_tokens":1452,"pruned_tokens":987}},
{"rank":6330,"domain":"defra.gov.uk","origin":"http://defra.gov.uk","robots":{"status":"ok","http":200,"url":"http://defra.gov.uk/robots.txt","bytes":65,"groups":1},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"missing","http":404,"bytes":1786},"home":{"status":"ok","http":200,"url":"https://www.gov.uk/government/organisations/department-for-environment-food-rural-affairs","html_tokens":36819,"markdown_tokens":67,"pruned_tokens":67}},
{"rank":6331,"domain":"cox.net","origin":"https://cox.net","robots":{"status":"html","http":200,"url":"https://www.cox.com/residential/home.html","bytes":91137,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":6332,"domain":"bonanza.com","origin":"https://bonanza.com","robots":{"status":"ok","http":200,"url":"https://www.bonanza.com/robots.txt","bytes":260,"groups":1},"bots":{"GPTBot":["partial","*",7,0],"ChatGPT-User":["partial","*",7,0],"OAI-SearchBot":["partial","*",7,0],"ClaudeBot":["partial","*",7,0],"Claude-User":["partial","*",7,0],"Claude-SearchBot":["partial","*",7,0],"anthropic-ai":["partial","*",7,0],"PerplexityBot":["partial","*",7,0],"Perplexity-User":["partial","*",7,0],"Google-Extended":["partial","*",7,0],"Applebot-Extended":["partial","*",7,0],"CCBot":["partial","*",7,0],"Bytespider":["partial","*",7,0],"Amazonbot":["partial","*",7,0],"meta-externalagent":["partial","*",7,0],"cohere-ai":["partial","*",7,0],"Diffbot":["partial","*",7,0],"ContextFetchBot":["partial","*",7,0]},"llms":{"status":"http_error","http":403,"bytes":5544},"home":{"status":"ok","http":200,"url":"https://www.bonanza.com/","html_tokens":29268,"markdown_tokens":52,"pruned_tokens":52}},
{"rank":6333,"domain":"challenges.fr","origin":"https://challenges.fr","robots":{"status":"ok","http":200,"url":"https://www.challenges.fr/robots.txt","bytes":3270,"groups":3},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["partial","named",20,2],"OAI-SearchBot":["partial","named",20,2],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",20,1],"Claude-SearchBot":["partial","named",20,2],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["partial","named",20,2],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",20,1]},"llms":{"status":"missing","http":404,"bytes":194},"home":{"status":"ok","http":200,"url":"https://www.challenges.fr/","html_tokens":176456,"markdown_tokens":1162,"pruned_tokens":146}},
{"rank":6334,"domain":"immobilienscout24.de","origin":"https://immobilienscout24.de","robots":{"status":"ok","http":200,"url":"https://www.immobilienscout24.de/robots.txt","bytes":1548,"groups":9},"bots":{"GPTBot":["partial","named",1,1],"ChatGPT-User":["allowed","named",0,1],"OAI-SearchBot":["allowed","named",0,1],"ClaudeBot":["partial","named",1,1],"Claude-User":["allowed","named",0,1],"Claude-SearchBot":["allowed","named",0,1],"anthropic-ai":["partial","*",22,5],"PerplexityBot":["allowed","named",0,1],"Perplexity-User":["partial","*",22,5],"Google-Extended":["partial","*",22,5],"Applebot-Extended":["partial","*",22,5],"CCBot":["partial","*",22,5],"Bytespider":["partial","*",22,5],"Amazonbot":["partial","*",22,5],"meta-externalagent":["partial","*",22,5],"cohere-ai":["partial","*",22,5],"Diffbot":["partial","*",22,5],"ContextFetchBot":["partial","*",22,5]},"llms":{"status":"missing","http":404,"bytes":5619},"home":{"status":"http_error","http":401,"url":"https://www.immobilienscout24.de/"}},
{"rank":6335,"domain":"cloudera.com","origin":"https://cloudera.com","robots":{"status":"ok","http":200,"url":"https://www.cloudera.com/robots.txt","bytes":3158,"groups":1},"bots":{"GPTBot":["partial","*",43,2],"ChatGPT-User":["partial","*",43,2],"OAI-SearchBot":["partial","*",43,2],"ClaudeBot":["partial","*",43,2],"Claude-User":["partial","*",43,2],"Claude-SearchBot":["partial","*",43,2],"anthropic-ai":["partial","*",43,2],"PerplexityBot":["partial","*",43,2],"Perplexity-User":["partial","*",43,2],"Google-Extended":["partial","*",43,2],"Applebot-Extended":["partial","*",43,2],"CCBot":["partial","*",43,2],"Bytespider":["partial","*",43,2],"Amazonbot":["partial","*",43,2],"meta-externalagent":["partial","*",43,2],"cohere-ai":["partial","*",43,2],"Diffbot":["partial","*",43,2],"ContextFetchBot":["partial","*",43,2]},"llms":{"status":"missing","http":404,"bytes":72875},"home":{"status":"ok","http":200,"url":"https://www.cloudera.com/","html_tokens":34183,"markdown_tokens":298,"pruned_tokens":231}},
{"rank":6336,"domain":"edpharmahub.com","origin":"https://edpharmahub.com","robots":{"status":"ok","http":200,"url":"https://edpharmahub.com/robots.txt","bytes":74,"groups":1},"bots":{"GPTBot":["partial","*",1,0],"ChatGPT-User":["partial","*",1,0],"OAI-SearchBot":["partial","*",1,0],"ClaudeBot":["partial","*",1,0],"Claude-User":["partial","*",1,0],"Claude-SearchBot":["partial","*",1,0],"anthropic-ai":["partial","*",1,0],"PerplexityBot":["partial","*",1,0],"Perplexity-User":["partial","*",1,0],"Google-Extended":["partial","*",1,0],"Applebot-Extended":["partial","*",1,0],"CCBot":["partial","*",1,0],"Bytespider":["partial","*",1,0],"Amazonbot":["partial","*",1,0],"meta-externalagent":["partial","*",1,0],"cohere-ai":["partial","*",1,0],"Diffbot":["partial","*",1,0],"ContextFetchBot":["partial","*",1,0]},"llms":{"status":"missing","http":404,"bytes":644},"home":{"status":"ok","http":200,"url":"https://edpharmahub.com/","html_tokens":4133,"markdown_tokens":206,"pruned_tokens":89}},
{"rank":6337,"domain":"m-w.com","origin":"https://m-w.com","robots":{"status":"ok","http":200,"url":"https://www.merriam-webster.com/robots.txt","bytes":708,"groups":1},"bots":{"GPTBot":["partial","*",7,0],"ChatGPT-User":["partial","*",7,0],"OAI-SearchBot":["partial","*",7,0],"ClaudeBot":["partial","*",7,0],"Claude-User":["partial","*",7,0],"Claude-SearchBot":["partial","*",7,0],"anthropic-ai":["partial","*",7,0],"PerplexityBot":["partial","*",7,0],"Perplexity-User":["partial","*",7,0],"Google-Extended":["partial","*",7,0],"Applebot-Extended":["partial","*",7,0],"CCBot":["partial","*",7,0],"Bytespider":["partial","*",7,0],"Amazonbot":["partial","*",7,0],"meta-externalagent":["partial","*",7,0],"cohere-ai":["partial","*",7,0],"Diffbot":["partial","*",7,0],"ContextFetchBot":["partial","*",7,0]},"llms":{"status":"missing","http":404,"bytes":146},"home":{"status":"http_error","http":403,"url":"https://www.merriam-webster.com/"}},
{"rank":6338,"domain":"direct.gov.uk","origin":"https://www.direct.gov.uk","robots":{"status":"ok","http":200,"url":"https://www.direct.gov.uk/robots.txt","bytes":70,"groups":1},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"missing","http":404,"bytes":1647},"home":{"status":"ok","http":200,"url":"https://www.gov.uk","html_tokens":21379,"markdown_tokens":26,"pruned_tokens":26}},
{"rank":6339,"domain":"etihad.com","origin":"https://etihad.com","robots":{"status":"unreachable","http":null,"url":"https://etihad.com/robots.txt","bytes":0,"groups":0,"error":"<StreamReset stream_id:1, error_code:2, remote_reset:True>"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":6340,"domain":"accc.gov.au","origin":"https://accc.gov.au","robots":{"status":"refused","http":403,"url":"https://www.accc.gov.au/robots.txt","bytes":387,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":6341,"domain":"crave.ca","origin":"https://crave.ca","robots":{"status":"ok","http":200,"url":"https://www.crave.ca/robots.txt","bytes":406,"groups":1},"bots":{"GPTBot":["partial","*",15,0],"ChatGPT-User":["partial","*",15,0],"OAI-SearchBot":["partial","*",15,0],"ClaudeBot":["partial","*",15,0],"Claude-User":["partial","*",15,0],"Claude-SearchBot":["partial","*",15,0],"anthropic-ai":["partial","*",15,0],"PerplexityBot":["partial","*",15,0],"Perplexity-User":["partial","*",15,0],"Google-Extended":["partial","*",15,0],"Applebot-Extended":["partial","*",15,0],"CCBot":["partial","*",15,0],"Bytespider":["partial","*",15,0],"Amazonbot":["partial","*",15,0],"meta-externalagent":["partial","*",15,0],"cohere-ai":["partial","*",15,0],"Diffbot":["partial","*",15,0],"ContextFetchBot":["partial","*",15,0]},"llms":{"status":"html","http":200,"bytes":1811},"home":{"status":"ok","http":200,"url":"https://www.crave.ca/","html_tokens":453,"markdown_tokens":0,"pruned_tokens":0}},
{"rank":6342,"domain":"jetblue.com","origin":"https://jetblue.com","robots":{"status":"ok","http":200,"url":"https://www.jetblue.com/robots.txt","bytes":255,"groups":1},"bots":{"GPTBot":["partial","*",3,1],"ChatGPT-User":["partial","*",3,1],"OAI-SearchBot":["partial","*",3,1],"ClaudeBot":["partial","*",3,1],"Claude-User":["partial","*",3,1],"Claude-SearchBot":["partial","*",3,1],"anthropic-ai":["partial","*",3,1],"PerplexityBot":["partial","*",3,1],"Perplexity-User":["partial","*",3,1],"Google-Extended":["partial","*",3,1],"Applebot-Extended":["partial","*",3,1],"CCBot":["partial","*",3,1],"Bytespider":["partial","*",3,1],"Amazonbot":["partial","*",3,1],"meta-externalagent":["partial","*",3,1],"cohere-ai":["partial","*",3,1],"Diffbot":["partial","*",3,1],"ContextFetchBot":["partial","*",3,1]},"llms":{"status":"missing","http":404,"bytes":37162},"home":{"status":"ok","http":200,"url":"https://www.jetblue.com/","html_tokens":165263,"markdown_tokens":35,"pruned_tokens":35}},
{"rank":6343,"domain":"bms.com","origin":"https://bms.com","robots":{"status":"ok","http":200,"url":"https://www.bms.com/robots.txt","bytes":260,"groups":1},"bots":{"GPTBot":["partial","*",7,0],"ChatGPT-User":["partial","*",7,0],"OAI-SearchBot":["partial","*",7,0],"ClaudeBot":["partial","*",7,0],"Claude-User":["partial","*",7,0],"Claude-SearchBot":["partial","*",7,0],"anthropic-ai":["partial","*",7,0],"PerplexityBot":["partial","*",7,0],"Perplexity-User":["partial","*",7,0],"Google-Extended":["partial","*",7,0],"Applebot-Extended":["partial","*",7,0],"CCBot":["partial","*",7,0],"Bytespider":["partial","*",7,0],"Amazonbot":["partial","*",7,0],"meta-externalagent":["partial","*",7,0],"cohere-ai":["partial","*",7,0],"Diffbot":["partial","*",7,0],"ContextFetchBot":["partial","*",7,0]},"llms":{"status":"missing","http":404,"bytes":82362},"home":{"status":"ok","http":200,"url":"https://www.bms.com/","html_tokens":21892,"markdown_tokens":300,"pruned_tokens":142}},
{"rank":6344,"domain":"104.com.tw","origin":"https://104.com.tw","robots":{"status":"refused","http":403,"url":"https://www.104.com.tw/robots.txt","bytes":5615,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":6345,"domain":"pl.ua","origin":"https://pl.ua","robots":{"status":"unreachable","http":null,"url":"https://pl.ua/robots.txt","bytes":0,"groups":0,"error":"cannot resolve pl.ua"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":6346,"domain":"captivate.fm","origin":"https://captivate.fm","robots":{"status":"ok","http":200,"url":"https://www.captivate.fm/robots.txt","bytes":860,"groups":1},"bots":{"GPTBot":["partial","*",11,1],"ChatGPT-User":["partial","*",11,1],"OAI-SearchBot":["partial","*",11,1],"ClaudeBot":["partial","*",11,1],"Claude-User":["partial","*",11,1],"Claude-SearchBot":["partial","*",11,1],"anthropic-ai":["partial","*",11,1],"PerplexityBot":["partial","*",11,1],"Perplexity-User":["partial","*",11,1],"Google-Extended":["partial","*",11,1],"Applebot-Extended":["partial","*",11,1],"CCBot":["partial","*",11,1],"Bytespider":["partial","*",11,1],"Amazonbot":["partial","*",11,1],"meta-externalagent":["partial","*",11,1],"cohere-ai":["partial","*",11,1],"Diffbot":["partial","*",11,1],"ContextFetchBot":["partial","*",11,1]},"llms":{"status":"missing","http":404,"bytes":62990},"home":{"status":"ok","http":200,"url":"https://www.captivate.fm/","html_tokens":71807,"markdown_tokens":2802,"pruned_tokens":386}},
{"rank":6347,"domain":"info.gov.hk","origin":"https://www.info.gov.hk","robots":{"status":"ok","http":200,"url":"https://www.info.gov.hk/robots.txt","bytes":299,"groups":1},"bots":{"GPTBot":["partial","*",1,0],"ChatGPT-User":["partial","*",1,0],"OAI-SearchBot":["partial","*",1,0],"ClaudeBot":["partial","*",1,0],"Claude-User":["partial","*",1,0],"Claude-SearchBot":["partial","*",1,0],"anthropic-ai":["partial","*",1,0],"PerplexityBot":["partial","*",1,0],"Perplexity-User":["partial","*",1,0],"Google-Extended":["partial","*",1,0],"Applebot-Extended":["partial","*",1,0],"CCBot":["partial","*",1,0],"Bytespider":["partial","*",1,0],"Amazonbot":["partial","*",1,0],"meta-externalagent":["partial","*",1,0],"cohere-ai":["partial","*",1,0],"Diffbot":["partial","*",1,0],"ContextFetchBot":["partial","*",1,0]},"llms":{"status":"missing","http":404,"bytes":9018},"home":{"status":"ok","http":200,"url":"https://www.info.gov.hk/","html_tokens":775,"markdown_tokens":101,"pruned_tokens":82}},
{"rank":6348,"domain":"experiment.com","origin":"https://experiment.com","robots":{"status":"ok","http":200,"url":"https://experiment.com/robots.txt","bytes":87,"groups":1},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":1687},"home":{"status":"ok","http":200,"url":"https://experiment.com/","html_tokens":17668,"markdown_tokens":2036,"pruned_tokens":84}},
{"rank":6349,"domain":"conference-board.org","origin":"https://conference-board.org","robots":{"status":"ok","http":200,"url":"https://www.conference-board.org/robots.txt","bytes":306209,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"html","http":200,"bytes":0},"home":{"status":"ok","http":200,"url":"https://www.conference-board.org/north-america/","html_tokens":115257,"markdown_tokens":216,"pruned_tokens":196}},
{"rank":6350,"domain":"petfinder.com","origin":"https://petfinder.com","robots":{"status":"refused","http":403,"url":"https://www.petfinder.com/robots.txt","bytes":35327,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":6351,"domain":"greekreporter.com","origin":"https://greekreporter.com","robots":{"status":"ok","http":200,"url":"https://greekreporter.com/robots.txt","bytes":68,"groups":1},"bots":{"GPTBot":["partial","*",1,1],"ChatGPT-User":["partial","*",1,1],"OAI-SearchBot":["partial","*",1,1],"ClaudeBot":["partial","*",1,1],"Claude-User":["partial","*",1,1],"Claude-SearchBot":["partial","*",1,1],"anthropic-ai":["partial","*",1,1],"PerplexityBot":["partial","*",1,1],"Perplexity-User":["partial","*",1,1],"Google-Extended":["partial","*",1,1],"Applebot-Extended":["partial","*",1,1],"CCBot":["partial","*",1,1],"Bytespider":["partial","*",1,1],"Amazonbot":["partial","*",1,1],"meta-externalagent":["partial","*",1,1],"cohere-ai":["partial","*",1,1],"Diffbot":["partial","*",1,1],"ContextFetchBot":["partial","*",1,1]},"llms":{"status":"missing","http":404,"bytes":404346},"home":{"status":"http_error","http":403,"url":"https://greekreporter.com/"}},
{"rank":6352,"domain":"lejdd.fr","origin":"https://lejdd.fr","robots":{"status":"ok","http":200,"url":"https://www.lejdd.fr/robots.txt","bytes":572,"groups":10},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["blocked","named",1,0],"OAI-SearchBot":["partial","*",1,0],"ClaudeBot":["partial","*",1,0],"Claude-User":["partial","*",1,0],"Claude-SearchBot":["partial","*",1,0],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["partial","*",1,0],"Perplexity-User":["partial","*",1,0],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["partial","*",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["partial","*",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["partial","*",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["partial","*",1,0],"ContextFetchBot":["partial","*",1,0]},"llms":{"status":"missing","http":404,"bytes":60161},"home":{"status":"http_error","http":403,"url":"https://www.lejdd.fr/"}},
{"rank":6353,"domain":"supermicro.com","origin":"https://supermicro.com","robots":{"status":"ok","http":200,"url":"https://www.supermicro.com/robots.txt","bytes":147,"groups":1},"bots":{"GPTBot":["partial","*",4,0],"ChatGPT-User":["partial","*",4,0],"OAI-SearchBot":["partial","*",4,0],"ClaudeBot":["partial","*",4,0],"Claude-User":["partial","*",4,0],"Claude-SearchBot":["partial","*",4,0],"anthropic-ai":["partial","*",4,0],"PerplexityBot":["partial","*",4,0],"Perplexity-User":["partial","*",4,0],"Google-Extended":["partial","*",4,0],"Applebot-Extended":["partial","*",4,0],"CCBot":["partial","*",4,0],"Bytespider":["partial","*",4,0],"Amazonbot":["partial","*",4,0],"meta-externalagent":["partial","*",4,0],"cohere-ai":["partial","*",4,0],"Diffbot":["partial","*",4,0],"ContextFetchBot":["partial","*",4,0]},"llms":{"status":"html","http":200,"bytes":191154},"home":{"status":"ok","http":200,"url":"https://www.supermicro.com/en/","html_tokens":47748,"markdown_tokens":246,"pruned_tokens":204}},
{"rank":6354,"domain":"km.ua","origin":"http://km.ua","robots":{"status":"ok","http":200,"url":"http://km.ua/robots.txt","bytes":35,"groups":1},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"missing","http":404,"bytes":236},"home":{"status":"ok","http":200,"url":"http://km.ua/","html_tokens":1409,"markdown_tokens":368,"pruned_tokens":13}},
{"rank":6355,"domain":"tempo.co","origin":"https://tempo.co","robots":{"status":"ok","http":200,"url":"https://www.tempo.co/robots.txt","bytes":4828,"groups":87},"bots":{"GPTBot":["blocked","named",3,0],"ChatGPT-User":["blocked","named",2,0],"OAI-SearchBot":["blocked","named",2,0],"ClaudeBot":["blocked","named",2,0],"Claude-User":["blocked","named",2,0],"Claude-SearchBot":["blocked","named",1,0],"anthropic-ai":["blocked","named",2,0],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["blocked","named",1,0],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",2,0],"Bytespider":["blocked","named",2,0],"Amazonbot":["blocked","named",2,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",2,0],"Diffbot":["blocked","named",2,0],"ContextFetchBot":["allowed","*",0,2]},"llms":{"status":"present","http":200,"bytes":1890},"home":{"status":"http_error","http":403,"url":"https://www.tempo.co/"}},
{"rank":6356,"domain":"chegg.com","origin":"https://chegg.com","robots":{"status":"ok","http":200,"url":"https://www.chegg.com/robots.txt","bytes":1614,"groups":4},"bots":{"GPTBot":["partial","*",57,0],"ChatGPT-User":["partial","*",57,0],"OAI-SearchBot":["partial","*",57,0],"ClaudeBot":["partial","*",57,0],"Claude-User":["partial","*",57,0],"Claude-SearchBot":["partial","*",57,0],"anthropic-ai":["partial","*",57,0],"PerplexityBot":["partial","*",57,0],"Perplexity-User":["partial","*",57,0],"Google-Extended":["partial","*",57,0],"Applebot-Extended":["partial","*",57,0],"CCBot":["partial","*",57,0],"Bytespider":["partial","*",57,0],"Amazonbot":["partial","*",57,0],"meta-externalagent":["partial","*",57,0],"cohere-ai":["partial","*",57,0],"Diffbot":["partial","*",57,0],"ContextFetchBot":["partial","*",57,0]},"llms":{"status":"missing","http":404,"bytes":1461},"home":{"status":"ok","http":200,"url":"https://www.chegg.com/","html_tokens":102402,"markdown_tokens":88,"pruned_tokens":88}},
{"rank":6357,"domain":"dpd.com","origin":"https://dpd.com","robots":{"status":"ok","http":200,"url":"https://www.dpd.com/robots.txt","bytes":10139,"groups":2},"bots":{"GPTBot":["partial","*",1,1],"ChatGPT-User":["partial","*",1,1],"OAI-SearchBot":["partial","*",1,1],"ClaudeBot":["partial","*",1,1],"Claude-User":["partial","*",1,1],"Claude-SearchBot":["partial","*",1,1],"anthropic-ai":["partial","*",1,1],"PerplexityBot":["partial","*",1,1],"Perplexity-User":["partial","*",1,1],"Google-Extended":["partial","*",1,1],"Applebot-Extended":["partial","*",1,1],"CCBot":["partial","*",1,1],"Bytespider":["partial","*",1,1],"Amazonbot":["partial","*",1,1],"meta-externalagent":["partial","*",1,1],"cohere-ai":["partial","*",1,1],"Diffbot":["partial","*",1,1],"ContextFetchBot":["partial","*",1,1]},"llms":{"status":"missing","http":404,"bytes":83861},"home":{"status":"ok","http":200,"url":"https://www.dpd.com/","html_tokens":32843,"markdown_tokens":151,"pruned_tokens":133}},
{"rank":6358,"domain":"federatedjournals.com","origin":"https://federatedjournals.com","robots":{"status":"ok","http":200,"url":"https://federatedjournals.com/robots.txt","bytes":128,"groups":1},"bots":{"GPTBot":["partial","*",3,0],"ChatGPT-User":["partial","*",3,0],"OAI-SearchBot":["partial","*",3,0],"ClaudeBot":["partial","*",3,0],"Claude-User":["partial","*",3,0],"Claude-SearchBot":["partial","*",3,0],"anthropic-ai":["partial","*",3,0],"PerplexityBot":["partial","*",3,0],"Perplexity-User":["partial","*",3,0],"Google-Extended":["partial","*",3,0],"Applebot-Extended":["partial","*",3,0],"CCBot":["partial","*",3,0],"Bytespider":["partial","*",3,0],"Amazonbot":["partial","*",3,0],"meta-externalagent":["partial","*",3,0],"cohere-ai":["partial","*",3,0],"Diffbot":["partial","*",3,0],"ContextFetchBot":["partial","*",3,0]},"llms":{"status":"missing","http":404,"bytes":36341},"home":{"status":"http_error","http":500,"url":"https://federatedjournals.com/"}},
{"rank":6359,"domain":"austlii.edu.au","origin":"https://austlii.edu.au","robots":{"status":"ok","http":200,"url":"https://austlii.edu.au/robots.txt","bytes":2676,"groups":32},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["allowed","*",0,1],"OAI-SearchBot":["allowed","*",0,1],"ClaudeBot":["blocked","named",1,0],"Claude-User":["allowed","*",0,1],"Claude-SearchBot":["allowed","*",0,1],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["allowed","*",0,1],"Perplexity-User":["allowed","*",0,1],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["allowed","*",0,1]},"llms":{"status":"http_error","http":403,"bytes":5503},"home":{"status":"http_error","http":403,"url":"https://austlii.edu.au/"}},
{"rank":6360,"domain":"librivox.org","origin":"https://librivox.org","robots":{"status":"ok","http":200,"url":"https://librivox.org/robots.txt","bytes":192,"groups":1},"bots":{"GPTBot":["partial","*",4,0],"ChatGPT-User":["partial","*",4,0],"OAI-SearchBot":["partial","*",4,0],"ClaudeBot":["partial","*",4,0],"Claude-User":["partial","*",4,0],"Claude-SearchBot":["partial","*",4,0],"anthropic-ai":["partial","*",4,0],"PerplexityBot":["partial","*",4,0],"Perplexity-User":["partial","*",4,0],"Google-Extended":["partial","*",4,0],"Applebot-Extended":["partial","*",4,0],"CCBot":["partial","*",4,0],"Bytespider":["partial","*",4,0],"Amazonbot":["partial","*",4,0],"meta-externalagent":["partial","*",4,0],"cohere-ai":["partial","*",4,0],"Diffbot":["partial","*",4,0],"ContextFetchBot":["partial","*",4,0]},"llms":{"status":"missing","http":404,"bytes":15759},"home":{"status":"ok","http":200,"url":"https://librivox.org/","html_tokens":10176,"markdown_tokens":400,"pruned_tokens":89}},
{"rank":6361,"domain":"calculator.net","origin":"https://calculator.net","robots":{"status":"ok","http":200,"url":"https://calculator.net/robots.txt","bytes":64,"groups":1},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"missing","http":404,"bytes":6211},"home":{"status":"ok","http":200,"url":"https://www.calculator.net/","html_tokens":2919,"markdown_tokens":66,"pruned_tokens":66}},
{"rank":6362,"domain":"geni.us","origin":"https://geni.us","robots":{"status":"ok","http":200,"url":"https://geni.us/robots.txt","bytes":24,"groups":1},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"missing","http":404,"bytes":2034},"home":{"status":"ok","http":200,"url":"https://geniuslink.com","html_tokens":5711,"markdown_tokens":316,"pruned_tokens":143}},
{"rank":6363,"domain":"travelocity.com","origin":"https://travelocity.com","robots":{"status":"ok","http":200,"url":"https://www.travelocity.com/robots.txt","bytes":2046,"groups":4},"bots":{"GPTBot":["partial","*",51,0],"ChatGPT-User":["partial","named",5,1],"OAI-SearchBot":["partial","named",5,1],"ClaudeBot":["partial","*",51,0],"Claude-User":["partial","named",5,1],"Claude-SearchBot":["partial","named",5,1],"anthropic-ai":["partial","*",51,0],"PerplexityBot":["partial","named",5,1],"Perplexity-User":["partial","named",5,1],"Google-Extended":["partial","*",51,0],"Applebot-Extended":["partial","*",51,0],"CCBot":["partial","*",51,0],"Bytespider":["partial","*",51,0],"Amazonbot":["partial","*",51,0],"meta-externalagent":["partial","*",51,0],"cohere-ai":["partial","*",51,0],"Diffbot":["partial","*",51,0],"ContextFetchBot":["partial","*",51,0]},"llms":{"status":"http_error","http":403,"bytes":243},"home":{"status":"http_error","http":429,"url":"https://www.travelocity.com/"}},
{"rank":6364,"domain":"sony.jp","origin":"https://sony.jp","robots":{"status":"ok","http":200,"url":"https://www.sony.jp/robots.txt","bytes":230,"groups":1},"bots":{"GPTBot":["partial","*",7,0],"ChatGPT-User":["partial","*",7,0],"OAI-SearchBot":["partial","*",7,0],"ClaudeBot":["partial","*",7,0],"Claude-User":["partial","*",7,0],"Claude-SearchBot":["partial","*",7,0],"anthropic-ai":["partial","*",7,0],"PerplexityBot":["partial","*",7,0],"Perplexity-User":["partial","*",7,0],"Google-Extended":["partial","*",7,0],"Applebot-Extended":["partial","*",7,0],"CCBot":["partial","*",7,0],"Bytespider":["partial","*",7,0],"Amazonbot":["partial","*",7,0],"meta-externalagent":["partial","*",7,0],"cohere-ai":["partial","*",7,0],"Diffbot":["partial","*",7,0],"ContextFetchBot":["partial","*",7,0]},"llms":{"status":"missing","http":404,"bytes":3347},"home":{"status":"ok","http":200,"url":"https://www.sony.jp/","html_tokens":63457,"markdown_tokens":20,"pruned_tokens":17}},
{"rank":6365,"domain":"roberthalf.com","origin":"https://roberthalf.com","robots":{"status":"ok","http":200,"url":"https://www.roberthalf.com/robots.txt","bytes":18592,"groups":1},"bots":{"GPTBot":["partial","*",146,0],"ChatGPT-User":["partial","*",146,0],"OAI-SearchBot":["partial","*",146,0],"ClaudeBot":["partial","*",146,0],"Claude-User":["partial","*",146,0],"Claude-SearchBot":["partial","*",146,0],"anthropic-ai":["partial","*",146,0],"PerplexityBot":["partial","*",146,0],"Perplexity-User":["partial","*",146,0],"Google-Extended":["partial","*",146,0],"Applebot-Extended":["partial","*",146,0],"CCBot":["partial","*",146,0],"Bytespider":["partial","*",146,0],"Amazonbot":["partial","*",146,0],"meta-externalagent":["partial","*",146,0],"cohere-ai":["partial","*",146,0],"Diffbot":["partial","*",146,0],"ContextFetchBot":["partial","*",146,0]},"llms":{"status":"missing","http":404,"bytes":110341},"home":{"status":"ok","http":200,"url":"https://www.roberthalf.com/us/en","html_tokens":37624,"markdown_tokens":197,"pruned_tokens":38}},
{"rank":6366,"domain":"anthropologie.com","origin":"https://anthropologie.com","robots":{"status":"refused","http":403,"url":"https://www.anthropologie.com/robots.txt","bytes":777,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":6367,"domain":"semana.com","origin":"https://semana.com","robots":{"status":"ok","http":200,"url":"https://www.semana.com/robots.txt","bytes":1293,"groups":2},"bots":{"GPTBot":["partial","*",22,5],"ChatGPT-User":["partial","*",22,5],"OAI-SearchBot":["partial","*",22,5],"ClaudeBot":["partial","*",22,5],"Claude-User":["partial","*",22,5],"Claude-SearchBot":["partial","*",22,5],"anthropic-ai":["partial","*",22,5],"PerplexityBot":["partial","*",22,5],"Perplexity-User":["partial","*",22,5],"Google-Extended":["partial","named",22,5],"Applebot-Extended":["partial","*",22,5],"CCBot":["partial","*",22,5],"Bytespider":["partial","*",22,5],"Amazonbot":["partial","*",22,5],"meta-externalagent":["partial","*",22,5],"cohere-ai":["partial","*",22,5],"Diffbot":["partial","*",22,5],"ContextFetchBot":["partial","*",22,5]},"llms":{"status":"missing","http":404,"bytes":91655},"home":{"status":"ok","http":200,"url":"https://www.semana.com/","html_tokens":350327,"markdown_tokens":222,"pruned_tokens":164}},
{"rank":6368,"domain":"realmoneyslots-mobile.com","origin":"https://realmoneyslots-mobile.com","robots":{"status":"ok","http":200,"url":"https://realmoneyslots-mobile.com/robots.txt","bytes":522,"groups":1},"bots":{"GPTBot":["partial","*",18,0],"ChatGPT-User":["partial","*",18,0],"OAI-SearchBot":["partial","*",18,0],"ClaudeBot":["partial","*",18,0],"Claude-User":["partial","*",18,0],"Claude-SearchBot":["partial","*",18,0],"anthropic-ai":["partial","*",18,0],"PerplexityBot":["partial","*",18,0],"Perplexity-User":["partial","*",18,0],"Google-Extended":["partial","*",18,0],"Applebot-Extended":["partial","*",18,0],"CCBot":["partial","*",18,0],"Bytespider":["partial","*",18,0],"Amazonbot":["partial","*",18,0],"meta-externalagent":["partial","*",18,0],"cohere-ai":["partial","*",18,0],"Diffbot":["partial","*",18,0],"ContextFetchBot":["partial","*",18,0]},"llms":{"status":"missing","http":404,"bytes":33949},"home":{"status":"ok","http":200,"url":"https://realmoneyslots-mobile.com/","html_tokens":21548,"markdown_tokens":2896,"pruned_tokens":165}},
{"rank":6369,"domain":"rnd.de","origin":"https://rnd.de","robots":{"status":"ok","http":200,"url":"https://www.rnd.de/robots.txt","bytes":1163,"groups":8},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["blocked","named",1,0],"OAI-SearchBot":["partial","*",17,3],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",17,3],"Claude-SearchBot":["partial","*",17,3],"anthropic-ai":["partial","*",17,3],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["partial","*",17,3],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["partial","*",17,3],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["partial","*",17,3],"meta-externalagent":["partial","*",17,3],"cohere-ai":["partial","*",17,3],"Diffbot":["partial","*",17,3],"ContextFetchBot":["partial","*",17,3]},"llms":{"status":"missing","http":404,"bytes":97909},"home":{"status":"http_error","http":403,"url":"https://www.rnd.de/"}},
{"rank":6370,"domain":"clubic.com","origin":"https://clubic.com","robots":{"status":"ok","http":200,"url":"https://www.clubic.com/robots.txt","bytes":587,"groups":2},"bots":{"GPTBot":["partial","*",19,2],"ChatGPT-User":["partial","*",19,2],"OAI-SearchBot":["partial","*",19,2],"ClaudeBot":["partial","*",19,2],"Claude-User":["partial","*",19,2],"Claude-SearchBot":["partial","*",19,2],"anthropic-ai":["partial","*",19,2],"PerplexityBot":["partial","*",19,2],"Perplexity-User":["partial","*",19,2],"Google-Extended":["partial","*",19,2],"Applebot-Extended":["partial","*",19,2],"CCBot":["partial","*",19,2],"Bytespider":["partial","*",19,2],"Amazonbot":["partial","*",19,2],"meta-externalagent":["partial","*",19,2],"cohere-ai":["partial","*",19,2],"Diffbot":["partial","*",19,2],"ContextFetchBot":["partial","*",19,2]},"llms":{"status":"missing","http":404,"bytes":0},"home":{"status":"ok","http":200,"url":"https://www.clubic.com","html_tokens":323731,"markdown_tokens":253,"pruned_tokens":91}},
{"rank":6371,"domain":"wyo.gov","origin":"https://www.wyo.gov","robots":{"status":"missing","http":404,"url":"https://www.wyo.gov/robots.txt","bytes":142084,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":142119},"home":{"status":"ok","http":200,"url":"https://www.wyo.gov/","html_tokens":46302,"markdown_tokens":132,"pruned_tokens":132}},
{"rank":6372,"domain":"teacherspayteachers.com","origin":"https://teacherspayteachers.com","robots":{"status":"ok","http":200,"url":"https://www.teacherspayteachers.com/robots.txt","bytes":1188,"groups":7},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["partial","*",32,2],"OAI-SearchBot":["partial","*",32,2],"ClaudeBot":["partial","*",32,2],"Claude-User":["partial","*",32,2],"Claude-SearchBot":["partial","*",32,2],"anthropic-ai":["partial","*",32,2],"PerplexityBot":["partial","*",32,2],"Perplexity-User":["partial","*",32,2],"Google-Extended":["partial","*",32,2],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["partial","*",32,2],"Amazonbot":["partial","*",32,2],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["partial","*",32,2],"Diffbot":["partial","*",32,2],"ContextFetchBot":["partial","*",32,2]},"llms":{"status":"missing","http":404,"bytes":164331},"home":{"status":"ok","http":200,"url":"https://www.teacherspayteachers.com/","html_tokens":93882,"markdown_tokens":21,"pruned_tokens":21}},
{"rank":6373,"domain":"eol.cn","origin":"https://eol.cn","robots":{"status":"ok","http":200,"url":"https://www.eol.cn/robots.txt","bytes":120,"groups":2},"bots":{"GPTBot":["partial","*",4,0],"ChatGPT-User":["partial","*",4,0],"OAI-SearchBot":["partial","*",4,0],"ClaudeBot":["partial","*",4,0],"Claude-User":["partial","*",4,0],"Claude-SearchBot":["partial","*",4,0],"anthropic-ai":["partial","*",4,0],"PerplexityBot":["partial","*",4,0],"Perplexity-User":["partial","*",4,0],"Google-Extended":["partial","*",4,0],"Applebot-Extended":["partial","*",4,0],"CCBot":["partial","*",4,0],"Bytespider":["partial","*",4,0],"Amazonbot":["partial","*",4,0],"meta-externalagent":["partial","*",4,0],"cohere-ai":["partial","*",4,0],"Diffbot":["partial","*",4,0],"ContextFetchBot":["partial","*",4,0]},"llms":{"status":"other_text","http":200,"bytes":134},"home":{"status":"ok","http":200,"url":"https://www.eol.cn/","html_tokens":31630,"markdown_tokens":4,"pruned_tokens":4}},
{"rank":6374,"domain":"presstv.ir","origin":"https://presstv.ir","robots":{"status":"missing","http":404,"url":"https://www.presstv.co.uk/robots.txt","bytes":0,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":0},"home":{"status":"ok","http":200,"url":"https://www.presstv.co.uk/","html_tokens":20645,"markdown_tokens":276,"pruned_tokens":102}},
{"rank":6375,"domain":"biz.id","origin":"https://biz.id","robots":{"status":"unreachable","http":null,"url":"https://biz.id/robots.txt","bytes":0,"groups":0,"error":"cannot resolve biz.id"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":6376,"domain":"worldathletics.org","origin":"https://worldathletics.org","robots":{"status":"ok","http":200,"url":"https://worldathletics.org/robots.txt","bytes":106,"groups":1},"bots":{"GPTBot":["allowed","*",0,2],"ChatGPT-User":["allowed","*",0,2],"OAI-SearchBot":["allowed","*",0,2],"ClaudeBot":["allowed","*",0,2],"Claude-User":["allowed","*",0,2],"Claude-SearchBot":["allowed","*",0,2],"anthropic-ai":["allowed","*",0,2],"PerplexityBot":["allowed","*",0,2],"Perplexity-User":["allowed","*",0,2],"Google-Extended":["allowed","*",0,2],"Applebot-Extended":["allowed","*",0,2],"CCBot":["allowed","*",0,2],"Bytespider":["allowed","*",0,2],"Amazonbot":["allowed","*",0,2],"meta-externalagent":["allowed","*",0,2],"cohere-ai":["allowed","*",0,2],"Diffbot":["allowed","*",0,2],"ContextFetchBot":["allowed","*",0,2]},"llms":{"status":"missing","http":404,"bytes":19931},"home":{"status":"ok","http":200,"url":"https://worldathletics.org/","html_tokens":289412,"markdown_tokens":118,"pruned_tokens":54}},
{"rank":6377,"domain":"wellfound.com","origin":"https://wellfound.com","robots":{"status":"ok","http":200,"url":"https://wellfound.com/robots.txt","bytes":964,"groups":1},"bots":{"GPTBot":["partial","*",31,0],"ChatGPT-User":["partial","*",31,0],"OAI-SearchBot":["partial","*",31,0],"ClaudeBot":["partial","*",31,0],"Claude-User":["partial","*",31,0],"Claude-SearchBot":["partial","*",31,0],"anthropic-ai":["partial","*",31,0],"PerplexityBot":["partial","*",31,0],"Perplexity-User":["partial","*",31,0],"Google-Extended":["partial","*",31,0],"Applebot-Extended":["partial","*",31,0],"CCBot":["partial","*",31,0],"Bytespider":["partial","*",31,0],"Amazonbot":["partial","*",31,0],"meta-externalagent":["partial","*",31,0],"cohere-ai":["partial","*",31,0],"Diffbot":["partial","*",31,0],"ContextFetchBot":["partial","*",31,0]},"llms":{"status":"missing","http":404,"bytes":0},"home":{"status":"ok","http":200,"url":"https://wellfound.com/","html_tokens":31886,"markdown_tokens":692,"pruned_tokens":319}},
{"rank":6378,"domain":"cmail20.com","origin":"https://cmail20.com","robots":{"status":"ok","http":200,"url":"https://cmail20.com/robots.txt","bytes":26,"groups":1},"bots":{"GPTBot":["blocked","*",1,0],"ChatGPT-User":["blocked","*",1,0],"OAI-SearchBot":["blocked","*",1,0],"ClaudeBot":["blocked","*",1,0],"Claude-User":["blocked","*",1,0],"Claude-SearchBot":["blocked","*",1,0],"anthropic-ai":["blocked","*",1,0],"PerplexityBot":["blocked","*",1,0],"Perplexity-User":["blocked","*",1,0],"Google-Extended":["blocked","*",1,0],"Applebot-Extended":["blocked","*",1,0],"CCBot":["blocked","*",1,0],"Bytespider":["blocked","*",1,0],"Amazonbot":["blocked","*",1,0],"meta-externalagent":["blocked","*",1,0],"cohere-ai":["blocked","*",1,0],"Diffbot":["blocked","*",1,0],"ContextFetchBot":["blocked","*",1,0]},"llms":{"status":"disallowed"},"home":{"status":"disallowed"}},
{"rank":6379,"domain":"overleaf.com","origin":"https://overleaf.com","robots":{"status":"ok","http":200,"url":"https://www.overleaf.com/robots.txt","bytes":165,"groups":1},"bots":{"GPTBot":["partial","*",4,1],"ChatGPT-User":["partial","*",4,1],"OAI-SearchBot":["partial","*",4,1],"ClaudeBot":["partial","*",4,1],"Claude-User":["partial","*",4,1],"Claude-SearchBot":["partial","*",4,1],"anthropic-ai":["partial","*",4,1],"PerplexityBot":["partial","*",4,1],"Perplexity-User":["partial","*",4,1],"Google-Extended":["partial","*",4,1],"Applebot-Extended":["partial","*",4,1],"CCBot":["partial","*",4,1],"Bytespider":["partial","*",4,1],"Amazonbot":["partial","*",4,1],"meta-externalagent":["partial","*",4,1],"cohere-ai":["partial","*",4,1],"Diffbot":["partial","*",4,1],"ContextFetchBot":["partial","*",4,1]},"llms":{"status":"missing","http":404,"bytes":35416},"home":{"status":"ok","http":200,"url":"https://www.overleaf.com/","html_tokens":14727,"markdown_tokens":730,"pruned_tokens":246}},
{"rank":6380,"domain":"google.lk","origin":"https://google.lk","robots":{"status":"ok","http":200,"url":"https://www.google.lk/robots.txt","bytes":6624,"groups":4},"bots":{"GPTBot":["partial","*",169,64],"ChatGPT-User":["partial","*",169,64],"OAI-SearchBot":["partial","*",169,64],"ClaudeBot":["partial","*",169,64],"Claude-User":["partial","*",169,64],"Claude-SearchBot":["partial","*",169,64],"anthropic-ai":["partial","*",169,64],"PerplexityBot":["partial","*",169,64],"Perplexity-User":["partial","*",169,64],"Google-Extended":["partial","*",169,64],"Applebot-Extended":["partial","*",169,64],"CCBot":["partial","*",169,64],"Bytespider":["partial","*",169,64],"Amazonbot":["partial","*",169,64],"meta-externalagent":["partial","*",169,64],"cohere-ai":["partial","*",169,64],"Diffbot":["partial","*",169,64],"ContextFetchBot":["partial","*",169,64]},"llms":{"status":"missing","http":404,"bytes":1569},"home":{"status":"ok","http":200,"url":"https://www.google.com/","html_tokens":21364,"markdown_tokens":30,"pruned_tokens":30}},
{"rank":6381,"domain":"designspiration.com","origin":"https://designspiration.com","robots":{"status":"ok","http":200,"url":"https://www.designspiration.com/robots.txt","bytes":2073,"groups":38},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["blocked","named",1,0],"OAI-SearchBot":["partial","*",8,0],"ClaudeBot":["partial","*",8,0],"Claude-User":["partial","*",8,0],"Claude-SearchBot":["partial","*",8,0],"anthropic-ai":["partial","*",8,0],"PerplexityBot":["partial","*",8,0],"Perplexity-User":["partial","*",8,0],"Google-Extended":["partial","*",8,0],"Applebot-Extended":["partial","*",8,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["partial","*",8,0],"meta-externalagent":["partial","*",8,0],"cohere-ai":["partial","*",8,0],"Diffbot":["partial","*",8,0],"ContextFetchBot":["partial","*",8,0]},"llms":{"status":"missing","http":404,"bytes":21774},"home":{"status":"ok","http":200,"url":"https://www.designspiration.com/","html_tokens":5564,"markdown_tokens":40,"pruned_tokens":40}},
{"rank":6382,"domain":"eenews.net","origin":"https://eenews.net","robots":{"status":"ok","http":200,"url":"https://www.eenews.net/robots.txt","bytes":814,"groups":3},"bots":{"GPTBot":["partial","*",1,1],"ChatGPT-User":["partial","*",1,1],"OAI-SearchBot":["partial","*",1,1],"ClaudeBot":["partial","*",1,1],"Claude-User":["partial","*",1,1],"Claude-SearchBot":["partial","*",1,1],"anthropic-ai":["partial","*",1,1],"PerplexityBot":["partial","*",1,1],"Perplexity-User":["partial","*",1,1],"Google-Extended":["partial","named",1,10],"Applebot-Extended":["partial","*",1,1],"CCBot":["partial","*",1,1],"Bytespider":["partial","*",1,1],"Amazonbot":["partial","named",1,1],"meta-externalagent":["partial","*",1,1],"cohere-ai":["partial","*",1,1],"Diffbot":["partial","*",1,1],"ContextFetchBot":["partial","*",1,1]},"llms":{"status":"http_error","http":403,"bytes":5541},"home":{"status":"http_error","http":403,"url":"https://eenews.net/"}},
{"rank":6383,"domain":"autocar.co.uk","origin":"https://autocar.co.uk","robots":{"status":"html","http":202,"url":"https://www.autocar.co.uk/robots.txt","bytes":2415,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":6384,"domain":"urssaf.fr","origin":"https://urssaf.fr","robots":{"status":"unreachable","http":null,"url":"https://urssaf.fr/robots.txt","bytes":0,"groups":0,"error":"ReadError"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":6385,"domain":"lamborghini.com","origin":"https://lamborghini.com","robots":{"status":"ok","http":200,"url":"https://www.lamborghini.com/robots.txt","bytes":727,"groups":1},"bots":{"GPTBot":["partial","*",1,1],"ChatGPT-User":["partial","*",1,1],"OAI-SearchBot":["partial","*",1,1],"ClaudeBot":["partial","*",1,1],"Claude-User":["partial","*",1,1],"Claude-SearchBot":["partial","*",1,1],"anthropic-ai":["partial","*",1,1],"PerplexityBot":["partial","*",1,1],"Perplexity-User":["partial","*",1,1],"Google-Extended":["partial","*",1,1],"Applebot-Extended":["partial","*",1,1],"CCBot":["partial","*",1,1],"Bytespider":["partial","*",1,1],"Amazonbot":["partial","*",1,1],"meta-externalagent":["partial","*",1,1],"cohere-ai":["partial","*",1,1],"Diffbot":["partial","*",1,1],"ContextFetchBot":["partial","*",1,1]},"llms":{"status":"missing","http":404,"bytes":222208},"home":{"status":"ok","http":200,"url":"https://www.lamborghini.com/en-en","html_tokens":112806,"markdown_tokens":47,"pruned_tokens":47}},
{"rank":6386,"domain":"refworld.org","origin":"https://refworld.org","robots":{"status":"refused","http":403,"url":"https://www.refworld.org/robots.txt","bytes":5533,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":6387,"domain":"shazam.com","origin":"https://shazam.com","robots":{"status":"ok","http":200,"url":"https://www.shazam.com/robots.txt","bytes":1178,"groups":6},"bots":{"GPTBot":["partial","*",5,1],"ChatGPT-User":["partial","*",5,1],"OAI-SearchBot":["partial","*",5,1],"ClaudeBot":["partial","*",5,1],"Claude-User":["partial","*",5,1],"Claude-SearchBot":["partial","*",5,1],"anthropic-ai":["partial","*",5,1],"PerplexityBot":["partial","*",5,1],"Perplexity-User":["partial","*",5,1],"Google-Extended":["partial","*",5,1],"Applebot-Extended":["partial","*",5,1],"CCBot":["partial","*",5,1],"Bytespider":["partial","*",5,1],"Amazonbot":["partial","*",5,1],"meta-externalagent":["partial","*",5,1],"cohere-ai":["partial","*",5,1],"Diffbot":["partial","*",5,1],"ContextFetchBot":["partial","*",5,1]},"llms":{"status":"present","http":200,"bytes":6241},"home":{"status":"ok","http":200,"url":"https://www.shazam.com/","html_tokens":385007,"markdown_tokens":22,"pruned_tokens":22}},
{"rank":6388,"domain":"imarcgroup.com","origin":"https://imarcgroup.com","robots":{"status":"ok","http":200,"url":"https://www.imarcgroup.com/robots.txt","bytes":1549,"groups":1},"bots":{"GPTBot":["partial","*",19,0],"ChatGPT-User":["partial","*",19,0],"OAI-SearchBot":["partial","*",19,0],"ClaudeBot":["partial","*",19,0],"Claude-User":["partial","*",19,0],"Claude-SearchBot":["partial","*",19,0],"anthropic-ai":["partial","*",19,0],"PerplexityBot":["partial","*",19,0],"Perplexity-User":["partial","*",19,0],"Google-Extended":["partial","*",19,0],"Applebot-Extended":["partial","*",19,0],"CCBot":["partial","*",19,0],"Bytespider":["partial","*",19,0],"Amazonbot":["partial","*",19,0],"meta-externalagent":["partial","*",19,0],"cohere-ai":["partial","*",19,0],"Diffbot":["partial","*",19,0],"ContextFetchBot":["partial","*",19,0]},"llms":{"status":"present","http":200,"bytes":23877},"home":{"status":"ok","http":200,"url":"https://www.imarcgroup.com/","html_tokens":66046,"markdown_tokens":462,"pruned_tokens":440}},
{"rank":6389,"domain":"food52.com","origin":"https://food52.com","robots":{"status":"refused","http":429,"url":"https://food52.com/robots.txt","bytes":34219,"groups":0,"error":"HTTP 429"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":6390,"domain":"sandiegozoo.org","origin":"https://sandiegozoo.org","robots":{"status":"ok","http":200,"url":"https://sandiegozoo.org/robots.txt","bytes":351,"groups":1},"bots":{"GPTBot":["partial","*",15,0],"ChatGPT-User":["partial","*",15,0],"OAI-SearchBot":["partial","*",15,0],"ClaudeBot":["partial","*",15,0],"Claude-User":["partial","*",15,0],"Claude-SearchBot":["partial","*",15,0],"anthropic-ai":["partial","*",15,0],"PerplexityBot":["partial","*",15,0],"Perplexity-User":["partial","*",15,0],"Google-Extended":["partial","*",15,0],"Applebot-Extended":["partial","*",15,0],"CCBot":["partial","*",15,0],"Bytespider":["partial","*",15,0],"Amazonbot":["partial","*",15,0],"meta-externalagent":["partial","*",15,0],"cohere-ai":["partial","*",15,0],"Diffbot":["partial","*",15,0],"ContextFetchBot":["partial","*",15,0]},"llms":{"status":"missing","http":404,"bytes":397},"home":{"status":"disallowed","error":"robots.txt of sandiegozoowildlifealliance.org not readable (refused)"}},
{"rank":6391,"domain":"moovitapp.com","origin":"https://moovitapp.com","robots":{"status":"html","http":202,"url":"https://moovitapp.com/robots.txt","bytes":1995,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":6392,"domain":"lacity.gov","origin":"https://lacity.gov","robots":{"status":"refused","http":403,"url":"https://lacity.gov/robots.txt","bytes":374,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":6393,"domain":"inf.br","origin":"https://inf.br","robots":{"status":"unreachable","http":null,"url":"https://inf.br/robots.txt","bytes":0,"groups":0,"error":"cannot resolve inf.br"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":6394,"domain":"linksys.com","origin":"https://linksys.com","robots":{"status":"html","http":200,"url":"https://www.linksys.com/robots.txt","bytes":527,"groups":1},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":6395,"domain":"cmake.org","origin":"https://cmake.org","robots":{"status":"ok","http":200,"url":"https://cmake.org/robots.txt","bytes":1621,"groups":9},"bots":{"GPTBot":["partial","*",34,0],"ChatGPT-User":["partial","*",34,0],"OAI-SearchBot":["partial","*",34,0],"ClaudeBot":["partial","*",34,0],"Claude-User":["partial","*",34,0],"Claude-SearchBot":["partial","*",34,0],"anthropic-ai":["partial","*",34,0],"PerplexityBot":["partial","*",34,0],"Perplexity-User":["partial","*",34,0],"Google-Extended":["partial","*",34,0],"Applebot-Extended":["partial","*",34,0],"CCBot":["partial","*",34,0],"Bytespider":["partial","*",34,0],"Amazonbot":["partial","*",34,0],"meta-externalagent":["partial","*",34,0],"cohere-ai":["partial","*",34,0],"Diffbot":["partial","*",34,0],"ContextFetchBot":["partial","*",34,0]},"llms":{"status":"crawl_delay","error":"Crawl-delay 240 s"},"home":{"status":"crawl_delay","error":"Crawl-delay 240 s"}},
{"rank":6396,"domain":"gmane.org","origin":"https://gmane.org","robots":{"status":"unreachable","http":null,"url":"https://gmane.org/robots.txt","bytes":0,"groups":0,"error":"cannot resolve gmane.org"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":6397,"domain":"framagit.org","origin":"https://framagit.org","robots":{"status":"ok","http":200,"url":"https://framagit.org/robots.txt","bytes":3355,"groups":5},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["partial","*",88,3],"OAI-SearchBot":["partial","*",88,3],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",88,3],"Claude-SearchBot":["partial","*",88,3],"anthropic-ai":["partial","*",88,3],"PerplexityBot":["partial","*",88,3],"Perplexity-User":["partial","*",88,3],"Google-Extended":["partial","*",88,3],"Applebot-Extended":["partial","*",88,3],"CCBot":["partial","*",88,3],"Bytespider":["partial","*",88,3],"Amazonbot":["partial","*",88,3],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["partial","*",88,3],"Diffbot":["partial","*",88,3],"ContextFetchBot":["partial","*",88,3]},"llms":{"status":"html","http":200,"bytes":4729},"home":{"status":"ok","http":200,"url":"https://framagit.org/public/projects","html_tokens":6025,"markdown_tokens":4,"pruned_tokens":4}},
{"rank":6398,"domain":"unal.edu.co","origin":"http://unal.edu.co","robots":{"status":"refused","http":403,"url":"http://unal.edu.co/robots.txt","bytes":118,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":6399,"domain":"berlios.de","origin":"https://berlios.de","robots":{"status":"ok","http":200,"url":"https://berlios.de/robots.txt","bytes":67,"groups":1},"bots":{"GPTBot":["partial","*",1,1],"ChatGPT-User":["partial","*",1,1],"OAI-SearchBot":["partial","*",1,1],"ClaudeBot":["partial","*",1,1],"Claude-User":["partial","*",1,1],"Claude-SearchBot":["partial","*",1,1],"anthropic-ai":["partial","*",1,1],"PerplexityBot":["partial","*",1,1],"Perplexity-User":["partial","*",1,1],"Google-Extended":["partial","*",1,1],"Applebot-Extended":["partial","*",1,1],"CCBot":["partial","*",1,1],"Bytespider":["partial","*",1,1],"Amazonbot":["partial","*",1,1],"meta-externalagent":["partial","*",1,1],"cohere-ai":["partial","*",1,1],"Diffbot":["partial","*",1,1],"ContextFetchBot":["partial","*",1,1]},"llms":{"status":"missing","http":404,"bytes":39744},"home":{"status":"ok","http":200,"url":"https://berlios.de/","html_tokens":13942,"markdown_tokens":828,"pruned_tokens":105}},
{"rank":6400,"domain":"naic.org","origin":"https://naic.org","robots":{"status":"ok","http":200,"url":"https://naic.org/robots.txt","bytes":22534,"groups":1},"bots":{"GPTBot":["allowed","*",0,1],"ChatGPT-User":["allowed","*",0,1],"OAI-SearchBot":["allowed","*",0,1],"ClaudeBot":["allowed","*",0,1],"Claude-User":["allowed","*",0,1],"Claude-SearchBot":["allowed","*",0,1],"anthropic-ai":["allowed","*",0,1],"PerplexityBot":["allowed","*",0,1],"Perplexity-User":["allowed","*",0,1],"Google-Extended":["allowed","*",0,1],"Applebot-Extended":["allowed","*",0,1],"CCBot":["allowed","*",0,1],"Bytespider":["allowed","*",0,1],"Amazonbot":["allowed","*",0,1],"meta-externalagent":["allowed","*",0,1],"cohere-ai":["allowed","*",0,1],"Diffbot":["allowed","*",0,1],"ContextFetchBot":["allowed","*",0,1]},"llms":{"status":"disallowed","error":"robots.txt of content.naic.org not readable (refused)"},"home":{"status":"disallowed","error":"robots.txt of content.naic.org not readable (refused)"}},
{"rank":6401,"domain":"metlife.com","origin":"https://metlife.com","robots":{"status":"ok","http":200,"url":"https://www.metlife.com/robots.txt","bytes":2036,"groups":4},"bots":{"GPTBot":["partial","*",10,0],"ChatGPT-User":["partial","*",10,0],"OAI-SearchBot":["partial","*",10,0],"ClaudeBot":["partial","*",10,0],"Claude-User":["partial","*",10,0],"Claude-SearchBot":["partial","*",10,0],"anthropic-ai":["partial","*",10,0],"PerplexityBot":["partial","*",10,0],"Perplexity-User":["partial","*",10,0],"Google-Extended":["partial","*",10,0],"Applebot-Extended":["partial","*",10,0],"CCBot":["partial","*",10,0],"Bytespider":["partial","*",10,0],"Amazonbot":["partial","*",10,0],"meta-externalagent":["partial","*",10,0],"cohere-ai":["partial","*",10,0],"Diffbot":["partial","*",10,0],"ContextFetchBot":["partial","*",10,0]},"llms":{"status":"present","http":200,"bytes":2197},"home":{"status":"ok","http":200,"url":"https://www.metlife.com/","html_tokens":78942,"markdown_tokens":3946,"pruned_tokens":1278}},
{"rank":6402,"domain":"ray-ban.com","origin":"https://ray-ban.com","robots":{"status":"refused","http":403,"url":"https://ray-ban.com/robots.txt","bytes":377,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":6403,"domain":"themehorse.com","origin":"https://themehorse.com","robots":{"status":"ok","http":200,"url":"https://themehorse.com/robots.txt","bytes":298,"groups":2},"bots":{"GPTBot":["partial","*",7,0],"ChatGPT-User":["partial","*",7,0],"OAI-SearchBot":["partial","*",7,0],"ClaudeBot":["partial","*",7,0],"Claude-User":["partial","*",7,0],"Claude-SearchBot":["partial","*",7,0],"anthropic-ai":["partial","*",7,0],"PerplexityBot":["partial","*",7,0],"Perplexity-User":["partial","*",7,0],"Google-Extended":["partial","*",7,0],"Applebot-Extended":["partial","*",7,0],"CCBot":["partial","*",7,0],"Bytespider":["partial","*",7,0],"Amazonbot":["partial","*",7,0],"meta-externalagent":["partial","*",7,0],"cohere-ai":["partial","*",7,0],"Diffbot":["partial","*",7,0],"ContextFetchBot":["partial","*",7,0]},"llms":{"status":"missing","http":404,"bytes":84437},"home":{"status":"ok","http":200,"url":"https://www.themehorse.com/","html_tokens":13923,"markdown_tokens":301,"pruned_tokens":77}},
{"rank":6404,"domain":"click2houston.com","origin":"https://click2houston.com","robots":{"status":"ok","http":200,"url":"https://www.click2houston.com/robots.txt","bytes":1184,"groups":9},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["partial","*",5,0],"OAI-SearchBot":["partial","*",5,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",5,0],"Claude-SearchBot":["partial","*",5,0],"anthropic-ai":["partial","*",5,0],"PerplexityBot":["partial","*",5,0],"Perplexity-User":["partial","*",5,0],"Google-Extended":["partial","*",5,0],"Applebot-Extended":["partial","*",5,0],"CCBot":["partial","*",5,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["partial","*",5,0],"cohere-ai":["partial","*",5,0],"Diffbot":["partial","*",5,0],"ContextFetchBot":["partial","*",5,0]},"llms":{"status":"missing","http":404,"bytes":159835},"home":{"status":"ok","http":200,"url":"https://www.click2houston.com/","html_tokens":215091,"markdown_tokens":355,"pruned_tokens":118}},
{"rank":6405,"domain":"taiwannews.com.tw","origin":"https://taiwannews.com.tw","robots":{"status":"ok","http":200,"url":"https://www.taiwannews.com.tw/robots.txt","bytes":190,"groups":1},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"missing","http":404,"bytes":8054},"home":{"status":"ok","http":200,"url":"https://www.taiwannews.com.tw/","html_tokens":66411,"markdown_tokens":346,"pruned_tokens":154}},
{"rank":6406,"domain":"kissmetrics.com","origin":"https://kissmetrics.com","robots":{"status":"refused","http":403,"url":"https://neilpatel.com","bytes":5478,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":6407,"domain":"99bitcoins.com","origin":"https://99bitcoins.com","robots":{"status":"ok","http":200,"url":"https://99bitcoins.com/robots.txt","bytes":1183,"groups":1},"bots":{"GPTBot":["partial","*",6,2],"ChatGPT-User":["partial","*",6,2],"OAI-SearchBot":["partial","*",6,2],"ClaudeBot":["partial","*",6,2],"Claude-User":["partial","*",6,2],"Claude-SearchBot":["partial","*",6,2],"anthropic-ai":["partial","*",6,2],"PerplexityBot":["partial","*",6,2],"Perplexity-User":["partial","*",6,2],"Google-Extended":["partial","*",6,2],"Applebot-Extended":["partial","*",6,2],"CCBot":["partial","*",6,2],"Bytespider":["partial","*",6,2],"Amazonbot":["partial","*",6,2],"meta-externalagent":["partial","*",6,2],"cohere-ai":["partial","*",6,2],"Diffbot":["partial","*",6,2],"ContextFetchBot":["partial","*",6,2]},"llms":{"status":"missing","http":404,"bytes":116796},"home":{"status":"http_error","http":403,"url":"https://99bitcoins.com/"}},
{"rank":6408,"domain":"wisegeek.com","origin":"https://wisegeek.com","robots":{"status":"ok","http":200,"url":"https://www.wisegeek.com/robots.txt","bytes":340,"groups":2},"bots":{"GPTBot":["partial","*",9,0],"ChatGPT-User":["partial","*",9,0],"OAI-SearchBot":["partial","*",9,0],"ClaudeBot":["partial","*",9,0],"Claude-User":["partial","*",9,0],"Claude-SearchBot":["partial","*",9,0],"anthropic-ai":["partial","*",9,0],"PerplexityBot":["partial","*",9,0],"Perplexity-User":["partial","*",9,0],"Google-Extended":["partial","*",9,0],"Applebot-Extended":["partial","*",9,0],"CCBot":["partial","*",9,0],"Bytespider":["partial","*",9,0],"Amazonbot":["partial","*",9,0],"meta-externalagent":["partial","*",9,0],"cohere-ai":["partial","*",9,0],"Diffbot":["partial","*",9,0],"ContextFetchBot":["partial","*",9,0]},"llms":{"status":"missing","http":404,"bytes":84092},"home":{"status":"ok","http":200,"url":"https://www.wisegeek.com/","html_tokens":17382,"markdown_tokens":414,"pruned_tokens":29}},
{"rank":6409,"domain":"lazada.sg","origin":"https://lazada.sg","robots":{"status":"ok","http":200,"url":"https://www.lazada.sg/robots.txt","bytes":221,"groups":4},"bots":{"GPTBot":["allowed","named",0,1],"ChatGPT-User":["allowed","named",0,1],"OAI-SearchBot":["allowed","named",0,1],"ClaudeBot":["partial","*",4,0],"Claude-User":["partial","*",4,0],"Claude-SearchBot":["partial","*",4,0],"anthropic-ai":["partial","*",4,0],"PerplexityBot":["partial","*",4,0],"Perplexity-User":["partial","*",4,0],"Google-Extended":["partial","*",4,0],"Applebot-Extended":["partial","*",4,0],"CCBot":["partial","*",4,0],"Bytespider":["partial","*",4,0],"Amazonbot":["partial","*",4,0],"meta-externalagent":["partial","*",4,0],"cohere-ai":["partial","*",4,0],"Diffbot":["partial","*",4,0],"ContextFetchBot":["partial","*",4,0]},"llms":{"status":"present","http":200,"bytes":3122},"home":{"status":"ok","http":200,"url":"https://www.lazada.sg/","html_tokens":117358,"markdown_tokens":826,"pruned_tokens":102}},
{"rank":6410,"domain":"mheducation.com","origin":"https://mheducation.com","robots":{"status":"ok","http":200,"url":"https://www.mheducation.com/robots.txt","bytes":258,"groups":2},"bots":{"GPTBot":["partial","*",2,0],"ChatGPT-User":["partial","*",2,0],"OAI-SearchBot":["partial","*",2,0],"ClaudeBot":["partial","*",2,0],"Claude-User":["partial","*",2,0],"Claude-SearchBot":["partial","*",2,0],"anthropic-ai":["partial","*",2,0],"PerplexityBot":["partial","*",2,0],"Perplexity-User":["partial","*",2,0],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["partial","*",2,0],"CCBot":["partial","*",2,0],"Bytespider":["partial","*",2,0],"Amazonbot":["partial","*",2,0],"meta-externalagent":["partial","*",2,0],"cohere-ai":["partial","*",2,0],"Diffbot":["partial","*",2,0],"ContextFetchBot":["partial","*",2,0]},"llms":{"status":"missing","http":404,"bytes":223534},"home":{"status":"ok","http":200,"url":"https://www.mheducation.com/","html_tokens":98526,"markdown_tokens":427,"pruned_tokens":150}},
{"rank":6411,"domain":"dutchnews.nl","origin":"https://dutchnews.nl","robots":{"status":"ok","http":200,"url":"https://dutchnews.nl/robots.txt","bytes":2007,"groups":2},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["partial","*",72,1],"OAI-SearchBot":["partial","*",72,1],"ClaudeBot":["partial","*",72,1],"Claude-User":["partial","*",72,1],"Claude-SearchBot":["partial","*",72,1],"anthropic-ai":["partial","*",72,1],"PerplexityBot":["partial","*",72,1],"Perplexity-User":["partial","*",72,1],"Google-Extended":["partial","*",72,1],"Applebot-Extended":["partial","*",72,1],"CCBot":["partial","*",72,1],"Bytespider":["partial","*",72,1],"Amazonbot":["partial","*",72,1],"meta-externalagent":["partial","*",72,1],"cohere-ai":["partial","*",72,1],"Diffbot":["partial","*",72,1],"ContextFetchBot":["partial","*",72,1]},"llms":{"status":"timeout","error":"timed out after 10 s"},"home":{"status":"ok","http":200,"url":"https://www.dutchnews.nl/","html_tokens":49301,"markdown_tokens":94,"pruned_tokens":94}},
{"rank":6412,"domain":"jointcommission.org","origin":"https://jointcommission.org","robots":{"status":"refused","http":403,"url":"https://www.jointcommission.org//robots.txt","bytes":207977,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":6413,"domain":"ulb.ac.be","origin":"https://ulb.ac.be","robots":{"status":"ok","http":200,"url":"https://www.ulb.be/robots.txt","bytes":198,"groups":1},"bots":{"GPTBot":["partial","*",6,0],"ChatGPT-User":["partial","*",6,0],"OAI-SearchBot":["partial","*",6,0],"ClaudeBot":["partial","*",6,0],"Claude-User":["partial","*",6,0],"Claude-SearchBot":["partial","*",6,0],"anthropic-ai":["partial","*",6,0],"PerplexityBot":["partial","*",6,0],"Perplexity-User":["partial","*",6,0],"Google-Extended":["partial","*",6,0],"Applebot-Extended":["partial","*",6,0],"CCBot":["partial","*",6,0],"Bytespider":["partial","*",6,0],"Amazonbot":["partial","*",6,0],"meta-externalagent":["partial","*",6,0],"cohere-ai":["partial","*",6,0],"Diffbot":["partial","*",6,0],"ContextFetchBot":["partial","*",6,0]},"llms":{"status":"missing","http":404,"bytes":815},"home":{"status":"ok","http":200,"url":"https://www.ulb.be/","html_tokens":95453,"markdown_tokens":1176,"pruned_tokens":119}},
{"rank":6414,"domain":"geek.com","origin":"https://geek.com","robots":{"status":"ok","http":200,"url":"https://geek.com/robots.txt","bytes":5164,"groups":1},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["blocked","named",1,0],"OAI-SearchBot":["blocked","named",1,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["blocked","named",1,0],"Claude-SearchBot":["blocked","named",1,0],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["blocked","named",1,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"http_error","http":403,"bytes":263},"home":{"status":"ok","http":200,"url":"https://geek.com/","html_tokens":23635,"markdown_tokens":3535,"pruned_tokens":101}},
{"rank":6415,"domain":"cisecurity.org","origin":"https://cisecurity.org","robots":{"status":"ok","http":200,"url":"https://www.cisecurity.org/robots.txt","bytes":94,"groups":1},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"missing","http":404,"bytes":5848},"home":{"status":"ok","http":200,"url":"https://www.cisecurity.org/","html_tokens":71322,"markdown_tokens":151,"pruned_tokens":151}},
{"rank":6416,"domain":"caf.fr","origin":"https://caf.fr","robots":{"status":"ok","http":200,"url":"https://caf.fr/robots.txt","bytes":6592,"groups":1},"bots":{"GPTBot":["partial","*",69,18],"ChatGPT-User":["partial","*",69,18],"OAI-SearchBot":["partial","*",69,18],"ClaudeBot":["partial","*",69,18],"Claude-User":["partial","*",69,18],"Claude-SearchBot":["partial","*",69,18],"anthropic-ai":["partial","*",69,18],"PerplexityBot":["partial","*",69,18],"Perplexity-User":["partial","*",69,18],"Google-Extended":["partial","*",69,18],"Applebot-Extended":["partial","*",69,18],"CCBot":["partial","*",69,18],"Bytespider":["partial","*",69,18],"Amazonbot":["partial","*",69,18],"meta-externalagent":["partial","*",69,18],"cohere-ai":["partial","*",69,18],"Diffbot":["partial","*",69,18],"ContextFetchBot":["partial","*",69,18]},"llms":{"status":"missing","http":404,"bytes":601},"home":{"status":"ok","http":200,"url":"https://caf.fr/","html_tokens":31321,"markdown_tokens":891,"pruned_tokens":163}},
{"rank":6417,"domain":"ard.de","origin":"https://ard.de","robots":{"status":"ok","http":200,"url":"https://www.ard.de/robots.txt","bytes":2182,"groups":21},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["allowed","*",0,1],"OAI-SearchBot":["allowed","*",0,1],"ClaudeBot":["blocked","named",1,0],"Claude-User":["allowed","*",0,1],"Claude-SearchBot":["allowed","*",0,1],"anthropic-ai":["allowed","*",0,1],"PerplexityBot":["allowed","*",0,1],"Perplexity-User":["allowed","*",0,1],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["allowed","*",0,1],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["allowed","*",0,1],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["allowed","*",0,1]},"llms":{"status":"missing","http":404,"bytes":82963},"home":{"status":"ok","http":200,"url":"https://www.ard.de/","html_tokens":21903,"markdown_tokens":52,"pruned_tokens":28}},
{"rank":6418,"domain":"ableton.com","origin":"https://ableton.com","robots":{"status":"ok","http":200,"url":"https://www.ableton.com/robots.txt","bytes":222,"groups":2},"bots":{"GPTBot":["partial","*",3,0],"ChatGPT-User":["partial","*",3,0],"OAI-SearchBot":["partial","*",3,0],"ClaudeBot":["partial","*",3,0],"Claude-User":["partial","*",3,0],"Claude-SearchBot":["partial","*",3,0],"anthropic-ai":["partial","*",3,0],"PerplexityBot":["partial","*",3,0],"Perplexity-User":["partial","*",3,0],"Google-Extended":["partial","*",3,0],"Applebot-Extended":["partial","*",3,0],"CCBot":["partial","*",3,0],"Bytespider":["partial","*",3,0],"Amazonbot":["partial","*",3,0],"meta-externalagent":["partial","*",3,0],"cohere-ai":["partial","*",3,0],"Diffbot":["partial","*",3,0],"ContextFetchBot":["partial","*",3,0]},"llms":{"status":"missing","http":404,"bytes":104850},"home":{"status":"ok","http":200,"url":"https://www.ableton.com/","html_tokens":31108,"markdown_tokens":15,"pruned_tokens":15}},
{"rank":6419,"domain":"mapleprimes.com","origin":"https://mapleprimes.com","robots":{"status":"ok","http":200,"url":"https://mapleprimes.com/robots.txt","bytes":558,"groups":1},"bots":{"GPTBot":["partial","*",20,0],"ChatGPT-User":["partial","*",20,0],"OAI-SearchBot":["partial","*",20,0],"ClaudeBot":["partial","*",20,0],"Claude-User":["partial","*",20,0],"Claude-SearchBot":["partial","*",20,0],"anthropic-ai":["partial","*",20,0],"PerplexityBot":["partial","*",20,0],"Perplexity-User":["partial","*",20,0],"Google-Extended":["partial","*",20,0],"Applebot-Extended":["partial","*",20,0],"CCBot":["partial","*",20,0],"Bytespider":["partial","*",20,0],"Amazonbot":["partial","*",20,0],"meta-externalagent":["partial","*",20,0],"cohere-ai":["partial","*",20,0],"Diffbot":["partial","*",20,0],"ContextFetchBot":["partial","*",20,0]},"llms":{"status":"missing","http":404,"bytes":1245},"home":{"status":"ok","http":200,"url":"https://mapleprimes.com/","html_tokens":21425,"markdown_tokens":200,"pruned_tokens":42}},
{"rank":6420,"domain":"unido.org","origin":"https://unido.org","robots":{"status":"refused","http":403,"url":"https://unido.org/robots.txt","bytes":5525,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":6421,"domain":"vangoghmuseum.nl","origin":"https://vangoghmuseum.nl","robots":{"status":"ok","http":200,"url":"https://www.vangoghmuseum.nl/robots.txt","bytes":25,"groups":1},"bots":{"GPTBot":["allowed","*",0,1],"ChatGPT-User":["allowed","*",0,1],"OAI-SearchBot":["allowed","*",0,1],"ClaudeBot":["allowed","*",0,1],"Claude-User":["allowed","*",0,1],"Claude-SearchBot":["allowed","*",0,1],"anthropic-ai":["allowed","*",0,1],"PerplexityBot":["allowed","*",0,1],"Perplexity-User":["allowed","*",0,1],"Google-Extended":["allowed","*",0,1],"Applebot-Extended":["allowed","*",0,1],"CCBot":["allowed","*",0,1],"Bytespider":["allowed","*",0,1],"Amazonbot":["allowed","*",0,1],"meta-externalagent":["allowed","*",0,1],"cohere-ai":["allowed","*",0,1],"Diffbot":["allowed","*",0,1],"ContextFetchBot":["allowed","*",0,1]},"llms":{"status":"missing","http":404,"bytes":65395},"home":{"status":"ok","http":200,"url":"https://www.vangoghmuseum.nl/nl","html_tokens":28534,"markdown_tokens":167,"pruned_tokens":82}},
{"rank":6422,"domain":"qantas.com","origin":"https://qantas.com","robots":{"status":"timeout","http":null,"url":"https://qantas.com/robots.txt","bytes":0,"groups":0,"error":"timed out after 10 s"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":6423,"domain":"woodmarttheme.com","origin":"https://woodmarttheme.com","robots":{"status":"ok","http":200,"url":"https://woodmarttheme.com/robots.txt","bytes":121,"groups":1},"bots":{"GPTBot":["allowed","*",0,1],"ChatGPT-User":["allowed","*",0,1],"OAI-SearchBot":["allowed","*",0,1],"ClaudeBot":["allowed","*",0,1],"Claude-User":["allowed","*",0,1],"Claude-SearchBot":["allowed","*",0,1],"anthropic-ai":["allowed","*",0,1],"PerplexityBot":["allowed","*",0,1],"Perplexity-User":["allowed","*",0,1],"Google-Extended":["allowed","*",0,1],"Applebot-Extended":["allowed","*",0,1],"CCBot":["allowed","*",0,1],"Bytespider":["allowed","*",0,1],"Amazonbot":["allowed","*",0,1],"meta-externalagent":["allowed","*",0,1],"cohere-ai":["allowed","*",0,1],"Diffbot":["allowed","*",0,1],"ContextFetchBot":["allowed","*",0,1]},"llms":{"status":"present","http":200,"bytes":33018},"home":{"status":"ok","http":200,"url":"https://woodmarttheme.com/","html_tokens":23884,"markdown_tokens":2469,"pruned_tokens":45}},
{"rank":6424,"domain":"wjla.com","origin":"https://wjla.com","robots":{"status":"ok","http":200,"url":"https://wjla.com/robots.txt","bytes":565,"groups":12},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["blocked","named",1,0],"OAI-SearchBot":["partial","*",1,1],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",1,1],"Claude-SearchBot":["partial","*",1,1],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["partial","*",1,1],"Perplexity-User":["partial","*",1,1],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["partial","*",1,1],"Amazonbot":["partial","*",1,1],"meta-externalagent":["partial","*",1,1],"cohere-ai":["blocked","named",1,0],"Diffbot":["partial","*",1,1],"ContextFetchBot":["partial","*",1,1]},"llms":{"status":"http_error","http":400,"bytes":45},"home":{"status":"ok","http":200,"url":"https://wjla.com/","html_tokens":124938,"markdown_tokens":62,"pruned_tokens":62}},
{"rank":6425,"domain":"nieuwsblad.be","origin":"https://nieuwsblad.be","robots":{"status":"ok","http":200,"url":"https://www.nieuwsblad.be/robots.txt","bytes":2700,"groups":40},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["blocked","named",1,0],"OAI-SearchBot":["blocked","named",1,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["blocked","named",1,0],"Claude-SearchBot":["blocked","named",1,0],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["blocked","named",1,0],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["partial","*",1,1],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",1,1]},"llms":{"status":"http_error","http":403,"bytes":4546},"home":{"status":"http_error","http":403,"url":"https://www.nieuwsblad.be/"}},
{"rank":6426,"domain":"dolby.com","origin":"https://dolby.com","robots":{"status":"ok","http":200,"url":"https://www.dolby.com/robots.txt","bytes":699,"groups":3},"bots":{"GPTBot":["allowed","named",0,1],"ChatGPT-User":["allowed","named",0,1],"OAI-SearchBot":["allowed","named",0,1],"ClaudeBot":["allowed","named",0,1],"Claude-User":["partial","*",2,1],"Claude-SearchBot":["partial","*",2,1],"anthropic-ai":["allowed","named",0,1],"PerplexityBot":["allowed","named",0,1],"Perplexity-User":["partial","*",2,1],"Google-Extended":["partial","*",2,1],"Applebot-Extended":["partial","*",2,1],"CCBot":["allowed","named",0,1],"Bytespider":["allowed","named",0,1],"Amazonbot":["allowed","named",0,1],"meta-externalagent":["partial","*",2,1],"cohere-ai":["partial","*",2,1],"Diffbot":["partial","*",2,1],"ContextFetchBot":["partial","*",2,1]},"llms":{"status":"html","http":200,"bytes":114847},"home":{"status":"ok","http":200,"url":"https://www.dolby.com/","html_tokens":30577,"markdown_tokens":116,"pruned_tokens":39}},
{"rank":6427,"domain":"newcastle.edu.au","origin":"https://newcastle.edu.au","robots":{"status":"refused","http":403,"url":"https://www.newcastle.edu.au/robots.txt","bytes":5558,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":6428,"domain":"openstack.org","origin":"https://openstack.org","robots":{"status":"ok","http":200,"url":"https://www.openstack.org/robots.txt","bytes":145,"groups":2},"bots":{"GPTBot":["allowed","*",0,1],"ChatGPT-User":["allowed","*",0,1],"OAI-SearchBot":["allowed","*",0,1],"ClaudeBot":["allowed","*",0,1],"Claude-User":["allowed","*",0,1],"Claude-SearchBot":["allowed","*",0,1],"anthropic-ai":["allowed","*",0,1],"PerplexityBot":["allowed","*",0,1],"Perplexity-User":["allowed","*",0,1],"Google-Extended":["allowed","*",0,1],"Applebot-Extended":["allowed","*",0,1],"CCBot":["allowed","*",0,1],"Bytespider":["allowed","*",0,1],"Amazonbot":["allowed","*",0,1],"meta-externalagent":["allowed","*",0,1],"cohere-ai":["allowed","*",0,1],"Diffbot":["allowed","*",0,1],"ContextFetchBot":["allowed","*",0,1]},"llms":{"status":"missing","http":404,"bytes":214974},"home":{"status":"ok","http":200,"url":"https://www.openstack.org/","html_tokens":62517,"markdown_tokens":173,"pruned_tokens":173}},
{"rank":6429,"domain":"honda.co.jp","origin":"https://www.honda.co.jp","robots":{"status":"ok","http":200,"url":"https://www.honda.co.jp/robots.txt","bytes":95,"groups":1},"bots":{"GPTBot":["partial","*",1,0],"ChatGPT-User":["partial","*",1,0],"OAI-SearchBot":["partial","*",1,0],"ClaudeBot":["partial","*",1,0],"Claude-User":["partial","*",1,0],"Claude-SearchBot":["partial","*",1,0],"anthropic-ai":["partial","*",1,0],"PerplexityBot":["partial","*",1,0],"Perplexity-User":["partial","*",1,0],"Google-Extended":["partial","*",1,0],"Applebot-Extended":["partial","*",1,0],"CCBot":["partial","*",1,0],"Bytespider":["partial","*",1,0],"Amazonbot":["partial","*",1,0],"meta-externalagent":["partial","*",1,0],"cohere-ai":["partial","*",1,0],"Diffbot":["partial","*",1,0],"ContextFetchBot":["partial","*",1,0]},"llms":{"status":"present","http":200,"bytes":16274},"home":{"status":"ok","http":200,"url":"https://www.honda.co.jp/","html_tokens":19798,"markdown_tokens":13,"pruned_tokens":13}},
{"rank":6430,"domain":"sevenrooms.com","origin":"https://sevenrooms.com","robots":{"status":"ok","http":200,"url":"https://sevenrooms.com/robots.txt","bytes":68,"groups":1},"bots":{"GPTBot":["allowed","*",0,1],"ChatGPT-User":["allowed","*",0,1],"OAI-SearchBot":["allowed","*",0,1],"ClaudeBot":["allowed","*",0,1],"Claude-User":["allowed","*",0,1],"Claude-SearchBot":["allowed","*",0,1],"anthropic-ai":["allowed","*",0,1],"PerplexityBot":["allowed","*",0,1],"Perplexity-User":["allowed","*",0,1],"Google-Extended":["allowed","*",0,1],"Applebot-Extended":["allowed","*",0,1],"CCBot":["allowed","*",0,1],"Bytespider":["allowed","*",0,1],"Amazonbot":["allowed","*",0,1],"meta-externalagent":["allowed","*",0,1],"cohere-ai":["allowed","*",0,1],"Diffbot":["allowed","*",0,1],"ContextFetchBot":["allowed","*",0,1]},"llms":{"status":"missing","http":404,"bytes":30246},"home":{"status":"ok","http":200,"url":"https://sevenrooms.com/","html_tokens":109827,"markdown_tokens":312,"pruned_tokens":150}},
{"rank":6431,"domain":"mehrnews.com","origin":"https://mehrnews.com","robots":{"status":"ok","http":200,"url":"https://www.mehrnews.com/robots.txt","bytes":2325,"groups":1},"bots":{"GPTBot":["partial","*",1,0],"ChatGPT-User":["partial","*",1,0],"OAI-SearchBot":["partial","*",1,0],"ClaudeBot":["partial","*",1,0],"Claude-User":["partial","*",1,0],"Claude-SearchBot":["partial","*",1,0],"anthropic-ai":["partial","*",1,0],"PerplexityBot":["partial","*",1,0],"Perplexity-User":["partial","*",1,0],"Google-Extended":["partial","*",1,0],"Applebot-Extended":["partial","*",1,0],"CCBot":["partial","*",1,0],"Bytespider":["partial","*",1,0],"Amazonbot":["partial","*",1,0],"meta-externalagent":["partial","*",1,0],"cohere-ai":["partial","*",1,0],"Diffbot":["partial","*",1,0],"ContextFetchBot":["partial","*",1,0]},"llms":{"status":"missing","http":404,"bytes":2004},"home":{"status":"ok","http":200,"url":"https://www.mehrnews.com/","html_tokens":39455,"markdown_tokens":599,"pruned_tokens":173}},
{"rank":6432,"domain":"scoop.co.nz","origin":"https://scoop.co.nz","robots":{"status":"ok","http":200,"url":"https://www.scoop.co.nz/robots.txt","bytes":5464,"groups":128},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["partial","*",2,0],"OAI-SearchBot":["partial","*",2,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",2,0],"Claude-SearchBot":["partial","*",2,0],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["partial","*",2,0],"Perplexity-User":["partial","*",2,0],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["partial","*",2,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",2,0]},"llms":{"status":"missing","http":404,"bytes":28518},"home":{"status":"ok","http":200,"url":"https://www.scoop.co.nz/","html_tokens":23419,"markdown_tokens":47,"pruned_tokens":47}},
{"rank":6433,"domain":"canterbury.ac.nz","origin":"https://canterbury.ac.nz","robots":{"status":"ok","http":200,"url":"https://www.canterbury.ac.nz/robots.txt","bytes":207,"groups":3},"bots":{"GPTBot":["partial","*",1,1],"ChatGPT-User":["partial","*",1,1],"OAI-SearchBot":["partial","*",1,1],"ClaudeBot":["partial","*",1,1],"Claude-User":["partial","*",1,1],"Claude-SearchBot":["partial","*",1,1],"anthropic-ai":["partial","*",1,1],"PerplexityBot":["partial","*",1,1],"Perplexity-User":["partial","*",1,1],"Google-Extended":["partial","*",1,1],"Applebot-Extended":["partial","*",1,1],"CCBot":["partial","*",1,1],"Bytespider":["partial","*",1,1],"Amazonbot":["partial","*",1,1],"meta-externalagent":["partial","*",1,1],"cohere-ai":["partial","*",1,1],"Diffbot":["partial","*",1,1],"ContextFetchBot":["partial","*",1,1]},"llms":{"status":"missing","http":404,"bytes":179505},"home":{"status":"ok","http":200,"url":"https://www.canterbury.ac.nz/","html_tokens":85957,"markdown_tokens":101,"pruned_tokens":101}},
{"rank":6434,"domain":"neurosciencenews.com","origin":"https://neurosciencenews.com","robots":{"status":"ok","http":200,"url":"https://neurosciencenews.com/robots.txt","bytes":445,"groups":5},"bots":{"GPTBot":["partial","*",4,0],"ChatGPT-User":["partial","*",4,0],"OAI-SearchBot":["partial","*",4,0],"ClaudeBot":["partial","*",4,0],"Claude-User":["partial","*",4,0],"Claude-SearchBot":["partial","*",4,0],"anthropic-ai":["partial","*",4,0],"PerplexityBot":["partial","*",4,0],"Perplexity-User":["partial","*",4,0],"Google-Extended":["partial","*",4,0],"Applebot-Extended":["partial","*",4,0],"CCBot":["partial","*",4,0],"Bytespider":["partial","*",4,0],"Amazonbot":["partial","*",4,0],"meta-externalagent":["partial","*",4,0],"cohere-ai":["partial","*",4,0],"Diffbot":["partial","*",4,0],"ContextFetchBot":["partial","*",4,0]},"llms":{"status":"timeout","error":"timed out after 10 s"},"home":{"status":"ok","http":200,"url":"https://neurosciencenews.com/","html_tokens":50592,"markdown_tokens":190,"pruned_tokens":150}},
{"rank":6435,"domain":"rehairhub.com","origin":"https://rehairhub.com","robots":{"status":"ok","http":200,"url":"https://rehairhub.com/robots.txt","bytes":76,"groups":1},"bots":{"GPTBot":["partial","*",1,0],"ChatGPT-User":["partial","*",1,0],"OAI-SearchBot":["partial","*",1,0],"ClaudeBot":["partial","*",1,0],"Claude-User":["partial","*",1,0],"Claude-SearchBot":["partial","*",1,0],"anthropic-ai":["partial","*",1,0],"PerplexityBot":["partial","*",1,0],"Perplexity-User":["partial","*",1,0],"Google-Extended":["partial","*",1,0],"Applebot-Extended":["partial","*",1,0],"CCBot":["partial","*",1,0],"Bytespider":["partial","*",1,0],"Amazonbot":["partial","*",1,0],"meta-externalagent":["partial","*",1,0],"cohere-ai":["partial","*",1,0],"Diffbot":["partial","*",1,0],"ContextFetchBot":["partial","*",1,0]},"llms":{"status":"missing","http":404,"bytes":642},"home":{"status":"ok","http":200,"url":"https://rehairhub.com/","html_tokens":3671,"markdown_tokens":367,"pruned_tokens":172}},
{"rank":6436,"domain":"r7.com","origin":"https://r7.com","robots":{"status":"ok","http":200,"url":"https://www.r7.com/robots.txt","bytes":1171,"groups":11},"bots":{"GPTBot":["allowed","named",0,1],"ChatGPT-User":["partial","*",23,1],"OAI-SearchBot":["partial","*",23,1],"ClaudeBot":["partial","*",23,1],"Claude-User":["partial","*",23,1],"Claude-SearchBot":["partial","*",23,1],"anthropic-ai":["partial","*",23,1],"PerplexityBot":["partial","*",23,1],"Perplexity-User":["partial","*",23,1],"Google-Extended":["allowed","named",0,1],"Applebot-Extended":["partial","*",23,1],"CCBot":["partial","*",23,1],"Bytespider":["partial","*",23,1],"Amazonbot":["partial","*",23,1],"meta-externalagent":["partial","*",23,1],"cohere-ai":["partial","*",23,1],"Diffbot":["partial","*",23,1],"ContextFetchBot":["partial","*",23,1]},"llms":{"status":"missing","http":404,"bytes":1437548},"home":{"status":"ok","http":200,"url":"https://www.r7.com/","html_tokens":633192,"markdown_tokens":18,"pruned_tokens":18}},
{"rank":6437,"domain":"jhsph.edu","origin":"https://jhsph.edu","robots":{"status":"refused","http":403,"url":"https://publichealth.jhu.edu/","bytes":5491,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":6438,"domain":"japan.travel","origin":"https://www.japan.travel","robots":{"status":"ok","http":200,"url":"https://www.japan.travel/robots.txt","bytes":136,"groups":1},"bots":{"GPTBot":["partial","*",3,1],"ChatGPT-User":["partial","*",3,1],"OAI-SearchBot":["partial","*",3,1],"ClaudeBot":["partial","*",3,1],"Claude-User":["partial","*",3,1],"Claude-SearchBot":["partial","*",3,1],"anthropic-ai":["partial","*",3,1],"PerplexityBot":["partial","*",3,1],"Perplexity-User":["partial","*",3,1],"Google-Extended":["partial","*",3,1],"Applebot-Extended":["partial","*",3,1],"CCBot":["partial","*",3,1],"Bytespider":["partial","*",3,1],"Amazonbot":["partial","*",3,1],"meta-externalagent":["partial","*",3,1],"cohere-ai":["partial","*",3,1],"Diffbot":["partial","*",3,1],"ContextFetchBot":["partial","*",3,1]},"llms":{"status":"missing","http":404,"bytes":139046},"home":{"status":"ok","http":200,"url":"https://www.japan.travel/en/","html_tokens":70184,"markdown_tokens":77,"pruned_tokens":33}},
{"rank":6439,"domain":"freddiemac.com","origin":"https://freddiemac.com","robots":{"status":"ok","http":200,"url":"https://www.freddiemac.com/robots.txt","bytes":2302,"groups":1},"bots":{"GPTBot":["partial","*",42,18],"ChatGPT-User":["partial","*",42,18],"OAI-SearchBot":["partial","*",42,18],"ClaudeBot":["partial","*",42,18],"Claude-User":["partial","*",42,18],"Claude-SearchBot":["partial","*",42,18],"anthropic-ai":["partial","*",42,18],"PerplexityBot":["partial","*",42,18],"Perplexity-User":["partial","*",42,18],"Google-Extended":["partial","*",42,18],"Applebot-Extended":["partial","*",42,18],"CCBot":["partial","*",42,18],"Bytespider":["partial","*",42,18],"Amazonbot":["partial","*",42,18],"meta-externalagent":["partial","*",42,18],"cohere-ai":["partial","*",42,18],"Diffbot":["partial","*",42,18],"ContextFetchBot":["partial","*",42,18]},"llms":{"status":"missing","http":404,"bytes":174},"home":{"status":"ok","http":200,"url":"https://www.freddiemac.com/","html_tokens":25354,"markdown_tokens":109,"pruned_tokens":58}},
{"rank":6440,"domain":"trendhunter.com","origin":"https://trendhunter.com","robots":{"status":"refused","http":403,"url":"https://www.trendhunter.com/robots.txt","bytes":3959,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":6441,"domain":"bangordailynews.com","origin":"https://bangordailynews.com","robots":{"status":"ok","http":200,"url":"https://www.bangordailynews.com/robots.txt","bytes":813,"groups":7},"bots":{"GPTBot":["partial","*",5,0],"ChatGPT-User":["partial","*",5,0],"OAI-SearchBot":["allowed","named",0,1],"ClaudeBot":["partial","*",5,0],"Claude-User":["partial","*",5,0],"Claude-SearchBot":["allowed","named",0,1],"anthropic-ai":["partial","*",5,0],"PerplexityBot":["allowed","named",0,1],"Perplexity-User":["partial","*",5,0],"Google-Extended":["partial","*",5,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["partial","*",5,0],"Bytespider":["partial","*",5,0],"Amazonbot":["partial","*",5,0],"meta-externalagent":["partial","*",5,0],"cohere-ai":["partial","*",5,0],"Diffbot":["partial","*",5,0],"ContextFetchBot":["partial","*",5,0]},"llms":{"status":"missing","http":404,"bytes":241548},"home":{"status":"ok","http":200,"url":"https://www.bangordailynews.com/","html_tokens":109988,"markdown_tokens":1616,"pruned_tokens":10}},
{"rank":6442,"domain":"slate.fr","origin":"https://slate.fr","robots":{"status":"ok","http":200,"url":"https://www.slate.fr/robots.txt","bytes":4112,"groups":80},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["blocked","named",1,0],"OAI-SearchBot":["blocked","named",1,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",33,17],"Claude-SearchBot":["partial","*",33,17],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["partial","*",33,17],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["partial","*",33,17],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["partial","*",33,17],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",33,17]},"llms":{"status":"missing","http":404,"bytes":41994},"home":{"status":"ok","http":200,"url":"https://www.slate.fr/","html_tokens":49566,"markdown_tokens":78,"pruned_tokens":78}},
{"rank":6443,"domain":"thenorthface.com","origin":"https://thenorthface.com","robots":{"status":"refused","http":403,"url":"https://thenorthface.com/robots.txt","bytes":380,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":6444,"domain":"thenewhumanitarian.org","origin":"https://thenewhumanitarian.org","robots":{"status":"ok","http":200,"url":"https://www.thenewhumanitarian.org/robots.txt","bytes":2141,"groups":1},"bots":{"GPTBot":["partial","*",35,22],"ChatGPT-User":["partial","*",35,22],"OAI-SearchBot":["partial","*",35,22],"ClaudeBot":["partial","*",35,22],"Claude-User":["partial","*",35,22],"Claude-SearchBot":["partial","*",35,22],"anthropic-ai":["partial","*",35,22],"PerplexityBot":["partial","*",35,22],"Perplexity-User":["partial","*",35,22],"Google-Extended":["partial","*",35,22],"Applebot-Extended":["partial","*",35,22],"CCBot":["partial","*",35,22],"Bytespider":["partial","*",35,22],"Amazonbot":["partial","*",35,22],"meta-externalagent":["partial","*",35,22],"cohere-ai":["partial","*",35,22],"Diffbot":["partial","*",35,22],"ContextFetchBot":["partial","*",35,22]},"llms":{"status":"missing","http":404,"bytes":2820},"home":{"status":"http_error","http":403,"url":"https://thenewhumanitarian.org/"}},
{"rank":6445,"domain":"moonfruit.com","origin":"https://moonfruit.com","robots":{"status":"ok","http":200,"url":"https://moonfruit.com/robots.txt","bytes":0,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":198726},"home":{"status":"ok","http":200,"url":"https://moonfruit.com/","html_tokens":59279,"markdown_tokens":146,"pruned_tokens":146}},
{"rank":6446,"domain":"substackcdn.com","origin":"https://substackcdn.com","robots":{"status":"ok","http":200,"url":"https://substackcdn.com/robots.txt","bytes":24,"groups":1},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"http_error","http":403,"bytes":5672},"home":{"status":"http_error","http":403,"url":"https://substackcdn.com/"}},
{"rank":6447,"domain":"toscana.it","origin":"https://toscana.it","robots":{"status":"unreachable","http":null,"url":"https://toscana.it/robots.txt","bytes":0,"groups":0,"error":"cannot resolve toscana.it"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":6448,"domain":"nhsinform.scot","origin":"https://nhsinform.scot","robots":{"status":"refused","http":403,"url":"https://nhsinform.scot/robots.txt","bytes":38,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":6449,"domain":"mnn.com","origin":"https://mnn.com","robots":{"status":"ok","http":200,"url":"https://www.treehugger.com/robots.txt","bytes":3473,"groups":6},"bots":{"GPTBot":["partial","named",1,0],"ChatGPT-User":["partial","named",1,0],"OAI-SearchBot":["partial","named",1,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["blocked","named",1,0],"Claude-SearchBot":["partial","named",1,2],"anthropic-ai":["partial","named",1,2],"PerplexityBot":["partial","named",1,2],"Perplexity-User":["blocked","named",1,0],"Google-Extended":["partial","named",1,2],"Applebot-Extended":["blocked","named",1,0],"CCBot":["partial","named",1,2],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["partial","named",1,2],"cohere-ai":["partial","named",1,2],"Diffbot":["partial","*",2,0],"ContextFetchBot":["partial","*",2,0]},"llms":{"status":"http_error","http":402,"bytes":612},"home":{"status":"http_error","http":402,"url":"https://www.treehugger.com/"}},
{"rank":6450,"domain":"legalsportsreport.com","origin":"https://legalsportsreport.com","robots":{"status":"ok","http":200,"url":"https://www.legalsportsreport.com/robots.txt","bytes":1026,"groups":8},"bots":{"GPTBot":["partial","*",28,1],"ChatGPT-User":["partial","*",28,1],"OAI-SearchBot":["partial","*",28,1],"ClaudeBot":["partial","*",28,1],"Claude-User":["partial","*",28,1],"Claude-SearchBot":["partial","*",28,1],"anthropic-ai":["partial","*",28,1],"PerplexityBot":["partial","*",28,1],"Perplexity-User":["partial","*",28,1],"Google-Extended":["partial","*",28,1],"Applebot-Extended":["partial","*",28,1],"CCBot":["partial","*",28,1],"Bytespider":["partial","*",28,1],"Amazonbot":["partial","*",28,1],"meta-externalagent":["partial","*",28,1],"cohere-ai":["partial","*",28,1],"Diffbot":["partial","*",28,1],"ContextFetchBot":["partial","*",28,1]},"llms":{"status":"http_error","http":403,"bytes":4554},"home":{"status":"http_error","http":403,"url":"https://www.legalsportsreport.com/"}},
{"rank":6451,"domain":"greasyfork.org","origin":"https://greasyfork.org","robots":{"status":"ok","http":200,"url":"https://greasyfork.org/robots.txt","bytes":29,"groups":1},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"missing","http":404,"bytes":0},"home":{"status":"ok","http":200,"url":"https://greasyfork.org/en","html_tokens":7108,"markdown_tokens":652,"pruned_tokens":107}},
{"rank":6452,"domain":"hainan.gov.cn","origin":"https://hainan.gov.cn","robots":{"status":"timeout","http":null,"url":"https://hainan.gov.cn/robots.txt","bytes":0,"groups":0,"error":"ConnectTimeout"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":6453,"domain":"8theme.com","origin":"https://8theme.com","robots":{"status":"ok","http":200,"url":"https://8theme.com/robots.txt","bytes":7253,"groups":2},"bots":{"GPTBot":["partial","*",66,8],"ChatGPT-User":["partial","*",66,8],"OAI-SearchBot":["partial","*",66,8],"ClaudeBot":["partial","*",66,8],"Claude-User":["partial","*",66,8],"Claude-SearchBot":["partial","*",66,8],"anthropic-ai":["partial","*",66,8],"PerplexityBot":["partial","*",66,8],"Perplexity-User":["partial","*",66,8],"Google-Extended":["partial","*",66,8],"Applebot-Extended":["partial","*",66,8],"CCBot":["partial","*",66,8],"Bytespider":["partial","*",66,8],"Amazonbot":["partial","*",66,8],"meta-externalagent":["partial","*",66,8],"cohere-ai":["partial","*",66,8],"Diffbot":["partial","*",66,8],"ContextFetchBot":["partial","*",66,8]},"llms":{"status":"missing","http":404,"bytes":99677},"home":{"status":"ok","http":200,"url":"https://www.8theme.com/","html_tokens":88133,"markdown_tokens":572,"pruned_tokens":200}},
{"rank":6454,"domain":"billetweb.fr","origin":"https://billetweb.fr","robots":{"status":"ok","http":200,"url":"https://www.billetweb.fr/robots.txt","bytes":401,"groups":1},"bots":{"GPTBot":["partial","*",11,1],"ChatGPT-User":["partial","*",11,1],"OAI-SearchBot":["partial","*",11,1],"ClaudeBot":["partial","*",11,1],"Claude-User":["partial","*",11,1],"Claude-SearchBot":["partial","*",11,1],"anthropic-ai":["partial","*",11,1],"PerplexityBot":["partial","*",11,1],"Perplexity-User":["partial","*",11,1],"Google-Extended":["partial","*",11,1],"Applebot-Extended":["partial","*",11,1],"CCBot":["partial","*",11,1],"Bytespider":["partial","*",11,1],"Amazonbot":["partial","*",11,1],"meta-externalagent":["partial","*",11,1],"cohere-ai":["partial","*",11,1],"Diffbot":["partial","*",11,1],"ContextFetchBot":["partial","*",11,1]},"llms":{"status":"missing","http":404,"bytes":6294},"home":{"status":"ok","http":200,"url":"https://www.billetweb.fr/fr-fr/","html_tokens":86404,"markdown_tokens":591,"pruned_tokens":157}},
{"rank":6455,"domain":"viki.com","origin":"https://viki.com","robots":{"status":"ok","http":200,"url":"https://www.viki.com/robots.txt","bytes":341,"groups":1},"bots":{"GPTBot":["partial","*",8,1],"ChatGPT-User":["partial","*",8,1],"OAI-SearchBot":["partial","*",8,1],"ClaudeBot":["partial","*",8,1],"Claude-User":["partial","*",8,1],"Claude-SearchBot":["partial","*",8,1],"anthropic-ai":["partial","*",8,1],"PerplexityBot":["partial","*",8,1],"Perplexity-User":["partial","*",8,1],"Google-Extended":["partial","*",8,1],"Applebot-Extended":["partial","*",8,1],"CCBot":["partial","*",8,1],"Bytespider":["partial","*",8,1],"Amazonbot":["partial","*",8,1],"meta-externalagent":["partial","*",8,1],"cohere-ai":["partial","*",8,1],"Diffbot":["partial","*",8,1],"ContextFetchBot":["partial","*",8,1]},"llms":{"status":"missing","http":404,"bytes":63233},"home":{"status":"ok","http":200,"url":"https://www.viki.com/","html_tokens":39863,"markdown_tokens":21,"pruned_tokens":21}},
{"rank":6456,"domain":"edf.fr","origin":"https://edf.fr","robots":{"status":"ok","http":200,"url":"https://www.edf.fr/robots.txt","bytes":5024,"groups":3},"bots":{"GPTBot":["partial","named",37,18],"ChatGPT-User":["partial","named",37,18],"OAI-SearchBot":["partial","named",37,18],"ClaudeBot":["partial","named",37,18],"Claude-User":["partial","named",37,18],"Claude-SearchBot":["partial","named",37,18],"anthropic-ai":["partial","named",37,18],"PerplexityBot":["partial","named",37,18],"Perplexity-User":["partial","named",37,18],"Google-Extended":["partial","named",37,18],"Applebot-Extended":["partial","named",37,18],"CCBot":["partial","named",37,18],"Bytespider":["partial","*",37,18],"Amazonbot":["partial","named",37,18],"meta-externalagent":["partial","named",37,18],"cohere-ai":["partial","*",37,18],"Diffbot":["partial","*",37,18],"ContextFetchBot":["partial","*",37,18]},"llms":{"status":"http_error","http":403,"bytes":7575},"home":{"status":"ok","http":200,"url":"https://www.edf.fr/","html_tokens":20097,"markdown_tokens":102,"pruned_tokens":102}},
{"rank":6457,"domain":"piriform.com","origin":"https://www.piriform.com","robots":{"status":"refused","http":403,"url":"https://www.ccleaner.com/robots.txt","bytes":384,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":6458,"domain":"rbb24.de","origin":"https://rbb24.de","robots":{"status":"ok","http":200,"url":"https://www.rbb24.de/robots.txt","bytes":3253,"groups":32},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["partial","*",4,0],"OAI-SearchBot":["partial","*",4,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",4,0],"Claude-SearchBot":["partial","*",4,0],"anthropic-ai":["partial","*",4,0],"PerplexityBot":["partial","*",4,0],"Perplexity-User":["partial","*",4,0],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["partial","*",4,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["partial","*",4,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",4,0]},"llms":{"status":"missing","http":404,"bytes":35113},"home":{"status":"ok","http":200,"url":"https://www.rbb24.de/","html_tokens":97182,"markdown_tokens":265,"pruned_tokens":96}},
{"rank":6459,"domain":"endocrine.org","origin":"https://endocrine.org","robots":{"status":"ok","http":200,"url":"https://www.endocrine.org/robots.txt","bytes":50,"groups":1},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"html","http":200,"bytes":80112},"home":{"status":"ok","http":200,"url":"https://www.endocrine.org/","html_tokens":27779,"markdown_tokens":768,"pruned_tokens":110}},
{"rank":6460,"domain":"timesofmalta.com","origin":"https://timesofmalta.com","robots":{"status":"refused","http":403,"url":"https://timesofmalta.com/robots.txt","bytes":5626,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":6461,"domain":"shinystat.com","origin":"https://shinystat.com","robots":{"status":"ok","http":200,"url":"https://www.shinystat.com/robots.txt","bytes":240,"groups":1},"bots":{"GPTBot":["partial","*",6,0],"ChatGPT-User":["partial","*",6,0],"OAI-SearchBot":["partial","*",6,0],"ClaudeBot":["partial","*",6,0],"Claude-User":["partial","*",6,0],"Claude-SearchBot":["partial","*",6,0],"anthropic-ai":["partial","*",6,0],"PerplexityBot":["partial","*",6,0],"Perplexity-User":["partial","*",6,0],"Google-Extended":["partial","*",6,0],"Applebot-Extended":["partial","*",6,0],"CCBot":["partial","*",6,0],"Bytespider":["partial","*",6,0],"Amazonbot":["partial","*",6,0],"meta-externalagent":["partial","*",6,0],"cohere-ai":["partial","*",6,0],"Diffbot":["partial","*",6,0],"ContextFetchBot":["partial","*",6,0]},"llms":{"status":"missing","http":404,"bytes":30273},"home":{"status":"ok","http":200,"url":"https://www.shinystat.com/","html_tokens":17544,"markdown_tokens":183,"pruned_tokens":18}},
{"rank":6462,"domain":"unitn.it","origin":"https://unitn.it","robots":{"status":"ok","http":200,"url":"https://www.unitn.it/robots.txt","bytes":2027,"groups":1},"bots":{"GPTBot":["partial","*",34,18],"ChatGPT-User":["partial","*",34,18],"OAI-SearchBot":["partial","*",34,18],"ClaudeBot":["partial","*",34,18],"Claude-User":["partial","*",34,18],"Claude-SearchBot":["partial","*",34,18],"anthropic-ai":["partial","*",34,18],"PerplexityBot":["partial","*",34,18],"Perplexity-User":["partial","*",34,18],"Google-Extended":["partial","*",34,18],"Applebot-Extended":["partial","*",34,18],"CCBot":["partial","*",34,18],"Bytespider":["partial","*",34,18],"Amazonbot":["partial","*",34,18],"meta-externalagent":["partial","*",34,18],"cohere-ai":["partial","*",34,18],"Diffbot":["partial","*",34,18],"ContextFetchBot":["partial","*",34,18]},"llms":{"status":"missing","http":404,"bytes":547},"home":{"status":"ok","http":200,"url":"https://www.unitn.it/it","html_tokens":118942,"markdown_tokens":34,"pruned_tokens":34}},
{"rank":6463,"domain":"utu.fi","origin":"https://utu.fi","robots":{"status":"ok","http":200,"url":"https://www.utu.fi/robots.txt","bytes":2027,"groups":1},"bots":{"GPTBot":["partial","*",34,18],"ChatGPT-User":["partial","*",34,18],"OAI-SearchBot":["partial","*",34,18],"ClaudeBot":["partial","*",34,18],"Claude-User":["partial","*",34,18],"Claude-SearchBot":["partial","*",34,18],"anthropic-ai":["partial","*",34,18],"PerplexityBot":["partial","*",34,18],"Perplexity-User":["partial","*",34,18],"Google-Extended":["partial","*",34,18],"Applebot-Extended":["partial","*",34,18],"CCBot":["partial","*",34,18],"Bytespider":["partial","*",34,18],"Amazonbot":["partial","*",34,18],"meta-externalagent":["partial","*",34,18],"cohere-ai":["partial","*",34,18],"Diffbot":["partial","*",34,18],"ContextFetchBot":["partial","*",34,18]},"llms":{"status":"missing","http":404,"bytes":146},"home":{"status":"ok","http":200,"url":"https://www.utu.fi/fi","html_tokens":17821,"markdown_tokens":62,"pruned_tokens":62}},
{"rank":6464,"domain":"putty.org","origin":"https://putty.org","robots":{"status":"missing","http":404,"url":"https://putty.org/robots.txt","bytes":433,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":433},"home":{"status":"ok","http":200,"url":"https://putty.org/","html_tokens":1277,"markdown_tokens":421,"pruned_tokens":97}},
{"rank":6465,"domain":"xfyun.cn","origin":"https://xfyun.cn","robots":{"status":"missing","http":404,"url":"https://www.xfyun.cn/robots.txt","bytes":28641,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":28641},"home":{"status":"ok","http":200,"url":"https://www.xfyun.cn/","html_tokens":46156,"markdown_tokens":18,"pruned_tokens":18}},
{"rank":6466,"domain":"byrdie.com","origin":"https://byrdie.com","robots":{"status":"ok","http":200,"url":"https://www.byrdie.com/robots.txt","bytes":3465,"groups":6},"bots":{"GPTBot":["partial","named",1,0],"ChatGPT-User":["partial","named",1,0],"OAI-SearchBot":["partial","named",1,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["blocked","named",1,0],"Claude-SearchBot":["partial","named",1,2],"anthropic-ai":["partial","named",1,2],"PerplexityBot":["partial","named",1,2],"Perplexity-User":["blocked","named",1,0],"Google-Extended":["partial","named",1,2],"Applebot-Extended":["blocked","named",1,0],"CCBot":["partial","named",1,2],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["partial","named",1,2],"cohere-ai":["partial","named",1,2],"Diffbot":["partial","*",2,0],"ContextFetchBot":["partial","*",2,0]},"llms":{"status":"http_error","http":402,"bytes":612},"home":{"status":"http_error","http":402,"url":"https://www.byrdie.com/"}},
{"rank":6467,"domain":"bis.gov","origin":"https://www.bis.gov","robots":{"status":"ok","http":200,"url":"https://www.bis.gov/robots.txt","bytes":22,"groups":1},"bots":{"GPTBot":["allowed","*",0,1],"ChatGPT-User":["allowed","*",0,1],"OAI-SearchBot":["allowed","*",0,1],"ClaudeBot":["allowed","*",0,1],"Claude-User":["allowed","*",0,1],"Claude-SearchBot":["allowed","*",0,1],"anthropic-ai":["allowed","*",0,1],"PerplexityBot":["allowed","*",0,1],"Perplexity-User":["allowed","*",0,1],"Google-Extended":["allowed","*",0,1],"Applebot-Extended":["allowed","*",0,1],"CCBot":["allowed","*",0,1],"Bytespider":["allowed","*",0,1],"Amazonbot":["allowed","*",0,1],"meta-externalagent":["allowed","*",0,1],"cohere-ai":["allowed","*",0,1],"Diffbot":["allowed","*",0,1],"ContextFetchBot":["allowed","*",0,1]},"llms":{"status":"missing","http":404,"bytes":47550},"home":{"status":"ok","http":200,"url":"https://www.bis.gov/","html_tokens":18973,"markdown_tokens":488,"pruned_tokens":35}},
{"rank":6468,"domain":"whu.edu.cn","origin":"https://whu.edu.cn","robots":{"status":"ok","http":200,"url":"https://www.whu.edu.cn","bytes":122462,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"disallowed","error":"robots.txt of www.whu.edu.cn not readable (unreachable)"},"home":{"status":"disallowed","error":"robots.txt of www.whu.edu.cn not readable (unreachable)"}},
{"rank":6469,"domain":"india.gov.in","origin":"https://india.gov.in","robots":{"status":"refused","http":403,"url":"https://www.india.gov.in/robots.txt","bytes":388,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":6470,"domain":"jacksonville.com","origin":"https://jacksonville.com","robots":{"status":"ok","http":200,"url":"https://www.jacksonville.com/robots.txt","bytes":14073,"groups":288},"bots":{"GPTBot":["partial","named",1,4],"ChatGPT-User":["partial","named",1,4],"OAI-SearchBot":["partial","named",1,4],"ClaudeBot":["blocked","named",1,0],"Claude-User":["blocked","named",1,0],"Claude-SearchBot":["blocked","named",1,0],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["partial","named",1,4],"Perplexity-User":["blocked","named",1,0],"Google-Extended":["partial","named",1,4],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",2,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",88,0]},"llms":{"status":"http_error","http":402,"bytes":254},"home":{"status":"http_error","http":402,"url":"https://www.jacksonville.com/"}},
{"rank":6471,"domain":"univ-lyon1.fr","origin":"https://univ-lyon1.fr","robots":{"status":"html","http":200,"url":"https://www.univ-lyon1.fr","bytes":179936,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":6472,"domain":"tvn24.pl","origin":"https://tvn24.pl","robots":{"status":"ok","http":200,"url":"https://tvn24.pl/robots.txt","bytes":92,"groups":2},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["partial","*",2,0],"OAI-SearchBot":["partial","*",2,0],"ClaudeBot":["partial","*",2,0],"Claude-User":["partial","*",2,0],"Claude-SearchBot":["partial","*",2,0],"anthropic-ai":["partial","*",2,0],"PerplexityBot":["partial","*",2,0],"Perplexity-User":["partial","*",2,0],"Google-Extended":["partial","*",2,0],"Applebot-Extended":["partial","*",2,0],"CCBot":["partial","*",2,0],"Bytespider":["partial","*",2,0],"Amazonbot":["partial","*",2,0],"meta-externalagent":["partial","*",2,0],"cohere-ai":["partial","*",2,0],"Diffbot":["partial","*",2,0],"ContextFetchBot":["partial","*",2,0]},"llms":{"status":"missing","http":404,"bytes":108682},"home":{"status":"ok","http":200,"url":"https://tvn24.pl/","html_tokens":191547,"markdown_tokens":96,"pruned_tokens":96}},
{"rank":6473,"domain":"globaldata.com","origin":"https://globaldata.com","robots":{"status":"ok","http":200,"url":"https://www.globaldata.com/robots.txt","bytes":2932,"groups":2},"bots":{"GPTBot":["allowed","named",1,13],"ChatGPT-User":["partial","*",26,0],"OAI-SearchBot":["allowed","named",1,13],"ClaudeBot":["allowed","named",1,13],"Claude-User":["allowed","named",1,13],"Claude-SearchBot":["allowed","named",1,13],"anthropic-ai":["partial","*",26,0],"PerplexityBot":["allowed","named",1,13],"Perplexity-User":["allowed","named",1,13],"Google-Extended":["allowed","named",1,13],"Applebot-Extended":["allowed","named",1,13],"CCBot":["allowed","named",1,13],"Bytespider":["allowed","named",1,13],"Amazonbot":["allowed","named",1,13],"meta-externalagent":["allowed","named",1,13],"cohere-ai":["allowed","named",1,13],"Diffbot":["allowed","named",1,13],"ContextFetchBot":["partial","*",26,0]},"llms":{"status":"missing","http":404,"bytes":743211},"home":{"status":"ok","http":200,"url":"https://www.globaldata.com/","html_tokens":215882,"markdown_tokens":363,"pruned_tokens":103}},
{"rank":6474,"domain":"miaminewtimes.com","origin":"https://miaminewtimes.com","robots":{"status":"ok","http":200,"url":"https://www.miaminewtimes.com/robots.txt","bytes":1416,"groups":2},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["blocked","named",1,0],"OAI-SearchBot":["blocked","named",1,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["blocked","named",1,0],"Claude-SearchBot":["blocked","named",1,0],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["blocked","named",1,0],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",1,1]},"llms":{"status":"missing","http":404,"bytes":118088},"home":{"status":"ok","http":200,"url":"https://www.miaminewtimes.com/","html_tokens":64774,"markdown_tokens":56,"pruned_tokens":56}},
{"rank":6475,"domain":"google.com.ec","origin":"https://google.com.ec","robots":{"status":"ok","http":200,"url":"https://www.google.com.ec/robots.txt","bytes":6624,"groups":4},"bots":{"GPTBot":["partial","*",169,64],"ChatGPT-User":["partial","*",169,64],"OAI-SearchBot":["partial","*",169,64],"ClaudeBot":["partial","*",169,64],"Claude-User":["partial","*",169,64],"Claude-SearchBot":["partial","*",169,64],"anthropic-ai":["partial","*",169,64],"PerplexityBot":["partial","*",169,64],"Perplexity-User":["partial","*",169,64],"Google-Extended":["partial","*",169,64],"Applebot-Extended":["partial","*",169,64],"CCBot":["partial","*",169,64],"Bytespider":["partial","*",169,64],"Amazonbot":["partial","*",169,64],"meta-externalagent":["partial","*",169,64],"cohere-ai":["partial","*",169,64],"Diffbot":["partial","*",169,64],"ContextFetchBot":["partial","*",169,64]},"llms":{"status":"missing","http":404,"bytes":1569},"home":{"status":"ok","http":200,"url":"https://www.google.com/","html_tokens":21368,"markdown_tokens":30,"pruned_tokens":30}},
{"rank":6476,"domain":"yokohama.lg.jp","origin":"https://yokohama.lg.jp","robots":{"status":"unreachable","http":null,"url":"https://yokohama.lg.jp/robots.txt","bytes":0,"groups":0,"error":"cannot resolve yokohama.lg.jp"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":6477,"domain":"themify.me","origin":"https://themify.me","robots":{"status":"ok","http":200,"url":"https://themify.me/robots.txt","bytes":147,"groups":1},"bots":{"GPTBot":["partial","*",7,0],"ChatGPT-User":["partial","*",7,0],"OAI-SearchBot":["partial","*",7,0],"ClaudeBot":["partial","*",7,0],"Claude-User":["partial","*",7,0],"Claude-SearchBot":["partial","*",7,0],"anthropic-ai":["partial","*",7,0],"PerplexityBot":["partial","*",7,0],"Perplexity-User":["partial","*",7,0],"Google-Extended":["partial","*",7,0],"Applebot-Extended":["partial","*",7,0],"CCBot":["partial","*",7,0],"Bytespider":["partial","*",7,0],"Amazonbot":["partial","*",7,0],"meta-externalagent":["partial","*",7,0],"cohere-ai":["partial","*",7,0],"Diffbot":["partial","*",7,0],"ContextFetchBot":["partial","*",7,0]},"llms":{"status":"missing","http":404,"bytes":59254},"home":{"status":"ok","http":200,"url":"https://themify.me/","html_tokens":29869,"markdown_tokens":440,"pruned_tokens":220}},
{"rank":6478,"domain":"samsclub.com","origin":"https://samsclub.com","robots":{"status":"html","http":200,"url":"https://www.samsclub.com/robots.txt","bytes":244,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":6479,"domain":"lavoixdunord.fr","origin":"https://lavoixdunord.fr","robots":{"status":"ok","http":200,"url":"https://www.lavoixdunord.fr/robots.txt","bytes":6485,"groups":3},"bots":{"GPTBot":["partial","named",1,11],"ChatGPT-User":["partial","named",1,11],"OAI-SearchBot":["partial","named",1,11],"ClaudeBot":["partial","named",1,11],"Claude-User":["partial","named",1,11],"Claude-SearchBot":["partial","named",1,11],"anthropic-ai":["partial","named",1,11],"PerplexityBot":["partial","named",1,11],"Perplexity-User":["partial","named",1,11],"Google-Extended":["partial","named",1,11],"Applebot-Extended":["partial","named",1,11],"CCBot":["partial","named",1,11],"Bytespider":["partial","named",1,11],"Amazonbot":["partial","named",1,11],"meta-externalagent":["partial","named",1,11],"cohere-ai":["partial","named",1,11],"Diffbot":["partial","named",1,11],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"present","http":200,"bytes":2093},"home":{"status":"ok","http":200,"url":"https://www.lavoixdunord.fr/","html_tokens":138631,"markdown_tokens":62,"pruned_tokens":62}},
{"rank":6480,"domain":"sfexaminer.com","origin":"https://sfexaminer.com","robots":{"status":"ok","http":200,"url":"https://sfexaminer.com/robots.txt","bytes":4129,"groups":31},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["blocked","named",1,0],"OAI-SearchBot":["blocked","named",1,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",37,0],"Claude-SearchBot":["partial","*",37,0],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["partial","*",37,0],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["partial","*",37,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",37,0]},"llms":{"status":"http_error","http":429,"bytes":64},"home":{"status":"ok","http":200,"url":"https://www.sfexaminer.com/","html_tokens":130951,"markdown_tokens":185,"pruned_tokens":139}},
{"rank":6481,"domain":"break.com","origin":"https://break.com","robots":{"status":"ok","http":200,"url":"https://break.com/robots.txt","bytes":87,"groups":1},"bots":{"GPTBot":["allowed","*",0,1],"ChatGPT-User":["allowed","*",0,1],"OAI-SearchBot":["allowed","*",0,1],"ClaudeBot":["allowed","*",0,1],"Claude-User":["allowed","*",0,1],"Claude-SearchBot":["allowed","*",0,1],"anthropic-ai":["allowed","*",0,1],"PerplexityBot":["allowed","*",0,1],"Perplexity-User":["allowed","*",0,1],"Google-Extended":["allowed","*",0,1],"Applebot-Extended":["allowed","*",0,1],"CCBot":["allowed","*",0,1],"Bytespider":["allowed","*",0,1],"Amazonbot":["allowed","*",0,1],"meta-externalagent":["allowed","*",0,1],"cohere-ai":["allowed","*",0,1],"Diffbot":["allowed","*",0,1],"ContextFetchBot":["allowed","*",0,1]},"llms":{"status":"html","http":200,"bytes":250434},"home":{"status":"ok","http":200,"url":"https://break.com/","html_tokens":62604,"markdown_tokens":1078,"pruned_tokens":136}},
{"rank":6482,"domain":"jitsi.org","origin":"https://jitsi.org","robots":{"status":"ok","http":200,"url":"https://jitsi.org/robots.txt","bytes":166,"groups":1},"bots":{"GPTBot":["partial","*",3,1],"ChatGPT-User":["partial","*",3,1],"OAI-SearchBot":["partial","*",3,1],"ClaudeBot":["partial","*",3,1],"Claude-User":["partial","*",3,1],"Claude-SearchBot":["partial","*",3,1],"anthropic-ai":["partial","*",3,1],"PerplexityBot":["partial","*",3,1],"Perplexity-User":["partial","*",3,1],"Google-Extended":["partial","*",3,1],"Applebot-Extended":["partial","*",3,1],"CCBot":["partial","*",3,1],"Bytespider":["partial","*",3,1],"Amazonbot":["partial","*",3,1],"meta-externalagent":["partial","*",3,1],"cohere-ai":["partial","*",3,1],"Diffbot":["partial","*",3,1],"ContextFetchBot":["partial","*",3,1]},"llms":{"status":"missing","http":404,"bytes":60},"home":{"status":"ok","http":200,"url":"https://jitsi.org/","html_tokens":11607,"markdown_tokens":155,"pruned_tokens":120}},
{"rank":6483,"domain":"iapp.org","origin":"https://iapp.org","robots":{"status":"ok","http":200,"url":"https://iapp.org/robots.txt","bytes":154,"groups":1},"bots":{"GPTBot":["partial","*",4,1],"ChatGPT-User":["partial","*",4,1],"OAI-SearchBot":["partial","*",4,1],"ClaudeBot":["partial","*",4,1],"Claude-User":["partial","*",4,1],"Claude-SearchBot":["partial","*",4,1],"anthropic-ai":["partial","*",4,1],"PerplexityBot":["partial","*",4,1],"Perplexity-User":["partial","*",4,1],"Google-Extended":["partial","*",4,1],"Applebot-Extended":["partial","*",4,1],"CCBot":["partial","*",4,1],"Bytespider":["partial","*",4,1],"Amazonbot":["partial","*",4,1],"meta-externalagent":["partial","*",4,1],"cohere-ai":["partial","*",4,1],"Diffbot":["partial","*",4,1],"ContextFetchBot":["partial","*",4,1]},"llms":{"status":"missing","http":404,"bytes":184507},"home":{"status":"ok","http":200,"url":"https://iapp.org/","html_tokens":120012,"markdown_tokens":281,"pruned_tokens":170}},
{"rank":6484,"domain":"iis.net","origin":"https://iis.net","robots":{"status":"html","http":200,"url":"https://www.iis.net/","bytes":25092,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":6485,"domain":"theparisreview.org","origin":"https://theparisreview.org","robots":{"status":"ok","http":200,"url":"https://www.theparisreview.org/robots.txt","bytes":442,"groups":3},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["partial","*",13,0],"OAI-SearchBot":["partial","*",13,0],"ClaudeBot":["partial","*",13,0],"Claude-User":["partial","*",13,0],"Claude-SearchBot":["partial","*",13,0],"anthropic-ai":["partial","*",13,0],"PerplexityBot":["partial","*",13,0],"Perplexity-User":["partial","*",13,0],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["partial","*",13,0],"CCBot":["partial","*",13,0],"Bytespider":["partial","*",13,0],"Amazonbot":["partial","*",13,0],"meta-externalagent":["partial","*",13,0],"cohere-ai":["partial","*",13,0],"Diffbot":["partial","*",13,0],"ContextFetchBot":["partial","*",13,0]},"llms":{"status":"missing","http":404,"bytes":4532},"home":{"status":"http_error","http":403,"url":"https://theparisreview.org/"}},
{"rank":6486,"domain":"suite101.com","origin":"https://suite101.com","robots":{"status":"ok","http":200,"url":"https://suite101.com/robots.txt","bytes":97,"groups":1},"bots":{"GPTBot":["partial","*",1,2],"ChatGPT-User":["partial","*",1,2],"OAI-SearchBot":["partial","*",1,2],"ClaudeBot":["partial","*",1,2],"Claude-User":["partial","*",1,2],"Claude-SearchBot":["partial","*",1,2],"anthropic-ai":["partial","*",1,2],"PerplexityBot":["partial","*",1,2],"Perplexity-User":["partial","*",1,2],"Google-Extended":["partial","*",1,2],"Applebot-Extended":["partial","*",1,2],"CCBot":["partial","*",1,2],"Bytespider":["partial","*",1,2],"Amazonbot":["partial","*",1,2],"meta-externalagent":["partial","*",1,2],"cohere-ai":["partial","*",1,2],"Diffbot":["partial","*",1,2],"ContextFetchBot":["partial","*",1,2]},"llms":{"status":"missing","http":404,"bytes":147585},"home":{"status":"ok","http":200,"url":"https://suite101.com/","html_tokens":77346,"markdown_tokens":496,"pruned_tokens":47}},
{"rank":6487,"domain":"aasm.org","origin":"https://aasm.org","robots":{"status":"ok","http":200,"url":"https://aasm.org/robots.txt","bytes":3017,"groups":14},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["blocked","named",1,0],"OAI-SearchBot":["blocked","named",1,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",11,1],"Claude-SearchBot":["partial","*",11,1],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["partial","*",11,1],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",11,1]},"llms":{"status":"http_error","http":403,"bytes":5539},"home":{"status":"http_error","http":403,"url":"https://aasm.org/"}},
{"rank":6488,"domain":"izvestia.ru","origin":"http://izvestia.ru","robots":{"status":"ok","http":200,"url":"https://iz.ru/robots.txt","bytes":4144,"groups":3},"bots":{"GPTBot":["partial","*",18,3],"ChatGPT-User":["partial","*",18,3],"OAI-SearchBot":["partial","*",18,3],"ClaudeBot":["partial","*",18,3],"Claude-User":["partial","*",18,3],"Claude-SearchBot":["partial","*",18,3],"anthropic-ai":["partial","*",18,3],"PerplexityBot":["partial","*",18,3],"Perplexity-User":["partial","*",18,3],"Google-Extended":["partial","*",18,3],"Applebot-Extended":["partial","*",18,3],"CCBot":["partial","*",18,3],"Bytespider":["partial","*",18,3],"Amazonbot":["partial","*",18,3],"meta-externalagent":["partial","*",18,3],"cohere-ai":["partial","*",18,3],"Diffbot":["partial","*",18,3],"ContextFetchBot":["partial","*",18,3]},"llms":{"status":"disallowed","error":"robots.txt of iz.ru not readable (refused)"},"home":{"status":"disallowed","error":"robots.txt of iz.ru not readable (refused)"}},
{"rank":6489,"domain":"imperva.com","origin":"https://imperva.com","robots":{"status":"ok","http":200,"url":"https://www.imperva.com/robots.txt","bytes":2513,"groups":6},"bots":{"GPTBot":["partial","*",18,1],"ChatGPT-User":["partial","*",18,1],"OAI-SearchBot":["partial","*",18,1],"ClaudeBot":["partial","*",18,1],"Claude-User":["partial","*",18,1],"Claude-SearchBot":["partial","*",18,1],"anthropic-ai":["partial","*",18,1],"PerplexityBot":["partial","*",18,1],"Perplexity-User":["partial","*",18,1],"Google-Extended":["partial","*",18,1],"Applebot-Extended":["partial","*",18,1],"CCBot":["partial","*",18,1],"Bytespider":["partial","*",18,1],"Amazonbot":["partial","*",18,1],"meta-externalagent":["partial","*",18,1],"cohere-ai":["partial","*",18,1],"Diffbot":["partial","*",18,1],"ContextFetchBot":["partial","*",18,1]},"llms":{"status":"missing","http":404,"bytes":174364},"home":{"status":"ok","http":200,"url":"https://www.imperva.com/","html_tokens":62004,"markdown_tokens":166,"pruned_tokens":8}},
{"rank":6490,"domain":"draxe.com","origin":"https://draxe.com","robots":{"status":"ok","http":200,"url":"https://draxe.com/robots.txt","bytes":63,"groups":1},"bots":{"GPTBot":["allowed","*",0,1],"ChatGPT-User":["allowed","*",0,1],"OAI-SearchBot":["allowed","*",0,1],"ClaudeBot":["allowed","*",0,1],"Claude-User":["allowed","*",0,1],"Claude-SearchBot":["allowed","*",0,1],"anthropic-ai":["allowed","*",0,1],"PerplexityBot":["allowed","*",0,1],"Perplexity-User":["allowed","*",0,1],"Google-Extended":["allowed","*",0,1],"Applebot-Extended":["allowed","*",0,1],"CCBot":["allowed","*",0,1],"Bytespider":["allowed","*",0,1],"Amazonbot":["allowed","*",0,1],"meta-externalagent":["allowed","*",0,1],"cohere-ai":["allowed","*",0,1],"Diffbot":["allowed","*",0,1],"ContextFetchBot":["allowed","*",0,1]},"llms":{"status":"present","http":200,"bytes":1872},"home":{"status":"http_error","http":429,"url":"https://draxe.com/"}},
{"rank":6491,"domain":"nvcasinos.io","origin":"https://nvcasinos.io","robots":{"status":"ok","http":200,"url":"https://nvcasinos.win/robots.txt","bytes":638,"groups":5},"bots":{"GPTBot":["partial","*",3,16],"ChatGPT-User":["partial","*",3,16],"OAI-SearchBot":["partial","*",3,16],"ClaudeBot":["partial","*",3,16],"Claude-User":["partial","*",3,16],"Claude-SearchBot":["partial","*",3,16],"anthropic-ai":["partial","*",3,16],"PerplexityBot":["partial","*",3,16],"Perplexity-User":["partial","*",3,16],"Google-Extended":["partial","*",3,16],"Applebot-Extended":["partial","*",3,16],"CCBot":["partial","*",3,16],"Bytespider":["partial","*",3,16],"Amazonbot":["partial","*",3,16],"meta-externalagent":["partial","*",3,16],"cohere-ai":["partial","*",3,16],"Diffbot":["partial","*",3,16],"ContextFetchBot":["partial","*",3,16]},"llms":{"status":"missing","http":404,"bytes":21869},"home":{"status":"ok","http":200,"url":"https://nvcasinos.win/de/","html_tokens":25406,"markdown_tokens":3365,"pruned_tokens":161}},
{"rank":6492,"domain":"tom.com","origin":"https://tom.com","robots":{"status":"unreachable","http":null,"url":"https://tom.com/robots.txt","bytes":0,"groups":0,"error":"[SSL: CERTIFICATE_VERIFY_FAILED] certificate verify failed: unable to get local issuer certificate (_ssl.c:1028)"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":6493,"domain":"esportsinsider.com","origin":"https://esportsinsider.com","robots":{"status":"refused","http":403,"url":"https://esportsinsider.com/robots.txt","bytes":5577,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":6494,"domain":"biblestudytools.com","origin":"https://biblestudytools.com","robots":{"status":"ok","http":200,"url":"https://www.biblestudytools.com/robots.txt","bytes":124,"groups":1},"bots":{"GPTBot":["partial","*",1,1],"ChatGPT-User":["partial","*",1,1],"OAI-SearchBot":["partial","*",1,1],"ClaudeBot":["partial","*",1,1],"Claude-User":["partial","*",1,1],"Claude-SearchBot":["partial","*",1,1],"anthropic-ai":["partial","*",1,1],"PerplexityBot":["partial","*",1,1],"Perplexity-User":["partial","*",1,1],"Google-Extended":["partial","*",1,1],"Applebot-Extended":["partial","*",1,1],"CCBot":["partial","*",1,1],"Bytespider":["partial","*",1,1],"Amazonbot":["partial","*",1,1],"meta-externalagent":["partial","*",1,1],"cohere-ai":["partial","*",1,1],"Diffbot":["partial","*",1,1],"ContextFetchBot":["partial","*",1,1]},"llms":{"status":"missing","http":404,"bytes":93664},"home":{"status":"ok","http":200,"url":"https://www.biblestudytools.com/","html_tokens":43244,"markdown_tokens":473,"pruned_tokens":211}},
{"rank":6495,"domain":"ama.org","origin":"https://ama.org","robots":{"status":"ok","http":200,"url":"https://www.ama.org/robots.txt","bytes":558,"groups":3},"bots":{"GPTBot":["partial","*",9,2],"ChatGPT-User":["partial","*",9,2],"OAI-SearchBot":["partial","*",9,2],"ClaudeBot":["partial","*",9,2],"Claude-User":["partial","*",9,2],"Claude-SearchBot":["partial","*",9,2],"anthropic-ai":["partial","*",9,2],"PerplexityBot":["partial","*",9,2],"Perplexity-User":["partial","*",9,2],"Google-Extended":["partial","*",9,2],"Applebot-Extended":["partial","*",9,2],"CCBot":["partial","*",9,2],"Bytespider":["partial","*",9,2],"Amazonbot":["partial","*",9,2],"meta-externalagent":["partial","*",9,2],"cohere-ai":["partial","*",9,2],"Diffbot":["partial","*",9,2],"ContextFetchBot":["partial","*",9,2]},"llms":{"status":"missing","http":404,"bytes":77917},"home":{"status":"ok","http":200,"url":"https://www.ama.org/","html_tokens":35737,"markdown_tokens":546,"pruned_tokens":64}},
{"rank":6496,"domain":"aetna.com","origin":"https://aetna.com","robots":{"status":"ok","http":200,"url":"https://www.aetna.com/robots.txt","bytes":10826,"groups":3},"bots":{"GPTBot":["partial","*",51,2],"ChatGPT-User":["partial","*",51,2],"OAI-SearchBot":["partial","*",51,2],"ClaudeBot":["partial","*",51,2],"Claude-User":["partial","*",51,2],"Claude-SearchBot":["partial","*",51,2],"anthropic-ai":["partial","*",51,2],"PerplexityBot":["partial","*",51,2],"Perplexity-User":["partial","*",51,2],"Google-Extended":["partial","*",51,2],"Applebot-Extended":["partial","*",51,2],"CCBot":["partial","*",51,2],"Bytespider":["partial","*",51,2],"Amazonbot":["partial","*",51,2],"meta-externalagent":["partial","*",51,2],"cohere-ai":["partial","*",51,2],"Diffbot":["partial","*",51,2],"ContextFetchBot":["partial","*",51,2]},"llms":{"status":"missing","http":404,"bytes":49382},"home":{"status":"ok","http":200,"url":"https://www.aetna.com/","html_tokens":34343,"markdown_tokens":2464,"pruned_tokens":798}},
{"rank":6497,"domain":"gold.ac.uk","origin":"https://gold.ac.uk","robots":{"status":"ok","http":200,"url":"https://www.gold.ac.uk/robots.txt","bytes":329,"groups":1},"bots":{"GPTBot":["partial","*",7,0],"ChatGPT-User":["partial","*",7,0],"OAI-SearchBot":["partial","*",7,0],"ClaudeBot":["partial","*",7,0],"Claude-User":["partial","*",7,0],"Claude-SearchBot":["partial","*",7,0],"anthropic-ai":["partial","*",7,0],"PerplexityBot":["partial","*",7,0],"Perplexity-User":["partial","*",7,0],"Google-Extended":["partial","*",7,0],"Applebot-Extended":["partial","*",7,0],"CCBot":["partial","*",7,0],"Bytespider":["partial","*",7,0],"Amazonbot":["partial","*",7,0],"meta-externalagent":["partial","*",7,0],"cohere-ai":["partial","*",7,0],"Diffbot":["partial","*",7,0],"ContextFetchBot":["partial","*",7,0]},"llms":{"status":"missing","http":404,"bytes":23967},"home":{"status":"ok","http":200,"url":"https://www.gold.ac.uk/","html_tokens":10917,"markdown_tokens":91,"pruned_tokens":91}},
{"rank":6498,"domain":"restofworld.org","origin":"https://restofworld.org","robots":{"status":"ok","http":200,"url":"https://restofworld.org/robots.txt","bytes":1213,"groups":18},"bots":{"GPTBot":["partial","named",1,1],"ChatGPT-User":["partial","named",1,1],"OAI-SearchBot":["partial","*",5,2],"ClaudeBot":["partial","named",1,1],"Claude-User":["partial","*",5,2],"Claude-SearchBot":["partial","*",5,2],"anthropic-ai":["partial","named",1,1],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["partial","*",5,2],"Google-Extended":["partial","named",1,1],"Applebot-Extended":["partial","named",1,1],"CCBot":["partial","named",1,1],"Bytespider":["partial","named",1,1],"Amazonbot":["partial","*",5,2],"meta-externalagent":["partial","*",5,2],"cohere-ai":["partial","named",1,1],"Diffbot":["partial","*",5,2],"ContextFetchBot":["partial","*",5,2]},"llms":{"status":"missing","http":404,"bytes":38781},"home":{"status":"ok","http":200,"url":"https://restofworld.org/","html_tokens":42235,"markdown_tokens":202,"pruned_tokens":27}},
{"rank":6499,"domain":"spaceweather.com","origin":"https://spaceweather.com","robots":{"status":"missing","http":404,"url":"https://spaceweather.com/robots.txt","bytes":178,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":178},"home":{"status":"ok","http":200,"url":"https://spaceweather.com/","html_tokens":34650,"markdown_tokens":3915,"pruned_tokens":589}},
{"rank":6500,"domain":"jssor.com","origin":"https://jssor.com","robots":{"status":"ok","http":200,"url":"https://www.jssor.com/robots.txt","bytes":266,"groups":1},"bots":{"GPTBot":["partial","*",6,0],"ChatGPT-User":["partial","*",6,0],"OAI-SearchBot":["partial","*",6,0],"ClaudeBot":["partial","*",6,0],"Claude-User":["partial","*",6,0],"Claude-SearchBot":["partial","*",6,0],"anthropic-ai":["partial","*",6,0],"PerplexityBot":["partial","*",6,0],"Perplexity-User":["partial","*",6,0],"Google-Extended":["partial","*",6,0],"Applebot-Extended":["partial","*",6,0],"CCBot":["partial","*",6,0],"Bytespider":["partial","*",6,0],"Amazonbot":["partial","*",6,0],"meta-externalagent":["partial","*",6,0],"cohere-ai":["partial","*",6,0],"Diffbot":["partial","*",6,0],"ContextFetchBot":["partial","*",6,0]},"llms":{"status":"http_error","http":500,"bytes":1069},"home":{"status":"http_error","http":403,"url":"https://jssor.com/"}},
{"rank":6501,"domain":"oklahoma.gov","origin":"https://oklahoma.gov","robots":{"status":"ok","http":200,"url":"https://oklahoma.gov/robots.txt","bytes":64,"groups":1},"bots":{"GPTBot":["allowed","*",0,1],"ChatGPT-User":["allowed","*",0,1],"OAI-SearchBot":["allowed","*",0,1],"ClaudeBot":["allowed","*",0,1],"Claude-User":["allowed","*",0,1],"Claude-SearchBot":["allowed","*",0,1],"anthropic-ai":["allowed","*",0,1],"PerplexityBot":["allowed","*",0,1],"Perplexity-User":["allowed","*",0,1],"Google-Extended":["allowed","*",0,1],"Applebot-Extended":["allowed","*",0,1],"CCBot":["allowed","*",0,1],"Bytespider":["allowed","*",0,1],"Amazonbot":["allowed","*",0,1],"meta-externalagent":["allowed","*",0,1],"cohere-ai":["allowed","*",0,1],"Diffbot":["allowed","*",0,1],"ContextFetchBot":["allowed","*",0,1]},"llms":{"status":"missing","http":404,"bytes":196},"home":{"status":"ok","http":200,"url":"https://oklahoma.gov/","html_tokens":25821,"markdown_tokens":425,"pruned_tokens":140}},
{"rank":6502,"domain":"getmonero.org","origin":"https://getmonero.org","robots":{"status":"ok","http":200,"url":"https://www.getmonero.org/robots.txt","bytes":1248,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":19405},"home":{"status":"ok","http":200,"url":"https://www.getmonero.org/","html_tokens":8467,"markdown_tokens":438,"pruned_tokens":141}},
{"rank":6503,"domain":"hi-ho.ne.jp","origin":"https://hi-ho.ne.jp","robots":{"status":"timeout","http":null,"url":"https://hi-ho.ne.jp/robots.txt","bytes":0,"groups":0,"error":"ConnectTimeout"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":6504,"domain":"global.honda","origin":"https://global.honda","robots":{"status":"missing","http":404,"url":"https://global.honda/robots.txt","bytes":50423,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":50421},"home":{"status":"ok","http":200,"url":"https://global.honda/en/","html_tokens":18315,"markdown_tokens":204,"pruned_tokens":167}},
{"rank":6505,"domain":"acehardware.com","origin":"https://acehardware.com","robots":{"status":"refused","http":403,"url":"https://www.acehardware.com/robots.txt","bytes":5536,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":6506,"domain":"visitlondon.com","origin":"https://visitlondon.com","robots":{"status":"ok","http":200,"url":"https://www.visitlondon.com/robots.txt","bytes":655,"groups":3},"bots":{"GPTBot":["partial","*",19,0],"ChatGPT-User":["partial","*",19,0],"OAI-SearchBot":["partial","*",19,0],"ClaudeBot":["partial","*",19,0],"Claude-User":["partial","*",19,0],"Claude-SearchBot":["partial","*",19,0],"anthropic-ai":["partial","*",19,0],"PerplexityBot":["partial","*",19,0],"Perplexity-User":["partial","*",19,0],"Google-Extended":["partial","*",19,0],"Applebot-Extended":["partial","*",19,0],"CCBot":["partial","*",19,0],"Bytespider":["partial","*",19,0],"Amazonbot":["partial","*",19,0],"meta-externalagent":["partial","*",19,0],"cohere-ai":["partial","*",19,0],"Diffbot":["partial","*",19,0],"ContextFetchBot":["partial","*",19,0]},"llms":{"status":"empty","http":200,"bytes":0},"home":{"status":"http_error","http":403,"url":"https://www.visitlondon.com/"}},
{"rank":6507,"domain":"care.com","origin":"https://care.com","robots":{"status":"ok","http":200,"url":"https://www.care.com/robots.txt","bytes":1870,"groups":8},"bots":{"GPTBot":["partial","*",28,3],"ChatGPT-User":["partial","*",28,3],"OAI-SearchBot":["partial","*",28,3],"ClaudeBot":["partial","*",28,3],"Claude-User":["partial","*",28,3],"Claude-SearchBot":["partial","*",28,3],"anthropic-ai":["partial","*",28,3],"PerplexityBot":["partial","*",28,3],"Perplexity-User":["partial","*",28,3],"Google-Extended":["partial","*",28,3],"Applebot-Extended":["partial","*",28,3],"CCBot":["partial","*",28,3],"Bytespider":["partial","*",28,3],"Amazonbot":["partial","*",28,3],"meta-externalagent":["partial","*",28,3],"cohere-ai":["partial","*",28,3],"Diffbot":["partial","*",28,3],"ContextFetchBot":["partial","*",28,3]},"llms":{"status":"present","http":200,"bytes":12125},"home":{"status":"ok","http":200,"url":"https://www.care.com/","html_tokens":88704,"markdown_tokens":263,"pruned_tokens":161}},
{"rank":6508,"domain":"avanafast.com","origin":"https://avanafast.com","robots":{"status":"ok","http":200,"url":"https://avanafast.com/robots.txt","bytes":74,"groups":1},"bots":{"GPTBot":["partial","*",1,0],"ChatGPT-User":["partial","*",1,0],"OAI-SearchBot":["partial","*",1,0],"ClaudeBot":["partial","*",1,0],"Claude-User":["partial","*",1,0],"Claude-SearchBot":["partial","*",1,0],"anthropic-ai":["partial","*",1,0],"PerplexityBot":["partial","*",1,0],"Perplexity-User":["partial","*",1,0],"Google-Extended":["partial","*",1,0],"Applebot-Extended":["partial","*",1,0],"CCBot":["partial","*",1,0],"Bytespider":["partial","*",1,0],"Amazonbot":["partial","*",1,0],"meta-externalagent":["partial","*",1,0],"cohere-ai":["partial","*",1,0],"Diffbot":["partial","*",1,0],"ContextFetchBot":["partial","*",1,0]},"llms":{"status":"missing","http":404,"bytes":642},"home":{"status":"ok","http":200,"url":"https://avanafast.com/","html_tokens":9765,"markdown_tokens":3737,"pruned_tokens":510}},
{"rank":6509,"domain":"normattiva.it","origin":"https://www.normattiva.it","robots":{"status":"ok","http":200,"url":"https://www.normattiva.it/robots.txt","bytes":234,"groups":3},"bots":{"GPTBot":["partial","*",2,1],"ChatGPT-User":["partial","*",2,1],"OAI-SearchBot":["partial","*",2,1],"ClaudeBot":["partial","*",2,1],"Claude-User":["partial","*",2,1],"Claude-SearchBot":["partial","*",2,1],"anthropic-ai":["partial","*",2,1],"PerplexityBot":["partial","*",2,1],"Perplexity-User":["partial","*",2,1],"Google-Extended":["partial","*",2,1],"Applebot-Extended":["partial","*",2,1],"CCBot":["partial","*",2,1],"Bytespider":["partial","*",2,1],"Amazonbot":["partial","*",2,1],"meta-externalagent":["partial","*",2,1],"cohere-ai":["partial","*",2,1],"Diffbot":["partial","*",2,1],"ContextFetchBot":["partial","*",2,1]},"llms":{"status":"http_error","http":409,"bytes":501},"home":{"status":"ok","http":200,"url":"https://www.normattiva.it/","html_tokens":11696,"markdown_tokens":141,"pruned_tokens":69}},
{"rank":6510,"domain":"williams-sonoma.com","origin":"https://williams-sonoma.com","robots":{"status":"ok","http":200,"url":"https://www.williams-sonoma.com/robots.txt","bytes":1175,"groups":2},"bots":{"GPTBot":["partial","*",17,1],"ChatGPT-User":["partial","*",17,1],"OAI-SearchBot":["partial","*",17,1],"ClaudeBot":["partial","*",17,1],"Claude-User":["partial","*",17,1],"Claude-SearchBot":["partial","*",17,1],"anthropic-ai":["partial","*",17,1],"PerplexityBot":["partial","*",17,1],"Perplexity-User":["partial","*",17,1],"Google-Extended":["partial","*",17,1],"Applebot-Extended":["partial","*",17,1],"CCBot":["partial","*",17,1],"Bytespider":["partial","*",17,1],"Amazonbot":["partial","*",17,1],"meta-externalagent":["partial","*",17,1],"cohere-ai":["partial","*",17,1],"Diffbot":["partial","*",17,1],"ContextFetchBot":["partial","*",17,1]},"llms":{"status":"present","http":200,"bytes":17511},"home":{"status":"ok","http":200,"url":"https://www.williams-sonoma.com/","html_tokens":138893,"markdown_tokens":4,"pruned_tokens":4}},
{"rank":6511,"domain":"theoatmeal.com","origin":"https://theoatmeal.com","robots":{"status":"missing","http":404,"url":"https://theoatmeal.com/robots.txt","bytes":8610,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":8608},"home":{"status":"ok","http":200,"url":"https://theoatmeal.com/","html_tokens":16477,"markdown_tokens":52,"pruned_tokens":52}},
{"rank":6512,"domain":"start.me","origin":"https://start.me","robots":{"status":"ok","http":200,"url":"https://start.me/robots.txt","bytes":77,"groups":1},"bots":{"GPTBot":["partial","*",1,0],"ChatGPT-User":["partial","*",1,0],"OAI-SearchBot":["partial","*",1,0],"ClaudeBot":["partial","*",1,0],"Claude-User":["partial","*",1,0],"Claude-SearchBot":["partial","*",1,0],"anthropic-ai":["partial","*",1,0],"PerplexityBot":["partial","*",1,0],"Perplexity-User":["partial","*",1,0],"Google-Extended":["partial","*",1,0],"Applebot-Extended":["partial","*",1,0],"CCBot":["partial","*",1,0],"Bytespider":["partial","*",1,0],"Amazonbot":["partial","*",1,0],"meta-externalagent":["partial","*",1,0],"cohere-ai":["partial","*",1,0],"Diffbot":["partial","*",1,0],"ContextFetchBot":["partial","*",1,0]},"llms":{"status":"missing","http":404,"bytes":4131},"home":{"status":"http_error","http":403,"url":"https://start.me/"}},
{"rank":6513,"domain":"forum24.ru","origin":"https://forum24.ru","robots":{"status":"ok","http":200,"url":"https://forum24.ru/robots.txt","bytes":0,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"other_text","http":200,"bytes":43},"home":{"status":"ok","http":200,"url":"https://forum24.ru/","html_tokens":3277,"markdown_tokens":201,"pruned_tokens":141}},
{"rank":6514,"domain":"sme.sk","origin":"https://sme.sk","robots":{"status":"ok","http":200,"url":"https://www.sme.sk/robots.txt","bytes":3803,"groups":3},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["blocked","named",1,0],"OAI-SearchBot":["blocked","named",1,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["blocked","named",1,0],"Claude-SearchBot":["blocked","named",1,0],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["blocked","named",1,0],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",1,1]},"llms":{"status":"present","http":200,"bytes":9567},"home":{"status":"http_error","http":403,"url":"https://sme.sk/"}},
{"rank":6515,"domain":"balkaninsight.com","origin":"https://balkaninsight.com","robots":{"status":"ok","http":200,"url":"https://balkaninsight.com/robots.txt","bytes":445,"groups":3},"bots":{"GPTBot":["blocked","*",3,0],"ChatGPT-User":["blocked","*",3,0],"OAI-SearchBot":["blocked","*",3,0],"ClaudeBot":["blocked","*",3,0],"Claude-User":["blocked","*",3,0],"Claude-SearchBot":["blocked","*",3,0],"anthropic-ai":["blocked","*",3,0],"PerplexityBot":["blocked","*",3,0],"Perplexity-User":["blocked","*",3,0],"Google-Extended":["blocked","*",3,0],"Applebot-Extended":["blocked","*",3,0],"CCBot":["blocked","*",3,0],"Bytespider":["blocked","*",3,0],"Amazonbot":["blocked","*",3,0],"meta-externalagent":["blocked","*",3,0],"cohere-ai":["blocked","*",3,0],"Diffbot":["blocked","*",3,0],"ContextFetchBot":["blocked","*",3,0]},"llms":{"status":"disallowed"},"home":{"status":"disallowed"}},
{"rank":6516,"domain":"google.com.lb","origin":"https://google.com.lb","robots":{"status":"ok","http":200,"url":"https://www.google.com.lb/robots.txt","bytes":6624,"groups":4},"bots":{"GPTBot":["partial","*",169,64],"ChatGPT-User":["partial","*",169,64],"OAI-SearchBot":["partial","*",169,64],"ClaudeBot":["partial","*",169,64],"Claude-User":["partial","*",169,64],"Claude-SearchBot":["partial","*",169,64],"anthropic-ai":["partial","*",169,64],"PerplexityBot":["partial","*",169,64],"Perplexity-User":["partial","*",169,64],"Google-Extended":["partial","*",169,64],"Applebot-Extended":["partial","*",169,64],"CCBot":["partial","*",169,64],"Bytespider":["partial","*",169,64],"Amazonbot":["partial","*",169,64],"meta-externalagent":["partial","*",169,64],"cohere-ai":["partial","*",169,64],"Diffbot":["partial","*",169,64],"ContextFetchBot":["partial","*",169,64]},"llms":{"status":"missing","http":404,"bytes":1569},"home":{"status":"ok","http":200,"url":"https://www.google.com/","html_tokens":21290,"markdown_tokens":30,"pruned_tokens":30}},
{"rank":6517,"domain":"metapress.com","origin":"https://metapress.com","robots":{"status":"ok","http":200,"url":"https://metapress.com/robots.txt","bytes":73,"groups":1},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"other_text","http":200,"bytes":1746},"home":{"status":"ok","http":200,"url":"https://metapress.com/","html_tokens":21690,"markdown_tokens":64,"pruned_tokens":64}},
{"rank":6518,"domain":"playbill.com","origin":"https://playbill.com","robots":{"status":"ok","http":200,"url":"https://playbill.com/robots.txt","bytes":202,"groups":1},"bots":{"GPTBot":["partial","*",4,0],"ChatGPT-User":["partial","*",4,0],"OAI-SearchBot":["partial","*",4,0],"ClaudeBot":["partial","*",4,0],"Claude-User":["partial","*",4,0],"Claude-SearchBot":["partial","*",4,0],"anthropic-ai":["partial","*",4,0],"PerplexityBot":["partial","*",4,0],"Perplexity-User":["partial","*",4,0],"Google-Extended":["partial","*",4,0],"Applebot-Extended":["partial","*",4,0],"CCBot":["partial","*",4,0],"Bytespider":["partial","*",4,0],"Amazonbot":["partial","*",4,0],"meta-externalagent":["partial","*",4,0],"cohere-ai":["partial","*",4,0],"Diffbot":["partial","*",4,0],"ContextFetchBot":["partial","*",4,0]},"llms":{"status":"missing","http":404,"bytes":104234},"home":{"status":"ok","http":200,"url":"https://playbill.com/","html_tokens":41303,"markdown_tokens":86,"pruned_tokens":68}},
{"rank":6519,"domain":"3cat.cat","origin":"https://3cat.cat","robots":{"status":"ok","http":200,"url":"https://www.3cat.cat/robots.txt","bytes":640,"groups":2},"bots":{"GPTBot":["partial","*",18,0],"ChatGPT-User":["partial","*",18,0],"OAI-SearchBot":["partial","*",18,0],"ClaudeBot":["partial","*",18,0],"Claude-User":["partial","*",18,0],"Claude-SearchBot":["partial","*",18,0],"anthropic-ai":["partial","*",18,0],"PerplexityBot":["partial","*",18,0],"Perplexity-User":["partial","*",18,0],"Google-Extended":["partial","*",18,0],"Applebot-Extended":["partial","*",18,0],"CCBot":["partial","*",18,0],"Bytespider":["partial","*",18,0],"Amazonbot":["partial","*",18,0],"meta-externalagent":["partial","*",18,0],"cohere-ai":["partial","*",18,0],"Diffbot":["partial","*",18,0],"ContextFetchBot":["partial","*",18,0]},"llms":{"status":"missing","http":404,"bytes":33167},"home":{"status":"ok","http":200,"url":"https://www.3cat.cat/3cat/","html_tokens":64896,"markdown_tokens":167,"pruned_tokens":167}},
{"rank":6520,"domain":"burberry.com","origin":"https://burberry.com","robots":{"status":"ok","http":200,"url":"https://www.burberry.com/robots.txt","bytes":688,"groups":2},"bots":{"GPTBot":["partial","*",15,0],"ChatGPT-User":["partial","*",15,0],"OAI-SearchBot":["partial","*",15,0],"ClaudeBot":["partial","*",15,0],"Claude-User":["partial","*",15,0],"Claude-SearchBot":["partial","*",15,0],"anthropic-ai":["partial","*",15,0],"PerplexityBot":["partial","*",15,0],"Perplexity-User":["partial","*",15,0],"Google-Extended":["partial","*",15,0],"Applebot-Extended":["partial","*",15,0],"CCBot":["partial","*",15,0],"Bytespider":["partial","*",15,0],"Amazonbot":["partial","*",15,0],"meta-externalagent":["partial","*",15,0],"cohere-ai":["partial","*",15,0],"Diffbot":["partial","*",15,0],"ContextFetchBot":["partial","*",15,0]},"llms":{"status":"html","http":200,"bytes":177186},"home":{"status":"ok","http":200,"url":"https://www.burberry.com/","html_tokens":44258,"markdown_tokens":685,"pruned_tokens":175}},
{"rank":6521,"domain":"google.ba","origin":"https://google.ba","robots":{"status":"ok","http":200,"url":"https://www.google.ba/robots.txt","bytes":6624,"groups":4},"bots":{"GPTBot":["partial","*",169,64],"ChatGPT-User":["partial","*",169,64],"OAI-SearchBot":["partial","*",169,64],"ClaudeBot":["partial","*",169,64],"Claude-User":["partial","*",169,64],"Claude-SearchBot":["partial","*",169,64],"anthropic-ai":["partial","*",169,64],"PerplexityBot":["partial","*",169,64],"Perplexity-User":["partial","*",169,64],"Google-Extended":["partial","*",169,64],"Applebot-Extended":["partial","*",169,64],"CCBot":["partial","*",169,64],"Bytespider":["partial","*",169,64],"Amazonbot":["partial","*",169,64],"meta-externalagent":["partial","*",169,64],"cohere-ai":["partial","*",169,64],"Diffbot":["partial","*",169,64],"ContextFetchBot":["partial","*",169,64]},"llms":{"status":"missing","http":404,"bytes":1569},"home":{"status":"ok","http":200,"url":"https://www.google.com/","html_tokens":21303,"markdown_tokens":30,"pruned_tokens":30}},
{"rank":6522,"domain":"kleinanzeigen.de","origin":"https://kleinanzeigen.de","robots":{"status":"ok","http":200,"url":"https://www.kleinanzeigen.de/robots.txt","bytes":11141,"groups":10},"bots":{"GPTBot":["partial","named",39,1],"ChatGPT-User":["partial","named",39,1],"OAI-SearchBot":["allowed","named",0,1],"ClaudeBot":["partial","*",251,1],"Claude-User":["partial","*",251,1],"Claude-SearchBot":["partial","*",251,1],"anthropic-ai":["partial","*",251,1],"PerplexityBot":["partial","named",39,1],"Perplexity-User":["partial","*",251,1],"Google-Extended":["partial","*",251,1],"Applebot-Extended":["partial","*",251,1],"CCBot":["partial","*",251,1],"Bytespider":["partial","*",251,1],"Amazonbot":["partial","*",251,1],"meta-externalagent":["partial","*",251,1],"cohere-ai":["partial","*",251,1],"Diffbot":["partial","*",251,1],"ContextFetchBot":["partial","*",251,1]},"llms":{"status":"missing","http":404,"bytes":5019},"home":{"status":"ok","http":200,"url":"https://www.kleinanzeigen.de/","html_tokens":84886,"markdown_tokens":152,"pruned_tokens":152}},
{"rank":6523,"domain":"padi.com","origin":"https://padi.com","robots":{"status":"ok","http":200,"url":"https://www.padi.com/robots.txt","bytes":2662,"groups":11},"bots":{"GPTBot":["allowed","named",0,1],"ChatGPT-User":["allowed","named",0,2],"OAI-SearchBot":["allowed","named",0,1],"ClaudeBot":["allowed","named",0,1],"Claude-User":["partial","*",22,19],"Claude-SearchBot":["partial","*",22,19],"anthropic-ai":["partial","*",22,19],"PerplexityBot":["allowed","named",0,1],"Perplexity-User":["partial","*",22,19],"Google-Extended":["partial","*",22,19],"Applebot-Extended":["partial","*",22,19],"CCBot":["partial","*",22,19],"Bytespider":["partial","*",22,19],"Amazonbot":["allowed","named",0,1],"meta-externalagent":["allowed","named",0,1],"cohere-ai":["partial","*",22,19],"Diffbot":["partial","*",22,19],"ContextFetchBot":["partial","*",22,19]},"llms":{"status":"missing","http":404,"bytes":186},"home":{"status":"ok","http":200,"url":"https://www.padi.com/","html_tokens":41875,"markdown_tokens":317,"pruned_tokens":84}},
{"rank":6524,"domain":"01net.com","origin":"https://01net.com","robots":{"status":"ok","http":200,"url":"https://www.01net.com/robots.txt","bytes":1046,"groups":19},"bots":{"GPTBot":["partial","*",4,1],"ChatGPT-User":["partial","*",4,1],"OAI-SearchBot":["partial","*",4,1],"ClaudeBot":["partial","*",4,1],"Claude-User":["partial","*",4,1],"Claude-SearchBot":["partial","*",4,1],"anthropic-ai":["partial","*",4,1],"PerplexityBot":["partial","*",4,1],"Perplexity-User":["partial","*",4,1],"Google-Extended":["partial","*",4,1],"Applebot-Extended":["partial","*",4,1],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["partial","*",4,1],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",4,1]},"llms":{"status":"present","http":200,"bytes":2342},"home":{"status":"ok","http":200,"url":"https://www.01net.com/","html_tokens":43049,"markdown_tokens":103,"pruned_tokens":103}},
{"rank":6525,"domain":"uva.es","origin":"https://uva.es","robots":{"status":"ok","http":200,"url":"https://uva.es/robots.txt","bytes":425,"groups":1},"bots":{"GPTBot":["partial","*",8,1],"ChatGPT-User":["partial","*",8,1],"OAI-SearchBot":["partial","*",8,1],"ClaudeBot":["partial","*",8,1],"Claude-User":["partial","*",8,1],"Claude-SearchBot":["partial","*",8,1],"anthropic-ai":["partial","*",8,1],"PerplexityBot":["partial","*",8,1],"Perplexity-User":["partial","*",8,1],"Google-Extended":["partial","*",8,1],"Applebot-Extended":["partial","*",8,1],"CCBot":["partial","*",8,1],"Bytespider":["partial","*",8,1],"Amazonbot":["partial","*",8,1],"meta-externalagent":["partial","*",8,1],"cohere-ai":["partial","*",8,1],"Diffbot":["partial","*",8,1],"ContextFetchBot":["partial","*",8,1]},"llms":{"status":"missing","http":404,"bytes":5689},"home":{"status":"ok","http":200,"url":"https://uva.es/","html_tokens":30602,"markdown_tokens":134,"pruned_tokens":133}},
{"rank":6526,"domain":"stocksnap.io","origin":"https://stocksnap.io","robots":{"status":"ok","http":200,"url":"https://stocksnap.io/robots.txt","bytes":114,"groups":1},"bots":{"GPTBot":["allowed","*",1,0],"ChatGPT-User":["allowed","*",1,0],"OAI-SearchBot":["allowed","*",1,0],"ClaudeBot":["allowed","*",1,0],"Claude-User":["allowed","*",1,0],"Claude-SearchBot":["allowed","*",1,0],"anthropic-ai":["allowed","*",1,0],"PerplexityBot":["allowed","*",1,0],"Perplexity-User":["allowed","*",1,0],"Google-Extended":["allowed","*",1,0],"Applebot-Extended":["allowed","*",1,0],"CCBot":["allowed","*",1,0],"Bytespider":["allowed","*",1,0],"Amazonbot":["allowed","*",1,0],"meta-externalagent":["allowed","*",1,0],"cohere-ai":["allowed","*",1,0],"Diffbot":["allowed","*",1,0],"ContextFetchBot":["allowed","*",1,0]},"llms":{"status":"missing","http":404,"bytes":20268},"home":{"status":"http_error","http":403,"url":"https://stocksnap.io/"}},
{"rank":6527,"domain":"act.org","origin":"https://act.org","robots":{"status":"ok","http":200,"url":"https://www.act.org/robots.txt","bytes":99,"groups":1},"bots":{"GPTBot":["partial","*",1,1],"ChatGPT-User":["partial","*",1,1],"OAI-SearchBot":["partial","*",1,1],"ClaudeBot":["partial","*",1,1],"Claude-User":["partial","*",1,1],"Claude-SearchBot":["partial","*",1,1],"anthropic-ai":["partial","*",1,1],"PerplexityBot":["partial","*",1,1],"Perplexity-User":["partial","*",1,1],"Google-Extended":["partial","*",1,1],"Applebot-Extended":["partial","*",1,1],"CCBot":["partial","*",1,1],"Bytespider":["partial","*",1,1],"Amazonbot":["partial","*",1,1],"meta-externalagent":["partial","*",1,1],"cohere-ai":["partial","*",1,1],"Diffbot":["partial","*",1,1],"ContextFetchBot":["partial","*",1,1]},"llms":{"status":"missing","http":404,"bytes":54411},"home":{"status":"ok","http":200,"url":"https://www.act.org/","html_tokens":28866,"markdown_tokens":452,"pruned_tokens":83}},
{"rank":6528,"domain":"videogamer.com","origin":"https://videogamer.com","robots":{"status":"ok","http":200,"url":"https://www.videogamer.com/robots.txt","bytes":558,"groups":2},"bots":{"GPTBot":["partial","*",10,1],"ChatGPT-User":["partial","*",10,1],"OAI-SearchBot":["partial","*",10,1],"ClaudeBot":["partial","*",10,1],"Claude-User":["partial","*",10,1],"Claude-SearchBot":["partial","*",10,1],"anthropic-ai":["partial","*",10,1],"PerplexityBot":["partial","*",10,1],"Perplexity-User":["partial","*",10,1],"Google-Extended":["partial","*",10,1],"Applebot-Extended":["partial","*",10,1],"CCBot":["partial","*",10,1],"Bytespider":["partial","*",10,1],"Amazonbot":["partial","*",10,1],"meta-externalagent":["partial","*",10,1],"cohere-ai":["partial","*",10,1],"Diffbot":["partial","*",10,1],"ContextFetchBot":["partial","*",10,1]},"llms":{"status":"missing","http":404,"bytes":51864},"home":{"status":"ok","http":200,"url":"https://www.videogamer.com/","html_tokens":38365,"markdown_tokens":1080,"pruned_tokens":100}},
{"rank":6529,"domain":"trellis.net","origin":"https://trellis.net","robots":{"status":"ok","http":200,"url":"https://trellis.net/robots.txt","bytes":259,"groups":1},"bots":{"GPTBot":["partial","*",5,0],"ChatGPT-User":["partial","*",5,0],"OAI-SearchBot":["partial","*",5,0],"ClaudeBot":["partial","*",5,0],"Claude-User":["partial","*",5,0],"Claude-SearchBot":["partial","*",5,0],"anthropic-ai":["partial","*",5,0],"PerplexityBot":["partial","*",5,0],"Perplexity-User":["partial","*",5,0],"Google-Extended":["partial","*",5,0],"Applebot-Extended":["partial","*",5,0],"CCBot":["partial","*",5,0],"Bytespider":["partial","*",5,0],"Amazonbot":["partial","*",5,0],"meta-externalagent":["partial","*",5,0],"cohere-ai":["partial","*",5,0],"Diffbot":["partial","*",5,0],"ContextFetchBot":["partial","*",5,0]},"llms":{"status":"missing","http":404,"bytes":76475},"home":{"status":"ok","http":200,"url":"https://trellis.net/","html_tokens":31543,"markdown_tokens":222,"pruned_tokens":71}},
{"rank":6530,"domain":"blueyonder.co.uk","origin":"http://blueyonder.co.uk","robots":{"status":"html","http":200,"url":"https://www.virginmedia.com/","bytes":688084,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":6531,"domain":"ironman.com","origin":"https://ironman.com","robots":{"status":"ok","http":200,"url":"https://ironman.com/robots.txt","bytes":7973,"groups":20},"bots":{"GPTBot":["partial","named",3,2],"ChatGPT-User":["blocked","named",1,0],"OAI-SearchBot":["partial","*",120,22],"ClaudeBot":["partial","named",3,2],"Claude-User":["blocked","named",1,0],"Claude-SearchBot":["partial","*",120,22],"anthropic-ai":["partial","*",120,22],"PerplexityBot":["partial","*",120,22],"Perplexity-User":["blocked","named",1,0],"Google-Extended":["partial","*",120,22],"Applebot-Extended":["partial","*",120,22],"CCBot":["partial","named",3,2],"Bytespider":["partial","*",120,22],"Amazonbot":["partial","*",120,22],"meta-externalagent":["partial","named",3,2],"cohere-ai":["partial","*",120,22],"Diffbot":["partial","*",120,22],"ContextFetchBot":["partial","*",120,22]},"llms":{"status":"present","http":200,"bytes":1422},"home":{"status":"ok","http":200,"url":"https://ironman.com/","html_tokens":228966,"markdown_tokens":506,"pruned_tokens":232}},
{"rank":6532,"domain":"ralphlauren.com","origin":"https://ralphlauren.com","robots":{"status":"ok","http":200,"url":"https://www.ralphlauren.com/robots.txt","bytes":2426,"groups":6},"bots":{"GPTBot":["partial","*",72,0],"ChatGPT-User":["partial","*",72,0],"OAI-SearchBot":["partial","*",72,0],"ClaudeBot":["partial","*",72,0],"Claude-User":["partial","*",72,0],"Claude-SearchBot":["partial","*",72,0],"anthropic-ai":["partial","*",72,0],"PerplexityBot":["partial","*",72,0],"Perplexity-User":["partial","*",72,0],"Google-Extended":["partial","*",72,0],"Applebot-Extended":["partial","*",72,0],"CCBot":["partial","*",72,0],"Bytespider":["partial","*",72,0],"Amazonbot":["partial","*",72,0],"meta-externalagent":["partial","*",72,0],"cohere-ai":["partial","*",72,0],"Diffbot":["partial","*",72,0],"ContextFetchBot":["partial","*",72,0]},"llms":{"status":"http_error","http":307,"bytes":5645},"home":{"status":"ok","http":307,"url":"https://www.ralphlauren.com/","html_tokens":1412,"markdown_tokens":0,"pruned_tokens":0}},
{"rank":6533,"domain":"courrierinternational.com","origin":"https://courrierinternational.com","robots":{"status":"ok","http":200,"url":"https://www.courrierinternational.com/robots.txt","bytes":6985,"groups":3},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["blocked","named",1,0],"OAI-SearchBot":["partial","*",44,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",44,0],"Claude-SearchBot":["partial","*",44,0],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["partial","*",44,0],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",2,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",2,0],"ContextFetchBot":["partial","*",44,0]},"llms":{"status":"missing","http":404,"bytes":38049},"home":{"status":"ok","http":200,"url":"https://www.courrierinternational.com/","html_tokens":57180,"markdown_tokens":817,"pruned_tokens":169}},
{"rank":6534,"domain":"architizer.com","origin":"https://architizer.com","robots":{"status":"ok","http":200,"url":"https://architizer.com/robots.txt","bytes":661,"groups":2},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["partial","*",24,0],"OAI-SearchBot":["partial","*",24,0],"ClaudeBot":["partial","*",24,0],"Claude-User":["partial","*",24,0],"Claude-SearchBot":["partial","*",24,0],"anthropic-ai":["partial","*",24,0],"PerplexityBot":["partial","*",24,0],"Perplexity-User":["partial","*",24,0],"Google-Extended":["partial","*",24,0],"Applebot-Extended":["partial","*",24,0],"CCBot":["partial","*",24,0],"Bytespider":["partial","*",24,0],"Amazonbot":["partial","*",24,0],"meta-externalagent":["partial","*",24,0],"cohere-ai":["partial","*",24,0],"Diffbot":["partial","*",24,0],"ContextFetchBot":["partial","*",24,0]},"llms":{"status":"missing","http":404,"bytes":61080},"home":{"status":"http_error","http":403,"url":"https://architizer.com/"}},
{"rank":6535,"domain":"aiaa.org","origin":"https://aiaa.org","robots":{"status":"ok","http":200,"url":"https://aiaa.org/robots.txt","bytes":704,"groups":7},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["partial","*",5,0],"OAI-SearchBot":["partial","*",5,0],"ClaudeBot":["partial","*",5,0],"Claude-User":["partial","*",5,0],"Claude-SearchBot":["partial","*",5,0],"anthropic-ai":["partial","*",5,0],"PerplexityBot":["partial","*",5,0],"Perplexity-User":["partial","*",5,0],"Google-Extended":["partial","*",5,0],"Applebot-Extended":["partial","*",5,0],"CCBot":["partial","*",5,0],"Bytespider":["partial","*",5,0],"Amazonbot":["partial","*",5,0],"meta-externalagent":["partial","*",5,0],"cohere-ai":["partial","*",5,0],"Diffbot":["partial","*",5,0],"ContextFetchBot":["partial","*",5,0]},"llms":{"status":"crawl_delay","error":"Crawl-delay 120 s"},"home":{"status":"crawl_delay","error":"Crawl-delay 120 s"}},
{"rank":6536,"domain":"tf1info.fr","origin":"https://tf1info.fr","robots":{"status":"ok","http":200,"url":"https://www.tf1info.fr/robots.txt","bytes":8284,"groups":4},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["blocked","named",1,0],"OAI-SearchBot":["partial","*",21,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["blocked","named",1,0],"Claude-SearchBot":["partial","*",21,0],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["partial","*",21,0],"Perplexity-User":["blocked","named",1,0],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",21,0]},"llms":{"status":"missing","http":404,"bytes":19187},"home":{"status":"ok","http":200,"url":"https://www.tf1info.fr/","html_tokens":443899,"markdown_tokens":834,"pruned_tokens":40}},
{"rank":6537,"domain":"hiroshima-u.ac.jp","origin":"https://www.hiroshima-u.ac.jp","robots":{"status":"ok","http":200,"url":"https://www.hiroshima-u.ac.jp/robots.txt","bytes":2141,"groups":1},"bots":{"GPTBot":["partial","*",35,22],"ChatGPT-User":["partial","*",35,22],"OAI-SearchBot":["partial","*",35,22],"ClaudeBot":["partial","*",35,22],"Claude-User":["partial","*",35,22],"Claude-SearchBot":["partial","*",35,22],"anthropic-ai":["partial","*",35,22],"PerplexityBot":["partial","*",35,22],"Perplexity-User":["partial","*",35,22],"Google-Extended":["partial","*",35,22],"Applebot-Extended":["partial","*",35,22],"CCBot":["partial","*",35,22],"Bytespider":["partial","*",35,22],"Amazonbot":["partial","*",35,22],"meta-externalagent":["partial","*",35,22],"cohere-ai":["partial","*",35,22],"Diffbot":["partial","*",35,22],"ContextFetchBot":["partial","*",35,22]},"llms":{"status":"missing","http":404,"bytes":195},"home":{"status":"ok","http":200,"url":"https://www.hiroshima-u.ac.jp/","html_tokens":17833,"markdown_tokens":3,"pruned_tokens":3}},
{"rank":6538,"domain":"isciii.es","origin":"https://isciii.es","robots":{"status":"server_error","http":500,"url":"https://isciii.es/robots.txt","bytes":27188,"groups":0,"error":"HTTP 500"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":6539,"domain":"technetbloggers.de","origin":"https://technetbloggers.de","robots":{"status":"ok","http":200,"url":"https://technetbloggers.de/robots.txt","bytes":94,"groups":1},"bots":{"GPTBot":["partial","*",2,0],"ChatGPT-User":["partial","*",2,0],"OAI-SearchBot":["partial","*",2,0],"ClaudeBot":["partial","*",2,0],"Claude-User":["partial","*",2,0],"Claude-SearchBot":["partial","*",2,0],"anthropic-ai":["partial","*",2,0],"PerplexityBot":["partial","*",2,0],"Perplexity-User":["partial","*",2,0],"Google-Extended":["partial","*",2,0],"Applebot-Extended":["partial","*",2,0],"CCBot":["partial","*",2,0],"Bytespider":["partial","*",2,0],"Amazonbot":["partial","*",2,0],"meta-externalagent":["partial","*",2,0],"cohere-ai":["partial","*",2,0],"Diffbot":["partial","*",2,0],"ContextFetchBot":["partial","*",2,0]},"llms":{"status":"missing","http":404,"bytes":15594},"home":{"status":"ok","http":200,"url":"https://technetbloggers.de/","html_tokens":8529,"markdown_tokens":10,"pruned_tokens":10}},
{"rank":6540,"domain":"jku.at","origin":"https://jku.at","robots":{"status":"ok","http":200,"url":"https://www.jku.at/robots.txt","bytes":1073,"groups":10},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["partial","*",14,0],"OAI-SearchBot":["partial","*",14,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",14,0],"Claude-SearchBot":["partial","*",14,0],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["partial","*",14,0],"Perplexity-User":["partial","*",14,0],"Google-Extended":["partial","*",14,0],"Applebot-Extended":["partial","*",14,0],"CCBot":["blocked","named",1,0],"Bytespider":["partial","*",14,0],"Amazonbot":["partial","*",14,0],"meta-externalagent":["partial","*",14,0],"cohere-ai":["partial","*",14,0],"Diffbot":["partial","*",14,0],"ContextFetchBot":["partial","*",14,0]},"llms":{"status":"missing","http":404,"bytes":173165},"home":{"status":"ok","http":200,"url":"https://www.jku.at/","html_tokens":53389,"markdown_tokens":326,"pruned_tokens":73}},
{"rank":6541,"domain":"gamblerzone.ca","origin":"https://gamblerzone.ca","robots":{"status":"ok","http":200,"url":"https://gamblerzone.ca/robots.txt","bytes":533,"groups":1},"bots":{"GPTBot":["partial","*",19,0],"ChatGPT-User":["partial","*",19,0],"OAI-SearchBot":["partial","*",19,0],"ClaudeBot":["partial","*",19,0],"Claude-User":["partial","*",19,0],"Claude-SearchBot":["partial","*",19,0],"anthropic-ai":["partial","*",19,0],"PerplexityBot":["partial","*",19,0],"Perplexity-User":["partial","*",19,0],"Google-Extended":["partial","*",19,0],"Applebot-Extended":["partial","*",19,0],"CCBot":["partial","*",19,0],"Bytespider":["partial","*",19,0],"Amazonbot":["partial","*",19,0],"meta-externalagent":["partial","*",19,0],"cohere-ai":["partial","*",19,0],"Diffbot":["partial","*",19,0],"ContextFetchBot":["partial","*",19,0]},"llms":{"status":"missing","http":404,"bytes":43105},"home":{"status":"ok","http":200,"url":"https://gamblerzone.ca/","html_tokens":24829,"markdown_tokens":1859,"pruned_tokens":82}},
{"rank":6542,"domain":"mchs.gov.ru","origin":"https://mchs.gov.ru","robots":{"status":"ok","http":200,"url":"https://mchs.gov.ru/robots.txt","bytes":292,"groups":1},"bots":{"GPTBot":["partial","*",9,1],"ChatGPT-User":["partial","*",9,1],"OAI-SearchBot":["partial","*",9,1],"ClaudeBot":["partial","*",9,1],"Claude-User":["partial","*",9,1],"Claude-SearchBot":["partial","*",9,1],"anthropic-ai":["partial","*",9,1],"PerplexityBot":["partial","*",9,1],"Perplexity-User":["partial","*",9,1],"Google-Extended":["partial","*",9,1],"Applebot-Extended":["partial","*",9,1],"CCBot":["partial","*",9,1],"Bytespider":["partial","*",9,1],"Amazonbot":["partial","*",9,1],"meta-externalagent":["partial","*",9,1],"cohere-ai":["partial","*",9,1],"Diffbot":["partial","*",9,1],"ContextFetchBot":["partial","*",9,1]},"llms":{"status":"missing","http":404,"bytes":82433},"home":{"status":"ok","http":200,"url":"https://mchs.gov.ru/","html_tokens":24777,"markdown_tokens":83,"pruned_tokens":83}},
{"rank":6543,"domain":"aawsat.com","origin":"https://aawsat.com","robots":{"status":"ok","http":200,"url":"https://aawsat.com/robots.txt","bytes":2016,"groups":1},"bots":{"GPTBot":["partial","*",28,18],"ChatGPT-User":["partial","*",28,18],"OAI-SearchBot":["partial","*",28,18],"ClaudeBot":["partial","*",28,18],"Claude-User":["partial","*",28,18],"Claude-SearchBot":["partial","*",28,18],"anthropic-ai":["partial","*",28,18],"PerplexityBot":["partial","*",28,18],"Perplexity-User":["partial","*",28,18],"Google-Extended":["partial","*",28,18],"Applebot-Extended":["partial","*",28,18],"CCBot":["partial","*",28,18],"Bytespider":["partial","*",28,18],"Amazonbot":["partial","*",28,18],"meta-externalagent":["partial","*",28,18],"cohere-ai":["partial","*",28,18],"Diffbot":["partial","*",28,18],"ContextFetchBot":["partial","*",28,18]},"llms":{"status":"missing","http":404,"bytes":652},"home":{"status":"ok","http":200,"url":"https://aawsat.com/","html_tokens":111733,"markdown_tokens":409,"pruned_tokens":160}},
{"rank":6544,"domain":"bpifrance.fr","origin":"https://bpifrance.fr","robots":{"status":"refused","http":403,"url":"https://www.bpifrance.fr/robots.txt","bytes":919,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":6545,"domain":"visitsealife.com","origin":"https://visitsealife.com","robots":{"status":"ok","http":200,"url":"https://visitsealife.com/robots.txt","bytes":84,"groups":1},"bots":{"GPTBot":["partial","*",1,0],"ChatGPT-User":["partial","*",1,0],"OAI-SearchBot":["partial","*",1,0],"ClaudeBot":["partial","*",1,0],"Claude-User":["partial","*",1,0],"Claude-SearchBot":["partial","*",1,0],"anthropic-ai":["partial","*",1,0],"PerplexityBot":["partial","*",1,0],"Perplexity-User":["partial","*",1,0],"Google-Extended":["partial","*",1,0],"Applebot-Extended":["partial","*",1,0],"CCBot":["partial","*",1,0],"Bytespider":["partial","*",1,0],"Amazonbot":["partial","*",1,0],"meta-externalagent":["partial","*",1,0],"cohere-ai":["partial","*",1,0],"Diffbot":["partial","*",1,0],"ContextFetchBot":["partial","*",1,0]},"llms":{"status":"missing","http":404,"bytes":0},"home":{"status":"ok","http":200,"url":"https://visitsealife.com/","html_tokens":29513,"markdown_tokens":21,"pruned_tokens":21}},
{"rank":6546,"domain":"themefreesia.com","origin":"https://themefreesia.com","robots":{"status":"ok","http":200,"url":"https://themefreesia.com/robots.txt","bytes":145,"groups":1},"bots":{"GPTBot":["partial","*",6,0],"ChatGPT-User":["partial","*",6,0],"OAI-SearchBot":["partial","*",6,0],"ClaudeBot":["partial","*",6,0],"Claude-User":["partial","*",6,0],"Claude-SearchBot":["partial","*",6,0],"anthropic-ai":["partial","*",6,0],"PerplexityBot":["partial","*",6,0],"Perplexity-User":["partial","*",6,0],"Google-Extended":["partial","*",6,0],"Applebot-Extended":["partial","*",6,0],"CCBot":["partial","*",6,0],"Bytespider":["partial","*",6,0],"Amazonbot":["partial","*",6,0],"meta-externalagent":["partial","*",6,0],"cohere-ai":["partial","*",6,0],"Diffbot":["partial","*",6,0],"ContextFetchBot":["partial","*",6,0]},"llms":{"status":"http_error","http":403,"bytes":5521},"home":{"status":"http_error","http":403,"url":"https://themefreesia.com/"}},
{"rank":6547,"domain":"carscoops.com","origin":"https://carscoops.com","robots":{"status":"ok","http":200,"url":"https://carscoops.com/robots.txt","bytes":209,"groups":1},"bots":{"GPTBot":["partial","*",4,0],"ChatGPT-User":["partial","*",4,0],"OAI-SearchBot":["partial","*",4,0],"ClaudeBot":["partial","*",4,0],"Claude-User":["partial","*",4,0],"Claude-SearchBot":["partial","*",4,0],"anthropic-ai":["partial","*",4,0],"PerplexityBot":["partial","*",4,0],"Perplexity-User":["partial","*",4,0],"Google-Extended":["partial","*",4,0],"Applebot-Extended":["partial","*",4,0],"CCBot":["partial","*",4,0],"Bytespider":["partial","*",4,0],"Amazonbot":["partial","*",4,0],"meta-externalagent":["partial","*",4,0],"cohere-ai":["partial","*",4,0],"Diffbot":["partial","*",4,0],"ContextFetchBot":["partial","*",4,0]},"llms":{"status":"html","http":200,"bytes":70675},"home":{"status":"ok","http":200,"url":"https://www.carscoops.com/","html_tokens":17669,"markdown_tokens":379,"pruned_tokens":165}},
{"rank":6548,"domain":"artofmanliness.com","origin":"https://artofmanliness.com","robots":{"status":"ok","http":200,"url":"https://www.artofmanliness.com/robots.txt","bytes":371,"groups":1},"bots":{"GPTBot":["partial","*",5,1],"ChatGPT-User":["partial","*",5,1],"OAI-SearchBot":["partial","*",5,1],"ClaudeBot":["partial","*",5,1],"Claude-User":["partial","*",5,1],"Claude-SearchBot":["partial","*",5,1],"anthropic-ai":["partial","*",5,1],"PerplexityBot":["partial","*",5,1],"Perplexity-User":["partial","*",5,1],"Google-Extended":["partial","*",5,1],"Applebot-Extended":["partial","*",5,1],"CCBot":["partial","*",5,1],"Bytespider":["partial","*",5,1],"Amazonbot":["partial","*",5,1],"meta-externalagent":["partial","*",5,1],"cohere-ai":["partial","*",5,1],"Diffbot":["partial","*",5,1],"ContextFetchBot":["partial","*",5,1]},"llms":{"status":"missing","http":404,"bytes":119940},"home":{"status":"ok","http":200,"url":"https://www.artofmanliness.com/","html_tokens":35182,"markdown_tokens":20,"pruned_tokens":20}},
{"rank":6549,"domain":"protonvpn.com","origin":"https://protonvpn.com","robots":{"status":"ok","http":200,"url":"https://protonvpn.com/robots.txt","bytes":86,"groups":1},"bots":{"GPTBot":["allowed","*",0,1],"ChatGPT-User":["allowed","*",0,1],"OAI-SearchBot":["allowed","*",0,1],"ClaudeBot":["allowed","*",0,1],"Claude-User":["allowed","*",0,1],"Claude-SearchBot":["allowed","*",0,1],"anthropic-ai":["allowed","*",0,1],"PerplexityBot":["allowed","*",0,1],"Perplexity-User":["allowed","*",0,1],"Google-Extended":["allowed","*",0,1],"Applebot-Extended":["allowed","*",0,1],"CCBot":["allowed","*",0,1],"Bytespider":["allowed","*",0,1],"Amazonbot":["allowed","*",0,1],"meta-externalagent":["allowed","*",0,1],"cohere-ai":["allowed","*",0,1],"Diffbot":["allowed","*",0,1],"ContextFetchBot":["allowed","*",0,1]},"llms":{"status":"missing","http":404,"bytes":124796},"home":{"status":"ok","http":200,"url":"https://protonvpn.com/","html_tokens":106588,"markdown_tokens":980,"pruned_tokens":233}},
{"rank":6550,"domain":"bmfsfj.de","origin":"https://bmfsfj.de","robots":{"status":"ok","http":200,"url":"https://www.bmbfsfj.bund.de/robots.txt","bytes":130,"groups":1},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"missing","http":404,"bytes":66507},"home":{"status":"ok","http":200,"url":"https://www.bmbfsfj.bund.de/","html_tokens":26109,"markdown_tokens":222,"pruned_tokens":161}},
{"rank":6551,"domain":"google.ci","origin":"https://google.ci","robots":{"status":"ok","http":200,"url":"https://www.google.ci/robots.txt","bytes":6624,"groups":4},"bots":{"GPTBot":["partial","*",169,64],"ChatGPT-User":["partial","*",169,64],"OAI-SearchBot":["partial","*",169,64],"ClaudeBot":["partial","*",169,64],"Claude-User":["partial","*",169,64],"Claude-SearchBot":["partial","*",169,64],"anthropic-ai":["partial","*",169,64],"PerplexityBot":["partial","*",169,64],"Perplexity-User":["partial","*",169,64],"Google-Extended":["partial","*",169,64],"Applebot-Extended":["partial","*",169,64],"CCBot":["partial","*",169,64],"Bytespider":["partial","*",169,64],"Amazonbot":["partial","*",169,64],"meta-externalagent":["partial","*",169,64],"cohere-ai":["partial","*",169,64],"Diffbot":["partial","*",169,64],"ContextFetchBot":["partial","*",169,64]},"llms":{"status":"missing","http":404,"bytes":1569},"home":{"status":"ok","http":200,"url":"https://www.google.com/","html_tokens":21455,"markdown_tokens":30,"pruned_tokens":30}},
{"rank":6552,"domain":"ebaumsworld.com","origin":"https://ebaumsworld.com","robots":{"status":"ok","http":200,"url":"https://www.ebaumsworld.com/robots.txt","bytes":1683,"groups":1},"bots":{"GPTBot":["partial","*",71,0],"ChatGPT-User":["partial","*",71,0],"OAI-SearchBot":["partial","*",71,0],"ClaudeBot":["partial","*",71,0],"Claude-User":["partial","*",71,0],"Claude-SearchBot":["partial","*",71,0],"anthropic-ai":["partial","*",71,0],"PerplexityBot":["partial","*",71,0],"Perplexity-User":["partial","*",71,0],"Google-Extended":["partial","*",71,0],"Applebot-Extended":["partial","*",71,0],"CCBot":["partial","*",71,0],"Bytespider":["partial","*",71,0],"Amazonbot":["partial","*",71,0],"meta-externalagent":["partial","*",71,0],"cohere-ai":["partial","*",71,0],"Diffbot":["partial","*",71,0],"ContextFetchBot":["partial","*",71,0]},"llms":{"status":"http_error","http":403,"bytes":919},"home":{"status":"http_error","http":403,"url":"https://ebaumsworld.com/"}},
{"rank":6553,"domain":"namu.wiki","origin":"https://namu.wiki","robots":{"status":"ok","http":200,"url":"https://namu.wiki/robots.txt","bytes":501,"groups":1},"bots":{"GPTBot":["partial","*",1,26],"ChatGPT-User":["partial","*",1,26],"OAI-SearchBot":["partial","*",1,26],"ClaudeBot":["partial","*",1,26],"Claude-User":["partial","*",1,26],"Claude-SearchBot":["partial","*",1,26],"anthropic-ai":["partial","*",1,26],"PerplexityBot":["partial","*",1,26],"Perplexity-User":["partial","*",1,26],"Google-Extended":["partial","*",1,26],"Applebot-Extended":["partial","*",1,26],"CCBot":["partial","*",1,26],"Bytespider":["partial","*",1,26],"Amazonbot":["partial","*",1,26],"meta-externalagent":["partial","*",1,26],"cohere-ai":["partial","*",1,26],"Diffbot":["partial","*",1,26],"ContextFetchBot":["partial","*",1,26]},"llms":{"status":"disallowed"},"home":{"status":"http_error","http":403,"url":"https://namu.wiki/"}},
{"rank":6554,"domain":"has-sante.fr","origin":"https://has-sante.fr","robots":{"status":"unreachable","http":null,"url":"https://has-sante.fr/robots.txt","bytes":0,"groups":0,"error":"[SSL: CERTIFICATE_VERIFY_FAILED] certificate verify failed: unable to get local issuer certificate (_ssl.c:1028)"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":6555,"domain":"prada.com","origin":"https://prada.com","robots":{"status":"unreachable","http":null,"url":"https://prada.com/robots.txt","bytes":0,"groups":0,"error":"<StreamReset stream_id:1, error_code:2, remote_reset:True>"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":6556,"domain":"format.com","origin":"https://format.com","robots":{"status":"ok","http":200,"url":"https://www.format.com/robots.txt","bytes":3780,"groups":7},"bots":{"GPTBot":["partial","named",32,3],"ChatGPT-User":["partial","*",44,3],"OAI-SearchBot":["partial","*",44,3],"ClaudeBot":["partial","named",32,3],"Claude-User":["partial","*",44,3],"Claude-SearchBot":["partial","*",44,3],"anthropic-ai":["partial","*",44,3],"PerplexityBot":["partial","*",44,3],"Perplexity-User":["partial","*",44,3],"Google-Extended":["partial","*",44,3],"Applebot-Extended":["blocked","named",1,0],"CCBot":["partial","named",32,3],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["partial","*",44,3],"Diffbot":["partial","*",44,3],"ContextFetchBot":["partial","*",44,3]},"llms":{"status":"other_text","http":200,"bytes":8385},"home":{"status":"ok","http":200,"url":"https://www.format.com/","html_tokens":134893,"markdown_tokens":1018,"pruned_tokens":190}},
{"rank":6557,"domain":"nspcc.org.uk","origin":"https://nspcc.org.uk","robots":{"status":"ok","http":200,"url":"https://www.nspcc.org.uk/robots.txt","bytes":124,"groups":1},"bots":{"GPTBot":["partial","*",3,0],"ChatGPT-User":["partial","*",3,0],"OAI-SearchBot":["partial","*",3,0],"ClaudeBot":["partial","*",3,0],"Claude-User":["partial","*",3,0],"Claude-SearchBot":["partial","*",3,0],"anthropic-ai":["partial","*",3,0],"PerplexityBot":["partial","*",3,0],"Perplexity-User":["partial","*",3,0],"Google-Extended":["partial","*",3,0],"Applebot-Extended":["partial","*",3,0],"CCBot":["partial","*",3,0],"Bytespider":["partial","*",3,0],"Amazonbot":["partial","*",3,0],"meta-externalagent":["partial","*",3,0],"cohere-ai":["partial","*",3,0],"Diffbot":["partial","*",3,0],"ContextFetchBot":["partial","*",3,0]},"llms":{"status":"missing","http":404,"bytes":0},"home":{"status":"http_error","http":403,"url":"https://www.nspcc.org.uk/"}},
{"rank":6558,"domain":"feedspot.com","origin":"https://feedspot.com","robots":{"status":"ok","http":200,"url":"https://www.feedspot.com/robots.txt","bytes":859,"groups":12},"bots":{"GPTBot":["partial","*",15,4],"ChatGPT-User":["partial","*",15,4],"OAI-SearchBot":["partial","*",15,4],"ClaudeBot":["partial","*",15,4],"Claude-User":["partial","*",15,4],"Claude-SearchBot":["partial","*",15,4],"anthropic-ai":["partial","*",15,4],"PerplexityBot":["partial","*",15,4],"Perplexity-User":["partial","*",15,4],"Google-Extended":["partial","*",15,4],"Applebot-Extended":["partial","*",15,4],"CCBot":["partial","*",15,4],"Bytespider":["partial","*",15,4],"Amazonbot":["partial","*",15,4],"meta-externalagent":["partial","*",15,4],"cohere-ai":["partial","*",15,4],"Diffbot":["partial","*",15,4],"ContextFetchBot":["partial","*",15,4]},"llms":{"status":"http_error","http":403,"bytes":5483},"home":{"status":"http_error","http":403,"url":"https://www.feedspot.com/"}},
{"rank":6559,"domain":"versobooks.com","origin":"https://versobooks.com","robots":{"status":"ok","http":200,"url":"https://www.versobooks.com/robots.txt","bytes":3668,"groups":2},"bots":{"GPTBot":["partial","*",40,23],"ChatGPT-User":["partial","*",40,23],"OAI-SearchBot":["partial","*",40,23],"ClaudeBot":["partial","*",40,23],"Claude-User":["partial","*",40,23],"Claude-SearchBot":["partial","*",40,23],"anthropic-ai":["partial","*",40,23],"PerplexityBot":["partial","*",40,23],"Perplexity-User":["partial","*",40,23],"Google-Extended":["partial","*",40,23],"Applebot-Extended":["partial","*",40,23],"CCBot":["partial","*",40,23],"Bytespider":["partial","*",40,23],"Amazonbot":["partial","*",40,23],"meta-externalagent":["partial","*",40,23],"cohere-ai":["partial","*",40,23],"Diffbot":["partial","*",40,23],"ContextFetchBot":["partial","*",40,23]},"llms":{"status":"present","http":200,"bytes":5726},"home":{"status":"ok","http":200,"url":"https://www.versobooks.com/","html_tokens":44841,"markdown_tokens":168,"pruned_tokens":92}},
{"rank":6560,"domain":"yell.com","origin":"https://yell.com","robots":{"status":"refused","http":403,"url":"https://yell.com/robots.txt","bytes":4541,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":6561,"domain":"venngage.com","origin":"https://venngage.com","robots":{"status":"ok","http":200,"url":"https://venngage.com/robots.txt","bytes":840,"groups":1},"bots":{"GPTBot":["partial","*",20,0],"ChatGPT-User":["partial","*",20,0],"OAI-SearchBot":["partial","*",20,0],"ClaudeBot":["partial","*",20,0],"Claude-User":["partial","*",20,0],"Claude-SearchBot":["partial","*",20,0],"anthropic-ai":["partial","*",20,0],"PerplexityBot":["partial","*",20,0],"Perplexity-User":["partial","*",20,0],"Google-Extended":["partial","*",20,0],"Applebot-Extended":["partial","*",20,0],"CCBot":["partial","*",20,0],"Bytespider":["partial","*",20,0],"Amazonbot":["partial","*",20,0],"meta-externalagent":["partial","*",20,0],"cohere-ai":["partial","*",20,0],"Diffbot":["partial","*",20,0],"ContextFetchBot":["partial","*",20,0]},"llms":{"status":"missing","http":404,"bytes":111615},"home":{"status":"ok","http":200,"url":"https://venngage.com/","html_tokens":120495,"markdown_tokens":1339,"pruned_tokens":267}},
{"rank":6562,"domain":"jqplot.com","origin":"http://jqplot.com","robots":{"status":"ok","http":200,"url":"http://www.jqplot.com/robots.txt","bytes":55,"groups":1},"bots":{"GPTBot":["partial","*",2,0],"ChatGPT-User":["partial","*",2,0],"OAI-SearchBot":["partial","*",2,0],"ClaudeBot":["partial","*",2,0],"Claude-User":["partial","*",2,0],"Claude-SearchBot":["partial","*",2,0],"anthropic-ai":["partial","*",2,0],"PerplexityBot":["partial","*",2,0],"Perplexity-User":["partial","*",2,0],"Google-Extended":["partial","*",2,0],"Applebot-Extended":["partial","*",2,0],"CCBot":["partial","*",2,0],"Bytespider":["partial","*",2,0],"Amazonbot":["partial","*",2,0],"meta-externalagent":["partial","*",2,0],"cohere-ai":["partial","*",2,0],"Diffbot":["partial","*",2,0],"ContextFetchBot":["partial","*",2,0]},"llms":{"status":"missing","http":404,"bytes":315},"home":{"status":"ok","http":200,"url":"http://www.jqplot.com/","html_tokens":1260,"markdown_tokens":281,"pruned_tokens":133}},
{"rank":6563,"domain":"moj.gov.cn","origin":"https://www.moj.gov.cn","robots":{"status":"missing","http":404,"url":"https://www.moj.gov.cn/robots.txt","bytes":1298,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":1298},"home":{"status":"ok","http":200,"url":"https://www.moj.gov.cn/","html_tokens":23027,"markdown_tokens":16,"pruned_tokens":16}},
{"rank":6564,"domain":"sizzling-hot-za-darmo.com","origin":"https://sizzling-hot-za-darmo.com","robots":{"status":"ok","http":200,"url":"https://sizzling-hot-za-darmo.com/robots.txt","bytes":554,"groups":1},"bots":{"GPTBot":["partial","*",19,0],"ChatGPT-User":["partial","*",19,0],"OAI-SearchBot":["partial","*",19,0],"ClaudeBot":["partial","*",19,0],"Claude-User":["partial","*",19,0],"Claude-SearchBot":["partial","*",19,0],"anthropic-ai":["partial","*",19,0],"PerplexityBot":["partial","*",19,0],"Perplexity-User":["partial","*",19,0],"Google-Extended":["partial","*",19,0],"Applebot-Extended":["partial","*",19,0],"CCBot":["partial","*",19,0],"Bytespider":["partial","*",19,0],"Amazonbot":["partial","*",19,0],"meta-externalagent":["partial","*",19,0],"cohere-ai":["partial","*",19,0],"Diffbot":["partial","*",19,0],"ContextFetchBot":["partial","*",19,0]},"llms":{"status":"missing","http":404,"bytes":15},"home":{"status":"http_error","http":404,"url":"https://sizzling-hot-za-darmo.com/"}},
{"rank":6565,"domain":"odn.ne.jp","origin":"https://odn.ne.jp","robots":{"status":"timeout","http":null,"url":"https://odn.ne.jp/robots.txt","bytes":0,"groups":0,"error":"ConnectTimeout"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":6566,"domain":"merchantcircle.com","origin":"https://merchantcircle.com","robots":{"status":"ok","http":200,"url":"https://www.merchantcircle.com/robots.txt","bytes":6939,"groups":18},"bots":{"GPTBot":["blocked","*",1,0],"ChatGPT-User":["blocked","*",1,0],"OAI-SearchBot":["blocked","*",1,0],"ClaudeBot":["blocked","*",1,0],"Claude-User":["blocked","*",1,0],"Claude-SearchBot":["blocked","*",1,0],"anthropic-ai":["blocked","*",1,0],"PerplexityBot":["blocked","*",1,0],"Perplexity-User":["blocked","*",1,0],"Google-Extended":["blocked","*",1,0],"Applebot-Extended":["blocked","*",1,0],"CCBot":["blocked","*",1,0],"Bytespider":["blocked","*",1,0],"Amazonbot":["blocked","*",1,0],"meta-externalagent":["blocked","*",1,0],"cohere-ai":["blocked","*",1,0],"Diffbot":["blocked","*",1,0],"ContextFetchBot":["blocked","*",1,0]},"llms":{"status":"disallowed"},"home":{"status":"disallowed"}},
{"rank":6567,"domain":"sch.gr","origin":"https://sch.gr","robots":{"status":"ok","http":200,"url":"https://sch.gr/robots.txt","bytes":2201,"groups":7},"bots":{"GPTBot":["partial","named",1,0],"ChatGPT-User":["partial","*",28,2],"OAI-SearchBot":["partial","*",28,2],"ClaudeBot":["partial","named",1,0],"Claude-User":["partial","*",28,2],"Claude-SearchBot":["partial","*",28,2],"anthropic-ai":["partial","*",28,2],"PerplexityBot":["partial","named",1,0],"Perplexity-User":["partial","*",28,2],"Google-Extended":["partial","*",28,2],"Applebot-Extended":["partial","*",28,2],"CCBot":["partial","named",1,0],"Bytespider":["partial","named",1,0],"Amazonbot":["partial","named",1,0],"meta-externalagent":["partial","*",28,2],"cohere-ai":["partial","*",28,2],"Diffbot":["partial","*",28,2],"ContextFetchBot":["partial","*",28,2]},"llms":{"status":"missing","http":404,"bytes":79896},"home":{"status":"ok","http":200,"url":"https://www.sch.gr/","html_tokens":48522,"markdown_tokens":112,"pruned_tokens":84}},
{"rank":6568,"domain":"allthingsd.com","origin":"https://allthingsd.com","robots":{"status":"refused","http":403,"url":"https://allthingsd.com/robots.txt","bytes":243,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":6569,"domain":"dlapiper.com","origin":"https://dlapiper.com","robots":{"status":"refused","http":429,"url":"https://dlapiper.com/robots.txt","bytes":34223,"groups":0,"error":"HTTP 429"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":6570,"domain":"euronext.com","origin":"https://euronext.com","robots":{"status":"ok","http":200,"url":"https://www.euronext.com/robots.txt","bytes":2432,"groups":1},"bots":{"GPTBot":["partial","*",32,18],"ChatGPT-User":["partial","*",32,18],"OAI-SearchBot":["partial","*",32,18],"ClaudeBot":["partial","*",32,18],"Claude-User":["partial","*",32,18],"Claude-SearchBot":["partial","*",32,18],"anthropic-ai":["partial","*",32,18],"PerplexityBot":["partial","*",32,18],"Perplexity-User":["partial","*",32,18],"Google-Extended":["partial","*",32,18],"Applebot-Extended":["partial","*",32,18],"CCBot":["partial","*",32,18],"Bytespider":["partial","*",32,18],"Amazonbot":["partial","*",32,18],"meta-externalagent":["partial","*",32,18],"cohere-ai":["partial","*",32,18],"Diffbot":["partial","*",32,18],"ContextFetchBot":["partial","*",32,18]},"llms":{"status":"missing","http":404,"bytes":190},"home":{"status":"ok","http":200,"url":"https://www.euronext.com/en","html_tokens":114266,"markdown_tokens":406,"pruned_tokens":222}},
{"rank":6571,"domain":"pulitzer.org","origin":"https://pulitzer.org","robots":{"status":"ok","http":200,"url":"https://www.pulitzer.org/robots.txt","bytes":2027,"groups":1},"bots":{"GPTBot":["partial","*",34,18],"ChatGPT-User":["partial","*",34,18],"OAI-SearchBot":["partial","*",34,18],"ClaudeBot":["partial","*",34,18],"Claude-User":["partial","*",34,18],"Claude-SearchBot":["partial","*",34,18],"anthropic-ai":["partial","*",34,18],"PerplexityBot":["partial","*",34,18],"Perplexity-User":["partial","*",34,18],"Google-Extended":["partial","*",34,18],"Applebot-Extended":["partial","*",34,18],"CCBot":["partial","*",34,18],"Bytespider":["partial","*",34,18],"Amazonbot":["partial","*",34,18],"meta-externalagent":["partial","*",34,18],"cohere-ai":["partial","*",34,18],"Diffbot":["partial","*",34,18],"ContextFetchBot":["partial","*",34,18]},"llms":{"status":"missing","http":404,"bytes":190},"home":{"status":"http_error","http":403,"url":"https://pulitzer.org/"}},
{"rank":6572,"domain":"56.com","origin":"https://56.com","robots":{"status":"html","http":200,"url":"https://www.56.com/","bytes":268306,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":6573,"domain":"upmc.com","origin":"https://upmc.com","robots":{"status":"ok","http":200,"url":"https://www.upmc.com/robots.txt","bytes":831,"groups":2},"bots":{"GPTBot":["partial","*",12,0],"ChatGPT-User":["partial","*",12,0],"OAI-SearchBot":["partial","*",12,0],"ClaudeBot":["partial","*",12,0],"Claude-User":["partial","*",12,0],"Claude-SearchBot":["partial","*",12,0],"anthropic-ai":["partial","*",12,0],"PerplexityBot":["partial","*",12,0],"Perplexity-User":["partial","*",12,0],"Google-Extended":["partial","*",12,0],"Applebot-Extended":["partial","*",12,0],"CCBot":["partial","*",12,0],"Bytespider":["partial","*",12,0],"Amazonbot":["partial","*",12,0],"meta-externalagent":["partial","*",12,0],"cohere-ai":["partial","*",12,0],"Diffbot":["partial","*",12,0],"ContextFetchBot":["partial","*",12,0]},"llms":{"status":"missing","http":404,"bytes":159928},"home":{"status":"ok","http":200,"url":"https://www.upmc.com/","html_tokens":43373,"markdown_tokens":209,"pruned_tokens":175}},
{"rank":6574,"domain":"eventbrite.com.au","origin":"https://eventbrite.com.au","robots":{"status":"ok","http":200,"url":"https://eventbrite.com.au/robots.txt","bytes":7565,"groups":135},"bots":{"GPTBot":["partial","*",51,2],"ChatGPT-User":["partial","*",51,2],"OAI-SearchBot":["partial","*",51,2],"ClaudeBot":["partial","*",51,2],"Claude-User":["partial","*",51,2],"Claude-SearchBot":["partial","*",51,2],"anthropic-ai":["partial","*",51,2],"PerplexityBot":["partial","*",51,2],"Perplexity-User":["partial","*",51,2],"Google-Extended":["partial","*",51,2],"Applebot-Extended":["partial","*",51,2],"CCBot":["blocked","named",1,0],"Bytespider":["partial","*",51,2],"Amazonbot":["partial","*",51,2],"meta-externalagent":["partial","*",51,2],"cohere-ai":["partial","*",51,2],"Diffbot":["partial","*",51,2],"ContextFetchBot":["partial","*",51,2]},"llms":{"status":"missing","http":404,"bytes":79367},"home":{"status":"ok","http":200,"url":"https://www.eventbrite.com.au","html_tokens":52436,"markdown_tokens":64,"pruned_tokens":64}},
{"rank":6575,"domain":"hotbot.com","origin":"https://hotbot.com","robots":{"status":"ok","http":200,"url":"https://www.hotbot.com/robots.txt","bytes":84,"groups":1},"bots":{"GPTBot":["partial","*",2,3],"ChatGPT-User":["partial","*",2,3],"OAI-SearchBot":["partial","*",2,3],"ClaudeBot":["partial","*",2,3],"Claude-User":["partial","*",2,3],"Claude-SearchBot":["partial","*",2,3],"anthropic-ai":["partial","*",2,3],"PerplexityBot":["partial","*",2,3],"Perplexity-User":["partial","*",2,3],"Google-Extended":["partial","*",2,3],"Applebot-Extended":["partial","*",2,3],"CCBot":["partial","*",2,3],"Bytespider":["partial","*",2,3],"Amazonbot":["partial","*",2,3],"meta-externalagent":["partial","*",2,3],"cohere-ai":["partial","*",2,3],"Diffbot":["partial","*",2,3],"ContextFetchBot":["partial","*",2,3]},"llms":{"status":"missing","http":404,"bytes":50628},"home":{"status":"ok","http":200,"url":"https://www.hotbot.com/","html_tokens":25272,"markdown_tokens":26,"pruned_tokens":26}},
{"rank":6576,"domain":"convertkit.com","origin":"https://convertkit.com","robots":{"status":"ok","http":200,"url":"https://kit.com/robots.txt","bytes":2720,"groups":1},"bots":{"GPTBot":["partial","*",5,0],"ChatGPT-User":["partial","*",5,0],"OAI-SearchBot":["partial","*",5,0],"ClaudeBot":["partial","*",5,0],"Claude-User":["partial","*",5,0],"Claude-SearchBot":["partial","*",5,0],"anthropic-ai":["partial","*",5,0],"PerplexityBot":["partial","*",5,0],"Perplexity-User":["partial","*",5,0],"Google-Extended":["partial","*",5,0],"Applebot-Extended":["partial","*",5,0],"CCBot":["partial","*",5,0],"Bytespider":["partial","*",5,0],"Amazonbot":["partial","*",5,0],"meta-externalagent":["partial","*",5,0],"cohere-ai":["partial","*",5,0],"Diffbot":["partial","*",5,0],"ContextFetchBot":["partial","*",5,0]},"llms":{"status":"missing","http":404,"bytes":101340},"home":{"status":"http_error","http":403,"url":"https://kit.com/"}},
{"rank":6577,"domain":"buff.ly","origin":"https://buff.ly","robots":{"status":"ok","http":200,"url":"https://buff.ly/robots.txt","bytes":61,"groups":1},"bots":{"GPTBot":["allowed","*",0,1],"ChatGPT-User":["allowed","*",0,1],"OAI-SearchBot":["allowed","*",0,1],"ClaudeBot":["allowed","*",0,1],"Claude-User":["allowed","*",0,1],"Claude-SearchBot":["allowed","*",0,1],"anthropic-ai":["allowed","*",0,1],"PerplexityBot":["allowed","*",0,1],"Perplexity-User":["allowed","*",0,1],"Google-Extended":["allowed","*",0,1],"Applebot-Extended":["allowed","*",0,1],"CCBot":["allowed","*",0,1],"Bytespider":["allowed","*",0,1],"Amazonbot":["allowed","*",0,1],"meta-externalagent":["allowed","*",0,1],"cohere-ai":["allowed","*",0,1],"Diffbot":["allowed","*",0,1],"ContextFetchBot":["allowed","*",0,1]},"llms":{"status":"html","http":200,"bytes":615341},"home":{"status":"ok","http":200,"url":"https://buffer.com/","html_tokens":153803,"markdown_tokens":576,"pruned_tokens":103}},
{"rank":6578,"domain":"vwthemes.com","origin":"https://vwthemes.com","robots":{"status":"ok","http":200,"url":"https://www.vwthemes.com/robots.txt","bytes":3753,"groups":2},"bots":{"GPTBot":["partial","*",40,23],"ChatGPT-User":["partial","*",40,23],"OAI-SearchBot":["partial","*",40,23],"ClaudeBot":["partial","*",40,23],"Claude-User":["partial","*",40,23],"Claude-SearchBot":["partial","*",40,23],"anthropic-ai":["partial","*",40,23],"PerplexityBot":["partial","*",40,23],"Perplexity-User":["partial","*",40,23],"Google-Extended":["partial","*",40,23],"Applebot-Extended":["partial","*",40,23],"CCBot":["partial","*",40,23],"Bytespider":["partial","*",40,23],"Amazonbot":["partial","*",40,23],"meta-externalagent":["partial","*",40,23],"cohere-ai":["partial","*",40,23],"Diffbot":["partial","*",40,23],"ContextFetchBot":["partial","*",40,23]},"llms":{"status":"present","http":200,"bytes":66519},"home":{"status":"ok","http":200,"url":"https://www.vwthemes.com/","html_tokens":192774,"markdown_tokens":51976,"pruned_tokens":602}},
{"rank":6579,"domain":"williamhill.com","origin":"https://williamhill.com","robots":{"status":"ok","http":200,"url":"https://www.williamhill.com/robots.txt","bytes":92,"groups":1},"bots":{"GPTBot":["partial","*",3,0],"ChatGPT-User":["partial","*",3,0],"OAI-SearchBot":["partial","*",3,0],"ClaudeBot":["partial","*",3,0],"Claude-User":["partial","*",3,0],"Claude-SearchBot":["partial","*",3,0],"anthropic-ai":["partial","*",3,0],"PerplexityBot":["partial","*",3,0],"Perplexity-User":["partial","*",3,0],"Google-Extended":["partial","*",3,0],"Applebot-Extended":["partial","*",3,0],"CCBot":["partial","*",3,0],"Bytespider":["partial","*",3,0],"Amazonbot":["partial","*",3,0],"meta-externalagent":["partial","*",3,0],"cohere-ai":["partial","*",3,0],"Diffbot":["partial","*",3,0],"ContextFetchBot":["partial","*",3,0]},"llms":{"status":"error","error":"more than 5 redirects"},"home":{"status":"ok","http":200,"url":"https://www.williamhill.com/","html_tokens":23322,"markdown_tokens":325,"pruned_tokens":152}},
{"rank":6580,"domain":"unb.br","origin":"https://unb.br","robots":{"status":"ok","http":200,"url":"https://unb.br/robots.txt","bytes":871,"groups":1},"bots":{"GPTBot":["partial","*",14,0],"ChatGPT-User":["partial","*",14,0],"OAI-SearchBot":["partial","*",14,0],"ClaudeBot":["partial","*",14,0],"Claude-User":["partial","*",14,0],"Claude-SearchBot":["partial","*",14,0],"anthropic-ai":["partial","*",14,0],"PerplexityBot":["partial","*",14,0],"Perplexity-User":["partial","*",14,0],"Google-Extended":["partial","*",14,0],"Applebot-Extended":["partial","*",14,0],"CCBot":["partial","*",14,0],"Bytespider":["partial","*",14,0],"Amazonbot":["partial","*",14,0],"meta-externalagent":["partial","*",14,0],"cohere-ai":["partial","*",14,0],"Diffbot":["partial","*",14,0],"ContextFetchBot":["partial","*",14,0]},"llms":{"status":"missing","http":404,"bytes":3789},"home":{"status":"ok","http":200,"url":"https://unb.br/","html_tokens":26621,"markdown_tokens":63,"pruned_tokens":18}},
{"rank":6581,"domain":"blogbright.net","origin":"https://blogbright.net","robots":{"status":"ok","http":200,"url":"https://blogbright.net/robots.txt","bytes":90,"groups":1},"bots":{"GPTBot":["partial","*",2,0],"ChatGPT-User":["partial","*",2,0],"OAI-SearchBot":["partial","*",2,0],"ClaudeBot":["partial","*",2,0],"Claude-User":["partial","*",2,0],"Claude-SearchBot":["partial","*",2,0],"anthropic-ai":["partial","*",2,0],"PerplexityBot":["partial","*",2,0],"Perplexity-User":["partial","*",2,0],"Google-Extended":["partial","*",2,0],"Applebot-Extended":["partial","*",2,0],"CCBot":["partial","*",2,0],"Bytespider":["partial","*",2,0],"Amazonbot":["partial","*",2,0],"meta-externalagent":["partial","*",2,0],"cohere-ai":["partial","*",2,0],"Diffbot":["partial","*",2,0],"ContextFetchBot":["partial","*",2,0]},"llms":{"status":"missing","http":404,"bytes":39792},"home":{"status":"ok","http":200,"url":"https://blogbright.net/","html_tokens":18542,"markdown_tokens":70,"pruned_tokens":70}},
{"rank":6582,"domain":"clickorlando.com","origin":"https://clickorlando.com","robots":{"status":"ok","http":200,"url":"https://www.clickorlando.com/robots.txt","bytes":1179,"groups":9},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["partial","*",5,0],"OAI-SearchBot":["partial","*",5,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",5,0],"Claude-SearchBot":["partial","*",5,0],"anthropic-ai":["partial","*",5,0],"PerplexityBot":["partial","*",5,0],"Perplexity-User":["partial","*",5,0],"Google-Extended":["partial","*",5,0],"Applebot-Extended":["partial","*",5,0],"CCBot":["partial","*",5,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["partial","*",5,0],"cohere-ai":["partial","*",5,0],"Diffbot":["partial","*",5,0],"ContextFetchBot":["partial","*",5,0]},"llms":{"status":"missing","http":404,"bytes":181910},"home":{"status":"ok","http":200,"url":"https://www.clickorlando.com/","html_tokens":273512,"markdown_tokens":7508,"pruned_tokens":124}},
{"rank":6583,"domain":"dnv.com","origin":"https://dnv.com","robots":{"status":"ok","http":200,"url":"https://www.dnv.com/robots.txt","bytes":3005,"groups":10},"bots":{"GPTBot":["partial","named",6,1],"ChatGPT-User":["partial","named",6,1],"OAI-SearchBot":["partial","named",6,1],"ClaudeBot":["partial","named",6,1],"Claude-User":["partial","named",6,1],"Claude-SearchBot":["partial","named",6,1],"anthropic-ai":["partial","*",6,1],"PerplexityBot":["partial","named",6,1],"Perplexity-User":["partial","*",6,1],"Google-Extended":["partial","named",6,1],"Applebot-Extended":["partial","named",6,1],"CCBot":["partial","*",6,1],"Bytespider":["partial","*",6,1],"Amazonbot":["partial","*",6,1],"meta-externalagent":["partial","*",6,1],"cohere-ai":["partial","*",6,1],"Diffbot":["partial","*",6,1],"ContextFetchBot":["partial","*",6,1]},"llms":{"status":"missing","http":404,"bytes":252112},"home":{"status":"ok","http":200,"url":"https://www.dnv.com/","html_tokens":53639,"markdown_tokens":126,"pruned_tokens":126}},
{"rank":6584,"domain":"imslp.org","origin":"https://imslp.org","robots":{"status":"ok","http":200,"url":"https://imslp.org/robots.txt","bytes":356,"groups":2},"bots":{"GPTBot":["partial","*",11,0],"ChatGPT-User":["partial","*",11,0],"OAI-SearchBot":["partial","*",11,0],"ClaudeBot":["partial","*",11,0],"Claude-User":["partial","*",11,0],"Claude-SearchBot":["partial","*",11,0],"anthropic-ai":["partial","*",11,0],"PerplexityBot":["partial","*",11,0],"Perplexity-User":["partial","*",11,0],"Google-Extended":["partial","*",11,0],"Applebot-Extended":["partial","*",11,0],"CCBot":["partial","*",11,0],"Bytespider":["partial","*",11,0],"Amazonbot":["partial","*",11,0],"meta-externalagent":["partial","*",11,0],"cohere-ai":["partial","*",11,0],"Diffbot":["partial","*",11,0],"ContextFetchBot":["partial","*",11,0]},"llms":{"status":"http_error","http":502,"bytes":166},"home":{"status":"ok","http":200,"url":"https://imslp.org/","html_tokens":10637,"markdown_tokens":220,"pruned_tokens":174}},
{"rank":6585,"domain":"morningconsult.com","origin":"https://morningconsult.com","robots":{"status":"ok","http":200,"url":"https://morningconsult.com/robots.txt","bytes":213,"groups":1},"bots":{"GPTBot":["partial","*",5,0],"ChatGPT-User":["partial","*",5,0],"OAI-SearchBot":["partial","*",5,0],"ClaudeBot":["partial","*",5,0],"Claude-User":["partial","*",5,0],"Claude-SearchBot":["partial","*",5,0],"anthropic-ai":["partial","*",5,0],"PerplexityBot":["partial","*",5,0],"Perplexity-User":["partial","*",5,0],"Google-Extended":["partial","*",5,0],"Applebot-Extended":["partial","*",5,0],"CCBot":["partial","*",5,0],"Bytespider":["partial","*",5,0],"Amazonbot":["partial","*",5,0],"meta-externalagent":["partial","*",5,0],"cohere-ai":["partial","*",5,0],"Diffbot":["partial","*",5,0],"ContextFetchBot":["partial","*",5,0]},"llms":{"status":"present","http":200,"bytes":10277},"home":{"status":"ok","http":200,"url":"https://morningconsult.com/","html_tokens":51924,"markdown_tokens":762,"pruned_tokens":106}},
{"rank":6586,"domain":"shimano.com","origin":"https://shimano.com","robots":{"status":"refused","http":403,"url":"https://www.shimano.com/robots.txt","bytes":383,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":6587,"domain":"animoto.com","origin":"https://animoto.com","robots":{"status":"refused","http":403,"url":"https://animoto.com/robots.txt","bytes":919,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":6588,"domain":"fujian.gov.cn","origin":"https://fujian.gov.cn","robots":{"status":"missing","http":404,"url":"https://fujian.gov.cn/robots.txt","bytes":3402,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":3402},"home":{"status":"ok","http":200,"url":"https://fujian.gov.cn/","html_tokens":97695,"markdown_tokens":78,"pruned_tokens":34}},
{"rank":6589,"domain":"tinder.com","origin":"https://tinder.com","robots":{"status":"ok","http":200,"url":"https://tinder.com/robots.txt","bytes":134,"groups":1},"bots":{"GPTBot":["partial","*",4,0],"ChatGPT-User":["partial","*",4,0],"OAI-SearchBot":["partial","*",4,0],"ClaudeBot":["partial","*",4,0],"Claude-User":["partial","*",4,0],"Claude-SearchBot":["partial","*",4,0],"anthropic-ai":["partial","*",4,0],"PerplexityBot":["partial","*",4,0],"Perplexity-User":["partial","*",4,0],"Google-Extended":["partial","*",4,0],"Applebot-Extended":["partial","*",4,0],"CCBot":["partial","*",4,0],"Bytespider":["partial","*",4,0],"Amazonbot":["partial","*",4,0],"meta-externalagent":["partial","*",4,0],"cohere-ai":["partial","*",4,0],"Diffbot":["partial","*",4,0],"ContextFetchBot":["partial","*",4,0]},"llms":{"status":"html","http":200,"bytes":556408},"home":{"status":"ok","http":200,"url":"https://tinder.com/","html_tokens":143298,"markdown_tokens":21,"pruned_tokens":21}},
{"rank":6590,"domain":"stdaily.com","origin":"https://stdaily.com","robots":{"status":"ok","http":200,"url":"https://www.stdaily.com/robots.txt","bytes":70,"groups":1},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"missing","http":404,"bytes":14872},"home":{"status":"ok","http":200,"url":"https://www.stdaily.com/","html_tokens":16893,"markdown_tokens":264,"pruned_tokens":172}},
{"rank":6591,"domain":"wantedly.com","origin":"https://wantedly.com","robots":{"status":"ok","http":200,"url":"https://www.wantedly.com/robots.txt","bytes":880,"groups":1},"bots":{"GPTBot":["partial","*",28,0],"ChatGPT-User":["partial","*",28,0],"OAI-SearchBot":["partial","*",28,0],"ClaudeBot":["partial","*",28,0],"Claude-User":["partial","*",28,0],"Claude-SearchBot":["partial","*",28,0],"anthropic-ai":["partial","*",28,0],"PerplexityBot":["partial","*",28,0],"Perplexity-User":["partial","*",28,0],"Google-Extended":["partial","*",28,0],"Applebot-Extended":["partial","*",28,0],"CCBot":["partial","*",28,0],"Bytespider":["partial","*",28,0],"Amazonbot":["partial","*",28,0],"meta-externalagent":["partial","*",28,0],"cohere-ai":["partial","*",28,0],"Diffbot":["partial","*",28,0],"ContextFetchBot":["partial","*",28,0]},"llms":{"status":"missing","http":404,"bytes":0},"home":{"status":"ok","http":200,"url":"https://www.wantedly.com/","html_tokens":7840,"markdown_tokens":31,"pruned_tokens":31}},
{"rank":6592,"domain":"arts.gov","origin":"https://arts.gov","robots":{"status":"ok","http":200,"url":"https://www.arts.gov/robots.txt","bytes":4558,"groups":7},"bots":{"GPTBot":["partial","*",71,18],"ChatGPT-User":["blocked","named",1,0],"OAI-SearchBot":["partial","*",71,18],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",71,18],"Claude-SearchBot":["partial","*",71,18],"anthropic-ai":["partial","*",71,18],"PerplexityBot":["partial","*",71,18],"Perplexity-User":["partial","*",71,18],"Google-Extended":["partial","*",71,18],"Applebot-Extended":["partial","*",71,18],"CCBot":["partial","*",71,18],"Bytespider":["partial","*",71,18],"Amazonbot":["partial","*",71,18],"meta-externalagent":["partial","*",71,18],"cohere-ai":["partial","*",71,18],"Diffbot":["partial","*",71,18],"ContextFetchBot":["partial","*",71,18]},"llms":{"status":"crawl_delay","error":"Crawl-delay 300 s"},"home":{"status":"crawl_delay","error":"Crawl-delay 300 s"}},
{"rank":6593,"domain":"gigazine.net","origin":"https://gigazine.net","robots":{"status":"ok","http":200,"url":"https://gigazine.net/robots.txt","bytes":2694,"groups":12},"bots":{"GPTBot":["partial","*",14,3],"ChatGPT-User":["partial","*",14,3],"OAI-SearchBot":["partial","*",14,3],"ClaudeBot":["partial","*",14,3],"Claude-User":["partial","*",14,3],"Claude-SearchBot":["partial","*",14,3],"anthropic-ai":["partial","*",14,3],"PerplexityBot":["partial","*",14,3],"Perplexity-User":["partial","*",14,3],"Google-Extended":["partial","*",14,3],"Applebot-Extended":["partial","*",14,3],"CCBot":["partial","*",14,3],"Bytespider":["partial","*",14,3],"Amazonbot":["partial","*",14,3],"meta-externalagent":["partial","*",14,3],"cohere-ai":["partial","*",14,3],"Diffbot":["partial","*",14,3],"ContextFetchBot":["partial","*",14,3]},"llms":{"status":"crawl_delay","error":"Crawl-delay 100 s"},"home":{"status":"crawl_delay","error":"Crawl-delay 100 s"}},
{"rank":6594,"domain":"mediadelivery.net","origin":"https://mediadelivery.net","robots":{"status":"unreachable","http":null,"url":"https://mediadelivery.net/robots.txt","bytes":0,"groups":0,"error":"cannot resolve mediadelivery.net"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":6595,"domain":"coupang.com","origin":"https://coupang.com","robots":{"status":"refused","http":403,"url":"https://www.coupang.com/robots.txt","bytes":383,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":6596,"domain":"npo.nl","origin":"https://npo.nl","robots":{"status":"ok","http":200,"url":"https://npo.nl/robots.txt","bytes":420,"groups":1},"bots":{"GPTBot":["partial","*",11,0],"ChatGPT-User":["partial","*",11,0],"OAI-SearchBot":["partial","*",11,0],"ClaudeBot":["partial","*",11,0],"Claude-User":["partial","*",11,0],"Claude-SearchBot":["partial","*",11,0],"anthropic-ai":["partial","*",11,0],"PerplexityBot":["partial","*",11,0],"Perplexity-User":["partial","*",11,0],"Google-Extended":["partial","*",11,0],"Applebot-Extended":["partial","*",11,0],"CCBot":["partial","*",11,0],"Bytespider":["partial","*",11,0],"Amazonbot":["partial","*",11,0],"meta-externalagent":["partial","*",11,0],"cohere-ai":["partial","*",11,0],"Diffbot":["partial","*",11,0],"ContextFetchBot":["partial","*",11,0]},"llms":{"status":"missing","http":404,"bytes":42016},"home":{"status":"ok","http":200,"url":"https://npo.nl/","html_tokens":212579,"markdown_tokens":5747,"pruned_tokens":135}},
{"rank":6597,"domain":"torontomu.ca","origin":"https://torontomu.ca","robots":{"status":"ok","http":200,"url":"https://www.torontomu.ca/robots.txt","bytes":4555,"groups":2},"bots":{"GPTBot":["partial","*",115,0],"ChatGPT-User":["partial","*",115,0],"OAI-SearchBot":["partial","*",115,0],"ClaudeBot":["partial","*",115,0],"Claude-User":["partial","*",115,0],"Claude-SearchBot":["partial","*",115,0],"anthropic-ai":["partial","*",115,0],"PerplexityBot":["partial","*",115,0],"Perplexity-User":["partial","*",115,0],"Google-Extended":["partial","*",115,0],"Applebot-Extended":["partial","*",115,0],"CCBot":["partial","*",115,0],"Bytespider":["partial","*",115,0],"Amazonbot":["partial","*",115,0],"meta-externalagent":["partial","*",115,0],"cohere-ai":["partial","*",115,0],"Diffbot":["partial","*",115,0],"ContextFetchBot":["partial","*",115,0]},"llms":{"status":"missing","http":404,"bytes":18946},"home":{"status":"ok","http":200,"url":"https://www.torontomu.ca/","html_tokens":17467,"markdown_tokens":197,"pruned_tokens":108}},
{"rank":6598,"domain":"radaronline.com","origin":"https://radaronline.com","robots":{"status":"ok","http":200,"url":"https://radaronline.com/robots.txt","bytes":3696,"groups":1},"bots":{"GPTBot":["partial","*",152,0],"ChatGPT-User":["partial","*",152,0],"OAI-SearchBot":["partial","*",152,0],"ClaudeBot":["partial","*",152,0],"Claude-User":["partial","*",152,0],"Claude-SearchBot":["partial","*",152,0],"anthropic-ai":["partial","*",152,0],"PerplexityBot":["partial","*",152,0],"Perplexity-User":["partial","*",152,0],"Google-Extended":["partial","*",152,0],"Applebot-Extended":["partial","*",152,0],"CCBot":["partial","*",152,0],"Bytespider":["partial","*",152,0],"Amazonbot":["partial","*",152,0],"meta-externalagent":["partial","*",152,0],"cohere-ai":["partial","*",152,0],"Diffbot":["partial","*",152,0],"ContextFetchBot":["partial","*",152,0]},"llms":{"status":"missing","http":404,"bytes":56105},"home":{"status":"ok","http":200,"url":"https://radaronline.com/","html_tokens":68474,"markdown_tokens":636,"pruned_tokens":96}},
{"rank":6599,"domain":"retailmenot.com","origin":"https://retailmenot.com","robots":{"status":"ok","http":200,"url":"https://www.retailmenot.com/robots.txt","bytes":1883,"groups":10},"bots":{"GPTBot":["partial","*",49,9],"ChatGPT-User":["partial","*",49,9],"OAI-SearchBot":["partial","*",49,9],"ClaudeBot":["partial","*",49,9],"Claude-User":["partial","*",49,9],"Claude-SearchBot":["partial","*",49,9],"anthropic-ai":["partial","*",49,9],"PerplexityBot":["partial","*",49,9],"Perplexity-User":["partial","*",49,9],"Google-Extended":["partial","*",49,9],"Applebot-Extended":["partial","*",49,9],"CCBot":["partial","*",49,9],"Bytespider":["partial","*",49,9],"Amazonbot":["partial","*",49,9],"meta-externalagent":["partial","*",49,9],"cohere-ai":["partial","*",49,9],"Diffbot":["partial","*",49,9],"ContextFetchBot":["partial","*",49,9]},"llms":{"status":"http_error","http":403,"bytes":5555},"home":{"status":"http_error","http":403,"url":"https://www.retailmenot.com/"}},
{"rank":6600,"domain":"musicradar.com","origin":"https://musicradar.com","robots":{"status":"ok","http":200,"url":"https://www.musicradar.com/robots.txt","bytes":1716,"groups":5},"bots":{"GPTBot":["partial","*",21,0],"ChatGPT-User":["partial","*",21,0],"OAI-SearchBot":["partial","*",21,0],"ClaudeBot":["partial","*",21,0],"Claude-User":["partial","*",21,0],"Claude-SearchBot":["partial","*",21,0],"anthropic-ai":["partial","*",21,0],"PerplexityBot":["partial","*",21,0],"Perplexity-User":["partial","*",21,0],"Google-Extended":["partial","*",21,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["partial","*",21,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["partial","*",21,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",21,0]},"llms":{"status":"missing","http":404,"bytes":1092093},"home":{"status":"ok","http":200,"url":"https://www.musicradar.com/","html_tokens":413614,"markdown_tokens":58,"pruned_tokens":58}},
{"rank":6601,"domain":"mathsisfun.com","origin":"https://mathsisfun.com","robots":{"status":"ok","http":200,"url":"https://mathsisfun.com/robots.txt","bytes":380,"groups":3},"bots":{"GPTBot":["partial","*",8,0],"ChatGPT-User":["partial","*",8,0],"OAI-SearchBot":["partial","*",8,0],"ClaudeBot":["partial","*",8,0],"Claude-User":["partial","*",8,0],"Claude-SearchBot":["partial","*",8,0],"anthropic-ai":["partial","*",8,0],"PerplexityBot":["partial","*",8,0],"Perplexity-User":["partial","*",8,0],"Google-Extended":["partial","*",8,0],"Applebot-Extended":["partial","*",8,0],"CCBot":["partial","*",8,0],"Bytespider":["partial","*",8,0],"Amazonbot":["partial","*",8,0],"meta-externalagent":["partial","*",8,0],"cohere-ai":["partial","*",8,0],"Diffbot":["partial","*",8,0],"ContextFetchBot":["partial","*",8,0]},"llms":{"status":"missing","http":404,"bytes":5269},"home":{"status":"ok","http":200,"url":"https://mathsisfun.com/","html_tokens":5650,"markdown_tokens":99,"pruned_tokens":18}},
{"rank":6602,"domain":"regex101.com","origin":"https://regex101.com","robots":{"status":"ok","http":200,"url":"https://regex101.com/robots.txt","bytes":95,"groups":1},"bots":{"GPTBot":["partial","*",1,0],"ChatGPT-User":["partial","*",1,0],"OAI-SearchBot":["partial","*",1,0],"ClaudeBot":["partial","*",1,0],"Claude-User":["partial","*",1,0],"Claude-SearchBot":["partial","*",1,0],"anthropic-ai":["partial","*",1,0],"PerplexityBot":["partial","*",1,0],"Perplexity-User":["partial","*",1,0],"Google-Extended":["partial","*",1,0],"Applebot-Extended":["partial","*",1,0],"CCBot":["partial","*",1,0],"Bytespider":["partial","*",1,0],"Amazonbot":["partial","*",1,0],"meta-externalagent":["partial","*",1,0],"cohere-ai":["partial","*",1,0],"Diffbot":["partial","*",1,0],"ContextFetchBot":["partial","*",1,0]},"llms":{"status":"present","http":200,"bytes":3442},"home":{"status":"ok","http":200,"url":"https://regex101.com/","html_tokens":16417,"markdown_tokens":44,"pruned_tokens":44}},
{"rank":6603,"domain":"betterhelp.com","origin":"https://betterhelp.com","robots":{"status":"ok","http":200,"url":"https://www.betterhelp.com/robots.txt","bytes":881,"groups":2},"bots":{"GPTBot":["partial","*",13,0],"ChatGPT-User":["partial","*",13,0],"OAI-SearchBot":["partial","*",13,0],"ClaudeBot":["partial","*",13,0],"Claude-User":["partial","*",13,0],"Claude-SearchBot":["partial","*",13,0],"anthropic-ai":["partial","*",13,0],"PerplexityBot":["partial","*",13,0],"Perplexity-User":["partial","*",13,0],"Google-Extended":["partial","*",13,0],"Applebot-Extended":["partial","*",13,0],"CCBot":["partial","*",13,0],"Bytespider":["partial","*",13,0],"Amazonbot":["partial","*",13,0],"meta-externalagent":["partial","*",13,0],"cohere-ai":["partial","*",13,0],"Diffbot":["partial","*",13,0],"ContextFetchBot":["partial","*",13,0]},"llms":{"status":"present","http":200,"bytes":7359},"home":{"status":"ok","http":200,"url":"https://www.betterhelp.com/","html_tokens":28047,"markdown_tokens":1231,"pruned_tokens":564}},
{"rank":6604,"domain":"poki.com","origin":"https://poki.com","robots":{"status":"ok","http":200,"url":"https://poki.com/robots.txt","bytes":1735,"groups":2},"bots":{"GPTBot":["partial","*",3,0],"ChatGPT-User":["partial","*",3,0],"OAI-SearchBot":["partial","*",3,0],"ClaudeBot":["partial","*",3,0],"Claude-User":["partial","*",3,0],"Claude-SearchBot":["partial","*",3,0],"anthropic-ai":["partial","*",3,0],"PerplexityBot":["partial","*",3,0],"Perplexity-User":["partial","*",3,0],"Google-Extended":["partial","*",3,0],"Applebot-Extended":["partial","*",3,0],"CCBot":["partial","*",3,0],"Bytespider":["partial","*",3,0],"Amazonbot":["partial","*",3,0],"meta-externalagent":["partial","*",3,0],"cohere-ai":["partial","*",3,0],"Diffbot":["partial","*",3,0],"ContextFetchBot":["partial","*",3,0]},"llms":{"status":"missing","http":404,"bytes":206214},"home":{"status":"ok","http":200,"url":"https://poki.com/","html_tokens":107554,"markdown_tokens":1177,"pruned_tokens":115}},
{"rank":6605,"domain":"iccsafe.org","origin":"https://iccsafe.org","robots":{"status":"ok","http":200,"url":"https://iccsafe.org/robots.txt","bytes":1083,"groups":6},"bots":{"GPTBot":["partial","*",11,1],"ChatGPT-User":["partial","*",11,1],"OAI-SearchBot":["partial","*",11,1],"ClaudeBot":["partial","*",11,1],"Claude-User":["partial","*",11,1],"Claude-SearchBot":["partial","*",11,1],"anthropic-ai":["partial","*",11,1],"PerplexityBot":["partial","*",11,1],"Perplexity-User":["partial","*",11,1],"Google-Extended":["partial","*",11,1],"Applebot-Extended":["partial","*",11,1],"CCBot":["partial","*",11,1],"Bytespider":["partial","*",11,1],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["partial","*",11,1],"cohere-ai":["partial","*",11,1],"Diffbot":["partial","*",11,1],"ContextFetchBot":["partial","*",11,1]},"llms":{"status":"missing","http":404,"bytes":168227},"home":{"status":"ok","http":200,"url":"https://www.iccsafe.org/","html_tokens":70126,"markdown_tokens":597,"pruned_tokens":154}},
{"rank":6606,"domain":"skyscanner.com","origin":"https://skyscanner.com","robots":{"status":"ok","http":200,"url":"https://www.skyscanner.com/robots.txt","bytes":10882,"groups":11},"bots":{"GPTBot":["partial","*",324,1],"ChatGPT-User":["partial","*",324,1],"OAI-SearchBot":["partial","*",324,1],"ClaudeBot":["partial","*",324,1],"Claude-User":["partial","*",324,1],"Claude-SearchBot":["partial","*",324,1],"anthropic-ai":["partial","*",324,1],"PerplexityBot":["partial","*",324,1],"Perplexity-User":["partial","*",324,1],"Google-Extended":["partial","*",324,1],"Applebot-Extended":["partial","*",324,1],"CCBot":["partial","*",324,1],"Bytespider":["partial","*",324,1],"Amazonbot":["partial","*",324,1],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["partial","*",324,1],"Diffbot":["partial","*",324,1],"ContextFetchBot":["partial","*",324,1]},"llms":{"status":"missing","http":404,"bytes":147},"home":{"status":"ok","http":200,"url":"https://www.skyscanner.com/","html_tokens":65931,"markdown_tokens":0,"pruned_tokens":0}},
{"rank":6607,"domain":"asic.gov.au","origin":"https://asic.gov.au","robots":{"status":"ok","http":200,"url":"https://www.asic.gov.au/robots.txt","bytes":59,"groups":1},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"missing","http":404,"bytes":65807},"home":{"status":"ok","http":200,"url":"https://www.asic.gov.au/","html_tokens":21505,"markdown_tokens":333,"pruned_tokens":175}},
{"rank":6608,"domain":"esy.es","origin":"https://esy.es","robots":{"status":"unreachable","http":null,"url":"https://esy.es/robots.txt","bytes":0,"groups":0,"error":"cannot resolve esy.es"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":6609,"domain":"belta.by","origin":"https://belta.by","robots":{"status":"ok","http":200,"url":"https://belta.by/robots.txt","bytes":2935,"groups":2},"bots":{"GPTBot":["partial","*",32,1],"ChatGPT-User":["partial","*",32,1],"OAI-SearchBot":["partial","*",32,1],"ClaudeBot":["partial","*",32,1],"Claude-User":["partial","*",32,1],"Claude-SearchBot":["partial","*",32,1],"anthropic-ai":["partial","*",32,1],"PerplexityBot":["partial","*",32,1],"Perplexity-User":["partial","*",32,1],"Google-Extended":["partial","*",32,1],"Applebot-Extended":["partial","*",32,1],"CCBot":["partial","*",32,1],"Bytespider":["partial","*",32,1],"Amazonbot":["partial","*",32,1],"meta-externalagent":["partial","*",32,1],"cohere-ai":["partial","*",32,1],"Diffbot":["partial","*",32,1],"ContextFetchBot":["partial","*",32,1]},"llms":{"status":"missing","http":404,"bytes":2946},"home":{"status":"ok","http":200,"url":"https://belta.by/","html_tokens":57771,"markdown_tokens":189,"pruned_tokens":44}},
{"rank":6610,"domain":"google.as","origin":"https://google.as","robots":{"status":"ok","http":200,"url":"https://www.google.as/robots.txt","bytes":6624,"groups":4},"bots":{"GPTBot":["partial","*",169,64],"ChatGPT-User":["partial","*",169,64],"OAI-SearchBot":["partial","*",169,64],"ClaudeBot":["partial","*",169,64],"Claude-User":["partial","*",169,64],"Claude-SearchBot":["partial","*",169,64],"anthropic-ai":["partial","*",169,64],"PerplexityBot":["partial","*",169,64],"Perplexity-User":["partial","*",169,64],"Google-Extended":["partial","*",169,64],"Applebot-Extended":["partial","*",169,64],"CCBot":["partial","*",169,64],"Bytespider":["partial","*",169,64],"Amazonbot":["partial","*",169,64],"meta-externalagent":["partial","*",169,64],"cohere-ai":["partial","*",169,64],"Diffbot":["partial","*",169,64],"ContextFetchBot":["partial","*",169,64]},"llms":{"status":"missing","http":404,"bytes":1569},"home":{"status":"ok","http":200,"url":"https://www.google.com/","html_tokens":21337,"markdown_tokens":30,"pruned_tokens":30}},
{"rank":6611,"domain":"wikinews.org","origin":"https://wikinews.org","robots":{"status":"ok","http":200,"url":"https://en.wikinews.org/robots.txt","bytes":17073,"groups":33},"bots":{"GPTBot":["partial","*",206,4],"ChatGPT-User":["partial","*",206,4],"OAI-SearchBot":["partial","*",206,4],"ClaudeBot":["partial","*",206,4],"Claude-User":["partial","*",206,4],"Claude-SearchBot":["partial","*",206,4],"anthropic-ai":["partial","*",206,4],"PerplexityBot":["partial","*",206,4],"Perplexity-User":["partial","*",206,4],"Google-Extended":["partial","*",206,4],"Applebot-Extended":["partial","*",206,4],"CCBot":["partial","*",206,4],"Bytespider":["partial","*",206,4],"Amazonbot":["partial","*",206,4],"meta-externalagent":["partial","*",206,4],"cohere-ai":["partial","*",206,4],"Diffbot":["partial","*",206,4],"ContextFetchBot":["partial","*",206,4]},"llms":{"status":"missing","http":404,"bytes":1868},"home":{"status":"ok","http":200,"url":"https://www.wikinews.org/","html_tokens":15233,"markdown_tokens":19,"pruned_tokens":20}},
{"rank":6612,"domain":"eventbrite.fr","origin":"https://eventbrite.fr","robots":{"status":"ok","http":200,"url":"https://eventbrite.fr/robots.txt","bytes":7499,"groups":135},"bots":{"GPTBot":["partial","*",51,2],"ChatGPT-User":["partial","*",51,2],"OAI-SearchBot":["partial","*",51,2],"ClaudeBot":["partial","*",51,2],"Claude-User":["partial","*",51,2],"Claude-SearchBot":["partial","*",51,2],"anthropic-ai":["partial","*",51,2],"PerplexityBot":["partial","*",51,2],"Perplexity-User":["partial","*",51,2],"Google-Extended":["partial","*",51,2],"Applebot-Extended":["partial","*",51,2],"CCBot":["blocked","named",1,0],"Bytespider":["partial","*",51,2],"Amazonbot":["partial","*",51,2],"meta-externalagent":["partial","*",51,2],"cohere-ai":["partial","*",51,2],"Diffbot":["partial","*",51,2],"ContextFetchBot":["partial","*",51,2]},"llms":{"status":"missing","http":404,"bytes":76982},"home":{"status":"ok","http":200,"url":"https://www.eventbrite.fr","html_tokens":53087,"markdown_tokens":66,"pruned_tokens":66}},
{"rank":6613,"domain":"adnkronos.com","origin":"https://www.adnkronos.com","robots":{"status":"ok","http":200,"url":"https://www.adnkronos.com/robots.txt","bytes":2069,"groups":2},"bots":{"GPTBot":["partial","*",29,0],"ChatGPT-User":["partial","*",29,0],"OAI-SearchBot":["partial","*",29,0],"ClaudeBot":["partial","*",29,0],"Claude-User":["partial","*",29,0],"Claude-SearchBot":["partial","*",29,0],"anthropic-ai":["partial","*",29,0],"PerplexityBot":["partial","*",29,0],"Perplexity-User":["partial","*",29,0],"Google-Extended":["partial","*",29,0],"Applebot-Extended":["partial","*",29,0],"CCBot":["partial","*",29,0],"Bytespider":["partial","*",29,0],"Amazonbot":["partial","*",29,0],"meta-externalagent":["partial","*",29,0],"cohere-ai":["partial","*",29,0],"Diffbot":["partial","*",29,0],"ContextFetchBot":["partial","*",29,0]},"llms":{"status":"missing","http":404,"bytes":113072},"home":{"status":"ok","http":200,"url":"https://www.adnkronos.com/","html_tokens":62108,"markdown_tokens":1412,"pruned_tokens":161}},
{"rank":6614,"domain":"campusfrance.org","origin":"https://campusfrance.org","robots":{"status":"ok","http":200,"url":"https://www.campusfrance.org/robots.txt","bytes":2027,"groups":1},"bots":{"GPTBot":["partial","*",34,18],"ChatGPT-User":["partial","*",34,18],"OAI-SearchBot":["partial","*",34,18],"ClaudeBot":["partial","*",34,18],"Claude-User":["partial","*",34,18],"Claude-SearchBot":["partial","*",34,18],"anthropic-ai":["partial","*",34,18],"PerplexityBot":["partial","*",34,18],"Perplexity-User":["partial","*",34,18],"Google-Extended":["partial","*",34,18],"Applebot-Extended":["partial","*",34,18],"CCBot":["partial","*",34,18],"Bytespider":["partial","*",34,18],"Amazonbot":["partial","*",34,18],"meta-externalagent":["partial","*",34,18],"cohere-ai":["partial","*",34,18],"Diffbot":["partial","*",34,18],"ContextFetchBot":["partial","*",34,18]},"llms":{"status":"missing","http":404,"bytes":194},"home":{"status":"ok","http":200,"url":"https://www.campusfrance.org/fr","html_tokens":46915,"markdown_tokens":316,"pruned_tokens":171}},
{"rank":6615,"domain":"pagesjaunes.fr","origin":"https://pagesjaunes.fr","robots":{"status":"ok","http":200,"url":"https://www.pagesjaunes.fr/robots.txt","bytes":3178,"groups":3},"bots":{"GPTBot":["partial","*",105,1],"ChatGPT-User":["partial","*",105,1],"OAI-SearchBot":["partial","*",105,1],"ClaudeBot":["partial","*",105,1],"Claude-User":["partial","*",105,1],"Claude-SearchBot":["partial","*",105,1],"anthropic-ai":["partial","*",105,1],"PerplexityBot":["partial","*",105,1],"Perplexity-User":["partial","*",105,1],"Google-Extended":["partial","*",105,1],"Applebot-Extended":["partial","*",105,1],"CCBot":["blocked","named",1,0],"Bytespider":["partial","*",105,1],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["partial","*",105,1],"cohere-ai":["partial","*",105,1],"Diffbot":["partial","*",105,1],"ContextFetchBot":["partial","*",105,1]},"llms":{"status":"http_error","http":403,"bytes":6133},"home":{"status":"http_error","http":403,"url":"https://pagesjaunes.fr/"}},
{"rank":6616,"domain":"elcorteingles.es","origin":"https://elcorteingles.es","robots":{"status":"refused","http":403,"url":"https://www.elcorteingles.es/robots.txt","bytes":386,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":6617,"domain":"7news.com.au","origin":"https://7news.com.au","robots":{"status":"ok","http":200,"url":"https://7news.com.au/robots.txt","bytes":392,"groups":4},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["blocked","named",2,0],"OAI-SearchBot":["partial","*",1,0],"ClaudeBot":["partial","*",1,0],"Claude-User":["partial","*",1,0],"Claude-SearchBot":["partial","*",1,0],"anthropic-ai":["partial","*",1,0],"PerplexityBot":["partial","*",1,0],"Perplexity-User":["partial","*",1,0],"Google-Extended":["partial","*",1,0],"Applebot-Extended":["partial","*",1,0],"CCBot":["partial","*",1,0],"Bytespider":["partial","*",1,0],"Amazonbot":["partial","*",1,0],"meta-externalagent":["partial","*",1,0],"cohere-ai":["partial","*",1,0],"Diffbot":["partial","*",1,0],"ContextFetchBot":["partial","*",1,0]},"llms":{"status":"missing","http":404,"bytes":319579},"home":{"status":"ok","http":200,"url":"https://7news.com.au/","html_tokens":257839,"markdown_tokens":991,"pruned_tokens":112}},
{"rank":6618,"domain":"fortra.com","origin":"https://fortra.com","robots":{"status":"refused","http":403,"url":"https://fortra.com/robots.txt","bytes":560,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":6619,"domain":"simplyrecipes.com","origin":"https://simplyrecipes.com","robots":{"status":"ok","http":200,"url":"https://www.simplyrecipes.com/robots.txt","bytes":3479,"groups":6},"bots":{"GPTBot":["partial","named",1,0],"ChatGPT-User":["partial","named",1,0],"OAI-SearchBot":["partial","named",1,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["blocked","named",1,0],"Claude-SearchBot":["partial","named",1,2],"anthropic-ai":["partial","named",1,2],"PerplexityBot":["partial","named",1,2],"Perplexity-User":["blocked","named",1,0],"Google-Extended":["partial","named",1,2],"Applebot-Extended":["blocked","named",1,0],"CCBot":["partial","named",1,2],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["partial","named",1,2],"cohere-ai":["partial","named",1,2],"Diffbot":["partial","*",2,0],"ContextFetchBot":["partial","*",2,0]},"llms":{"status":"http_error","http":402,"bytes":612},"home":{"status":"http_error","http":402,"url":"https://www.simplyrecipes.com/"}},
{"rank":6620,"domain":"richmond.com","origin":"https://richmond.com","robots":{"status":"ok","http":200,"url":"https://richmond.com/robots.txt","bytes":8886,"groups":1},"bots":{"GPTBot":["partial","*",155,0],"ChatGPT-User":["partial","*",155,0],"OAI-SearchBot":["partial","*",155,0],"ClaudeBot":["partial","*",155,0],"Claude-User":["partial","*",155,0],"Claude-SearchBot":["partial","*",155,0],"anthropic-ai":["partial","*",155,0],"PerplexityBot":["partial","*",155,0],"Perplexity-User":["partial","*",155,0],"Google-Extended":["partial","*",155,0],"Applebot-Extended":["partial","*",155,0],"CCBot":["partial","*",155,0],"Bytespider":["partial","*",155,0],"Amazonbot":["partial","*",155,0],"meta-externalagent":["partial","*",155,0],"cohere-ai":["partial","*",155,0],"Diffbot":["partial","*",155,0],"ContextFetchBot":["partial","*",155,0]},"llms":{"status":"http_error","http":429,"bytes":64},"home":{"status":"ok","http":200,"url":"https://richmond.com/","html_tokens":131860,"markdown_tokens":1172,"pruned_tokens":116}},
{"rank":6621,"domain":"aa.org","origin":"https://aa.org","robots":{"status":"ok","http":200,"url":"https://www.aa.org/robots.txt","bytes":2027,"groups":1},"bots":{"GPTBot":["partial","*",34,18],"ChatGPT-User":["partial","*",34,18],"OAI-SearchBot":["partial","*",34,18],"ClaudeBot":["partial","*",34,18],"Claude-User":["partial","*",34,18],"Claude-SearchBot":["partial","*",34,18],"anthropic-ai":["partial","*",34,18],"PerplexityBot":["partial","*",34,18],"Perplexity-User":["partial","*",34,18],"Google-Extended":["partial","*",34,18],"Applebot-Extended":["partial","*",34,18],"CCBot":["partial","*",34,18],"Bytespider":["partial","*",34,18],"Amazonbot":["partial","*",34,18],"meta-externalagent":["partial","*",34,18],"cohere-ai":["partial","*",34,18],"Diffbot":["partial","*",34,18],"ContextFetchBot":["partial","*",34,18]},"llms":{"status":"missing","http":404,"bytes":318},"home":{"status":"ok","http":200,"url":"https://www.aa.org/","html_tokens":20690,"markdown_tokens":149,"pruned_tokens":149}},
{"rank":6622,"domain":"greatist.com","origin":"https://greatist.com","robots":{"status":"ok","http":200,"url":"https://greatist.com/robots.txt","bytes":4405,"groups":29},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["partial","*",71,32],"OAI-SearchBot":["partial","*",71,32],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",71,32],"Claude-SearchBot":["partial","*",71,32],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["partial","*",71,32],"Perplexity-User":["partial","*",71,32],"Google-Extended":["partial","*",71,32],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["partial","*",71,32],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",71,32]},"llms":{"status":"missing","http":404,"bytes":87046},"home":{"status":"ok","http":200,"url":"https://greatist.com/","html_tokens":45946,"markdown_tokens":56,"pruned_tokens":56}},
{"rank":6623,"domain":"l-tike.com","origin":"https://l-tike.com","robots":{"status":"unreachable","http":null,"url":"https://l-tike.com/robots.txt","bytes":0,"groups":0,"error":"<StreamReset stream_id:1, error_code:2, remote_reset:True>"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":6624,"domain":"eleconomista.com.mx","origin":"https://eleconomista.com.mx","robots":{"status":"ok","http":200,"url":"https://www.eleconomista.com.mx/robots.txt","bytes":386,"groups":1},"bots":{"GPTBot":["partial","*",15,0],"ChatGPT-User":["partial","*",15,0],"OAI-SearchBot":["partial","*",15,0],"ClaudeBot":["partial","*",15,0],"Claude-User":["partial","*",15,0],"Claude-SearchBot":["partial","*",15,0],"anthropic-ai":["partial","*",15,0],"PerplexityBot":["partial","*",15,0],"Perplexity-User":["partial","*",15,0],"Google-Extended":["partial","*",15,0],"Applebot-Extended":["partial","*",15,0],"CCBot":["partial","*",15,0],"Bytespider":["partial","*",15,0],"Amazonbot":["partial","*",15,0],"meta-externalagent":["partial","*",15,0],"cohere-ai":["partial","*",15,0],"Diffbot":["partial","*",15,0],"ContextFetchBot":["partial","*",15,0]},"llms":{"status":"missing","http":404,"bytes":120145},"home":{"status":"ok","http":200,"url":"https://www.eleconomista.com.mx/","html_tokens":88804,"markdown_tokens":18,"pruned_tokens":18}},
{"rank":6625,"domain":"youcanbook.me","origin":"https://youcanbook.me","robots":{"status":"ok","http":200,"url":"https://youcanbook.me/robots.txt","bytes":165,"groups":1},"bots":{"GPTBot":["partial","*",5,0],"ChatGPT-User":["partial","*",5,0],"OAI-SearchBot":["partial","*",5,0],"ClaudeBot":["partial","*",5,0],"Claude-User":["partial","*",5,0],"Claude-SearchBot":["partial","*",5,0],"anthropic-ai":["partial","*",5,0],"PerplexityBot":["partial","*",5,0],"Perplexity-User":["partial","*",5,0],"Google-Extended":["partial","*",5,0],"Applebot-Extended":["partial","*",5,0],"CCBot":["partial","*",5,0],"Bytespider":["partial","*",5,0],"Amazonbot":["partial","*",5,0],"meta-externalagent":["partial","*",5,0],"cohere-ai":["partial","*",5,0],"Diffbot":["partial","*",5,0],"ContextFetchBot":["partial","*",5,0]},"llms":{"status":"present","http":200,"bytes":16339},"home":{"status":"ok","http":200,"url":"https://youcanbook.me/","html_tokens":66190,"markdown_tokens":1129,"pruned_tokens":114}},
{"rank":6626,"domain":"bjnews.com.cn","origin":"https://www.bjnews.com.cn","robots":{"status":"ok","http":200,"url":"https://www.bjnews.com.cn/robots.txt","bytes":28,"groups":1},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"missing","http":404,"bytes":1567},"home":{"status":"ok","http":200,"url":"https://www.bjnews.com.cn/","html_tokens":41448,"markdown_tokens":276,"pruned_tokens":131}},
{"rank":6627,"domain":"act.gov.au","origin":"https://act.gov.au","robots":{"status":"refused","http":403,"url":"https://act.gov.au/robots.txt","bytes":5505,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":6628,"domain":"filmaffinity.com","origin":"https://filmaffinity.com","robots":{"status":"refused","http":403,"url":"https://filmaffinity.com/robots.txt","bytes":5575,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":6629,"domain":"kotlinlang.org","origin":"https://kotlinlang.org","robots":{"status":"ok","http":200,"url":"https://kotlinlang.org/robots.txt","bytes":97,"groups":1},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"present","http":200,"bytes":49198},"home":{"status":"ok","http":200,"url":"https://kotlinlang.org/","html_tokens":18360,"markdown_tokens":387,"pruned_tokens":74}},
{"rank":6630,"domain":"joga-casino.com","origin":"https://joga-casino.com","robots":{"status":"ok","http":200,"url":"https://joga-casino.com/robots.txt","bytes":534,"groups":1},"bots":{"GPTBot":["partial","*",19,0],"ChatGPT-User":["partial","*",19,0],"OAI-SearchBot":["partial","*",19,0],"ClaudeBot":["partial","*",19,0],"Claude-User":["partial","*",19,0],"Claude-SearchBot":["partial","*",19,0],"anthropic-ai":["partial","*",19,0],"PerplexityBot":["partial","*",19,0],"Perplexity-User":["partial","*",19,0],"Google-Extended":["partial","*",19,0],"Applebot-Extended":["partial","*",19,0],"CCBot":["partial","*",19,0],"Bytespider":["partial","*",19,0],"Amazonbot":["partial","*",19,0],"meta-externalagent":["partial","*",19,0],"cohere-ai":["partial","*",19,0],"Diffbot":["partial","*",19,0],"ContextFetchBot":["partial","*",19,0]},"llms":{"status":"missing","http":404,"bytes":28892},"home":{"status":"ok","http":200,"url":"https://joga-casino.com/","html_tokens":26093,"markdown_tokens":2477,"pruned_tokens":116}},
{"rank":6631,"domain":"tno.nl","origin":"https://tno.nl","robots":{"status":"ok","http":200,"url":"https://www.tno.nl/robots.txt","bytes":278,"groups":1},"bots":{"GPTBot":["partial","*",6,1],"ChatGPT-User":["partial","*",6,1],"OAI-SearchBot":["partial","*",6,1],"ClaudeBot":["partial","*",6,1],"Claude-User":["partial","*",6,1],"Claude-SearchBot":["partial","*",6,1],"anthropic-ai":["partial","*",6,1],"PerplexityBot":["partial","*",6,1],"Perplexity-User":["partial","*",6,1],"Google-Extended":["partial","*",6,1],"Applebot-Extended":["partial","*",6,1],"CCBot":["partial","*",6,1],"Bytespider":["partial","*",6,1],"Amazonbot":["partial","*",6,1],"meta-externalagent":["partial","*",6,1],"cohere-ai":["partial","*",6,1],"Diffbot":["partial","*",6,1],"ContextFetchBot":["partial","*",6,1]},"llms":{"status":"present","http":200,"bytes":19837},"home":{"status":"ok","http":200,"url":"https://www.tno.nl/nl","html_tokens":34241,"markdown_tokens":104,"pruned_tokens":62}},
{"rank":6632,"domain":"seeedstudio.com","origin":"https://seeedstudio.com","robots":{"status":"ok","http":200,"url":"https://www.seeedstudio.com/robots.txt","bytes":2991,"groups":3},"bots":{"GPTBot":["partial","named",2,2],"ChatGPT-User":["partial","*",102,2],"OAI-SearchBot":["partial","*",102,2],"ClaudeBot":["partial","*",102,2],"Claude-User":["partial","*",102,2],"Claude-SearchBot":["partial","*",102,2],"anthropic-ai":["partial","*",102,2],"PerplexityBot":["partial","*",102,2],"Perplexity-User":["partial","*",102,2],"Google-Extended":["partial","*",102,2],"Applebot-Extended":["partial","*",102,2],"CCBot":["partial","*",102,2],"Bytespider":["partial","*",102,2],"Amazonbot":["partial","*",102,2],"meta-externalagent":["partial","*",102,2],"cohere-ai":["partial","*",102,2],"Diffbot":["partial","*",102,2],"ContextFetchBot":["partial","*",102,2]},"llms":{"status":"missing","http":404,"bytes":538752},"home":{"status":"ok","http":200,"url":"https://www.seeedstudio.com/","html_tokens":77242,"markdown_tokens":14,"pruned_tokens":14}},
{"rank":6633,"domain":"oprahdaily.com","origin":"https://oprahdaily.com","robots":{"status":"ok","http":200,"url":"https://oprahdaily.com/robots.txt","bytes":965,"groups":3},"bots":{"GPTBot":["partial","named",32,0],"ChatGPT-User":["partial","named",32,0],"OAI-SearchBot":["partial","named",32,0],"ClaudeBot":["partial","*",19,0],"Claude-User":["partial","*",19,0],"Claude-SearchBot":["partial","*",19,0],"anthropic-ai":["partial","*",19,0],"PerplexityBot":["partial","*",19,0],"Perplexity-User":["partial","*",19,0],"Google-Extended":["partial","*",19,0],"Applebot-Extended":["partial","*",19,0],"CCBot":["partial","*",19,0],"Bytespider":["partial","*",19,0],"Amazonbot":["partial","*",19,0],"meta-externalagent":["partial","*",19,0],"cohere-ai":["partial","*",19,0],"Diffbot":["partial","*",19,0],"ContextFetchBot":["partial","*",19,0]},"llms":{"status":"missing","http":404,"bytes":84},"home":{"status":"ok","http":200,"url":"https://www.oprahdaily.com/","html_tokens":174745,"markdown_tokens":30,"pruned_tokens":30}},
{"rank":6634,"domain":"momentjs.com","origin":"https://momentjs.com","robots":{"status":"missing","http":404,"url":"https://momentjs.com/robots.txt","bytes":9379,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":9379},"home":{"status":"ok","http":200,"url":"https://momentjs.com/","html_tokens":5278,"markdown_tokens":271,"pruned_tokens":129}},
{"rank":6635,"domain":"openresty.com","origin":"https://openresty.com","robots":{"status":"ok","http":200,"url":"https://openresty.com/robots.txt","bytes":221,"groups":1},"bots":{"GPTBot":["partial","*",1,0],"ChatGPT-User":["partial","*",1,0],"OAI-SearchBot":["partial","*",1,0],"ClaudeBot":["partial","*",1,0],"Claude-User":["partial","*",1,0],"Claude-SearchBot":["partial","*",1,0],"anthropic-ai":["partial","*",1,0],"PerplexityBot":["partial","*",1,0],"Perplexity-User":["partial","*",1,0],"Google-Extended":["partial","*",1,0],"Applebot-Extended":["partial","*",1,0],"CCBot":["partial","*",1,0],"Bytespider":["partial","*",1,0],"Amazonbot":["partial","*",1,0],"meta-externalagent":["partial","*",1,0],"cohere-ai":["partial","*",1,0],"Diffbot":["partial","*",1,0],"ContextFetchBot":["partial","*",1,0]},"llms":{"status":"missing","http":404,"bytes":12890},"home":{"status":"ok","http":200,"url":"https://openresty.com/en/","html_tokens":7463,"markdown_tokens":80,"pruned_tokens":37}},
{"rank":6636,"domain":"teespring.com","origin":"https://teespring.com","robots":{"status":"refused","http":403,"url":"https://teespring.com/robots.txt","bytes":4907,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":6637,"domain":"loccitane.com","origin":"https://www.loccitane.com","robots":{"status":"ok","http":200,"url":"https://www.loccitane.com/robots.txt","bytes":1581,"groups":1},"bots":{"GPTBot":["partial","*",25,12],"ChatGPT-User":["partial","*",25,12],"OAI-SearchBot":["partial","*",25,12],"ClaudeBot":["partial","*",25,12],"Claude-User":["partial","*",25,12],"Claude-SearchBot":["partial","*",25,12],"anthropic-ai":["partial","*",25,12],"PerplexityBot":["partial","*",25,12],"Perplexity-User":["partial","*",25,12],"Google-Extended":["partial","*",25,12],"Applebot-Extended":["partial","*",25,12],"CCBot":["partial","*",25,12],"Bytespider":["partial","*",25,12],"Amazonbot":["partial","*",25,12],"meta-externalagent":["partial","*",25,12],"cohere-ai":["partial","*",25,12],"Diffbot":["partial","*",25,12],"ContextFetchBot":["partial","*",25,12]},"llms":{"status":"missing","http":410,"bytes":99426},"home":{"status":"ok","http":200,"url":"https://www.loccitane.com/","html_tokens":25181,"markdown_tokens":9,"pruned_tokens":9}},
{"rank":6638,"domain":"subscribe.ru","origin":"https://subscribe.ru","robots":{"status":"ok","http":200,"url":"https://subscribe.ru/robots.txt","bytes":265,"groups":2},"bots":{"GPTBot":["partial","*",9,0],"ChatGPT-User":["partial","*",9,0],"OAI-SearchBot":["partial","*",9,0],"ClaudeBot":["partial","*",9,0],"Claude-User":["partial","*",9,0],"Claude-SearchBot":["partial","*",9,0],"anthropic-ai":["partial","*",9,0],"PerplexityBot":["partial","*",9,0],"Perplexity-User":["partial","*",9,0],"Google-Extended":["partial","*",9,0],"Applebot-Extended":["partial","*",9,0],"CCBot":["partial","*",9,0],"Bytespider":["partial","*",9,0],"Amazonbot":["partial","*",9,0],"meta-externalagent":["partial","*",9,0],"cohere-ai":["partial","*",9,0],"Diffbot":["partial","*",9,0],"ContextFetchBot":["partial","*",9,0]},"llms":{"status":"missing","http":404,"bytes":13281},"home":{"status":"ok","http":200,"url":"https://subscribe.ru/","html_tokens":31720,"markdown_tokens":1021,"pruned_tokens":149}},
{"rank":6639,"domain":"lululemon.com","origin":"https://lululemon.com","robots":{"status":"timeout","http":null,"url":"https://lululemon.com/robots.txt","bytes":0,"groups":0,"error":"timed out after 10 s"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":6640,"domain":"svoboda.org","origin":"https://svoboda.org","robots":{"status":"ok","http":200,"url":"https://www.svoboda.org/robots.txt","bytes":473,"groups":2},"bots":{"GPTBot":["partial","*",16,0],"ChatGPT-User":["partial","*",16,0],"OAI-SearchBot":["partial","*",16,0],"ClaudeBot":["partial","*",16,0],"Claude-User":["partial","*",16,0],"Claude-SearchBot":["partial","*",16,0],"anthropic-ai":["partial","*",16,0],"PerplexityBot":["partial","*",16,0],"Perplexity-User":["partial","*",16,0],"Google-Extended":["partial","*",16,0],"Applebot-Extended":["partial","*",16,0],"CCBot":["partial","*",16,0],"Bytespider":["partial","*",16,0],"Amazonbot":["partial","*",16,0],"meta-externalagent":["partial","*",16,0],"cohere-ai":["partial","*",16,0],"Diffbot":["partial","*",16,0],"ContextFetchBot":["partial","*",16,0]},"llms":{"status":"missing","http":404,"bytes":34799},"home":{"status":"ok","http":200,"url":"https://www.svoboda.org/","html_tokens":34082,"markdown_tokens":24,"pruned_tokens":24}},
{"rank":6641,"domain":"themezhut.com","origin":"https://themezhut.com","robots":{"status":"ok","http":200,"url":"https://themezhut.com/robots.txt","bytes":171,"groups":1},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"missing","http":404,"bytes":60641},"home":{"status":"ok","http":200,"url":"https://themezhut.com/","html_tokens":17793,"markdown_tokens":99,"pruned_tokens":99}},
{"rank":6642,"domain":"etix.com","origin":"https://etix.com","robots":{"status":"ok","http":200,"url":"https://etix.com/robots.txt","bytes":2772,"groups":1},"bots":{"GPTBot":["partial","*",48,24],"ChatGPT-User":["partial","*",48,24],"OAI-SearchBot":["partial","*",48,24],"ClaudeBot":["partial","*",48,24],"Claude-User":["partial","*",48,24],"Claude-SearchBot":["partial","*",48,24],"anthropic-ai":["partial","*",48,24],"PerplexityBot":["partial","*",48,24],"Perplexity-User":["partial","*",48,24],"Google-Extended":["partial","*",48,24],"Applebot-Extended":["partial","*",48,24],"CCBot":["partial","*",48,24],"Bytespider":["partial","*",48,24],"Amazonbot":["partial","*",48,24],"meta-externalagent":["partial","*",48,24],"cohere-ai":["partial","*",48,24],"Diffbot":["partial","*",48,24],"ContextFetchBot":["partial","*",48,24]},"llms":{"status":"missing","http":404,"bytes":196},"home":{"status":"ok","http":202,"url":"https://etix.com/ticket/","html_tokens":505,"markdown_tokens":40,"pruned_tokens":40}},
{"rank":6643,"domain":"ketodermic.org","origin":"https://ketodermic.org","robots":{"status":"ok","http":200,"url":"https://ketodermic.org/robots.txt","bytes":75,"groups":1},"bots":{"GPTBot":["partial","*",1,0],"ChatGPT-User":["partial","*",1,0],"OAI-SearchBot":["partial","*",1,0],"ClaudeBot":["partial","*",1,0],"Claude-User":["partial","*",1,0],"Claude-SearchBot":["partial","*",1,0],"anthropic-ai":["partial","*",1,0],"PerplexityBot":["partial","*",1,0],"Perplexity-User":["partial","*",1,0],"Google-Extended":["partial","*",1,0],"Applebot-Extended":["partial","*",1,0],"CCBot":["partial","*",1,0],"Bytespider":["partial","*",1,0],"Amazonbot":["partial","*",1,0],"meta-externalagent":["partial","*",1,0],"cohere-ai":["partial","*",1,0],"Diffbot":["partial","*",1,0],"ContextFetchBot":["partial","*",1,0]},"llms":{"status":"missing","http":404,"bytes":643},"home":{"status":"ok","http":200,"url":"https://ketodermic.org/","html_tokens":13180,"markdown_tokens":3146,"pruned_tokens":154}},
{"rank":6644,"domain":"hoover.org","origin":"https://hoover.org","robots":{"status":"refused","http":403,"url":"https://hoover.org/robots.txt","bytes":560,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":6645,"domain":"tiffany.com","origin":"https://tiffany.com","robots":{"status":"refused","http":403,"url":"https://tiffany.com/robots.txt","bytes":375,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":6646,"domain":"ksu.edu","origin":"https://ksu.edu","robots":{"status":"ok","http":200,"url":"https://www.k-state.edu/robots.txt","bytes":4170,"groups":1},"bots":{"GPTBot":["partial","*",101,0],"ChatGPT-User":["partial","*",101,0],"OAI-SearchBot":["partial","*",101,0],"ClaudeBot":["partial","*",101,0],"Claude-User":["partial","*",101,0],"Claude-SearchBot":["partial","*",101,0],"anthropic-ai":["partial","*",101,0],"PerplexityBot":["partial","*",101,0],"Perplexity-User":["partial","*",101,0],"Google-Extended":["partial","*",101,0],"Applebot-Extended":["partial","*",101,0],"CCBot":["partial","*",101,0],"Bytespider":["partial","*",101,0],"Amazonbot":["partial","*",101,0],"meta-externalagent":["partial","*",101,0],"cohere-ai":["partial","*",101,0],"Diffbot":["partial","*",101,0],"ContextFetchBot":["partial","*",101,0]},"llms":{"status":"missing","http":404,"bytes":42112},"home":{"status":"ok","http":200,"url":"https://www.k-state.edu/","html_tokens":20048,"markdown_tokens":725,"pruned_tokens":160}},
{"rank":6647,"domain":"koreanair.com","origin":"https://koreanair.com","robots":{"status":"unreachable","http":null,"url":"https://koreanair.com/robots.txt","bytes":0,"groups":0,"error":"<StreamReset stream_id:1, error_code:2, remote_reset:True>"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":6648,"domain":"fireeye.com","origin":"https://fireeye.com","robots":{"status":"server_error","http":530,"url":"https://www.fireeye.com/robots.txt","bytes":8241,"groups":0,"error":"HTTP 530"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":6649,"domain":"google.cat","origin":"https://google.cat","robots":{"status":"ok","http":200,"url":"https://www.google.cat/robots.txt","bytes":6624,"groups":4},"bots":{"GPTBot":["partial","*",169,64],"ChatGPT-User":["partial","*",169,64],"OAI-SearchBot":["partial","*",169,64],"ClaudeBot":["partial","*",169,64],"Claude-User":["partial","*",169,64],"Claude-SearchBot":["partial","*",169,64],"anthropic-ai":["partial","*",169,64],"PerplexityBot":["partial","*",169,64],"Perplexity-User":["partial","*",169,64],"Google-Extended":["partial","*",169,64],"Applebot-Extended":["partial","*",169,64],"CCBot":["partial","*",169,64],"Bytespider":["partial","*",169,64],"Amazonbot":["partial","*",169,64],"meta-externalagent":["partial","*",169,64],"cohere-ai":["partial","*",169,64],"Diffbot":["partial","*",169,64],"ContextFetchBot":["partial","*",169,64]},"llms":{"status":"missing","http":404,"bytes":1569},"home":{"status":"ok","http":200,"url":"https://www.google.com/","html_tokens":21251,"markdown_tokens":30,"pruned_tokens":30}},
{"rank":6650,"domain":"oxfam.org.uk","origin":"https://oxfam.org.uk","robots":{"status":"ok","http":200,"url":"https://www.oxfam.org.uk/robots.txt","bytes":89,"groups":1},"bots":{"GPTBot":["partial","*",1,1],"ChatGPT-User":["partial","*",1,1],"OAI-SearchBot":["partial","*",1,1],"ClaudeBot":["partial","*",1,1],"Claude-User":["partial","*",1,1],"Claude-SearchBot":["partial","*",1,1],"anthropic-ai":["partial","*",1,1],"PerplexityBot":["partial","*",1,1],"Perplexity-User":["partial","*",1,1],"Google-Extended":["partial","*",1,1],"Applebot-Extended":["partial","*",1,1],"CCBot":["partial","*",1,1],"Bytespider":["partial","*",1,1],"Amazonbot":["partial","*",1,1],"meta-externalagent":["partial","*",1,1],"cohere-ai":["partial","*",1,1],"Diffbot":["partial","*",1,1],"ContextFetchBot":["partial","*",1,1]},"llms":{"status":"missing","http":404,"bytes":151920},"home":{"status":"ok","http":200,"url":"https://www.oxfam.org.uk/","html_tokens":107975,"markdown_tokens":257,"pruned_tokens":32}},
{"rank":6651,"domain":"newsvine.com","origin":"https://newsvine.com","robots":{"status":"timeout","http":null,"url":"https://newsvine.com/robots.txt","bytes":0,"groups":0,"error":"ConnectTimeout"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":6652,"domain":"teslamotors.com","origin":"https://www.teslamotors.com","robots":{"status":"refused","http":403,"url":"https://www.tesla.com/robots.txt","bytes":381,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":6653,"domain":"sketch.com","origin":"https://sketch.com","robots":{"status":"ok","http":200,"url":"https://www.sketch.com/robots.txt","bytes":790,"groups":2},"bots":{"GPTBot":["partial","*",9,0],"ChatGPT-User":["partial","*",9,0],"OAI-SearchBot":["partial","*",9,0],"ClaudeBot":["partial","*",9,0],"Claude-User":["partial","*",9,0],"Claude-SearchBot":["partial","*",9,0],"anthropic-ai":["partial","*",9,0],"PerplexityBot":["partial","*",9,0],"Perplexity-User":["partial","*",9,0],"Google-Extended":["partial","*",9,0],"Applebot-Extended":["partial","*",9,0],"CCBot":["partial","*",9,0],"Bytespider":["partial","*",9,0],"Amazonbot":["partial","*",9,0],"meta-externalagent":["partial","*",9,0],"cohere-ai":["partial","*",9,0],"Diffbot":["partial","*",9,0],"ContextFetchBot":["partial","*",9,0]},"llms":{"status":"present","http":200,"bytes":44928},"home":{"status":"ok","http":200,"url":"https://www.sketch.com/","html_tokens":115753,"markdown_tokens":800,"pruned_tokens":107}},
{"rank":6654,"domain":"cdlib.org","origin":"https://cdlib.org","robots":{"status":"ok","http":200,"url":"https://cdlib.org/robots.txt","bytes":128,"groups":1},"bots":{"GPTBot":["partial","*",2,0],"ChatGPT-User":["partial","*",2,0],"OAI-SearchBot":["partial","*",2,0],"ClaudeBot":["partial","*",2,0],"Claude-User":["partial","*",2,0],"Claude-SearchBot":["partial","*",2,0],"anthropic-ai":["partial","*",2,0],"PerplexityBot":["partial","*",2,0],"Perplexity-User":["partial","*",2,0],"Google-Extended":["partial","*",2,0],"Applebot-Extended":["partial","*",2,0],"CCBot":["partial","*",2,0],"Bytespider":["partial","*",2,0],"Amazonbot":["partial","*",2,0],"meta-externalagent":["partial","*",2,0],"cohere-ai":["partial","*",2,0],"Diffbot":["partial","*",2,0],"ContextFetchBot":["partial","*",2,0]},"llms":{"status":"missing","http":404,"bytes":61075},"home":{"status":"ok","http":200,"url":"https://cdlib.org/","html_tokens":18721,"markdown_tokens":100,"pruned_tokens":100}},
{"rank":6655,"domain":"kennedy-center.org","origin":"https://kennedy-center.org","robots":{"status":"ok","http":200,"url":"https://www.kennedy-center.org/robots.txt","bytes":378,"groups":2},"bots":{"GPTBot":["partial","*",9,0],"ChatGPT-User":["partial","*",9,0],"OAI-SearchBot":["partial","*",9,0],"ClaudeBot":["partial","*",9,0],"Claude-User":["partial","*",9,0],"Claude-SearchBot":["partial","*",9,0],"anthropic-ai":["partial","*",9,0],"PerplexityBot":["partial","*",9,0],"Perplexity-User":["partial","*",9,0],"Google-Extended":["partial","*",9,0],"Applebot-Extended":["partial","*",9,0],"CCBot":["partial","*",9,0],"Bytespider":["partial","*",9,0],"Amazonbot":["partial","*",9,0],"meta-externalagent":["partial","*",9,0],"cohere-ai":["partial","*",9,0],"Diffbot":["partial","*",9,0],"ContextFetchBot":["partial","*",9,0]},"llms":{"status":"html","http":200,"bytes":69662},"home":{"status":"http_error","http":403,"url":"https://www.kennedy-center.org/"}},
{"rank":6656,"domain":"1c-bitrix.ru","origin":"https://1c-bitrix.ru","robots":{"status":"ok","http":200,"url":"https://1c-bitrix.ru/robots.txt","bytes":4851,"groups":3},"bots":{"GPTBot":["partial","*",63,4],"ChatGPT-User":["partial","*",63,4],"OAI-SearchBot":["partial","*",63,4],"ClaudeBot":["partial","*",63,4],"Claude-User":["partial","*",63,4],"Claude-SearchBot":["partial","*",63,4],"anthropic-ai":["partial","*",63,4],"PerplexityBot":["partial","*",63,4],"Perplexity-User":["partial","*",63,4],"Google-Extended":["partial","*",63,4],"Applebot-Extended":["partial","*",63,4],"CCBot":["partial","*",63,4],"Bytespider":["partial","*",63,4],"Amazonbot":["partial","*",63,4],"meta-externalagent":["partial","*",63,4],"cohere-ai":["partial","*",63,4],"Diffbot":["partial","*",63,4],"ContextFetchBot":["partial","*",63,4]},"llms":{"status":"missing","http":404,"bytes":1131},"home":{"status":"ok","http":200,"url":"https://www.1c-bitrix.ru/","html_tokens":48796,"markdown_tokens":70,"pruned_tokens":70}},
{"rank":6657,"domain":"geo.de","origin":"https://geo.de","robots":{"status":"ok","http":200,"url":"https://www.geo.de/robots.txt","bytes":600,"groups":11},"bots":{"GPTBot":["partial","named",1,1],"ChatGPT-User":["partial","named",1,1],"OAI-SearchBot":["partial","*",8,0],"ClaudeBot":["partial","*",8,0],"Claude-User":["partial","*",8,0],"Claude-SearchBot":["partial","*",8,0],"anthropic-ai":["partial","*",8,0],"PerplexityBot":["partial","*",8,0],"Perplexity-User":["partial","*",8,0],"Google-Extended":["partial","named",1,1],"Applebot-Extended":["partial","named",1,1],"CCBot":["blocked","named",1,0],"Bytespider":["partial","*",8,0],"Amazonbot":["partial","*",8,0],"meta-externalagent":["partial","*",8,0],"cohere-ai":["partial","*",8,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",8,0]},"llms":{"status":"missing","http":404,"bytes":161587},"home":{"status":"ok","http":200,"url":"https://www.geo.de/","html_tokens":97405,"markdown_tokens":1126,"pruned_tokens":95}},
{"rank":6658,"domain":"sbs.co.kr","origin":"https://sbs.co.kr","robots":{"status":"missing","http":404,"url":"https://www.sbs.co.kr/robots.txt","bytes":301,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":279},"home":{"status":"ok","http":200,"url":"https://www.sbs.co.kr/","html_tokens":101451,"markdown_tokens":9,"pruned_tokens":9}},
{"rank":6659,"domain":"nautil.us","origin":"https://nautil.us","robots":{"status":"ok","http":200,"url":"https://nautil.us/robots.txt","bytes":346,"groups":2},"bots":{"GPTBot":["partial","*",7,0],"ChatGPT-User":["partial","*",7,0],"OAI-SearchBot":["partial","*",7,0],"ClaudeBot":["partial","*",7,0],"Claude-User":["partial","*",7,0],"Claude-SearchBot":["partial","*",7,0],"anthropic-ai":["partial","*",7,0],"PerplexityBot":["partial","*",7,0],"Perplexity-User":["partial","*",7,0],"Google-Extended":["partial","*",7,0],"Applebot-Extended":["partial","*",7,0],"CCBot":["partial","*",7,0],"Bytespider":["partial","*",7,0],"Amazonbot":["partial","*",7,0],"meta-externalagent":["partial","*",7,0],"cohere-ai":["partial","*",7,0],"Diffbot":["partial","*",7,0],"ContextFetchBot":["partial","*",7,0]},"llms":{"status":"missing","http":404,"bytes":9},"home":{"status":"ok","http":200,"url":"https://nautil.us/","html_tokens":118859,"markdown_tokens":217,"pruned_tokens":14}},
{"rank":6660,"domain":"google.co.cr","origin":"https://google.co.cr","robots":{"status":"ok","http":200,"url":"https://www.google.co.cr/robots.txt","bytes":6624,"groups":4},"bots":{"GPTBot":["partial","*",169,64],"ChatGPT-User":["partial","*",169,64],"OAI-SearchBot":["partial","*",169,64],"ClaudeBot":["partial","*",169,64],"Claude-User":["partial","*",169,64],"Claude-SearchBot":["partial","*",169,64],"anthropic-ai":["partial","*",169,64],"PerplexityBot":["partial","*",169,64],"Perplexity-User":["partial","*",169,64],"Google-Extended":["partial","*",169,64],"Applebot-Extended":["partial","*",169,64],"CCBot":["partial","*",169,64],"Bytespider":["partial","*",169,64],"Amazonbot":["partial","*",169,64],"meta-externalagent":["partial","*",169,64],"cohere-ai":["partial","*",169,64],"Diffbot":["partial","*",169,64],"ContextFetchBot":["partial","*",169,64]},"llms":{"status":"missing","http":404,"bytes":1569},"home":{"status":"ok","http":200,"url":"https://www.google.com/","html_tokens":21351,"markdown_tokens":30,"pruned_tokens":30}},
{"rank":6661,"domain":"gleam.io","origin":"https://gleam.io","robots":{"status":"ok","http":200,"url":"https://gleam.io/robots.txt","bytes":951,"groups":1},"bots":{"GPTBot":["partial","*",29,0],"ChatGPT-User":["partial","*",29,0],"OAI-SearchBot":["partial","*",29,0],"ClaudeBot":["partial","*",29,0],"Claude-User":["partial","*",29,0],"Claude-SearchBot":["partial","*",29,0],"anthropic-ai":["partial","*",29,0],"PerplexityBot":["partial","*",29,0],"Perplexity-User":["partial","*",29,0],"Google-Extended":["partial","*",29,0],"Applebot-Extended":["partial","*",29,0],"CCBot":["partial","*",29,0],"Bytespider":["partial","*",29,0],"Amazonbot":["partial","*",29,0],"meta-externalagent":["partial","*",29,0],"cohere-ai":["partial","*",29,0],"Diffbot":["partial","*",29,0],"ContextFetchBot":["partial","*",29,0]},"llms":{"status":"http_error","http":403,"bytes":60083},"home":{"status":"http_error","http":403,"url":"https://gleam.io/"}},
{"rank":6662,"domain":"storenvy.com","origin":"https://storenvy.com","robots":{"status":"ok","http":200,"url":"https://www.storenvy.com/robots.txt","bytes":545,"groups":1},"bots":{"GPTBot":["partial","*",19,0],"ChatGPT-User":["partial","*",19,0],"OAI-SearchBot":["partial","*",19,0],"ClaudeBot":["partial","*",19,0],"Claude-User":["partial","*",19,0],"Claude-SearchBot":["partial","*",19,0],"anthropic-ai":["partial","*",19,0],"PerplexityBot":["partial","*",19,0],"Perplexity-User":["partial","*",19,0],"Google-Extended":["partial","*",19,0],"Applebot-Extended":["partial","*",19,0],"CCBot":["partial","*",19,0],"Bytespider":["partial","*",19,0],"Amazonbot":["partial","*",19,0],"meta-externalagent":["partial","*",19,0],"cohere-ai":["partial","*",19,0],"Diffbot":["partial","*",19,0],"ContextFetchBot":["partial","*",19,0]},"llms":{"status":"present","http":200,"bytes":2610},"home":{"status":"ok","http":200,"url":"https://www.storenvy.com/","html_tokens":25322,"markdown_tokens":40,"pruned_tokens":38}},
{"rank":6663,"domain":"google.com.do","origin":"https://google.com.do","robots":{"status":"ok","http":200,"url":"https://www.google.com.do/robots.txt","bytes":6624,"groups":4},"bots":{"GPTBot":["partial","*",169,64],"ChatGPT-User":["partial","*",169,64],"OAI-SearchBot":["partial","*",169,64],"ClaudeBot":["partial","*",169,64],"Claude-User":["partial","*",169,64],"Claude-SearchBot":["partial","*",169,64],"anthropic-ai":["partial","*",169,64],"PerplexityBot":["partial","*",169,64],"Perplexity-User":["partial","*",169,64],"Google-Extended":["partial","*",169,64],"Applebot-Extended":["partial","*",169,64],"CCBot":["partial","*",169,64],"Bytespider":["partial","*",169,64],"Amazonbot":["partial","*",169,64],"meta-externalagent":["partial","*",169,64],"cohere-ai":["partial","*",169,64],"Diffbot":["partial","*",169,64],"ContextFetchBot":["partial","*",169,64]},"llms":{"status":"missing","http":404,"bytes":1569},"home":{"status":"ok","http":200,"url":"https://www.google.com/","html_tokens":21342,"markdown_tokens":30,"pruned_tokens":30}},
{"rank":6664,"domain":"gastromedix.shop","origin":"https://gastromedix.shop","robots":{"status":"ok","http":200,"url":"https://gastromedix.shop/robots.txt","bytes":1248,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":645},"home":{"status":"ok","http":200,"url":"https://gastromedix.shop/","html_tokens":4038,"markdown_tokens":2291,"pruned_tokens":154}},
{"rank":6665,"domain":"umk.pl","origin":"https://umk.pl","robots":{"status":"ok","http":200,"url":"https://www.umk.pl/robots.txt","bytes":110,"groups":1},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"missing","http":404,"bytes":56651},"home":{"status":"ok","http":200,"url":"https://www.umk.pl/","html_tokens":29834,"markdown_tokens":729,"pruned_tokens":160}},
{"rank":6666,"domain":"cmail19.com","origin":"https://cmail19.com","robots":{"status":"ok","http":200,"url":"https://cmail19.com/robots.txt","bytes":26,"groups":1},"bots":{"GPTBot":["blocked","*",1,0],"ChatGPT-User":["blocked","*",1,0],"OAI-SearchBot":["blocked","*",1,0],"ClaudeBot":["blocked","*",1,0],"Claude-User":["blocked","*",1,0],"Claude-SearchBot":["blocked","*",1,0],"anthropic-ai":["blocked","*",1,0],"PerplexityBot":["blocked","*",1,0],"Perplexity-User":["blocked","*",1,0],"Google-Extended":["blocked","*",1,0],"Applebot-Extended":["blocked","*",1,0],"CCBot":["blocked","*",1,0],"Bytespider":["blocked","*",1,0],"Amazonbot":["blocked","*",1,0],"meta-externalagent":["blocked","*",1,0],"cohere-ai":["blocked","*",1,0],"Diffbot":["blocked","*",1,0],"ContextFetchBot":["blocked","*",1,0]},"llms":{"status":"disallowed"},"home":{"status":"disallowed"}},
{"rank":6667,"domain":"sharethis.com","origin":"https://sharethis.com","robots":{"status":"ok","http":200,"url":"https://sharethis.com/robots.txt","bytes":171,"groups":1},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"missing","http":404,"bytes":130245},"home":{"status":"ok","http":200,"url":"https://sharethis.com/","html_tokens":35214,"markdown_tokens":351,"pruned_tokens":170}},
{"rank":6668,"domain":"dc.com","origin":"https://dc.com","robots":{"status":"ok","http":200,"url":"https://www.dc.com/robots.txt","bytes":115,"groups":2},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["partial","*",1,0],"OAI-SearchBot":["partial","*",1,0],"ClaudeBot":["partial","*",1,0],"Claude-User":["partial","*",1,0],"Claude-SearchBot":["partial","*",1,0],"anthropic-ai":["partial","*",1,0],"PerplexityBot":["partial","*",1,0],"Perplexity-User":["partial","*",1,0],"Google-Extended":["partial","*",1,0],"Applebot-Extended":["partial","*",1,0],"CCBot":["partial","*",1,0],"Bytespider":["partial","*",1,0],"Amazonbot":["partial","*",1,0],"meta-externalagent":["partial","*",1,0],"cohere-ai":["partial","*",1,0],"Diffbot":["partial","*",1,0],"ContextFetchBot":["partial","*",1,0]},"llms":{"status":"http_error","http":403,"bytes":118},"home":{"status":"ok","http":200,"url":"https://www.dc.com/","html_tokens":60873,"markdown_tokens":148,"pruned_tokens":104}},
{"rank":6669,"domain":"gia.edu","origin":"https://gia.edu","robots":{"status":"ok","http":200,"url":"https://www.gia.edu/robots.txt","bytes":1239,"groups":2},"bots":{"GPTBot":["partial","*",33,0],"ChatGPT-User":["partial","*",33,0],"OAI-SearchBot":["partial","*",33,0],"ClaudeBot":["partial","*",33,0],"Claude-User":["partial","*",33,0],"Claude-SearchBot":["partial","*",33,0],"anthropic-ai":["partial","*",33,0],"PerplexityBot":["partial","*",33,0],"Perplexity-User":["partial","*",33,0],"Google-Extended":["partial","*",33,0],"Applebot-Extended":["partial","*",33,0],"CCBot":["partial","*",33,0],"Bytespider":["partial","*",33,0],"Amazonbot":["partial","*",33,0],"meta-externalagent":["partial","*",33,0],"cohere-ai":["partial","*",33,0],"Diffbot":["partial","*",33,0],"ContextFetchBot":["partial","*",33,0]},"llms":{"status":"timeout","error":"timed out after 10 s"},"home":{"status":"ok","http":200,"url":"https://www.gia.edu/","html_tokens":98498,"markdown_tokens":25,"pruned_tokens":25}},
{"rank":6670,"domain":"ne.gov","origin":"https://ne.gov","robots":{"status":"ok","http":200,"url":"https://www.ne.gov/robots.txt","bytes":97,"groups":1},"bots":{"GPTBot":["partial","*",2,2],"ChatGPT-User":["partial","*",2,2],"OAI-SearchBot":["partial","*",2,2],"ClaudeBot":["partial","*",2,2],"Claude-User":["partial","*",2,2],"Claude-SearchBot":["partial","*",2,2],"anthropic-ai":["partial","*",2,2],"PerplexityBot":["partial","*",2,2],"Perplexity-User":["partial","*",2,2],"Google-Extended":["partial","*",2,2],"Applebot-Extended":["partial","*",2,2],"CCBot":["partial","*",2,2],"Bytespider":["partial","*",2,2],"Amazonbot":["partial","*",2,2],"meta-externalagent":["partial","*",2,2],"cohere-ai":["partial","*",2,2],"Diffbot":["partial","*",2,2],"ContextFetchBot":["partial","*",2,2]},"llms":{"status":"missing","http":404,"bytes":10077},"home":{"status":"ok","http":200,"url":"https://www.ne.gov/","html_tokens":8076,"markdown_tokens":35,"pruned_tokens":35}},
{"rank":6671,"domain":"mondelezinternational.com","origin":"https://mondelezinternational.com","robots":{"status":"ok","http":200,"url":"https://www.mondelezinternational.com/robots.txt","bytes":949,"groups":12},"bots":{"GPTBot":["partial","named",2,1],"ChatGPT-User":["partial","named",2,1],"OAI-SearchBot":["partial","named",2,1],"ClaudeBot":["partial","named",2,1],"Claude-User":["partial","named",2,1],"Claude-SearchBot":["partial","named",2,1],"anthropic-ai":["partial","*",2,1],"PerplexityBot":["partial","named",2,1],"Perplexity-User":["partial","named",2,1],"Google-Extended":["partial","named",2,1],"Applebot-Extended":["partial","*",2,1],"CCBot":["partial","*",2,1],"Bytespider":["partial","*",2,1],"Amazonbot":["partial","*",2,1],"meta-externalagent":["partial","*",2,1],"cohere-ai":["partial","*",2,1],"Diffbot":["partial","*",2,1],"ContextFetchBot":["partial","*",2,1]},"llms":{"status":"missing","http":404,"bytes":930117},"home":{"status":"ok","http":200,"url":"https://www.mondelezinternational.com/","html_tokens":235397,"markdown_tokens":24,"pruned_tokens":24}},
{"rank":6672,"domain":"biggerpockets.com","origin":"https://biggerpockets.com","robots":{"status":"ok","http":200,"url":"https://www.biggerpockets.com/robots.txt","bytes":6494,"groups":15},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["partial","*",170,1],"OAI-SearchBot":["partial","*",170,1],"ClaudeBot":["partial","*",170,1],"Claude-User":["partial","*",170,1],"Claude-SearchBot":["partial","*",170,1],"anthropic-ai":["partial","*",170,1],"PerplexityBot":["partial","*",170,1],"Perplexity-User":["partial","*",170,1],"Google-Extended":["partial","*",170,1],"Applebot-Extended":["partial","*",170,1],"CCBot":["partial","*",170,1],"Bytespider":["partial","*",170,1],"Amazonbot":["partial","*",170,1],"meta-externalagent":["partial","*",170,1],"cohere-ai":["partial","*",170,1],"Diffbot":["partial","*",170,1],"ContextFetchBot":["partial","*",170,1]},"llms":{"status":"http_error","http":406,"bytes":0},"home":{"status":"ok","http":200,"url":"https://www.biggerpockets.com/","html_tokens":4574,"markdown_tokens":122,"pruned_tokens":122}},
{"rank":6673,"domain":"minoximedx.com","origin":"https://minoximedx.com","robots":{"status":"ok","http":200,"url":"https://minoximedx.com/robots.txt","bytes":77,"groups":1},"bots":{"GPTBot":["partial","*",1,0],"ChatGPT-User":["partial","*",1,0],"OAI-SearchBot":["partial","*",1,0],"ClaudeBot":["partial","*",1,0],"Claude-User":["partial","*",1,0],"Claude-SearchBot":["partial","*",1,0],"anthropic-ai":["partial","*",1,0],"PerplexityBot":["partial","*",1,0],"Perplexity-User":["partial","*",1,0],"Google-Extended":["partial","*",1,0],"Applebot-Extended":["partial","*",1,0],"CCBot":["partial","*",1,0],"Bytespider":["partial","*",1,0],"Amazonbot":["partial","*",1,0],"meta-externalagent":["partial","*",1,0],"cohere-ai":["partial","*",1,0],"Diffbot":["partial","*",1,0],"ContextFetchBot":["partial","*",1,0]},"llms":{"status":"missing","http":404,"bytes":643},"home":{"status":"ok","http":200,"url":"https://minoximedx.com/","html_tokens":13150,"markdown_tokens":2715,"pruned_tokens":525}},
{"rank":6674,"domain":"asce.org","origin":"https://asce.org","robots":{"status":"ok","http":200,"url":"https://www.asce.org/robots.txt","bytes":427,"groups":1},"bots":{"GPTBot":["partial","*",15,0],"ChatGPT-User":["partial","*",15,0],"OAI-SearchBot":["partial","*",15,0],"ClaudeBot":["partial","*",15,0],"Claude-User":["partial","*",15,0],"Claude-SearchBot":["partial","*",15,0],"anthropic-ai":["partial","*",15,0],"PerplexityBot":["partial","*",15,0],"Perplexity-User":["partial","*",15,0],"Google-Extended":["partial","*",15,0],"Applebot-Extended":["partial","*",15,0],"CCBot":["partial","*",15,0],"Bytespider":["partial","*",15,0],"Amazonbot":["partial","*",15,0],"meta-externalagent":["partial","*",15,0],"cohere-ai":["partial","*",15,0],"Diffbot":["partial","*",15,0],"ContextFetchBot":["partial","*",15,0]},"llms":{"status":"missing","http":404,"bytes":120329},"home":{"status":"ok","http":200,"url":"https://www.asce.org/","html_tokens":39986,"markdown_tokens":314,"pruned_tokens":130}},
{"rank":6675,"domain":"zhiye.com","origin":"https://zhiye.com","robots":{"status":"unreachable","http":null,"url":"https://zhiye.com/robots.txt","bytes":0,"groups":0,"error":"cannot resolve zhiye.com"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":6676,"domain":"te.ua","origin":"https://te.ua","robots":{"status":"unreachable","http":null,"url":"https://te.ua/robots.txt","bytes":0,"groups":0,"error":"cannot resolve te.ua"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":6677,"domain":"okfn.de","origin":"https://okfn.de","robots":{"status":"ok","http":200,"url":"https://okfn.de/robots.txt","bytes":16,"groups":1},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"missing","http":404,"bytes":37538},"home":{"status":"ok","http":200,"url":"https://okfn.de/","html_tokens":12774,"markdown_tokens":179,"pruned_tokens":173}},
{"rank":6678,"domain":"bittorrent.com","origin":"https://bittorrent.com","robots":{"status":"ok","http":200,"url":"https://www.bittorrent.com/robots.txt","bytes":602,"groups":8},"bots":{"GPTBot":["allowed","named",0,1],"ChatGPT-User":["partial","*",8,0],"OAI-SearchBot":["allowed","named",0,1],"ClaudeBot":["allowed","named",0,1],"Claude-User":["partial","*",8,0],"Claude-SearchBot":["partial","*",8,0],"anthropic-ai":["allowed","named",0,1],"PerplexityBot":["allowed","named",0,1],"Perplexity-User":["partial","*",8,0],"Google-Extended":["allowed","named",0,1],"Applebot-Extended":["partial","*",8,0],"CCBot":["partial","*",8,0],"Bytespider":["partial","*",8,0],"Amazonbot":["partial","*",8,0],"meta-externalagent":["partial","*",8,0],"cohere-ai":["partial","*",8,0],"Diffbot":["partial","*",8,0],"ContextFetchBot":["partial","*",8,0]},"llms":{"status":"present","http":200,"bytes":3282},"home":{"status":"ok","http":200,"url":"https://www.bittorrent.com/","html_tokens":129008,"markdown_tokens":30,"pruned_tokens":30}},
{"rank":6679,"domain":"cushmanwakefield.com","origin":"https://cushmanwakefield.com","robots":{"status":"refused","http":403,"url":"https://www.cushmanwakefield.com/en","bytes":179,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":6680,"domain":"newsbusters.org","origin":"https://newsbusters.org","robots":{"status":"refused","http":403,"url":"https://newsbusters.org/robots.txt","bytes":560,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":6681,"domain":"slimgogo.com","origin":"https://slimgogo.com","robots":{"status":"ok","http":200,"url":"https://slimgogo.com/robots.txt","bytes":72,"groups":1},"bots":{"GPTBot":["partial","*",1,0],"ChatGPT-User":["partial","*",1,0],"OAI-SearchBot":["partial","*",1,0],"ClaudeBot":["partial","*",1,0],"Claude-User":["partial","*",1,0],"Claude-SearchBot":["partial","*",1,0],"anthropic-ai":["partial","*",1,0],"PerplexityBot":["partial","*",1,0],"Perplexity-User":["partial","*",1,0],"Google-Extended":["partial","*",1,0],"Applebot-Extended":["partial","*",1,0],"CCBot":["partial","*",1,0],"Bytespider":["partial","*",1,0],"Amazonbot":["partial","*",1,0],"meta-externalagent":["partial","*",1,0],"cohere-ai":["partial","*",1,0],"Diffbot":["partial","*",1,0],"ContextFetchBot":["partial","*",1,0]},"llms":{"status":"missing","http":404,"bytes":641},"home":{"status":"ok","http":200,"url":"https://slimgogo.com/","html_tokens":3268,"markdown_tokens":329,"pruned_tokens":174}},
{"rank":6682,"domain":"shanghai.gov.cn","origin":"https://shanghai.gov.cn","robots":{"status":"missing","http":404,"url":"https://www.shanghai.gov.cn/robots.txt","bytes":1091,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":1091},"home":{"status":"ok","http":200,"url":"https://www.shanghai.gov.cn/","html_tokens":29797,"markdown_tokens":16,"pruned_tokens":16}},
{"rank":6683,"domain":"edmedix.com","origin":"https://edmedix.com","robots":{"status":"ok","http":200,"url":"https://edmedix.com/robots.txt","bytes":143,"groups":1},"bots":{"GPTBot":["partial","*",5,0],"ChatGPT-User":["partial","*",5,0],"OAI-SearchBot":["partial","*",5,0],"ClaudeBot":["partial","*",5,0],"Claude-User":["partial","*",5,0],"Claude-SearchBot":["partial","*",5,0],"anthropic-ai":["partial","*",5,0],"PerplexityBot":["partial","*",5,0],"Perplexity-User":["partial","*",5,0],"Google-Extended":["partial","*",5,0],"Applebot-Extended":["partial","*",5,0],"CCBot":["partial","*",5,0],"Bytespider":["partial","*",5,0],"Amazonbot":["partial","*",5,0],"meta-externalagent":["partial","*",5,0],"cohere-ai":["partial","*",5,0],"Diffbot":["partial","*",5,0],"ContextFetchBot":["partial","*",5,0]},"llms":{"status":"missing","http":404,"bytes":640},"home":{"status":"ok","http":200,"url":"https://edmedix.com/","html_tokens":12704,"markdown_tokens":4460,"pruned_tokens":285}},
{"rank":6684,"domain":"wikiversity.org","origin":"https://wikiversity.org","robots":{"status":"ok","http":200,"url":"https://en.wikiversity.org/robots.txt","bytes":16213,"groups":33},"bots":{"GPTBot":["partial","*",220,4],"ChatGPT-User":["partial","*",220,4],"OAI-SearchBot":["partial","*",220,4],"ClaudeBot":["partial","*",220,4],"Claude-User":["partial","*",220,4],"Claude-SearchBot":["partial","*",220,4],"anthropic-ai":["partial","*",220,4],"PerplexityBot":["partial","*",220,4],"Perplexity-User":["partial","*",220,4],"Google-Extended":["partial","*",220,4],"Applebot-Extended":["partial","*",220,4],"CCBot":["partial","*",220,4],"Bytespider":["partial","*",220,4],"Amazonbot":["partial","*",220,4],"meta-externalagent":["partial","*",220,4],"cohere-ai":["partial","*",220,4],"Diffbot":["partial","*",220,4],"ContextFetchBot":["partial","*",220,4]},"llms":{"status":"missing","http":404,"bytes":1868},"home":{"status":"ok","http":200,"url":"https://www.wikiversity.org/","html_tokens":15464,"markdown_tokens":11,"pruned_tokens":11}},
{"rank":6685,"domain":"sunysb.edu","origin":"https://sunysb.edu","robots":{"status":"html","http":200,"url":"https://www.stonybrook.edu/","bytes":147804,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":6686,"domain":"cults3d.com","origin":"https://cults3d.com","robots":{"status":"refused","http":403,"url":"https://cults3d.com/robots.txt","bytes":5399,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":6687,"domain":"plotly.com","origin":"https://plotly.com","robots":{"status":"ok","http":200,"url":"https://plotly.com/robots.txt","bytes":489,"groups":1},"bots":{"GPTBot":["partial","*",9,0],"ChatGPT-User":["partial","*",9,0],"OAI-SearchBot":["partial","*",9,0],"ClaudeBot":["partial","*",9,0],"Claude-User":["partial","*",9,0],"Claude-SearchBot":["partial","*",9,0],"anthropic-ai":["partial","*",9,0],"PerplexityBot":["partial","*",9,0],"Perplexity-User":["partial","*",9,0],"Google-Extended":["partial","*",9,0],"Applebot-Extended":["partial","*",9,0],"CCBot":["partial","*",9,0],"Bytespider":["partial","*",9,0],"Amazonbot":["partial","*",9,0],"meta-externalagent":["partial","*",9,0],"cohere-ai":["partial","*",9,0],"Diffbot":["partial","*",9,0],"ContextFetchBot":["partial","*",9,0]},"llms":{"status":"present","http":200,"bytes":4760},"home":{"status":"ok","http":200,"url":"https://plotly.com/","html_tokens":122693,"markdown_tokens":1284,"pruned_tokens":258}},
{"rank":6688,"domain":"bucknell.edu","origin":"https://bucknell.edu","robots":{"status":"refused","http":403,"url":"https://bucknell.edu/robots.txt","bytes":560,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":6689,"domain":"miamidade.gov","origin":"https://miamidade.gov","robots":{"status":"missing","http":404,"url":"https://www.miamidade.gov/robots.txt","bytes":2625,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":2625},"home":{"status":"ok","http":200,"url":"https://www.miamidade.gov/","html_tokens":85,"markdown_tokens":0,"pruned_tokens":0}},
{"rank":6690,"domain":"wgntv.com","origin":"https://wgntv.com","robots":{"status":"ok","http":200,"url":"https://wgntv.com/robots.txt","bytes":2297,"groups":40},"bots":{"GPTBot":["blocked","named",2,0],"ChatGPT-User":["blocked","named",1,0],"OAI-SearchBot":["blocked","named",1,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",12,1],"Claude-SearchBot":["partial","*",12,1],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["blocked","named",2,0],"Perplexity-User":["partial","*",12,1],"Google-Extended":["blocked","named",2,0],"Applebot-Extended":["blocked","named",2,0],"CCBot":["blocked","named",2,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["partial","*",12,1],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",12,1]},"llms":{"status":"http_error","http":403,"bytes":6514},"home":{"status":"http_error","http":403,"url":"https://wgntv.com/"}},
{"rank":6691,"domain":"rechtspraak.nl","origin":"https://rechtspraak.nl","robots":{"status":"refused","http":403,"url":"https://rechtspraak.nl/robots.txt","bytes":378,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":6692,"domain":"wcvb.com","origin":"https://wcvb.com","robots":{"status":"ok","http":200,"url":"https://wcvb.com/robots.txt","bytes":3763,"groups":48},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["blocked","named",1,0],"OAI-SearchBot":["partial","*",22,2],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",22,2],"Claude-SearchBot":["partial","*",22,2],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["partial","*",22,2],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",2,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",22,2]},"llms":{"status":"missing","http":404,"bytes":0},"home":{"status":"ok","http":200,"url":"https://www.wcvb.com/","html_tokens":152174,"markdown_tokens":67,"pruned_tokens":67}},
{"rank":6693,"domain":"ehu.eus","origin":"https://ehu.eus","robots":{"status":"ok","http":200,"url":"https://www.ehu.eus/robots.txt","bytes":1054,"groups":3},"bots":{"GPTBot":["partial","*",14,0],"ChatGPT-User":["partial","*",14,0],"OAI-SearchBot":["partial","*",14,0],"ClaudeBot":["partial","*",14,0],"Claude-User":["partial","*",14,0],"Claude-SearchBot":["partial","*",14,0],"anthropic-ai":["partial","*",14,0],"PerplexityBot":["partial","*",14,0],"Perplexity-User":["partial","*",14,0],"Google-Extended":["partial","*",14,0],"Applebot-Extended":["partial","*",14,0],"CCBot":["partial","*",14,0],"Bytespider":["partial","*",14,0],"Amazonbot":["partial","named",3,0],"meta-externalagent":["partial","*",14,0],"cohere-ai":["partial","*",14,0],"Diffbot":["partial","*",14,0],"ContextFetchBot":["partial","*",14,0]},"llms":{"status":"missing","http":404,"bytes":3149},"home":{"status":"ok","http":200,"url":"https://www.ehu.eus/en/en-home","html_tokens":23521,"markdown_tokens":9,"pruned_tokens":9}},
{"rank":6694,"domain":"gamblinginsider.com","origin":"https://gamblinginsider.com","robots":{"status":"ok","http":200,"url":"https://www.gamblinginsider.com/robots.txt","bytes":887,"groups":1},"bots":{"GPTBot":["partial","*",10,5],"ChatGPT-User":["partial","*",10,5],"OAI-SearchBot":["partial","*",10,5],"ClaudeBot":["partial","*",10,5],"Claude-User":["partial","*",10,5],"Claude-SearchBot":["partial","*",10,5],"anthropic-ai":["partial","*",10,5],"PerplexityBot":["partial","*",10,5],"Perplexity-User":["partial","*",10,5],"Google-Extended":["partial","*",10,5],"Applebot-Extended":["partial","*",10,5],"CCBot":["partial","*",10,5],"Bytespider":["partial","*",10,5],"Amazonbot":["partial","*",10,5],"meta-externalagent":["partial","*",10,5],"cohere-ai":["partial","*",10,5],"Diffbot":["partial","*",10,5],"ContextFetchBot":["partial","*",10,5]},"llms":{"status":"missing","http":404,"bytes":82821},"home":{"status":"http_error","http":403,"url":"https://gamblinginsider.com/"}},
{"rank":6695,"domain":"ing.com","origin":"https://ing.com","robots":{"status":"ok","http":200,"url":"https://ing.com/robots.txt","bytes":127,"groups":3},"bots":{"GPTBot":["partial","*",1,0],"ChatGPT-User":["partial","*",1,0],"OAI-SearchBot":["partial","*",1,0],"ClaudeBot":["partial","*",1,0],"Claude-User":["partial","*",1,0],"Claude-SearchBot":["partial","*",1,0],"anthropic-ai":["partial","*",1,0],"PerplexityBot":["partial","*",1,0],"Perplexity-User":["partial","*",1,0],"Google-Extended":["partial","*",1,0],"Applebot-Extended":["partial","*",1,0],"CCBot":["partial","*",1,0],"Bytespider":["partial","*",1,0],"Amazonbot":["partial","*",1,0],"meta-externalagent":["partial","*",1,0],"cohere-ai":["partial","*",1,0],"Diffbot":["partial","*",1,0],"ContextFetchBot":["partial","*",1,0]},"llms":{"status":"missing","http":404,"bytes":330730},"home":{"status":"ok","http":200,"url":"https://ing.com/","html_tokens":91056,"markdown_tokens":388,"pruned_tokens":167}},
{"rank":6696,"domain":"google.bs","origin":"https://google.bs","robots":{"status":"ok","http":200,"url":"https://www.google.bs/robots.txt","bytes":6624,"groups":4},"bots":{"GPTBot":["partial","*",169,64],"ChatGPT-User":["partial","*",169,64],"OAI-SearchBot":["partial","*",169,64],"ClaudeBot":["partial","*",169,64],"Claude-User":["partial","*",169,64],"Claude-SearchBot":["partial","*",169,64],"anthropic-ai":["partial","*",169,64],"PerplexityBot":["partial","*",169,64],"Perplexity-User":["partial","*",169,64],"Google-Extended":["partial","*",169,64],"Applebot-Extended":["partial","*",169,64],"CCBot":["partial","*",169,64],"Bytespider":["partial","*",169,64],"Amazonbot":["partial","*",169,64],"meta-externalagent":["partial","*",169,64],"cohere-ai":["partial","*",169,64],"Diffbot":["partial","*",169,64],"ContextFetchBot":["partial","*",169,64]},"llms":{"status":"missing","http":404,"bytes":1569},"home":{"status":"ok","http":200,"url":"https://www.google.com/","html_tokens":21324,"markdown_tokens":30,"pruned_tokens":30}},
{"rank":6697,"domain":"lifesitenews.com","origin":"https://lifesitenews.com","robots":{"status":"ok","http":200,"url":"https://www.lifesitenews.com/robots.txt","bytes":290,"groups":8},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["blocked","named",1,0],"OAI-SearchBot":["partial","*",1,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",1,0],"Claude-SearchBot":["partial","*",1,0],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["partial","*",1,0],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["partial","*",1,0],"CCBot":["partial","*",1,0],"Bytespider":["partial","*",1,0],"Amazonbot":["partial","*",1,0],"meta-externalagent":["partial","*",1,0],"cohere-ai":["partial","*",1,0],"Diffbot":["partial","*",1,0],"ContextFetchBot":["partial","*",1,0]},"llms":{"status":"missing","http":404,"bytes":121742},"home":{"status":"ok","http":200,"url":"https://www.lifesitenews.com/","html_tokens":35573,"markdown_tokens":36,"pruned_tokens":36}},
{"rank":6698,"domain":"quoteinvestigator.com","origin":"https://quoteinvestigator.com","robots":{"status":"ok","http":200,"url":"https://quoteinvestigator.com/robots.txt","bytes":24,"groups":1},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"missing","http":404,"bytes":0},"home":{"status":"error","error":"unsupported content encoding br"}},
{"rank":6699,"domain":"unenvironment.org","origin":"https://unenvironment.org","robots":{"status":"ok","http":200,"url":"https://www.unep.org/robots.txt","bytes":5128,"groups":33},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["partial","*",50,19],"OAI-SearchBot":["partial","*",50,19],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",50,19],"Claude-SearchBot":["partial","*",50,19],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["partial","*",50,19],"Perplexity-User":["partial","*",50,19],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",50,19]},"llms":{"status":"missing","http":404,"bytes":1243},"home":{"status":"http_error","http":403,"url":"https://www.unep.org/en"}},
{"rank":6700,"domain":"www.gov.si","origin":"https://www.gov.si","robots":{"status":"ok","http":200,"url":"https://www.gov.si/robots.txt","bytes":112,"groups":1},"bots":{"GPTBot":["partial","*",3,0],"ChatGPT-User":["partial","*",3,0],"OAI-SearchBot":["partial","*",3,0],"ClaudeBot":["partial","*",3,0],"Claude-User":["partial","*",3,0],"Claude-SearchBot":["partial","*",3,0],"anthropic-ai":["partial","*",3,0],"PerplexityBot":["partial","*",3,0],"Perplexity-User":["partial","*",3,0],"Google-Extended":["partial","*",3,0],"Applebot-Extended":["partial","*",3,0],"CCBot":["partial","*",3,0],"Bytespider":["partial","*",3,0],"Amazonbot":["partial","*",3,0],"meta-externalagent":["partial","*",3,0],"cohere-ai":["partial","*",3,0],"Diffbot":["partial","*",3,0],"ContextFetchBot":["partial","*",3,0]},"llms":{"status":"missing","http":404,"bytes":4424},"home":{"status":"ok","http":200,"url":"https://www.gov.si/","html_tokens":7386,"markdown_tokens":32,"pruned_tokens":32}},
{"rank":6701,"domain":"uni-bayreuth.de","origin":"https://uni-bayreuth.de","robots":{"status":"ok","http":200,"url":"https://www.uni-bayreuth.de/robots.txt","bytes":84,"groups":1},"bots":{"GPTBot":["partial","*",1,0],"ChatGPT-User":["partial","*",1,0],"OAI-SearchBot":["partial","*",1,0],"ClaudeBot":["partial","*",1,0],"Claude-User":["partial","*",1,0],"Claude-SearchBot":["partial","*",1,0],"anthropic-ai":["partial","*",1,0],"PerplexityBot":["partial","*",1,0],"Perplexity-User":["partial","*",1,0],"Google-Extended":["partial","*",1,0],"Applebot-Extended":["partial","*",1,0],"CCBot":["partial","*",1,0],"Bytespider":["partial","*",1,0],"Amazonbot":["partial","*",1,0],"meta-externalagent":["partial","*",1,0],"cohere-ai":["partial","*",1,0],"Diffbot":["partial","*",1,0],"ContextFetchBot":["partial","*",1,0]},"llms":{"status":"missing","http":404,"bytes":1163},"home":{"status":"ok","http":200,"url":"https://www.uni-bayreuth.de/","html_tokens":115655,"markdown_tokens":79,"pruned_tokens":79}},
{"rank":6702,"domain":"whatsappbusiness.com","origin":"https://whatsappbusiness.com","robots":{"status":"ok","http":200,"url":"https://whatsappbusiness.com/robots.txt","bytes":178,"groups":1},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"missing","http":404,"bytes":229625},"home":{"status":"ok","http":200,"url":"https://whatsappbusiness.com/","html_tokens":64034,"markdown_tokens":201,"pruned_tokens":87}},
{"rank":6703,"domain":"adminforge.de","origin":"https://adminforge.de","robots":{"status":"ok","http":200,"url":"https://adminforge.de/robots.txt","bytes":180,"groups":1},"bots":{"GPTBot":["allowed","*",0,1],"ChatGPT-User":["allowed","*",0,1],"OAI-SearchBot":["allowed","*",0,1],"ClaudeBot":["allowed","*",0,1],"Claude-User":["allowed","*",0,1],"Claude-SearchBot":["allowed","*",0,1],"anthropic-ai":["allowed","*",0,1],"PerplexityBot":["allowed","*",0,1],"Perplexity-User":["allowed","*",0,1],"Google-Extended":["allowed","*",0,1],"Applebot-Extended":["allowed","*",0,1],"CCBot":["allowed","*",0,1],"Bytespider":["allowed","*",0,1],"Amazonbot":["allowed","*",0,1],"meta-externalagent":["allowed","*",0,1],"cohere-ai":["allowed","*",0,1],"Diffbot":["allowed","*",0,1],"ContextFetchBot":["allowed","*",0,1]},"llms":{"status":"present","http":200,"bytes":6106},"home":{"status":"ok","http":200,"url":"https://adminforge.de/","html_tokens":60116,"markdown_tokens":183,"pruned_tokens":171}},
{"rank":6704,"domain":"chateauversailles.fr","origin":"https://chateauversailles.fr","robots":{"status":"ok","http":200,"url":"https://www.chateauversailles.fr/robots.txt","bytes":2548,"groups":1},"bots":{"GPTBot":["partial","*",36,18],"ChatGPT-User":["partial","*",36,18],"OAI-SearchBot":["partial","*",36,18],"ClaudeBot":["partial","*",36,18],"Claude-User":["partial","*",36,18],"Claude-SearchBot":["partial","*",36,18],"anthropic-ai":["partial","*",36,18],"PerplexityBot":["partial","*",36,18],"Perplexity-User":["partial","*",36,18],"Google-Extended":["partial","*",36,18],"Applebot-Extended":["partial","*",36,18],"CCBot":["partial","*",36,18],"Bytespider":["partial","*",36,18],"Amazonbot":["partial","*",36,18],"meta-externalagent":["partial","*",36,18],"cohere-ai":["partial","*",36,18],"Diffbot":["partial","*",36,18],"ContextFetchBot":["partial","*",36,18]},"llms":{"status":"missing","http":404,"bytes":198},"home":{"status":"ok","http":200,"url":"https://www.chateauversailles.fr/","html_tokens":31336,"markdown_tokens":745,"pruned_tokens":145}},
{"rank":6705,"domain":"ns.nl","origin":"https://ns.nl","robots":{"status":"ok","http":200,"url":"https://www.ns.nl/robots.txt","bytes":213,"groups":2},"bots":{"GPTBot":["partial","*",5,0],"ChatGPT-User":["partial","*",5,0],"OAI-SearchBot":["partial","*",5,0],"ClaudeBot":["partial","*",5,0],"Claude-User":["partial","*",5,0],"Claude-SearchBot":["partial","*",5,0],"anthropic-ai":["partial","*",5,0],"PerplexityBot":["partial","*",5,0],"Perplexity-User":["partial","*",5,0],"Google-Extended":["partial","*",5,0],"Applebot-Extended":["partial","*",5,0],"CCBot":["partial","*",5,0],"Bytespider":["partial","*",5,0],"Amazonbot":["partial","*",5,0],"meta-externalagent":["partial","*",5,0],"cohere-ai":["partial","*",5,0],"Diffbot":["partial","*",5,0],"ContextFetchBot":["partial","*",5,0]},"llms":{"status":"missing","http":404,"bytes":133635},"home":{"status":"ok","http":200,"url":"https://www.ns.nl/","html_tokens":94485,"markdown_tokens":26,"pruned_tokens":26}},
{"rank":6706,"domain":"pubpub.org","origin":"https://pubpub.org","robots":{"status":"ok","http":200,"url":"https://www.pubpub.org/robots.txt","bytes":41,"groups":1},"bots":{"GPTBot":["partial","*",1,0],"ChatGPT-User":["partial","*",1,0],"OAI-SearchBot":["partial","*",1,0],"ClaudeBot":["partial","*",1,0],"Claude-User":["partial","*",1,0],"Claude-SearchBot":["partial","*",1,0],"anthropic-ai":["partial","*",1,0],"PerplexityBot":["partial","*",1,0],"Perplexity-User":["partial","*",1,0],"Google-Extended":["partial","*",1,0],"Applebot-Extended":["partial","*",1,0],"CCBot":["partial","*",1,0],"Bytespider":["partial","*",1,0],"Amazonbot":["partial","*",1,0],"meta-externalagent":["partial","*",1,0],"cohere-ai":["partial","*",1,0],"Diffbot":["partial","*",1,0],"ContextFetchBot":["partial","*",1,0]},"llms":{"status":"missing","http":404,"bytes":10233},"home":{"status":"http_error","http":403,"url":"https://pubpub.org/"}},
{"rank":6707,"domain":"sytes.net","origin":"http://sytes.net","robots":{"status":"html","http":200,"url":"http://freeddns.noip.com?d=sytes.net&u=c3l0ZXMubmV0L3JvYm90cy50eHQ=","bytes":60536,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":6708,"domain":"bankofcanada.ca","origin":"https://bankofcanada.ca","robots":{"status":"ok","http":200,"url":"https://www.bankofcanada.ca/robots.txt","bytes":120,"groups":1},"bots":{"GPTBot":["partial","*",1,1],"ChatGPT-User":["partial","*",1,1],"OAI-SearchBot":["partial","*",1,1],"ClaudeBot":["partial","*",1,1],"Claude-User":["partial","*",1,1],"Claude-SearchBot":["partial","*",1,1],"anthropic-ai":["partial","*",1,1],"PerplexityBot":["partial","*",1,1],"Perplexity-User":["partial","*",1,1],"Google-Extended":["partial","*",1,1],"Applebot-Extended":["partial","*",1,1],"CCBot":["partial","*",1,1],"Bytespider":["partial","*",1,1],"Amazonbot":["partial","*",1,1],"meta-externalagent":["partial","*",1,1],"cohere-ai":["partial","*",1,1],"Diffbot":["partial","*",1,1],"ContextFetchBot":["partial","*",1,1]},"llms":{"status":"missing","http":404,"bytes":125028},"home":{"status":"ok","http":200,"url":"https://www.bankofcanada.ca/","html_tokens":53097,"markdown_tokens":373,"pruned_tokens":156}},
{"rank":6709,"domain":"richmond.edu","origin":"https://richmond.edu","robots":{"status":"unreachable","http":null,"url":"https://richmond.edu/robots.txt","bytes":0,"groups":0,"error":"[SSL: CERTIFICATE_VERIFY_FAILED] certificate verify failed: unable to get local issuer certificate (_ssl.c:1028)"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":6710,"domain":"deseretnews.com","origin":"https://deseretnews.com","robots":{"status":"ok","http":200,"url":"https://www.deseret.com/robots.txt","bytes":990,"groups":18},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["allowed","named",0,1],"OAI-SearchBot":["allowed","named",0,1],"ClaudeBot":["blocked","named",1,0],"Claude-User":["allowed","named",0,1],"Claude-SearchBot":["allowed","named",0,1],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","named",0,1],"Perplexity-User":["allowed","named",0,1],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"missing","http":404,"bytes":160464},"home":{"status":"ok","http":200,"url":"https://www.deseret.com/","html_tokens":86710,"markdown_tokens":1887,"pruned_tokens":175}},
{"rank":6711,"domain":"userapi.com","origin":"https://userapi.com","robots":{"status":"ok","http":200,"url":"https://vk.com/robots.txt","bytes":6175,"groups":3},"bots":{"GPTBot":["partial","*",79,11],"ChatGPT-User":["partial","*",79,11],"OAI-SearchBot":["partial","*",79,11],"ClaudeBot":["partial","*",79,11],"Claude-User":["partial","*",79,11],"Claude-SearchBot":["partial","*",79,11],"anthropic-ai":["partial","*",79,11],"PerplexityBot":["partial","*",79,11],"Perplexity-User":["partial","*",79,11],"Google-Extended":["partial","*",79,11],"Applebot-Extended":["partial","*",79,11],"CCBot":["partial","*",79,11],"Bytespider":["partial","*",79,11],"Amazonbot":["partial","*",79,11],"meta-externalagent":["partial","*",79,11],"cohere-ai":["partial","*",79,11],"Diffbot":["partial","*",79,11],"ContextFetchBot":["partial","*",79,11]},"llms":{"status":"missing","http":404,"bytes":13067},"home":{"status":"ok","http":200,"url":"https://m.vk.com/","html_tokens":27051,"markdown_tokens":27,"pruned_tokens":27}},
{"rank":6712,"domain":"psypost.org","origin":"https://psypost.org","robots":{"status":"ok","http":200,"url":"https://www.psypost.org/robots.txt","bytes":200,"groups":1},"bots":{"GPTBot":["partial","*",4,2],"ChatGPT-User":["partial","*",4,2],"OAI-SearchBot":["partial","*",4,2],"ClaudeBot":["partial","*",4,2],"Claude-User":["partial","*",4,2],"Claude-SearchBot":["partial","*",4,2],"anthropic-ai":["partial","*",4,2],"PerplexityBot":["partial","*",4,2],"Perplexity-User":["partial","*",4,2],"Google-Extended":["partial","*",4,2],"Applebot-Extended":["partial","*",4,2],"CCBot":["partial","*",4,2],"Bytespider":["partial","*",4,2],"Amazonbot":["partial","*",4,2],"meta-externalagent":["partial","*",4,2],"cohere-ai":["partial","*",4,2],"Diffbot":["partial","*",4,2],"ContextFetchBot":["partial","*",4,2]},"llms":{"status":"http_error","http":403,"bytes":5542},"home":{"status":"http_error","http":403,"url":"https://psypost.org/"}},
{"rank":6713,"domain":"cmswire.com","origin":"https://cmswire.com","robots":{"status":"ok","http":200,"url":"https://www.cmswire.com/robots.txt","bytes":818,"groups":4},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["partial","named",18,0],"OAI-SearchBot":["partial","*",18,0],"ClaudeBot":["partial","*",18,0],"Claude-User":["partial","*",18,0],"Claude-SearchBot":["partial","*",18,0],"anthropic-ai":["partial","*",18,0],"PerplexityBot":["partial","*",18,0],"Perplexity-User":["partial","*",18,0],"Google-Extended":["partial","*",18,0],"Applebot-Extended":["partial","*",18,0],"CCBot":["blocked","named",1,0],"Bytespider":["partial","*",18,0],"Amazonbot":["partial","*",18,0],"meta-externalagent":["partial","*",18,0],"cohere-ai":["partial","*",18,0],"Diffbot":["partial","*",18,0],"ContextFetchBot":["partial","*",18,0]},"llms":{"status":"missing","http":404,"bytes":319814},"home":{"status":"ok","http":200,"url":"https://www.cmswire.com/","html_tokens":127107,"markdown_tokens":155,"pruned_tokens":155}},
{"rank":6714,"domain":"qobuz.com","origin":"https://qobuz.com","robots":{"status":"ok","http":200,"url":"https://www.qobuz.com/robots.txt","bytes":1238,"groups":3},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["allowed","named",0,1],"OAI-SearchBot":["partial","*",24,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["allowed","named",0,1],"Claude-SearchBot":["partial","*",24,0],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["partial","*",24,0],"Perplexity-User":["allowed","named",0,1],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["partial","*",24,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",24,0]},"llms":{"status":"missing","http":404,"bytes":126497},"home":{"status":"ok","http":200,"url":"https://www.qobuz.com/us-en/discover","html_tokens":35733,"markdown_tokens":327,"pruned_tokens":59}},
{"rank":6715,"domain":"xanga.com","origin":"https://xanga.com","robots":{"status":"ok","http":200,"url":"https://xanga.com/robots.txt","bytes":27,"groups":1},"bots":{"GPTBot":["blocked","*",1,0],"ChatGPT-User":["blocked","*",1,0],"OAI-SearchBot":["blocked","*",1,0],"ClaudeBot":["blocked","*",1,0],"Claude-User":["blocked","*",1,0],"Claude-SearchBot":["blocked","*",1,0],"anthropic-ai":["blocked","*",1,0],"PerplexityBot":["blocked","*",1,0],"Perplexity-User":["blocked","*",1,0],"Google-Extended":["blocked","*",1,0],"Applebot-Extended":["blocked","*",1,0],"CCBot":["blocked","*",1,0],"Bytespider":["blocked","*",1,0],"Amazonbot":["blocked","*",1,0],"meta-externalagent":["blocked","*",1,0],"cohere-ai":["blocked","*",1,0],"Diffbot":["blocked","*",1,0],"ContextFetchBot":["blocked","*",1,0]},"llms":{"status":"disallowed"},"home":{"status":"disallowed"}},
{"rank":6716,"domain":"rics.org","origin":"https://rics.org","robots":{"status":"ok","http":200,"url":"https://www.rics.org/robots.txt","bytes":490,"groups":1},"bots":{"GPTBot":["partial","*",11,0],"ChatGPT-User":["partial","*",11,0],"OAI-SearchBot":["partial","*",11,0],"ClaudeBot":["partial","*",11,0],"Claude-User":["partial","*",11,0],"Claude-SearchBot":["partial","*",11,0],"anthropic-ai":["partial","*",11,0],"PerplexityBot":["partial","*",11,0],"Perplexity-User":["partial","*",11,0],"Google-Extended":["partial","*",11,0],"Applebot-Extended":["partial","*",11,0],"CCBot":["partial","*",11,0],"Bytespider":["partial","*",11,0],"Amazonbot":["partial","*",11,0],"meta-externalagent":["partial","*",11,0],"cohere-ai":["partial","*",11,0],"Diffbot":["partial","*",11,0],"ContextFetchBot":["partial","*",11,0]},"llms":{"status":"missing","http":404,"bytes":192481},"home":{"status":"ok","http":200,"url":"https://www.rics.org/","html_tokens":85414,"markdown_tokens":293,"pruned_tokens":78}},
{"rank":6717,"domain":"fundable.com","origin":"https://fundable.com","robots":{"status":"ok","http":200,"url":"https://www.fundable.com/robots.txt","bytes":59,"groups":1},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"missing","http":404,"bytes":66715},"home":{"status":"ok","http":200,"url":"https://www.fundable.com/","html_tokens":23690,"markdown_tokens":222,"pruned_tokens":180}},
{"rank":6718,"domain":"cloudsecurityalliance.org","origin":"https://cloudsecurityalliance.org","robots":{"status":"ok","http":200,"url":"https://cloudsecurityalliance.org/robots.txt","bytes":251,"groups":1},"bots":{"GPTBot":["partial","*",2,0],"ChatGPT-User":["partial","*",2,0],"OAI-SearchBot":["partial","*",2,0],"ClaudeBot":["partial","*",2,0],"Claude-User":["partial","*",2,0],"Claude-SearchBot":["partial","*",2,0],"anthropic-ai":["partial","*",2,0],"PerplexityBot":["partial","*",2,0],"Perplexity-User":["partial","*",2,0],"Google-Extended":["partial","*",2,0],"Applebot-Extended":["partial","*",2,0],"CCBot":["partial","*",2,0],"Bytespider":["partial","*",2,0],"Amazonbot":["partial","*",2,0],"meta-externalagent":["partial","*",2,0],"cohere-ai":["partial","*",2,0],"Diffbot":["partial","*",2,0],"ContextFetchBot":["partial","*",2,0]},"llms":{"status":"present","http":200,"bytes":18367},"home":{"status":"ok","http":200,"url":"https://cloudsecurityalliance.org/","html_tokens":29877,"markdown_tokens":396,"pruned_tokens":132}},
{"rank":6719,"domain":"sentinelone.com","origin":"https://sentinelone.com","robots":{"status":"ok","http":200,"url":"https://sentinelone.com/robots.txt","bytes":813,"groups":2},"bots":{"GPTBot":["partial","*",4,2],"ChatGPT-User":["partial","*",4,2],"OAI-SearchBot":["partial","*",4,2],"ClaudeBot":["partial","*",4,2],"Claude-User":["partial","*",4,2],"Claude-SearchBot":["partial","*",4,2],"anthropic-ai":["partial","*",4,2],"PerplexityBot":["partial","*",4,2],"Perplexity-User":["partial","*",4,2],"Google-Extended":["partial","*",4,2],"Applebot-Extended":["partial","*",4,2],"CCBot":["partial","*",4,2],"Bytespider":["partial","*",4,2],"Amazonbot":["partial","*",4,2],"meta-externalagent":["partial","*",4,2],"cohere-ai":["partial","*",4,2],"Diffbot":["partial","*",4,2],"ContextFetchBot":["partial","*",4,2]},"llms":{"status":"present","http":200,"bytes":6307},"home":{"status":"ok","http":200,"url":"https://www.sentinelone.com/","html_tokens":97550,"markdown_tokens":113,"pruned_tokens":113}},
{"rank":6720,"domain":"city-journal.org","origin":"https://city-journal.org","robots":{"status":"html","http":200,"url":"https://www.city-journal.org/robots.txt","bytes":163076,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":6721,"domain":"terbimed.com","origin":"https://terbimed.com","robots":{"status":"ok","http":200,"url":"https://terbimed.com/robots.txt","bytes":75,"groups":1},"bots":{"GPTBot":["partial","*",1,0],"ChatGPT-User":["partial","*",1,0],"OAI-SearchBot":["partial","*",1,0],"ClaudeBot":["partial","*",1,0],"Claude-User":["partial","*",1,0],"Claude-SearchBot":["partial","*",1,0],"anthropic-ai":["partial","*",1,0],"PerplexityBot":["partial","*",1,0],"Perplexity-User":["partial","*",1,0],"Google-Extended":["partial","*",1,0],"Applebot-Extended":["partial","*",1,0],"CCBot":["partial","*",1,0],"Bytespider":["partial","*",1,0],"Amazonbot":["partial","*",1,0],"meta-externalagent":["partial","*",1,0],"cohere-ai":["partial","*",1,0],"Diffbot":["partial","*",1,0],"ContextFetchBot":["partial","*",1,0]},"llms":{"status":"missing","http":404,"bytes":641},"home":{"status":"ok","http":200,"url":"https://terbimed.com/","html_tokens":20174,"markdown_tokens":7972,"pruned_tokens":142}},
{"rank":6722,"domain":"yummly.com","origin":"https://yummly.com","robots":{"status":"unreachable","http":null,"url":"https://yummly.com/robots.txt","bytes":0,"groups":0,"error":"All connection attempts failed"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":6723,"domain":"qualys.com","origin":"https://qualys.com","robots":{"status":"ok","http":200,"url":"https://www.qualys.com/robots.txt","bytes":1693,"groups":1},"bots":{"GPTBot":["partial","*",30,1],"ChatGPT-User":["partial","*",30,1],"OAI-SearchBot":["partial","*",30,1],"ClaudeBot":["partial","*",30,1],"Claude-User":["partial","*",30,1],"Claude-SearchBot":["partial","*",30,1],"anthropic-ai":["partial","*",30,1],"PerplexityBot":["partial","*",30,1],"Perplexity-User":["partial","*",30,1],"Google-Extended":["partial","*",30,1],"Applebot-Extended":["partial","*",30,1],"CCBot":["partial","*",30,1],"Bytespider":["partial","*",30,1],"Amazonbot":["partial","*",30,1],"meta-externalagent":["partial","*",30,1],"cohere-ai":["partial","*",30,1],"Diffbot":["partial","*",30,1],"ContextFetchBot":["partial","*",30,1]},"llms":{"status":"present","http":200,"bytes":9690},"home":{"status":"ok","http":200,"url":"https://www.qualys.com/","html_tokens":61823,"markdown_tokens":253,"pruned_tokens":253}},
{"rank":6724,"domain":"carleton.edu","origin":"https://carleton.edu","robots":{"status":"ok","http":200,"url":"https://www.carleton.edu/robots.txt","bytes":305,"groups":1},"bots":{"GPTBot":["partial","*",10,0],"ChatGPT-User":["partial","*",10,0],"OAI-SearchBot":["partial","*",10,0],"ClaudeBot":["partial","*",10,0],"Claude-User":["partial","*",10,0],"Claude-SearchBot":["partial","*",10,0],"anthropic-ai":["partial","*",10,0],"PerplexityBot":["partial","*",10,0],"Perplexity-User":["partial","*",10,0],"Google-Extended":["partial","*",10,0],"Applebot-Extended":["partial","*",10,0],"CCBot":["partial","*",10,0],"Bytespider":["partial","*",10,0],"Amazonbot":["partial","*",10,0],"meta-externalagent":["partial","*",10,0],"cohere-ai":["partial","*",10,0],"Diffbot":["partial","*",10,0],"ContextFetchBot":["partial","*",10,0]},"llms":{"status":"missing","http":404,"bytes":26018},"home":{"status":"ok","http":200,"url":"https://www.carleton.edu/","html_tokens":48062,"markdown_tokens":288,"pruned_tokens":114}},
{"rank":6725,"domain":"marketingland.com","origin":"https://marketingland.com","robots":{"status":"refused","http":403,"url":"https://martech.org/robots.txt/","bytes":464,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":6726,"domain":"unisa.edu.au","origin":"https://unisa.edu.au","robots":{"status":"unreachable","http":null,"url":"https://unisa.edu.au/robots.txt","bytes":0,"groups":0,"error":"[SSL: CERTIFICATE_VERIFY_FAILED] certificate verify failed: unable to get local issuer certificate (_ssl.c:1028)"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":6727,"domain":"cardplayer.com","origin":"https://cardplayer.com","robots":{"status":"ok","http":200,"url":"https://www.cardplayer.com/robots.txt","bytes":726,"groups":1},"bots":{"GPTBot":["partial","*",11,0],"ChatGPT-User":["partial","*",11,0],"OAI-SearchBot":["partial","*",11,0],"ClaudeBot":["partial","*",11,0],"Claude-User":["partial","*",11,0],"Claude-SearchBot":["partial","*",11,0],"anthropic-ai":["partial","*",11,0],"PerplexityBot":["partial","*",11,0],"Perplexity-User":["partial","*",11,0],"Google-Extended":["partial","*",11,0],"Applebot-Extended":["partial","*",11,0],"CCBot":["partial","*",11,0],"Bytespider":["partial","*",11,0],"Amazonbot":["partial","*",11,0],"meta-externalagent":["partial","*",11,0],"cohere-ai":["partial","*",11,0],"Diffbot":["partial","*",11,0],"ContextFetchBot":["partial","*",11,0]},"llms":{"status":"missing","http":404,"bytes":63819},"home":{"status":"ok","http":200,"url":"https://www.cardplayer.com/","html_tokens":44830,"markdown_tokens":198,"pruned_tokens":198}},
{"rank":6728,"domain":"webself.net","origin":"https://webself.net","robots":{"status":"ok","http":200,"url":"https://www.webself.net/robots.txt","bytes":182,"groups":1},"bots":{"GPTBot":["partial","*",1,1],"ChatGPT-User":["partial","*",1,1],"OAI-SearchBot":["partial","*",1,1],"ClaudeBot":["partial","*",1,1],"Claude-User":["partial","*",1,1],"Claude-SearchBot":["partial","*",1,1],"anthropic-ai":["partial","*",1,1],"PerplexityBot":["partial","*",1,1],"Perplexity-User":["partial","*",1,1],"Google-Extended":["partial","*",1,1],"Applebot-Extended":["partial","*",1,1],"CCBot":["partial","*",1,1],"Bytespider":["partial","*",1,1],"Amazonbot":["partial","*",1,1],"meta-externalagent":["partial","*",1,1],"cohere-ai":["partial","*",1,1],"Diffbot":["partial","*",1,1],"ContextFetchBot":["partial","*",1,1]},"llms":{"status":"missing","http":404,"bytes":21444},"home":{"status":"ok","http":200,"url":"https://www.webself.net/","html_tokens":14723,"markdown_tokens":190,"pruned_tokens":59}},
{"rank":6729,"domain":"waymo.com","origin":"https://waymo.com","robots":{"status":"ok","http":200,"url":"https://waymo.com/robots.txt","bytes":63,"groups":1},"bots":{"GPTBot":["allowed","*",0,1],"ChatGPT-User":["allowed","*",0,1],"OAI-SearchBot":["allowed","*",0,1],"ClaudeBot":["allowed","*",0,1],"Claude-User":["allowed","*",0,1],"Claude-SearchBot":["allowed","*",0,1],"anthropic-ai":["allowed","*",0,1],"PerplexityBot":["allowed","*",0,1],"Perplexity-User":["allowed","*",0,1],"Google-Extended":["allowed","*",0,1],"Applebot-Extended":["allowed","*",0,1],"CCBot":["allowed","*",0,1],"Bytespider":["allowed","*",0,1],"Amazonbot":["allowed","*",0,1],"meta-externalagent":["allowed","*",0,1],"cohere-ai":["allowed","*",0,1],"Diffbot":["allowed","*",0,1],"ContextFetchBot":["allowed","*",0,1]},"llms":{"status":"present","http":200,"bytes":1768},"home":{"status":"ok","http":200,"url":"https://waymo.com/","html_tokens":84310,"markdown_tokens":790,"pruned_tokens":361}},
{"rank":6730,"domain":"parisjetaime.com","origin":"https://parisjetaime.com","robots":{"status":"ok","http":200,"url":"https://parisjetaime.com/robots.txt","bytes":3712,"groups":28},"bots":{"GPTBot":["allowed","named",0,1],"ChatGPT-User":["allowed","named",0,1],"OAI-SearchBot":["allowed","named",0,1],"ClaudeBot":["allowed","named",0,1],"Claude-User":["partial","*",26,1],"Claude-SearchBot":["partial","*",26,1],"anthropic-ai":["allowed","named",0,1],"PerplexityBot":["allowed","named",0,1],"Perplexity-User":["allowed","named",0,1],"Google-Extended":["allowed","named",0,1],"Applebot-Extended":["allowed","named",0,1],"CCBot":["allowed","named",0,1],"Bytespider":["allowed","named",0,1],"Amazonbot":["allowed","named",0,1],"meta-externalagent":["allowed","named",0,1],"cohere-ai":["allowed","named",0,1],"Diffbot":["partial","*",26,1],"ContextFetchBot":["partial","*",26,1]},"llms":{"status":"missing","http":404,"bytes":321587},"home":{"status":"http_error","http":403,"url":"https://parisjetaime.com/"}},
{"rank":6731,"domain":"parismatch.com","origin":"https://parismatch.com","robots":{"status":"ok","http":200,"url":"https://www.parismatch.com/robots.txt","bytes":2235,"groups":5},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["partial","named",1,2],"OAI-SearchBot":["partial","named",1,2],"ClaudeBot":["partial","named",1,2],"Claude-User":["partial","named",1,2],"Claude-SearchBot":["partial","named",1,2],"anthropic-ai":["partial","named",1,2],"PerplexityBot":["partial","named",1,2],"Perplexity-User":["partial","*",7,0],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["partial","named",1,2],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["partial","named",1,2],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",7,0]},"llms":{"status":"missing","http":404,"bytes":55601},"home":{"status":"ok","http":200,"url":"https://www.parismatch.com/","html_tokens":97516,"markdown_tokens":48,"pruned_tokens":48}},
{"rank":6732,"domain":"unive.it","origin":"https://unive.it","robots":{"status":"ok","http":200,"url":"https://www.unive.it/robots.txt","bytes":38,"groups":1},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":196},"home":{"status":"ok","http":200,"url":"https://www.unive.it/","html_tokens":38991,"markdown_tokens":2908,"pruned_tokens":108}},
{"rank":6733,"domain":"adyen.com","origin":"https://adyen.com","robots":{"status":"ok","http":200,"url":"https://www.adyen.com/robots.txt","bytes":64,"groups":1},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"present","http":200,"bytes":618773},"home":{"status":"ok","http":200,"url":"https://www.adyen.com/","html_tokens":292902,"markdown_tokens":145,"pruned_tokens":145}},
{"rank":6734,"domain":"mindbodyonline.com","origin":"https://mindbodyonline.com","robots":{"status":"ok","http":200,"url":"https://www.mindbodyonline.com/robots.txt","bytes":1648,"groups":1},"bots":{"GPTBot":["partial","*",19,18],"ChatGPT-User":["partial","*",19,18],"OAI-SearchBot":["partial","*",19,18],"ClaudeBot":["partial","*",19,18],"Claude-User":["partial","*",19,18],"Claude-SearchBot":["partial","*",19,18],"anthropic-ai":["partial","*",19,18],"PerplexityBot":["partial","*",19,18],"Perplexity-User":["partial","*",19,18],"Google-Extended":["partial","*",19,18],"Applebot-Extended":["partial","*",19,18],"CCBot":["partial","*",19,18],"Bytespider":["partial","*",19,18],"Amazonbot":["partial","*",19,18],"meta-externalagent":["partial","*",19,18],"cohere-ai":["partial","*",19,18],"Diffbot":["partial","*",19,18],"ContextFetchBot":["partial","*",19,18]},"llms":{"status":"missing","http":404,"bytes":557},"home":{"status":"ok","http":200,"url":"https://www.mindbodyonline.com/","html_tokens":90208,"markdown_tokens":2718,"pruned_tokens":350}},
{"rank":6735,"domain":"ratp.fr","origin":"https://ratp.fr","robots":{"status":"refused","http":403,"url":"https://www.ratp.fr/","bytes":5476,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":6736,"domain":"skrill.com","origin":"https://skrill.com","robots":{"status":"ok","http":200,"url":"https://www.skrill.com/robots.txt","bytes":969,"groups":1},"bots":{"GPTBot":["partial","*",34,1],"ChatGPT-User":["partial","*",34,1],"OAI-SearchBot":["partial","*",34,1],"ClaudeBot":["partial","*",34,1],"Claude-User":["partial","*",34,1],"Claude-SearchBot":["partial","*",34,1],"anthropic-ai":["partial","*",34,1],"PerplexityBot":["partial","*",34,1],"Perplexity-User":["partial","*",34,1],"Google-Extended":["partial","*",34,1],"Applebot-Extended":["partial","*",34,1],"CCBot":["partial","*",34,1],"Bytespider":["partial","*",34,1],"Amazonbot":["partial","*",34,1],"meta-externalagent":["partial","*",34,1],"cohere-ai":["partial","*",34,1],"Diffbot":["partial","*",34,1],"ContextFetchBot":["partial","*",34,1]},"llms":{"status":"missing","http":404,"bytes":40724},"home":{"status":"ok","http":200,"url":"https://www.skrill.com/en-us/","html_tokens":15907,"markdown_tokens":127,"pruned_tokens":42}},
{"rank":6737,"domain":"wellandgood.com","origin":"https://wellandgood.com","robots":{"status":"html","http":200,"url":"https://www.theskimm.com/","bytes":2654767,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":6738,"domain":"americanbanker.com","origin":"https://americanbanker.com","robots":{"status":"ok","http":200,"url":"https://www.americanbanker.com/robots.txt","bytes":1358,"groups":26},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["partial","*",5,0],"OAI-SearchBot":["partial","*",5,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",5,0],"Claude-SearchBot":["partial","*",5,0],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["partial","*",5,0],"Perplexity-User":["partial","*",5,0],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",5,0]},"llms":{"status":"missing","http":404,"bytes":213883},"home":{"status":"ok","http":200,"url":"https://www.americanbanker.com/","html_tokens":161362,"markdown_tokens":1310,"pruned_tokens":150}},
{"rank":6739,"domain":"njit.edu","origin":"https://njit.edu","robots":{"status":"ok","http":200,"url":"https://www.njit.edu/robots.txt","bytes":2026,"groups":1},"bots":{"GPTBot":["partial","*",34,18],"ChatGPT-User":["partial","*",34,18],"OAI-SearchBot":["partial","*",34,18],"ClaudeBot":["partial","*",34,18],"Claude-User":["partial","*",34,18],"Claude-SearchBot":["partial","*",34,18],"anthropic-ai":["partial","*",34,18],"PerplexityBot":["partial","*",34,18],"Perplexity-User":["partial","*",34,18],"Google-Extended":["partial","*",34,18],"Applebot-Extended":["partial","*",34,18],"CCBot":["partial","*",34,18],"Bytespider":["partial","*",34,18],"Amazonbot":["partial","*",34,18],"meta-externalagent":["partial","*",34,18],"cohere-ai":["partial","*",34,18],"Diffbot":["partial","*",34,18],"ContextFetchBot":["partial","*",34,18]},"llms":{"status":"missing","http":404,"bytes":186},"home":{"status":"ok","http":200,"url":"https://www.njit.edu/","html_tokens":46067,"markdown_tokens":51,"pruned_tokens":51}},
{"rank":6740,"domain":"interfax.com.ua","origin":"https://interfax.com.ua","robots":{"status":"ok","http":200,"url":"https://interfax.com.ua/robots.txt","bytes":841,"groups":7},"bots":{"GPTBot":["partial","*",17,0],"ChatGPT-User":["partial","*",17,0],"OAI-SearchBot":["partial","*",17,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",17,0],"Claude-SearchBot":["partial","*",17,0],"anthropic-ai":["partial","*",17,0],"PerplexityBot":["partial","*",17,0],"Perplexity-User":["partial","*",17,0],"Google-Extended":["partial","*",17,0],"Applebot-Extended":["partial","*",17,0],"CCBot":["partial","*",17,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["partial","*",17,0],"Diffbot":["partial","*",17,0],"ContextFetchBot":["partial","*",17,0]},"llms":{"status":"missing","http":404,"bytes":6386},"home":{"status":"ok","http":200,"url":"https://interfax.com.ua/","html_tokens":44170,"markdown_tokens":12,"pruned_tokens":12}},
{"rank":6741,"domain":"healthcentral.com","origin":"https://healthcentral.com","robots":{"status":"refused","http":403,"url":"https://www.healthcentral.com/robots.txt","bytes":777,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":6742,"domain":"readspeaker.com","origin":"https://readspeaker.com","robots":{"status":"ok","http":200,"url":"https://www.readspeaker.com/robots.txt","bytes":2590,"groups":20},"bots":{"GPTBot":["partial","named",3,1],"ChatGPT-User":["partial","named",3,1],"OAI-SearchBot":["partial","named",3,1],"ClaudeBot":["partial","named",3,1],"Claude-User":["partial","named",3,1],"Claude-SearchBot":["partial","named",3,1],"anthropic-ai":["partial","named",3,1],"PerplexityBot":["partial","named",3,1],"Perplexity-User":["partial","named",3,1],"Google-Extended":["partial","named",3,1],"Applebot-Extended":["partial","named",3,1],"CCBot":["partial","named",3,1],"Bytespider":["partial","named",3,1],"Amazonbot":["partial","named",3,1],"meta-externalagent":["partial","named",3,1],"cohere-ai":["partial","named",3,1],"Diffbot":["partial","*",3,0],"ContextFetchBot":["partial","*",3,0]},"llms":{"status":"present","http":200,"bytes":50319},"home":{"status":"ok","http":200,"url":"https://www.readspeaker.com/","html_tokens":205668,"markdown_tokens":679,"pruned_tokens":226}},
{"rank":6743,"domain":"uipath.com","origin":"https://uipath.com","robots":{"status":"ok","http":200,"url":"https://www.uipath.com/robots.txt","bytes":3338,"groups":4},"bots":{"GPTBot":["partial","*",11,0],"ChatGPT-User":["partial","*",11,0],"OAI-SearchBot":["partial","*",11,0],"ClaudeBot":["partial","*",11,0],"Claude-User":["partial","*",11,0],"Claude-SearchBot":["partial","*",11,0],"anthropic-ai":["partial","*",11,0],"PerplexityBot":["partial","*",11,0],"Perplexity-User":["partial","*",11,0],"Google-Extended":["partial","*",11,0],"Applebot-Extended":["partial","*",11,0],"CCBot":["partial","*",11,0],"Bytespider":["partial","*",11,0],"Amazonbot":["partial","*",11,0],"meta-externalagent":["partial","*",11,0],"cohere-ai":["partial","*",11,0],"Diffbot":["partial","*",11,0],"ContextFetchBot":["partial","*",11,0]},"llms":{"status":"missing","http":404,"bytes":95930},"home":{"status":"ok","http":200,"url":"https://www.uipath.com/","html_tokens":182181,"markdown_tokens":933,"pruned_tokens":124}},
{"rank":6744,"domain":"airfrance.fr","origin":"https://airfrance.fr","robots":{"status":"timeout","http":null,"url":"https://airfrance.fr/robots.txt","bytes":0,"groups":0,"error":"timed out after 10 s"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":6745,"domain":"jhuapl.edu","origin":"https://jhuapl.edu","robots":{"status":"ok","http":200,"url":"https://www.jhuapl.edu/robots.txt","bytes":2141,"groups":1},"bots":{"GPTBot":["partial","*",35,22],"ChatGPT-User":["partial","*",35,22],"OAI-SearchBot":["partial","*",35,22],"ClaudeBot":["partial","*",35,22],"Claude-User":["partial","*",35,22],"Claude-SearchBot":["partial","*",35,22],"anthropic-ai":["partial","*",35,22],"PerplexityBot":["partial","*",35,22],"Perplexity-User":["partial","*",35,22],"Google-Extended":["partial","*",35,22],"Applebot-Extended":["partial","*",35,22],"CCBot":["partial","*",35,22],"Bytespider":["partial","*",35,22],"Amazonbot":["partial","*",35,22],"meta-externalagent":["partial","*",35,22],"cohere-ai":["partial","*",35,22],"Diffbot":["partial","*",35,22],"ContextFetchBot":["partial","*",35,22]},"llms":{"status":"missing","http":404,"bytes":188},"home":{"status":"ok","http":200,"url":"https://www.jhuapl.edu/","html_tokens":57262,"markdown_tokens":385,"pruned_tokens":101}},
{"rank":6746,"domain":"renewableenergyworld.com","origin":"https://renewableenergyworld.com","robots":{"status":"ok","http":200,"url":"https://www.renewableenergyworld.com/robots.txt","bytes":2112,"groups":2},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["partial","*",19,1],"OAI-SearchBot":["partial","*",19,1],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",19,1],"Claude-SearchBot":["partial","*",19,1],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["partial","*",19,1],"Perplexity-User":["partial","*",19,1],"Google-Extended":["partial","*",19,1],"Applebot-Extended":["partial","*",19,1],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["partial","*",19,1],"cohere-ai":["partial","*",19,1],"Diffbot":["partial","*",19,1],"ContextFetchBot":["partial","*",19,1]},"llms":{"status":"missing","http":404,"bytes":102394},"home":{"status":"ok","http":200,"url":"https://www.renewableenergyworld.com/","html_tokens":58795,"markdown_tokens":923,"pruned_tokens":133}},
{"rank":6747,"domain":"oeaw.ac.at","origin":"https://oeaw.ac.at","robots":{"status":"ok","http":200,"url":"https://www.oeaw.ac.at/robots.txt","bytes":613,"groups":3},"bots":{"GPTBot":["partial","*",16,0],"ChatGPT-User":["partial","*",16,0],"OAI-SearchBot":["partial","*",16,0],"ClaudeBot":["partial","*",16,0],"Claude-User":["partial","*",16,0],"Claude-SearchBot":["partial","*",16,0],"anthropic-ai":["partial","*",16,0],"PerplexityBot":["partial","*",16,0],"Perplexity-User":["partial","*",16,0],"Google-Extended":["partial","*",16,0],"Applebot-Extended":["partial","*",16,0],"CCBot":["partial","*",16,0],"Bytespider":["partial","*",16,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["partial","*",16,0],"cohere-ai":["partial","*",16,0],"Diffbot":["partial","*",16,0],"ContextFetchBot":["partial","*",16,0]},"llms":{"status":"http_error","http":502,"bytes":6398},"home":{"status":"http_error","http":403,"url":"https://www.oeaw.ac.at/"}},
{"rank":6748,"domain":"haufe.de","origin":"https://haufe.de","robots":{"status":"ok","http":200,"url":"https://www.haufe.de/robots.txt","bytes":1207,"groups":22},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["partial","*",2,3],"OAI-SearchBot":["partial","*",2,3],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",2,3],"Claude-SearchBot":["partial","*",2,3],"anthropic-ai":["partial","*",2,3],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["partial","*",2,3],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["partial","*",2,3],"Diffbot":["partial","*",2,3],"ContextFetchBot":["partial","*",2,3]},"llms":{"status":"missing","http":404,"bytes":106360},"home":{"status":"ok","http":200,"url":"https://www.haufe.de/","html_tokens":21391,"markdown_tokens":603,"pruned_tokens":148}},
{"rank":6749,"domain":"acsm.org","origin":"https://acsm.org","robots":{"status":"ok","http":200,"url":"https://acsm.org/robots.txt","bytes":166,"groups":1},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"other_text","http":200,"bytes":4361},"home":{"status":"ok","http":200,"url":"https://acsm.org/","html_tokens":119831,"markdown_tokens":291,"pruned_tokens":175}},
{"rank":6750,"domain":"setlist.fm","origin":"https://setlist.fm","robots":{"status":"ok","http":200,"url":"https://www.setlist.fm/robots.txt","bytes":2669,"groups":28},"bots":{"GPTBot":["partial","*",6,0],"ChatGPT-User":["partial","*",6,0],"OAI-SearchBot":["partial","*",6,0],"ClaudeBot":["partial","*",6,0],"Claude-User":["partial","*",6,0],"Claude-SearchBot":["partial","*",6,0],"anthropic-ai":["partial","*",6,0],"PerplexityBot":["partial","*",6,0],"Perplexity-User":["partial","*",6,0],"Google-Extended":["partial","*",6,0],"Applebot-Extended":["partial","*",6,0],"CCBot":["partial","*",6,0],"Bytespider":["partial","*",6,0],"Amazonbot":["partial","*",6,0],"meta-externalagent":["partial","*",6,0],"cohere-ai":["partial","*",6,0],"Diffbot":["partial","*",6,0],"ContextFetchBot":["partial","*",6,0]},"llms":{"status":"timeout","error":"timed out after 10 s"},"home":{"status":"ok","http":200,"url":"https://www.setlist.fm/","html_tokens":34856,"markdown_tokens":58,"pruned_tokens":58}},
{"rank":6751,"domain":"grundfos.com","origin":"https://grundfos.com","robots":{"status":"ok","http":200,"url":"https://www.grundfos.com/robots.txt","bytes":3303,"groups":1},"bots":{"GPTBot":["partial","*",2,0],"ChatGPT-User":["partial","*",2,0],"OAI-SearchBot":["partial","*",2,0],"ClaudeBot":["partial","*",2,0],"Claude-User":["partial","*",2,0],"Claude-SearchBot":["partial","*",2,0],"anthropic-ai":["partial","*",2,0],"PerplexityBot":["partial","*",2,0],"Perplexity-User":["partial","*",2,0],"Google-Extended":["partial","*",2,0],"Applebot-Extended":["partial","*",2,0],"CCBot":["partial","*",2,0],"Bytespider":["partial","*",2,0],"Amazonbot":["partial","*",2,0],"meta-externalagent":["partial","*",2,0],"cohere-ai":["partial","*",2,0],"Diffbot":["partial","*",2,0],"ContextFetchBot":["partial","*",2,0]},"llms":{"status":"missing","http":404,"bytes":94},"home":{"status":"ok","http":200,"url":"https://www.grundfos.com/","html_tokens":28885,"markdown_tokens":160,"pruned_tokens":98}},
{"rank":6752,"domain":"zjol.com.cn","origin":"https://zjol.com.cn","robots":{"status":"html","http":200,"url":"https://www.zjol.com.cn/robots.txt","bytes":8234,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":6753,"domain":"isc2.org","origin":"https://isc2.org","robots":{"status":"html","http":200,"url":"https://isc2.org/robots.txt","bytes":269,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":6754,"domain":"drive2.ru","origin":"https://drive2.ru","robots":{"status":"refused","http":403,"url":"https://www.drive2.ru/robots.txt","bytes":902,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":6755,"domain":"lazada.com","origin":"https://lazada.com","robots":{"status":"html","http":200,"url":"https://lazada.com/en/robots.txt/","bytes":600,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":6756,"domain":"citizen.org","origin":"https://citizen.org","robots":{"status":"ok","http":200,"url":"https://www.citizen.org/robots.txt","bytes":189,"groups":1},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"http_error","http":403,"bytes":5542},"home":{"status":"http_error","http":403,"url":"https://citizen.org/"}},
{"rank":6757,"domain":"dcu.ie","origin":"https://dcu.ie","robots":{"status":"ok","http":200,"url":"https://www.dcu.ie/robots.txt","bytes":2529,"groups":24},"bots":{"GPTBot":["partial","*",23,18],"ChatGPT-User":["partial","*",23,18],"OAI-SearchBot":["partial","*",23,18],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",23,18],"Claude-SearchBot":["partial","*",23,18],"anthropic-ai":["partial","*",23,18],"PerplexityBot":["partial","*",23,18],"Perplexity-User":["partial","*",23,18],"Google-Extended":["partial","*",23,18],"Applebot-Extended":["partial","*",23,18],"CCBot":["partial","*",23,18],"Bytespider":["partial","*",23,18],"Amazonbot":["partial","*",23,18],"meta-externalagent":["partial","*",23,18],"cohere-ai":["partial","*",23,18],"Diffbot":["partial","*",23,18],"ContextFetchBot":["partial","*",23,18]},"llms":{"status":"missing","http":404,"bytes":184},"home":{"status":"ok","http":200,"url":"https://www.dcu.ie/","html_tokens":40358,"markdown_tokens":105,"pruned_tokens":105}},
{"rank":6758,"domain":"ks.gov","origin":"https://ks.gov","robots":{"status":"timeout","http":null,"url":"https://ks.gov/robots.txt","bytes":0,"groups":0,"error":"ConnectTimeout"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":6759,"domain":"elearningindustry.com","origin":"https://elearningindustry.com","robots":{"status":"ok","http":200,"url":"https://elearningindustry.com/robots.txt","bytes":3462,"groups":49},"bots":{"GPTBot":["partial","*",67,6],"ChatGPT-User":["partial","*",67,6],"OAI-SearchBot":["partial","*",67,6],"ClaudeBot":["partial","*",67,6],"Claude-User":["partial","*",67,6],"Claude-SearchBot":["partial","*",67,6],"anthropic-ai":["partial","*",67,6],"PerplexityBot":["partial","*",67,6],"Perplexity-User":["partial","*",67,6],"Google-Extended":["partial","*",67,6],"Applebot-Extended":["partial","*",67,6],"CCBot":["partial","*",67,6],"Bytespider":["partial","*",67,6],"Amazonbot":["partial","*",67,6],"meta-externalagent":["partial","*",67,6],"cohere-ai":["partial","*",67,6],"Diffbot":["partial","*",67,6],"ContextFetchBot":["partial","*",67,6]},"llms":{"status":"missing","http":404,"bytes":189050},"home":{"status":"ok","http":200,"url":"https://elearningindustry.com/","html_tokens":91097,"markdown_tokens":117,"pruned_tokens":117}},
{"rank":6760,"domain":"google.ad","origin":"https://google.ad","robots":{"status":"ok","http":200,"url":"https://www.google.ad/robots.txt","bytes":6624,"groups":4},"bots":{"GPTBot":["partial","*",169,64],"ChatGPT-User":["partial","*",169,64],"OAI-SearchBot":["partial","*",169,64],"ClaudeBot":["partial","*",169,64],"Claude-User":["partial","*",169,64],"Claude-SearchBot":["partial","*",169,64],"anthropic-ai":["partial","*",169,64],"PerplexityBot":["partial","*",169,64],"Perplexity-User":["partial","*",169,64],"Google-Extended":["partial","*",169,64],"Applebot-Extended":["partial","*",169,64],"CCBot":["partial","*",169,64],"Bytespider":["partial","*",169,64],"Amazonbot":["partial","*",169,64],"meta-externalagent":["partial","*",169,64],"cohere-ai":["partial","*",169,64],"Diffbot":["partial","*",169,64],"ContextFetchBot":["partial","*",169,64]},"llms":{"status":"missing","http":404,"bytes":1569},"home":{"status":"ok","http":200,"url":"https://www.google.com/","html_tokens":21354,"markdown_tokens":30,"pruned_tokens":30}},
{"rank":6761,"domain":"etnews.com","origin":"https://etnews.com","robots":{"status":"ok","http":200,"url":"https://www.etnews.com/robots.txt","bytes":307,"groups":1},"bots":{"GPTBot":["allowed","*",0,1],"ChatGPT-User":["allowed","*",0,1],"OAI-SearchBot":["allowed","*",0,1],"ClaudeBot":["allowed","*",0,1],"Claude-User":["allowed","*",0,1],"Claude-SearchBot":["allowed","*",0,1],"anthropic-ai":["allowed","*",0,1],"PerplexityBot":["allowed","*",0,1],"Perplexity-User":["allowed","*",0,1],"Google-Extended":["allowed","*",0,1],"Applebot-Extended":["allowed","*",0,1],"CCBot":["allowed","*",0,1],"Bytespider":["allowed","*",0,1],"Amazonbot":["allowed","*",0,1],"meta-externalagent":["allowed","*",0,1],"cohere-ai":["allowed","*",0,1],"Diffbot":["allowed","*",0,1],"ContextFetchBot":["allowed","*",0,1]},"llms":{"status":"missing","http":404,"bytes":371},"home":{"status":"ok","http":200,"url":"https://www.etnews.com/","html_tokens":17524,"markdown_tokens":358,"pruned_tokens":162}},
{"rank":6762,"domain":"tidio.com","origin":"https://tidio.com","robots":{"status":"ok","http":200,"url":"https://www.tidio.com/robots.txt","bytes":777,"groups":2},"bots":{"GPTBot":["partial","*",5,0],"ChatGPT-User":["partial","*",5,0],"OAI-SearchBot":["partial","*",5,0],"ClaudeBot":["partial","*",5,0],"Claude-User":["partial","*",5,0],"Claude-SearchBot":["partial","*",5,0],"anthropic-ai":["partial","*",5,0],"PerplexityBot":["partial","*",5,0],"Perplexity-User":["partial","*",5,0],"Google-Extended":["partial","*",5,0],"Applebot-Extended":["partial","*",5,0],"CCBot":["partial","*",5,0],"Bytespider":["partial","*",5,0],"Amazonbot":["partial","*",5,0],"meta-externalagent":["partial","*",5,0],"cohere-ai":["partial","*",5,0],"Diffbot":["partial","*",5,0],"ContextFetchBot":["partial","*",5,0]},"llms":{"status":"other_text","http":200,"bytes":13275},"home":{"status":"ok","http":200,"url":"https://www.tidio.com/","html_tokens":108501,"markdown_tokens":1725,"pruned_tokens":216}},
{"rank":6763,"domain":"barcelo.com","origin":"https://barcelo.com","robots":{"status":"ok","http":200,"url":"https://www.barcelo.com/robots.txt","bytes":7178,"groups":1},"bots":{"GPTBot":["partial","*",133,3],"ChatGPT-User":["partial","*",133,3],"OAI-SearchBot":["partial","*",133,3],"ClaudeBot":["partial","*",133,3],"Claude-User":["partial","*",133,3],"Claude-SearchBot":["partial","*",133,3],"anthropic-ai":["partial","*",133,3],"PerplexityBot":["partial","*",133,3],"Perplexity-User":["partial","*",133,3],"Google-Extended":["partial","*",133,3],"Applebot-Extended":["partial","*",133,3],"CCBot":["partial","*",133,3],"Bytespider":["partial","*",133,3],"Amazonbot":["partial","*",133,3],"meta-externalagent":["partial","*",133,3],"cohere-ai":["partial","*",133,3],"Diffbot":["partial","*",133,3],"ContextFetchBot":["partial","*",133,3]},"llms":{"status":"present","http":200,"bytes":7138},"home":{"status":"ok","http":200,"url":"https://www.barcelo.com/en-us/","html_tokens":143234,"markdown_tokens":692,"pruned_tokens":118}},
{"rank":6764,"domain":"kottke.org","origin":"https://kottke.org","robots":{"status":"ok","http":200,"url":"https://kottke.org/robots.txt","bytes":486,"groups":8},"bots":{"GPTBot":["allowed","named",0,1],"ChatGPT-User":["partial","*",3,0],"OAI-SearchBot":["allowed","named",0,1],"ClaudeBot":["allowed","named",0,1],"Claude-User":["partial","*",3,0],"Claude-SearchBot":["partial","*",3,0],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["allowed","named",0,1],"Perplexity-User":["partial","*",3,0],"Google-Extended":["allowed","named",0,1],"Applebot-Extended":["partial","*",3,0],"CCBot":["blocked","named",1,0],"Bytespider":["partial","*",3,0],"Amazonbot":["partial","*",3,0],"meta-externalagent":["partial","*",3,0],"cohere-ai":["partial","*",3,0],"Diffbot":["partial","*",3,0],"ContextFetchBot":["partial","*",3,0]},"llms":{"status":"present","http":200,"bytes":982},"home":{"status":"ok","http":200,"url":"https://kottke.org/","html_tokens":89857,"markdown_tokens":5342,"pruned_tokens":51}},
{"rank":6765,"domain":"ds28perm.ru","origin":"https://ds28perm.ru","robots":{"status":"ok","http":200,"url":"https://ds28perm.ru/robots.txt","bytes":172992,"groups":32},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["allowed","*",0,1],"OAI-SearchBot":["allowed","*",0,1],"ClaudeBot":["blocked","named",1,0],"Claude-User":["allowed","*",0,1],"Claude-SearchBot":["allowed","*",0,1],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["allowed","*",0,1],"Perplexity-User":["allowed","*",0,1],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["allowed","*",0,1]},"llms":{"status":"html","http":200,"bytes":170683},"home":{"status":"ok","http":200,"url":"https://ds28perm.ru/","html_tokens":42646,"markdown_tokens":18,"pruned_tokens":18}},
{"rank":6766,"domain":"met.police.uk","origin":"https://met.police.uk","robots":{"status":"refused","http":403,"url":"https://www.met.police.uk/robots.txt","bytes":5576,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":6767,"domain":"bandlab.com","origin":"https://bandlab.com","robots":{"status":"ok","http":200,"url":"https://www.bandlab.com/robots.txt","bytes":1206,"groups":10},"bots":{"GPTBot":["partial","named",1,25],"ChatGPT-User":["partial","*",9,0],"OAI-SearchBot":["partial","*",9,0],"ClaudeBot":["partial","named",1,25],"Claude-User":["partial","*",9,0],"Claude-SearchBot":["partial","*",9,0],"anthropic-ai":["partial","*",9,0],"PerplexityBot":["partial","*",9,0],"Perplexity-User":["partial","*",9,0],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["partial","*",9,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["partial","*",9,0],"cohere-ai":["partial","*",9,0],"Diffbot":["partial","*",9,0],"ContextFetchBot":["partial","*",9,0]},"llms":{"status":"html","http":200,"bytes":4292},"home":{"status":"ok","http":200,"url":"https://www.bandlab.com/","html_tokens":1073,"markdown_tokens":0,"pruned_tokens":0}},
{"rank":6768,"domain":"nat.gov.tw","origin":"https://nat.gov.tw","robots":{"status":"unreachable","http":null,"url":"https://nat.gov.tw/robots.txt","bytes":0,"groups":0,"error":"cannot resolve nat.gov.tw"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":6769,"domain":"passiongames-es.com","origin":"https://passiongames-es.com","robots":{"status":"ok","http":200,"url":"https://passiongames-es.com/robots.txt","bytes":543,"groups":1},"bots":{"GPTBot":["partial","*",19,0],"ChatGPT-User":["partial","*",19,0],"OAI-SearchBot":["partial","*",19,0],"ClaudeBot":["partial","*",19,0],"Claude-User":["partial","*",19,0],"Claude-SearchBot":["partial","*",19,0],"anthropic-ai":["partial","*",19,0],"PerplexityBot":["partial","*",19,0],"Perplexity-User":["partial","*",19,0],"Google-Extended":["partial","*",19,0],"Applebot-Extended":["partial","*",19,0],"CCBot":["partial","*",19,0],"Bytespider":["partial","*",19,0],"Amazonbot":["partial","*",19,0],"meta-externalagent":["partial","*",19,0],"cohere-ai":["partial","*",19,0],"Diffbot":["partial","*",19,0],"ContextFetchBot":["partial","*",19,0]},"llms":{"status":"missing","http":404,"bytes":15439},"home":{"status":"ok","http":200,"url":"https://passiongames-es.com/","html_tokens":19310,"markdown_tokens":53,"pruned_tokens":53}},
{"rank":6770,"domain":"mybluehost.me","origin":"http://mybluehost.me","robots":{"status":"refused","http":409,"url":"http://mybluehost.me/robots.txt","bytes":7867,"groups":0,"error":"HTTP 409"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":6771,"domain":"gate.io","origin":"https://gate.io","robots":{"status":"ok","http":200,"url":"https://www.gate.com/robots.txt","bytes":5595,"groups":2},"bots":{"GPTBot":["partial","*",109,17],"ChatGPT-User":["partial","*",109,17],"OAI-SearchBot":["partial","*",109,17],"ClaudeBot":["partial","*",109,17],"Claude-User":["partial","*",109,17],"Claude-SearchBot":["partial","*",109,17],"anthropic-ai":["partial","*",109,17],"PerplexityBot":["partial","*",109,17],"Perplexity-User":["partial","*",109,17],"Google-Extended":["partial","*",109,17],"Applebot-Extended":["partial","*",109,17],"CCBot":["partial","*",109,17],"Bytespider":["partial","*",109,17],"Amazonbot":["partial","*",109,17],"meta-externalagent":["partial","*",109,17],"cohere-ai":["partial","*",109,17],"Diffbot":["partial","*",109,17],"ContextFetchBot":["partial","*",109,17]},"llms":{"status":"other_text","http":200,"bytes":11765},"home":{"status":"ok","http":200,"url":"https://www.gate.com/","html_tokens":51541,"markdown_tokens":6,"pruned_tokens":6}},
{"rank":6772,"domain":"climatecentral.org","origin":"https://climatecentral.org","robots":{"status":"ok","http":200,"url":"https://www.climatecentral.org/robots.txt","bytes":72,"groups":1},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"missing","http":404,"bytes":61932},"home":{"status":"ok","http":200,"url":"https://www.climatecentral.org/","html_tokens":60248,"markdown_tokens":192,"pruned_tokens":96}},
{"rank":6773,"domain":"udg.mx","origin":"https://udg.mx","robots":{"status":"ok","http":200,"url":"https://udg.mx/robots.txt","bytes":2027,"groups":1},"bots":{"GPTBot":["partial","*",34,18],"ChatGPT-User":["partial","*",34,18],"OAI-SearchBot":["partial","*",34,18],"ClaudeBot":["partial","*",34,18],"Claude-User":["partial","*",34,18],"Claude-SearchBot":["partial","*",34,18],"anthropic-ai":["partial","*",34,18],"PerplexityBot":["partial","*",34,18],"Perplexity-User":["partial","*",34,18],"Google-Extended":["partial","*",34,18],"Applebot-Extended":["partial","*",34,18],"CCBot":["partial","*",34,18],"Bytespider":["partial","*",34,18],"Amazonbot":["partial","*",34,18],"meta-externalagent":["partial","*",34,18],"cohere-ai":["partial","*",34,18],"Diffbot":["partial","*",34,18],"ContextFetchBot":["partial","*",34,18]},"llms":{"status":"http_error","http":403,"bytes":153},"home":{"status":"ok","http":200,"url":"https://udg.mx/","html_tokens":15067,"markdown_tokens":23,"pruned_tokens":23}},
{"rank":6774,"domain":"top500.org","origin":"https://top500.org","robots":{"status":"missing","http":404,"url":"https://top500.org/robots.txt/","bytes":45043,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":45043},"home":{"status":"ok","http":200,"url":"https://top500.org/","html_tokens":19077,"markdown_tokens":283,"pruned_tokens":132}},
{"rank":6775,"domain":"qwen.ai","origin":"https://qwen.ai","robots":{"status":"html","http":200,"url":"https://qwen.ai/","bytes":94344,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":6776,"domain":"google.ms","origin":"https://google.ms","robots":{"status":"ok","http":200,"url":"https://www.google.ms/robots.txt","bytes":6624,"groups":4},"bots":{"GPTBot":["partial","*",169,64],"ChatGPT-User":["partial","*",169,64],"OAI-SearchBot":["partial","*",169,64],"ClaudeBot":["partial","*",169,64],"Claude-User":["partial","*",169,64],"Claude-SearchBot":["partial","*",169,64],"anthropic-ai":["partial","*",169,64],"PerplexityBot":["partial","*",169,64],"Perplexity-User":["partial","*",169,64],"Google-Extended":["partial","*",169,64],"Applebot-Extended":["partial","*",169,64],"CCBot":["partial","*",169,64],"Bytespider":["partial","*",169,64],"Amazonbot":["partial","*",169,64],"meta-externalagent":["partial","*",169,64],"cohere-ai":["partial","*",169,64],"Diffbot":["partial","*",169,64],"ContextFetchBot":["partial","*",169,64]},"llms":{"status":"missing","http":404,"bytes":1569},"home":{"status":"ok","http":200,"url":"https://www.google.ms/","html_tokens":21352,"markdown_tokens":32,"pruned_tokens":32}},
{"rank":6777,"domain":"s.id","origin":"https://s.id","robots":{"status":"ok","http":200,"url":"https://s.id/robots.txt","bytes":21,"groups":1},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"missing","http":404,"bytes":1255},"home":{"status":"ok","http":200,"url":"https://home.s.id/gl/en","html_tokens":50144,"markdown_tokens":802,"pruned_tokens":75}},
{"rank":6778,"domain":"lesswrong.com","origin":"https://lesswrong.com","robots":{"status":"ok","http":200,"url":"https://www.lesswrong.com/robots.txt","bytes":486,"groups":2},"bots":{"GPTBot":["partial","*",13,0],"ChatGPT-User":["partial","*",13,0],"OAI-SearchBot":["partial","*",13,0],"ClaudeBot":["partial","*",13,0],"Claude-User":["partial","*",13,0],"Claude-SearchBot":["partial","*",13,0],"anthropic-ai":["partial","*",13,0],"PerplexityBot":["partial","*",13,0],"Perplexity-User":["partial","*",13,0],"Google-Extended":["partial","*",13,0],"Applebot-Extended":["partial","*",13,0],"CCBot":["partial","*",13,0],"Bytespider":["partial","*",13,0],"Amazonbot":["partial","*",13,0],"meta-externalagent":["partial","*",13,0],"cohere-ai":["partial","*",13,0],"Diffbot":["partial","*",13,0],"ContextFetchBot":["partial","*",13,0]},"llms":{"status":"missing","http":404,"bytes":138119},"home":{"status":"ok","http":200,"url":"https://www.lesswrong.com/","html_tokens":227209,"markdown_tokens":501,"pruned_tokens":169}},
{"rank":6779,"domain":"famfamfam.com","origin":"https://famfamfam.com","robots":{"status":"unreachable","http":null,"url":"https://famfamfam.com/robots.txt","bytes":0,"groups":0,"error":"cannot resolve famfamfam.com"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":6780,"domain":"mt.co.kr","origin":"https://mt.co.kr","robots":{"status":"ok","http":200,"url":"https://www.mt.co.kr/robots.txt","bytes":8697,"groups":58},"bots":{"GPTBot":["allowed","named",0,1],"ChatGPT-User":["partial","*",7,6],"OAI-SearchBot":["allowed","named",0,1],"ClaudeBot":["allowed","named",0,1],"Claude-User":["partial","*",7,6],"Claude-SearchBot":["partial","*",7,6],"anthropic-ai":["allowed","named",0,1],"PerplexityBot":["allowed","named",0,1],"Perplexity-User":["partial","*",7,6],"Google-Extended":["allowed","named",0,1],"Applebot-Extended":["partial","*",7,6],"CCBot":["allowed","named",0,1],"Bytespider":["blocked","named",1,0],"Amazonbot":["partial","*",7,6],"meta-externalagent":["partial","*",7,6],"cohere-ai":["allowed","named",0,1],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",7,6]},"llms":{"status":"missing","http":404,"bytes":42043},"home":{"status":"ok","http":200,"url":"https://www.mt.co.kr/","html_tokens":47418,"markdown_tokens":143,"pruned_tokens":143}},
{"rank":6781,"domain":"bonhams.com","origin":"https://bonhams.com","robots":{"status":"ok","http":200,"url":"https://www.bonhams.com/robots.txt","bytes":180,"groups":2},"bots":{"GPTBot":["partial","*",4,0],"ChatGPT-User":["partial","*",4,0],"OAI-SearchBot":["partial","*",4,0],"ClaudeBot":["partial","*",4,0],"Claude-User":["partial","*",4,0],"Claude-SearchBot":["partial","*",4,0],"anthropic-ai":["partial","*",4,0],"PerplexityBot":["partial","*",4,0],"Perplexity-User":["partial","*",4,0],"Google-Extended":["partial","*",4,0],"Applebot-Extended":["partial","*",4,0],"CCBot":["partial","*",4,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["partial","*",4,0],"meta-externalagent":["partial","*",4,0],"cohere-ai":["partial","*",4,0],"Diffbot":["partial","*",4,0],"ContextFetchBot":["partial","*",4,0]},"llms":{"status":"missing","http":404,"bytes":74186},"home":{"status":"ok","http":200,"url":"https://www.bonhams.com/","html_tokens":75385,"markdown_tokens":11,"pruned_tokens":11}},
{"rank":6782,"domain":"ernet.in","origin":"https://ernet.in","robots":{"status":"ok","http":200,"url":"https://xn--m1bd4e9a1b9i.xn--h2brj9c/robots.txt","bytes":2189,"groups":1},"bots":{"GPTBot":["partial","*",36,32],"ChatGPT-User":["partial","*",36,32],"OAI-SearchBot":["partial","*",36,32],"ClaudeBot":["partial","*",36,32],"Claude-User":["partial","*",36,32],"Claude-SearchBot":["partial","*",36,32],"anthropic-ai":["partial","*",36,32],"PerplexityBot":["partial","*",36,32],"Perplexity-User":["partial","*",36,32],"Google-Extended":["partial","*",36,32],"Applebot-Extended":["partial","*",36,32],"CCBot":["partial","*",36,32],"Bytespider":["partial","*",36,32],"Amazonbot":["partial","*",36,32],"meta-externalagent":["partial","*",36,32],"cohere-ai":["partial","*",36,32],"Diffbot":["partial","*",36,32],"ContextFetchBot":["partial","*",36,32]},"llms":{"status":"missing","http":404,"bytes":291},"home":{"status":"ok","http":200,"url":"https://xn--m1bd4e9a1b9i.xn--h2brj9c/","html_tokens":20858,"markdown_tokens":145,"pruned_tokens":145}},
{"rank":6783,"domain":"safer-networking.org","origin":"https://safer-networking.org","robots":{"status":"ok","http":200,"url":"https://www.safer-networking.org/robots.txt","bytes":182,"groups":1},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"missing","http":404,"bytes":50484},"home":{"status":"ok","http":200,"url":"https://www.safer-networking.org/","html_tokens":19337,"markdown_tokens":377,"pruned_tokens":294}},
{"rank":6784,"domain":"aemet.es","origin":"https://www.aemet.es","robots":{"status":"ok","http":200,"url":"https://www.aemet.es/robots.txt","bytes":351,"groups":1},"bots":{"GPTBot":["partial","*",7,0],"ChatGPT-User":["partial","*",7,0],"OAI-SearchBot":["partial","*",7,0],"ClaudeBot":["partial","*",7,0],"Claude-User":["partial","*",7,0],"Claude-SearchBot":["partial","*",7,0],"anthropic-ai":["partial","*",7,0],"PerplexityBot":["partial","*",7,0],"Perplexity-User":["partial","*",7,0],"Google-Extended":["partial","*",7,0],"Applebot-Extended":["partial","*",7,0],"CCBot":["partial","*",7,0],"Bytespider":["partial","*",7,0],"Amazonbot":["partial","*",7,0],"meta-externalagent":["partial","*",7,0],"cohere-ai":["partial","*",7,0],"Diffbot":["partial","*",7,0],"ContextFetchBot":["partial","*",7,0]},"llms":{"status":"missing","http":404,"bytes":10859},"home":{"status":"ok","http":200,"url":"https://www.aemet.es/es/portada","html_tokens":17075,"markdown_tokens":59,"pruned_tokens":59}},
{"rank":6785,"domain":"kitchenaid.com","origin":"https://kitchenaid.com","robots":{"status":"unreachable","http":null,"url":"https://kitchenaid.com/robots.txt","bytes":0,"groups":0,"error":"All connection attempts failed"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":6786,"domain":"furaffinity.net","origin":"https://furaffinity.net","robots":{"status":"ok","http":200,"url":"https://www.furaffinity.net/robots.txt","bytes":5326,"groups":2},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["blocked","named",1,0],"OAI-SearchBot":["blocked","named",1,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["blocked","named",1,0],"Claude-SearchBot":["blocked","named",1,0],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["blocked","named",1,0],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",45,0]},"llms":{"status":"missing","http":404,"bytes":2072},"home":{"status":"ok","http":200,"url":"https://www.furaffinity.net/","html_tokens":30518,"markdown_tokens":1992,"pruned_tokens":173}},
{"rank":6787,"domain":"ketorolix.com","origin":"https://ketorolix.com","robots":{"status":"ok","http":200,"url":"https://ketorolix.com/robots.txt","bytes":73,"groups":1},"bots":{"GPTBot":["partial","*",1,0],"ChatGPT-User":["partial","*",1,0],"OAI-SearchBot":["partial","*",1,0],"ClaudeBot":["partial","*",1,0],"Claude-User":["partial","*",1,0],"Claude-SearchBot":["partial","*",1,0],"anthropic-ai":["partial","*",1,0],"PerplexityBot":["partial","*",1,0],"Perplexity-User":["partial","*",1,0],"Google-Extended":["partial","*",1,0],"Applebot-Extended":["partial","*",1,0],"CCBot":["partial","*",1,0],"Bytespider":["partial","*",1,0],"Amazonbot":["partial","*",1,0],"meta-externalagent":["partial","*",1,0],"cohere-ai":["partial","*",1,0],"Diffbot":["partial","*",1,0],"ContextFetchBot":["partial","*",1,0]},"llms":{"status":"missing","http":404,"bytes":642},"home":{"status":"ok","http":200,"url":"https://ketorolix.com/","html_tokens":14959,"markdown_tokens":3372,"pruned_tokens":155}},
{"rank":6788,"domain":"physorg.com","origin":"http://physorg.com","robots":{"status":"ok","http":200,"url":"https://phys.org/robots.txt","bytes":1146,"groups":15},"bots":{"GPTBot":["partial","named",4,0],"ChatGPT-User":["partial","*",5,1],"OAI-SearchBot":["partial","*",5,1],"ClaudeBot":["partial","named",4,0],"Claude-User":["partial","*",5,1],"Claude-SearchBot":["partial","*",5,1],"anthropic-ai":["partial","*",5,1],"PerplexityBot":["partial","*",5,1],"Perplexity-User":["partial","*",5,1],"Google-Extended":["partial","named",2,0],"Applebot-Extended":["partial","named",2,0],"CCBot":["partial","named",4,0],"Bytespider":["partial","named",4,0],"Amazonbot":["partial","*",5,1],"meta-externalagent":["partial","*",5,1],"cohere-ai":["partial","named",4,0],"Diffbot":["partial","*",5,1],"ContextFetchBot":["partial","*",5,1]},"llms":{"status":"present","http":200,"bytes":2417},"home":{"status":"ok","http":200,"url":"https://phys.org/","html_tokens":51445,"markdown_tokens":2195,"pruned_tokens":144}},
{"rank":6789,"domain":"gwdg.de","origin":"https://gwdg.de","robots":{"status":"missing","http":404,"url":"https://gwdg.de/robots.txt","bytes":13141,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":13141},"home":{"status":"ok","http":200,"url":"https://gwdg.de/","html_tokens":4251,"markdown_tokens":185,"pruned_tokens":51}},
{"rank":6790,"domain":"mining.com","origin":"https://mining.com","robots":{"status":"refused","http":403,"url":"https://mining.com/robots.txt","bytes":919,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":6791,"domain":"ucas.com","origin":"https://ucas.com","robots":{"status":"ok","http":200,"url":"https://www.ucas.com/robots.txt","bytes":3172,"groups":1},"bots":{"GPTBot":["partial","*",47,32],"ChatGPT-User":["partial","*",47,32],"OAI-SearchBot":["partial","*",47,32],"ClaudeBot":["partial","*",47,32],"Claude-User":["partial","*",47,32],"Claude-SearchBot":["partial","*",47,32],"anthropic-ai":["partial","*",47,32],"PerplexityBot":["partial","*",47,32],"Perplexity-User":["partial","*",47,32],"Google-Extended":["partial","*",47,32],"Applebot-Extended":["partial","*",47,32],"CCBot":["partial","*",47,32],"Bytespider":["partial","*",47,32],"Amazonbot":["partial","*",47,32],"meta-externalagent":["partial","*",47,32],"cohere-ai":["partial","*",47,32],"Diffbot":["partial","*",47,32],"ContextFetchBot":["partial","*",47,32]},"llms":{"status":"missing","http":404,"bytes":547},"home":{"status":"ok","http":200,"url":"https://www.ucas.com/","html_tokens":35160,"markdown_tokens":183,"pruned_tokens":56}},
{"rank":6792,"domain":"arts.ac.uk","origin":"https://arts.ac.uk","robots":{"status":"refused","http":403,"url":"https://www.arts.ac.uk/robots.txt","bytes":5552,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":6793,"domain":"ktvu.com","origin":"https://ktvu.com","robots":{"status":"ok","http":200,"url":"https://www.ktvu.com/robots.txt","bytes":790,"groups":7},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["partial","*",11,0],"OAI-SearchBot":["partial","*",11,0],"ClaudeBot":["partial","*",11,0],"Claude-User":["partial","*",11,0],"Claude-SearchBot":["partial","*",11,0],"anthropic-ai":["partial","*",11,0],"PerplexityBot":["partial","*",11,0],"Perplexity-User":["partial","*",11,0],"Google-Extended":["partial","*",11,0],"Applebot-Extended":["partial","*",11,0],"CCBot":["partial","*",11,0],"Bytespider":["partial","*",11,0],"Amazonbot":["partial","*",11,0],"meta-externalagent":["partial","*",11,0],"cohere-ai":["partial","*",11,0],"Diffbot":["partial","*",11,0],"ContextFetchBot":["partial","*",11,0]},"llms":{"status":"missing","http":404,"bytes":0},"home":{"status":"ok","http":200,"url":"https://www.ktvu.com/","html_tokens":117429,"markdown_tokens":197,"pruned_tokens":149}},
{"rank":6794,"domain":"knoxnews.com","origin":"https://knoxnews.com","robots":{"status":"ok","http":200,"url":"https://www.knoxnews.com/robots.txt","bytes":14345,"groups":288},"bots":{"GPTBot":["partial","named",1,6],"ChatGPT-User":["partial","named",1,6],"OAI-SearchBot":["partial","named",1,6],"ClaudeBot":["blocked","named",1,0],"Claude-User":["blocked","named",1,0],"Claude-SearchBot":["blocked","named",1,0],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["partial","named",1,6],"Perplexity-User":["blocked","named",1,0],"Google-Extended":["partial","named",1,6],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",2,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",32,0]},"llms":{"status":"http_error","http":402,"bytes":254},"home":{"status":"http_error","http":402,"url":"https://www.knoxnews.com/"}},
{"rank":6795,"domain":"kleinezeitung.at","origin":"https://kleinezeitung.at","robots":{"status":"ok","http":200,"url":"https://www.kleinezeitung.at/robots.txt","bytes":2226,"groups":3},"bots":{"GPTBot":["partial","*",21,6],"ChatGPT-User":["partial","*",21,6],"OAI-SearchBot":["partial","*",21,6],"ClaudeBot":["partial","*",21,6],"Claude-User":["partial","*",21,6],"Claude-SearchBot":["partial","*",21,6],"anthropic-ai":["partial","*",21,6],"PerplexityBot":["partial","*",21,6],"Perplexity-User":["partial","*",21,6],"Google-Extended":["partial","*",21,6],"Applebot-Extended":["partial","*",21,6],"CCBot":["partial","*",21,6],"Bytespider":["partial","*",21,6],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["partial","*",21,6],"cohere-ai":["partial","*",21,6],"Diffbot":["partial","*",21,6],"ContextFetchBot":["partial","*",21,6]},"llms":{"status":"missing","http":404,"bytes":128736},"home":{"status":"ok","http":200,"url":"https://www.kleinezeitung.at/","html_tokens":80274,"markdown_tokens":31,"pruned_tokens":31}},
{"rank":6796,"domain":"fueleconomy.gov","origin":"https://fueleconomy.gov","robots":{"status":"unreachable","http":null,"url":"https://fueleconomy.gov/robots.txt","bytes":0,"groups":0,"error":"<StreamReset stream_id:1, error_code:8, remote_reset:True>"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":6797,"domain":"3sat.de","origin":"https://3sat.de","robots":{"status":"ok","http":200,"url":"https://www.3sat.de/robots.txt","bytes":40,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":73198},"home":{"status":"ok","http":200,"url":"https://www.3sat.de/","html_tokens":146926,"markdown_tokens":259,"pruned_tokens":150}},
{"rank":6798,"domain":"hkust.edu.hk","origin":"https://hkust.edu.hk","robots":{"status":"ok","http":200,"url":"https://hkust.edu.hk/robots.txt","bytes":2027,"groups":1},"bots":{"GPTBot":["partial","*",34,18],"ChatGPT-User":["partial","*",34,18],"OAI-SearchBot":["partial","*",34,18],"ClaudeBot":["partial","*",34,18],"Claude-User":["partial","*",34,18],"Claude-SearchBot":["partial","*",34,18],"anthropic-ai":["partial","*",34,18],"PerplexityBot":["partial","*",34,18],"Perplexity-User":["partial","*",34,18],"Google-Extended":["partial","*",34,18],"Applebot-Extended":["partial","*",34,18],"CCBot":["partial","*",34,18],"Bytespider":["partial","*",34,18],"Amazonbot":["partial","*",34,18],"meta-externalagent":["partial","*",34,18],"cohere-ai":["partial","*",34,18],"Diffbot":["partial","*",34,18],"ContextFetchBot":["partial","*",34,18]},"llms":{"status":"missing","http":404,"bytes":146},"home":{"status":"ok","http":200,"url":"https://hkust.edu.hk/","html_tokens":37657,"markdown_tokens":666,"pruned_tokens":220}},
{"rank":6799,"domain":"clickfunnels.com","origin":"https://clickfunnels.com","robots":{"status":"ok","http":200,"url":"https://www.clickfunnels.com/robots.txt","bytes":7816,"groups":4},"bots":{"GPTBot":["partial","named",50,0],"ChatGPT-User":["partial","named",16,0],"OAI-SearchBot":["partial","named",50,0],"ClaudeBot":["partial","named",50,0],"Claude-User":["partial","named",16,0],"Claude-SearchBot":["partial","named",50,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["partial","named",50,0],"Perplexity-User":["partial","named",16,0],"Google-Extended":["partial","named",50,0],"Applebot-Extended":["partial","named",50,0],"CCBot":["partial","named",50,0],"Bytespider":["partial","named",50,0],"Amazonbot":["partial","named",50,0],"meta-externalagent":["partial","named",50,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"missing","http":404,"bytes":0},"home":{"status":"ok","http":200,"url":"https://www.clickfunnels.com/","html_tokens":191060,"markdown_tokens":554,"pruned_tokens":291}},
{"rank":6800,"domain":"gob.bo","origin":"https://www.gob.bo","robots":{"status":"ok","http":200,"url":"https://www.gob.bo/robots.txt","bytes":98,"groups":1},"bots":{"GPTBot":["partial","*",2,1],"ChatGPT-User":["partial","*",2,1],"OAI-SearchBot":["partial","*",2,1],"ClaudeBot":["partial","*",2,1],"Claude-User":["partial","*",2,1],"Claude-SearchBot":["partial","*",2,1],"anthropic-ai":["partial","*",2,1],"PerplexityBot":["partial","*",2,1],"Perplexity-User":["partial","*",2,1],"Google-Extended":["partial","*",2,1],"Applebot-Extended":["partial","*",2,1],"CCBot":["partial","*",2,1],"Bytespider":["partial","*",2,1],"Amazonbot":["partial","*",2,1],"meta-externalagent":["partial","*",2,1],"cohere-ai":["partial","*",2,1],"Diffbot":["partial","*",2,1],"ContextFetchBot":["partial","*",2,1]},"llms":{"status":"missing","http":404,"bytes":32864},"home":{"status":"ok","http":200,"url":"https://www.gob.bo/","html_tokens":38858,"markdown_tokens":76,"pruned_tokens":53}},
{"rank":6801,"domain":"xonbet.net","origin":"https://xonbet.net","robots":{"status":"ok","http":200,"url":"https://xonbet.net/robots.txt","bytes":635,"groups":5},"bots":{"GPTBot":["partial","*",3,16],"ChatGPT-User":["partial","*",3,16],"OAI-SearchBot":["partial","*",3,16],"ClaudeBot":["partial","*",3,16],"Claude-User":["partial","*",3,16],"Claude-SearchBot":["partial","*",3,16],"anthropic-ai":["partial","*",3,16],"PerplexityBot":["partial","*",3,16],"Perplexity-User":["partial","*",3,16],"Google-Extended":["partial","*",3,16],"Applebot-Extended":["partial","*",3,16],"CCBot":["partial","*",3,16],"Bytespider":["partial","*",3,16],"Amazonbot":["partial","*",3,16],"meta-externalagent":["partial","*",3,16],"cohere-ai":["partial","*",3,16],"Diffbot":["partial","*",3,16],"ContextFetchBot":["partial","*",3,16]},"llms":{"status":"missing","http":404,"bytes":21452},"home":{"status":"ok","http":200,"url":"https://xonbet.net/","html_tokens":25388,"markdown_tokens":2486,"pruned_tokens":101}},
{"rank":6802,"domain":"bet.com","origin":"https://bet.com","robots":{"status":"ok","http":200,"url":"https://www.bet.com/robots.txt","bytes":66,"groups":2},"bots":{"GPTBot":["partial","*",1,0],"ChatGPT-User":["partial","*",1,0],"OAI-SearchBot":["partial","*",1,0],"ClaudeBot":["partial","*",1,0],"Claude-User":["partial","*",1,0],"Claude-SearchBot":["partial","*",1,0],"anthropic-ai":["partial","*",1,0],"PerplexityBot":["partial","*",1,0],"Perplexity-User":["partial","*",1,0],"Google-Extended":["partial","*",1,0],"Applebot-Extended":["partial","*",1,0],"CCBot":["partial","*",1,0],"Bytespider":["partial","*",1,0],"Amazonbot":["partial","*",1,0],"meta-externalagent":["partial","*",1,0],"cohere-ai":["partial","*",1,0],"Diffbot":["partial","*",1,0],"ContextFetchBot":["partial","*",1,0]},"llms":{"status":"missing","http":404,"bytes":808131},"home":{"status":"ok","http":200,"url":"https://www.bet.com","html_tokens":472153,"markdown_tokens":293,"pruned_tokens":127}},
{"rank":6803,"domain":"naacp.org","origin":"https://naacp.org","robots":{"status":"ok","http":200,"url":"https://naacp.org/robots.txt","bytes":2476,"groups":7},"bots":{"GPTBot":["partial","*",35,22],"ChatGPT-User":["partial","*",35,22],"OAI-SearchBot":["partial","*",35,22],"ClaudeBot":["partial","*",35,22],"Claude-User":["partial","*",35,22],"Claude-SearchBot":["partial","*",35,22],"anthropic-ai":["partial","*",35,22],"PerplexityBot":["partial","*",35,22],"Perplexity-User":["partial","*",35,22],"Google-Extended":["partial","*",35,22],"Applebot-Extended":["partial","*",35,22],"CCBot":["partial","*",35,22],"Bytespider":["partial","*",35,22],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["partial","*",35,22],"cohere-ai":["partial","*",35,22],"Diffbot":["partial","*",35,22],"ContextFetchBot":["partial","*",35,22]},"llms":{"status":"missing","http":404,"bytes":187},"home":{"status":"ok","http":200,"url":"https://naacp.org/","html_tokens":35744,"markdown_tokens":706,"pruned_tokens":272}},
{"rank":6804,"domain":"rapidapi.com","origin":"https://rapidapi.com","robots":{"status":"ok","http":200,"url":"https://rapidapi.com/robots.txt","bytes":1056,"groups":2},"bots":{"GPTBot":["partial","*",12,1],"ChatGPT-User":["partial","*",12,1],"OAI-SearchBot":["partial","*",12,1],"ClaudeBot":["partial","*",12,1],"Claude-User":["partial","*",12,1],"Claude-SearchBot":["partial","*",12,1],"anthropic-ai":["partial","*",12,1],"PerplexityBot":["partial","*",12,1],"Perplexity-User":["partial","*",12,1],"Google-Extended":["partial","*",12,1],"Applebot-Extended":["partial","*",12,1],"CCBot":["partial","*",12,1],"Bytespider":["partial","*",12,1],"Amazonbot":["partial","*",12,1],"meta-externalagent":["partial","*",12,1],"cohere-ai":["partial","*",12,1],"Diffbot":["partial","*",12,1],"ContextFetchBot":["partial","*",12,1]},"llms":{"status":"missing","http":404,"bytes":190129},"home":{"status":"ok","http":200,"url":"https://rapidapi.com/","html_tokens":53100,"markdown_tokens":0,"pruned_tokens":0}},
{"rank":6805,"domain":"antena3.com","origin":"https://antena3.com","robots":{"status":"ok","http":200,"url":"https://www.antena3.com/robots.txt","bytes":4105,"groups":1},"bots":{"GPTBot":["partial","*",95,0],"ChatGPT-User":["partial","*",95,0],"OAI-SearchBot":["partial","*",95,0],"ClaudeBot":["partial","*",95,0],"Claude-User":["partial","*",95,0],"Claude-SearchBot":["partial","*",95,0],"anthropic-ai":["partial","*",95,0],"PerplexityBot":["partial","*",95,0],"Perplexity-User":["partial","*",95,0],"Google-Extended":["partial","*",95,0],"Applebot-Extended":["partial","*",95,0],"CCBot":["partial","*",95,0],"Bytespider":["partial","*",95,0],"Amazonbot":["partial","*",95,0],"meta-externalagent":["partial","*",95,0],"cohere-ai":["partial","*",95,0],"Diffbot":["partial","*",95,0],"ContextFetchBot":["partial","*",95,0]},"llms":{"status":"missing","http":404,"bytes":377092},"home":{"status":"ok","http":200,"url":"https://www.antena3.com/","html_tokens":218315,"markdown_tokens":6,"pruned_tokens":6}},
{"rank":6806,"domain":"kvk.nl","origin":"https://kvk.nl","robots":{"status":"ok","http":200,"url":"https://www.kvk.nl/robots.txt","bytes":1187,"groups":1},"bots":{"GPTBot":["partial","*",18,0],"ChatGPT-User":["partial","*",18,0],"OAI-SearchBot":["partial","*",18,0],"ClaudeBot":["partial","*",18,0],"Claude-User":["partial","*",18,0],"Claude-SearchBot":["partial","*",18,0],"anthropic-ai":["partial","*",18,0],"PerplexityBot":["partial","*",18,0],"Perplexity-User":["partial","*",18,0],"Google-Extended":["partial","*",18,0],"Applebot-Extended":["partial","*",18,0],"CCBot":["partial","*",18,0],"Bytespider":["partial","*",18,0],"Amazonbot":["partial","*",18,0],"meta-externalagent":["partial","*",18,0],"cohere-ai":["partial","*",18,0],"Diffbot":["partial","*",18,0],"ContextFetchBot":["partial","*",18,0]},"llms":{"status":"present","http":200,"bytes":25},"home":{"status":"ok","http":200,"url":"https://www.kvk.nl/","html_tokens":76820,"markdown_tokens":49,"pruned_tokens":49}},
{"rank":6807,"domain":"local10.com","origin":"https://local10.com","robots":{"status":"ok","http":200,"url":"https://www.local10.com/robots.txt","bytes":468,"groups":2},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["partial","*",4,0],"OAI-SearchBot":["partial","*",4,0],"ClaudeBot":["partial","*",4,0],"Claude-User":["partial","*",4,0],"Claude-SearchBot":["partial","*",4,0],"anthropic-ai":["partial","*",4,0],"PerplexityBot":["partial","*",4,0],"Perplexity-User":["partial","*",4,0],"Google-Extended":["partial","*",4,0],"Applebot-Extended":["partial","*",4,0],"CCBot":["partial","*",4,0],"Bytespider":["partial","*",4,0],"Amazonbot":["partial","*",4,0],"meta-externalagent":["partial","*",4,0],"cohere-ai":["partial","*",4,0],"Diffbot":["partial","*",4,0],"ContextFetchBot":["partial","*",4,0]},"llms":{"status":"missing","http":404,"bytes":70201},"home":{"status":"ok","http":200,"url":"https://www.local10.com/","html_tokens":579956,"markdown_tokens":0,"pruned_tokens":0}},
{"rank":6808,"domain":"thoughtcatalog.com","origin":"https://thoughtcatalog.com","robots":{"status":"ok","http":200,"url":"https://thoughtcatalog.com/robots.txt","bytes":186,"groups":1},"bots":{"GPTBot":["partial","*",2,1],"ChatGPT-User":["partial","*",2,1],"OAI-SearchBot":["partial","*",2,1],"ClaudeBot":["partial","*",2,1],"Claude-User":["partial","*",2,1],"Claude-SearchBot":["partial","*",2,1],"anthropic-ai":["partial","*",2,1],"PerplexityBot":["partial","*",2,1],"Perplexity-User":["partial","*",2,1],"Google-Extended":["partial","*",2,1],"Applebot-Extended":["partial","*",2,1],"CCBot":["partial","*",2,1],"Bytespider":["partial","*",2,1],"Amazonbot":["partial","*",2,1],"meta-externalagent":["partial","*",2,1],"cohere-ai":["partial","*",2,1],"Diffbot":["partial","*",2,1],"ContextFetchBot":["partial","*",2,1]},"llms":{"status":"missing","http":404,"bytes":125906},"home":{"status":"ok","http":200,"url":"https://thoughtcatalog.com/","html_tokens":64152,"markdown_tokens":40,"pruned_tokens":40}},
{"rank":6809,"domain":"neimanmarcus.com","origin":"https://neimanmarcus.com","robots":{"status":"refused","http":403,"url":"https://www.neimanmarcus.com/robots.txt","bytes":776,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":6810,"domain":"corning.com","origin":"https://corning.com","robots":{"status":"refused","http":403,"url":"https://corning.com/robots.txt","bytes":373,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":6811,"domain":"up.ac.za","origin":"https://www.up.ac.za","robots":{"status":"ok","http":200,"url":"https://www.up.ac.za/robots.txt","bytes":1629,"groups":1},"bots":{"GPTBot":["partial","*",22,18],"ChatGPT-User":["partial","*",22,18],"OAI-SearchBot":["partial","*",22,18],"ClaudeBot":["partial","*",22,18],"Claude-User":["partial","*",22,18],"Claude-SearchBot":["partial","*",22,18],"anthropic-ai":["partial","*",22,18],"PerplexityBot":["partial","*",22,18],"Perplexity-User":["partial","*",22,18],"Google-Extended":["partial","*",22,18],"Applebot-Extended":["partial","*",22,18],"CCBot":["partial","*",22,18],"Bytespider":["partial","*",22,18],"Amazonbot":["partial","*",22,18],"meta-externalagent":["partial","*",22,18],"cohere-ai":["partial","*",22,18],"Diffbot":["partial","*",22,18],"ContextFetchBot":["partial","*",22,18]},"llms":{"status":"missing","http":404,"bytes":547},"home":{"status":"http_error","http":403,"url":"https://www.up.ac.za/"}},
{"rank":6812,"domain":"ceur-ws.org","origin":"https://ceur-ws.org","robots":{"status":"ok","http":200,"url":"https://ceur-ws.org/robots.txt","bytes":403,"groups":1},"bots":{"GPTBot":["partial","*",11,0],"ChatGPT-User":["partial","*",11,0],"OAI-SearchBot":["partial","*",11,0],"ClaudeBot":["partial","*",11,0],"Claude-User":["partial","*",11,0],"Claude-SearchBot":["partial","*",11,0],"anthropic-ai":["partial","*",11,0],"PerplexityBot":["partial","*",11,0],"Perplexity-User":["partial","*",11,0],"Google-Extended":["partial","*",11,0],"Applebot-Extended":["partial","*",11,0],"CCBot":["partial","*",11,0],"Bytespider":["partial","*",11,0],"Amazonbot":["partial","*",11,0],"meta-externalagent":["partial","*",11,0],"cohere-ai":["partial","*",11,0],"Diffbot":["partial","*",11,0],"ContextFetchBot":["partial","*",11,0]},"llms":{"status":"missing","http":404,"bytes":162},"home":{"status":"too_large","error":"response larger than 5 MB"}},
{"rank":6813,"domain":"sportsnet.ca","origin":"https://sportsnet.ca","robots":{"status":"ok","http":200,"url":"https://www.sportsnet.ca/robots.txt","bytes":220,"groups":2},"bots":{"GPTBot":["partial","*",3,1],"ChatGPT-User":["partial","*",3,1],"OAI-SearchBot":["partial","*",3,1],"ClaudeBot":["partial","*",3,1],"Claude-User":["partial","*",3,1],"Claude-SearchBot":["partial","*",3,1],"anthropic-ai":["partial","*",3,1],"PerplexityBot":["partial","*",3,1],"Perplexity-User":["partial","*",3,1],"Google-Extended":["partial","*",3,1],"Applebot-Extended":["partial","*",3,1],"CCBot":["partial","*",3,1],"Bytespider":["partial","*",3,1],"Amazonbot":["partial","*",3,1],"meta-externalagent":["partial","*",3,1],"cohere-ai":["partial","*",3,1],"Diffbot":["partial","*",3,1],"ContextFetchBot":["partial","*",3,1]},"llms":{"status":"http_error","http":403,"bytes":146},"home":{"status":"ok","http":200,"url":"https://www.sportsnet.ca/","html_tokens":58342,"markdown_tokens":5,"pruned_tokens":5}},
{"rank":6814,"domain":"belmond.com","origin":"https://belmond.com","robots":{"status":"ok","http":200,"url":"https://www.belmond.com/robots.txt","bytes":1028,"groups":2},"bots":{"GPTBot":["partial","*",15,14],"ChatGPT-User":["partial","*",15,14],"OAI-SearchBot":["partial","*",15,14],"ClaudeBot":["partial","*",15,14],"Claude-User":["partial","*",15,14],"Claude-SearchBot":["partial","*",15,14],"anthropic-ai":["partial","*",15,14],"PerplexityBot":["partial","*",15,14],"Perplexity-User":["partial","*",15,14],"Google-Extended":["partial","*",15,14],"Applebot-Extended":["partial","*",15,14],"CCBot":["partial","*",15,14],"Bytespider":["partial","*",15,14],"Amazonbot":["partial","*",15,14],"meta-externalagent":["partial","*",15,14],"cohere-ai":["partial","*",15,14],"Diffbot":["partial","*",15,14],"ContextFetchBot":["partial","*",15,14]},"llms":{"status":"missing","http":404,"bytes":13002},"home":{"status":"ok","http":200,"url":"https://www.belmond.com/en","html_tokens":158902,"markdown_tokens":75,"pruned_tokens":75}},
{"rank":6815,"domain":"spin.com","origin":"https://spin.com","robots":{"status":"ok","http":200,"url":"https://www.spinmagazine.com/robots.txt","bytes":446,"groups":5},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["partial","*",6,1],"OAI-SearchBot":["partial","*",6,1],"ClaudeBot":["partial","*",6,1],"Claude-User":["partial","*",6,1],"Claude-SearchBot":["partial","*",6,1],"anthropic-ai":["partial","*",6,1],"PerplexityBot":["partial","*",6,1],"Perplexity-User":["partial","*",6,1],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["partial","*",6,1],"CCBot":["blocked","named",1,0],"Bytespider":["partial","*",6,1],"Amazonbot":["partial","*",6,1],"meta-externalagent":["partial","*",6,1],"cohere-ai":["partial","*",6,1],"Diffbot":["partial","*",6,1],"ContextFetchBot":["partial","*",6,1]},"llms":{"status":"other_text","http":200,"bytes":4728},"home":{"status":"http_error","http":403,"url":"https://spin.com/"}},
{"rank":6816,"domain":"verizonwireless.com","origin":"https://verizonwireless.com","robots":{"status":"ok","http":200,"url":"https://www.verizon.com/robots.txt","bytes":7269,"groups":2},"bots":{"GPTBot":["partial","*",124,12],"ChatGPT-User":["partial","*",124,12],"OAI-SearchBot":["partial","*",124,12],"ClaudeBot":["partial","*",124,12],"Claude-User":["partial","*",124,12],"Claude-SearchBot":["partial","*",124,12],"anthropic-ai":["partial","*",124,12],"PerplexityBot":["partial","*",124,12],"Perplexity-User":["partial","*",124,12],"Google-Extended":["partial","*",124,12],"Applebot-Extended":["partial","*",124,12],"CCBot":["partial","*",124,12],"Bytespider":["partial","*",124,12],"Amazonbot":["partial","*",124,12],"meta-externalagent":["partial","*",124,12],"cohere-ai":["partial","*",124,12],"Diffbot":["partial","*",124,12],"ContextFetchBot":["partial","*",124,12]},"llms":{"status":"present","http":200,"bytes":5716},"home":{"status":"ok","http":200,"url":"https://www.verizon.com/","html_tokens":31292,"markdown_tokens":85,"pruned_tokens":49}},
{"rank":6817,"domain":"tripsavvy.com","origin":"https://tripsavvy.com","robots":{"status":"ok","http":200,"url":"https://www.tripsavvy.com/robots.txt","bytes":3471,"groups":6},"bots":{"GPTBot":["partial","named",1,0],"ChatGPT-User":["partial","named",1,0],"OAI-SearchBot":["partial","named",1,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["blocked","named",1,0],"Claude-SearchBot":["partial","named",1,2],"anthropic-ai":["partial","named",1,2],"PerplexityBot":["partial","named",1,2],"Perplexity-User":["blocked","named",1,0],"Google-Extended":["partial","named",1,2],"Applebot-Extended":["blocked","named",1,0],"CCBot":["partial","named",1,2],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["partial","named",1,2],"cohere-ai":["partial","named",1,2],"Diffbot":["partial","*",2,0],"ContextFetchBot":["partial","*",2,0]},"llms":{"status":"http_error","http":402,"bytes":612},"home":{"status":"http_error","http":402,"url":"https://www.tripsavvy.com/"}},
{"rank":6818,"domain":"afnor.org","origin":"https://afnor.org","robots":{"status":"ok","http":200,"url":"https://www.afnor.org/robots.txt","bytes":1257,"groups":16},"bots":{"GPTBot":["partial","named",1,1],"ChatGPT-User":["partial","*",21,2],"OAI-SearchBot":["partial","*",21,2],"ClaudeBot":["partial","named",1,1],"Claude-User":["partial","*",21,2],"Claude-SearchBot":["partial","*",21,2],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["partial","named",1,1],"Perplexity-User":["partial","*",21,2],"Google-Extended":["partial","named",1,1],"Applebot-Extended":["partial","named",1,1],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["partial","*",21,2],"meta-externalagent":["partial","*",21,2],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",21,2]},"llms":{"status":"missing","http":404,"bytes":9636},"home":{"status":"ok","http":200,"url":"https://www.afnor.org/","html_tokens":95629,"markdown_tokens":248,"pruned_tokens":88}},
{"rank":6819,"domain":"autoexpress.co.uk","origin":"https://autoexpress.co.uk","robots":{"status":"ok","http":200,"url":"https://www.autoexpress.co.uk/robots.txt","bytes":378,"groups":3},"bots":{"GPTBot":["partial","*",6,1],"ChatGPT-User":["partial","*",6,1],"OAI-SearchBot":["partial","*",6,1],"ClaudeBot":["partial","*",6,1],"Claude-User":["partial","*",6,1],"Claude-SearchBot":["partial","*",6,1],"anthropic-ai":["partial","*",6,1],"PerplexityBot":["partial","*",6,1],"Perplexity-User":["partial","*",6,1],"Google-Extended":["partial","*",6,1],"Applebot-Extended":["partial","*",6,1],"CCBot":["partial","*",6,1],"Bytespider":["partial","*",6,1],"Amazonbot":["partial","*",6,1],"meta-externalagent":["partial","*",6,1],"cohere-ai":["partial","*",6,1],"Diffbot":["partial","*",6,1],"ContextFetchBot":["partial","*",6,1]},"llms":{"status":"missing","http":404,"bytes":114},"home":{"status":"ok","http":200,"url":"https://www.autoexpress.co.uk/","html_tokens":352408,"markdown_tokens":1382,"pruned_tokens":39}},
{"rank":6820,"domain":"vite.dev","origin":"https://vite.dev","robots":{"status":"missing","http":404,"url":"https://vite.dev/robots.txt","bytes":1960,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"present","http":200,"bytes":3528},"home":{"status":"ok","http":200,"url":"https://vite.dev/","html_tokens":16377,"markdown_tokens":455,"pruned_tokens":99}},
{"rank":6821,"domain":"letelegramme.fr","origin":"https://letelegramme.fr","robots":{"status":"ok","http":200,"url":"https://www.letelegramme.fr/robots.txt","bytes":9942,"groups":6},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["partial","named",2,1],"OAI-SearchBot":["partial","named",2,1],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","named",2,1],"Claude-SearchBot":["partial","named",2,1],"anthropic-ai":["blocked","*",1,0],"PerplexityBot":["partial","named",2,1],"Perplexity-User":["partial","named",2,1],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","*",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["partial","named",2,1],"cohere-ai":["blocked","*",1,0],"Diffbot":["partial","named",2,1],"ContextFetchBot":["blocked","*",1,0]},"llms":{"status":"disallowed"},"home":{"status":"disallowed"}},
{"rank":6822,"domain":"caixa.gov.br","origin":"https://caixa.gov.br","robots":{"status":"ok","http":200,"url":"https://www.caixa.gov.br/robots.txt","bytes":217,"groups":1},"bots":{"GPTBot":["partial","*",6,0],"ChatGPT-User":["partial","*",6,0],"OAI-SearchBot":["partial","*",6,0],"ClaudeBot":["partial","*",6,0],"Claude-User":["partial","*",6,0],"Claude-SearchBot":["partial","*",6,0],"anthropic-ai":["partial","*",6,0],"PerplexityBot":["partial","*",6,0],"Perplexity-User":["partial","*",6,0],"Google-Extended":["partial","*",6,0],"Applebot-Extended":["partial","*",6,0],"CCBot":["partial","*",6,0],"Bytespider":["partial","*",6,0],"Amazonbot":["partial","*",6,0],"meta-externalagent":["partial","*",6,0],"cohere-ai":["partial","*",6,0],"Diffbot":["partial","*",6,0],"ContextFetchBot":["partial","*",6,0]},"llms":{"status":"missing","http":404,"bytes":658},"home":{"status":"ok","http":200,"url":"https://www.caixa.gov.br/Paginas/home-caixa.aspx","html_tokens":74371,"markdown_tokens":54,"pruned_tokens":54}},
{"rank":6823,"domain":"u.ae","origin":"https://u.ae","robots":{"status":"missing","http":404,"url":"https://u.ae/robots.txt","bytes":0,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":0},"home":{"status":"timeout","error":"timed out after 15 s"}},
{"rank":6824,"domain":"vfsglobal.com","origin":"https://vfsglobal.com","robots":{"status":"ok","http":200,"url":"https://vfsglobal.com/robots.txt","bytes":134,"groups":1},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"html","http":200,"bytes":1754},"home":{"status":"ok","http":200,"url":"https://www.vfsglobal.com/en/individuals/index.html","html_tokens":7674,"markdown_tokens":122,"pruned_tokens":122}},
{"rank":6825,"domain":"ancient-origins.net","origin":"https://ancient-origins.net","robots":{"status":"ok","http":200,"url":"https://www.ancient-origins.net/robots.txt","bytes":2512,"groups":2},"bots":{"GPTBot":["partial","*",33,18],"ChatGPT-User":["partial","*",33,18],"OAI-SearchBot":["partial","*",33,18],"ClaudeBot":["partial","*",33,18],"Claude-User":["partial","*",33,18],"Claude-SearchBot":["partial","*",33,18],"anthropic-ai":["partial","*",33,18],"PerplexityBot":["partial","*",33,18],"Perplexity-User":["partial","*",33,18],"Google-Extended":["partial","*",33,18],"Applebot-Extended":["partial","*",33,18],"CCBot":["partial","*",33,18],"Bytespider":["partial","*",33,18],"Amazonbot":["partial","*",33,18],"meta-externalagent":["partial","*",33,18],"cohere-ai":["partial","*",33,18],"Diffbot":["partial","*",33,18],"ContextFetchBot":["partial","*",33,18]},"llms":{"status":"timeout","error":"timed out after 10 s"},"home":{"status":"disallowed","error":"robots.txt of www.ancient-origins.net not readable (timeout)"}},
{"rank":6826,"domain":"manualslib.com","origin":"https://manualslib.com","robots":{"status":"ok","http":200,"url":"https://www.manualslib.com/robots.txt","bytes":1575,"groups":5},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["partial","*",8,1],"OAI-SearchBot":["partial","*",8,1],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",8,1],"Claude-SearchBot":["partial","*",8,1],"anthropic-ai":["partial","*",8,1],"PerplexityBot":["partial","*",8,1],"Perplexity-User":["partial","*",8,1],"Google-Extended":["partial","*",8,1],"Applebot-Extended":["partial","*",8,1],"CCBot":["partial","*",8,1],"Bytespider":["partial","*",8,1],"Amazonbot":["partial","*",8,1],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["partial","*",8,1],"Diffbot":["partial","*",8,1],"ContextFetchBot":["partial","*",8,1]},"llms":{"status":"missing","http":404,"bytes":162952},"home":{"status":"http_error","http":403,"url":"https://www.manualslib.com/"}},
{"rank":6827,"domain":"nbcboston.com","origin":"https://nbcboston.com","robots":{"status":"ok","http":200,"url":"https://www.nbcboston.com/robots.txt","bytes":10558,"groups":27},"bots":{"GPTBot":["partial","named",1,6],"ChatGPT-User":["partial","named",1,6],"OAI-SearchBot":["partial","*",13,3],"ClaudeBot":["partial","named",1,6],"Claude-User":["partial","*",13,3],"Claude-SearchBot":["partial","*",13,3],"anthropic-ai":["partial","named",1,6],"PerplexityBot":["partial","named",1,6],"Perplexity-User":["partial","*",13,3],"Google-Extended":["partial","named",1,6],"Applebot-Extended":["partial","*",13,3],"CCBot":["partial","named",1,6],"Bytespider":["partial","named",1,6],"Amazonbot":["partial","named",1,8],"meta-externalagent":["partial","*",13,3],"cohere-ai":["partial","*",13,3],"Diffbot":["partial","named",1,6],"ContextFetchBot":["partial","*",13,3]},"llms":{"status":"timeout","error":"timed out after 10 s"},"home":{"status":"timeout","error":"timed out after 15 s"}},
{"rank":6828,"domain":"fanniemae.com","origin":"https://fanniemae.com","robots":{"status":"ok","http":200,"url":"https://www.fanniemae.com/robots.txt","bytes":2504,"groups":1},"bots":{"GPTBot":["partial","*",35,18],"ChatGPT-User":["partial","*",35,18],"OAI-SearchBot":["partial","*",35,18],"ClaudeBot":["partial","*",35,18],"Claude-User":["partial","*",35,18],"Claude-SearchBot":["partial","*",35,18],"anthropic-ai":["partial","*",35,18],"PerplexityBot":["partial","*",35,18],"Perplexity-User":["partial","*",35,18],"Google-Extended":["partial","*",35,18],"Applebot-Extended":["partial","*",35,18],"CCBot":["partial","*",35,18],"Bytespider":["partial","*",35,18],"Amazonbot":["partial","*",35,18],"meta-externalagent":["partial","*",35,18],"cohere-ai":["partial","*",35,18],"Diffbot":["partial","*",35,18],"ContextFetchBot":["partial","*",35,18]},"llms":{"status":"missing","http":404,"bytes":191},"home":{"status":"http_error","http":403,"url":"https://www.fanniemae.com/"}},
{"rank":6829,"domain":"ticketleap.com","origin":"https://ticketleap.com","robots":{"status":"ok","http":200,"url":"https://www.ticketleap.com/robots.txt","bytes":272,"groups":1},"bots":{"GPTBot":["partial","*",3,1],"ChatGPT-User":["partial","*",3,1],"OAI-SearchBot":["partial","*",3,1],"ClaudeBot":["partial","*",3,1],"Claude-User":["partial","*",3,1],"Claude-SearchBot":["partial","*",3,1],"anthropic-ai":["partial","*",3,1],"PerplexityBot":["partial","*",3,1],"Perplexity-User":["partial","*",3,1],"Google-Extended":["partial","*",3,1],"Applebot-Extended":["partial","*",3,1],"CCBot":["partial","*",3,1],"Bytespider":["partial","*",3,1],"Amazonbot":["partial","*",3,1],"meta-externalagent":["partial","*",3,1],"cohere-ai":["partial","*",3,1],"Diffbot":["partial","*",3,1],"ContextFetchBot":["partial","*",3,1]},"llms":{"status":"http_error","http":403,"bytes":6318},"home":{"status":"http_error","http":403,"url":"https://www.ticketleap.com/"}},
{"rank":6830,"domain":"pipex.com","origin":"https://pipex.com","robots":{"status":"timeout","http":null,"url":"https://pipex.com/robots.txt","bytes":0,"groups":0,"error":"ConnectTimeout"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":6831,"domain":"freightwaves.com","origin":"https://freightwaves.com","robots":{"status":"ok","http":200,"url":"https://www.freightwaves.com/robots.txt","bytes":299,"groups":2},"bots":{"GPTBot":["partial","*",5,0],"ChatGPT-User":["partial","*",5,0],"OAI-SearchBot":["partial","*",5,0],"ClaudeBot":["partial","*",5,0],"Claude-User":["partial","*",5,0],"Claude-SearchBot":["partial","*",5,0],"anthropic-ai":["partial","*",5,0],"PerplexityBot":["partial","*",5,0],"Perplexity-User":["partial","*",5,0],"Google-Extended":["partial","*",5,0],"Applebot-Extended":["partial","*",5,0],"CCBot":["blocked","named",1,0],"Bytespider":["partial","*",5,0],"Amazonbot":["partial","*",5,0],"meta-externalagent":["partial","*",5,0],"cohere-ai":["partial","*",5,0],"Diffbot":["partial","*",5,0],"ContextFetchBot":["partial","*",5,0]},"llms":{"status":"missing","http":404,"bytes":92257},"home":{"status":"ok","http":200,"url":"https://www.freightwaves.com/","html_tokens":65097,"markdown_tokens":255,"pruned_tokens":135}},
{"rank":6832,"domain":"itar-tass.com","origin":"http://itar-tass.com","robots":{"status":"refused","http":403,"url":"https://tass.ru/robots.txt","bytes":1293,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":6833,"domain":"azfamily.com","origin":"https://azfamily.com","robots":{"status":"ok","http":200,"url":"https://www.azfamily.com/robots.txt","bytes":1205,"groups":2},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["blocked","named",1,0],"OAI-SearchBot":["partial","*",19,3],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",19,3],"Claude-SearchBot":["partial","*",19,3],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["partial","*",19,3],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["partial","*",19,3],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["partial","*",19,3],"cohere-ai":["partial","*",19,3],"Diffbot":["partial","*",19,3],"ContextFetchBot":["partial","*",19,3]},"llms":{"status":"missing","http":404,"bytes":103403},"home":{"status":"ok","http":200,"url":"https://www.azfamily.com/","html_tokens":151927,"markdown_tokens":171,"pruned_tokens":171}},
{"rank":6834,"domain":"pressdemocrat.com","origin":"https://pressdemocrat.com","robots":{"status":"ok","http":200,"url":"https://www.pressdemocrat.com/robots.txt","bytes":4489,"groups":68},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["blocked","named",1,0],"OAI-SearchBot":["blocked","named",1,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["blocked","named",1,0],"Claude-SearchBot":["blocked","named",1,0],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["blocked","named",1,0],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["partial","*",8,1],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",8,1]},"llms":{"status":"missing","http":404,"bytes":80857},"home":{"status":"ok","http":200,"url":"https://www.pressdemocrat.com/","html_tokens":68581,"markdown_tokens":451,"pruned_tokens":151}},
{"rank":6835,"domain":"casadellibro.com","origin":"https://casadellibro.com","robots":{"status":"refused","http":403,"url":"https://www.casadellibro.com/robots.txt","bytes":3668,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":6836,"domain":"buzznet.com","origin":"https://buzznet.com","robots":{"status":"ok","http":200,"url":"https://www.buzznet.com/robots.txt","bytes":525,"groups":6},"bots":{"GPTBot":["partial","*",13,1],"ChatGPT-User":["partial","*",13,1],"OAI-SearchBot":["partial","*",13,1],"ClaudeBot":["partial","*",13,1],"Claude-User":["partial","*",13,1],"Claude-SearchBot":["partial","*",13,1],"anthropic-ai":["partial","*",13,1],"PerplexityBot":["partial","*",13,1],"Perplexity-User":["partial","*",13,1],"Google-Extended":["partial","*",13,1],"Applebot-Extended":["partial","*",13,1],"CCBot":["partial","*",13,1],"Bytespider":["partial","*",13,1],"Amazonbot":["partial","*",13,1],"meta-externalagent":["partial","*",13,1],"cohere-ai":["partial","*",13,1],"Diffbot":["partial","*",13,1],"ContextFetchBot":["partial","*",13,1]},"llms":{"status":"missing","http":404,"bytes":60},"home":{"status":"ok","http":200,"url":"https://www.buzznet.com/","html_tokens":51001,"markdown_tokens":653,"pruned_tokens":160}},
{"rank":6837,"domain":"bnl.gov","origin":"https://bnl.gov","robots":{"status":"html","http":200,"url":"https://www.bnl.gov/world/","bytes":33479,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":6838,"domain":"360.com","origin":"https://360.com","robots":{"status":"missing","http":404,"url":"https://360.com/robots.txt","bytes":162,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":162},"home":{"status":"ok","http":200,"url":"https://360.com/","html_tokens":62003,"markdown_tokens":231,"pruned_tokens":80}},
{"rank":6839,"domain":"econlib.org","origin":"https://econlib.org","robots":{"status":"ok","http":200,"url":"https://www.econlib.org/robots.txt","bytes":115,"groups":2},"bots":{"GPTBot":["partial","*",1,0],"ChatGPT-User":["partial","*",1,0],"OAI-SearchBot":["partial","*",1,0],"ClaudeBot":["partial","*",1,0],"Claude-User":["partial","*",1,0],"Claude-SearchBot":["partial","*",1,0],"anthropic-ai":["partial","*",1,0],"PerplexityBot":["partial","*",1,0],"Perplexity-User":["partial","*",1,0],"Google-Extended":["partial","*",1,0],"Applebot-Extended":["partial","*",1,0],"CCBot":["partial","*",1,0],"Bytespider":["partial","*",1,0],"Amazonbot":["partial","*",1,0],"meta-externalagent":["partial","*",1,0],"cohere-ai":["partial","*",1,0],"Diffbot":["partial","*",1,0],"ContextFetchBot":["partial","*",1,0]},"llms":{"status":"http_error","http":403,"bytes":5542},"home":{"status":"http_error","http":403,"url":"https://econlib.org/"}},
{"rank":6840,"domain":"toolforge.org","origin":"https://toolforge.org","robots":{"status":"html","http":200,"url":"https://wikitech.wikimedia.org/wiki/Portal:Toolforge","bytes":60692,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":6841,"domain":"nmpa.gov.cn","origin":"https://www.nmpa.gov.cn","robots":{"status":"missing","http":404,"url":"https://www.nmpa.gov.cn/robots.txt","bytes":3638,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":3638},"home":{"status":"http_error","http":412,"url":"https://www.nmpa.gov.cn/"}},
{"rank":6842,"domain":"hcch.net","origin":"https://hcch.net","robots":{"status":"html","http":200,"url":"https://www.hcch.net","bytes":5381,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":6843,"domain":"austintexas.gov","origin":"https://austintexas.gov","robots":{"status":"ok","http":200,"url":"https://www.austintexas.gov/robots.txt","bytes":2302,"groups":1},"bots":{"GPTBot":["partial","*",48,18],"ChatGPT-User":["partial","*",48,18],"OAI-SearchBot":["partial","*",48,18],"ClaudeBot":["partial","*",48,18],"Claude-User":["partial","*",48,18],"Claude-SearchBot":["partial","*",48,18],"anthropic-ai":["partial","*",48,18],"PerplexityBot":["partial","*",48,18],"Perplexity-User":["partial","*",48,18],"Google-Extended":["partial","*",48,18],"Applebot-Extended":["partial","*",48,18],"CCBot":["partial","*",48,18],"Bytespider":["partial","*",48,18],"Amazonbot":["partial","*",48,18],"meta-externalagent":["partial","*",48,18],"cohere-ai":["partial","*",48,18],"Diffbot":["partial","*",48,18],"ContextFetchBot":["partial","*",48,18]},"llms":{"status":"missing","http":404,"bytes":327},"home":{"status":"ok","http":200,"url":"https://www.austintexas.gov/","html_tokens":47337,"markdown_tokens":17,"pruned_tokens":11}},
{"rank":6844,"domain":"dice.fm","origin":"https://dice.fm","robots":{"status":"ok","http":200,"url":"https://dice.fm/robots.txt","bytes":2860,"groups":33},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["partial","*",1,1],"OAI-SearchBot":["partial","*",1,1],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",1,1],"Claude-SearchBot":["partial","*",1,1],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["partial","*",1,1],"Perplexity-User":["partial","*",1,1],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",1,1]},"llms":{"status":"missing","http":404,"bytes":251070},"home":{"status":"ok","http":200,"url":"https://dice.fm/","html_tokens":95414,"markdown_tokens":171,"pruned_tokens":171}},
{"rank":6845,"domain":"ucoz.net","origin":"https://ucoz.net","robots":{"status":"ok","http":200,"url":"https://www.ucoz.ru/robots.txt","bytes":121,"groups":1},"bots":{"GPTBot":["partial","*",5,0],"ChatGPT-User":["partial","*",5,0],"OAI-SearchBot":["partial","*",5,0],"ClaudeBot":["partial","*",5,0],"Claude-User":["partial","*",5,0],"Claude-SearchBot":["partial","*",5,0],"anthropic-ai":["partial","*",5,0],"PerplexityBot":["partial","*",5,0],"Perplexity-User":["partial","*",5,0],"Google-Extended":["partial","*",5,0],"Applebot-Extended":["partial","*",5,0],"CCBot":["partial","*",5,0],"Bytespider":["partial","*",5,0],"Amazonbot":["partial","*",5,0],"meta-externalagent":["partial","*",5,0],"cohere-ai":["partial","*",5,0],"Diffbot":["partial","*",5,0],"ContextFetchBot":["partial","*",5,0]},"llms":{"status":"present","http":200,"bytes":33987},"home":{"status":"ok","http":200,"url":"https://www.ucoz.ru/","html_tokens":13583,"markdown_tokens":183,"pruned_tokens":133}},
{"rank":6846,"domain":"google.com.pa","origin":"https://google.com.pa","robots":{"status":"ok","http":200,"url":"https://www.google.com.pa/robots.txt","bytes":6624,"groups":4},"bots":{"GPTBot":["partial","*",169,64],"ChatGPT-User":["partial","*",169,64],"OAI-SearchBot":["partial","*",169,64],"ClaudeBot":["partial","*",169,64],"Claude-User":["partial","*",169,64],"Claude-SearchBot":["partial","*",169,64],"anthropic-ai":["partial","*",169,64],"PerplexityBot":["partial","*",169,64],"Perplexity-User":["partial","*",169,64],"Google-Extended":["partial","*",169,64],"Applebot-Extended":["partial","*",169,64],"CCBot":["partial","*",169,64],"Bytespider":["partial","*",169,64],"Amazonbot":["partial","*",169,64],"meta-externalagent":["partial","*",169,64],"cohere-ai":["partial","*",169,64],"Diffbot":["partial","*",169,64],"ContextFetchBot":["partial","*",169,64]},"llms":{"status":"missing","http":404,"bytes":1569},"home":{"status":"ok","http":200,"url":"https://www.google.com/","html_tokens":21446,"markdown_tokens":30,"pruned_tokens":30}},
{"rank":6847,"domain":"accaglobal.com","origin":"https://accaglobal.com","robots":{"status":"ok","http":200,"url":"https://www.accaglobal.com/robots.txt","bytes":2972,"groups":2},"bots":{"GPTBot":["partial","*",38,0],"ChatGPT-User":["partial","*",38,0],"OAI-SearchBot":["partial","*",38,0],"ClaudeBot":["partial","*",38,0],"Claude-User":["partial","*",38,0],"Claude-SearchBot":["partial","*",38,0],"anthropic-ai":["partial","*",38,0],"PerplexityBot":["partial","*",38,0],"Perplexity-User":["partial","*",38,0],"Google-Extended":["partial","*",38,0],"Applebot-Extended":["partial","*",38,0],"CCBot":["partial","*",38,0],"Bytespider":["partial","*",38,0],"Amazonbot":["partial","*",38,0],"meta-externalagent":["partial","*",38,0],"cohere-ai":["partial","*",38,0],"Diffbot":["partial","*",38,0],"ContextFetchBot":["partial","*",38,0]},"llms":{"status":"missing","http":404,"bytes":263966},"home":{"status":"ok","http":200,"url":"https://www.accaglobal.com/gb/en.html","html_tokens":32673,"markdown_tokens":80,"pruned_tokens":80}},
{"rank":6848,"domain":"asrock.com","origin":"https://asrock.com","robots":{"status":"ok","http":200,"url":"https://asrock.com/robots.txt","bytes":37,"groups":1},"bots":{"GPTBot":["partial","*",1,0],"ChatGPT-User":["partial","*",1,0],"OAI-SearchBot":["partial","*",1,0],"ClaudeBot":["partial","*",1,0],"Claude-User":["partial","*",1,0],"Claude-SearchBot":["partial","*",1,0],"anthropic-ai":["partial","*",1,0],"PerplexityBot":["partial","*",1,0],"Perplexity-User":["partial","*",1,0],"Google-Extended":["partial","*",1,0],"Applebot-Extended":["partial","*",1,0],"CCBot":["partial","*",1,0],"Bytespider":["partial","*",1,0],"Amazonbot":["partial","*",1,0],"meta-externalagent":["partial","*",1,0],"cohere-ai":["partial","*",1,0],"Diffbot":["partial","*",1,0],"ContextFetchBot":["partial","*",1,0]},"llms":{"status":"missing","http":404,"bytes":1245},"home":{"status":"ok","http":200,"url":"https://asrock.com/","html_tokens":53,"markdown_tokens":0,"pruned_tokens":0}},
{"rank":6849,"domain":"wwu.edu","origin":"https://wwu.edu","robots":{"status":"ok","http":200,"url":"https://www.wwu.edu/robots.txt","bytes":2027,"groups":1},"bots":{"GPTBot":["partial","*",34,18],"ChatGPT-User":["partial","*",34,18],"OAI-SearchBot":["partial","*",34,18],"ClaudeBot":["partial","*",34,18],"Claude-User":["partial","*",34,18],"Claude-SearchBot":["partial","*",34,18],"anthropic-ai":["partial","*",34,18],"PerplexityBot":["partial","*",34,18],"Perplexity-User":["partial","*",34,18],"Google-Extended":["partial","*",34,18],"Applebot-Extended":["partial","*",34,18],"CCBot":["partial","*",34,18],"Bytespider":["partial","*",34,18],"Amazonbot":["partial","*",34,18],"meta-externalagent":["partial","*",34,18],"cohere-ai":["partial","*",34,18],"Diffbot":["partial","*",34,18],"ContextFetchBot":["partial","*",34,18]},"llms":{"status":"missing","http":404,"bytes":185},"home":{"status":"ok","http":200,"url":"https://www.wwu.edu/","html_tokens":17571,"markdown_tokens":665,"pruned_tokens":172}},
{"rank":6850,"domain":"gocomics.com","origin":"https://gocomics.com","robots":{"status":"ok","http":200,"url":"https://www.gocomics.com/robots.txt","bytes":2953,"groups":5},"bots":{"GPTBot":["partial","named",1,1],"ChatGPT-User":["partial","named",1,1],"OAI-SearchBot":["partial","*",5,1],"ClaudeBot":["partial","named",1,1],"Claude-User":["partial","*",5,1],"Claude-SearchBot":["partial","*",5,1],"anthropic-ai":["partial","named",1,1],"PerplexityBot":["partial","named",1,1],"Perplexity-User":["partial","*",5,1],"Google-Extended":["partial","named",1,1],"Applebot-Extended":["partial","*",5,1],"CCBot":["partial","named",1,1],"Bytespider":["partial","named",1,1],"Amazonbot":["partial","*",5,1],"meta-externalagent":["partial","*",5,1],"cohere-ai":["partial","named",1,1],"Diffbot":["partial","named",1,1],"ContextFetchBot":["partial","*",5,1]},"llms":{"status":"present","http":200,"bytes":120993},"home":{"status":"ok","http":200,"url":"https://www.gocomics.com/","html_tokens":371552,"markdown_tokens":112,"pruned_tokens":112}},
{"rank":6851,"domain":"nerdist.com","origin":"https://nerdist.com","robots":{"status":"ok","http":200,"url":"https://nerdist.com/robots.txt","bytes":255,"groups":2},"bots":{"GPTBot":["partial","*",5,1],"ChatGPT-User":["partial","*",5,1],"OAI-SearchBot":["partial","*",5,1],"ClaudeBot":["partial","*",5,1],"Claude-User":["partial","*",5,1],"Claude-SearchBot":["partial","*",5,1],"anthropic-ai":["partial","*",5,1],"PerplexityBot":["partial","*",5,1],"Perplexity-User":["partial","*",5,1],"Google-Extended":["partial","*",5,1],"Applebot-Extended":["partial","*",5,1],"CCBot":["partial","*",5,1],"Bytespider":["partial","*",5,1],"Amazonbot":["partial","*",5,1],"meta-externalagent":["partial","*",5,1],"cohere-ai":["partial","*",5,1],"Diffbot":["partial","*",5,1],"ContextFetchBot":["partial","*",5,1]},"llms":{"status":"http_error","http":403,"bytes":5542},"home":{"status":"http_error","http":403,"url":"https://nerdist.com/"}},
{"rank":6852,"domain":"stuve.de","origin":"https://stuve.de","robots":{"status":"missing","http":404,"url":"https://stuve.de/robots.txt","bytes":71404,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":71404},"home":{"status":"ok","http":200,"url":"https://stuve.de/aktuelles","html_tokens":20899,"markdown_tokens":393,"pruned_tokens":205}},
{"rank":6853,"domain":"justsecurity.org","origin":"https://justsecurity.org","robots":{"status":"ok","http":200,"url":"https://www.justsecurity.org/robots.txt","bytes":541,"groups":1},"bots":{"GPTBot":["partial","*",9,1],"ChatGPT-User":["partial","*",9,1],"OAI-SearchBot":["partial","*",9,1],"ClaudeBot":["partial","*",9,1],"Claude-User":["partial","*",9,1],"Claude-SearchBot":["partial","*",9,1],"anthropic-ai":["partial","*",9,1],"PerplexityBot":["partial","*",9,1],"Perplexity-User":["partial","*",9,1],"Google-Extended":["partial","*",9,1],"Applebot-Extended":["partial","*",9,1],"CCBot":["partial","*",9,1],"Bytespider":["partial","*",9,1],"Amazonbot":["partial","*",9,1],"meta-externalagent":["partial","*",9,1],"cohere-ai":["partial","*",9,1],"Diffbot":["partial","*",9,1],"ContextFetchBot":["partial","*",9,1]},"llms":{"status":"present","http":200,"bytes":8344},"home":{"status":"ok","http":200,"url":"https://www.justsecurity.org/","html_tokens":68144,"markdown_tokens":936,"pruned_tokens":64}},
{"rank":6854,"domain":"vub.ac.be","origin":"https://vub.ac.be","robots":{"status":"ok","http":200,"url":"https://www.vub.be//robots.txt","bytes":2418,"groups":1},"bots":{"GPTBot":["partial","*",41,22],"ChatGPT-User":["partial","*",41,22],"OAI-SearchBot":["partial","*",41,22],"ClaudeBot":["partial","*",41,22],"Claude-User":["partial","*",41,22],"Claude-SearchBot":["partial","*",41,22],"anthropic-ai":["partial","*",41,22],"PerplexityBot":["partial","*",41,22],"Perplexity-User":["partial","*",41,22],"Google-Extended":["partial","*",41,22],"Applebot-Extended":["partial","*",41,22],"CCBot":["partial","*",41,22],"Bytespider":["partial","*",41,22],"Amazonbot":["partial","*",41,22],"meta-externalagent":["partial","*",41,22],"cohere-ai":["partial","*",41,22],"Diffbot":["partial","*",41,22],"ContextFetchBot":["partial","*",41,22]},"llms":{"status":"missing","http":404,"bytes":184},"home":{"status":"ok","http":200,"url":"https://www.vub.be/en","html_tokens":70027,"markdown_tokens":825,"pruned_tokens":247}},
{"rank":6855,"domain":"jboss.org","origin":"https://jboss.org","robots":{"status":"ok","http":200,"url":"https://www.jboss.org/robots.txt","bytes":331,"groups":1},"bots":{"GPTBot":["partial","*",10,0],"ChatGPT-User":["partial","*",10,0],"OAI-SearchBot":["partial","*",10,0],"ClaudeBot":["partial","*",10,0],"Claude-User":["partial","*",10,0],"Claude-SearchBot":["partial","*",10,0],"anthropic-ai":["partial","*",10,0],"PerplexityBot":["partial","*",10,0],"Perplexity-User":["partial","*",10,0],"Google-Extended":["partial","*",10,0],"Applebot-Extended":["partial","*",10,0],"CCBot":["partial","*",10,0],"Bytespider":["partial","*",10,0],"Amazonbot":["partial","*",10,0],"meta-externalagent":["partial","*",10,0],"cohere-ai":["partial","*",10,0],"Diffbot":["partial","*",10,0],"ContextFetchBot":["partial","*",10,0]},"llms":{"status":"missing","http":404,"bytes":1053},"home":{"status":"ok","http":200,"url":"https://www.jboss.org/","html_tokens":2057,"markdown_tokens":252,"pruned_tokens":167}},
{"rank":6856,"domain":"moneyhelper.org.uk","origin":"https://moneyhelper.org.uk","robots":{"status":"refused","http":403,"url":"https://moneyhelper.org.uk/robots.txt","bytes":5535,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":6857,"domain":"d3js.org","origin":"https://d3js.org","robots":{"status":"ok","http":200,"url":"https://d3js.org/robots.txt","bytes":1248,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":13660},"home":{"status":"ok","http":200,"url":"https://d3js.org/","html_tokens":17813,"markdown_tokens":142,"pruned_tokens":142}},
{"rank":6858,"domain":"halo.run","origin":"https://halo.run","robots":{"status":"ok","http":200,"url":"https://www.halo.run/robots.txt","bytes":84,"groups":1},"bots":{"GPTBot":["partial","*",1,1],"ChatGPT-User":["partial","*",1,1],"OAI-SearchBot":["partial","*",1,1],"ClaudeBot":["partial","*",1,1],"Claude-User":["partial","*",1,1],"Claude-SearchBot":["partial","*",1,1],"anthropic-ai":["partial","*",1,1],"PerplexityBot":["partial","*",1,1],"Perplexity-User":["partial","*",1,1],"Google-Extended":["partial","*",1,1],"Applebot-Extended":["partial","*",1,1],"CCBot":["partial","*",1,1],"Bytespider":["partial","*",1,1],"Amazonbot":["partial","*",1,1],"meta-externalagent":["partial","*",1,1],"cohere-ai":["partial","*",1,1],"Diffbot":["partial","*",1,1],"ContextFetchBot":["partial","*",1,1]},"llms":{"status":"missing","http":404,"bytes":2896},"home":{"status":"ok","http":200,"url":"https://www.halo.run/","html_tokens":26575,"markdown_tokens":39,"pruned_tokens":39}},
{"rank":6859,"domain":"princetonreview.com","origin":"https://princetonreview.com","robots":{"status":"ok","http":200,"url":"https://www.princetonreview.com/robots.txt","bytes":519,"groups":1},"bots":{"GPTBot":["partial","*",16,0],"ChatGPT-User":["partial","*",16,0],"OAI-SearchBot":["partial","*",16,0],"ClaudeBot":["partial","*",16,0],"Claude-User":["partial","*",16,0],"Claude-SearchBot":["partial","*",16,0],"anthropic-ai":["partial","*",16,0],"PerplexityBot":["partial","*",16,0],"Perplexity-User":["partial","*",16,0],"Google-Extended":["partial","*",16,0],"Applebot-Extended":["partial","*",16,0],"CCBot":["partial","*",16,0],"Bytespider":["partial","*",16,0],"Amazonbot":["partial","*",16,0],"meta-externalagent":["partial","*",16,0],"cohere-ai":["partial","*",16,0],"Diffbot":["partial","*",16,0],"ContextFetchBot":["partial","*",16,0]},"llms":{"status":"other_text","http":200,"bytes":10329},"home":{"status":"ok","http":200,"url":"https://www.princetonreview.com/","html_tokens":53414,"markdown_tokens":766,"pruned_tokens":113}},
{"rank":6860,"domain":"icm.edu.pl","origin":"https://icm.edu.pl","robots":{"status":"ok","http":200,"url":"https://icm.edu.pl/robots.txt","bytes":0,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":76621},"home":{"status":"ok","http":200,"url":"https://icm.edu.pl/","html_tokens":42304,"markdown_tokens":735,"pruned_tokens":135}},
{"rank":6861,"domain":"copyblogger.com","origin":"https://copyblogger.com","robots":{"status":"ok","http":200,"url":"https://copyblogger.com/robots.txt","bytes":156,"groups":1},"bots":{"GPTBot":["partial","*",3,2],"ChatGPT-User":["partial","*",3,2],"OAI-SearchBot":["partial","*",3,2],"ClaudeBot":["partial","*",3,2],"Claude-User":["partial","*",3,2],"Claude-SearchBot":["partial","*",3,2],"anthropic-ai":["partial","*",3,2],"PerplexityBot":["partial","*",3,2],"Perplexity-User":["partial","*",3,2],"Google-Extended":["partial","*",3,2],"Applebot-Extended":["partial","*",3,2],"CCBot":["partial","*",3,2],"Bytespider":["partial","*",3,2],"Amazonbot":["partial","*",3,2],"meta-externalagent":["partial","*",3,2],"cohere-ai":["partial","*",3,2],"Diffbot":["partial","*",3,2],"ContextFetchBot":["partial","*",3,2]},"llms":{"status":"missing","http":404,"bytes":68294},"home":{"status":"ok","http":200,"url":"https://copyblogger.com/","html_tokens":33789,"markdown_tokens":472,"pruned_tokens":138}},
{"rank":6862,"domain":"brainpickings.org","origin":"https://brainpickings.org","robots":{"status":"ok","http":200,"url":"https://www.themarginalian.org/robots.txt","bytes":123,"groups":1},"bots":{"GPTBot":["partial","*",1,1],"ChatGPT-User":["partial","*",1,1],"OAI-SearchBot":["partial","*",1,1],"ClaudeBot":["partial","*",1,1],"Claude-User":["partial","*",1,1],"Claude-SearchBot":["partial","*",1,1],"anthropic-ai":["partial","*",1,1],"PerplexityBot":["partial","*",1,1],"Perplexity-User":["partial","*",1,1],"Google-Extended":["partial","*",1,1],"Applebot-Extended":["partial","*",1,1],"CCBot":["partial","*",1,1],"Bytespider":["partial","*",1,1],"Amazonbot":["partial","*",1,1],"meta-externalagent":["partial","*",1,1],"cohere-ai":["partial","*",1,1],"Diffbot":["partial","*",1,1],"ContextFetchBot":["partial","*",1,1]},"llms":{"status":"missing","http":404,"bytes":56156},"home":{"status":"ok","http":200,"url":"https://www.themarginalian.org/","html_tokens":33149,"markdown_tokens":3596,"pruned_tokens":128}},
{"rank":6863,"domain":"clickmeeting.com","origin":"https://clickmeeting.com","robots":{"status":"ok","http":200,"url":"https://clickmeeting.com/robots.txt","bytes":166,"groups":5},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"other_text","http":200,"bytes":2394},"home":{"status":"ok","http":200,"url":"https://clickmeeting.com/","html_tokens":76105,"markdown_tokens":558,"pruned_tokens":147}},
{"rank":6864,"domain":"shapeways.com","origin":"https://shapeways.com","robots":{"status":"ok","http":200,"url":"https://www.shapeways.com/robots.txt","bytes":26,"groups":1},"bots":{"GPTBot":["blocked","*",1,0],"ChatGPT-User":["blocked","*",1,0],"OAI-SearchBot":["blocked","*",1,0],"ClaudeBot":["blocked","*",1,0],"Claude-User":["blocked","*",1,0],"Claude-SearchBot":["blocked","*",1,0],"anthropic-ai":["blocked","*",1,0],"PerplexityBot":["blocked","*",1,0],"Perplexity-User":["blocked","*",1,0],"Google-Extended":["blocked","*",1,0],"Applebot-Extended":["blocked","*",1,0],"CCBot":["blocked","*",1,0],"Bytespider":["blocked","*",1,0],"Amazonbot":["blocked","*",1,0],"meta-externalagent":["blocked","*",1,0],"cohere-ai":["blocked","*",1,0],"Diffbot":["blocked","*",1,0],"ContextFetchBot":["blocked","*",1,0]},"llms":{"status":"disallowed"},"home":{"status":"disallowed"}},
{"rank":6865,"domain":"jbc.org","origin":"https://jbc.org","robots":{"status":"ok","http":200,"url":"https://www.jbc.org/robots.txt","bytes":1822,"groups":22},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["blocked","named",1,0],"OAI-SearchBot":["partial","*",19,6],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",19,6],"Claude-SearchBot":["partial","*",19,6],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["partial","*",19,6],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["partial","*",19,6],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",19,6]},"llms":{"status":"http_error","http":403,"bytes":5517},"home":{"status":"http_error","http":403,"url":"https://jbc.org/"}},
{"rank":6866,"domain":"go2cloud.org","origin":"https://www.go2cloud.org","robots":{"status":"missing","http":404,"url":"https://www.go2cloud.org/robots.txt","bytes":0,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":0},"home":{"status":"http_error","http":404,"url":"https://www.go2cloud.org/"}},
{"rank":6867,"domain":"krita.org","origin":"https://krita.org","robots":{"status":"missing","http":404,"url":"https://krita.org/robots.txt","bytes":21259,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":21259},"home":{"status":"ok","http":200,"url":"https://krita.org/","html_tokens":54,"markdown_tokens":0,"pruned_tokens":0}},
{"rank":6868,"domain":"netent.com","origin":"https://netent.com","robots":{"status":"ok","http":200,"url":"https://netent.com/robots.txt","bytes":112,"groups":1},"bots":{"GPTBot":["allowed","*",0,1],"ChatGPT-User":["allowed","*",0,1],"OAI-SearchBot":["allowed","*",0,1],"ClaudeBot":["allowed","*",0,1],"Claude-User":["allowed","*",0,1],"Claude-SearchBot":["allowed","*",0,1],"anthropic-ai":["allowed","*",0,1],"PerplexityBot":["allowed","*",0,1],"Perplexity-User":["allowed","*",0,1],"Google-Extended":["allowed","*",0,1],"Applebot-Extended":["allowed","*",0,1],"CCBot":["allowed","*",0,1],"Bytespider":["allowed","*",0,1],"Amazonbot":["allowed","*",0,1],"meta-externalagent":["allowed","*",0,1],"cohere-ai":["allowed","*",0,1],"Diffbot":["allowed","*",0,1],"ContextFetchBot":["allowed","*",0,1]},"llms":{"status":"present","http":200,"bytes":2348},"home":{"status":"ok","http":200,"url":"https://netent.com/","html_tokens":203970,"markdown_tokens":259,"pruned_tokens":141}},
{"rank":6869,"domain":"shueisha.co.jp","origin":"https://shueisha.co.jp","robots":{"status":"ok","http":200,"url":"https://www.shueisha.co.jp/robots.txt","bytes":67,"groups":1},"bots":{"GPTBot":["partial","*",1,1],"ChatGPT-User":["partial","*",1,1],"OAI-SearchBot":["partial","*",1,1],"ClaudeBot":["partial","*",1,1],"Claude-User":["partial","*",1,1],"Claude-SearchBot":["partial","*",1,1],"anthropic-ai":["partial","*",1,1],"PerplexityBot":["partial","*",1,1],"Perplexity-User":["partial","*",1,1],"Google-Extended":["partial","*",1,1],"Applebot-Extended":["partial","*",1,1],"CCBot":["partial","*",1,1],"Bytespider":["partial","*",1,1],"Amazonbot":["partial","*",1,1],"meta-externalagent":["partial","*",1,1],"cohere-ai":["partial","*",1,1],"Diffbot":["partial","*",1,1],"ContextFetchBot":["partial","*",1,1]},"llms":{"status":"missing","http":404,"bytes":38354},"home":{"status":"ok","http":200,"url":"https://www.shueisha.co.jp/","html_tokens":18813,"markdown_tokens":35,"pruned_tokens":31}},
{"rank":6870,"domain":"amc.com","origin":"https://amc.com","robots":{"status":"ok","http":200,"url":"https://www.amc.com/robots.txt","bytes":117,"groups":1},"bots":{"GPTBot":["allowed","*",0,1],"ChatGPT-User":["allowed","*",0,1],"OAI-SearchBot":["allowed","*",0,1],"ClaudeBot":["allowed","*",0,1],"Claude-User":["allowed","*",0,1],"Claude-SearchBot":["allowed","*",0,1],"anthropic-ai":["allowed","*",0,1],"PerplexityBot":["allowed","*",0,1],"Perplexity-User":["allowed","*",0,1],"Google-Extended":["allowed","*",0,1],"Applebot-Extended":["allowed","*",0,1],"CCBot":["allowed","*",0,1],"Bytespider":["allowed","*",0,1],"Amazonbot":["allowed","*",0,1],"meta-externalagent":["allowed","*",0,1],"cohere-ai":["allowed","*",0,1],"Diffbot":["allowed","*",0,1],"ContextFetchBot":["allowed","*",0,1]},"llms":{"status":"missing","http":404,"bytes":9879},"home":{"status":"ok","http":200,"url":"https://www.amc.com/","html_tokens":88873,"markdown_tokens":78,"pruned_tokens":78}},
{"rank":6871,"domain":"lawfaremedia.org","origin":"https://lawfaremedia.org","robots":{"status":"ok","http":200,"url":"https://www.lawfaremedia.org/robots.txt","bytes":83,"groups":1},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"missing","http":404,"bytes":1193},"home":{"status":"http_error","http":403,"url":"https://lawfaremedia.org/"}},
{"rank":6872,"domain":"hexagon.com","origin":"https://hexagon.com","robots":{"status":"refused","http":403,"url":"https://hexagon.com/robots.txt","bytes":5527,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":6873,"domain":"fia.com","origin":"https://fia.com","robots":{"status":"ok","http":200,"url":"https://www.fia.com/robots.txt","bytes":2270,"groups":1},"bots":{"GPTBot":["partial","*",38,32],"ChatGPT-User":["partial","*",38,32],"OAI-SearchBot":["partial","*",38,32],"ClaudeBot":["partial","*",38,32],"Claude-User":["partial","*",38,32],"Claude-SearchBot":["partial","*",38,32],"anthropic-ai":["partial","*",38,32],"PerplexityBot":["partial","*",38,32],"Perplexity-User":["partial","*",38,32],"Google-Extended":["partial","*",38,32],"Applebot-Extended":["partial","*",38,32],"CCBot":["partial","*",38,32],"Bytespider":["partial","*",38,32],"Amazonbot":["partial","*",38,32],"meta-externalagent":["partial","*",38,32],"cohere-ai":["partial","*",38,32],"Diffbot":["partial","*",38,32],"ContextFetchBot":["partial","*",38,32]},"llms":{"status":"missing","http":404,"bytes":291},"home":{"status":"ok","http":200,"url":"https://www.fia.com/","html_tokens":264705,"markdown_tokens":279,"pruned_tokens":96}},
{"rank":6874,"domain":"utsa.edu","origin":"https://utsa.edu","robots":{"status":"ok","http":200,"url":"https://www.utsa.edu/robots.txt","bytes":837,"groups":2},"bots":{"GPTBot":["partial","*",30,0],"ChatGPT-User":["partial","*",30,0],"OAI-SearchBot":["partial","*",30,0],"ClaudeBot":["partial","*",30,0],"Claude-User":["partial","*",30,0],"Claude-SearchBot":["partial","*",30,0],"anthropic-ai":["partial","*",30,0],"PerplexityBot":["partial","*",30,0],"Perplexity-User":["partial","*",30,0],"Google-Extended":["partial","*",30,0],"Applebot-Extended":["partial","*",30,0],"CCBot":["partial","*",30,0],"Bytespider":["partial","*",30,0],"Amazonbot":["partial","*",30,0],"meta-externalagent":["partial","*",30,0],"cohere-ai":["partial","*",30,0],"Diffbot":["partial","*",30,0],"ContextFetchBot":["partial","*",30,0]},"llms":{"status":"html","http":200,"bytes":46775},"home":{"status":"ok","http":200,"url":"https://www.utsa.edu/","html_tokens":15374,"markdown_tokens":361,"pruned_tokens":143}},
{"rank":6875,"domain":"bmwk.de","origin":"https://www.bmwk.de","robots":{"status":"missing","http":404,"url":"https://www.bundeswirtschaftsministerium.de/robots.txt","bytes":133493,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":133596},"home":{"status":"ok","http":200,"url":"https://www.bundeswirtschaftsministerium.de/Navigation/DE/Home/home.html","html_tokens":29603,"markdown_tokens":13,"pruned_tokens":13}},
{"rank":6876,"domain":"ledevoir.com","origin":"https://ledevoir.com","robots":{"status":"ok","http":200,"url":"https://www.ledevoir.com/robots.txt","bytes":4180,"groups":89},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["blocked","named",1,0],"OAI-SearchBot":["blocked","named",1,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",7,0],"Claude-SearchBot":["partial","*",7,0],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["partial","*",7,0],"Perplexity-User":["partial","*",7,0],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["partial","*",7,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",7,0]},"llms":{"status":"missing","http":404,"bytes":48507},"home":{"status":"ok","http":200,"url":"https://www.ledevoir.com/","html_tokens":293241,"markdown_tokens":7716,"pruned_tokens":112}},
{"rank":6877,"domain":"audi.com","origin":"https://audi.com","robots":{"status":"ok","http":200,"url":"https://www.audi.com/robots.txt","bytes":130,"groups":1},"bots":{"GPTBot":["partial","*",1,0],"ChatGPT-User":["partial","*",1,0],"OAI-SearchBot":["partial","*",1,0],"ClaudeBot":["partial","*",1,0],"Claude-User":["partial","*",1,0],"Claude-SearchBot":["partial","*",1,0],"anthropic-ai":["partial","*",1,0],"PerplexityBot":["partial","*",1,0],"Perplexity-User":["partial","*",1,0],"Google-Extended":["partial","*",1,0],"Applebot-Extended":["partial","*",1,0],"CCBot":["partial","*",1,0],"Bytespider":["partial","*",1,0],"Amazonbot":["partial","*",1,0],"meta-externalagent":["partial","*",1,0],"cohere-ai":["partial","*",1,0],"Diffbot":["partial","*",1,0],"ContextFetchBot":["partial","*",1,0]},"llms":{"status":"present","http":200,"bytes":1833},"home":{"status":"ok","http":200,"url":"https://www.audi.com/en","html_tokens":17209,"markdown_tokens":38,"pruned_tokens":38}},
{"rank":6878,"domain":"bget.ru","origin":"https://bget.ru","robots":{"status":"html","http":200,"url":"https://bget.ru/robots.txt","bytes":10934,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":6879,"domain":"xserver.ne.jp","origin":"https://xserver.ne.jp","robots":{"status":"ok","http":200,"url":"https://www.xserver.ne.jp/robots.txt","bytes":173,"groups":1},"bots":{"GPTBot":["partial","*",5,0],"ChatGPT-User":["partial","*",5,0],"OAI-SearchBot":["partial","*",5,0],"ClaudeBot":["partial","*",5,0],"Claude-User":["partial","*",5,0],"Claude-SearchBot":["partial","*",5,0],"anthropic-ai":["partial","*",5,0],"PerplexityBot":["partial","*",5,0],"Perplexity-User":["partial","*",5,0],"Google-Extended":["partial","*",5,0],"Applebot-Extended":["partial","*",5,0],"CCBot":["partial","*",5,0],"Bytespider":["partial","*",5,0],"Amazonbot":["partial","*",5,0],"meta-externalagent":["partial","*",5,0],"cohere-ai":["partial","*",5,0],"Diffbot":["partial","*",5,0],"ContextFetchBot":["partial","*",5,0]},"llms":{"status":"missing","http":404,"bytes":66330},"home":{"status":"ok","http":200,"url":"https://www.xserver.ne.jp/","html_tokens":32220,"markdown_tokens":148,"pruned_tokens":148}},
{"rank":6880,"domain":"crisistextline.org","origin":"https://crisistextline.org","robots":{"status":"ok","http":200,"url":"https://crisistextline.org/robots.txt","bytes":1513,"groups":1},"bots":{"GPTBot":["partial","named",27,1],"ChatGPT-User":["partial","*",27,1],"OAI-SearchBot":["partial","*",27,1],"ClaudeBot":["partial","named",27,1],"Claude-User":["partial","*",27,1],"Claude-SearchBot":["partial","*",27,1],"anthropic-ai":["partial","named",27,1],"PerplexityBot":["partial","*",27,1],"Perplexity-User":["partial","*",27,1],"Google-Extended":["partial","named",27,1],"Applebot-Extended":["partial","named",27,1],"CCBot":["partial","named",27,1],"Bytespider":["partial","named",27,1],"Amazonbot":["partial","named",27,1],"meta-externalagent":["partial","named",27,1],"cohere-ai":["partial","named",27,1],"Diffbot":["partial","*",27,1],"ContextFetchBot":["partial","*",27,1]},"llms":{"status":"missing","http":404,"bytes":60},"home":{"status":"ok","http":200,"url":"https://www.crisistextline.org/","html_tokens":45905,"markdown_tokens":413,"pruned_tokens":83}},
{"rank":6881,"domain":"zum.de","origin":"https://zum.de","robots":{"status":"ok","http":200,"url":"https://www.zum.de/robots.txt","bytes":104,"groups":1},"bots":{"GPTBot":["partial","*",2,0],"ChatGPT-User":["partial","*",2,0],"OAI-SearchBot":["partial","*",2,0],"ClaudeBot":["partial","*",2,0],"Claude-User":["partial","*",2,0],"Claude-SearchBot":["partial","*",2,0],"anthropic-ai":["partial","*",2,0],"PerplexityBot":["partial","*",2,0],"Perplexity-User":["partial","*",2,0],"Google-Extended":["partial","*",2,0],"Applebot-Extended":["partial","*",2,0],"CCBot":["partial","*",2,0],"Bytespider":["partial","*",2,0],"Amazonbot":["partial","*",2,0],"meta-externalagent":["partial","*",2,0],"cohere-ai":["partial","*",2,0],"Diffbot":["partial","*",2,0],"ContextFetchBot":["partial","*",2,0]},"llms":{"status":"timeout","error":"timed out after 10 s"},"home":{"status":"ok","http":200,"url":"https://www.zum.de/portal","html_tokens":38081,"markdown_tokens":347,"pruned_tokens":232}},
{"rank":6882,"domain":"usip.org","origin":"https://usip.org","robots":{"status":"ok","http":200,"url":"https://www.usip.org/robots.txt","bytes":110,"groups":1},"bots":{"GPTBot":["partial","*",1,1],"ChatGPT-User":["partial","*",1,1],"OAI-SearchBot":["partial","*",1,1],"ClaudeBot":["partial","*",1,1],"Claude-User":["partial","*",1,1],"Claude-SearchBot":["partial","*",1,1],"anthropic-ai":["partial","*",1,1],"PerplexityBot":["partial","*",1,1],"Perplexity-User":["partial","*",1,1],"Google-Extended":["partial","*",1,1],"Applebot-Extended":["partial","*",1,1],"CCBot":["partial","*",1,1],"Bytespider":["partial","*",1,1],"Amazonbot":["partial","*",1,1],"meta-externalagent":["partial","*",1,1],"cohere-ai":["partial","*",1,1],"Diffbot":["partial","*",1,1],"ContextFetchBot":["partial","*",1,1]},"llms":{"status":"html","http":200,"bytes":109365},"home":{"status":"ok","http":200,"url":"https://www.usip.org/","html_tokens":27340,"markdown_tokens":45,"pruned_tokens":45}},
{"rank":6883,"domain":"desa.id","origin":"https://desa.id","robots":{"status":"unreachable","http":null,"url":"https://desa.id/robots.txt","bytes":0,"groups":0,"error":"cannot resolve desa.id"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":6884,"domain":"odni.gov","origin":"https://odni.gov","robots":{"status":"ok","http":200,"url":"https://www.odni.gov/robots.txt","bytes":779,"groups":3},"bots":{"GPTBot":["partial","*",2,1],"ChatGPT-User":["partial","*",2,1],"OAI-SearchBot":["partial","*",2,1],"ClaudeBot":["partial","*",2,1],"Claude-User":["partial","*",2,1],"Claude-SearchBot":["partial","*",2,1],"anthropic-ai":["partial","*",2,1],"PerplexityBot":["partial","*",2,1],"Perplexity-User":["partial","*",2,1],"Google-Extended":["partial","*",2,1],"Applebot-Extended":["partial","*",2,1],"CCBot":["partial","*",2,1],"Bytespider":["partial","*",2,1],"Amazonbot":["partial","*",2,1],"meta-externalagent":["partial","*",2,1],"cohere-ai":["partial","*",2,1],"Diffbot":["partial","*",2,1],"ContextFetchBot":["partial","*",2,1]},"llms":{"status":"timeout","error":"timed out after 10 s"},"home":{"status":"ok","http":200,"url":"https://www.odni.gov/","html_tokens":46818,"markdown_tokens":130,"pruned_tokens":130}},
{"rank":6885,"domain":"4chan.org","origin":"https://4chan.org","robots":{"status":"ok","http":200,"url":"https://4chan.org/robots.txt","bytes":64,"groups":1},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"missing","http":404,"bytes":2945},"home":{"status":"http_error","http":403,"url":"https://4chan.org/"}},
{"rank":6886,"domain":"oneplus.com","origin":"https://oneplus.com","robots":{"status":"ok","http":200,"url":"https://www.oneplus.com/robots.txt","bytes":4550,"groups":23},"bots":{"GPTBot":["partial","*",13,1],"ChatGPT-User":["partial","*",13,1],"OAI-SearchBot":["partial","*",13,1],"ClaudeBot":["partial","*",13,1],"Claude-User":["partial","*",13,1],"Claude-SearchBot":["partial","*",13,1],"anthropic-ai":["partial","*",13,1],"PerplexityBot":["partial","*",13,1],"Perplexity-User":["partial","*",13,1],"Google-Extended":["partial","*",13,1],"Applebot-Extended":["partial","*",13,1],"CCBot":["partial","*",13,1],"Bytespider":["partial","*",13,1],"Amazonbot":["partial","*",13,1],"meta-externalagent":["partial","*",13,1],"cohere-ai":["partial","*",13,1],"Diffbot":["partial","*",13,1],"ContextFetchBot":["partial","*",13,1]},"llms":{"status":"missing","http":404,"bytes":154576},"home":{"status":"ok","http":200,"url":"https://www.oneplus.com/us","html_tokens":30049,"markdown_tokens":6,"pruned_tokens":6}},
{"rank":6887,"domain":"ritsumei.ac.jp","origin":"https://www.ritsumei.ac.jp","robots":{"status":"ok","http":200,"url":"https://www.ritsumei.ac.jp/robots.txt","bytes":34,"groups":1},"bots":{"GPTBot":["partial","*",1,0],"ChatGPT-User":["partial","*",1,0],"OAI-SearchBot":["partial","*",1,0],"ClaudeBot":["partial","*",1,0],"Claude-User":["partial","*",1,0],"Claude-SearchBot":["partial","*",1,0],"anthropic-ai":["partial","*",1,0],"PerplexityBot":["partial","*",1,0],"Perplexity-User":["partial","*",1,0],"Google-Extended":["partial","*",1,0],"Applebot-Extended":["partial","*",1,0],"CCBot":["partial","*",1,0],"Bytespider":["partial","*",1,0],"Amazonbot":["partial","*",1,0],"meta-externalagent":["partial","*",1,0],"cohere-ai":["partial","*",1,0],"Diffbot":["partial","*",1,0],"ContextFetchBot":["partial","*",1,0]},"llms":{"status":"missing","http":404,"bytes":33509},"home":{"status":"ok","http":200,"url":"https://www.ritsumei.ac.jp/","html_tokens":16258,"markdown_tokens":385,"pruned_tokens":56}},
{"rank":6888,"domain":"ilgiornale.it","origin":"https://ilgiornale.it","robots":{"status":"ok","http":200,"url":"https://www.ilgiornale.it/robots.txt","bytes":947,"groups":1},"bots":{"GPTBot":["partial","*",10,0],"ChatGPT-User":["partial","*",10,0],"OAI-SearchBot":["partial","*",10,0],"ClaudeBot":["partial","*",10,0],"Claude-User":["partial","*",10,0],"Claude-SearchBot":["partial","*",10,0],"anthropic-ai":["partial","*",10,0],"PerplexityBot":["partial","*",10,0],"Perplexity-User":["partial","*",10,0],"Google-Extended":["partial","*",10,0],"Applebot-Extended":["partial","*",10,0],"CCBot":["partial","*",10,0],"Bytespider":["partial","*",10,0],"Amazonbot":["partial","*",10,0],"meta-externalagent":["partial","*",10,0],"cohere-ai":["partial","*",10,0],"Diffbot":["partial","*",10,0],"ContextFetchBot":["partial","*",10,0]},"llms":{"status":"missing","http":404,"bytes":111803},"home":{"status":"ok","http":200,"url":"https://www.ilgiornale.it/","html_tokens":138428,"markdown_tokens":569,"pruned_tokens":150}},
{"rank":6889,"domain":"rankmath.com","origin":"https://rankmath.com","robots":{"status":"ok","http":200,"url":"https://rankmath.com/robots.txt","bytes":247,"groups":2},"bots":{"GPTBot":["partial","*",2,2],"ChatGPT-User":["partial","*",2,2],"OAI-SearchBot":["partial","*",2,2],"ClaudeBot":["partial","*",2,2],"Claude-User":["partial","*",2,2],"Claude-SearchBot":["partial","*",2,2],"anthropic-ai":["partial","*",2,2],"PerplexityBot":["partial","*",2,2],"Perplexity-User":["partial","*",2,2],"Google-Extended":["partial","*",2,2],"Applebot-Extended":["partial","*",2,2],"CCBot":["partial","*",2,2],"Bytespider":["partial","*",2,2],"Amazonbot":["partial","*",2,2],"meta-externalagent":["partial","*",2,2],"cohere-ai":["partial","*",2,2],"Diffbot":["partial","*",2,2],"ContextFetchBot":["partial","*",2,2]},"llms":{"status":"missing","http":404,"bytes":80055},"home":{"status":"ok","http":200,"url":"https://rankmath.com/","html_tokens":76859,"markdown_tokens":740,"pruned_tokens":135}},
{"rank":6890,"domain":"kentucky.com","origin":"https://kentucky.com","robots":{"status":"unreachable","http":null,"url":"https://kentucky.com/robots.txt","bytes":0,"groups":0,"error":"<StreamReset stream_id:1, error_code:2, remote_reset:True>"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":6891,"domain":"itunes.com","origin":"https://itunes.com","robots":{"status":"html","http":200,"url":"https://www.apple.com/itunes/?cid=OAS-US-DOMAINS-itunes.com","bytes":134603,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":6892,"domain":"mindful.org","origin":"https://mindful.org","robots":{"status":"ok","http":200,"url":"https://www.mindful.org/robots.txt","bytes":200,"groups":1},"bots":{"GPTBot":["partial","*",5,1],"ChatGPT-User":["partial","*",5,1],"OAI-SearchBot":["partial","*",5,1],"ClaudeBot":["partial","*",5,1],"Claude-User":["partial","*",5,1],"Claude-SearchBot":["partial","*",5,1],"anthropic-ai":["partial","*",5,1],"PerplexityBot":["partial","*",5,1],"Perplexity-User":["partial","*",5,1],"Google-Extended":["partial","*",5,1],"Applebot-Extended":["partial","*",5,1],"CCBot":["partial","*",5,1],"Bytespider":["partial","*",5,1],"Amazonbot":["partial","*",5,1],"meta-externalagent":["partial","*",5,1],"cohere-ai":["partial","*",5,1],"Diffbot":["partial","*",5,1],"ContextFetchBot":["partial","*",5,1]},"llms":{"status":"other_text","http":200,"bytes":16398},"home":{"status":"ok","http":200,"url":"https://www.mindful.org/","html_tokens":271818,"markdown_tokens":397,"pruned_tokens":104}},
{"rank":6893,"domain":"thinkupthemes.com","origin":"https://thinkupthemes.com","robots":{"status":"ok","http":200,"url":"https://www.thinkupthemes.com/robots.txt","bytes":179,"groups":1},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"missing","http":404,"bytes":40393},"home":{"status":"ok","http":200,"url":"https://www.thinkupthemes.com/","html_tokens":29061,"markdown_tokens":48,"pruned_tokens":48}},
{"rank":6894,"domain":"brunel.ac.uk","origin":"https://brunel.ac.uk","robots":{"status":"ok","http":200,"url":"https://www.brunel.ac.uk/robots.txt","bytes":1698,"groups":1},"bots":{"GPTBot":["partial","*",46,1],"ChatGPT-User":["partial","*",46,1],"OAI-SearchBot":["partial","*",46,1],"ClaudeBot":["partial","*",46,1],"Claude-User":["partial","*",46,1],"Claude-SearchBot":["partial","*",46,1],"anthropic-ai":["partial","*",46,1],"PerplexityBot":["partial","*",46,1],"Perplexity-User":["partial","*",46,1],"Google-Extended":["partial","*",46,1],"Applebot-Extended":["partial","*",46,1],"CCBot":["partial","*",46,1],"Bytespider":["partial","*",46,1],"Amazonbot":["partial","*",46,1],"meta-externalagent":["partial","*",46,1],"cohere-ai":["partial","*",46,1],"Diffbot":["partial","*",46,1],"ContextFetchBot":["partial","*",46,1]},"llms":{"status":"missing","http":404,"bytes":14},"home":{"status":"ok","http":200,"url":"https://www.brunel.ac.uk/","html_tokens":309132,"markdown_tokens":210,"pruned_tokens":180}},
{"rank":6895,"domain":"panasonic.co.jp","origin":"https://panasonic.co.jp","robots":{"status":"missing","http":404,"url":"https://panasonic.co.jp/robots.txt","bytes":222,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":220},"home":{"status":"http_error","http":403,"url":"https://panasonic.co.jp/"}},
{"rank":6896,"domain":"yandex.net","origin":"https://yandex.net","robots":{"status":"missing","http":404,"url":"https://yandex.net/robots.txt","bytes":9,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":11769},"home":{"status":"ok","http":200,"url":"https://ya.ru/","html_tokens":121351,"markdown_tokens":22,"pruned_tokens":22}},
{"rank":6897,"domain":"actblue.com","origin":"https://actblue.com","robots":{"status":"ok","http":200,"url":"https://www.actblue.com/robots.txt","bytes":762,"groups":1},"bots":{"GPTBot":["partial","*",22,0],"ChatGPT-User":["partial","*",22,0],"OAI-SearchBot":["partial","*",22,0],"ClaudeBot":["partial","*",22,0],"Claude-User":["partial","*",22,0],"Claude-SearchBot":["partial","*",22,0],"anthropic-ai":["partial","*",22,0],"PerplexityBot":["partial","*",22,0],"Perplexity-User":["partial","*",22,0],"Google-Extended":["partial","*",22,0],"Applebot-Extended":["partial","*",22,0],"CCBot":["partial","*",22,0],"Bytespider":["partial","*",22,0],"Amazonbot":["partial","*",22,0],"meta-externalagent":["partial","*",22,0],"cohere-ai":["partial","*",22,0],"Diffbot":["partial","*",22,0],"ContextFetchBot":["partial","*",22,0]},"llms":{"status":"missing","http":404,"bytes":60},"home":{"status":"ok","http":200,"url":"https://www.actblue.com/","html_tokens":34513,"markdown_tokens":207,"pruned_tokens":84}},
{"rank":6898,"domain":"surferseo.com","origin":"https://surferseo.com","robots":{"status":"ok","http":200,"url":"https://surferseo.com/robots.txt/","bytes":144,"groups":1},"bots":{"GPTBot":["partial","*",5,0],"ChatGPT-User":["partial","*",5,0],"OAI-SearchBot":["partial","*",5,0],"ClaudeBot":["partial","*",5,0],"Claude-User":["partial","*",5,0],"Claude-SearchBot":["partial","*",5,0],"anthropic-ai":["partial","*",5,0],"PerplexityBot":["partial","*",5,0],"Perplexity-User":["partial","*",5,0],"Google-Extended":["partial","*",5,0],"Applebot-Extended":["partial","*",5,0],"CCBot":["partial","*",5,0],"Bytespider":["partial","*",5,0],"Amazonbot":["partial","*",5,0],"meta-externalagent":["partial","*",5,0],"cohere-ai":["partial","*",5,0],"Diffbot":["partial","*",5,0],"ContextFetchBot":["partial","*",5,0]},"llms":{"status":"missing","http":404,"bytes":59731},"home":{"status":"ok","http":200,"url":"https://surferseo.com/","html_tokens":105800,"markdown_tokens":1010,"pruned_tokens":65}},
{"rank":6899,"domain":"vodafone.de","origin":"https://vodafone.de","robots":{"status":"ok","http":200,"url":"https://www.vodafone.de/robots.txt","bytes":2109,"groups":10},"bots":{"GPTBot":["partial","*",39,3],"ChatGPT-User":["partial","*",39,3],"OAI-SearchBot":["partial","*",39,3],"ClaudeBot":["partial","*",39,3],"Claude-User":["partial","*",39,3],"Claude-SearchBot":["partial","*",39,3],"anthropic-ai":["partial","*",39,3],"PerplexityBot":["partial","*",39,3],"Perplexity-User":["partial","*",39,3],"Google-Extended":["partial","*",39,3],"Applebot-Extended":["partial","*",39,3],"CCBot":["partial","*",39,3],"Bytespider":["partial","*",39,3],"Amazonbot":["partial","named",2,0],"meta-externalagent":["partial","*",39,3],"cohere-ai":["partial","*",39,3],"Diffbot":["partial","*",39,3],"ContextFetchBot":["partial","*",39,3]},"llms":{"status":"missing","http":404,"bytes":114438},"home":{"status":"ok","http":200,"url":"https://www.vodafone.de/","html_tokens":36592,"markdown_tokens":186,"pruned_tokens":113}},
{"rank":6900,"domain":"simpleflying.com","origin":"https://simpleflying.com","robots":{"status":"ok","http":200,"url":"https://simpleflying.com/robots.txt","bytes":2037,"groups":2},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["partial","*",22,0],"OAI-SearchBot":["partial","*",22,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",22,0],"Claude-SearchBot":["partial","*",22,0],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["partial","*",22,0],"Perplexity-User":["partial","*",22,0],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",22,0]},"llms":{"status":"missing","http":404,"bytes":100649},"home":{"status":"ok","http":200,"url":"https://simpleflying.com/","html_tokens":243230,"markdown_tokens":304,"pruned_tokens":141}},
{"rank":6901,"domain":"uniroma2.it","origin":"https://uniroma2.it","robots":{"status":"ok","http":200,"url":"https://web.uniroma2.it/robots.txt","bytes":25,"groups":1},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"html","http":200,"bytes":47298},"home":{"status":"ok","http":200,"url":"https://web.uniroma2.it/","html_tokens":11816,"markdown_tokens":82,"pruned_tokens":48}},
{"rank":6902,"domain":"alibabagroup.com","origin":"https://alibabagroup.com","robots":{"status":"html","http":200,"url":"https://www.alibabagroup.com/en-US/robots.txt","bytes":58316,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":6903,"domain":"rtx.com","origin":"https://rtx.com","robots":{"status":"ok","http":200,"url":"https://www.rtx.com/robots.txt","bytes":232,"groups":1},"bots":{"GPTBot":["allowed","*",0,1],"ChatGPT-User":["allowed","*",0,1],"OAI-SearchBot":["allowed","*",0,1],"ClaudeBot":["allowed","*",0,1],"Claude-User":["allowed","*",0,1],"Claude-SearchBot":["allowed","*",0,1],"anthropic-ai":["allowed","*",0,1],"PerplexityBot":["allowed","*",0,1],"Perplexity-User":["allowed","*",0,1],"Google-Extended":["allowed","*",0,1],"Applebot-Extended":["allowed","*",0,1],"CCBot":["allowed","*",0,1],"Bytespider":["allowed","*",0,1],"Amazonbot":["allowed","*",0,1],"meta-externalagent":["allowed","*",0,1],"cohere-ai":["allowed","*",0,1],"Diffbot":["allowed","*",0,1],"ContextFetchBot":["allowed","*",0,1]},"llms":{"status":"missing","http":404,"bytes":103},"home":{"status":"ok","http":200,"url":"https://www.rtx.com/","html_tokens":17225,"markdown_tokens":133,"pruned_tokens":32}},
{"rank":6904,"domain":"brother.com","origin":"https://brother.com","robots":{"status":"missing","http":404,"url":"https://www.brother.com/robots.txt","bytes":3133,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":3133},"home":{"status":"ok","http":200,"url":"https://www.brother-usa.com/","html_tokens":217915,"markdown_tokens":80,"pruned_tokens":58}},
{"rank":6905,"domain":"thinglink.com","origin":"https://thinglink.com","robots":{"status":"ok","http":200,"url":"https://www.thinglink.com/robots.txt","bytes":500,"groups":1},"bots":{"GPTBot":["partial","*",18,0],"ChatGPT-User":["partial","*",18,0],"OAI-SearchBot":["partial","*",18,0],"ClaudeBot":["partial","*",18,0],"Claude-User":["partial","*",18,0],"Claude-SearchBot":["partial","*",18,0],"anthropic-ai":["partial","*",18,0],"PerplexityBot":["partial","*",18,0],"Perplexity-User":["partial","*",18,0],"Google-Extended":["partial","*",18,0],"Applebot-Extended":["partial","*",18,0],"CCBot":["partial","*",18,0],"Bytespider":["partial","*",18,0],"Amazonbot":["partial","*",18,0],"meta-externalagent":["partial","*",18,0],"cohere-ai":["partial","*",18,0],"Diffbot":["partial","*",18,0],"ContextFetchBot":["partial","*",18,0]},"llms":{"status":"missing","http":404,"bytes":2495},"home":{"status":"ok","http":200,"url":"https://www.thinglink.com/","html_tokens":73207,"markdown_tokens":68,"pruned_tokens":68}},
{"rank":6906,"domain":"webcrawler.com","origin":"https://www.webcrawler.com","robots":{"status":"ok","http":200,"url":"https://www.webcrawler.com/robots.txt","bytes":52,"groups":1},"bots":{"GPTBot":["partial","*",1,2],"ChatGPT-User":["partial","*",1,2],"OAI-SearchBot":["partial","*",1,2],"ClaudeBot":["partial","*",1,2],"Claude-User":["partial","*",1,2],"Claude-SearchBot":["partial","*",1,2],"anthropic-ai":["partial","*",1,2],"PerplexityBot":["partial","*",1,2],"Perplexity-User":["partial","*",1,2],"Google-Extended":["partial","*",1,2],"Applebot-Extended":["partial","*",1,2],"CCBot":["partial","*",1,2],"Bytespider":["partial","*",1,2],"Amazonbot":["partial","*",1,2],"meta-externalagent":["partial","*",1,2],"cohere-ai":["partial","*",1,2],"Diffbot":["partial","*",1,2],"ContextFetchBot":["partial","*",1,2]},"llms":{"status":"disallowed"},"home":{"status":"ok","http":200,"url":"https://www.webcrawler.com/","html_tokens":9988,"markdown_tokens":6,"pruned_tokens":6}},
{"rank":6907,"domain":"bark.com","origin":"https://bark.com","robots":{"status":"ok","http":200,"url":"https://www.bark.com/robots.txt","bytes":3790,"groups":8},"bots":{"GPTBot":["partial","*",44,0],"ChatGPT-User":["partial","*",44,0],"OAI-SearchBot":["partial","*",44,0],"ClaudeBot":["partial","*",44,0],"Claude-User":["partial","*",44,0],"Claude-SearchBot":["partial","*",44,0],"anthropic-ai":["partial","*",44,0],"PerplexityBot":["partial","*",44,0],"Perplexity-User":["partial","*",44,0],"Google-Extended":["partial","*",44,0],"Applebot-Extended":["partial","*",44,0],"CCBot":["partial","*",44,0],"Bytespider":["partial","*",44,0],"Amazonbot":["partial","*",44,0],"meta-externalagent":["partial","*",44,0],"cohere-ai":["partial","*",44,0],"Diffbot":["partial","*",44,0],"ContextFetchBot":["partial","*",44,0]},"llms":{"status":"missing","http":404,"bytes":319786},"home":{"status":"ok","http":200,"url":"https://www.bark.com/en/us/","html_tokens":104488,"markdown_tokens":15,"pruned_tokens":15}},
{"rank":6908,"domain":"nh-hotels.com","origin":"https://nh-hotels.com","robots":{"status":"refused","http":403,"url":"https://nh-hotels.com/robots.txt","bytes":379,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":6909,"domain":"ryerson.ca","origin":"https://ryerson.ca","robots":{"status":"ok","http":200,"url":"https://www.torontomu.ca/robots.txt","bytes":4555,"groups":2},"bots":{"GPTBot":["partial","*",115,0],"ChatGPT-User":["partial","*",115,0],"OAI-SearchBot":["partial","*",115,0],"ClaudeBot":["partial","*",115,0],"Claude-User":["partial","*",115,0],"Claude-SearchBot":["partial","*",115,0],"anthropic-ai":["partial","*",115,0],"PerplexityBot":["partial","*",115,0],"Perplexity-User":["partial","*",115,0],"Google-Extended":["partial","*",115,0],"Applebot-Extended":["partial","*",115,0],"CCBot":["partial","*",115,0],"Bytespider":["partial","*",115,0],"Amazonbot":["partial","*",115,0],"meta-externalagent":["partial","*",115,0],"cohere-ai":["partial","*",115,0],"Diffbot":["partial","*",115,0],"ContextFetchBot":["partial","*",115,0]},"llms":{"status":"missing","http":404,"bytes":18946},"home":{"status":"ok","http":200,"url":"https://www.torontomu.ca/","html_tokens":17467,"markdown_tokens":197,"pruned_tokens":89}},
{"rank":6910,"domain":"netbeans.org","origin":"https://netbeans.org","robots":{"status":"missing","http":404,"url":"https://netbeans.apache.org//robots.txt","bytes":315,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":315},"home":{"status":"ok","http":200,"url":"https://netbeans.apache.org/front/main/index.html","html_tokens":3325,"markdown_tokens":119,"pruned_tokens":119}},
{"rank":6911,"domain":"jussieu.fr","origin":"https://jussieu.fr","robots":{"status":"unreachable","http":null,"url":"https://jussieu.fr/robots.txt","bytes":0,"groups":0,"error":"cannot resolve jussieu.fr"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":6912,"domain":"rev.com","origin":"https://rev.com","robots":{"status":"ok","http":200,"url":"https://www.rev.com/robots.txt","bytes":390,"groups":3},"bots":{"GPTBot":["partial","*",5,0],"ChatGPT-User":["partial","*",5,0],"OAI-SearchBot":["partial","*",5,0],"ClaudeBot":["partial","*",5,0],"Claude-User":["partial","*",5,0],"Claude-SearchBot":["partial","*",5,0],"anthropic-ai":["partial","*",5,0],"PerplexityBot":["partial","*",5,0],"Perplexity-User":["partial","*",5,0],"Google-Extended":["partial","*",5,0],"Applebot-Extended":["partial","*",5,0],"CCBot":["partial","*",5,0],"Bytespider":["partial","*",5,0],"Amazonbot":["partial","*",5,0],"meta-externalagent":["partial","*",5,0],"cohere-ai":["partial","*",5,0],"Diffbot":["partial","*",5,0],"ContextFetchBot":["partial","*",5,0]},"llms":{"status":"present","http":200,"bytes":3216},"home":{"status":"ok","http":200,"url":"https://www.rev.com/","html_tokens":64794,"markdown_tokens":667,"pruned_tokens":317}},
{"rank":6913,"domain":"hugoboss.com","origin":"https://hugoboss.com","robots":{"status":"refused","http":403,"url":"https://www.hugoboss.com/robots.txt","bytes":384,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":6914,"domain":"aom.org","origin":"https://aom.org","robots":{"status":"ok","http":200,"url":"https://www.aom.org/robots.txt","bytes":185,"groups":1},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"http_error","http":403,"bytes":5538},"home":{"status":"http_error","http":403,"url":"https://aom.org/"}},
{"rank":6915,"domain":"betterup.com","origin":"https://betterup.com","robots":{"status":"ok","http":200,"url":"https://www.betterup.com/robots.txt","bytes":1766,"groups":8},"bots":{"GPTBot":["partial","named",5,0],"ChatGPT-User":["partial","named",6,1],"OAI-SearchBot":["partial","*",6,1],"ClaudeBot":["partial","named",5,0],"Claude-User":["partial","*",6,1],"Claude-SearchBot":["partial","*",6,1],"anthropic-ai":["partial","named",6,1],"PerplexityBot":["partial","named",6,1],"Perplexity-User":["partial","*",6,1],"Google-Extended":["partial","named",6,1],"Applebot-Extended":["partial","*",6,1],"CCBot":["partial","named",6,1],"Bytespider":["partial","*",6,1],"Amazonbot":["partial","*",6,1],"meta-externalagent":["partial","*",6,1],"cohere-ai":["partial","*",6,1],"Diffbot":["partial","*",6,1],"ContextFetchBot":["partial","*",6,1]},"llms":{"status":"present","http":200,"bytes":6075},"home":{"status":"ok","http":200,"url":"https://www.betterup.com/","html_tokens":40210,"markdown_tokens":547,"pruned_tokens":313}},
{"rank":6916,"domain":"argos.co.uk","origin":"https://argos.co.uk","robots":{"status":"refused","http":403,"url":"https://argos.co.uk/robots.txt","bytes":377,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":6917,"domain":"shorturl.fm","origin":"https://shorturl.fm","robots":{"status":"ok","http":200,"url":"https://shorturl.fm/robots.txt","bytes":221,"groups":1},"bots":{"GPTBot":["partial","*",7,0],"ChatGPT-User":["partial","*",7,0],"OAI-SearchBot":["partial","*",7,0],"ClaudeBot":["partial","*",7,0],"Claude-User":["partial","*",7,0],"Claude-SearchBot":["partial","*",7,0],"anthropic-ai":["partial","*",7,0],"PerplexityBot":["partial","*",7,0],"Perplexity-User":["partial","*",7,0],"Google-Extended":["partial","*",7,0],"Applebot-Extended":["partial","*",7,0],"CCBot":["partial","*",7,0],"Bytespider":["partial","*",7,0],"Amazonbot":["partial","*",7,0],"meta-externalagent":["partial","*",7,0],"cohere-ai":["partial","*",7,0],"Diffbot":["partial","*",7,0],"ContextFetchBot":["partial","*",7,0]},"llms":{"status":"missing","http":404,"bytes":7021},"home":{"status":"ok","http":200,"url":"https://shorturl.fm/","html_tokens":24751,"markdown_tokens":143,"pruned_tokens":117}},
{"rank":6918,"domain":"freenode.net","origin":"https://freenode.net","robots":{"status":"ok","http":200,"url":"https://freenode.net/robots.txt","bytes":119,"groups":1},"bots":{"GPTBot":["partial","*",3,1],"ChatGPT-User":["partial","*",3,1],"OAI-SearchBot":["partial","*",3,1],"ClaudeBot":["partial","*",3,1],"Claude-User":["partial","*",3,1],"Claude-SearchBot":["partial","*",3,1],"anthropic-ai":["partial","*",3,1],"PerplexityBot":["partial","*",3,1],"Perplexity-User":["partial","*",3,1],"Google-Extended":["partial","*",3,1],"Applebot-Extended":["partial","*",3,1],"CCBot":["partial","*",3,1],"Bytespider":["partial","*",3,1],"Amazonbot":["partial","*",3,1],"meta-externalagent":["partial","*",3,1],"cohere-ai":["partial","*",3,1],"Diffbot":["partial","*",3,1],"ContextFetchBot":["partial","*",3,1]},"llms":{"status":"missing","http":404,"bytes":22},"home":{"status":"ok","http":200,"url":"https://freenode.net/","html_tokens":8167,"markdown_tokens":574,"pruned_tokens":60}},
{"rank":6919,"domain":"chelseafc.com","origin":"https://chelseafc.com","robots":{"status":"ok","http":200,"url":"https://www.chelseafc.com/robots.txt","bytes":273,"groups":1},"bots":{"GPTBot":["allowed","*",0,1],"ChatGPT-User":["allowed","*",0,1],"OAI-SearchBot":["allowed","*",0,1],"ClaudeBot":["allowed","*",0,1],"Claude-User":["allowed","*",0,1],"Claude-SearchBot":["allowed","*",0,1],"anthropic-ai":["allowed","*",0,1],"PerplexityBot":["allowed","*",0,1],"Perplexity-User":["allowed","*",0,1],"Google-Extended":["allowed","*",0,1],"Applebot-Extended":["allowed","*",0,1],"CCBot":["allowed","*",0,1],"Bytespider":["allowed","*",0,1],"Amazonbot":["allowed","*",0,1],"meta-externalagent":["allowed","*",0,1],"cohere-ai":["allowed","*",0,1],"Diffbot":["allowed","*",0,1],"ContextFetchBot":["allowed","*",0,1]},"llms":{"status":"missing","http":404,"bytes":86562},"home":{"status":"ok","http":200,"url":"https://www.chelseafc.com/en","html_tokens":140211,"markdown_tokens":0,"pruned_tokens":0}},
{"rank":6920,"domain":"omnihotels.com","origin":"https://omnihotels.com","robots":{"status":"ok","http":200,"url":"https://www.omnihotels.com/robots.txt","bytes":154,"groups":1},"bots":{"GPTBot":["partial","*",1,1],"ChatGPT-User":["partial","*",1,1],"OAI-SearchBot":["partial","*",1,1],"ClaudeBot":["partial","*",1,1],"Claude-User":["partial","*",1,1],"Claude-SearchBot":["partial","*",1,1],"anthropic-ai":["partial","*",1,1],"PerplexityBot":["partial","*",1,1],"Perplexity-User":["partial","*",1,1],"Google-Extended":["partial","*",1,1],"Applebot-Extended":["partial","*",1,1],"CCBot":["partial","*",1,1],"Bytespider":["partial","*",1,1],"Amazonbot":["partial","*",1,1],"meta-externalagent":["partial","*",1,1],"cohere-ai":["partial","*",1,1],"Diffbot":["partial","*",1,1],"ContextFetchBot":["partial","*",1,1]},"llms":{"status":"missing","http":404,"bytes":131278},"home":{"status":"http_error","http":403,"url":"https://omnihotels.com/"}},
{"rank":6921,"domain":"arabianbusiness.com","origin":"https://arabianbusiness.com","robots":{"status":"refused","http":403,"url":"https://arabianbusiness.com/robots.txt","bytes":919,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":6922,"domain":"flir.com","origin":"https://flir.com","robots":{"status":"ok","http":200,"url":"https://www.flir.com/robots.txt","bytes":1052,"groups":3},"bots":{"GPTBot":["partial","*",2,0],"ChatGPT-User":["partial","*",2,0],"OAI-SearchBot":["partial","*",2,0],"ClaudeBot":["partial","*",2,0],"Claude-User":["partial","*",2,0],"Claude-SearchBot":["partial","*",2,0],"anthropic-ai":["partial","*",2,0],"PerplexityBot":["partial","*",2,0],"Perplexity-User":["partial","*",2,0],"Google-Extended":["partial","*",2,0],"Applebot-Extended":["partial","*",2,0],"CCBot":["partial","*",2,0],"Bytespider":["partial","*",2,0],"Amazonbot":["partial","*",2,0],"meta-externalagent":["partial","*",2,0],"cohere-ai":["partial","*",2,0],"Diffbot":["partial","*",2,0],"ContextFetchBot":["partial","*",2,0]},"llms":{"status":"missing","http":404,"bytes":432452},"home":{"status":"ok","http":200,"url":"https://www.flir.com/","html_tokens":110052,"markdown_tokens":4,"pruned_tokens":4}},
{"rank":6923,"domain":"yadvashem.org","origin":"https://yadvashem.org","robots":{"status":"refused","http":403,"url":"https://www.yadvashem.org/robots.txt","bytes":118,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":6924,"domain":"fnal.gov","origin":"https://fnal.gov","robots":{"status":"refused","http":403,"url":"https://www.fnal.gov/robots.txt","bytes":5485,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":6925,"domain":"yicai.com","origin":"https://www.yicai.com","robots":{"status":"ok","http":200,"url":"https://www.yicai.com/robots.txt","bytes":49,"groups":1},"bots":{"GPTBot":["partial","*",2,0],"ChatGPT-User":["partial","*",2,0],"OAI-SearchBot":["partial","*",2,0],"ClaudeBot":["partial","*",2,0],"Claude-User":["partial","*",2,0],"Claude-SearchBot":["partial","*",2,0],"anthropic-ai":["partial","*",2,0],"PerplexityBot":["partial","*",2,0],"Perplexity-User":["partial","*",2,0],"Google-Extended":["partial","*",2,0],"Applebot-Extended":["partial","*",2,0],"CCBot":["partial","*",2,0],"Bytespider":["partial","*",2,0],"Amazonbot":["partial","*",2,0],"meta-externalagent":["partial","*",2,0],"cohere-ai":["partial","*",2,0],"Diffbot":["partial","*",2,0],"ContextFetchBot":["partial","*",2,0]},"llms":{"status":"missing","http":404,"bytes":1829},"home":{"status":"ok","http":200,"url":"https://www.yicai.com/","html_tokens":268370,"markdown_tokens":43,"pruned_tokens":43}},
{"rank":6926,"domain":"google.bi","origin":"https://google.bi","robots":{"status":"ok","http":200,"url":"https://www.google.bi/robots.txt","bytes":6624,"groups":4},"bots":{"GPTBot":["partial","*",169,64],"ChatGPT-User":["partial","*",169,64],"OAI-SearchBot":["partial","*",169,64],"ClaudeBot":["partial","*",169,64],"Claude-User":["partial","*",169,64],"Claude-SearchBot":["partial","*",169,64],"anthropic-ai":["partial","*",169,64],"PerplexityBot":["partial","*",169,64],"Perplexity-User":["partial","*",169,64],"Google-Extended":["partial","*",169,64],"Applebot-Extended":["partial","*",169,64],"CCBot":["partial","*",169,64],"Bytespider":["partial","*",169,64],"Amazonbot":["partial","*",169,64],"meta-externalagent":["partial","*",169,64],"cohere-ai":["partial","*",169,64],"Diffbot":["partial","*",169,64],"ContextFetchBot":["partial","*",169,64]},"llms":{"status":"missing","http":404,"bytes":1569},"home":{"status":"ok","http":200,"url":"https://www.google.com/","html_tokens":21292,"markdown_tokens":30,"pruned_tokens":30}},
{"rank":6927,"domain":"bellingcat.com","origin":"https://bellingcat.com","robots":{"status":"ok","http":200,"url":"https://www.bellingcat.com/robots.txt","bytes":176,"groups":1},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"missing","http":404,"bytes":70448},"home":{"status":"ok","http":200,"url":"https://www.bellingcat.com/","html_tokens":32165,"markdown_tokens":353,"pruned_tokens":252}},
{"rank":6928,"domain":"cna.com.tw","origin":"https://cna.com.tw","robots":{"status":"ok","http":200,"url":"https://www.cna.com.tw/robots.txt","bytes":2230,"groups":3},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["partial","*",16,0],"OAI-SearchBot":["partial","*",16,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",16,0],"Claude-SearchBot":["partial","*",16,0],"anthropic-ai":["partial","*",16,0],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["partial","*",16,0],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["partial","*",16,0],"Diffbot":["partial","*",16,0],"ContextFetchBot":["partial","*",16,0]},"llms":{"status":"other_text","http":200,"bytes":8080},"home":{"status":"ok","http":200,"url":"https://www.cna.com.tw/","html_tokens":40477,"markdown_tokens":1930,"pruned_tokens":174}},
{"rank":6929,"domain":"allthatsinteresting.com","origin":"https://allthatsinteresting.com","robots":{"status":"ok","http":200,"url":"https://allthatsinteresting.com/robots.txt","bytes":511,"groups":1},"bots":{"GPTBot":["partial","*",8,2],"ChatGPT-User":["partial","*",8,2],"OAI-SearchBot":["partial","*",8,2],"ClaudeBot":["partial","*",8,2],"Claude-User":["partial","*",8,2],"Claude-SearchBot":["partial","*",8,2],"anthropic-ai":["partial","*",8,2],"PerplexityBot":["partial","*",8,2],"Perplexity-User":["partial","*",8,2],"Google-Extended":["partial","*",8,2],"Applebot-Extended":["partial","*",8,2],"CCBot":["partial","*",8,2],"Bytespider":["partial","*",8,2],"Amazonbot":["partial","*",8,2],"meta-externalagent":["partial","*",8,2],"cohere-ai":["partial","*",8,2],"Diffbot":["partial","*",8,2],"ContextFetchBot":["partial","*",8,2]},"llms":{"status":"missing","http":404,"bytes":52078},"home":{"status":"ok","http":200,"url":"https://allthatsinteresting.com/","html_tokens":29032,"markdown_tokens":0,"pruned_tokens":0}},
{"rank":6930,"domain":"37signals.com","origin":"https://37signals.com","robots":{"status":"missing","http":404,"url":"https://37signals.com/robots.txt","bytes":8289,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":8289},"home":{"status":"ok","http":200,"url":"https://37signals.com/","html_tokens":4264,"markdown_tokens":238,"pruned_tokens":67}},
{"rank":6931,"domain":"google.com.ai","origin":"https://google.com.ai","robots":{"status":"ok","http":200,"url":"https://www.google.com.ai/robots.txt","bytes":6624,"groups":4},"bots":{"GPTBot":["partial","*",169,64],"ChatGPT-User":["partial","*",169,64],"OAI-SearchBot":["partial","*",169,64],"ClaudeBot":["partial","*",169,64],"Claude-User":["partial","*",169,64],"Claude-SearchBot":["partial","*",169,64],"anthropic-ai":["partial","*",169,64],"PerplexityBot":["partial","*",169,64],"Perplexity-User":["partial","*",169,64],"Google-Extended":["partial","*",169,64],"Applebot-Extended":["partial","*",169,64],"CCBot":["partial","*",169,64],"Bytespider":["partial","*",169,64],"Amazonbot":["partial","*",169,64],"meta-externalagent":["partial","*",169,64],"cohere-ai":["partial","*",169,64],"Diffbot":["partial","*",169,64],"ContextFetchBot":["partial","*",169,64]},"llms":{"status":"missing","http":404,"bytes":1569},"home":{"status":"ok","http":200,"url":"https://www.google.com.ai/","html_tokens":21388,"markdown_tokens":32,"pruned_tokens":32}},
{"rank":6932,"domain":"neh.gov","origin":"https://neh.gov","robots":{"status":"ok","http":200,"url":"https://www.neh.gov/robots.txt","bytes":1477,"groups":1},"bots":{"GPTBot":["partial","*",33,18],"ChatGPT-User":["partial","*",33,18],"OAI-SearchBot":["partial","*",33,18],"ClaudeBot":["partial","*",33,18],"Claude-User":["partial","*",33,18],"Claude-SearchBot":["partial","*",33,18],"anthropic-ai":["partial","*",33,18],"PerplexityBot":["partial","*",33,18],"Perplexity-User":["partial","*",33,18],"Google-Extended":["partial","*",33,18],"Applebot-Extended":["partial","*",33,18],"CCBot":["partial","*",33,18],"Bytespider":["partial","*",33,18],"Amazonbot":["partial","*",33,18],"meta-externalagent":["partial","*",33,18],"cohere-ai":["partial","*",33,18],"Diffbot":["partial","*",33,18],"ContextFetchBot":["partial","*",33,18]},"llms":{"status":"crawl_delay","error":"Crawl-delay 300 s"},"home":{"status":"crawl_delay","error":"Crawl-delay 300 s"}},
{"rank":6933,"domain":"tdk.com","origin":"https://tdk.com","robots":{"status":"refused","http":403,"url":"https://tdk.com/robots.txt","bytes":199,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":6934,"domain":"therecord.media","origin":"https://therecord.media","robots":{"status":"missing","http":404,"url":"https://therecord.media/robots.txt","bytes":3989,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":3989},"home":{"status":"ok","http":200,"url":"https://therecord.media/","html_tokens":24637,"markdown_tokens":64,"pruned_tokens":64}},
{"rank":6935,"domain":"pochta.ru","origin":"https://pochta.ru","robots":{"status":"refused","http":417,"url":"https://pochta.ru/robots.txt","bytes":127597,"groups":0,"error":"HTTP 417"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":6936,"domain":"mcall.com","origin":"https://mcall.com","robots":{"status":"ok","http":200,"url":"https://www.mcall.com/robots.txt","bytes":4466,"groups":68},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["blocked","named",1,0],"OAI-SearchBot":["blocked","named",1,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["blocked","named",1,0],"Claude-SearchBot":["blocked","named",1,0],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["blocked","named",1,0],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["partial","*",8,1],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",8,1]},"llms":{"status":"missing","http":404,"bytes":90486},"home":{"status":"ok","http":200,"url":"https://www.mcall.com/","html_tokens":65426,"markdown_tokens":294,"pruned_tokens":157}},
{"rank":6937,"domain":"mailjet.com","origin":"https://mailjet.com","robots":{"status":"ok","http":200,"url":"https://mailjet.com/robots.txt","bytes":169,"groups":1},"bots":{"GPTBot":["partial","*",4,0],"ChatGPT-User":["partial","*",4,0],"OAI-SearchBot":["partial","*",4,0],"ClaudeBot":["partial","*",4,0],"Claude-User":["partial","*",4,0],"Claude-SearchBot":["partial","*",4,0],"anthropic-ai":["partial","*",4,0],"PerplexityBot":["partial","*",4,0],"Perplexity-User":["partial","*",4,0],"Google-Extended":["partial","*",4,0],"Applebot-Extended":["partial","*",4,0],"CCBot":["partial","*",4,0],"Bytespider":["partial","*",4,0],"Amazonbot":["partial","*",4,0],"meta-externalagent":["partial","*",4,0],"cohere-ai":["partial","*",4,0],"Diffbot":["partial","*",4,0],"ContextFetchBot":["partial","*",4,0]},"llms":{"status":"crawl_delay","error":"Crawl-delay 600 s"},"home":{"status":"crawl_delay","error":"Crawl-delay 600 s"}},
{"rank":6938,"domain":"freebeacon.com","origin":"https://freebeacon.com","robots":{"status":"ok","http":200,"url":"https://freebeacon.com/robots.txt","bytes":145,"groups":1},"bots":{"GPTBot":["partial","*",5,1],"ChatGPT-User":["partial","*",5,1],"OAI-SearchBot":["partial","*",5,1],"ClaudeBot":["partial","*",5,1],"Claude-User":["partial","*",5,1],"Claude-SearchBot":["partial","*",5,1],"anthropic-ai":["partial","*",5,1],"PerplexityBot":["partial","*",5,1],"Perplexity-User":["partial","*",5,1],"Google-Extended":["partial","*",5,1],"Applebot-Extended":["partial","*",5,1],"CCBot":["partial","*",5,1],"Bytespider":["partial","*",5,1],"Amazonbot":["partial","*",5,1],"meta-externalagent":["partial","*",5,1],"cohere-ai":["partial","*",5,1],"Diffbot":["partial","*",5,1],"ContextFetchBot":["partial","*",5,1]},"llms":{"status":"missing","http":404,"bytes":163608},"home":{"status":"ok","http":200,"url":"https://freebeacon.com/","html_tokens":56613,"markdown_tokens":387,"pruned_tokens":140}},
{"rank":6939,"domain":"campaignlive.co.uk","origin":"https://campaignlive.co.uk","robots":{"status":"ok","http":200,"url":"https://campaignlive.co.uk/robots.txt","bytes":2676,"groups":32},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["allowed","*",0,1],"OAI-SearchBot":["allowed","*",0,1],"ClaudeBot":["blocked","named",1,0],"Claude-User":["allowed","*",0,1],"Claude-SearchBot":["allowed","*",0,1],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["allowed","*",0,1],"Perplexity-User":["allowed","*",0,1],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["allowed","*",0,1]},"llms":{"status":"http_error","http":403,"bytes":5571},"home":{"status":"http_error","http":403,"url":"https://campaignlive.co.uk/"}},
{"rank":6940,"domain":"nsw.edu.au","origin":"https://nsw.edu.au","robots":{"status":"unreachable","http":null,"url":"https://nsw.edu.au/robots.txt","bytes":0,"groups":0,"error":"cannot resolve nsw.edu.au"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":6941,"domain":"memphis.edu","origin":"https://memphis.edu","robots":{"status":"ok","http":200,"url":"https://www.memphis.edu/robots.txt","bytes":4235,"groups":1},"bots":{"GPTBot":["partial","*",102,6],"ChatGPT-User":["partial","*",102,6],"OAI-SearchBot":["partial","*",102,6],"ClaudeBot":["partial","*",102,6],"Claude-User":["partial","*",102,6],"Claude-SearchBot":["partial","*",102,6],"anthropic-ai":["partial","*",102,6],"PerplexityBot":["partial","*",102,6],"Perplexity-User":["partial","*",102,6],"Google-Extended":["partial","*",102,6],"Applebot-Extended":["partial","*",102,6],"CCBot":["partial","*",102,6],"Bytespider":["partial","*",102,6],"Amazonbot":["partial","*",102,6],"meta-externalagent":["partial","*",102,6],"cohere-ai":["partial","*",102,6],"Diffbot":["partial","*",102,6],"ContextFetchBot":["partial","*",102,6]},"llms":{"status":"missing","http":404,"bytes":43863},"home":{"status":"ok","http":200,"url":"https://www.memphis.edu/","html_tokens":8470,"markdown_tokens":109,"pruned_tokens":54}},
{"rank":6942,"domain":"cope.es","origin":"https://cope.es","robots":{"status":"ok","http":200,"url":"https://www.cope.es/robots.txt","bytes":5773,"groups":153},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["blocked","named",1,0],"OAI-SearchBot":["blocked","named",1,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["blocked","named",1,0],"Claude-SearchBot":["blocked","named",1,0],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["blocked","named",1,0],"Google-Extended":["allowed","named",0,2],"Applebot-Extended":["partial","*",3,2],"CCBot":["blocked","named",1,0],"Bytespider":["allowed","named",1,1],"Amazonbot":["allowed","named",0,1],"meta-externalagent":["allowed","named",0,1],"cohere-ai":["partial","*",3,2],"Diffbot":["partial","*",3,2],"ContextFetchBot":["partial","*",3,2]},"llms":{"status":"missing","http":404,"bytes":99133},"home":{"status":"ok","http":200,"url":"https://www.cope.es/","html_tokens":214470,"markdown_tokens":794,"pruned_tokens":77}},
{"rank":6943,"domain":"php-editors.com","origin":"https://php-editors.com","robots":{"status":"unreachable","http":null,"url":"https://php-editors.com/robots.txt","bytes":0,"groups":0,"error":"[SSL: CERTIFICATE_VERIFY_FAILED] certificate verify failed: certificate has expired (_ssl.c:1028)"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":6944,"domain":"armani.com","origin":"https://armani.com","robots":{"status":"ok","http":200,"url":"https://www.armani.com/robots.txt","bytes":1469,"groups":8},"bots":{"GPTBot":["partial","*",32,0],"ChatGPT-User":["partial","*",32,0],"OAI-SearchBot":["partial","*",32,0],"ClaudeBot":["partial","*",32,0],"Claude-User":["partial","*",32,0],"Claude-SearchBot":["partial","*",32,0],"anthropic-ai":["partial","*",32,0],"PerplexityBot":["partial","*",32,0],"Perplexity-User":["partial","*",32,0],"Google-Extended":["partial","*",32,0],"Applebot-Extended":["partial","*",32,0],"CCBot":["partial","*",32,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["partial","*",32,0],"Diffbot":["partial","*",32,0],"ContextFetchBot":["partial","*",32,0]},"llms":{"status":"missing","http":404,"bytes":373062},"home":{"status":"ok","http":200,"url":"https://www.armani.com/en-us/","html_tokens":148824,"markdown_tokens":83,"pruned_tokens":29}},
{"rank":6945,"domain":"morguefile.com","origin":"https://morguefile.com","robots":{"status":"refused","http":403,"url":"https://morguefile.com/robots.txt","bytes":5552,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":6946,"domain":"tes.com","origin":"https://tes.com","robots":{"status":"ok","http":200,"url":"https://www.tes.com/robots.txt","bytes":1095,"groups":1},"bots":{"GPTBot":["partial","*",24,0],"ChatGPT-User":["partial","*",24,0],"OAI-SearchBot":["partial","*",24,0],"ClaudeBot":["partial","*",24,0],"Claude-User":["partial","*",24,0],"Claude-SearchBot":["partial","*",24,0],"anthropic-ai":["partial","*",24,0],"PerplexityBot":["partial","*",24,0],"Perplexity-User":["partial","*",24,0],"Google-Extended":["partial","*",24,0],"Applebot-Extended":["partial","*",24,0],"CCBot":["partial","*",24,0],"Bytespider":["partial","*",24,0],"Amazonbot":["partial","*",24,0],"meta-externalagent":["partial","*",24,0],"cohere-ai":["partial","*",24,0],"Diffbot":["partial","*",24,0],"ContextFetchBot":["partial","*",24,0]},"llms":{"status":"http_error","http":403,"bytes":146},"home":{"status":"ok","http":200,"url":"https://www.tes.com/","html_tokens":44092,"markdown_tokens":310,"pruned_tokens":114}},
{"rank":6947,"domain":"videopress.com","origin":"https://videopress.com","robots":{"status":"ok","http":200,"url":"https://videopress.com/robots.txt","bytes":628,"groups":1},"bots":{"GPTBot":["partial","*",10,1],"ChatGPT-User":["partial","*",10,1],"OAI-SearchBot":["partial","*",10,1],"ClaudeBot":["partial","*",10,1],"Claude-User":["partial","*",10,1],"Claude-SearchBot":["partial","*",10,1],"anthropic-ai":["partial","*",10,1],"PerplexityBot":["partial","*",10,1],"Perplexity-User":["partial","*",10,1],"Google-Extended":["partial","*",10,1],"Applebot-Extended":["partial","*",10,1],"CCBot":["partial","*",10,1],"Bytespider":["partial","*",10,1],"Amazonbot":["partial","*",10,1],"meta-externalagent":["partial","*",10,1],"cohere-ai":["partial","*",10,1],"Diffbot":["partial","*",10,1],"ContextFetchBot":["partial","*",10,1]},"llms":{"status":"missing","http":404,"bytes":56405},"home":{"status":"ok","http":200,"url":"https://videopress.com/","html_tokens":35942,"markdown_tokens":350,"pruned_tokens":230}},
{"rank":6948,"domain":"utas.edu.au","origin":"https://utas.edu.au","robots":{"status":"refused","http":403,"url":"https://utas.edu.au/robots.txt","bytes":5506,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":6949,"domain":"ekathimerini.com","origin":"https://ekathimerini.com","robots":{"status":"ok","http":200,"url":"https://www.ekathimerini.com/robots.txt","bytes":1503,"groups":26},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["blocked","named",1,0],"OAI-SearchBot":["partial","*",13,1],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",13,1],"Claude-SearchBot":["partial","*",13,1],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["partial","*",13,1],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["partial","*",13,1],"ContextFetchBot":["partial","*",13,1]},"llms":{"status":"missing","http":404,"bytes":146},"home":{"status":"http_error","http":403,"url":"https://ekathimerini.com/"}},
{"rank":6950,"domain":"google.com.om","origin":"https://google.com.om","robots":{"status":"ok","http":200,"url":"https://www.google.com.om/robots.txt","bytes":6624,"groups":4},"bots":{"GPTBot":["partial","*",169,64],"ChatGPT-User":["partial","*",169,64],"OAI-SearchBot":["partial","*",169,64],"ClaudeBot":["partial","*",169,64],"Claude-User":["partial","*",169,64],"Claude-SearchBot":["partial","*",169,64],"anthropic-ai":["partial","*",169,64],"PerplexityBot":["partial","*",169,64],"Perplexity-User":["partial","*",169,64],"Google-Extended":["partial","*",169,64],"Applebot-Extended":["partial","*",169,64],"CCBot":["partial","*",169,64],"Bytespider":["partial","*",169,64],"Amazonbot":["partial","*",169,64],"meta-externalagent":["partial","*",169,64],"cohere-ai":["partial","*",169,64],"Diffbot":["partial","*",169,64],"ContextFetchBot":["partial","*",169,64]},"llms":{"status":"missing","http":404,"bytes":1569},"home":{"status":"ok","http":200,"url":"https://www.google.com/","html_tokens":21286,"markdown_tokens":30,"pruned_tokens":30}},
{"rank":6951,"domain":"google.gg","origin":"https://google.gg","robots":{"status":"ok","http":200,"url":"https://www.google.gg/robots.txt","bytes":6624,"groups":4},"bots":{"GPTBot":["partial","*",169,64],"ChatGPT-User":["partial","*",169,64],"OAI-SearchBot":["partial","*",169,64],"ClaudeBot":["partial","*",169,64],"Claude-User":["partial","*",169,64],"Claude-SearchBot":["partial","*",169,64],"anthropic-ai":["partial","*",169,64],"PerplexityBot":["partial","*",169,64],"Perplexity-User":["partial","*",169,64],"Google-Extended":["partial","*",169,64],"Applebot-Extended":["partial","*",169,64],"CCBot":["partial","*",169,64],"Bytespider":["partial","*",169,64],"Amazonbot":["partial","*",169,64],"meta-externalagent":["partial","*",169,64],"cohere-ai":["partial","*",169,64],"Diffbot":["partial","*",169,64],"ContextFetchBot":["partial","*",169,64]},"llms":{"status":"missing","http":404,"bytes":1569},"home":{"status":"ok","http":200,"url":"https://www.google.com/","html_tokens":21351,"markdown_tokens":30,"pruned_tokens":30}},
{"rank":6952,"domain":"wolt.com","origin":"https://wolt.com","robots":{"status":"ok","http":200,"url":"https://wolt.com/robots.txt","bytes":62,"groups":1},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"missing","http":404,"bytes":1055118},"home":{"status":"ok","http":200,"url":"https://wolt.com/","html_tokens":336589,"markdown_tokens":112,"pruned_tokens":52}},
{"rank":6953,"domain":"omaha.com","origin":"https://omaha.com","robots":{"status":"ok","http":200,"url":"https://omaha.com/robots.txt","bytes":8138,"groups":31},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["blocked","named",1,0],"OAI-SearchBot":["blocked","named",1,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",100,0],"Claude-SearchBot":["partial","*",100,0],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["partial","*",100,0],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["partial","*",100,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",100,0]},"llms":{"status":"http_error","http":429,"bytes":64},"home":{"status":"ok","http":200,"url":"https://omaha.com/","html_tokens":128448,"markdown_tokens":1218,"pruned_tokens":116}},
{"rank":6954,"domain":"lombardia.it","origin":"https://lombardia.it","robots":{"status":"unreachable","http":null,"url":"https://lombardia.it/robots.txt","bytes":0,"groups":0,"error":"cannot resolve lombardia.it"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":6955,"domain":"unesp.br","origin":"https://unesp.br","robots":{"status":"ok","http":200,"url":"https://www2.unesp.br/robots.txt","bytes":196,"groups":1},"bots":{"GPTBot":["partial","*",2,1],"ChatGPT-User":["partial","*",2,1],"OAI-SearchBot":["partial","*",2,1],"ClaudeBot":["partial","*",2,1],"Claude-User":["partial","*",2,1],"Claude-SearchBot":["partial","*",2,1],"anthropic-ai":["partial","*",2,1],"PerplexityBot":["partial","*",2,1],"Perplexity-User":["partial","*",2,1],"Google-Extended":["partial","*",2,1],"Applebot-Extended":["partial","*",2,1],"CCBot":["partial","*",2,1],"Bytespider":["partial","*",2,1],"Amazonbot":["partial","*",2,1],"meta-externalagent":["partial","*",2,1],"cohere-ai":["partial","*",2,1],"Diffbot":["partial","*",2,1],"ContextFetchBot":["partial","*",2,1]},"llms":{"status":"html","http":200,"bytes":40626},"home":{"status":"ok","http":200,"url":"https://www2.unesp.br/","html_tokens":45210,"markdown_tokens":45,"pruned_tokens":45}},
{"rank":6956,"domain":"red-dot.org","origin":"https://red-dot.org","robots":{"status":"ok","http":200,"url":"https://www.red-dot.org/robots.txt","bytes":256,"groups":2},"bots":{"GPTBot":["partial","*",6,1],"ChatGPT-User":["partial","*",6,1],"OAI-SearchBot":["partial","*",6,1],"ClaudeBot":["partial","*",6,1],"Claude-User":["partial","*",6,1],"Claude-SearchBot":["partial","*",6,1],"anthropic-ai":["partial","*",6,1],"PerplexityBot":["partial","*",6,1],"Perplexity-User":["partial","*",6,1],"Google-Extended":["partial","*",6,1],"Applebot-Extended":["partial","*",6,1],"CCBot":["partial","*",6,1],"Bytespider":["partial","*",6,1],"Amazonbot":["partial","*",6,1],"meta-externalagent":["partial","*",6,1],"cohere-ai":["partial","*",6,1],"Diffbot":["partial","*",6,1],"ContextFetchBot":["partial","*",6,1]},"llms":{"status":"missing","http":404,"bytes":27767},"home":{"status":"ok","http":200,"url":"https://www.red-dot.org/","html_tokens":18024,"markdown_tokens":73,"pruned_tokens":69}},
{"rank":6957,"domain":"trunovskiy26raion.ru","origin":"https://trunovskiy26raion.ru","robots":{"status":"ok","http":200,"url":"https://trunovskiy26raion.ru/robots.txt","bytes":195,"groups":1},"bots":{"GPTBot":["partial","*",7,0],"ChatGPT-User":["partial","*",7,0],"OAI-SearchBot":["partial","*",7,0],"ClaudeBot":["partial","*",7,0],"Claude-User":["partial","*",7,0],"Claude-SearchBot":["partial","*",7,0],"anthropic-ai":["partial","*",7,0],"PerplexityBot":["partial","*",7,0],"Perplexity-User":["partial","*",7,0],"Google-Extended":["partial","*",7,0],"Applebot-Extended":["partial","*",7,0],"CCBot":["partial","*",7,0],"Bytespider":["partial","*",7,0],"Amazonbot":["partial","*",7,0],"meta-externalagent":["partial","*",7,0],"cohere-ai":["partial","*",7,0],"Diffbot":["partial","*",7,0],"ContextFetchBot":["partial","*",7,0]},"llms":{"status":"html","http":200,"bytes":15803},"home":{"status":"ok","http":200,"url":"https://smsb66.ru/","html_tokens":3676,"markdown_tokens":234,"pruned_tokens":175}},
{"rank":6958,"domain":"crateandbarrel.com","origin":"https://crateandbarrel.com","robots":{"status":"refused","http":403,"url":"https://www.crateandbarrel.com/robots.txt","bytes":710,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":6959,"domain":"logitechg.com","origin":"https://logitechg.com","robots":{"status":"ok","http":200,"url":"https://www.logitechg.com/robots.txt","bytes":384,"groups":1},"bots":{"GPTBot":["partial","*",16,1],"ChatGPT-User":["partial","*",16,1],"OAI-SearchBot":["partial","*",16,1],"ClaudeBot":["partial","*",16,1],"Claude-User":["partial","*",16,1],"Claude-SearchBot":["partial","*",16,1],"anthropic-ai":["partial","*",16,1],"PerplexityBot":["partial","*",16,1],"Perplexity-User":["partial","*",16,1],"Google-Extended":["partial","*",16,1],"Applebot-Extended":["partial","*",16,1],"CCBot":["partial","*",16,1],"Bytespider":["partial","*",16,1],"Amazonbot":["partial","*",16,1],"meta-externalagent":["partial","*",16,1],"cohere-ai":["partial","*",16,1],"Diffbot":["partial","*",16,1],"ContextFetchBot":["partial","*",16,1]},"llms":{"status":"present","http":200,"bytes":7872},"home":{"status":"ok","http":200,"url":"https://www.logitechg.com/en-us","html_tokens":109827,"markdown_tokens":676,"pruned_tokens":102}},
{"rank":6960,"domain":"theweek.in","origin":"https://theweek.in","robots":{"status":"ok","http":200,"url":"https://www.theweek.in/robots.txt","bytes":2082,"groups":4},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["blocked","named",1,0],"OAI-SearchBot":["partial","*",67,1],"ClaudeBot":["partial","*",67,1],"Claude-User":["partial","*",67,1],"Claude-SearchBot":["partial","*",67,1],"anthropic-ai":["partial","*",67,1],"PerplexityBot":["partial","*",67,1],"Perplexity-User":["partial","*",67,1],"Google-Extended":["partial","*",67,1],"Applebot-Extended":["partial","*",67,1],"CCBot":["partial","*",67,1],"Bytespider":["partial","*",67,1],"Amazonbot":["partial","*",67,1],"meta-externalagent":["partial","*",67,1],"cohere-ai":["partial","*",67,1],"Diffbot":["partial","*",67,1],"ContextFetchBot":["partial","*",67,1]},"llms":{"status":"missing","http":404,"bytes":12341},"home":{"status":"ok","http":200,"url":"https://www.theweek.in/","html_tokens":62614,"markdown_tokens":41,"pruned_tokens":41}},
{"rank":6961,"domain":"candid.org","origin":"https://candid.org","robots":{"status":"refused","http":403,"url":"https://candid.org/robots.txt","bytes":919,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":6962,"domain":"uncg.edu","origin":"https://uncg.edu","robots":{"status":"ok","http":200,"url":"https://www.uncg.edu/robots.txt","bytes":170,"groups":1},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"other_text","http":200,"bytes":6460},"home":{"status":"ok","http":200,"url":"https://www.uncg.edu/","html_tokens":38839,"markdown_tokens":306,"pruned_tokens":151}},
{"rank":6963,"domain":"google.com.mt","origin":"https://google.com.mt","robots":{"status":"ok","http":200,"url":"https://www.google.com.mt/robots.txt","bytes":6624,"groups":4},"bots":{"GPTBot":["partial","*",169,64],"ChatGPT-User":["partial","*",169,64],"OAI-SearchBot":["partial","*",169,64],"ClaudeBot":["partial","*",169,64],"Claude-User":["partial","*",169,64],"Claude-SearchBot":["partial","*",169,64],"anthropic-ai":["partial","*",169,64],"PerplexityBot":["partial","*",169,64],"Perplexity-User":["partial","*",169,64],"Google-Extended":["partial","*",169,64],"Applebot-Extended":["partial","*",169,64],"CCBot":["partial","*",169,64],"Bytespider":["partial","*",169,64],"Amazonbot":["partial","*",169,64],"meta-externalagent":["partial","*",169,64],"cohere-ai":["partial","*",169,64],"Diffbot":["partial","*",169,64],"ContextFetchBot":["partial","*",169,64]},"llms":{"status":"missing","http":404,"bytes":1569},"home":{"status":"ok","http":200,"url":"https://www.google.com/","html_tokens":21330,"markdown_tokens":30,"pruned_tokens":30}},
{"rank":6964,"domain":"homeoffice.gov.uk","origin":"http://homeoffice.gov.uk","robots":{"status":"ok","http":200,"url":"http://homeoffice.gov.uk/robots.txt","bytes":70,"groups":1},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"missing","http":404,"bytes":1682},"home":{"status":"ok","http":200,"url":"https://www.gov.uk/government/organisations/home-office","html_tokens":35485,"markdown_tokens":81,"pruned_tokens":81}},
{"rank":6965,"domain":"google.com.gt","origin":"https://google.com.gt","robots":{"status":"ok","http":200,"url":"https://www.google.com.gt/robots.txt","bytes":6624,"groups":4},"bots":{"GPTBot":["partial","*",169,64],"ChatGPT-User":["partial","*",169,64],"OAI-SearchBot":["partial","*",169,64],"ClaudeBot":["partial","*",169,64],"Claude-User":["partial","*",169,64],"Claude-SearchBot":["partial","*",169,64],"anthropic-ai":["partial","*",169,64],"PerplexityBot":["partial","*",169,64],"Perplexity-User":["partial","*",169,64],"Google-Extended":["partial","*",169,64],"Applebot-Extended":["partial","*",169,64],"CCBot":["partial","*",169,64],"Bytespider":["partial","*",169,64],"Amazonbot":["partial","*",169,64],"meta-externalagent":["partial","*",169,64],"cohere-ai":["partial","*",169,64],"Diffbot":["partial","*",169,64],"ContextFetchBot":["partial","*",169,64]},"llms":{"status":"missing","http":404,"bytes":1569},"home":{"status":"ok","http":200,"url":"https://www.google.com/","html_tokens":21316,"markdown_tokens":30,"pruned_tokens":30}},
{"rank":6966,"domain":"bluewin.ch","origin":"https://bluewin.ch","robots":{"status":"ok","http":200,"url":"https://www.bluewin.ch/robots.txt","bytes":3042,"groups":48},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["blocked","named",1,0],"OAI-SearchBot":["blocked","named",1,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",2,1],"Claude-SearchBot":["partial","*",2,1],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["blocked","named",1,0],"Google-Extended":["partial","*",2,1],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",2,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",2,1]},"llms":{"status":"html","http":200,"bytes":911810},"home":{"status":"ok","http":200,"url":"https://www.bluewin.ch/de","html_tokens":506126,"markdown_tokens":184,"pruned_tokens":76}},
{"rank":6967,"domain":"ccmixter.org","origin":"https://ccmixter.org","robots":{"status":"ok","http":200,"url":"https://ccmixter.org/robots.txt","bytes":6209,"groups":2},"bots":{"GPTBot":["partial","*",219,0],"ChatGPT-User":["partial","*",219,0],"OAI-SearchBot":["partial","*",219,0],"ClaudeBot":["partial","*",219,0],"Claude-User":["partial","*",219,0],"Claude-SearchBot":["partial","*",219,0],"anthropic-ai":["partial","*",219,0],"PerplexityBot":["partial","*",219,0],"Perplexity-User":["partial","*",219,0],"Google-Extended":["partial","*",219,0],"Applebot-Extended":["partial","*",219,0],"CCBot":["partial","*",219,0],"Bytespider":["partial","*",219,0],"Amazonbot":["partial","*",219,0],"meta-externalagent":["partial","*",219,0],"cohere-ai":["partial","*",219,0],"Diffbot":["partial","*",219,0],"ContextFetchBot":["partial","*",219,0]},"llms":{"status":"missing","http":404,"bytes":28102},"home":{"status":"ok","http":200,"url":"https://ccmixter.org/","html_tokens":9369,"markdown_tokens":443,"pruned_tokens":245}},
{"rank":6968,"domain":"warszawa.pl","origin":"https://www.warszawa.pl","robots":{"status":"ok","http":200,"url":"https://www.warszawa.pl/robots.txt","bytes":174,"groups":5},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"html","http":200,"bytes":101460},"home":{"status":"ok","http":200,"url":"https://www.warszawa.pl/","html_tokens":25266,"markdown_tokens":599,"pruned_tokens":136}},
{"rank":6969,"domain":"regeringen.se","origin":"https://regeringen.se","robots":{"status":"ok","http":200,"url":"https://www.regeringen.se/robots.txt","bytes":133,"groups":1},"bots":{"GPTBot":["partial","*",3,0],"ChatGPT-User":["partial","*",3,0],"OAI-SearchBot":["partial","*",3,0],"ClaudeBot":["partial","*",3,0],"Claude-User":["partial","*",3,0],"Claude-SearchBot":["partial","*",3,0],"anthropic-ai":["partial","*",3,0],"PerplexityBot":["partial","*",3,0],"Perplexity-User":["partial","*",3,0],"Google-Extended":["partial","*",3,0],"Applebot-Extended":["partial","*",3,0],"CCBot":["partial","*",3,0],"Bytespider":["partial","*",3,0],"Amazonbot":["partial","*",3,0],"meta-externalagent":["partial","*",3,0],"cohere-ai":["partial","*",3,0],"Diffbot":["partial","*",3,0],"ContextFetchBot":["partial","*",3,0]},"llms":{"status":"missing","http":404,"bytes":190345},"home":{"status":"ok","http":200,"url":"https://www.regeringen.se/","html_tokens":68519,"markdown_tokens":492,"pruned_tokens":156}},
{"rank":6970,"domain":"oakley.com","origin":"https://oakley.com","robots":{"status":"refused","http":403,"url":"https://www.oakley.com/robots.txt","bytes":380,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":6971,"domain":"futureoflife.org","origin":"https://futureoflife.org","robots":{"status":"ok","http":200,"url":"https://futureoflife.org/robots.txt","bytes":162,"groups":2},"bots":{"GPTBot":["partial","*",3,0],"ChatGPT-User":["partial","*",3,0],"OAI-SearchBot":["partial","*",3,0],"ClaudeBot":["partial","*",3,0],"Claude-User":["partial","*",3,0],"Claude-SearchBot":["partial","*",3,0],"anthropic-ai":["partial","*",3,0],"PerplexityBot":["partial","*",3,0],"Perplexity-User":["partial","*",3,0],"Google-Extended":["partial","*",3,0],"Applebot-Extended":["partial","*",3,0],"CCBot":["partial","*",3,0],"Bytespider":["partial","*",3,0],"Amazonbot":["partial","*",3,0],"meta-externalagent":["partial","*",3,0],"cohere-ai":["partial","*",3,0],"Diffbot":["partial","*",3,0],"ContextFetchBot":["partial","*",3,0]},"llms":{"status":"missing","http":404,"bytes":200344},"home":{"status":"ok","http":200,"url":"https://futureoflife.org/","html_tokens":87000,"markdown_tokens":210,"pruned_tokens":146}},
{"rank":6972,"domain":"toshiba.co.jp","origin":"https://www.toshiba.co.jp","robots":{"status":"ok","http":200,"url":"https://www.toshiba.co.jp/robots.txt","bytes":433,"groups":1},"bots":{"GPTBot":["partial","*",9,0],"ChatGPT-User":["partial","*",9,0],"OAI-SearchBot":["partial","*",9,0],"ClaudeBot":["partial","*",9,0],"Claude-User":["partial","*",9,0],"Claude-SearchBot":["partial","*",9,0],"anthropic-ai":["partial","*",9,0],"PerplexityBot":["partial","*",9,0],"Perplexity-User":["partial","*",9,0],"Google-Extended":["partial","*",9,0],"Applebot-Extended":["partial","*",9,0],"CCBot":["partial","*",9,0],"Bytespider":["partial","*",9,0],"Amazonbot":["partial","*",9,0],"meta-externalagent":["partial","*",9,0],"cohere-ai":["partial","*",9,0],"Diffbot":["partial","*",9,0],"ContextFetchBot":["partial","*",9,0]},"llms":{"status":"missing","http":404,"bytes":4507},"home":{"status":"ok","http":200,"url":"https://www.global.toshiba/jp/top.html","html_tokens":31894,"markdown_tokens":23,"pruned_tokens":23}},
{"rank":6973,"domain":"velog.io","origin":"https://velog.io","robots":{"status":"server_error","http":502,"url":"https://velog.io/robots.txt","bytes":122,"groups":0,"error":"HTTP 502"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":6974,"domain":"texasattorneygeneral.gov","origin":"https://texasattorneygeneral.gov","robots":{"status":"ok","http":200,"url":"https://www.texasattorneygeneral.gov/robots.txt","bytes":5897,"groups":23},"bots":{"GPTBot":["partial","*",62,18],"ChatGPT-User":["blocked","named",1,0],"OAI-SearchBot":["partial","*",62,18],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",62,18],"Claude-SearchBot":["partial","*",62,18],"anthropic-ai":["partial","*",62,18],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["partial","*",62,18],"Google-Extended":["partial","*",62,18],"Applebot-Extended":["partial","*",62,18],"CCBot":["partial","*",62,18],"Bytespider":["blocked","named",2,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["partial","*",62,18],"cohere-ai":["partial","*",62,18],"Diffbot":["partial","*",62,18],"ContextFetchBot":["partial","*",62,18]},"llms":{"status":"missing","http":404,"bytes":563},"home":{"status":"ok","http":200,"url":"https://www.texasattorneygeneral.gov/","html_tokens":33939,"markdown_tokens":110,"pruned_tokens":110}},
{"rank":6975,"domain":"guttmacher.org","origin":"https://guttmacher.org","robots":{"status":"ok","http":200,"url":"https://guttmacher.org/robots.txt","bytes":2027,"groups":1},"bots":{"GPTBot":["partial","*",34,18],"ChatGPT-User":["partial","*",34,18],"OAI-SearchBot":["partial","*",34,18],"ClaudeBot":["partial","*",34,18],"Claude-User":["partial","*",34,18],"Claude-SearchBot":["partial","*",34,18],"anthropic-ai":["partial","*",34,18],"PerplexityBot":["partial","*",34,18],"Perplexity-User":["partial","*",34,18],"Google-Extended":["partial","*",34,18],"Applebot-Extended":["partial","*",34,18],"CCBot":["partial","*",34,18],"Bytespider":["partial","*",34,18],"Amazonbot":["partial","*",34,18],"meta-externalagent":["partial","*",34,18],"cohere-ai":["partial","*",34,18],"Diffbot":["partial","*",34,18],"ContextFetchBot":["partial","*",34,18]},"llms":{"status":"missing","http":404,"bytes":192},"home":{"status":"ok","http":200,"url":"https://www.guttmacher.org/","html_tokens":17651,"markdown_tokens":201,"pruned_tokens":66}},
{"rank":6976,"domain":"rim.or.jp","origin":"https://www.rim.or.jp","robots":{"status":"missing","http":404,"url":"https://www.rim.or.jp/robots.txt","bytes":12357,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":12357},"home":{"status":"ok","http":200,"url":"https://www.rim.or.jp/","html_tokens":5092,"markdown_tokens":36,"pruned_tokens":36}},
{"rank":6977,"domain":"google.am","origin":"https://google.am","robots":{"status":"ok","http":200,"url":"https://www.google.am/robots.txt","bytes":6624,"groups":4},"bots":{"GPTBot":["partial","*",169,64],"ChatGPT-User":["partial","*",169,64],"OAI-SearchBot":["partial","*",169,64],"ClaudeBot":["partial","*",169,64],"Claude-User":["partial","*",169,64],"Claude-SearchBot":["partial","*",169,64],"anthropic-ai":["partial","*",169,64],"PerplexityBot":["partial","*",169,64],"Perplexity-User":["partial","*",169,64],"Google-Extended":["partial","*",169,64],"Applebot-Extended":["partial","*",169,64],"CCBot":["partial","*",169,64],"Bytespider":["partial","*",169,64],"Amazonbot":["partial","*",169,64],"meta-externalagent":["partial","*",169,64],"cohere-ai":["partial","*",169,64],"Diffbot":["partial","*",169,64],"ContextFetchBot":["partial","*",169,64]},"llms":{"status":"missing","http":404,"bytes":1569},"home":{"status":"ok","http":200,"url":"https://www.google.com/","html_tokens":21468,"markdown_tokens":30,"pruned_tokens":30}},
{"rank":6978,"domain":"google.lu","origin":"https://google.lu","robots":{"status":"ok","http":200,"url":"https://www.google.lu/robots.txt","bytes":6624,"groups":4},"bots":{"GPTBot":["partial","*",169,64],"ChatGPT-User":["partial","*",169,64],"OAI-SearchBot":["partial","*",169,64],"ClaudeBot":["partial","*",169,64],"Claude-User":["partial","*",169,64],"Claude-SearchBot":["partial","*",169,64],"anthropic-ai":["partial","*",169,64],"PerplexityBot":["partial","*",169,64],"Perplexity-User":["partial","*",169,64],"Google-Extended":["partial","*",169,64],"Applebot-Extended":["partial","*",169,64],"CCBot":["partial","*",169,64],"Bytespider":["partial","*",169,64],"Amazonbot":["partial","*",169,64],"meta-externalagent":["partial","*",169,64],"cohere-ai":["partial","*",169,64],"Diffbot":["partial","*",169,64],"ContextFetchBot":["partial","*",169,64]},"llms":{"status":"missing","http":404,"bytes":1569},"home":{"status":"ok","http":200,"url":"https://www.google.com/","html_tokens":21458,"markdown_tokens":30,"pruned_tokens":30}},
{"rank":6979,"domain":"domainmarkt.de","origin":"https://domainmarkt.de","robots":{"status":"missing","http":404,"url":"https://www.domainmarkt.de/robots.txt","bytes":1534,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":1534},"home":{"status":"ok","http":200,"url":"https://www.domainmarkt.de/","html_tokens":613,"markdown_tokens":20,"pruned_tokens":6}},
{"rank":6980,"domain":"tabletmag.com","origin":"https://tabletmag.com","robots":{"status":"ok","http":200,"url":"https://www.tabletmag.com/robots.txt","bytes":469,"groups":9},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["partial","*",1,1],"OAI-SearchBot":["partial","*",1,1],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",1,1],"Claude-SearchBot":["partial","*",1,1],"anthropic-ai":["partial","*",1,1],"PerplexityBot":["partial","*",1,1],"Perplexity-User":["partial","*",1,1],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["partial","*",1,1],"Diffbot":["partial","*",1,1],"ContextFetchBot":["partial","*",1,1]},"llms":{"status":"missing","http":404,"bytes":20029},"home":{"status":"http_error","http":403,"url":"https://www.tabletmag.com/"}},
{"rank":6981,"domain":"jacobinmag.com","origin":"https://jacobinmag.com","robots":{"status":"ok","http":200,"url":"https://jacobin.com/robots.txt","bytes":1248,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":2319},"home":{"status":"ok","http":200,"url":"https://jacobin.com/","html_tokens":15954,"markdown_tokens":814,"pruned_tokens":126}},
{"rank":6982,"domain":"ksta.de","origin":"https://ksta.de","robots":{"status":"ok","http":200,"url":"https://www.ksta.de/robots.txt","bytes":2339,"groups":2},"bots":{"GPTBot":["partial","named",35,0],"ChatGPT-User":["partial","named",35,0],"OAI-SearchBot":["partial","named",35,0],"ClaudeBot":["partial","named",35,0],"Claude-User":["partial","named",35,0],"Claude-SearchBot":["partial","named",35,0],"anthropic-ai":["partial","named",35,0],"PerplexityBot":["partial","named",35,0],"Perplexity-User":["partial","named",35,0],"Google-Extended":["partial","named",35,0],"Applebot-Extended":["partial","named",35,0],"CCBot":["partial","named",35,0],"Bytespider":["partial","named",35,0],"Amazonbot":["partial","named",35,0],"meta-externalagent":["partial","named",35,0],"cohere-ai":["partial","named",35,0],"Diffbot":["partial","named",35,0],"ContextFetchBot":["partial","*",16,2]},"llms":{"status":"missing","http":404,"bytes":30415},"home":{"status":"ok","http":200,"url":"https://www.ksta.de/","html_tokens":347327,"markdown_tokens":19,"pruned_tokens":19}},
{"rank":6983,"domain":"rzeszow.pl","origin":"https://rzeszow.pl","robots":{"status":"unreachable","http":null,"url":"https://rzeszow.pl/robots.txt","bytes":0,"groups":0,"error":"cannot resolve rzeszow.pl"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":6984,"domain":"smith.edu","origin":"https://smith.edu","robots":{"status":"ok","http":200,"url":"https://www.smith.edu/robots.txt","bytes":2212,"groups":1},"bots":{"GPTBot":["partial","*",36,18],"ChatGPT-User":["partial","*",36,18],"OAI-SearchBot":["partial","*",36,18],"ClaudeBot":["partial","*",36,18],"Claude-User":["partial","*",36,18],"Claude-SearchBot":["partial","*",36,18],"anthropic-ai":["partial","*",36,18],"PerplexityBot":["partial","*",36,18],"Perplexity-User":["partial","*",36,18],"Google-Extended":["partial","*",36,18],"Applebot-Extended":["partial","*",36,18],"CCBot":["partial","*",36,18],"Bytespider":["partial","*",36,18],"Amazonbot":["partial","*",36,18],"meta-externalagent":["partial","*",36,18],"cohere-ai":["partial","*",36,18],"Diffbot":["partial","*",36,18],"ContextFetchBot":["partial","*",36,18]},"llms":{"status":"missing","http":404,"bytes":187},"home":{"status":"ok","http":200,"url":"https://www.smith.edu/","html_tokens":22556,"markdown_tokens":129,"pruned_tokens":120}},
{"rank":6985,"domain":"53kf.com","origin":"https://53kf.com","robots":{"status":"missing","http":404,"url":"https://53kf.com/robots.txt","bytes":150,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":150},"home":{"status":"ok","http":200,"url":"https://www.53kf.com","html_tokens":22297,"markdown_tokens":344,"pruned_tokens":42}},
{"rank":6986,"domain":"covers.com","origin":"https://covers.com","robots":{"status":"ok","http":200,"url":"https://www.covers.com/robots.txt","bytes":1448,"groups":2},"bots":{"GPTBot":["partial","*",42,0],"ChatGPT-User":["partial","*",42,0],"OAI-SearchBot":["partial","*",42,0],"ClaudeBot":["partial","*",42,0],"Claude-User":["partial","*",42,0],"Claude-SearchBot":["partial","*",42,0],"anthropic-ai":["partial","*",42,0],"PerplexityBot":["partial","*",42,0],"Perplexity-User":["partial","*",42,0],"Google-Extended":["partial","*",42,0],"Applebot-Extended":["partial","*",42,0],"CCBot":["partial","*",42,0],"Bytespider":["partial","*",42,0],"Amazonbot":["partial","*",42,0],"meta-externalagent":["partial","*",42,0],"cohere-ai":["partial","*",42,0],"Diffbot":["partial","*",42,0],"ContextFetchBot":["partial","*",42,0]},"llms":{"status":"missing","http":404,"bytes":285254},"home":{"status":"ok","http":200,"url":"https://www.covers.com/","html_tokens":273790,"markdown_tokens":733,"pruned_tokens":91}},
{"rank":6987,"domain":"aeroport.fr","origin":"https://www.aeroport.fr","robots":{"status":"ok","http":200,"url":"https://www.aeroport.fr/robots.txt","bytes":2141,"groups":1},"bots":{"GPTBot":["partial","*",35,22],"ChatGPT-User":["partial","*",35,22],"OAI-SearchBot":["partial","*",35,22],"ClaudeBot":["partial","*",35,22],"Claude-User":["partial","*",35,22],"Claude-SearchBot":["partial","*",35,22],"anthropic-ai":["partial","*",35,22],"PerplexityBot":["partial","*",35,22],"Perplexity-User":["partial","*",35,22],"Google-Extended":["partial","*",35,22],"Applebot-Extended":["partial","*",35,22],"CCBot":["partial","*",35,22],"Bytespider":["partial","*",35,22],"Amazonbot":["partial","*",35,22],"meta-externalagent":["partial","*",35,22],"cohere-ai":["partial","*",35,22],"Diffbot":["partial","*",35,22],"ContextFetchBot":["partial","*",35,22]},"llms":{"status":"missing","http":404,"bytes":189},"home":{"status":"ok","http":200,"url":"https://www.aeroport.fr/","html_tokens":18337,"markdown_tokens":24,"pruned_tokens":24}},
{"rank":6988,"domain":"hupu.com","origin":"https://hupu.com","robots":{"status":"ok","http":200,"url":"https://www.hupu.com/robots.txt","bytes":814,"groups":17},"bots":{"GPTBot":["blocked","*",1,0],"ChatGPT-User":["blocked","*",1,0],"OAI-SearchBot":["blocked","*",1,0],"ClaudeBot":["blocked","*",1,0],"Claude-User":["blocked","*",1,0],"Claude-SearchBot":["blocked","*",1,0],"anthropic-ai":["blocked","*",1,0],"PerplexityBot":["blocked","*",1,0],"Perplexity-User":["blocked","*",1,0],"Google-Extended":["blocked","*",1,0],"Applebot-Extended":["blocked","*",1,0],"CCBot":["blocked","*",1,0],"Bytespider":["allowed","named",0,1],"Amazonbot":["blocked","*",1,0],"meta-externalagent":["blocked","*",1,0],"cohere-ai":["blocked","*",1,0],"Diffbot":["blocked","*",1,0],"ContextFetchBot":["blocked","*",1,0]},"llms":{"status":"disallowed"},"home":{"status":"disallowed"}},
{"rank":6989,"domain":"digitalcommerce360.com","origin":"https://digitalcommerce360.com","robots":{"status":"ok","http":200,"url":"https://www.digitalcommerce360.com/robots.txt","bytes":387,"groups":3},"bots":{"GPTBot":["partial","*",4,2],"ChatGPT-User":["partial","*",4,2],"OAI-SearchBot":["partial","*",4,2],"ClaudeBot":["partial","*",4,2],"Claude-User":["partial","*",4,2],"Claude-SearchBot":["partial","*",4,2],"anthropic-ai":["partial","*",4,2],"PerplexityBot":["partial","*",4,2],"Perplexity-User":["partial","*",4,2],"Google-Extended":["partial","*",4,2],"Applebot-Extended":["partial","*",4,2],"CCBot":["partial","*",4,2],"Bytespider":["partial","*",4,2],"Amazonbot":["partial","*",4,2],"meta-externalagent":["partial","*",4,2],"cohere-ai":["partial","*",4,2],"Diffbot":["partial","*",4,2],"ContextFetchBot":["partial","*",4,2]},"llms":{"status":"http_error","http":403,"bytes":5597},"home":{"status":"http_error","http":403,"url":"https://digitalcommerce360.com/"}},
{"rank":6990,"domain":"power-stars-777.com","origin":"https://power-stars-777.com","robots":{"status":"ok","http":200,"url":"https://power-stars-777.com/robots.txt","bytes":542,"groups":1},"bots":{"GPTBot":["partial","*",19,0],"ChatGPT-User":["partial","*",19,0],"OAI-SearchBot":["partial","*",19,0],"ClaudeBot":["partial","*",19,0],"Claude-User":["partial","*",19,0],"Claude-SearchBot":["partial","*",19,0],"anthropic-ai":["partial","*",19,0],"PerplexityBot":["partial","*",19,0],"Perplexity-User":["partial","*",19,0],"Google-Extended":["partial","*",19,0],"Applebot-Extended":["partial","*",19,0],"CCBot":["partial","*",19,0],"Bytespider":["partial","*",19,0],"Amazonbot":["partial","*",19,0],"meta-externalagent":["partial","*",19,0],"cohere-ai":["partial","*",19,0],"Diffbot":["partial","*",19,0],"ContextFetchBot":["partial","*",19,0]},"llms":{"status":"missing","http":404,"bytes":20298},"home":{"status":"ok","http":200,"url":"https://power-stars-777.com/","html_tokens":25658,"markdown_tokens":684,"pruned_tokens":162}},
{"rank":6991,"domain":"readwriteweb.com","origin":"https://readwriteweb.com","robots":{"status":"html","http":200,"url":"https://readwrite.com/","bytes":348730,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":6992,"domain":"sdu.edu.cn","origin":"https://sdu.edu.cn","robots":{"status":"missing","http":404,"url":"https://sdu.edu.cn/robots.txt","bytes":535,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":535},"home":{"status":"ok","http":200,"url":"https://sdu.edu.cn/","html_tokens":5205,"markdown_tokens":5,"pruned_tokens":5}},
{"rank":6993,"domain":"iqvia.com","origin":"https://iqvia.com","robots":{"status":"ok","http":200,"url":"https://www.iqvia.com/robots.txt","bytes":99,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":1245},"home":{"status":"ok","http":200,"url":"https://www.iqvia.com/","html_tokens":140166,"markdown_tokens":412,"pruned_tokens":282}},
{"rank":6994,"domain":"kgmedia.id","origin":"https://kgmedia.id","robots":{"status":"ok","http":200,"url":"https://www.kgmedia.id/robots.txt","bytes":66,"groups":1},"bots":{"GPTBot":["allowed","*",0,1],"ChatGPT-User":["allowed","*",0,1],"OAI-SearchBot":["allowed","*",0,1],"ClaudeBot":["allowed","*",0,1],"Claude-User":["allowed","*",0,1],"Claude-SearchBot":["allowed","*",0,1],"anthropic-ai":["allowed","*",0,1],"PerplexityBot":["allowed","*",0,1],"Perplexity-User":["allowed","*",0,1],"Google-Extended":["allowed","*",0,1],"Applebot-Extended":["allowed","*",0,1],"CCBot":["allowed","*",0,1],"Bytespider":["allowed","*",0,1],"Amazonbot":["allowed","*",0,1],"meta-externalagent":["allowed","*",0,1],"cohere-ai":["allowed","*",0,1],"Diffbot":["allowed","*",0,1],"ContextFetchBot":["allowed","*",0,1]},"llms":{"status":"missing","http":404,"bytes":1130},"home":{"status":"ok","http":200,"url":"https://www.kgmedia.id/","html_tokens":26181,"markdown_tokens":256,"pruned_tokens":119}},
{"rank":6995,"domain":"webnode.cz","origin":"https://webnode.cz","robots":{"status":"missing","http":404,"url":"https://webnode.cz/robots.txt","bytes":0,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"present","http":200,"bytes":3499},"home":{"status":"ok","http":200,"url":"https://www.webnode.com/cs/","html_tokens":61631,"markdown_tokens":78,"pruned_tokens":78}},
{"rank":6996,"domain":"goodfirms.co","origin":"https://goodfirms.co","robots":{"status":"ok","http":200,"url":"https://www.goodfirms.co/robots.txt","bytes":435,"groups":1},"bots":{"GPTBot":["partial","*",13,0],"ChatGPT-User":["partial","*",13,0],"OAI-SearchBot":["partial","*",13,0],"ClaudeBot":["partial","*",13,0],"Claude-User":["partial","*",13,0],"Claude-SearchBot":["partial","*",13,0],"anthropic-ai":["partial","*",13,0],"PerplexityBot":["partial","*",13,0],"Perplexity-User":["partial","*",13,0],"Google-Extended":["partial","*",13,0],"Applebot-Extended":["partial","*",13,0],"CCBot":["partial","*",13,0],"Bytespider":["partial","*",13,0],"Amazonbot":["partial","*",13,0],"meta-externalagent":["partial","*",13,0],"cohere-ai":["partial","*",13,0],"Diffbot":["partial","*",13,0],"ContextFetchBot":["partial","*",13,0]},"llms":{"status":"present","http":200,"bytes":10899},"home":{"status":"http_error","http":403,"url":"https://goodfirms.co/"}},
{"rank":6997,"domain":"ankiweb.net","origin":"https://ankiweb.net","robots":{"status":"ok","http":200,"url":"https://ankiweb.net/robots.txt","bytes":97,"groups":1},"bots":{"GPTBot":["partial","*",3,0],"ChatGPT-User":["partial","*",3,0],"OAI-SearchBot":["partial","*",3,0],"ClaudeBot":["partial","*",3,0],"Claude-User":["partial","*",3,0],"Claude-SearchBot":["partial","*",3,0],"anthropic-ai":["partial","*",3,0],"PerplexityBot":["partial","*",3,0],"Perplexity-User":["partial","*",3,0],"Google-Extended":["partial","*",3,0],"Applebot-Extended":["partial","*",3,0],"CCBot":["partial","*",3,0],"Bytespider":["partial","*",3,0],"Amazonbot":["partial","*",3,0],"meta-externalagent":["partial","*",3,0],"cohere-ai":["partial","*",3,0],"Diffbot":["partial","*",3,0],"ContextFetchBot":["partial","*",3,0]},"llms":{"status":"missing","http":404,"bytes":0},"home":{"status":"ok","http":200,"url":"https://ankiweb.net/","html_tokens":349,"markdown_tokens":6,"pruned_tokens":6}},
{"rank":6998,"domain":"suntory.co.jp","origin":"https://suntory.co.jp","robots":{"status":"refused","http":403,"url":"https://suntory.co.jp/robots.txt","bytes":379,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":6999,"domain":"mec.gov.br","origin":"https://mec.gov.br","robots":{"status":"refused","http":429,"url":"https://www.gov.br/mec/pt-br","bytes":2746,"groups":0,"error":"HTTP 429"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":7000,"domain":"aena.es","origin":"https://www.aena.es","robots":{"status":"ok","http":200,"url":"https://www.aena.es/robots.txt","bytes":1989,"groups":3},"bots":{"GPTBot":["partial","*",27,0],"ChatGPT-User":["partial","*",27,0],"OAI-SearchBot":["partial","*",27,0],"ClaudeBot":["partial","*",27,0],"Claude-User":["partial","*",27,0],"Claude-SearchBot":["partial","*",27,0],"anthropic-ai":["partial","*",27,0],"PerplexityBot":["partial","*",27,0],"Perplexity-User":["partial","*",27,0],"Google-Extended":["partial","*",27,0],"Applebot-Extended":["partial","*",27,0],"CCBot":["partial","*",27,0],"Bytespider":["partial","*",27,0],"Amazonbot":["partial","*",27,0],"meta-externalagent":["partial","*",27,0],"cohere-ai":["partial","*",27,0],"Diffbot":["partial","*",27,0],"ContextFetchBot":["partial","*",27,0]},"llms":{"status":"html","http":200,"bytes":15682},"home":{"status":"ok","http":200,"url":"https://www.aena.es/es/pasajeros/pasajeros.html","html_tokens":76659,"markdown_tokens":278,"pruned_tokens":74}},
{"rank":7001,"domain":"ufba.br","origin":"https://ufba.br","robots":{"status":"ok","http":200,"url":"https://ufba.br/robots.txt","bytes":23,"groups":1},"bots":{"GPTBot":["allowed","*",0,1],"ChatGPT-User":["allowed","*",0,1],"OAI-SearchBot":["allowed","*",0,1],"ClaudeBot":["allowed","*",0,1],"Claude-User":["allowed","*",0,1],"Claude-SearchBot":["allowed","*",0,1],"anthropic-ai":["allowed","*",0,1],"PerplexityBot":["allowed","*",0,1],"Perplexity-User":["allowed","*",0,1],"Google-Extended":["allowed","*",0,1],"Applebot-Extended":["allowed","*",0,1],"CCBot":["allowed","*",0,1],"Bytespider":["allowed","*",0,1],"Amazonbot":["allowed","*",0,1],"meta-externalagent":["allowed","*",0,1],"cohere-ai":["allowed","*",0,1],"Diffbot":["allowed","*",0,1],"ContextFetchBot":["allowed","*",0,1]},"llms":{"status":"missing","http":404,"bytes":188},"home":{"status":"ok","http":200,"url":"https://ufba.br/","html_tokens":14110,"markdown_tokens":8,"pruned_tokens":8}},
{"rank":7002,"domain":"ipl.org","origin":"https://ipl.org","robots":{"status":"html","http":202,"url":"https://ipl.org/robots.txt","bytes":2593,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":7003,"domain":"rockhall.com","origin":"https://rockhall.com","robots":{"status":"ok","http":200,"url":"https://rockhall.com/robots.txt","bytes":65,"groups":1},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"missing","http":404,"bytes":172528},"home":{"status":"ok","http":200,"url":"https://rockhall.com/","html_tokens":91439,"markdown_tokens":56,"pruned_tokens":56}},
{"rank":7004,"domain":"st-and.ac.uk","origin":"http://st-and.ac.uk","robots":{"status":"ok","http":200,"url":"https://www.st-andrews.ac.uk/robots.txt","bytes":602,"groups":4},"bots":{"GPTBot":["partial","*",20,0],"ChatGPT-User":["partial","*",20,0],"OAI-SearchBot":["partial","*",20,0],"ClaudeBot":["partial","*",20,0],"Claude-User":["partial","*",20,0],"Claude-SearchBot":["partial","*",20,0],"anthropic-ai":["partial","*",20,0],"PerplexityBot":["partial","*",20,0],"Perplexity-User":["partial","*",20,0],"Google-Extended":["partial","*",20,0],"Applebot-Extended":["partial","*",20,0],"CCBot":["partial","*",20,0],"Bytespider":["partial","*",20,0],"Amazonbot":["partial","*",20,0],"meta-externalagent":["partial","*",20,0],"cohere-ai":["partial","*",20,0],"Diffbot":["partial","*",20,0],"ContextFetchBot":["partial","*",20,0]},"llms":{"status":"html","http":200,"bytes":685374},"home":{"status":"ok","http":200,"url":"https://www.st-andrews.ac.uk/","html_tokens":7514,"markdown_tokens":286,"pruned_tokens":99}},
{"rank":7005,"domain":"it-recht-kanzlei.de","origin":"https://it-recht-kanzlei.de","robots":{"status":"ok","http":200,"url":"https://www.it-recht-kanzlei.de/robots.txt","bytes":217,"groups":1},"bots":{"GPTBot":["partial","*",4,0],"ChatGPT-User":["partial","*",4,0],"OAI-SearchBot":["partial","*",4,0],"ClaudeBot":["partial","*",4,0],"Claude-User":["partial","*",4,0],"Claude-SearchBot":["partial","*",4,0],"anthropic-ai":["partial","*",4,0],"PerplexityBot":["partial","*",4,0],"Perplexity-User":["partial","*",4,0],"Google-Extended":["partial","*",4,0],"Applebot-Extended":["partial","*",4,0],"CCBot":["partial","*",4,0],"Bytespider":["partial","*",4,0],"Amazonbot":["partial","*",4,0],"meta-externalagent":["partial","*",4,0],"cohere-ai":["partial","*",4,0],"Diffbot":["partial","*",4,0],"ContextFetchBot":["partial","*",4,0]},"llms":{"status":"missing","http":404,"bytes":42060},"home":{"status":"ok","http":200,"url":"https://www.it-recht-kanzlei.de/","html_tokens":99197,"markdown_tokens":354,"pruned_tokens":102}},
{"rank":7006,"domain":"sub.jp","origin":"http://sub.jp","robots":{"status":"missing","http":404,"url":"http://err.lolipop.jp/404.html","bytes":19241,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":19241},"home":{"status":"http_error","http":404,"url":"http://err.lolipop.jp/404.html"}},
{"rank":7007,"domain":"carrier.com","origin":"https://carrier.com","robots":{"status":"ok","http":200,"url":"https://www.carrier.com/robots.txt","bytes":503,"groups":1},"bots":{"GPTBot":["partial","*",9,0],"ChatGPT-User":["partial","*",9,0],"OAI-SearchBot":["partial","*",9,0],"ClaudeBot":["partial","*",9,0],"Claude-User":["partial","*",9,0],"Claude-SearchBot":["partial","*",9,0],"anthropic-ai":["partial","*",9,0],"PerplexityBot":["partial","*",9,0],"Perplexity-User":["partial","*",9,0],"Google-Extended":["partial","*",9,0],"Applebot-Extended":["partial","*",9,0],"CCBot":["partial","*",9,0],"Bytespider":["partial","*",9,0],"Amazonbot":["partial","*",9,0],"meta-externalagent":["partial","*",9,0],"cohere-ai":["partial","*",9,0],"Diffbot":["partial","*",9,0],"ContextFetchBot":["partial","*",9,0]},"llms":{"status":"html","http":200,"bytes":45230},"home":{"status":"ok","http":200,"url":"https://www.carrier.com/us/en/","html_tokens":35343,"markdown_tokens":129,"pruned_tokens":129}},
{"rank":7008,"domain":"fluke.com","origin":"https://fluke.com","robots":{"status":"ok","http":200,"url":"https://www.fluke.com/robots.txt","bytes":3756,"groups":5},"bots":{"GPTBot":["partial","*",92,18],"ChatGPT-User":["partial","*",92,18],"OAI-SearchBot":["partial","*",92,18],"ClaudeBot":["partial","*",92,18],"Claude-User":["partial","*",92,18],"Claude-SearchBot":["partial","*",92,18],"anthropic-ai":["partial","*",92,18],"PerplexityBot":["partial","*",92,18],"Perplexity-User":["partial","*",92,18],"Google-Extended":["partial","*",92,18],"Applebot-Extended":["partial","*",92,18],"CCBot":["partial","*",92,18],"Bytespider":["partial","*",92,18],"Amazonbot":["partial","*",92,18],"meta-externalagent":["partial","*",92,18],"cohere-ai":["partial","*",92,18],"Diffbot":["partial","*",92,18],"ContextFetchBot":["partial","*",92,18]},"llms":{"status":"missing","http":404,"bytes":548},"home":{"status":"ok","http":200,"url":"https://www.fluke.com/","html_tokens":39295,"markdown_tokens":314,"pruned_tokens":189}},
{"rank":7009,"domain":"ugm.ac.id","origin":"https://ugm.ac.id","robots":{"status":"ok","http":200,"url":"https://ugm.ac.id/robots.txt","bytes":241,"groups":1},"bots":{"GPTBot":["partial","*",7,2],"ChatGPT-User":["partial","*",7,2],"OAI-SearchBot":["partial","*",7,2],"ClaudeBot":["partial","*",7,2],"Claude-User":["partial","*",7,2],"Claude-SearchBot":["partial","*",7,2],"anthropic-ai":["partial","*",7,2],"PerplexityBot":["partial","*",7,2],"Perplexity-User":["partial","*",7,2],"Google-Extended":["partial","*",7,2],"Applebot-Extended":["partial","*",7,2],"CCBot":["partial","*",7,2],"Bytespider":["partial","*",7,2],"Amazonbot":["partial","*",7,2],"meta-externalagent":["partial","*",7,2],"cohere-ai":["partial","*",7,2],"Diffbot":["partial","*",7,2],"ContextFetchBot":["partial","*",7,2]},"llms":{"status":"missing","http":404,"bytes":139151},"home":{"status":"ok","http":200,"url":"https://ugm.ac.id/","html_tokens":70393,"markdown_tokens":73,"pruned_tokens":36}},
{"rank":7010,"domain":"mint.com","origin":"https://mint.com","robots":{"status":"ok","http":200,"url":"https://mint.intuit.com/robots.txt","bytes":1696,"groups":1},"bots":{"GPTBot":["partial","*",36,0],"ChatGPT-User":["partial","*",36,0],"OAI-SearchBot":["partial","*",36,0],"ClaudeBot":["partial","*",36,0],"Claude-User":["partial","*",36,0],"Claude-SearchBot":["partial","*",36,0],"anthropic-ai":["partial","*",36,0],"PerplexityBot":["partial","*",36,0],"Perplexity-User":["partial","*",36,0],"Google-Extended":["partial","*",36,0],"Applebot-Extended":["partial","*",36,0],"CCBot":["partial","*",36,0],"Bytespider":["partial","*",36,0],"Amazonbot":["partial","*",36,0],"meta-externalagent":["partial","*",36,0],"cohere-ai":["partial","*",36,0],"Diffbot":["partial","*",36,0],"ContextFetchBot":["partial","*",36,0]},"llms":{"status":"disallowed","error":"robots.txt of www.creditkarma.com disallows /error/notfound"},"home":{"status":"ok","http":200,"url":"https://mint.intuit.com/","html_tokens":11028,"markdown_tokens":23,"pruned_tokens":23}},
{"rank":7011,"domain":"openwall.com","origin":"https://openwall.com","robots":{"status":"ok","http":200,"url":"https://openwall.com/robots.txt","bytes":340,"groups":10},"bots":{"GPTBot":["partial","*",1,0],"ChatGPT-User":["partial","*",1,0],"OAI-SearchBot":["partial","*",1,0],"ClaudeBot":["partial","*",1,0],"Claude-User":["partial","*",1,0],"Claude-SearchBot":["partial","*",1,0],"anthropic-ai":["partial","*",1,0],"PerplexityBot":["partial","*",1,0],"Perplexity-User":["partial","*",1,0],"Google-Extended":["partial","*",1,0],"Applebot-Extended":["partial","*",1,0],"CCBot":["partial","*",1,0],"Bytespider":["partial","*",1,0],"Amazonbot":["partial","*",1,0],"meta-externalagent":["partial","*",1,0],"cohere-ai":["partial","*",1,0],"Diffbot":["partial","*",1,0],"ContextFetchBot":["partial","*",1,0]},"llms":{"status":"missing","http":404,"bytes":202},"home":{"status":"ok","http":200,"url":"https://openwall.com/","html_tokens":4575,"markdown_tokens":1877,"pruned_tokens":298}},
{"rank":7012,"domain":"yalemedicine.org","origin":"https://yalemedicine.org","robots":{"status":"ok","http":200,"url":"https://www.yalemedicine.org/robots.txt","bytes":1148,"groups":4},"bots":{"GPTBot":["partial","*",9,1],"ChatGPT-User":["partial","*",9,1],"OAI-SearchBot":["partial","*",9,1],"ClaudeBot":["partial","*",9,1],"Claude-User":["partial","*",9,1],"Claude-SearchBot":["partial","*",9,1],"anthropic-ai":["partial","*",9,1],"PerplexityBot":["partial","*",9,1],"Perplexity-User":["partial","*",9,1],"Google-Extended":["partial","*",9,1],"Applebot-Extended":["partial","*",9,1],"CCBot":["partial","*",9,1],"Bytespider":["blocked","named",1,0],"Amazonbot":["partial","*",9,1],"meta-externalagent":["partial","*",9,1],"cohere-ai":["partial","*",9,1],"Diffbot":["partial","*",9,1],"ContextFetchBot":["partial","*",9,1]},"llms":{"status":"missing","http":404,"bytes":35964},"home":{"status":"ok","http":200,"url":"https://www.yalemedicine.org/","html_tokens":17650,"markdown_tokens":79,"pruned_tokens":79}},
{"rank":7013,"domain":"pcgamesn.com","origin":"https://pcgamesn.com","robots":{"status":"ok","http":200,"url":"https://www.pcgamesn.com/robots.txt","bytes":4615,"groups":4},"bots":{"GPTBot":["partial","named",1,2],"ChatGPT-User":["partial","named",1,2],"OAI-SearchBot":["partial","named",1,2],"ClaudeBot":["partial","named",1,2],"Claude-User":["partial","named",1,2],"Claude-SearchBot":["partial","named",1,2],"anthropic-ai":["partial","named",1,2],"PerplexityBot":["partial","named",1,2],"Perplexity-User":["partial","named",1,2],"Google-Extended":["partial","*",20,0],"Applebot-Extended":["partial","named",1,2],"CCBot":["partial","named",1,2],"Bytespider":["partial","named",1,2],"Amazonbot":["partial","named",1,2],"meta-externalagent":["partial","named",1,2],"cohere-ai":["partial","named",1,2],"Diffbot":["partial","named",1,2],"ContextFetchBot":["partial","*",20,0]},"llms":{"status":"missing","http":404,"bytes":170553},"home":{"status":"ok","http":200,"url":"https://www.pcgamesn.com/","html_tokens":89388,"markdown_tokens":109,"pruned_tokens":109}},
{"rank":7014,"domain":"makerworld.com","origin":"https://makerworld.com","robots":{"status":"ok","http":200,"url":"https://makerworld.com/en/robots.txt","bytes":272,"groups":1},"bots":{"GPTBot":["partial","*",8,1],"ChatGPT-User":["partial","*",8,1],"OAI-SearchBot":["partial","*",8,1],"ClaudeBot":["partial","*",8,1],"Claude-User":["partial","*",8,1],"Claude-SearchBot":["partial","*",8,1],"anthropic-ai":["partial","*",8,1],"PerplexityBot":["partial","*",8,1],"Perplexity-User":["partial","*",8,1],"Google-Extended":["partial","*",8,1],"Applebot-Extended":["partial","*",8,1],"CCBot":["partial","*",8,1],"Bytespider":["partial","*",8,1],"Amazonbot":["partial","*",8,1],"meta-externalagent":["partial","*",8,1],"cohere-ai":["partial","*",8,1],"Diffbot":["partial","*",8,1],"ContextFetchBot":["partial","*",8,1]},"llms":{"status":"http_error","http":403,"bytes":5552},"home":{"status":"http_error","http":403,"url":"https://makerworld.com/"}},
{"rank":7015,"domain":"liga.net","origin":"https://liga.net","robots":{"status":"ok","http":200,"url":"https://www.liga.net/robots.txt","bytes":1558,"groups":4},"bots":{"GPTBot":["partial","*",39,2],"ChatGPT-User":["partial","*",39,2],"OAI-SearchBot":["partial","*",39,2],"ClaudeBot":["partial","*",39,2],"Claude-User":["partial","*",39,2],"Claude-SearchBot":["partial","*",39,2],"anthropic-ai":["partial","*",39,2],"PerplexityBot":["partial","*",39,2],"Perplexity-User":["partial","*",39,2],"Google-Extended":["partial","*",39,2],"Applebot-Extended":["partial","*",39,2],"CCBot":["partial","*",39,2],"Bytespider":["partial","*",39,2],"Amazonbot":["partial","*",39,2],"meta-externalagent":["partial","named",5,1],"cohere-ai":["partial","*",39,2],"Diffbot":["partial","*",39,2],"ContextFetchBot":["partial","*",39,2]},"llms":{"status":"present","http":200,"bytes":4002},"home":{"status":"ok","http":200,"url":"https://www.liga.net/ua","html_tokens":79489,"markdown_tokens":209,"pruned_tokens":165}},
{"rank":7016,"domain":"people.inc","origin":"https://people.inc","robots":{"status":"ok","http":200,"url":"https://www.people.inc/robots.txt","bytes":1994,"groups":26},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["blocked","named",1,0],"OAI-SearchBot":["blocked","named",1,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",2,0],"Claude-SearchBot":["partial","*",2,0],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["partial","*",2,0],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",2,0]},"llms":{"status":"missing","http":404,"bytes":7990},"home":{"status":"ok","http":200,"url":"https://www.people.inc/","html_tokens":74935,"markdown_tokens":15,"pruned_tokens":15}},
{"rank":7017,"domain":"viamichelin.com","origin":"https://viamichelin.com","robots":{"status":"ok","http":200,"url":"https://www.viamichelin.com/robots.txt","bytes":1601,"groups":5},"bots":{"GPTBot":["partial","*",47,0],"ChatGPT-User":["partial","*",47,0],"OAI-SearchBot":["partial","*",47,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",47,0],"Claude-SearchBot":["partial","*",47,0],"anthropic-ai":["partial","*",47,0],"PerplexityBot":["partial","*",47,0],"Perplexity-User":["partial","*",47,0],"Google-Extended":["partial","*",47,0],"Applebot-Extended":["partial","*",47,0],"CCBot":["partial","*",47,0],"Bytespider":["partial","*",47,0],"Amazonbot":["partial","*",47,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["partial","*",47,0],"Diffbot":["partial","*",47,0],"ContextFetchBot":["partial","*",47,0]},"llms":{"status":"missing","http":404,"bytes":7970},"home":{"status":"ok","http":200,"url":"https://www.viamichelin.com/","html_tokens":23538,"markdown_tokens":1106,"pruned_tokens":799}},
{"rank":7018,"domain":"expert.ru","origin":"https://expert.ru","robots":{"status":"ok","http":200,"url":"https://expert.ru/robots.txt","bytes":312,"groups":1},"bots":{"GPTBot":["partial","*",3,0],"ChatGPT-User":["partial","*",3,0],"OAI-SearchBot":["partial","*",3,0],"ClaudeBot":["partial","*",3,0],"Claude-User":["partial","*",3,0],"Claude-SearchBot":["partial","*",3,0],"anthropic-ai":["partial","*",3,0],"PerplexityBot":["partial","*",3,0],"Perplexity-User":["partial","*",3,0],"Google-Extended":["partial","*",3,0],"Applebot-Extended":["partial","*",3,0],"CCBot":["partial","*",3,0],"Bytespider":["partial","*",3,0],"Amazonbot":["partial","*",3,0],"meta-externalagent":["partial","*",3,0],"cohere-ai":["partial","*",3,0],"Diffbot":["partial","*",3,0],"ContextFetchBot":["partial","*",3,0]},"llms":{"status":"missing","http":404,"bytes":73109},"home":{"status":"ok","http":200,"url":"https://expert.ru/","html_tokens":44012,"markdown_tokens":202,"pruned_tokens":169}},
{"rank":7019,"domain":"btinternet.com","origin":"https://btinternet.com","robots":{"status":"timeout","http":null,"url":"https://btinternet.com/robots.txt","bytes":0,"groups":0,"error":"ConnectTimeout"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":7020,"domain":"memberclicks.net","origin":"https://memberclicks.net","robots":{"status":"ok","http":200,"url":"https://memberclicks.net/robots.txt","bytes":775,"groups":1},"bots":{"GPTBot":["partial","*",37,0],"ChatGPT-User":["partial","*",37,0],"OAI-SearchBot":["partial","*",37,0],"ClaudeBot":["partial","*",37,0],"Claude-User":["partial","*",37,0],"Claude-SearchBot":["partial","*",37,0],"anthropic-ai":["partial","*",37,0],"PerplexityBot":["partial","*",37,0],"Perplexity-User":["partial","*",37,0],"Google-Extended":["partial","*",37,0],"Applebot-Extended":["partial","*",37,0],"CCBot":["partial","*",37,0],"Bytespider":["partial","*",37,0],"Amazonbot":["partial","*",37,0],"meta-externalagent":["partial","*",37,0],"cohere-ai":["partial","*",37,0],"Diffbot":["partial","*",37,0],"ContextFetchBot":["partial","*",37,0]},"llms":{"status":"http_error","http":400,"bytes":226},"home":{"status":"http_error","http":400,"url":"https://memberclicks.net/"}},
{"rank":7021,"domain":"man.ac.uk","origin":"http://man.ac.uk","robots":{"status":"ok","http":200,"url":"http://www.man.ac.uk/robots.txt","bytes":40,"groups":1},"bots":{"GPTBot":["partial","*",1,0],"ChatGPT-User":["partial","*",1,0],"OAI-SearchBot":["partial","*",1,0],"ClaudeBot":["partial","*",1,0],"Claude-User":["partial","*",1,0],"Claude-SearchBot":["partial","*",1,0],"anthropic-ai":["partial","*",1,0],"PerplexityBot":["partial","*",1,0],"Perplexity-User":["partial","*",1,0],"Google-Extended":["partial","*",1,0],"Applebot-Extended":["partial","*",1,0],"CCBot":["partial","*",1,0],"Bytespider":["partial","*",1,0],"Amazonbot":["partial","*",1,0],"meta-externalagent":["partial","*",1,0],"cohere-ai":["partial","*",1,0],"Diffbot":["partial","*",1,0],"ContextFetchBot":["partial","*",1,0]},"llms":{"status":"missing","http":404,"bytes":206},"home":{"status":"ok","http":200,"url":"https://www.manchester.ac.uk/","html_tokens":17576,"markdown_tokens":102,"pruned_tokens":102}},
{"rank":7022,"domain":"mindsunlawyers.com","origin":"http://mindsunlawyers.com","robots":{"status":"missing","http":404,"url":"http://mindsunlawyers.com/robots.txt","bytes":5079,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":5075},"home":{"status":"ok","http":200,"url":"http://mindsunlawyers.com/","html_tokens":3098,"markdown_tokens":87,"pruned_tokens":87}},
{"rank":7023,"domain":"oyez.org","origin":"https://oyez.org","robots":{"status":"ok","http":200,"url":"https://www.oyez.org/robots.txt","bytes":78,"groups":1},"bots":{"GPTBot":["partial","*",2,0],"ChatGPT-User":["partial","*",2,0],"OAI-SearchBot":["partial","*",2,0],"ClaudeBot":["partial","*",2,0],"Claude-User":["partial","*",2,0],"Claude-SearchBot":["partial","*",2,0],"anthropic-ai":["partial","*",2,0],"PerplexityBot":["partial","*",2,0],"Perplexity-User":["partial","*",2,0],"Google-Extended":["partial","*",2,0],"Applebot-Extended":["partial","*",2,0],"CCBot":["partial","*",2,0],"Bytespider":["partial","*",2,0],"Amazonbot":["partial","*",2,0],"meta-externalagent":["partial","*",2,0],"cohere-ai":["partial","*",2,0],"Diffbot":["partial","*",2,0],"ContextFetchBot":["partial","*",2,0]},"llms":{"status":"html","http":200,"bytes":2453},"home":{"status":"ok","http":200,"url":"https://www.oyez.org/","html_tokens":614,"markdown_tokens":0,"pruned_tokens":0}},
{"rank":7024,"domain":"westfield.com","origin":"https://westfield.com","robots":{"status":"ok","http":200,"url":"https://www.westfield.com/robots.txt","bytes":940,"groups":1},"bots":{"GPTBot":["partial","*",36,0],"ChatGPT-User":["partial","*",36,0],"OAI-SearchBot":["partial","*",36,0],"ClaudeBot":["partial","*",36,0],"Claude-User":["partial","*",36,0],"Claude-SearchBot":["partial","*",36,0],"anthropic-ai":["partial","*",36,0],"PerplexityBot":["partial","*",36,0],"Perplexity-User":["partial","*",36,0],"Google-Extended":["partial","*",36,0],"Applebot-Extended":["partial","*",36,0],"CCBot":["partial","*",36,0],"Bytespider":["partial","*",36,0],"Amazonbot":["partial","*",36,0],"meta-externalagent":["partial","*",36,0],"cohere-ai":["partial","*",36,0],"Diffbot":["partial","*",36,0],"ContextFetchBot":["partial","*",36,0]},"llms":{"status":"missing","http":404,"bytes":140069},"home":{"status":"ok","http":200,"url":"https://www.westfield.com/","html_tokens":138103,"markdown_tokens":33,"pruned_tokens":33}},
{"rank":7025,"domain":"d-nb.info","origin":"https://d-nb.info","robots":{"status":"ok","http":200,"url":"https://d-nb.info/robots.txt","bytes":217,"groups":1},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"missing","http":404,"bytes":3830},"home":{"status":"ok","http":200,"url":"https://portal.dnb.de/index.htm","html_tokens":41,"markdown_tokens":0,"pruned_tokens":0}},
{"rank":7026,"domain":"smb.museum","origin":"https://smb.museum","robots":{"status":"ok","http":200,"url":"https://www.smb.museum/robots.txt","bytes":2747,"groups":2},"bots":{"GPTBot":["allowed","named",0,0],"ChatGPT-User":["allowed","named",0,0],"OAI-SearchBot":["allowed","named",0,0],"ClaudeBot":["allowed","named",0,0],"Claude-User":["allowed","named",0,0],"Claude-SearchBot":["allowed","named",0,0],"anthropic-ai":["allowed","named",0,0],"PerplexityBot":["allowed","named",0,0],"Perplexity-User":["allowed","named",0,0],"Google-Extended":["allowed","named",0,0],"Applebot-Extended":["allowed","named",0,0],"CCBot":["allowed","named",0,0],"Bytespider":["allowed","named",0,0],"Amazonbot":["allowed","named",0,0],"meta-externalagent":["allowed","named",0,0],"cohere-ai":["allowed","named",0,0],"Diffbot":["allowed","named",0,0],"ContextFetchBot":["partial","*",19,0]},"llms":{"status":"html","http":200,"bytes":29010},"home":{"status":"ok","http":200,"url":"https://www.smb.museum/home/","html_tokens":29418,"markdown_tokens":133,"pruned_tokens":133}},
{"rank":7027,"domain":"glamourmagazine.co.uk","origin":"https://glamourmagazine.co.uk","robots":{"status":"ok","http":200,"url":"https://www.glamourmagazine.co.uk/robots.txt","bytes":2293,"groups":3},"bots":{"GPTBot":["partial","*",14,12],"ChatGPT-User":["partial","*",14,12],"OAI-SearchBot":["partial","*",14,12],"ClaudeBot":["blocked","named",1,0],"Claude-User":["blocked","named",1,0],"Claude-SearchBot":["blocked","named",1,0],"anthropic-ai":["partial","*",14,12],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["blocked","named",1,0],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",14,12]},"llms":{"status":"missing","http":404,"bytes":502120},"home":{"status":"ok","http":200,"url":"https://www.glamourmagazine.co.uk/","html_tokens":365723,"markdown_tokens":404,"pruned_tokens":140}},
{"rank":7028,"domain":"juris.de","origin":"https://juris.de","robots":{"status":"timeout","http":null,"url":"https://juris.de/robots.txt","bytes":0,"groups":0,"error":"ConnectTimeout"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":7029,"domain":"google.com.qa","origin":"https://google.com.qa","robots":{"status":"ok","http":200,"url":"https://www.google.com.qa/robots.txt","bytes":6624,"groups":4},"bots":{"GPTBot":["partial","*",169,64],"ChatGPT-User":["partial","*",169,64],"OAI-SearchBot":["partial","*",169,64],"ClaudeBot":["partial","*",169,64],"Claude-User":["partial","*",169,64],"Claude-SearchBot":["partial","*",169,64],"anthropic-ai":["partial","*",169,64],"PerplexityBot":["partial","*",169,64],"Perplexity-User":["partial","*",169,64],"Google-Extended":["partial","*",169,64],"Applebot-Extended":["partial","*",169,64],"CCBot":["partial","*",169,64],"Bytespider":["partial","*",169,64],"Amazonbot":["partial","*",169,64],"meta-externalagent":["partial","*",169,64],"cohere-ai":["partial","*",169,64],"Diffbot":["partial","*",169,64],"ContextFetchBot":["partial","*",169,64]},"llms":{"status":"missing","http":404,"bytes":1569},"home":{"status":"ok","http":200,"url":"https://www.google.com/","html_tokens":21453,"markdown_tokens":30,"pruned_tokens":30}},
{"rank":7030,"domain":"npg.org.uk","origin":"https://npg.org.uk","robots":{"status":"ok","http":200,"url":"https://npg.org.uk/robots.txt","bytes":2676,"groups":32},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["allowed","*",0,1],"OAI-SearchBot":["allowed","*",0,1],"ClaudeBot":["blocked","named",1,0],"Claude-User":["allowed","*",0,1],"Claude-SearchBot":["allowed","*",0,1],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["allowed","*",0,1],"Perplexity-User":["allowed","*",0,1],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["allowed","*",0,1]},"llms":{"status":"http_error","http":403,"bytes":5541},"home":{"status":"http_error","http":403,"url":"https://npg.org.uk/"}},
{"rank":7031,"domain":"onsemi.com","origin":"https://onsemi.com","robots":{"status":"ok","http":200,"url":"https://www.onsemi.com/robots.txt","bytes":478,"groups":4},"bots":{"GPTBot":["partial","*",9,1],"ChatGPT-User":["partial","*",9,1],"OAI-SearchBot":["partial","*",9,1],"ClaudeBot":["partial","*",9,1],"Claude-User":["partial","*",9,1],"Claude-SearchBot":["partial","*",9,1],"anthropic-ai":["partial","*",9,1],"PerplexityBot":["partial","*",9,1],"Perplexity-User":["partial","*",9,1],"Google-Extended":["partial","*",9,1],"Applebot-Extended":["partial","*",9,1],"CCBot":["partial","*",9,1],"Bytespider":["partial","*",9,1],"Amazonbot":["partial","*",9,1],"meta-externalagent":["partial","*",9,1],"cohere-ai":["partial","*",9,1],"Diffbot":["partial","*",9,1],"ContextFetchBot":["partial","*",9,1]},"llms":{"status":"timeout","error":"timed out after 10 s"},"home":{"status":"ok","http":200,"url":"https://www.onsemi.com/","html_tokens":50751,"markdown_tokens":65,"pruned_tokens":65}},
{"rank":7032,"domain":"podcastaddict.com","origin":"https://podcastaddict.com","robots":{"status":"ok","http":200,"url":"https://podcastaddict.com/robots.txt","bytes":1500,"groups":15},"bots":{"GPTBot":["partial","*",7,10],"ChatGPT-User":["partial","*",7,10],"OAI-SearchBot":["partial","*",7,10],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",7,10],"Claude-SearchBot":["partial","*",7,10],"anthropic-ai":["partial","*",7,10],"PerplexityBot":["partial","*",7,10],"Perplexity-User":["partial","*",7,10],"Google-Extended":["partial","*",7,10],"Applebot-Extended":["partial","*",7,10],"CCBot":["partial","*",7,10],"Bytespider":["partial","*",7,10],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["partial","*",7,10],"cohere-ai":["partial","*",7,10],"Diffbot":["partial","*",7,10],"ContextFetchBot":["partial","*",7,10]},"llms":{"status":"present","http":200,"bytes":1975},"home":{"status":"http_error","http":403,"url":"https://podcastaddict.com/"}},
{"rank":7033,"domain":"sears.com","origin":"https://sears.com","robots":{"status":"ok","http":200,"url":"https://www.sears.com/robots.txt","bytes":1720,"groups":1},"bots":{"GPTBot":["partial","*",42,0],"ChatGPT-User":["partial","*",42,0],"OAI-SearchBot":["partial","*",42,0],"ClaudeBot":["partial","*",42,0],"Claude-User":["partial","*",42,0],"Claude-SearchBot":["partial","*",42,0],"anthropic-ai":["partial","*",42,0],"PerplexityBot":["partial","*",42,0],"Perplexity-User":["partial","*",42,0],"Google-Extended":["partial","*",42,0],"Applebot-Extended":["partial","*",42,0],"CCBot":["partial","*",42,0],"Bytespider":["partial","*",42,0],"Amazonbot":["partial","*",42,0],"meta-externalagent":["partial","*",42,0],"cohere-ai":["partial","*",42,0],"Diffbot":["partial","*",42,0],"ContextFetchBot":["partial","*",42,0]},"llms":{"status":"html","http":200,"bytes":72337},"home":{"status":"ok","http":200,"url":"https://www.sears.com/","html_tokens":6562,"markdown_tokens":0,"pruned_tokens":0}},
{"rank":7034,"domain":"google.com.ly","origin":"https://google.com.ly","robots":{"status":"ok","http":200,"url":"https://www.google.com.ly/robots.txt","bytes":6624,"groups":4},"bots":{"GPTBot":["partial","*",169,64],"ChatGPT-User":["partial","*",169,64],"OAI-SearchBot":["partial","*",169,64],"ClaudeBot":["partial","*",169,64],"Claude-User":["partial","*",169,64],"Claude-SearchBot":["partial","*",169,64],"anthropic-ai":["partial","*",169,64],"PerplexityBot":["partial","*",169,64],"Perplexity-User":["partial","*",169,64],"Google-Extended":["partial","*",169,64],"Applebot-Extended":["partial","*",169,64],"CCBot":["partial","*",169,64],"Bytespider":["partial","*",169,64],"Amazonbot":["partial","*",169,64],"meta-externalagent":["partial","*",169,64],"cohere-ai":["partial","*",169,64],"Diffbot":["partial","*",169,64],"ContextFetchBot":["partial","*",169,64]},"llms":{"status":"missing","http":404,"bytes":1569},"home":{"status":"ok","http":200,"url":"https://www.google.com/","html_tokens":21290,"markdown_tokens":30,"pruned_tokens":30}},
{"rank":7035,"domain":"shu.ac.uk","origin":"https://www.shu.ac.uk","robots":{"status":"ok","http":200,"url":"https://www.shu.ac.uk/robots.txt","bytes":310,"groups":1},"bots":{"GPTBot":["partial","*",9,0],"ChatGPT-User":["partial","*",9,0],"OAI-SearchBot":["partial","*",9,0],"ClaudeBot":["partial","*",9,0],"Claude-User":["partial","*",9,0],"Claude-SearchBot":["partial","*",9,0],"anthropic-ai":["partial","*",9,0],"PerplexityBot":["partial","*",9,0],"Perplexity-User":["partial","*",9,0],"Google-Extended":["partial","*",9,0],"Applebot-Extended":["partial","*",9,0],"CCBot":["partial","*",9,0],"Bytespider":["partial","*",9,0],"Amazonbot":["partial","*",9,0],"meta-externalagent":["partial","*",9,0],"cohere-ai":["partial","*",9,0],"Diffbot":["partial","*",9,0],"ContextFetchBot":["partial","*",9,0]},"llms":{"status":"present","http":200,"bytes":6206},"home":{"status":"ok","http":200,"url":"https://www.shu.ac.uk/","html_tokens":16087,"markdown_tokens":65,"pruned_tokens":48}},
{"rank":7036,"domain":"hemingwayapp.com","origin":"https://hemingwayapp.com","robots":{"status":"missing","http":404,"url":"https://hemingwayapp.com/robots.txt","bytes":90709,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":90709},"home":{"status":"ok","http":200,"url":"https://hemingwayapp.com/","html_tokens":25723,"markdown_tokens":13,"pruned_tokens":13}},
{"rank":7037,"domain":"google.com.cu","origin":"https://google.com.cu","robots":{"status":"ok","http":200,"url":"https://www.google.com.cu/robots.txt","bytes":6624,"groups":4},"bots":{"GPTBot":["partial","*",169,64],"ChatGPT-User":["partial","*",169,64],"OAI-SearchBot":["partial","*",169,64],"ClaudeBot":["partial","*",169,64],"Claude-User":["partial","*",169,64],"Claude-SearchBot":["partial","*",169,64],"anthropic-ai":["partial","*",169,64],"PerplexityBot":["partial","*",169,64],"Perplexity-User":["partial","*",169,64],"Google-Extended":["partial","*",169,64],"Applebot-Extended":["partial","*",169,64],"CCBot":["partial","*",169,64],"Bytespider":["partial","*",169,64],"Amazonbot":["partial","*",169,64],"meta-externalagent":["partial","*",169,64],"cohere-ai":["partial","*",169,64],"Diffbot":["partial","*",169,64],"ContextFetchBot":["partial","*",169,64]},"llms":{"status":"missing","http":404,"bytes":1569},"home":{"status":"ok","http":200,"url":"https://www.google.com/","html_tokens":21458,"markdown_tokens":30,"pruned_tokens":30}},
{"rank":7038,"domain":"adsoftheworld.com","origin":"https://adsoftheworld.com","robots":{"status":"refused","http":403,"url":"http://www.adsoftheworld.com/robots.txt","bytes":118,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":7039,"domain":"sebrae.com.br","origin":"https://sebrae.com.br","robots":{"status":"ok","http":200,"url":"https://sebrae.com.br/robots.txt","bytes":67,"groups":1},"bots":{"GPTBot":["allowed","*",0,1],"ChatGPT-User":["allowed","*",0,1],"OAI-SearchBot":["allowed","*",0,1],"ClaudeBot":["allowed","*",0,1],"Claude-User":["allowed","*",0,1],"Claude-SearchBot":["allowed","*",0,1],"anthropic-ai":["allowed","*",0,1],"PerplexityBot":["allowed","*",0,1],"Perplexity-User":["allowed","*",0,1],"Google-Extended":["allowed","*",0,1],"Applebot-Extended":["allowed","*",0,1],"CCBot":["allowed","*",0,1],"Bytespider":["allowed","*",0,1],"Amazonbot":["allowed","*",0,1],"meta-externalagent":["allowed","*",0,1],"cohere-ai":["allowed","*",0,1],"Diffbot":["allowed","*",0,1],"ContextFetchBot":["allowed","*",0,1]},"llms":{"status":"missing","http":404,"bytes":119625},"home":{"status":"ok","http":200,"url":"https://sebrae.com.br/","html_tokens":43238,"markdown_tokens":54,"pruned_tokens":40}},
{"rank":7040,"domain":"breakingdefense.com","origin":"https://breakingdefense.com","robots":{"status":"ok","http":200,"url":"https://breakingdefense.com/robots.txt","bytes":327,"groups":3},"bots":{"GPTBot":["partial","*",2,1],"ChatGPT-User":["partial","*",2,1],"OAI-SearchBot":["partial","*",2,1],"ClaudeBot":["partial","*",2,1],"Claude-User":["partial","*",2,1],"Claude-SearchBot":["partial","*",2,1],"anthropic-ai":["partial","*",2,1],"PerplexityBot":["partial","*",2,1],"Perplexity-User":["partial","*",2,1],"Google-Extended":["partial","*",2,1],"Applebot-Extended":["partial","*",2,1],"CCBot":["partial","*",2,1],"Bytespider":["partial","*",2,1],"Amazonbot":["partial","*",2,1],"meta-externalagent":["partial","*",2,1],"cohere-ai":["partial","*",2,1],"Diffbot":["partial","*",2,1],"ContextFetchBot":["partial","*",2,1]},"llms":{"status":"http_error","http":403,"bytes":5849},"home":{"status":"http_error","http":403,"url":"https://breakingdefense.com/"}},
{"rank":7041,"domain":"insiderintelligence.com","origin":"https://insiderintelligence.com","robots":{"status":"ok","http":200,"url":"https://www.emarketer.com/robots.txt","bytes":3776,"groups":2},"bots":{"GPTBot":["partial","*",127,1],"ChatGPT-User":["partial","*",127,1],"OAI-SearchBot":["partial","*",127,1],"ClaudeBot":["partial","*",127,1],"Claude-User":["partial","*",127,1],"Claude-SearchBot":["partial","*",127,1],"anthropic-ai":["partial","*",127,1],"PerplexityBot":["partial","*",127,1],"Perplexity-User":["partial","*",127,1],"Google-Extended":["partial","named",2,0],"Applebot-Extended":["partial","*",127,1],"CCBot":["partial","*",127,1],"Bytespider":["partial","*",127,1],"Amazonbot":["partial","*",127,1],"meta-externalagent":["partial","*",127,1],"cohere-ai":["partial","*",127,1],"Diffbot":["partial","*",127,1],"ContextFetchBot":["partial","*",127,1]},"llms":{"status":"missing","http":404,"bytes":105164},"home":{"status":"http_error","http":403,"url":"https://www.emarketer.com/"}},
{"rank":7042,"domain":"aur.archlinux.org","origin":"https://aur.archlinux.org","robots":{"status":"ok","http":200,"url":"https://aur.archlinux.org/robots.txt","bytes":458,"groups":1},"bots":{"GPTBot":["partial","*",4,2],"ChatGPT-User":["partial","*",4,2],"OAI-SearchBot":["partial","*",4,2],"ClaudeBot":["partial","*",4,2],"Claude-User":["partial","*",4,2],"Claude-SearchBot":["partial","*",4,2],"anthropic-ai":["partial","*",4,2],"PerplexityBot":["partial","*",4,2],"Perplexity-User":["partial","*",4,2],"Google-Extended":["partial","*",4,2],"Applebot-Extended":["partial","*",4,2],"CCBot":["partial","*",4,2],"Bytespider":["partial","*",4,2],"Amazonbot":["partial","*",4,2],"meta-externalagent":["partial","*",4,2],"cohere-ai":["partial","*",4,2],"Diffbot":["partial","*",4,2],"ContextFetchBot":["partial","*",4,2]},"llms":{"status":"missing","http":404,"bytes":9667},"home":{"status":"ok","http":200,"url":"https://aur.archlinux.org/","html_tokens":5392,"markdown_tokens":415,"pruned_tokens":120}},
{"rank":7043,"domain":"wayboo.cn","origin":"http://wayboo.cn","robots":{"status":"missing","http":404,"url":"http://wayboo.cn/robots.txt","bytes":168,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":168},"home":{"status":"ok","http":200,"url":"http://wayboo.cn/","html_tokens":10291,"markdown_tokens":206,"pruned_tokens":158}},
{"rank":7044,"domain":"getpostman.com","origin":"https://getpostman.com","robots":{"status":"ok","http":200,"url":"https://www.postman.com/robots.txt","bytes":3118,"groups":2},"bots":{"GPTBot":["partial","named",5,1],"ChatGPT-User":["partial","named",5,1],"OAI-SearchBot":["partial","named",5,1],"ClaudeBot":["partial","named",5,1],"Claude-User":["partial","named",5,1],"Claude-SearchBot":["partial","named",5,1],"anthropic-ai":["partial","named",5,1],"PerplexityBot":["partial","named",5,1],"Perplexity-User":["partial","named",5,1],"Google-Extended":["partial","named",5,1],"Applebot-Extended":["partial","named",5,1],"CCBot":["partial","named",5,1],"Bytespider":["blocked","named",1,0],"Amazonbot":["partial","named",5,1],"meta-externalagent":["partial","named",5,1],"cohere-ai":["partial","named",5,1],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",5,1]},"llms":{"status":"present","http":200,"bytes":21271},"home":{"status":"ok","http":200,"url":"https://www.postman.com/","html_tokens":125795,"markdown_tokens":337,"pruned_tokens":101}},
{"rank":7045,"domain":"tnt.com","origin":"https://www.tnt.com","robots":{"status":"ok","http":200,"url":"https://www.tnt.com/robots.txt","bytes":1216,"groups":3},"bots":{"GPTBot":["partial","*",20,0],"ChatGPT-User":["partial","*",20,0],"OAI-SearchBot":["partial","*",20,0],"ClaudeBot":["partial","*",20,0],"Claude-User":["partial","*",20,0],"Claude-SearchBot":["partial","*",20,0],"anthropic-ai":["partial","*",20,0],"PerplexityBot":["partial","*",20,0],"Perplexity-User":["partial","*",20,0],"Google-Extended":["partial","*",20,0],"Applebot-Extended":["partial","*",20,0],"CCBot":["partial","*",20,0],"Bytespider":["partial","*",20,0],"Amazonbot":["partial","*",20,0],"meta-externalagent":["partial","*",20,0],"cohere-ai":["partial","*",20,0],"Diffbot":["partial","*",20,0],"ContextFetchBot":["partial","*",20,0]},"llms":{"status":"missing","http":404,"bytes":196},"home":{"status":"ok","http":200,"url":"https://www.tnt.com/express/en_us/site/home.html","html_tokens":6875,"markdown_tokens":40,"pruned_tokens":40}},
{"rank":7046,"domain":"ncronline.org","origin":"https://ncronline.org","robots":{"status":"ok","http":200,"url":"https://ncronline.org/robots.txt","bytes":2027,"groups":1},"bots":{"GPTBot":["partial","*",34,18],"ChatGPT-User":["partial","*",34,18],"OAI-SearchBot":["partial","*",34,18],"ClaudeBot":["partial","*",34,18],"Claude-User":["partial","*",34,18],"Claude-SearchBot":["partial","*",34,18],"anthropic-ai":["partial","*",34,18],"PerplexityBot":["partial","*",34,18],"Perplexity-User":["partial","*",34,18],"Google-Extended":["partial","*",34,18],"Applebot-Extended":["partial","*",34,18],"CCBot":["partial","*",34,18],"Bytespider":["partial","*",34,18],"Amazonbot":["partial","*",34,18],"meta-externalagent":["partial","*",34,18],"cohere-ai":["partial","*",34,18],"Diffbot":["partial","*",34,18],"ContextFetchBot":["partial","*",34,18]},"llms":{"status":"missing","http":404,"bytes":552},"home":{"status":"http_error","http":403,"url":"https://ncronline.org/"}},
{"rank":7047,"domain":"istudiosg.com","origin":"https://istudiosg.com","robots":{"status":"ok","http":200,"url":"https://www.istudiosg.com/robots.txt","bytes":3663,"groups":2},"bots":{"GPTBot":["partial","*",40,23],"ChatGPT-User":["partial","*",40,23],"OAI-SearchBot":["partial","*",40,23],"ClaudeBot":["partial","*",40,23],"Claude-User":["partial","*",40,23],"Claude-SearchBot":["partial","*",40,23],"anthropic-ai":["partial","*",40,23],"PerplexityBot":["partial","*",40,23],"Perplexity-User":["partial","*",40,23],"Google-Extended":["partial","*",40,23],"Applebot-Extended":["partial","*",40,23],"CCBot":["partial","*",40,23],"Bytespider":["partial","*",40,23],"Amazonbot":["partial","*",40,23],"meta-externalagent":["partial","*",40,23],"cohere-ai":["partial","*",40,23],"Diffbot":["partial","*",40,23],"ContextFetchBot":["partial","*",40,23]},"llms":{"status":"present","http":200,"bytes":5818},"home":{"status":"ok","http":200,"url":"https://www.istudiosg.com/","html_tokens":60892,"markdown_tokens":76,"pruned_tokens":76}},
{"rank":7048,"domain":"google.co.ma","origin":"https://google.co.ma","robots":{"status":"ok","http":200,"url":"https://www.google.co.ma/robots.txt","bytes":6624,"groups":4},"bots":{"GPTBot":["partial","*",169,64],"ChatGPT-User":["partial","*",169,64],"OAI-SearchBot":["partial","*",169,64],"ClaudeBot":["partial","*",169,64],"Claude-User":["partial","*",169,64],"Claude-SearchBot":["partial","*",169,64],"anthropic-ai":["partial","*",169,64],"PerplexityBot":["partial","*",169,64],"Perplexity-User":["partial","*",169,64],"Google-Extended":["partial","*",169,64],"Applebot-Extended":["partial","*",169,64],"CCBot":["partial","*",169,64],"Bytespider":["partial","*",169,64],"Amazonbot":["partial","*",169,64],"meta-externalagent":["partial","*",169,64],"cohere-ai":["partial","*",169,64],"Diffbot":["partial","*",169,64],"ContextFetchBot":["partial","*",169,64]},"llms":{"status":"missing","http":404,"bytes":1569},"home":{"status":"ok","http":200,"url":"https://www.google.com/","html_tokens":21307,"markdown_tokens":30,"pruned_tokens":30}},
{"rank":7049,"domain":"soccerway.com","origin":"https://soccerway.com","robots":{"status":"ok","http":200,"url":"https://www.soccerway.com/robots.txt","bytes":2553,"groups":17},"bots":{"GPTBot":["partial","*",57,0],"ChatGPT-User":["partial","*",57,0],"OAI-SearchBot":["partial","*",57,0],"ClaudeBot":["partial","*",57,0],"Claude-User":["partial","*",57,0],"Claude-SearchBot":["partial","*",57,0],"anthropic-ai":["partial","*",57,0],"PerplexityBot":["partial","*",57,0],"Perplexity-User":["partial","*",57,0],"Google-Extended":["partial","*",57,0],"Applebot-Extended":["partial","*",57,0],"CCBot":["partial","*",57,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["partial","*",57,0],"meta-externalagent":["partial","*",57,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",57,0]},"llms":{"status":"missing","http":404,"bytes":4539},"home":{"status":"ok","http":200,"url":"https://www.soccerway.com/","html_tokens":160175,"markdown_tokens":285,"pruned_tokens":125}},
{"rank":7050,"domain":"google.com.ag","origin":"https://google.com.ag","robots":{"status":"ok","http":200,"url":"https://www.google.com.ag/robots.txt","bytes":6624,"groups":4},"bots":{"GPTBot":["partial","*",169,64],"ChatGPT-User":["partial","*",169,64],"OAI-SearchBot":["partial","*",169,64],"ClaudeBot":["partial","*",169,64],"Claude-User":["partial","*",169,64],"Claude-SearchBot":["partial","*",169,64],"anthropic-ai":["partial","*",169,64],"PerplexityBot":["partial","*",169,64],"Perplexity-User":["partial","*",169,64],"Google-Extended":["partial","*",169,64],"Applebot-Extended":["partial","*",169,64],"CCBot":["partial","*",169,64],"Bytespider":["partial","*",169,64],"Amazonbot":["partial","*",169,64],"meta-externalagent":["partial","*",169,64],"cohere-ai":["partial","*",169,64],"Diffbot":["partial","*",169,64],"ContextFetchBot":["partial","*",169,64]},"llms":{"status":"missing","http":404,"bytes":1569},"home":{"status":"ok","http":200,"url":"https://www.google.com/","html_tokens":21287,"markdown_tokens":30,"pruned_tokens":30}},
{"rank":7051,"domain":"google.com.gi","origin":"https://google.com.gi","robots":{"status":"ok","http":200,"url":"https://www.google.com.gi/robots.txt","bytes":6624,"groups":4},"bots":{"GPTBot":["partial","*",169,64],"ChatGPT-User":["partial","*",169,64],"OAI-SearchBot":["partial","*",169,64],"ClaudeBot":["partial","*",169,64],"Claude-User":["partial","*",169,64],"Claude-SearchBot":["partial","*",169,64],"anthropic-ai":["partial","*",169,64],"PerplexityBot":["partial","*",169,64],"Perplexity-User":["partial","*",169,64],"Google-Extended":["partial","*",169,64],"Applebot-Extended":["partial","*",169,64],"CCBot":["partial","*",169,64],"Bytespider":["partial","*",169,64],"Amazonbot":["partial","*",169,64],"meta-externalagent":["partial","*",169,64],"cohere-ai":["partial","*",169,64],"Diffbot":["partial","*",169,64],"ContextFetchBot":["partial","*",169,64]},"llms":{"status":"missing","http":404,"bytes":1569},"home":{"status":"ok","http":200,"url":"https://www.google.com/","html_tokens":21357,"markdown_tokens":30,"pruned_tokens":30}},
{"rank":7052,"domain":"thepetitionsite.com","origin":"https://thepetitionsite.com","robots":{"status":"ok","http":200,"url":"https://www.thepetitionsite.com/robots.txt","bytes":173,"groups":1},"bots":{"GPTBot":["partial","*",6,0],"ChatGPT-User":["partial","*",6,0],"OAI-SearchBot":["partial","*",6,0],"ClaudeBot":["partial","*",6,0],"Claude-User":["partial","*",6,0],"Claude-SearchBot":["partial","*",6,0],"anthropic-ai":["partial","*",6,0],"PerplexityBot":["partial","*",6,0],"Perplexity-User":["partial","*",6,0],"Google-Extended":["partial","*",6,0],"Applebot-Extended":["partial","*",6,0],"CCBot":["partial","*",6,0],"Bytespider":["partial","*",6,0],"Amazonbot":["partial","*",6,0],"meta-externalagent":["partial","*",6,0],"cohere-ai":["partial","*",6,0],"Diffbot":["partial","*",6,0],"ContextFetchBot":["partial","*",6,0]},"llms":{"status":"missing","http":404,"bytes":24926},"home":{"status":"ok","http":200,"url":"https://www.thepetitionsite.com/","html_tokens":12202,"markdown_tokens":152,"pruned_tokens":152}},
{"rank":7053,"domain":"surveymonkey.co.uk","origin":"https://surveymonkey.co.uk","robots":{"status":"ok","http":200,"url":"https://uk.surveymonkey.com/robots.txt","bytes":711,"groups":1},"bots":{"GPTBot":["partial","*",21,6],"ChatGPT-User":["partial","*",21,6],"OAI-SearchBot":["partial","*",21,6],"ClaudeBot":["partial","*",21,6],"Claude-User":["partial","*",21,6],"Claude-SearchBot":["partial","*",21,6],"anthropic-ai":["partial","*",21,6],"PerplexityBot":["partial","*",21,6],"Perplexity-User":["partial","*",21,6],"Google-Extended":["partial","*",21,6],"Applebot-Extended":["partial","*",21,6],"CCBot":["partial","*",21,6],"Bytespider":["partial","*",21,6],"Amazonbot":["partial","*",21,6],"meta-externalagent":["partial","*",21,6],"cohere-ai":["partial","*",21,6],"Diffbot":["partial","*",21,6],"ContextFetchBot":["partial","*",21,6]},"llms":{"status":"present","http":200,"bytes":13693},"home":{"status":"ok","http":200,"url":"https://uk.surveymonkey.com/","html_tokens":144378,"markdown_tokens":70,"pruned_tokens":45}},
{"rank":7054,"domain":"citysearch.com","origin":"https://citysearch.com","robots":{"status":"ok","http":200,"url":"https://citysearch.com/robots.txt","bytes":67,"groups":1},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"missing","http":404,"bytes":2166},"home":{"status":"ok","http":200,"url":"https://citysearch.com/","html_tokens":542,"markdown_tokens":0,"pruned_tokens":0}},
{"rank":7055,"domain":"ulg.ac.be","origin":"https://ulg.ac.be","robots":{"status":"server_error","http":500,"url":"https://ulg.ac.be/robots.txt","bytes":73692,"groups":0,"error":"HTTP 500"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":7056,"domain":"graphviz.org","origin":"https://graphviz.org","robots":{"status":"ok","http":200,"url":"https://graphviz.org/robots.txt","bytes":14,"groups":1},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"missing","http":404,"bytes":6155},"home":{"status":"ok","http":200,"url":"https://graphviz.org/","html_tokens":25733,"markdown_tokens":322,"pruned_tokens":220}},
{"rank":7057,"domain":"wmagazine.com","origin":"https://wmagazine.com","robots":{"status":"ok","http":200,"url":"https://www.wmagazine.com/robots.txt","bytes":292,"groups":2},"bots":{"GPTBot":["partial","*",1,0],"ChatGPT-User":["partial","*",1,0],"OAI-SearchBot":["partial","*",1,0],"ClaudeBot":["partial","*",1,0],"Claude-User":["partial","*",1,0],"Claude-SearchBot":["partial","*",1,0],"anthropic-ai":["partial","*",1,0],"PerplexityBot":["partial","*",1,0],"Perplexity-User":["partial","*",1,0],"Google-Extended":["partial","*",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["partial","*",1,0],"Bytespider":["partial","*",1,0],"Amazonbot":["partial","*",1,0],"meta-externalagent":["partial","*",1,0],"cohere-ai":["partial","*",1,0],"Diffbot":["partial","*",1,0],"ContextFetchBot":["partial","*",1,0]},"llms":{"status":"missing","http":404,"bytes":25719},"home":{"status":"ok","http":200,"url":"https://www.wmagazine.com","html_tokens":74977,"markdown_tokens":23,"pruned_tokens":23}},
{"rank":7058,"domain":"wikiart.org","origin":"https://wikiart.org","robots":{"status":"ok","http":200,"url":"https://www.wikiart.org/robots.txt","bytes":85,"groups":1},"bots":{"GPTBot":["allowed","*",0,1],"ChatGPT-User":["allowed","*",0,1],"OAI-SearchBot":["allowed","*",0,1],"ClaudeBot":["allowed","*",0,1],"Claude-User":["allowed","*",0,1],"Claude-SearchBot":["allowed","*",0,1],"anthropic-ai":["allowed","*",0,1],"PerplexityBot":["allowed","*",0,1],"Perplexity-User":["allowed","*",0,1],"Google-Extended":["allowed","*",0,1],"Applebot-Extended":["allowed","*",0,1],"CCBot":["allowed","*",0,1],"Bytespider":["allowed","*",0,1],"Amazonbot":["allowed","*",0,1],"meta-externalagent":["allowed","*",0,1],"cohere-ai":["allowed","*",0,1],"Diffbot":["allowed","*",0,1],"ContextFetchBot":["allowed","*",0,1]},"llms":{"status":"missing","http":404,"bytes":298592},"home":{"status":"ok","http":200,"url":"https://www.wikiart.org/","html_tokens":100127,"markdown_tokens":76,"pruned_tokens":76}},
{"rank":7059,"domain":"realestate.com.au","origin":"https://realestate.com.au","robots":{"status":"ok","http":200,"url":"https://www.realestate.com.au/robots.txt","bytes":2094,"groups":30},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["partial","*",2,0],"OAI-SearchBot":["partial","*",2,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["blocked","named",1,0],"Claude-SearchBot":["blocked","named",1,0],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["blocked","named",1,0],"Google-Extended":["partial","*",2,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["partial","*",2,0],"ContextFetchBot":["partial","*",2,0]},"llms":{"status":"missing","http":404,"bytes":43865},"home":{"status":"http_error","http":429,"url":"https://www.realestate.com.au/"}},
{"rank":7060,"domain":"chicagobusiness.com","origin":"https://chicagobusiness.com","robots":{"status":"refused","http":403,"url":"https://www.chicagobusiness.com/robots.txt","bytes":391,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":7061,"domain":"allenpress.com","origin":"https://allenpress.com","robots":{"status":"refused","http":403,"url":"https://allenpress.com/robots.txt","bytes":103,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":7062,"domain":"dogpile.com","origin":"https://dogpile.com","robots":{"status":"ok","http":200,"url":"https://www.dogpile.com/robots.txt","bytes":94,"groups":1},"bots":{"GPTBot":["partial","*",4,1],"ChatGPT-User":["partial","*",4,1],"OAI-SearchBot":["partial","*",4,1],"ClaudeBot":["partial","*",4,1],"Claude-User":["partial","*",4,1],"Claude-SearchBot":["partial","*",4,1],"anthropic-ai":["partial","*",4,1],"PerplexityBot":["partial","*",4,1],"Perplexity-User":["partial","*",4,1],"Google-Extended":["partial","*",4,1],"Applebot-Extended":["partial","*",4,1],"CCBot":["partial","*",4,1],"Bytespider":["partial","*",4,1],"Amazonbot":["partial","*",4,1],"meta-externalagent":["partial","*",4,1],"cohere-ai":["partial","*",4,1],"Diffbot":["partial","*",4,1],"ContextFetchBot":["partial","*",4,1]},"llms":{"status":"missing","http":404,"bytes":54755},"home":{"status":"ok","http":200,"url":"https://www.dogpile.com/","html_tokens":18333,"markdown_tokens":6,"pruned_tokens":6}},
{"rank":7063,"domain":"polymarket.com","origin":"https://polymarket.com","robots":{"status":"ok","http":200,"url":"https://polymarket.com/robots.txt","bytes":130,"groups":1},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"present","http":200,"bytes":9642},"home":{"status":"ok","http":200,"url":"https://polymarket.com/","html_tokens":449449,"markdown_tokens":37,"pruned_tokens":37}},
{"rank":7064,"domain":"peacocktv.com","origin":"https://peacocktv.com","robots":{"status":"ok","http":200,"url":"https://www.peacocktv.com/robots.txt","bytes":703,"groups":7},"bots":{"GPTBot":["partial","*",3,0],"ChatGPT-User":["partial","*",3,0],"OAI-SearchBot":["partial","*",3,0],"ClaudeBot":["partial","*",3,0],"Claude-User":["partial","*",3,0],"Claude-SearchBot":["partial","*",3,0],"anthropic-ai":["partial","*",3,0],"PerplexityBot":["partial","*",3,0],"Perplexity-User":["partial","*",3,0],"Google-Extended":["partial","*",3,0],"Applebot-Extended":["partial","*",3,0],"CCBot":["partial","*",3,0],"Bytespider":["partial","*",3,0],"Amazonbot":["partial","*",3,0],"meta-externalagent":["allowed","named",0,1],"cohere-ai":["partial","*",3,0],"Diffbot":["partial","*",3,0],"ContextFetchBot":["partial","*",3,0]},"llms":{"status":"missing","http":404,"bytes":407883},"home":{"status":"ok","http":200,"url":"https://www.peacocktv.com/","html_tokens":174278,"markdown_tokens":907,"pruned_tokens":91}},
{"rank":7065,"domain":"standaard.be","origin":"https://standaard.be","robots":{"status":"ok","http":200,"url":"https://www.standaard.be/robots.txt","bytes":2761,"groups":40},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["blocked","named",1,0],"OAI-SearchBot":["blocked","named",1,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["blocked","named",1,0],"Claude-SearchBot":["blocked","named",1,0],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["blocked","named",1,0],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["partial","*",2,2],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",2,2]},"llms":{"status":"http_error","http":403,"bytes":4545},"home":{"status":"http_error","http":403,"url":"https://www.standaard.be/"}},
{"rank":7066,"domain":"rarediseases.org","origin":"https://rarediseases.org","robots":{"status":"ok","http":200,"url":"https://rarediseases.org/robots.txt","bytes":3502,"groups":12},"bots":{"GPTBot":["allowed","named",0,1],"ChatGPT-User":["partial","*",16,5],"OAI-SearchBot":["partial","*",16,5],"ClaudeBot":["allowed","named",0,1],"Claude-User":["partial","*",16,5],"Claude-SearchBot":["partial","*",16,5],"anthropic-ai":["partial","*",16,5],"PerplexityBot":["partial","*",16,5],"Perplexity-User":["partial","*",16,5],"Google-Extended":["allowed","named",0,1],"Applebot-Extended":["allowed","named",0,1],"CCBot":["allowed","named",0,1],"Bytespider":["allowed","named",0,1],"Amazonbot":["allowed","named",0,1],"meta-externalagent":["allowed","named",0,1],"cohere-ai":["partial","*",16,5],"Diffbot":["partial","*",16,5],"ContextFetchBot":["partial","*",16,5]},"llms":{"status":"http_error","http":403,"bytes":6217},"home":{"status":"ok","http":200,"url":"https://rarediseases.org/","html_tokens":170829,"markdown_tokens":850,"pruned_tokens":156}},
{"rank":7067,"domain":"nvcasinos.win","origin":"https://nvcasinos.win","robots":{"status":"ok","http":200,"url":"https://nvcasinos.win/robots.txt","bytes":638,"groups":5},"bots":{"GPTBot":["partial","*",3,16],"ChatGPT-User":["partial","*",3,16],"OAI-SearchBot":["partial","*",3,16],"ClaudeBot":["partial","*",3,16],"Claude-User":["partial","*",3,16],"Claude-SearchBot":["partial","*",3,16],"anthropic-ai":["partial","*",3,16],"PerplexityBot":["partial","*",3,16],"Perplexity-User":["partial","*",3,16],"Google-Extended":["partial","*",3,16],"Applebot-Extended":["partial","*",3,16],"CCBot":["partial","*",3,16],"Bytespider":["partial","*",3,16],"Amazonbot":["partial","*",3,16],"meta-externalagent":["partial","*",3,16],"cohere-ai":["partial","*",3,16],"Diffbot":["partial","*",3,16],"ContextFetchBot":["partial","*",3,16]},"llms":{"status":"missing","http":404,"bytes":21869},"home":{"status":"ok","http":200,"url":"https://nvcasinos.win/","html_tokens":25403,"markdown_tokens":3339,"pruned_tokens":108}},
{"rank":7068,"domain":"thinkquest.org","origin":"https://thinkquest.org","robots":{"status":"timeout","http":null,"url":"https://thinkquest.org/robots.txt","bytes":0,"groups":0,"error":"ConnectTimeout"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":7069,"domain":"dice.com","origin":"https://dice.com","robots":{"status":"ok","http":200,"url":"https://www.dice.com/robots.txt","bytes":1167,"groups":2},"bots":{"GPTBot":["partial","*",35,5],"ChatGPT-User":["partial","*",35,5],"OAI-SearchBot":["partial","*",35,5],"ClaudeBot":["partial","*",35,5],"Claude-User":["partial","*",35,5],"Claude-SearchBot":["partial","*",35,5],"anthropic-ai":["partial","*",35,5],"PerplexityBot":["partial","*",35,5],"Perplexity-User":["partial","*",35,5],"Google-Extended":["partial","*",35,5],"Applebot-Extended":["partial","*",35,5],"CCBot":["partial","*",35,5],"Bytespider":["partial","*",35,5],"Amazonbot":["partial","*",35,5],"meta-externalagent":["partial","*",35,5],"cohere-ai":["partial","*",35,5],"Diffbot":["partial","*",35,5],"ContextFetchBot":["partial","*",35,5]},"llms":{"status":"html","http":200,"bytes":74647},"home":{"status":"ok","http":200,"url":"https://www.dice.com/","html_tokens":33646,"markdown_tokens":248,"pruned_tokens":203}},
{"rank":7070,"domain":"cryptoslate.com","origin":"https://cryptoslate.com","robots":{"status":"ok","http":200,"url":"https://cryptoslate.com/robots.txt","bytes":173,"groups":1},"bots":{"GPTBot":["partial","*",3,0],"ChatGPT-User":["partial","*",3,0],"OAI-SearchBot":["partial","*",3,0],"ClaudeBot":["partial","*",3,0],"Claude-User":["partial","*",3,0],"Claude-SearchBot":["partial","*",3,0],"anthropic-ai":["partial","*",3,0],"PerplexityBot":["partial","*",3,0],"Perplexity-User":["partial","*",3,0],"Google-Extended":["partial","*",3,0],"Applebot-Extended":["partial","*",3,0],"CCBot":["partial","*",3,0],"Bytespider":["partial","*",3,0],"Amazonbot":["partial","*",3,0],"meta-externalagent":["partial","*",3,0],"cohere-ai":["partial","*",3,0],"Diffbot":["partial","*",3,0],"ContextFetchBot":["partial","*",3,0]},"llms":{"status":"present","http":200,"bytes":10351},"home":{"status":"ok","http":200,"url":"https://cryptoslate.com/","html_tokens":62258,"markdown_tokens":384,"pruned_tokens":31}},
{"rank":7071,"domain":"activehosted.com","origin":"https://activehosted.com","robots":{"status":"missing","http":404,"url":"https://activehosted.com/robots.txt","bytes":341,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":341},"home":{"status":"ok","http":200,"url":"https://www.activecampaign.com/","html_tokens":114729,"markdown_tokens":517,"pruned_tokens":101}},
{"rank":7072,"domain":"fee.org","origin":"https://fee.org","robots":{"status":"ok","http":200,"url":"https://fee.org/robots.txt","bytes":131,"groups":1},"bots":{"GPTBot":["partial","*",1,2],"ChatGPT-User":["partial","*",1,2],"OAI-SearchBot":["partial","*",1,2],"ClaudeBot":["partial","*",1,2],"Claude-User":["partial","*",1,2],"Claude-SearchBot":["partial","*",1,2],"anthropic-ai":["partial","*",1,2],"PerplexityBot":["partial","*",1,2],"Perplexity-User":["partial","*",1,2],"Google-Extended":["partial","*",1,2],"Applebot-Extended":["partial","*",1,2],"CCBot":["partial","*",1,2],"Bytespider":["partial","*",1,2],"Amazonbot":["partial","*",1,2],"meta-externalagent":["partial","*",1,2],"cohere-ai":["partial","*",1,2],"Diffbot":["partial","*",1,2],"ContextFetchBot":["partial","*",1,2]},"llms":{"status":"missing","http":404,"bytes":70068},"home":{"status":"ok","http":200,"url":"https://fee.org/","html_tokens":33892,"markdown_tokens":332,"pruned_tokens":148}},
{"rank":7073,"domain":"google.com.na","origin":"https://google.com.na","robots":{"status":"ok","http":200,"url":"https://www.google.com.na/robots.txt","bytes":6624,"groups":4},"bots":{"GPTBot":["partial","*",169,64],"ChatGPT-User":["partial","*",169,64],"OAI-SearchBot":["partial","*",169,64],"ClaudeBot":["partial","*",169,64],"Claude-User":["partial","*",169,64],"Claude-SearchBot":["partial","*",169,64],"anthropic-ai":["partial","*",169,64],"PerplexityBot":["partial","*",169,64],"Perplexity-User":["partial","*",169,64],"Google-Extended":["partial","*",169,64],"Applebot-Extended":["partial","*",169,64],"CCBot":["partial","*",169,64],"Bytespider":["partial","*",169,64],"Amazonbot":["partial","*",169,64],"meta-externalagent":["partial","*",169,64],"cohere-ai":["partial","*",169,64],"Diffbot":["partial","*",169,64],"ContextFetchBot":["partial","*",169,64]},"llms":{"status":"missing","http":404,"bytes":1569},"home":{"status":"ok","http":200,"url":"https://www.google.com/","html_tokens":21360,"markdown_tokens":30,"pruned_tokens":30}},
{"rank":7074,"domain":"reservix.de","origin":"https://reservix.de","robots":{"status":"refused","http":403,"url":"https://www.reservix.de/robots.txt","bytes":919,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":7075,"domain":"screendaily.com","origin":"https://screendaily.com","robots":{"status":"ok","http":200,"url":"https://www.screendaily.com/robots.txt","bytes":721,"groups":1},"bots":{"GPTBot":["partial","*",27,0],"ChatGPT-User":["partial","*",27,0],"OAI-SearchBot":["partial","*",27,0],"ClaudeBot":["partial","*",27,0],"Claude-User":["partial","*",27,0],"Claude-SearchBot":["partial","*",27,0],"anthropic-ai":["partial","*",27,0],"PerplexityBot":["partial","*",27,0],"Perplexity-User":["partial","*",27,0],"Google-Extended":["partial","*",27,0],"Applebot-Extended":["partial","*",27,0],"CCBot":["partial","*",27,0],"Bytespider":["partial","*",27,0],"Amazonbot":["partial","*",27,0],"meta-externalagent":["partial","*",27,0],"cohere-ai":["partial","*",27,0],"Diffbot":["partial","*",27,0],"ContextFetchBot":["partial","*",27,0]},"llms":{"status":"missing","http":404,"bytes":88581},"home":{"status":"ok","http":200,"url":"https://www.screendaily.com/","html_tokens":43052,"markdown_tokens":94,"pruned_tokens":94}},
{"rank":7076,"domain":"theamericanconservative.com","origin":"https://theamericanconservative.com","robots":{"status":"ok","http":200,"url":"http://www.theamericanconservative.com/robots.txt","bytes":108,"groups":2},"bots":{"GPTBot":["partial","*",2,0],"ChatGPT-User":["partial","*",2,0],"OAI-SearchBot":["partial","*",2,0],"ClaudeBot":["partial","*",2,0],"Claude-User":["partial","*",2,0],"Claude-SearchBot":["partial","*",2,0],"anthropic-ai":["partial","*",2,0],"PerplexityBot":["partial","*",2,0],"Perplexity-User":["partial","*",2,0],"Google-Extended":["partial","*",2,0],"Applebot-Extended":["partial","*",2,0],"CCBot":["partial","*",2,0],"Bytespider":["partial","*",2,0],"Amazonbot":["partial","*",2,0],"meta-externalagent":["partial","*",2,0],"cohere-ai":["partial","*",2,0],"Diffbot":["partial","*",2,0],"ContextFetchBot":["partial","*",2,0]},"llms":{"status":"http_error","http":403,"bytes":5602},"home":{"status":"http_error","http":403,"url":"https://theamericanconservative.com/"}},
{"rank":7077,"domain":"chipotle.com","origin":"https://chipotle.com","robots":{"status":"ok","http":200,"url":"https://chipotle.com/robots.txt","bytes":122,"groups":1},"bots":{"GPTBot":["partial","*",1,0],"ChatGPT-User":["partial","*",1,0],"OAI-SearchBot":["partial","*",1,0],"ClaudeBot":["partial","*",1,0],"Claude-User":["partial","*",1,0],"Claude-SearchBot":["partial","*",1,0],"anthropic-ai":["partial","*",1,0],"PerplexityBot":["partial","*",1,0],"Perplexity-User":["partial","*",1,0],"Google-Extended":["partial","*",1,0],"Applebot-Extended":["partial","*",1,0],"CCBot":["partial","*",1,0],"Bytespider":["partial","*",1,0],"Amazonbot":["partial","*",1,0],"meta-externalagent":["partial","*",1,0],"cohere-ai":["partial","*",1,0],"Diffbot":["partial","*",1,0],"ContextFetchBot":["partial","*",1,0]},"llms":{"status":"missing","http":404,"bytes":43083},"home":{"status":"ok","http":200,"url":"https://www.chipotle.com/","html_tokens":17328,"markdown_tokens":108,"pruned_tokens":98}},
{"rank":7078,"domain":"childrenshospital.org","origin":"https://childrenshospital.org","robots":{"status":"ok","http":200,"url":"https://www.childrenshospital.org/robots.txt","bytes":2005,"groups":1},"bots":{"GPTBot":["partial","*",33,18],"ChatGPT-User":["partial","*",33,18],"OAI-SearchBot":["partial","*",33,18],"ClaudeBot":["partial","*",33,18],"Claude-User":["partial","*",33,18],"Claude-SearchBot":["partial","*",33,18],"anthropic-ai":["partial","*",33,18],"PerplexityBot":["partial","*",33,18],"Perplexity-User":["partial","*",33,18],"Google-Extended":["partial","*",33,18],"Applebot-Extended":["partial","*",33,18],"CCBot":["partial","*",33,18],"Bytespider":["partial","*",33,18],"Amazonbot":["partial","*",33,18],"meta-externalagent":["partial","*",33,18],"cohere-ai":["partial","*",33,18],"Diffbot":["partial","*",33,18],"ContextFetchBot":["partial","*",33,18]},"llms":{"status":"missing","http":404,"bytes":411},"home":{"status":"ok","http":200,"url":"https://www.childrenshospital.org/","html_tokens":290,"markdown_tokens":21,"pruned_tokens":21}},
{"rank":7079,"domain":"blogspot.be","origin":"https://blogspot.be","robots":{"status":"ok","http":200,"url":"https://blogspot.be/robots.txt","bytes":6624,"groups":4},"bots":{"GPTBot":["partial","*",169,64],"ChatGPT-User":["partial","*",169,64],"OAI-SearchBot":["partial","*",169,64],"ClaudeBot":["partial","*",169,64],"Claude-User":["partial","*",169,64],"Claude-SearchBot":["partial","*",169,64],"anthropic-ai":["partial","*",169,64],"PerplexityBot":["partial","*",169,64],"Perplexity-User":["partial","*",169,64],"Google-Extended":["partial","*",169,64],"Applebot-Extended":["partial","*",169,64],"CCBot":["partial","*",169,64],"Bytespider":["partial","*",169,64],"Amazonbot":["partial","*",169,64],"meta-externalagent":["partial","*",169,64],"cohere-ai":["partial","*",169,64],"Diffbot":["partial","*",169,64],"ContextFetchBot":["partial","*",169,64]},"llms":{"status":"missing","http":404,"bytes":1569},"home":{"status":"ok","http":200,"url":"https://www.google.com/","html_tokens":21355,"markdown_tokens":30,"pruned_tokens":30}},
{"rank":7080,"domain":"uc.cn","origin":"https://uc.cn","robots":{"status":"html","http":200,"url":"https://www.uc.cn/robots.txt","bytes":142144,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":7081,"domain":"mcu.es","origin":"https://www.mcu.es","robots":{"status":"refused","http":429,"url":"https://www.mcu.es/robots.txt","bytes":267,"groups":0,"error":"HTTP 429"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":7082,"domain":"newzealand.com","origin":"https://newzealand.com","robots":{"status":"refused","http":403,"url":"https://www.newzealand.com/robots.txt","bytes":386,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":7083,"domain":"ncku.edu.tw","origin":"https://ncku.edu.tw","robots":{"status":"ok","http":200,"url":"https://ncku.edu.tw/robots.txt","bytes":55,"groups":1},"bots":{"GPTBot":["partial","*",1,2],"ChatGPT-User":["partial","*",1,2],"OAI-SearchBot":["partial","*",1,2],"ClaudeBot":["partial","*",1,2],"Claude-User":["partial","*",1,2],"Claude-SearchBot":["partial","*",1,2],"anthropic-ai":["partial","*",1,2],"PerplexityBot":["partial","*",1,2],"Perplexity-User":["partial","*",1,2],"Google-Extended":["partial","*",1,2],"Applebot-Extended":["partial","*",1,2],"CCBot":["partial","*",1,2],"Bytespider":["partial","*",1,2],"Amazonbot":["partial","*",1,2],"meta-externalagent":["partial","*",1,2],"cohere-ai":["partial","*",1,2],"Diffbot":["partial","*",1,2],"ContextFetchBot":["partial","*",1,2]},"llms":{"status":"disallowed"},"home":{"status":"ok","http":200,"url":"https://ncku.edu.tw/","html_tokens":322,"markdown_tokens":5,"pruned_tokens":5}},
{"rank":7084,"domain":"trmilkandhoneyatl.com","origin":"https://trmilkandhoneyatl.com","robots":{"status":"ok","http":200,"url":"https://www.trmilkandhoneyatl.com/robots.txt","bytes":115,"groups":1},"bots":{"GPTBot":["partial","*",2,1],"ChatGPT-User":["partial","*",2,1],"OAI-SearchBot":["partial","*",2,1],"ClaudeBot":["partial","*",2,1],"Claude-User":["partial","*",2,1],"Claude-SearchBot":["partial","*",2,1],"anthropic-ai":["partial","*",2,1],"PerplexityBot":["partial","*",2,1],"Perplexity-User":["partial","*",2,1],"Google-Extended":["partial","*",2,1],"Applebot-Extended":["partial","*",2,1],"CCBot":["partial","*",2,1],"Bytespider":["partial","*",2,1],"Amazonbot":["partial","*",2,1],"meta-externalagent":["partial","*",2,1],"cohere-ai":["partial","*",2,1],"Diffbot":["partial","*",2,1],"ContextFetchBot":["partial","*",2,1]},"llms":{"status":"missing","http":404,"bytes":1183},"home":{"status":"ok","http":200,"url":"https://www.trmilkandhoneyatl.com/","html_tokens":3092,"markdown_tokens":222,"pruned_tokens":119}},
{"rank":7085,"domain":"bmo.com","origin":"https://bmo.com","robots":{"status":"unreachable","http":null,"url":"https://bmo.com/robots.txt","bytes":0,"groups":0,"error":"<StreamReset stream_id:1, error_code:2, remote_reset:True>"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":7086,"domain":"no-ip.com","origin":"https://no-ip.com","robots":{"status":"ok","http":200,"url":"https://www.noip.com/robots.txt","bytes":611,"groups":1},"bots":{"GPTBot":["partial","*",18,1],"ChatGPT-User":["partial","*",18,1],"OAI-SearchBot":["partial","*",18,1],"ClaudeBot":["partial","*",18,1],"Claude-User":["partial","*",18,1],"Claude-SearchBot":["partial","*",18,1],"anthropic-ai":["partial","*",18,1],"PerplexityBot":["partial","*",18,1],"Perplexity-User":["partial","*",18,1],"Google-Extended":["partial","*",18,1],"Applebot-Extended":["partial","*",18,1],"CCBot":["partial","*",18,1],"Bytespider":["partial","*",18,1],"Amazonbot":["partial","*",18,1],"meta-externalagent":["partial","*",18,1],"cohere-ai":["partial","*",18,1],"Diffbot":["partial","*",18,1],"ContextFetchBot":["partial","*",18,1]},"llms":{"status":"present","http":200,"bytes":8953},"home":{"status":"ok","http":200,"url":"https://www.noip.com/","html_tokens":49052,"markdown_tokens":633,"pruned_tokens":175}},
{"rank":7087,"domain":"pnc.com","origin":"https://pnc.com","robots":{"status":"ok","http":200,"url":"https://www.pnc.com/robots.txt","bytes":911,"groups":1},"bots":{"GPTBot":["partial","*",7,1],"ChatGPT-User":["partial","*",7,1],"OAI-SearchBot":["partial","*",7,1],"ClaudeBot":["partial","*",7,1],"Claude-User":["partial","*",7,1],"Claude-SearchBot":["partial","*",7,1],"anthropic-ai":["partial","*",7,1],"PerplexityBot":["partial","*",7,1],"Perplexity-User":["partial","*",7,1],"Google-Extended":["partial","*",7,1],"Applebot-Extended":["partial","*",7,1],"CCBot":["partial","*",7,1],"Bytespider":["partial","*",7,1],"Amazonbot":["partial","*",7,1],"meta-externalagent":["partial","*",7,1],"cohere-ai":["partial","*",7,1],"Diffbot":["partial","*",7,1],"ContextFetchBot":["partial","*",7,1]},"llms":{"status":"missing","http":404,"bytes":1208},"home":{"status":"ok","http":200,"url":"https://www.pnc.com/en/personal-banking.html","html_tokens":52171,"markdown_tokens":510,"pruned_tokens":173}},
{"rank":7088,"domain":"eatthis.com","origin":"https://eatthis.com","robots":{"status":"ok","http":200,"url":"https://eatthis.com/robots.txt","bytes":2676,"groups":32},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["allowed","*",0,1],"OAI-SearchBot":["allowed","*",0,1],"ClaudeBot":["blocked","named",1,0],"Claude-User":["allowed","*",0,1],"Claude-SearchBot":["allowed","*",0,1],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["allowed","*",0,1],"Perplexity-User":["allowed","*",0,1],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["allowed","*",0,1]},"llms":{"status":"missing","http":404,"bytes":102907},"home":{"status":"http_error","http":403,"url":"https://eatthis.com/"}},
{"rank":7089,"domain":"upworthy.com","origin":"https://upworthy.com","robots":{"status":"ok","http":200,"url":"https://www.upworthy.com/robots.txt","bytes":362,"groups":2},"bots":{"GPTBot":["partial","*",2,0],"ChatGPT-User":["partial","*",2,0],"OAI-SearchBot":["partial","*",2,0],"ClaudeBot":["partial","*",2,0],"Claude-User":["partial","*",2,0],"Claude-SearchBot":["partial","*",2,0],"anthropic-ai":["partial","*",2,0],"PerplexityBot":["partial","*",2,0],"Perplexity-User":["partial","*",2,0],"Google-Extended":["partial","*",2,0],"Applebot-Extended":["partial","*",2,0],"CCBot":["partial","*",2,0],"Bytespider":["partial","*",2,0],"Amazonbot":["partial","*",2,0],"meta-externalagent":["partial","*",2,0],"cohere-ai":["partial","*",2,0],"Diffbot":["partial","*",2,0],"ContextFetchBot":["partial","*",2,0]},"llms":{"status":"missing","http":404,"bytes":154990},"home":{"status":"ok","http":200,"url":"https://www.upworthy.com/","html_tokens":73357,"markdown_tokens":68,"pruned_tokens":68}},
{"rank":7090,"domain":"counterpointresearch.com","origin":"https://counterpointresearch.com","robots":{"status":"ok","http":200,"url":"https://counterpointresearch.com/robots.txt","bytes":1312,"groups":20},"bots":{"GPTBot":["partial","named",1,1],"ChatGPT-User":["partial","named",1,1],"OAI-SearchBot":["partial","named",1,1],"ClaudeBot":["partial","named",1,1],"Claude-User":["partial","named",1,1],"Claude-SearchBot":["partial","named",1,1],"anthropic-ai":["partial","*",1,1],"PerplexityBot":["partial","named",1,1],"Perplexity-User":["partial","named",1,1],"Google-Extended":["partial","named",1,1],"Applebot-Extended":["partial","named",1,1],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["partial","named",1,1],"meta-externalagent":["partial","named",1,1],"cohere-ai":["partial","named",1,1],"Diffbot":["partial","*",1,1],"ContextFetchBot":["partial","*",1,1]},"llms":{"status":"present","http":200,"bytes":1123},"home":{"status":"http_error","http":403,"url":"https://counterpointresearch.com/"}},
{"rank":7091,"domain":"framaforms.org","origin":"https://framaforms.org","robots":{"status":"ok","http":200,"url":"https://framaforms.org/robots.txt","bytes":2965,"groups":2},"bots":{"GPTBot":["partial","*",46,32],"ChatGPT-User":["partial","*",46,32],"OAI-SearchBot":["partial","*",46,32],"ClaudeBot":["partial","*",46,32],"Claude-User":["partial","*",46,32],"Claude-SearchBot":["partial","*",46,32],"anthropic-ai":["partial","*",46,32],"PerplexityBot":["partial","*",46,32],"Perplexity-User":["partial","*",46,32],"Google-Extended":["partial","*",46,32],"Applebot-Extended":["partial","*",46,32],"CCBot":["partial","*",46,32],"Bytespider":["partial","*",46,32],"Amazonbot":["partial","*",46,32],"meta-externalagent":["partial","*",46,32],"cohere-ai":["partial","*",46,32],"Diffbot":["partial","*",46,32],"ContextFetchBot":["partial","*",46,32]},"llms":{"status":"missing","http":404,"bytes":291},"home":{"status":"ok","http":200,"url":"https://framaforms.org/abc/","html_tokens":13064,"markdown_tokens":775,"pruned_tokens":350}},
{"rank":7092,"domain":"jinshuju.net","origin":"https://jinshuju.net","robots":{"status":"ok","http":200,"url":"https://jinshuju.net/robots.txt","bytes":428,"groups":1},"bots":{"GPTBot":["partial","*",7,0],"ChatGPT-User":["partial","*",7,0],"OAI-SearchBot":["partial","*",7,0],"ClaudeBot":["partial","*",7,0],"Claude-User":["partial","*",7,0],"Claude-SearchBot":["partial","*",7,0],"anthropic-ai":["partial","*",7,0],"PerplexityBot":["partial","*",7,0],"Perplexity-User":["partial","*",7,0],"Google-Extended":["partial","*",7,0],"Applebot-Extended":["partial","*",7,0],"CCBot":["partial","*",7,0],"Bytespider":["partial","*",7,0],"Amazonbot":["partial","*",7,0],"meta-externalagent":["partial","*",7,0],"cohere-ai":["partial","*",7,0],"Diffbot":["partial","*",7,0],"ContextFetchBot":["partial","*",7,0]},"llms":{"status":"other_text","http":200,"bytes":10509},"home":{"status":"ok","http":200,"url":"https://jinshuju.net/","html_tokens":157031,"markdown_tokens":20,"pruned_tokens":20}},
{"rank":7093,"domain":"vwo.com","origin":"https://vwo.com","robots":{"status":"ok","http":200,"url":"https://wingify.com/robots.txt","bytes":1856,"groups":34},"bots":{"GPTBot":["allowed","named",0,1],"ChatGPT-User":["allowed","named",0,1],"OAI-SearchBot":["allowed","named",0,1],"ClaudeBot":["allowed","named",0,1],"Claude-User":["allowed","named",0,1],"Claude-SearchBot":["allowed","named",0,1],"anthropic-ai":["partial","*",2,0],"PerplexityBot":["allowed","named",0,1],"Perplexity-User":["allowed","named",0,1],"Google-Extended":["allowed","named",0,1],"Applebot-Extended":["allowed","named",0,1],"CCBot":["partial","named",15,1],"Bytespider":["allowed","named",0,1],"Amazonbot":["allowed","named",0,1],"meta-externalagent":["allowed","named",0,2],"cohere-ai":["allowed","named",0,1],"Diffbot":["allowed","named",0,1],"ContextFetchBot":["partial","*",2,0]},"llms":{"status":"present","http":200,"bytes":87650},"home":{"status":"ok","http":200,"url":"https://wingify.com/","html_tokens":163926,"markdown_tokens":1670,"pruned_tokens":148}},
{"rank":7094,"domain":"pap.pl","origin":"https://pap.pl","robots":{"status":"ok","http":200,"url":"https://www.pap.pl/robots.txt","bytes":1402,"groups":2},"bots":{"GPTBot":["partial","named",1,2],"ChatGPT-User":["partial","named",1,2],"OAI-SearchBot":["partial","named",1,2],"ClaudeBot":["partial","named",1,2],"Claude-User":["partial","named",1,2],"Claude-SearchBot":["partial","named",1,2],"anthropic-ai":["partial","*",30,0],"PerplexityBot":["partial","named",1,2],"Perplexity-User":["partial","named",1,2],"Google-Extended":["partial","*",30,0],"Applebot-Extended":["partial","*",30,0],"CCBot":["partial","named",1,2],"Bytespider":["partial","named",1,2],"Amazonbot":["partial","named",1,2],"meta-externalagent":["partial","named",1,2],"cohere-ai":["partial","*",30,0],"Diffbot":["partial","*",30,0],"ContextFetchBot":["partial","*",30,0]},"llms":{"status":"missing","http":404,"bytes":317},"home":{"status":"ok","http":200,"url":"https://www.pap.pl/","html_tokens":53,"markdown_tokens":0,"pruned_tokens":0}},
{"rank":7095,"domain":"edgesuite.net","origin":"https://edgesuite.net","robots":{"status":"unreachable","http":null,"url":"https://edgesuite.net/robots.txt","bytes":0,"groups":0,"error":"cannot resolve edgesuite.net"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":7096,"domain":"webofknowledge.com","origin":"https://webofknowledge.com","robots":{"status":"html","http":200,"url":"https://www.webofscience.com/wos/","bytes":3063,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":7097,"domain":"brownbook.net","origin":"https://brownbook.net","robots":{"status":"ok","http":200,"url":"https://www.brownbook.net/robots.txt","bytes":1452,"groups":4},"bots":{"GPTBot":["allowed","named",0,1],"ChatGPT-User":["partial","*",39,1],"OAI-SearchBot":["partial","*",39,1],"ClaudeBot":["allowed","named",0,1],"Claude-User":["partial","*",39,1],"Claude-SearchBot":["partial","*",39,1],"anthropic-ai":["partial","*",39,1],"PerplexityBot":["partial","*",39,1],"Perplexity-User":["partial","*",39,1],"Google-Extended":["partial","*",39,1],"Applebot-Extended":["partial","*",39,1],"CCBot":["partial","*",39,1],"Bytespider":["partial","*",39,1],"Amazonbot":["partial","*",39,1],"meta-externalagent":["partial","*",39,1],"cohere-ai":["partial","*",39,1],"Diffbot":["partial","*",39,1],"ContextFetchBot":["partial","*",39,1]},"llms":{"status":"present","http":200,"bytes":962},"home":{"status":"ok","http":200,"url":"https://www.brownbook.net/","html_tokens":104821,"markdown_tokens":65,"pruned_tokens":65}},
{"rank":7098,"domain":"religionnews.com","origin":"https://religionnews.com","robots":{"status":"ok","http":200,"url":"https://religionnews.com/robots.txt","bytes":514,"groups":2},"bots":{"GPTBot":["partial","*",9,1],"ChatGPT-User":["partial","*",9,1],"OAI-SearchBot":["partial","*",9,1],"ClaudeBot":["partial","*",9,1],"Claude-User":["partial","*",9,1],"Claude-SearchBot":["partial","*",9,1],"anthropic-ai":["partial","*",9,1],"PerplexityBot":["partial","*",9,1],"Perplexity-User":["partial","*",9,1],"Google-Extended":["partial","*",9,1],"Applebot-Extended":["partial","*",9,1],"CCBot":["partial","*",9,1],"Bytespider":["partial","*",9,1],"Amazonbot":["partial","*",9,1],"meta-externalagent":["partial","*",9,1],"cohere-ai":["partial","*",9,1],"Diffbot":["partial","*",9,1],"ContextFetchBot":["partial","*",9,1]},"llms":{"status":"missing","http":404,"bytes":180956},"home":{"status":"ok","http":200,"url":"https://religionnews.com/","html_tokens":65787,"markdown_tokens":71,"pruned_tokens":71}},
{"rank":7099,"domain":"zurb.com","origin":"https://zurb.com","robots":{"status":"ok","http":200,"url":"https://zurb.com/robots.txt","bytes":739,"groups":1},"bots":{"GPTBot":["partial","*",8,0],"ChatGPT-User":["partial","*",8,0],"OAI-SearchBot":["partial","*",8,0],"ClaudeBot":["partial","*",8,0],"Claude-User":["partial","*",8,0],"Claude-SearchBot":["partial","*",8,0],"anthropic-ai":["partial","*",8,0],"PerplexityBot":["partial","*",8,0],"Perplexity-User":["partial","*",8,0],"Google-Extended":["partial","*",8,0],"Applebot-Extended":["partial","*",8,0],"CCBot":["partial","*",8,0],"Bytespider":["partial","*",8,0],"Amazonbot":["partial","*",8,0],"meta-externalagent":["partial","*",8,0],"cohere-ai":["partial","*",8,0],"Diffbot":["partial","*",8,0],"ContextFetchBot":["partial","*",8,0]},"llms":{"status":"http_error","http":500,"bytes":182},"home":{"status":"ok","http":200,"url":"https://zurb.com/","html_tokens":16553,"markdown_tokens":336,"pruned_tokens":261}},
{"rank":7100,"domain":"uni-paderborn.de","origin":"https://uni-paderborn.de","robots":{"status":"missing","http":404,"url":"https://www.uni-paderborn.de/robots.txt","bytes":61066,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":61066},"home":{"status":"ok","http":200,"url":"https://www.uni-paderborn.de/","html_tokens":37692,"markdown_tokens":509,"pruned_tokens":436}},
{"rank":7101,"domain":"minds.com","origin":"https://minds.com","robots":{"status":"ok","http":200,"url":"https://www.minds.com/robots.txt","bytes":141,"groups":1},"bots":{"GPTBot":["partial","*",1,2],"ChatGPT-User":["partial","*",1,2],"OAI-SearchBot":["partial","*",1,2],"ClaudeBot":["partial","*",1,2],"Claude-User":["partial","*",1,2],"Claude-SearchBot":["partial","*",1,2],"anthropic-ai":["partial","*",1,2],"PerplexityBot":["partial","*",1,2],"Perplexity-User":["partial","*",1,2],"Google-Extended":["partial","*",1,2],"Applebot-Extended":["partial","*",1,2],"CCBot":["partial","*",1,2],"Bytespider":["partial","*",1,2],"Amazonbot":["partial","*",1,2],"meta-externalagent":["partial","*",1,2],"cohere-ai":["partial","*",1,2],"Diffbot":["partial","*",1,2],"ContextFetchBot":["partial","*",1,2]},"llms":{"status":"missing","http":404,"bytes":326791},"home":{"status":"ok","http":200,"url":"https://www.minds.com/","html_tokens":75980,"markdown_tokens":320,"pruned_tokens":106}},
{"rank":7102,"domain":"threadreaderapp.com","origin":"https://threadreaderapp.com","robots":{"status":"ok","http":200,"url":"https://threadreaderapp.com/robots.txt","bytes":483,"groups":1},"bots":{"GPTBot":["partial","*",18,0],"ChatGPT-User":["partial","*",18,0],"OAI-SearchBot":["partial","*",18,0],"ClaudeBot":["partial","*",18,0],"Claude-User":["partial","*",18,0],"Claude-SearchBot":["partial","*",18,0],"anthropic-ai":["partial","*",18,0],"PerplexityBot":["partial","*",18,0],"Perplexity-User":["partial","*",18,0],"Google-Extended":["partial","*",18,0],"Applebot-Extended":["partial","*",18,0],"CCBot":["partial","*",18,0],"Bytespider":["partial","*",18,0],"Amazonbot":["partial","*",18,0],"meta-externalagent":["partial","*",18,0],"cohere-ai":["partial","*",18,0],"Diffbot":["partial","*",18,0],"ContextFetchBot":["partial","*",18,0]},"llms":{"status":"missing","http":404,"bytes":19457},"home":{"status":"ok","http":200,"url":"https://threadreaderapp.com/","html_tokens":13480,"markdown_tokens":86,"pruned_tokens":86}},
{"rank":7103,"domain":"unige.it","origin":"https://unige.it","robots":{"status":"ok","http":200,"url":"https://unige.it/robots.txt","bytes":2063,"groups":1},"bots":{"GPTBot":["partial","*",34,20],"ChatGPT-User":["partial","*",34,20],"OAI-SearchBot":["partial","*",34,20],"ClaudeBot":["partial","*",34,20],"Claude-User":["partial","*",34,20],"Claude-SearchBot":["partial","*",34,20],"anthropic-ai":["partial","*",34,20],"PerplexityBot":["partial","*",34,20],"Perplexity-User":["partial","*",34,20],"Google-Extended":["partial","*",34,20],"Applebot-Extended":["partial","*",34,20],"CCBot":["partial","*",34,20],"Bytespider":["partial","*",34,20],"Amazonbot":["partial","*",34,20],"meta-externalagent":["partial","*",34,20],"cohere-ai":["partial","*",34,20],"Diffbot":["partial","*",34,20],"ContextFetchBot":["partial","*",34,20]},"llms":{"status":"present","http":200,"bytes":8768},"home":{"status":"ok","http":200,"url":"https://unige.it/","html_tokens":29136,"markdown_tokens":74,"pruned_tokens":74}},
{"rank":7104,"domain":"straightdope.com","origin":"https://straightdope.com","robots":{"status":"ok","http":200,"url":"https://straightdope.com/robots.txt","bytes":1007,"groups":3},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["blocked","named",1,0],"OAI-SearchBot":["partial","*",2,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",2,0],"Claude-SearchBot":["partial","*",2,0],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["partial","*",2,0],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",2,0]},"llms":{"status":"missing","http":404,"bytes":0},"home":{"status":"ok","http":200,"url":"https://straightdope.com/","html_tokens":41665,"markdown_tokens":472,"pruned_tokens":174}},
{"rank":7105,"domain":"washingtonmonthly.com","origin":"https://washingtonmonthly.com","robots":{"status":"ok","http":200,"url":"https://washingtonmonthly.com/robots.txt","bytes":286,"groups":1},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"missing","http":404,"bytes":194637},"home":{"status":"ok","http":200,"url":"https://washingtonmonthly.com/","html_tokens":87610,"markdown_tokens":117,"pruned_tokens":102}},
{"rank":7106,"domain":"photo.net","origin":"https://photo.net","robots":{"status":"ok","http":200,"url":"https://www.photo.net/robots.txt","bytes":2676,"groups":32},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["allowed","*",0,1],"OAI-SearchBot":["allowed","*",0,1],"ClaudeBot":["blocked","named",1,0],"Claude-User":["allowed","*",0,1],"Claude-SearchBot":["allowed","*",0,1],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["allowed","*",0,1],"Perplexity-User":["allowed","*",0,1],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["allowed","*",0,1]},"llms":{"status":"http_error","http":403,"bytes":5524},"home":{"status":"http_error","http":403,"url":"https://www.photo.net/"}},
{"rank":7107,"domain":"apartments.com","origin":"https://apartments.com","robots":{"status":"refused","http":403,"url":"https://apartments.com/robots.txt","bytes":378,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":7108,"domain":"acl.gov","origin":"https://acl.gov","robots":{"status":"ok","http":200,"url":"https://acl.gov/robots.txt","bytes":2027,"groups":1},"bots":{"GPTBot":["partial","*",34,18],"ChatGPT-User":["partial","*",34,18],"OAI-SearchBot":["partial","*",34,18],"ClaudeBot":["partial","*",34,18],"Claude-User":["partial","*",34,18],"Claude-SearchBot":["partial","*",34,18],"anthropic-ai":["partial","*",34,18],"PerplexityBot":["partial","*",34,18],"Perplexity-User":["partial","*",34,18],"Google-Extended":["partial","*",34,18],"Applebot-Extended":["partial","*",34,18],"CCBot":["partial","*",34,18],"Bytespider":["partial","*",34,18],"Amazonbot":["partial","*",34,18],"meta-externalagent":["partial","*",34,18],"cohere-ai":["partial","*",34,18],"Diffbot":["partial","*",34,18],"ContextFetchBot":["partial","*",34,18]},"llms":{"status":"missing","http":404,"bytes":180},"home":{"status":"ok","http":200,"url":"https://acl.gov/","html_tokens":21273,"markdown_tokens":135,"pruned_tokens":52}},
{"rank":7109,"domain":"bsh-group.com","origin":"https://bsh-group.com","robots":{"status":"ok","http":200,"url":"https://www.bsh-group.com/robots.txt","bytes":1267,"groups":1},"bots":{"GPTBot":["partial","*",3,4],"ChatGPT-User":["partial","*",3,4],"OAI-SearchBot":["partial","*",3,4],"ClaudeBot":["partial","*",3,4],"Claude-User":["partial","*",3,4],"Claude-SearchBot":["partial","*",3,4],"anthropic-ai":["partial","*",3,4],"PerplexityBot":["partial","*",3,4],"Perplexity-User":["partial","*",3,4],"Google-Extended":["partial","*",3,4],"Applebot-Extended":["partial","*",3,4],"CCBot":["partial","*",3,4],"Bytespider":["partial","*",3,4],"Amazonbot":["partial","*",3,4],"meta-externalagent":["partial","*",3,4],"cohere-ai":["partial","*",3,4],"Diffbot":["partial","*",3,4],"ContextFetchBot":["partial","*",3,4]},"llms":{"status":"missing","http":404,"bytes":242500},"home":{"status":"ok","http":200,"url":"https://www.bsh-group.com/","html_tokens":88139,"markdown_tokens":73,"pruned_tokens":69}},
{"rank":7110,"domain":"omron.com","origin":"https://www.omron.com","robots":{"status":"refused","http":403,"url":"https://www.omron.com/robots.txt","bytes":381,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":7111,"domain":"linuxserver.io","origin":"https://linuxserver.io","robots":{"status":"ok","http":200,"url":"https://www.linuxserver.io/robots.txt","bytes":379,"groups":1},"bots":{"GPTBot":["partial","*",13,7],"ChatGPT-User":["partial","*",13,7],"OAI-SearchBot":["partial","*",13,7],"ClaudeBot":["partial","*",13,7],"Claude-User":["partial","*",13,7],"Claude-SearchBot":["partial","*",13,7],"anthropic-ai":["partial","*",13,7],"PerplexityBot":["partial","*",13,7],"Perplexity-User":["partial","*",13,7],"Google-Extended":["partial","*",13,7],"Applebot-Extended":["partial","*",13,7],"CCBot":["partial","*",13,7],"Bytespider":["partial","*",13,7],"Amazonbot":["partial","*",13,7],"meta-externalagent":["partial","*",13,7],"cohere-ai":["partial","*",13,7],"Diffbot":["partial","*",13,7],"ContextFetchBot":["partial","*",13,7]},"llms":{"status":"missing","http":404,"bytes":43371},"home":{"status":"ok","http":200,"url":"https://www.linuxserver.io/","html_tokens":12608,"markdown_tokens":518,"pruned_tokens":88}},
{"rank":7112,"domain":"google.com.np","origin":"https://google.com.np","robots":{"status":"ok","http":200,"url":"https://www.google.com.np/robots.txt","bytes":6624,"groups":4},"bots":{"GPTBot":["partial","*",169,64],"ChatGPT-User":["partial","*",169,64],"OAI-SearchBot":["partial","*",169,64],"ClaudeBot":["partial","*",169,64],"Claude-User":["partial","*",169,64],"Claude-SearchBot":["partial","*",169,64],"anthropic-ai":["partial","*",169,64],"PerplexityBot":["partial","*",169,64],"Perplexity-User":["partial","*",169,64],"Google-Extended":["partial","*",169,64],"Applebot-Extended":["partial","*",169,64],"CCBot":["partial","*",169,64],"Bytespider":["partial","*",169,64],"Amazonbot":["partial","*",169,64],"meta-externalagent":["partial","*",169,64],"cohere-ai":["partial","*",169,64],"Diffbot":["partial","*",169,64],"ContextFetchBot":["partial","*",169,64]},"llms":{"status":"missing","http":404,"bytes":1569},"home":{"status":"ok","http":200,"url":"https://www.google.com/","html_tokens":21302,"markdown_tokens":30,"pruned_tokens":30}},
{"rank":7113,"domain":"airnow.gov","origin":"https://www.airnow.gov","robots":{"status":"missing","http":404,"url":"https://www.airnow.gov/robots.txt","bytes":51473,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":51473},"home":{"status":"ok","http":200,"url":"https://www.airnow.gov/","html_tokens":26302,"markdown_tokens":45,"pruned_tokens":45}},
{"rank":7114,"domain":"mb.com.ph","origin":"https://mb.com.ph","robots":{"status":"refused","http":403,"url":"https://mb.com.ph/robots.txt","bytes":5504,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":7115,"domain":"firenze.it","origin":"https://firenze.it","robots":{"status":"unreachable","http":null,"url":"https://firenze.it/robots.txt","bytes":0,"groups":0,"error":"cannot resolve firenze.it"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":7116,"domain":"kontur.ru","origin":"https://kontur.ru","robots":{"status":"ok","http":200,"url":"https://kontur.ru/robots.txt","bytes":6059,"groups":2},"bots":{"GPTBot":["partial","*",67,1],"ChatGPT-User":["partial","*",67,1],"OAI-SearchBot":["partial","*",67,1],"ClaudeBot":["partial","*",67,1],"Claude-User":["partial","*",67,1],"Claude-SearchBot":["partial","*",67,1],"anthropic-ai":["partial","*",67,1],"PerplexityBot":["partial","*",67,1],"Perplexity-User":["partial","*",67,1],"Google-Extended":["partial","*",67,1],"Applebot-Extended":["partial","*",67,1],"CCBot":["partial","*",67,1],"Bytespider":["partial","*",67,1],"Amazonbot":["partial","*",67,1],"meta-externalagent":["partial","*",67,1],"cohere-ai":["partial","*",67,1],"Diffbot":["partial","*",67,1],"ContextFetchBot":["partial","*",67,1]},"llms":{"status":"present","http":200,"bytes":41018},"home":{"status":"ok","http":200,"url":"https://kontur.ru/","html_tokens":35095,"markdown_tokens":38,"pruned_tokens":38}},
{"rank":7117,"domain":"finextra.com","origin":"https://finextra.com","robots":{"status":"ok","http":200,"url":"https://www.finextra.com/robots.txt","bytes":3543,"groups":6},"bots":{"GPTBot":["partial","*",15,0],"ChatGPT-User":["partial","*",15,0],"OAI-SearchBot":["partial","*",15,0],"ClaudeBot":["partial","*",15,0],"Claude-User":["partial","*",15,0],"Claude-SearchBot":["partial","*",15,0],"anthropic-ai":["partial","*",15,0],"PerplexityBot":["partial","*",15,0],"Perplexity-User":["partial","*",15,0],"Google-Extended":["partial","*",15,0],"Applebot-Extended":["partial","*",15,0],"CCBot":["partial","*",15,0],"Bytespider":["partial","*",15,0],"Amazonbot":["partial","*",15,0],"meta-externalagent":["partial","*",15,0],"cohere-ai":["partial","*",15,0],"Diffbot":["partial","*",15,0],"ContextFetchBot":["partial","*",15,0]},"llms":{"status":"missing","http":404,"bytes":5223},"home":{"status":"http_error","http":403,"url":"https://finextra.com/"}},
{"rank":7118,"domain":"salvationarmy.org","origin":"https://salvationarmy.org","robots":{"status":"ok","http":200,"url":"https://www.salvationarmy.org/robots.txt","bytes":2027,"groups":1},"bots":{"GPTBot":["partial","*",34,18],"ChatGPT-User":["partial","*",34,18],"OAI-SearchBot":["partial","*",34,18],"ClaudeBot":["partial","*",34,18],"Claude-User":["partial","*",34,18],"Claude-SearchBot":["partial","*",34,18],"anthropic-ai":["partial","*",34,18],"PerplexityBot":["partial","*",34,18],"Perplexity-User":["partial","*",34,18],"Google-Extended":["partial","*",34,18],"Applebot-Extended":["partial","*",34,18],"CCBot":["partial","*",34,18],"Bytespider":["partial","*",34,18],"Amazonbot":["partial","*",34,18],"meta-externalagent":["partial","*",34,18],"cohere-ai":["partial","*",34,18],"Diffbot":["partial","*",34,18],"ContextFetchBot":["partial","*",34,18]},"llms":{"status":"missing","http":404,"bytes":146},"home":{"status":"ok","http":200,"url":"https://www.salvationarmy.org/","html_tokens":14091,"markdown_tokens":71,"pruned_tokens":71}},
{"rank":7119,"domain":"iqair.com","origin":"https://iqair.com","robots":{"status":"refused","http":429,"url":"https://iqair.com/robots.txt","bytes":34222,"groups":0,"error":"HTTP 429"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":7120,"domain":"yankodesign.com","origin":"https://yankodesign.com","robots":{"status":"ok","http":200,"url":"https://www.yankodesign.com/robots.txt","bytes":226,"groups":1},"bots":{"GPTBot":["partial","*",1,1],"ChatGPT-User":["partial","*",1,1],"OAI-SearchBot":["partial","*",1,1],"ClaudeBot":["partial","*",1,1],"Claude-User":["partial","*",1,1],"Claude-SearchBot":["partial","*",1,1],"anthropic-ai":["partial","*",1,1],"PerplexityBot":["partial","*",1,1],"Perplexity-User":["partial","*",1,1],"Google-Extended":["partial","*",1,1],"Applebot-Extended":["partial","*",1,1],"CCBot":["partial","*",1,1],"Bytespider":["partial","*",1,1],"Amazonbot":["partial","*",1,1],"meta-externalagent":["partial","*",1,1],"cohere-ai":["partial","*",1,1],"Diffbot":["partial","*",1,1],"ContextFetchBot":["partial","*",1,1]},"llms":{"status":"other_text","http":200,"bytes":603593},"home":{"status":"ok","http":200,"url":"https://www.yankodesign.com/","html_tokens":33286,"markdown_tokens":643,"pruned_tokens":145}},
{"rank":7121,"domain":"777playslots.com","origin":"https://777playslots.com","robots":{"status":"ok","http":200,"url":"https://777playslots.com/robots.txt","bytes":486,"groups":2},"bots":{"GPTBot":["partial","*",16,0],"ChatGPT-User":["partial","*",16,0],"OAI-SearchBot":["partial","*",16,0],"ClaudeBot":["partial","*",16,0],"Claude-User":["partial","*",16,0],"Claude-SearchBot":["partial","*",16,0],"anthropic-ai":["partial","*",16,0],"PerplexityBot":["partial","*",16,0],"Perplexity-User":["partial","*",16,0],"Google-Extended":["partial","*",16,0],"Applebot-Extended":["partial","*",16,0],"CCBot":["partial","*",16,0],"Bytespider":["partial","*",16,0],"Amazonbot":["partial","*",16,0],"meta-externalagent":["partial","*",16,0],"cohere-ai":["partial","*",16,0],"Diffbot":["partial","*",16,0],"ContextFetchBot":["partial","*",16,0]},"llms":{"status":"missing","http":404,"bytes":17988},"home":{"status":"ok","http":200,"url":"https://777playslots.com/","html_tokens":9542,"markdown_tokens":798,"pruned_tokens":116}},
{"rank":7122,"domain":"razorpay.com","origin":"https://razorpay.com","robots":{"status":"ok","http":200,"url":"https://razorpay.com/robots.txt","bytes":1353,"groups":5},"bots":{"GPTBot":["partial","named",11,1],"ChatGPT-User":["partial","named",3,1],"OAI-SearchBot":["partial","named",3,1],"ClaudeBot":["partial","named",3,1],"Claude-User":["partial","*",10,1],"Claude-SearchBot":["partial","*",10,1],"anthropic-ai":["partial","named",11,1],"PerplexityBot":["partial","named",3,1],"Perplexity-User":["partial","*",10,1],"Google-Extended":["partial","named",11,1],"Applebot-Extended":["partial","*",10,1],"CCBot":["partial","named",11,1],"Bytespider":["blocked","named",1,0],"Amazonbot":["partial","*",10,1],"meta-externalagent":["partial","named",11,1],"cohere-ai":["partial","*",10,1],"Diffbot":["partial","*",10,1],"ContextFetchBot":["partial","*",10,1]},"llms":{"status":"present","http":200,"bytes":19872},"home":{"status":"not_html","error":"unsupported content type text/markdown"}},
{"rank":7123,"domain":"derwesten.de","origin":"https://derwesten.de","robots":{"status":"ok","http":200,"url":"https://www.derwesten.de/robots.txt","bytes":3270,"groups":2},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["blocked","named",1,0],"OAI-SearchBot":["blocked","named",1,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["blocked","named",1,0],"Claude-SearchBot":["blocked","named",1,0],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["blocked","named",1,0],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["partial","*",8,1],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",8,1]},"llms":{"status":"missing","http":404,"bytes":118334},"home":{"status":"ok","http":200,"url":"https://www.derwesten.de/","html_tokens":55184,"markdown_tokens":77,"pruned_tokens":77}},
{"rank":7124,"domain":"urlscan.io","origin":"https://urlscan.io","robots":{"status":"ok","http":200,"url":"https://urlscan.io/robots.txt","bytes":318,"groups":2},"bots":{"GPTBot":["partial","*",11,0],"ChatGPT-User":["partial","*",11,0],"OAI-SearchBot":["partial","*",11,0],"ClaudeBot":["partial","*",11,0],"Claude-User":["partial","*",11,0],"Claude-SearchBot":["partial","*",11,0],"anthropic-ai":["partial","*",11,0],"PerplexityBot":["partial","*",11,0],"Perplexity-User":["partial","*",11,0],"Google-Extended":["partial","*",11,0],"Applebot-Extended":["partial","*",11,0],"CCBot":["partial","*",11,0],"Bytespider":["partial","*",11,0],"Amazonbot":["partial","*",11,0],"meta-externalagent":["partial","*",11,0],"cohere-ai":["partial","*",11,0],"Diffbot":["partial","*",11,0],"ContextFetchBot":["partial","*",11,0]},"llms":{"status":"missing","http":404,"bytes":8673},"home":{"status":"ok","http":200,"url":"https://urlscan.io/","html_tokens":11495,"markdown_tokens":10,"pruned_tokens":10}},
{"rank":7125,"domain":"stardock.com","origin":"https://stardock.com","robots":{"status":"ok","http":200,"url":"https://www.stardock.com/robots.txt","bytes":659,"groups":1},"bots":{"GPTBot":["partial","*",11,0],"ChatGPT-User":["partial","*",11,0],"OAI-SearchBot":["partial","*",11,0],"ClaudeBot":["partial","*",11,0],"Claude-User":["partial","*",11,0],"Claude-SearchBot":["partial","*",11,0],"anthropic-ai":["partial","*",11,0],"PerplexityBot":["partial","*",11,0],"Perplexity-User":["partial","*",11,0],"Google-Extended":["partial","*",11,0],"Applebot-Extended":["partial","*",11,0],"CCBot":["partial","*",11,0],"Bytespider":["partial","*",11,0],"Amazonbot":["partial","*",11,0],"meta-externalagent":["partial","*",11,0],"cohere-ai":["partial","*",11,0],"Diffbot":["partial","*",11,0],"ContextFetchBot":["partial","*",11,0]},"llms":{"status":"missing","http":404,"bytes":28355},"home":{"status":"ok","http":200,"url":"https://www.stardock.com/","html_tokens":21375,"markdown_tokens":147,"pruned_tokens":52}},
{"rank":7126,"domain":"viewsonic.com","origin":"https://viewsonic.com","robots":{"status":"ok","http":200,"url":"https://www.viewsonic.com/robots.txt","bytes":1581,"groups":8},"bots":{"GPTBot":["partial","*",43,1],"ChatGPT-User":["partial","*",43,1],"OAI-SearchBot":["partial","*",43,1],"ClaudeBot":["partial","*",43,1],"Claude-User":["partial","*",43,1],"Claude-SearchBot":["partial","*",43,1],"anthropic-ai":["partial","*",43,1],"PerplexityBot":["partial","*",43,1],"Perplexity-User":["partial","*",43,1],"Google-Extended":["partial","*",43,1],"Applebot-Extended":["partial","*",43,1],"CCBot":["partial","*",43,1],"Bytespider":["partial","*",43,1],"Amazonbot":["partial","*",43,1],"meta-externalagent":["partial","*",43,1],"cohere-ai":["partial","*",43,1],"Diffbot":["partial","*",43,1],"ContextFetchBot":["partial","*",43,1]},"llms":{"status":"missing","http":404,"bytes":215925},"home":{"status":"ok","http":200,"url":"https://www.viewsonic.com/","html_tokens":78091,"markdown_tokens":1112,"pruned_tokens":361}},
{"rank":7127,"domain":"threatpost.com","origin":"https://threatpost.com","robots":{"status":"ok","http":200,"url":"https://threatpost.com/robots.txt","bytes":1589,"groups":3},"bots":{"GPTBot":["partial","*",15,2],"ChatGPT-User":["partial","*",15,2],"OAI-SearchBot":["partial","*",15,2],"ClaudeBot":["partial","*",15,2],"Claude-User":["partial","*",15,2],"Claude-SearchBot":["partial","*",15,2],"anthropic-ai":["partial","*",15,2],"PerplexityBot":["partial","*",15,2],"Perplexity-User":["partial","*",15,2],"Google-Extended":["partial","*",15,2],"Applebot-Extended":["partial","*",15,2],"CCBot":["partial","*",15,2],"Bytespider":["partial","*",15,2],"Amazonbot":["partial","*",15,2],"meta-externalagent":["partial","*",15,2],"cohere-ai":["partial","*",15,2],"Diffbot":["partial","*",15,2],"ContextFetchBot":["partial","*",15,2]},"llms":{"status":"missing","http":404,"bytes":70810},"home":{"status":"ok","http":200,"url":"https://threatpost.com/","html_tokens":40505,"markdown_tokens":185,"pruned_tokens":162}},
{"rank":7128,"domain":"uni-kassel.de","origin":"https://uni-kassel.de","robots":{"status":"ok","http":200,"url":"https://www.uni-kassel.de/robots.txt","bytes":1017,"groups":1},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"missing","http":404,"bytes":236},"home":{"status":"ok","http":200,"url":"https://www.uni-kassel.de/uni/index.html","html_tokens":25160,"markdown_tokens":47,"pruned_tokens":47}},
{"rank":7129,"domain":"dianping.com","origin":"https://dianping.com","robots":{"status":"html","http":200,"url":"https://www.dianping.com/","bytes":2907,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":7130,"domain":"chapman.edu","origin":"https://chapman.edu","robots":{"status":"ok","http":200,"url":"https://www.chapman.edu/robots.txt","bytes":3132,"groups":1},"bots":{"GPTBot":["partial","*",60,0],"ChatGPT-User":["partial","*",60,0],"OAI-SearchBot":["partial","*",60,0],"ClaudeBot":["partial","*",60,0],"Claude-User":["partial","*",60,0],"Claude-SearchBot":["partial","*",60,0],"anthropic-ai":["partial","*",60,0],"PerplexityBot":["partial","*",60,0],"Perplexity-User":["partial","*",60,0],"Google-Extended":["partial","*",60,0],"Applebot-Extended":["partial","*",60,0],"CCBot":["partial","*",60,0],"Bytespider":["partial","*",60,0],"Amazonbot":["partial","*",60,0],"meta-externalagent":["partial","*",60,0],"cohere-ai":["partial","*",60,0],"Diffbot":["partial","*",60,0],"ContextFetchBot":["partial","*",60,0]},"llms":{"status":"html","http":200,"bytes":183213},"home":{"status":"ok","http":200,"url":"https://www.chapman.edu/","html_tokens":51925,"markdown_tokens":769,"pruned_tokens":340}},
{"rank":7131,"domain":"ecfr.eu","origin":"https://ecfr.eu","robots":{"status":"ok","http":200,"url":"https://ecfr.eu/robots.txt","bytes":161,"groups":1},"bots":{"GPTBot":["partial","*",4,1],"ChatGPT-User":["partial","*",4,1],"OAI-SearchBot":["partial","*",4,1],"ClaudeBot":["partial","*",4,1],"Claude-User":["partial","*",4,1],"Claude-SearchBot":["partial","*",4,1],"anthropic-ai":["partial","*",4,1],"PerplexityBot":["partial","*",4,1],"Perplexity-User":["partial","*",4,1],"Google-Extended":["partial","*",4,1],"Applebot-Extended":["partial","*",4,1],"CCBot":["partial","*",4,1],"Bytespider":["partial","*",4,1],"Amazonbot":["partial","*",4,1],"meta-externalagent":["partial","*",4,1],"cohere-ai":["partial","*",4,1],"Diffbot":["partial","*",4,1],"ContextFetchBot":["partial","*",4,1]},"llms":{"status":"missing","http":404,"bytes":72569},"home":{"status":"ok","http":200,"url":"https://ecfr.eu/","html_tokens":53543,"markdown_tokens":1204,"pruned_tokens":219}},
{"rank":7132,"domain":"australianfreepokies.com","origin":"https://australianfreepokies.com","robots":{"status":"ok","http":200,"url":"https://australianfreepokies.com/robots.txt","bytes":552,"groups":1},"bots":{"GPTBot":["partial","*",19,0],"ChatGPT-User":["partial","*",19,0],"OAI-SearchBot":["partial","*",19,0],"ClaudeBot":["partial","*",19,0],"Claude-User":["partial","*",19,0],"Claude-SearchBot":["partial","*",19,0],"anthropic-ai":["partial","*",19,0],"PerplexityBot":["partial","*",19,0],"Perplexity-User":["partial","*",19,0],"Google-Extended":["partial","*",19,0],"Applebot-Extended":["partial","*",19,0],"CCBot":["partial","*",19,0],"Bytespider":["partial","*",19,0],"Amazonbot":["partial","*",19,0],"meta-externalagent":["partial","*",19,0],"cohere-ai":["partial","*",19,0],"Diffbot":["partial","*",19,0],"ContextFetchBot":["partial","*",19,0]},"llms":{"status":"missing","http":404,"bytes":15},"home":{"status":"http_error","http":404,"url":"https://australianfreepokies.com/"}},
{"rank":7133,"domain":"unifr.ch","origin":"https://unifr.ch","robots":{"status":"ok","http":200,"url":"https://www.unifr.ch/robots.txt","bytes":579,"groups":2},"bots":{"GPTBot":["partial","*",9,1],"ChatGPT-User":["partial","*",9,1],"OAI-SearchBot":["partial","*",9,1],"ClaudeBot":["partial","*",9,1],"Claude-User":["partial","*",9,1],"Claude-SearchBot":["partial","*",9,1],"anthropic-ai":["partial","*",9,1],"PerplexityBot":["partial","*",9,1],"Perplexity-User":["partial","*",9,1],"Google-Extended":["partial","*",9,1],"Applebot-Extended":["partial","*",9,1],"CCBot":["partial","*",9,1],"Bytespider":["partial","*",9,1],"Amazonbot":["partial","*",9,1],"meta-externalagent":["partial","*",9,1],"cohere-ai":["partial","*",9,1],"Diffbot":["partial","*",9,1],"ContextFetchBot":["partial","*",9,1]},"llms":{"status":"present","http":200,"bytes":11907},"home":{"status":"ok","http":200,"url":"https://www.unifr.ch/home/en/","html_tokens":15956,"markdown_tokens":107,"pruned_tokens":107}},
{"rank":7134,"domain":"emailverification.info","origin":"https://emailverification.info","robots":{"status":"html","http":200,"url":"https://emailverification.info/","bytes":3559,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":7135,"domain":"ipbes.net","origin":"https://ipbes.net","robots":{"status":"ok","http":200,"url":"https://www.ipbes.net/robots.txt","bytes":2464,"groups":2},"bots":{"GPTBot":["partial","*",35,18],"ChatGPT-User":["partial","*",35,18],"OAI-SearchBot":["partial","*",35,18],"ClaudeBot":["partial","*",35,18],"Claude-User":["partial","*",35,18],"Claude-SearchBot":["partial","*",35,18],"anthropic-ai":["partial","*",35,18],"PerplexityBot":["partial","*",35,18],"Perplexity-User":["partial","*",35,18],"Google-Extended":["partial","*",35,18],"Applebot-Extended":["partial","*",35,18],"CCBot":["partial","*",35,18],"Bytespider":["partial","*",35,18],"Amazonbot":["partial","*",35,18],"meta-externalagent":["partial","*",35,18],"cohere-ai":["partial","*",35,18],"Diffbot":["partial","*",35,18],"ContextFetchBot":["partial","*",35,18]},"llms":{"status":"http_error","http":403,"bytes":513},"home":{"status":"ok","http":200,"url":"https://www.ipbes.net/","html_tokens":31478,"markdown_tokens":31,"pruned_tokens":28}},
{"rank":7136,"domain":"takeda.com","origin":"https://takeda.com","robots":{"status":"ok","http":200,"url":"https://www.takeda.com/robots.txt","bytes":6361,"groups":2},"bots":{"GPTBot":["allowed","named",0,1],"ChatGPT-User":["allowed","*",0,1],"OAI-SearchBot":["allowed","*",0,1],"ClaudeBot":["allowed","*",0,1],"Claude-User":["allowed","*",0,1],"Claude-SearchBot":["allowed","*",0,1],"anthropic-ai":["allowed","*",0,1],"PerplexityBot":["allowed","*",0,1],"Perplexity-User":["allowed","*",0,1],"Google-Extended":["allowed","*",0,1],"Applebot-Extended":["allowed","*",0,1],"CCBot":["allowed","*",0,1],"Bytespider":["allowed","*",0,1],"Amazonbot":["allowed","*",0,1],"meta-externalagent":["allowed","*",0,1],"cohere-ai":["allowed","*",0,1],"Diffbot":["allowed","*",0,1],"ContextFetchBot":["allowed","*",0,1]},"llms":{"status":"present","http":200,"bytes":21894},"home":{"status":"ok","http":200,"url":"https://www.takeda.com/","html_tokens":167571,"markdown_tokens":465,"pruned_tokens":26}},
{"rank":7137,"domain":"litmus.com","origin":"https://litmus.com","robots":{"status":"ok","http":200,"url":"https://litmus.com/robots.txt","bytes":26,"groups":1},"bots":{"GPTBot":["blocked","*",1,0],"ChatGPT-User":["blocked","*",1,0],"OAI-SearchBot":["blocked","*",1,0],"ClaudeBot":["blocked","*",1,0],"Claude-User":["blocked","*",1,0],"Claude-SearchBot":["blocked","*",1,0],"anthropic-ai":["blocked","*",1,0],"PerplexityBot":["blocked","*",1,0],"Perplexity-User":["blocked","*",1,0],"Google-Extended":["blocked","*",1,0],"Applebot-Extended":["blocked","*",1,0],"CCBot":["blocked","*",1,0],"Bytespider":["blocked","*",1,0],"Amazonbot":["blocked","*",1,0],"meta-externalagent":["blocked","*",1,0],"cohere-ai":["blocked","*",1,0],"Diffbot":["blocked","*",1,0],"ContextFetchBot":["blocked","*",1,0]},"llms":{"status":"disallowed"},"home":{"status":"disallowed"}},
{"rank":7138,"domain":"google.co.uz","origin":"https://google.co.uz","robots":{"status":"ok","http":200,"url":"https://www.google.co.uz/robots.txt","bytes":6624,"groups":4},"bots":{"GPTBot":["partial","*",169,64],"ChatGPT-User":["partial","*",169,64],"OAI-SearchBot":["partial","*",169,64],"ClaudeBot":["partial","*",169,64],"Claude-User":["partial","*",169,64],"Claude-SearchBot":["partial","*",169,64],"anthropic-ai":["partial","*",169,64],"PerplexityBot":["partial","*",169,64],"Perplexity-User":["partial","*",169,64],"Google-Extended":["partial","*",169,64],"Applebot-Extended":["partial","*",169,64],"CCBot":["partial","*",169,64],"Bytespider":["partial","*",169,64],"Amazonbot":["partial","*",169,64],"meta-externalagent":["partial","*",169,64],"cohere-ai":["partial","*",169,64],"Diffbot":["partial","*",169,64],"ContextFetchBot":["partial","*",169,64]},"llms":{"status":"missing","http":404,"bytes":1569},"home":{"status":"ok","http":200,"url":"https://www.google.com/","html_tokens":21459,"markdown_tokens":30,"pruned_tokens":30}},
{"rank":7139,"domain":"uc.cl","origin":"https://uc.cl","robots":{"status":"ok","http":200,"url":"https://www.uc.cl/robots.txt","bytes":72,"groups":1},"bots":{"GPTBot":["partial","*",1,0],"ChatGPT-User":["partial","*",1,0],"OAI-SearchBot":["partial","*",1,0],"ClaudeBot":["partial","*",1,0],"Claude-User":["partial","*",1,0],"Claude-SearchBot":["partial","*",1,0],"anthropic-ai":["partial","*",1,0],"PerplexityBot":["partial","*",1,0],"Perplexity-User":["partial","*",1,0],"Google-Extended":["partial","*",1,0],"Applebot-Extended":["partial","*",1,0],"CCBot":["partial","*",1,0],"Bytespider":["partial","*",1,0],"Amazonbot":["partial","*",1,0],"meta-externalagent":["partial","*",1,0],"cohere-ai":["partial","*",1,0],"Diffbot":["partial","*",1,0],"ContextFetchBot":["partial","*",1,0]},"llms":{"status":"missing","http":404,"bytes":101361},"home":{"status":"ok","http":200,"url":"https://www.uc.cl/","html_tokens":40055,"markdown_tokens":109,"pruned_tokens":84}},
{"rank":7140,"domain":"culture.ru","origin":"https://culture.ru","robots":{"status":"timeout","http":null,"url":"https://culture.ru/robots.txt","bytes":0,"groups":0,"error":"ConnectTimeout"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":7141,"domain":"seranking.com","origin":"https://seranking.com","robots":{"status":"ok","http":200,"url":"https://seranking.com/robots.txt","bytes":2385,"groups":11},"bots":{"GPTBot":["partial","named",3,1],"ChatGPT-User":["partial","named",2,1],"OAI-SearchBot":["partial","named",2,1],"ClaudeBot":["partial","named",3,1],"Claude-User":["partial","named",2,1],"Claude-SearchBot":["partial","named",2,1],"anthropic-ai":["partial","*",7,14],"PerplexityBot":["partial","named",2,1],"Perplexity-User":["partial","named",2,1],"Google-Extended":["partial","named",3,1],"Applebot-Extended":["partial","*",7,14],"CCBot":["partial","*",7,14],"Bytespider":["partial","*",7,14],"Amazonbot":["partial","*",7,14],"meta-externalagent":["partial","*",7,14],"cohere-ai":["partial","*",7,14],"Diffbot":["partial","*",7,14],"ContextFetchBot":["partial","*",7,14]},"llms":{"status":"missing","http":404,"bytes":409558},"home":{"status":"ok","http":200,"url":"https://seranking.com/","html_tokens":195895,"markdown_tokens":814,"pruned_tokens":80}},
{"rank":7142,"domain":"roadandtrack.com","origin":"https://roadandtrack.com","robots":{"status":"ok","http":200,"url":"https://roadandtrack.com/robots.txt","bytes":969,"groups":3},"bots":{"GPTBot":["partial","named",32,0],"ChatGPT-User":["partial","named",32,0],"OAI-SearchBot":["partial","named",32,0],"ClaudeBot":["partial","*",19,0],"Claude-User":["partial","*",19,0],"Claude-SearchBot":["partial","*",19,0],"anthropic-ai":["partial","*",19,0],"PerplexityBot":["partial","*",19,0],"Perplexity-User":["partial","*",19,0],"Google-Extended":["partial","*",19,0],"Applebot-Extended":["partial","*",19,0],"CCBot":["partial","*",19,0],"Bytespider":["partial","*",19,0],"Amazonbot":["partial","*",19,0],"meta-externalagent":["partial","*",19,0],"cohere-ai":["partial","*",19,0],"Diffbot":["partial","*",19,0],"ContextFetchBot":["partial","*",19,0]},"llms":{"status":"missing","http":404,"bytes":84},"home":{"status":"ok","http":200,"url":"https://www.roadandtrack.com/","html_tokens":196756,"markdown_tokens":115,"pruned_tokens":115}},
{"rank":7143,"domain":"pandora.net","origin":"https://pandora.net","robots":{"status":"ok","http":200,"url":"https://www.pandora.net/robots.txt","bytes":2864,"groups":9},"bots":{"GPTBot":["partial","*",20,23],"ChatGPT-User":["partial","*",20,23],"OAI-SearchBot":["partial","*",20,23],"ClaudeBot":["partial","*",20,23],"Claude-User":["partial","*",20,23],"Claude-SearchBot":["partial","*",20,23],"anthropic-ai":["partial","*",20,23],"PerplexityBot":["partial","*",20,23],"Perplexity-User":["partial","*",20,23],"Google-Extended":["partial","*",20,23],"Applebot-Extended":["partial","*",20,23],"CCBot":["partial","*",20,23],"Bytespider":["partial","*",20,23],"Amazonbot":["partial","*",20,23],"meta-externalagent":["partial","*",20,23],"cohere-ai":["partial","*",20,23],"Diffbot":["partial","*",20,23],"ContextFetchBot":["partial","*",20,23]},"llms":{"status":"disallowed","error":"robots.txt of us.pandora.net not readable (refused)"},"home":{"status":"disallowed","error":"robots.txt of us.pandora.net not readable (refused)"}},
{"rank":7144,"domain":"piie.com","origin":"https://piie.com","robots":{"status":"refused","http":403,"url":"https://piie.com/robots.txt","bytes":560,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":7145,"domain":"google.jo","origin":"https://google.jo","robots":{"status":"ok","http":200,"url":"https://www.google.jo/robots.txt","bytes":6624,"groups":4},"bots":{"GPTBot":["partial","*",169,64],"ChatGPT-User":["partial","*",169,64],"OAI-SearchBot":["partial","*",169,64],"ClaudeBot":["partial","*",169,64],"Claude-User":["partial","*",169,64],"Claude-SearchBot":["partial","*",169,64],"anthropic-ai":["partial","*",169,64],"PerplexityBot":["partial","*",169,64],"Perplexity-User":["partial","*",169,64],"Google-Extended":["partial","*",169,64],"Applebot-Extended":["partial","*",169,64],"CCBot":["partial","*",169,64],"Bytespider":["partial","*",169,64],"Amazonbot":["partial","*",169,64],"meta-externalagent":["partial","*",169,64],"cohere-ai":["partial","*",169,64],"Diffbot":["partial","*",169,64],"ContextFetchBot":["partial","*",169,64]},"llms":{"status":"missing","http":404,"bytes":1569},"home":{"status":"ok","http":200,"url":"https://www.google.com/","html_tokens":21352,"markdown_tokens":30,"pruned_tokens":30}},
{"rank":7146,"domain":"dtf.ru","origin":"https://dtf.ru","robots":{"status":"ok","http":200,"url":"https://dtf.ru/robots.txt","bytes":556,"groups":1},"bots":{"GPTBot":["partial","*",25,0],"ChatGPT-User":["partial","*",25,0],"OAI-SearchBot":["partial","*",25,0],"ClaudeBot":["partial","*",25,0],"Claude-User":["partial","*",25,0],"Claude-SearchBot":["partial","*",25,0],"anthropic-ai":["partial","*",25,0],"PerplexityBot":["partial","*",25,0],"Perplexity-User":["partial","*",25,0],"Google-Extended":["partial","*",25,0],"Applebot-Extended":["partial","*",25,0],"CCBot":["partial","*",25,0],"Bytespider":["partial","*",25,0],"Amazonbot":["partial","*",25,0],"meta-externalagent":["partial","*",25,0],"cohere-ai":["partial","*",25,0],"Diffbot":["partial","*",25,0],"ContextFetchBot":["partial","*",25,0]},"llms":{"status":"missing","http":404,"bytes":757473},"home":{"status":"ok","http":200,"url":"https://dtf.ru/","html_tokens":177752,"markdown_tokens":88,"pruned_tokens":88}},
{"rank":7147,"domain":"visitessexma.com","origin":"https://visitessexma.com","robots":{"status":"ok","http":200,"url":"https://visitessexma.com/robots.txt","bytes":59,"groups":1},"bots":{"GPTBot":["partial","*",2,1],"ChatGPT-User":["partial","*",2,1],"OAI-SearchBot":["partial","*",2,1],"ClaudeBot":["partial","*",2,1],"Claude-User":["partial","*",2,1],"Claude-SearchBot":["partial","*",2,1],"anthropic-ai":["partial","*",2,1],"PerplexityBot":["partial","*",2,1],"Perplexity-User":["partial","*",2,1],"Google-Extended":["partial","*",2,1],"Applebot-Extended":["partial","*",2,1],"CCBot":["partial","*",2,1],"Bytespider":["partial","*",2,1],"Amazonbot":["partial","*",2,1],"meta-externalagent":["partial","*",2,1],"cohere-ai":["partial","*",2,1],"Diffbot":["partial","*",2,1],"ContextFetchBot":["partial","*",2,1]},"llms":{"status":"missing","http":404,"bytes":1549},"home":{"status":"ok","http":200,"url":"https://visitessexma.com/","html_tokens":9275,"markdown_tokens":1357,"pruned_tokens":129}},
{"rank":7148,"domain":"siam.org","origin":"https://siam.org","robots":{"status":"ok","http":200,"url":"https://www.siam.org/robots.txt","bytes":89,"groups":1},"bots":{"GPTBot":["partial","*",1,0],"ChatGPT-User":["partial","*",1,0],"OAI-SearchBot":["partial","*",1,0],"ClaudeBot":["partial","*",1,0],"Claude-User":["partial","*",1,0],"Claude-SearchBot":["partial","*",1,0],"anthropic-ai":["partial","*",1,0],"PerplexityBot":["partial","*",1,0],"Perplexity-User":["partial","*",1,0],"Google-Extended":["partial","*",1,0],"Applebot-Extended":["partial","*",1,0],"CCBot":["partial","*",1,0],"Bytespider":["partial","*",1,0],"Amazonbot":["partial","*",1,0],"meta-externalagent":["partial","*",1,0],"cohere-ai":["partial","*",1,0],"Diffbot":["partial","*",1,0],"ContextFetchBot":["partial","*",1,0]},"llms":{"status":"missing","http":404,"bytes":0},"home":{"status":"http_error","http":403,"url":"https://siam.org/"}},
{"rank":7149,"domain":"chiba.jp","origin":"https://chiba.jp","robots":{"status":"unreachable","http":null,"url":"https://chiba.jp/robots.txt","bytes":0,"groups":0,"error":"cannot resolve chiba.jp"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":7150,"domain":"bravotv.com","origin":"https://bravotv.com","robots":{"status":"ok","http":200,"url":"https://www.bravotv.com/robots.txt","bytes":3276,"groups":23},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["partial","*",31,19],"OAI-SearchBot":["partial","*",31,19],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",31,19],"Claude-SearchBot":["partial","*",31,19],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["partial","*",31,19],"Perplexity-User":["partial","*",31,19],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["partial","*",31,19],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",31,19]},"llms":{"status":"present","http":200,"bytes":2449},"home":{"status":"ok","http":200,"url":"https://www.bravotv.com/","html_tokens":37805,"markdown_tokens":343,"pruned_tokens":343}},
{"rank":7151,"domain":"google.dm","origin":"https://google.dm","robots":{"status":"ok","http":200,"url":"https://www.google.dm/robots.txt","bytes":6624,"groups":4},"bots":{"GPTBot":["partial","*",169,64],"ChatGPT-User":["partial","*",169,64],"OAI-SearchBot":["partial","*",169,64],"ClaudeBot":["partial","*",169,64],"Claude-User":["partial","*",169,64],"Claude-SearchBot":["partial","*",169,64],"anthropic-ai":["partial","*",169,64],"PerplexityBot":["partial","*",169,64],"Perplexity-User":["partial","*",169,64],"Google-Extended":["partial","*",169,64],"Applebot-Extended":["partial","*",169,64],"CCBot":["partial","*",169,64],"Bytespider":["partial","*",169,64],"Amazonbot":["partial","*",169,64],"meta-externalagent":["partial","*",169,64],"cohere-ai":["partial","*",169,64],"Diffbot":["partial","*",169,64],"ContextFetchBot":["partial","*",169,64]},"llms":{"status":"missing","http":404,"bytes":1569},"home":{"status":"ok","http":200,"url":"https://www.google.com/","html_tokens":21438,"markdown_tokens":30,"pruned_tokens":30}},
{"rank":7152,"domain":"lnk.bio","origin":"https://lnk.bio","robots":{"status":"ok","http":200,"url":"https://lnk.bio/robots.txt","bytes":111,"groups":2},"bots":{"GPTBot":["allowed","*",0,1],"ChatGPT-User":["allowed","*",0,1],"OAI-SearchBot":["allowed","*",0,1],"ClaudeBot":["allowed","*",0,1],"Claude-User":["allowed","*",0,1],"Claude-SearchBot":["allowed","*",0,1],"anthropic-ai":["allowed","*",0,1],"PerplexityBot":["allowed","*",0,1],"Perplexity-User":["allowed","*",0,1],"Google-Extended":["allowed","*",0,1],"Applebot-Extended":["allowed","*",0,1],"CCBot":["allowed","*",0,1],"Bytespider":["allowed","*",0,1],"Amazonbot":["allowed","*",0,1],"meta-externalagent":["allowed","*",0,1],"cohere-ai":["allowed","*",0,1],"Diffbot":["allowed","*",0,1],"ContextFetchBot":["allowed","*",0,1]},"llms":{"status":"present","http":200,"bytes":4079},"home":{"status":"http_error","http":403,"url":"https://lnk.bio/"}},
{"rank":7153,"domain":"historicengland.org.uk","origin":"https://historicengland.org.uk","robots":{"status":"refused","http":403,"url":"https://historicengland.org.uk/robots.txt","bytes":5560,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":7154,"domain":"itsfoss.com","origin":"https://itsfoss.com","robots":{"status":"ok","http":200,"url":"https://itsfoss.com/robots.txt","bytes":118,"groups":1},"bots":{"GPTBot":["partial","*",4,0],"ChatGPT-User":["partial","*",4,0],"OAI-SearchBot":["partial","*",4,0],"ClaudeBot":["partial","*",4,0],"Claude-User":["partial","*",4,0],"Claude-SearchBot":["partial","*",4,0],"anthropic-ai":["partial","*",4,0],"PerplexityBot":["partial","*",4,0],"Perplexity-User":["partial","*",4,0],"Google-Extended":["partial","*",4,0],"Applebot-Extended":["partial","*",4,0],"CCBot":["partial","*",4,0],"Bytespider":["partial","*",4,0],"Amazonbot":["partial","*",4,0],"meta-externalagent":["partial","*",4,0],"cohere-ai":["partial","*",4,0],"Diffbot":["partial","*",4,0],"ContextFetchBot":["partial","*",4,0]},"llms":{"status":"present","http":200,"bytes":51149},"home":{"status":"ok","http":200,"url":"https://itsfoss.com/","html_tokens":39794,"markdown_tokens":127,"pruned_tokens":127}},
{"rank":7155,"domain":"openssh.com","origin":"https://openssh.com","robots":{"status":"missing","http":404,"url":"https://www.openssh.org/robots.txt","bytes":494,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":494},"home":{"status":"ok","http":200,"url":"https://www.openssh.org/","html_tokens":765,"markdown_tokens":227,"pruned_tokens":227}},
{"rank":7156,"domain":"xilinx.com","origin":"https://www.xilinx.com","robots":{"status":"ok","http":200,"url":"https://www.xilinx.com/robots.txt","bytes":2204,"groups":1},"bots":{"GPTBot":["partial","*",29,0],"ChatGPT-User":["partial","*",29,0],"OAI-SearchBot":["partial","*",29,0],"ClaudeBot":["partial","*",29,0],"Claude-User":["partial","*",29,0],"Claude-SearchBot":["partial","*",29,0],"anthropic-ai":["partial","*",29,0],"PerplexityBot":["partial","*",29,0],"Perplexity-User":["partial","*",29,0],"Google-Extended":["partial","*",29,0],"Applebot-Extended":["partial","*",29,0],"CCBot":["partial","*",29,0],"Bytespider":["partial","*",29,0],"Amazonbot":["partial","*",29,0],"meta-externalagent":["partial","*",29,0],"cohere-ai":["partial","*",29,0],"Diffbot":["partial","*",29,0],"ContextFetchBot":["partial","*",29,0]},"llms":{"status":"missing","http":404,"bytes":153431},"home":{"status":"disallowed","error":"robots.txt of www.amd.com not readable (unreachable)"}},
{"rank":7157,"domain":"filehippo.com","origin":"https://filehippo.com","robots":{"status":"refused","http":412,"url":"https://filehippo.com/robots.txt","bytes":0,"groups":0,"error":"HTTP 412"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":7158,"domain":"paralympic.org","origin":"https://www.paralympic.org","robots":{"status":"ok","http":200,"url":"https://www.paralympic.org/robots.txt","bytes":5111,"groups":1},"bots":{"GPTBot":["partial","*",100,18],"ChatGPT-User":["partial","*",100,18],"OAI-SearchBot":["partial","*",100,18],"ClaudeBot":["partial","*",100,18],"Claude-User":["partial","*",100,18],"Claude-SearchBot":["partial","*",100,18],"anthropic-ai":["partial","*",100,18],"PerplexityBot":["partial","*",100,18],"Perplexity-User":["partial","*",100,18],"Google-Extended":["partial","*",100,18],"Applebot-Extended":["partial","*",100,18],"CCBot":["partial","*",100,18],"Bytespider":["partial","*",100,18],"Amazonbot":["partial","*",100,18],"meta-externalagent":["partial","*",100,18],"cohere-ai":["partial","*",100,18],"Diffbot":["partial","*",100,18],"ContextFetchBot":["partial","*",100,18]},"llms":{"status":"missing","http":404,"bytes":192},"home":{"status":"ok","http":200,"url":"https://www.paralympic.org/","html_tokens":35953,"markdown_tokens":784,"pruned_tokens":553}},
{"rank":7159,"domain":"rutracker.org","origin":"https://rutracker.org","robots":{"status":"refused","http":403,"url":"https://rutracker.org/robots.txt","bytes":5423,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":7160,"domain":"abc15.com","origin":"https://abc15.com","robots":{"status":"ok","http":200,"url":"https://www.abc15.com/robots.txt","bytes":385,"groups":4},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["partial","*",5,0],"OAI-SearchBot":["partial","*",5,0],"ClaudeBot":["partial","*",5,0],"Claude-User":["partial","*",5,0],"Claude-SearchBot":["partial","*",5,0],"anthropic-ai":["partial","*",5,0],"PerplexityBot":["partial","*",5,0],"Perplexity-User":["partial","*",5,0],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["partial","*",5,0],"CCBot":["partial","*",5,0],"Bytespider":["partial","*",5,0],"Amazonbot":["partial","*",5,0],"meta-externalagent":["partial","*",5,0],"cohere-ai":["partial","*",5,0],"Diffbot":["partial","*",5,0],"ContextFetchBot":["partial","*",5,0]},"llms":{"status":"missing","http":404,"bytes":79126},"home":{"status":"ok","http":200,"url":"https://www.abc15.com/","html_tokens":65557,"markdown_tokens":198,"pruned_tokens":24}},
{"rank":7161,"domain":"gavi.org","origin":"https://gavi.org","robots":{"status":"ok","http":200,"url":"https://www.gavi.org/robots.txt","bytes":3480,"groups":1},"bots":{"GPTBot":["partial","*",49,38],"ChatGPT-User":["partial","*",49,38],"OAI-SearchBot":["partial","*",49,38],"ClaudeBot":["partial","*",49,38],"Claude-User":["partial","*",49,38],"Claude-SearchBot":["partial","*",49,38],"anthropic-ai":["partial","*",49,38],"PerplexityBot":["partial","*",49,38],"Perplexity-User":["partial","*",49,38],"Google-Extended":["partial","*",49,38],"Applebot-Extended":["partial","*",49,38],"CCBot":["partial","*",49,38],"Bytespider":["partial","*",49,38],"Amazonbot":["partial","*",49,38],"meta-externalagent":["partial","*",49,38],"cohere-ai":["partial","*",49,38],"Diffbot":["partial","*",49,38],"ContextFetchBot":["partial","*",49,38]},"llms":{"status":"present","http":200,"bytes":5237},"home":{"status":"ok","http":200,"url":"https://www.gavi.org/","html_tokens":52640,"markdown_tokens":633,"pruned_tokens":374}},
{"rank":7162,"domain":"active24.cz","origin":"https://active24.cz","robots":{"status":"ok","http":200,"url":"https://www.active24.cz/robots.txt","bytes":119,"groups":1},"bots":{"GPTBot":["partial","*",1,1],"ChatGPT-User":["partial","*",1,1],"OAI-SearchBot":["partial","*",1,1],"ClaudeBot":["partial","*",1,1],"Claude-User":["partial","*",1,1],"Claude-SearchBot":["partial","*",1,1],"anthropic-ai":["partial","*",1,1],"PerplexityBot":["partial","*",1,1],"Perplexity-User":["partial","*",1,1],"Google-Extended":["partial","*",1,1],"Applebot-Extended":["partial","*",1,1],"CCBot":["partial","*",1,1],"Bytespider":["partial","*",1,1],"Amazonbot":["partial","*",1,1],"meta-externalagent":["partial","*",1,1],"cohere-ai":["partial","*",1,1],"Diffbot":["partial","*",1,1],"ContextFetchBot":["partial","*",1,1]},"llms":{"status":"missing","http":404,"bytes":208880},"home":{"status":"ok","http":200,"url":"https://www.active24.cz/","html_tokens":80967,"markdown_tokens":319,"pruned_tokens":81}},
{"rank":7163,"domain":"brownpapertickets.com","origin":"https://brownpapertickets.com","robots":{"status":"ok","http":200,"url":"https://brownpapertickets.com/robots.txt","bytes":233,"groups":2},"bots":{"GPTBot":["partial","*",4,0],"ChatGPT-User":["partial","*",4,0],"OAI-SearchBot":["partial","*",4,0],"ClaudeBot":["partial","*",4,0],"Claude-User":["partial","*",4,0],"Claude-SearchBot":["partial","*",4,0],"anthropic-ai":["partial","*",4,0],"PerplexityBot":["partial","*",4,0],"Perplexity-User":["partial","*",4,0],"Google-Extended":["partial","*",4,0],"Applebot-Extended":["partial","*",4,0],"CCBot":["partial","*",4,0],"Bytespider":["partial","*",4,0],"Amazonbot":["partial","*",4,0],"meta-externalagent":["partial","*",4,0],"cohere-ai":["partial","*",4,0],"Diffbot":["partial","*",4,0],"ContextFetchBot":["partial","*",4,0]},"llms":{"status":"missing","http":404,"bytes":18718},"home":{"status":"ok","http":200,"url":"https://brownpapertickets.com/","html_tokens":15541,"markdown_tokens":110,"pruned_tokens":102}},
{"rank":7164,"domain":"wwf.de","origin":"https://wwf.de","robots":{"status":"ok","http":200,"url":"https://www.wwf.de/robots.txt","bytes":1165,"groups":3},"bots":{"GPTBot":["partial","*",13,15],"ChatGPT-User":["partial","*",13,15],"OAI-SearchBot":["partial","*",13,15],"ClaudeBot":["partial","*",13,15],"Claude-User":["partial","*",13,15],"Claude-SearchBot":["partial","*",13,15],"anthropic-ai":["partial","*",13,15],"PerplexityBot":["partial","*",13,15],"Perplexity-User":["partial","*",13,15],"Google-Extended":["partial","*",13,15],"Applebot-Extended":["partial","*",13,15],"CCBot":["partial","*",13,15],"Bytespider":["partial","*",13,15],"Amazonbot":["partial","*",13,15],"meta-externalagent":["partial","*",13,15],"cohere-ai":["partial","*",13,15],"Diffbot":["partial","*",13,15],"ContextFetchBot":["partial","*",13,15]},"llms":{"status":"missing","http":404,"bytes":36681},"home":{"status":"ok","http":200,"url":"https://www.wwf.de/","html_tokens":20548,"markdown_tokens":759,"pruned_tokens":138}},
{"rank":7165,"domain":"nihr.ac.uk","origin":"https://www.nihr.ac.uk","robots":{"status":"refused","http":403,"url":"https://www.nihr.ac.uk/robots.txt","bytes":919,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":7166,"domain":"skitterphoto.com","origin":"https://skitterphoto.com","robots":{"status":"ok","http":200,"url":"https://skitterphoto.com/robots.txt","bytes":22,"groups":1},"bots":{"GPTBot":["allowed","*",0,1],"ChatGPT-User":["allowed","*",0,1],"OAI-SearchBot":["allowed","*",0,1],"ClaudeBot":["allowed","*",0,1],"Claude-User":["allowed","*",0,1],"Claude-SearchBot":["allowed","*",0,1],"anthropic-ai":["allowed","*",0,1],"PerplexityBot":["allowed","*",0,1],"Perplexity-User":["allowed","*",0,1],"Google-Extended":["allowed","*",0,1],"Applebot-Extended":["allowed","*",0,1],"CCBot":["allowed","*",0,1],"Bytespider":["allowed","*",0,1],"Amazonbot":["allowed","*",0,1],"meta-externalagent":["allowed","*",0,1],"cohere-ai":["allowed","*",0,1],"Diffbot":["allowed","*",0,1],"ContextFetchBot":["allowed","*",0,1]},"llms":{"status":"missing","http":404,"bytes":24917},"home":{"status":"ok","http":200,"url":"https://skitterphoto.com/","html_tokens":8993,"markdown_tokens":138,"pruned_tokens":43}},
{"rank":7167,"domain":"vcahospitals.com","origin":"https://vcahospitals.com","robots":{"status":"ok","http":200,"url":"https://vcahospitals.com/robots.txt","bytes":523,"groups":1},"bots":{"GPTBot":["partial","*",7,0],"ChatGPT-User":["partial","*",7,0],"OAI-SearchBot":["partial","*",7,0],"ClaudeBot":["partial","*",7,0],"Claude-User":["partial","*",7,0],"Claude-SearchBot":["partial","*",7,0],"anthropic-ai":["partial","*",7,0],"PerplexityBot":["partial","*",7,0],"Perplexity-User":["partial","*",7,0],"Google-Extended":["partial","*",7,0],"Applebot-Extended":["partial","*",7,0],"CCBot":["partial","*",7,0],"Bytespider":["partial","*",7,0],"Amazonbot":["partial","*",7,0],"meta-externalagent":["partial","*",7,0],"cohere-ai":["partial","*",7,0],"Diffbot":["partial","*",7,0],"ContextFetchBot":["partial","*",7,0]},"llms":{"status":"missing","http":404,"bytes":1606},"home":{"status":"ok","http":200,"url":"https://vcahospitals.com/","html_tokens":71951,"markdown_tokens":699,"pruned_tokens":173}},
{"rank":7168,"domain":"wework.com","origin":"https://wework.com","robots":{"status":"refused","http":403,"url":"https://www.wework.com/robots.txt","bytes":5615,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":7169,"domain":"loseit.com","origin":"https://loseit.com","robots":{"status":"ok","http":200,"url":"https://www.loseit.com/robots.txt","bytes":160,"groups":2},"bots":{"GPTBot":["partial","*",1,0],"ChatGPT-User":["partial","*",1,0],"OAI-SearchBot":["partial","*",1,0],"ClaudeBot":["partial","*",1,0],"Claude-User":["partial","*",1,0],"Claude-SearchBot":["partial","*",1,0],"anthropic-ai":["partial","*",1,0],"PerplexityBot":["partial","*",1,0],"Perplexity-User":["partial","*",1,0],"Google-Extended":["partial","*",1,0],"Applebot-Extended":["partial","*",1,0],"CCBot":["partial","*",1,0],"Bytespider":["partial","*",1,0],"Amazonbot":["partial","*",1,0],"meta-externalagent":["partial","*",1,0],"cohere-ai":["partial","*",1,0],"Diffbot":["partial","*",1,0],"ContextFetchBot":["partial","*",1,0]},"llms":{"status":"other_text","http":200,"bytes":4308},"home":{"status":"ok","http":200,"url":"https://www.loseit.com/","html_tokens":3060,"markdown_tokens":430,"pruned_tokens":50}},
{"rank":7170,"domain":"tug.org","origin":"https://tug.org","robots":{"status":"ok","http":200,"url":"https://tug.org/robots.txt","bytes":1489,"groups":19},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["blocked","named",1,0],"OAI-SearchBot":["partial","*",26,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",26,0],"Claude-SearchBot":["partial","*",26,0],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["partial","*",26,0],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["partial","*",26,0],"CCBot":["partial","*",26,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["partial","*",26,0],"cohere-ai":["partial","*",26,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",26,0]},"llms":{"status":"missing","http":404,"bytes":236},"home":{"status":"ok","http":200,"url":"https://tug.org/","html_tokens":6003,"markdown_tokens":1435,"pruned_tokens":261}},
{"rank":7171,"domain":"trendyol.com","origin":"https://trendyol.com","robots":{"status":"ok","http":200,"url":"https://www.trendyol.com/robots.txt","bytes":16163,"groups":4},"bots":{"GPTBot":["partial","*",567,52],"ChatGPT-User":["partial","*",567,52],"OAI-SearchBot":["partial","*",567,52],"ClaudeBot":["partial","*",567,52],"Claude-User":["partial","*",567,52],"Claude-SearchBot":["partial","*",567,52],"anthropic-ai":["partial","*",567,52],"PerplexityBot":["partial","*",567,52],"Perplexity-User":["partial","*",567,52],"Google-Extended":["partial","*",567,52],"Applebot-Extended":["partial","*",567,52],"CCBot":["partial","*",567,52],"Bytespider":["partial","*",567,52],"Amazonbot":["partial","*",567,52],"meta-externalagent":["partial","*",567,52],"cohere-ai":["partial","*",567,52],"Diffbot":["partial","*",567,52],"ContextFetchBot":["partial","*",567,52]},"llms":{"status":"present","http":200,"bytes":4526},"home":{"status":"http_error","http":403,"url":"https://www.trendyol.com/"}},
{"rank":7172,"domain":"listverse.com","origin":"https://listverse.com","robots":{"status":"ok","http":200,"url":"https://listverse.com/robots.txt","bytes":171,"groups":1},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"missing","http":404,"bytes":56894},"home":{"status":"ok","http":200,"url":"https://listverse.com/","html_tokens":31192,"markdown_tokens":76,"pruned_tokens":62}},
{"rank":7173,"domain":"rbc.cn","origin":"https://rbc.cn","robots":{"status":"unreachable","http":null,"url":"https://rbc.cn/robots.txt","bytes":0,"groups":0,"error":"cannot resolve rbc.cn"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":7174,"domain":"u-strasbg.fr","origin":"https://u-strasbg.fr","robots":{"status":"unreachable","http":null,"url":"https://u-strasbg.fr/robots.txt","bytes":0,"groups":0,"error":"cannot resolve u-strasbg.fr"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":7175,"domain":"ring.com","origin":"https://ring.com","robots":{"status":"ok","http":200,"url":"https://ring.com/robots.txt","bytes":1935,"groups":2},"bots":{"GPTBot":["partial","named",16,1],"ChatGPT-User":["partial","named",16,1],"OAI-SearchBot":["partial","*",20,0],"ClaudeBot":["partial","named",16,1],"Claude-User":["partial","*",20,0],"Claude-SearchBot":["partial","*",20,0],"anthropic-ai":["partial","named",16,1],"PerplexityBot":["partial","named",16,1],"Perplexity-User":["partial","*",20,0],"Google-Extended":["partial","named",16,1],"Applebot-Extended":["partial","named",16,1],"CCBot":["partial","*",20,0],"Bytespider":["partial","*",20,0],"Amazonbot":["partial","*",20,0],"meta-externalagent":["partial","*",20,0],"cohere-ai":["partial","named",16,1],"Diffbot":["partial","*",20,0],"ContextFetchBot":["partial","*",20,0]},"llms":{"status":"present","http":200,"bytes":7398},"home":{"status":"ok","http":200,"url":"https://ring.com/","html_tokens":45257,"markdown_tokens":197,"pruned_tokens":31}},
{"rank":7176,"domain":"eui.eu","origin":"https://eui.eu","robots":{"status":"missing","http":404,"url":"https://www.eui.eu/robots.txt","bytes":14,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":14},"home":{"status":"ok","http":200,"url":"https://www.eui.eu/en/home","html_tokens":24060,"markdown_tokens":1472,"pruned_tokens":380}},
{"rank":7177,"domain":"pantip.com","origin":"https://pantip.com","robots":{"status":"ok","http":200,"url":"https://pantip.com/robots.txt","bytes":220,"groups":6},"bots":{"GPTBot":["partial","*",1,0],"ChatGPT-User":["partial","*",1,0],"OAI-SearchBot":["partial","*",1,0],"ClaudeBot":["partial","*",1,0],"Claude-User":["partial","*",1,0],"Claude-SearchBot":["partial","*",1,0],"anthropic-ai":["partial","*",1,0],"PerplexityBot":["partial","*",1,0],"Perplexity-User":["partial","*",1,0],"Google-Extended":["partial","*",1,0],"Applebot-Extended":["partial","*",1,0],"CCBot":["partial","*",1,0],"Bytespider":["partial","*",1,0],"Amazonbot":["partial","*",1,0],"meta-externalagent":["partial","*",1,0],"cohere-ai":["partial","*",1,0],"Diffbot":["partial","*",1,0],"ContextFetchBot":["partial","*",1,0]},"llms":{"status":"missing","http":404,"bytes":20730},"home":{"status":"ok","http":200,"url":"https://pantip.com/","html_tokens":32026,"markdown_tokens":190,"pruned_tokens":128}},
{"rank":7178,"domain":"clover.com","origin":"https://clover.com","robots":{"status":"ok","http":200,"url":"https://www.clover.com/robots.txt","bytes":1155,"groups":3},"bots":{"GPTBot":["partial","*",14,7],"ChatGPT-User":["partial","*",14,7],"OAI-SearchBot":["partial","*",14,7],"ClaudeBot":["partial","*",14,7],"Claude-User":["partial","*",14,7],"Claude-SearchBot":["partial","*",14,7],"anthropic-ai":["partial","*",14,7],"PerplexityBot":["partial","*",14,7],"Perplexity-User":["partial","*",14,7],"Google-Extended":["partial","*",14,7],"Applebot-Extended":["partial","*",14,7],"CCBot":["partial","*",14,7],"Bytespider":["partial","*",14,7],"Amazonbot":["partial","*",14,7],"meta-externalagent":["partial","*",14,7],"cohere-ai":["partial","*",14,7],"Diffbot":["partial","*",14,7],"ContextFetchBot":["partial","*",14,7]},"llms":{"status":"missing","http":404,"bytes":127},"home":{"status":"ok","http":200,"url":"https://www.clover.com/","html_tokens":70753,"markdown_tokens":473,"pruned_tokens":79}},
{"rank":7179,"domain":"pwc.co.uk","origin":"https://pwc.co.uk","robots":{"status":"refused","http":403,"url":"https://www.pwc.co.uk/robots.txt","bytes":385,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":7180,"domain":"freakonomics.com","origin":"https://freakonomics.com","robots":{"status":"ok","http":200,"url":"https://freakonomics.com/robots.txt","bytes":174,"groups":1},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"html","http":200,"bytes":56821},"home":{"status":"ok","http":200,"url":"https://freakonomics.com/","html_tokens":14200,"markdown_tokens":143,"pruned_tokens":67}},
{"rank":7181,"domain":"kahoot.it","origin":"https://kahoot.it","robots":{"status":"ok","http":200,"url":"https://kahoot.it/robots.txt","bytes":490,"groups":12},"bots":{"GPTBot":["partial","*",1,1],"ChatGPT-User":["partial","*",1,1],"OAI-SearchBot":["partial","*",1,1],"ClaudeBot":["partial","*",1,1],"Claude-User":["partial","*",1,1],"Claude-SearchBot":["partial","*",1,1],"anthropic-ai":["partial","*",1,1],"PerplexityBot":["partial","*",1,1],"Perplexity-User":["partial","*",1,1],"Google-Extended":["partial","*",1,1],"Applebot-Extended":["partial","*",1,1],"CCBot":["partial","*",1,1],"Bytespider":["partial","*",1,1],"Amazonbot":["partial","*",1,1],"meta-externalagent":["partial","*",1,1],"cohere-ai":["partial","*",1,1],"Diffbot":["partial","*",1,1],"ContextFetchBot":["partial","*",1,1]},"llms":{"status":"disallowed"},"home":{"status":"ok","http":200,"url":"https://kahoot.it/","html_tokens":1501,"markdown_tokens":0,"pruned_tokens":0}},
{"rank":7182,"domain":"baseball-reference.com","origin":"https://baseball-reference.com","robots":{"status":"refused","http":403,"url":"https://baseball-reference.com/robots.txt","bytes":5539,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":7183,"domain":"movavi.com","origin":"https://movavi.com","robots":{"status":"ok","http":200,"url":"https://www.movavi.com/robots.txt","bytes":640,"groups":1},"bots":{"GPTBot":["partial","*",18,3],"ChatGPT-User":["partial","*",18,3],"OAI-SearchBot":["partial","*",18,3],"ClaudeBot":["partial","*",18,3],"Claude-User":["partial","*",18,3],"Claude-SearchBot":["partial","*",18,3],"anthropic-ai":["partial","*",18,3],"PerplexityBot":["partial","*",18,3],"Perplexity-User":["partial","*",18,3],"Google-Extended":["partial","*",18,3],"Applebot-Extended":["partial","*",18,3],"CCBot":["partial","*",18,3],"Bytespider":["partial","*",18,3],"Amazonbot":["partial","*",18,3],"meta-externalagent":["partial","*",18,3],"cohere-ai":["partial","*",18,3],"Diffbot":["partial","*",18,3],"ContextFetchBot":["partial","*",18,3]},"llms":{"status":"present","http":200,"bytes":2385},"home":{"status":"ok","http":200,"url":"https://www.movavi.com/","html_tokens":102557,"markdown_tokens":1794,"pruned_tokens":722}},
{"rank":7184,"domain":"theworlds50best.com","origin":"https://theworlds50best.com","robots":{"status":"ok","http":200,"url":"https://www.the50.com/robots.txt","bytes":4177,"groups":5},"bots":{"GPTBot":["partial","named",1,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["partial","named",1,0],"ClaudeBot":["partial","named",1,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["partial","named",1,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["blocked","named",1,0],"Bytespider":["partial","named",1,0],"Amazonbot":["partial","named",1,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":29738},"home":{"status":"ok","http":200,"url":"https://www.the50.com/","html_tokens":13978,"markdown_tokens":74,"pruned_tokens":74}},
{"rank":7185,"domain":"lyxshs.com","origin":"http://lyxshs.com","robots":{"status":"ok","http":200,"url":"http://lyxshs.com/robots.txt","bytes":155,"groups":1},"bots":{"GPTBot":["partial","*",4,3],"ChatGPT-User":["partial","*",4,3],"OAI-SearchBot":["partial","*",4,3],"ClaudeBot":["partial","*",4,3],"Claude-User":["partial","*",4,3],"Claude-SearchBot":["partial","*",4,3],"anthropic-ai":["partial","*",4,3],"PerplexityBot":["partial","*",4,3],"Perplexity-User":["partial","*",4,3],"Google-Extended":["partial","*",4,3],"Applebot-Extended":["partial","*",4,3],"CCBot":["partial","*",4,3],"Bytespider":["partial","*",4,3],"Amazonbot":["partial","*",4,3],"meta-externalagent":["partial","*",4,3],"cohere-ai":["partial","*",4,3],"Diffbot":["partial","*",4,3],"ContextFetchBot":["partial","*",4,3]},"llms":{"status":"html","http":200,"bytes":3368},"home":{"status":"ok","http":200,"url":"http://www.lyxshs.com/","html_tokens":4962,"markdown_tokens":145,"pruned_tokens":99}},
{"rank":7186,"domain":"flightglobal.com","origin":"https://flightglobal.com","robots":{"status":"ok","http":200,"url":"https://flightglobal.com/robots.txt","bytes":2676,"groups":32},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["allowed","*",0,1],"OAI-SearchBot":["allowed","*",0,1],"ClaudeBot":["blocked","named",1,0],"Claude-User":["allowed","*",0,1],"Claude-SearchBot":["allowed","*",0,1],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["allowed","*",0,1],"Perplexity-User":["allowed","*",0,1],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["allowed","*",0,1]},"llms":{"status":"missing","http":404,"bytes":41707},"home":{"status":"ok","http":200,"url":"https://www.flightglobal.com/","html_tokens":24314,"markdown_tokens":956,"pruned_tokens":90}},
{"rank":7187,"domain":"well.com","origin":"https://www.well.com","robots":{"status":"ok","http":200,"url":"https://www.well.com/robots.txt","bytes":207,"groups":2},"bots":{"GPTBot":["partial","*",7,0],"ChatGPT-User":["partial","*",7,0],"OAI-SearchBot":["partial","*",7,0],"ClaudeBot":["partial","*",7,0],"Claude-User":["partial","*",7,0],"Claude-SearchBot":["partial","*",7,0],"anthropic-ai":["partial","*",7,0],"PerplexityBot":["partial","*",7,0],"Perplexity-User":["partial","*",7,0],"Google-Extended":["partial","*",7,0],"Applebot-Extended":["partial","*",7,0],"CCBot":["partial","*",7,0],"Bytespider":["partial","*",7,0],"Amazonbot":["partial","*",7,0],"meta-externalagent":["partial","*",7,0],"cohere-ai":["partial","*",7,0],"Diffbot":["partial","*",7,0],"ContextFetchBot":["partial","*",7,0]},"llms":{"status":"missing","http":404,"bytes":60},"home":{"status":"ok","http":200,"url":"https://www.well.com/","html_tokens":66887,"markdown_tokens":1171,"pruned_tokens":120}},
{"rank":7188,"domain":"xmission.com","origin":"https://xmission.com","robots":{"status":"ok","http":200,"url":"https://xmission.com/robots.txt","bytes":317,"groups":1},"bots":{"GPTBot":["partial","*",6,1],"ChatGPT-User":["partial","*",6,1],"OAI-SearchBot":["partial","*",6,1],"ClaudeBot":["partial","*",6,1],"Claude-User":["partial","*",6,1],"Claude-SearchBot":["partial","*",6,1],"anthropic-ai":["partial","*",6,1],"PerplexityBot":["partial","*",6,1],"Perplexity-User":["partial","*",6,1],"Google-Extended":["partial","*",6,1],"Applebot-Extended":["partial","*",6,1],"CCBot":["partial","*",6,1],"Bytespider":["partial","*",6,1],"Amazonbot":["partial","*",6,1],"meta-externalagent":["partial","*",6,1],"cohere-ai":["partial","*",6,1],"Diffbot":["partial","*",6,1],"ContextFetchBot":["partial","*",6,1]},"llms":{"status":"missing","http":404,"bytes":31272},"home":{"status":"ok","http":200,"url":"https://xmission.com/","html_tokens":12791,"markdown_tokens":193,"pruned_tokens":138}},
{"rank":7189,"domain":"lidalawyer.com","origin":"http://lidalawyer.com","robots":{"status":"missing","http":404,"url":"http://lidalawyer.com/robots.txt","bytes":1231,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":1231},"home":{"status":"ok","http":200,"url":"http://lidalawyer.com/","html_tokens":3789,"markdown_tokens":19,"pruned_tokens":19}},
{"rank":7190,"domain":"idealista.com","origin":"https://idealista.com","robots":{"status":"ok","http":200,"url":"https://www.idealista.com/robots.txt","bytes":16126,"groups":1},"bots":{"GPTBot":["partial","*",186,345],"ChatGPT-User":["partial","*",186,345],"OAI-SearchBot":["partial","*",186,345],"ClaudeBot":["partial","*",186,345],"Claude-User":["partial","*",186,345],"Claude-SearchBot":["partial","*",186,345],"anthropic-ai":["partial","*",186,345],"PerplexityBot":["partial","*",186,345],"Perplexity-User":["partial","*",186,345],"Google-Extended":["partial","*",186,345],"Applebot-Extended":["partial","*",186,345],"CCBot":["partial","*",186,345],"Bytespider":["partial","*",186,345],"Amazonbot":["partial","*",186,345],"meta-externalagent":["partial","*",186,345],"cohere-ai":["partial","*",186,345],"Diffbot":["partial","*",186,345],"ContextFetchBot":["partial","*",186,345]},"llms":{"status":"http_error","http":403,"bytes":773},"home":{"status":"http_error","http":403,"url":"https://www.idealista.com/"}},
{"rank":7191,"domain":"fide.com","origin":"https://fide.com","robots":{"status":"missing","http":404,"url":"https://fide.com/robots.txt","bytes":1075606,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":1075606},"home":{"status":"ok","http":200,"url":"https://www.fide.com/","html_tokens":123936,"markdown_tokens":4,"pruned_tokens":4}},
{"rank":7192,"domain":"afisha.ru","origin":"https://afisha.ru","robots":{"status":"ok","http":200,"url":"https://www.afisha.ru/robots.txt","bytes":7730,"groups":4},"bots":{"GPTBot":["partial","*",62,0],"ChatGPT-User":["partial","*",62,0],"OAI-SearchBot":["partial","*",62,0],"ClaudeBot":["partial","*",62,0],"Claude-User":["partial","*",62,0],"Claude-SearchBot":["partial","*",62,0],"anthropic-ai":["partial","*",62,0],"PerplexityBot":["partial","*",62,0],"Perplexity-User":["partial","*",62,0],"Google-Extended":["partial","*",62,0],"Applebot-Extended":["partial","*",62,0],"CCBot":["partial","*",62,0],"Bytespider":["partial","*",62,0],"Amazonbot":["partial","*",62,0],"meta-externalagent":["partial","*",62,0],"cohere-ai":["partial","*",62,0],"Diffbot":["partial","*",62,0],"ContextFetchBot":["partial","*",62,0]},"llms":{"status":"missing","http":404,"bytes":106167},"home":{"status":"ok","http":200,"url":"https://www.afisha.ru/","html_tokens":926,"markdown_tokens":0,"pruned_tokens":0}},
{"rank":7193,"domain":"diyanglawfirm.com","origin":"https://www.diyanglawfirm.com","robots":{"status":"ok","http":200,"url":"https://www.diyanglawfirm.com/robots.txt","bytes":716,"groups":4},"bots":{"GPTBot":["allowed","*",0,1],"ChatGPT-User":["allowed","*",0,1],"OAI-SearchBot":["allowed","*",0,1],"ClaudeBot":["allowed","*",0,1],"Claude-User":["allowed","*",0,1],"Claude-SearchBot":["allowed","*",0,1],"anthropic-ai":["allowed","*",0,1],"PerplexityBot":["allowed","*",0,1],"Perplexity-User":["allowed","*",0,1],"Google-Extended":["allowed","*",0,1],"Applebot-Extended":["allowed","*",0,1],"CCBot":["allowed","*",0,1],"Bytespider":["allowed","*",0,1],"Amazonbot":["allowed","*",0,1],"meta-externalagent":["allowed","*",0,1],"cohere-ai":["allowed","*",0,1],"Diffbot":["allowed","*",0,1],"ContextFetchBot":["allowed","*",0,1]},"llms":{"status":"missing","http":404,"bytes":0},"home":{"status":"ok","http":200,"url":"https://www.diyanglawfirm.com/","html_tokens":7965,"markdown_tokens":270,"pruned_tokens":164}},
{"rank":7194,"domain":"dfg.de","origin":"https://dfg.de","robots":{"status":"ok","http":200,"url":"https://www.dfg.de/robots.txt","bytes":738,"groups":1},"bots":{"GPTBot":["partial","*",5,0],"ChatGPT-User":["partial","*",5,0],"OAI-SearchBot":["partial","*",5,0],"ClaudeBot":["partial","*",5,0],"Claude-User":["partial","*",5,0],"Claude-SearchBot":["partial","*",5,0],"anthropic-ai":["partial","*",5,0],"PerplexityBot":["partial","*",5,0],"Perplexity-User":["partial","*",5,0],"Google-Extended":["partial","*",5,0],"Applebot-Extended":["partial","*",5,0],"CCBot":["partial","*",5,0],"Bytespider":["partial","*",5,0],"Amazonbot":["partial","*",5,0],"meta-externalagent":["partial","*",5,0],"cohere-ai":["partial","*",5,0],"Diffbot":["partial","*",5,0],"ContextFetchBot":["partial","*",5,0]},"llms":{"status":"missing","http":404,"bytes":343805},"home":{"status":"ok","http":200,"url":"https://www.dfg.de/","html_tokens":95472,"markdown_tokens":84,"pruned_tokens":84}},
{"rank":7195,"domain":"earth.org","origin":"https://earth.org","robots":{"status":"ok","http":200,"url":"https://earth.org/robots.txt","bytes":167,"groups":1},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"missing","http":404,"bytes":91913},"home":{"status":"ok","http":200,"url":"https://earth.org/","html_tokens":55689,"markdown_tokens":68,"pruned_tokens":68}},
{"rank":7196,"domain":"fi.it","origin":"https://fi.it","robots":{"status":"unreachable","http":null,"url":"https://fi.it/robots.txt","bytes":0,"groups":0,"error":"cannot resolve fi.it"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":7197,"domain":"artificialanalysis.ai","origin":"https://artificialanalysis.ai","robots":{"status":"ok","http":200,"url":"https://artificialanalysis.ai/robots.txt","bytes":399,"groups":1},"bots":{"GPTBot":["allowed","*",0,1],"ChatGPT-User":["allowed","*",0,1],"OAI-SearchBot":["allowed","*",0,1],"ClaudeBot":["allowed","*",0,1],"Claude-User":["allowed","*",0,1],"Claude-SearchBot":["allowed","*",0,1],"anthropic-ai":["allowed","*",0,1],"PerplexityBot":["allowed","*",0,1],"Perplexity-User":["allowed","*",0,1],"Google-Extended":["allowed","*",0,1],"Applebot-Extended":["allowed","*",0,1],"CCBot":["allowed","*",0,1],"Bytespider":["allowed","*",0,1],"Amazonbot":["allowed","*",0,1],"meta-externalagent":["allowed","*",0,1],"cohere-ai":["allowed","*",0,1],"Diffbot":["allowed","*",0,1],"ContextFetchBot":["allowed","*",0,1]},"llms":{"status":"missing","http":404,"bytes":14784},"home":{"status":"ok","http":200,"url":"https://artificialanalysis.ai/","html_tokens":369485,"markdown_tokens":424,"pruned_tokens":52}},
{"rank":7198,"domain":"ansys.com","origin":"https://ansys.com","robots":{"status":"ok","http":200,"url":"https://ansys.synopsys.com/robots.txt","bytes":751,"groups":1},"bots":{"GPTBot":["partial","*",7,0],"ChatGPT-User":["partial","*",7,0],"OAI-SearchBot":["partial","*",7,0],"ClaudeBot":["partial","*",7,0],"Claude-User":["partial","*",7,0],"Claude-SearchBot":["partial","*",7,0],"anthropic-ai":["partial","*",7,0],"PerplexityBot":["partial","*",7,0],"Perplexity-User":["partial","*",7,0],"Google-Extended":["partial","*",7,0],"Applebot-Extended":["partial","*",7,0],"CCBot":["partial","*",7,0],"Bytespider":["partial","*",7,0],"Amazonbot":["partial","*",7,0],"meta-externalagent":["partial","*",7,0],"cohere-ai":["partial","*",7,0],"Diffbot":["partial","*",7,0],"ContextFetchBot":["partial","*",7,0]},"llms":{"status":"present","http":200,"bytes":33544},"home":{"status":"ok","http":200,"url":"https://ansys.synopsys.com/","html_tokens":268886,"markdown_tokens":230,"pruned_tokens":127}},
{"rank":7199,"domain":"vektor-inc.co.jp","origin":"http://vektor-inc.co.jp","robots":{"status":"ok","http":200,"url":"https://www.vektor-inc.co.jp/robots.txt","bytes":393,"groups":2},"bots":{"GPTBot":["partial","*",6,1],"ChatGPT-User":["partial","*",6,1],"OAI-SearchBot":["partial","*",6,1],"ClaudeBot":["partial","*",6,1],"Claude-User":["partial","*",6,1],"Claude-SearchBot":["partial","*",6,1],"anthropic-ai":["partial","*",6,1],"PerplexityBot":["partial","*",6,1],"Perplexity-User":["partial","*",6,1],"Google-Extended":["partial","*",6,1],"Applebot-Extended":["partial","*",6,1],"CCBot":["partial","*",6,1],"Bytespider":["partial","*",6,1],"Amazonbot":["partial","*",6,1],"meta-externalagent":["partial","*",6,1],"cohere-ai":["partial","*",6,1],"Diffbot":["partial","*",6,1],"ContextFetchBot":["partial","*",6,1]},"llms":{"status":"disallowed","error":"robots.txt of www.vektor-inc.co.jp not readable (timeout)"},"home":{"status":"disallowed","error":"robots.txt of www.vektor-inc.co.jp not readable (timeout)"}},
{"rank":7200,"domain":"underarmour.com","origin":"https://underarmour.com","robots":{"status":"ok","http":200,"url":"https://www.underarmour.com/robots.txt","bytes":1367,"groups":1},"bots":{"GPTBot":["partial","*",40,1],"ChatGPT-User":["partial","*",40,1],"OAI-SearchBot":["partial","*",40,1],"ClaudeBot":["partial","*",40,1],"Claude-User":["partial","*",40,1],"Claude-SearchBot":["partial","*",40,1],"anthropic-ai":["partial","*",40,1],"PerplexityBot":["partial","*",40,1],"Perplexity-User":["partial","*",40,1],"Google-Extended":["partial","*",40,1],"Applebot-Extended":["partial","*",40,1],"CCBot":["partial","*",40,1],"Bytespider":["partial","*",40,1],"Amazonbot":["partial","*",40,1],"meta-externalagent":["partial","*",40,1],"cohere-ai":["partial","*",40,1],"Diffbot":["partial","*",40,1],"ContextFetchBot":["partial","*",40,1]},"llms":{"status":"present","http":200,"bytes":19502},"home":{"status":"ok","http":200,"url":"https://www.underarmour.com/en-us/","html_tokens":168728,"markdown_tokens":30,"pruned_tokens":30}},
{"rank":7201,"domain":"business-humanrights.org","origin":"https://business-humanrights.org","robots":{"status":"ok","http":200,"url":"https://www.business-humanrights.org/robots.txt","bytes":38,"groups":1},"bots":{"GPTBot":["allowed","*",0,1],"ChatGPT-User":["allowed","*",0,1],"OAI-SearchBot":["allowed","*",0,1],"ClaudeBot":["allowed","*",0,1],"Claude-User":["allowed","*",0,1],"Claude-SearchBot":["allowed","*",0,1],"anthropic-ai":["allowed","*",0,1],"PerplexityBot":["allowed","*",0,1],"Perplexity-User":["allowed","*",0,1],"Google-Extended":["allowed","*",0,1],"Applebot-Extended":["allowed","*",0,1],"CCBot":["allowed","*",0,1],"Bytespider":["allowed","*",0,1],"Amazonbot":["allowed","*",0,1],"meta-externalagent":["allowed","*",0,1],"cohere-ai":["allowed","*",0,1],"Diffbot":["allowed","*",0,1],"ContextFetchBot":["allowed","*",0,1]},"llms":{"status":"missing","http":404,"bytes":111938},"home":{"status":"ok","http":200,"url":"https://www.business-humanrights.org/en/","html_tokens":39283,"markdown_tokens":130,"pruned_tokens":114}},
{"rank":7202,"domain":"destructoid.com","origin":"https://destructoid.com","robots":{"status":"refused","http":403,"url":"https://destructoid.com/robots.txt","bytes":5553,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":7203,"domain":"wimbledon.com","origin":"https://wimbledon.com","robots":{"status":"ok","http":200,"url":"https://www.wimbledon.com/robots.txt","bytes":500,"groups":1},"bots":{"GPTBot":["partial","*",13,0],"ChatGPT-User":["partial","*",13,0],"OAI-SearchBot":["partial","*",13,0],"ClaudeBot":["partial","*",13,0],"Claude-User":["partial","*",13,0],"Claude-SearchBot":["partial","*",13,0],"anthropic-ai":["partial","*",13,0],"PerplexityBot":["partial","*",13,0],"Perplexity-User":["partial","*",13,0],"Google-Extended":["partial","*",13,0],"Applebot-Extended":["partial","*",13,0],"CCBot":["partial","*",13,0],"Bytespider":["partial","*",13,0],"Amazonbot":["partial","*",13,0],"meta-externalagent":["partial","*",13,0],"cohere-ai":["partial","*",13,0],"Diffbot":["partial","*",13,0],"ContextFetchBot":["partial","*",13,0]},"llms":{"status":"missing","http":404,"bytes":214168},"home":{"status":"ok","http":200,"url":"https://www.wimbledon.com/","html_tokens":76500,"markdown_tokens":0,"pruned_tokens":0}},
{"rank":7204,"domain":"vietnamplus.vn","origin":"https://vietnamplus.vn","robots":{"status":"ok","http":200,"url":"https://www.vietnamplus.vn/robots.txt","bytes":2566,"groups":5},"bots":{"GPTBot":["partial","*",74,4],"ChatGPT-User":["partial","*",74,4],"OAI-SearchBot":["partial","*",74,4],"ClaudeBot":["partial","*",74,4],"Claude-User":["partial","*",74,4],"Claude-SearchBot":["partial","*",74,4],"anthropic-ai":["partial","*",74,4],"PerplexityBot":["partial","*",74,4],"Perplexity-User":["partial","*",74,4],"Google-Extended":["partial","*",74,4],"Applebot-Extended":["partial","*",74,4],"CCBot":["partial","*",74,4],"Bytespider":["partial","*",74,4],"Amazonbot":["partial","*",74,4],"meta-externalagent":["partial","*",74,4],"cohere-ai":["partial","*",74,4],"Diffbot":["partial","*",74,4],"ContextFetchBot":["partial","*",74,4]},"llms":{"status":"timeout","error":"timed out after 10 s"},"home":{"status":"disallowed","error":"robots.txt of www.vietnamplus.vn not readable (timeout)"}},
{"rank":7205,"domain":"google.sc","origin":"https://google.sc","robots":{"status":"ok","http":200,"url":"https://www.google.sc/robots.txt","bytes":6624,"groups":4},"bots":{"GPTBot":["partial","*",169,64],"ChatGPT-User":["partial","*",169,64],"OAI-SearchBot":["partial","*",169,64],"ClaudeBot":["partial","*",169,64],"Claude-User":["partial","*",169,64],"Claude-SearchBot":["partial","*",169,64],"anthropic-ai":["partial","*",169,64],"PerplexityBot":["partial","*",169,64],"Perplexity-User":["partial","*",169,64],"Google-Extended":["partial","*",169,64],"Applebot-Extended":["partial","*",169,64],"CCBot":["partial","*",169,64],"Bytespider":["partial","*",169,64],"Amazonbot":["partial","*",169,64],"meta-externalagent":["partial","*",169,64],"cohere-ai":["partial","*",169,64],"Diffbot":["partial","*",169,64],"ContextFetchBot":["partial","*",169,64]},"llms":{"status":"missing","http":404,"bytes":1569},"home":{"status":"ok","http":200,"url":"https://www.google.com/","html_tokens":21300,"markdown_tokens":30,"pruned_tokens":30}},
{"rank":7206,"domain":"directv.com","origin":"https://directv.com","robots":{"status":"ok","http":200,"url":"https://www.directv.com/robots.txt","bytes":718,"groups":2},"bots":{"GPTBot":["partial","*",18,2],"ChatGPT-User":["partial","*",18,2],"OAI-SearchBot":["partial","*",18,2],"ClaudeBot":["partial","*",18,2],"Claude-User":["partial","*",18,2],"Claude-SearchBot":["partial","*",18,2],"anthropic-ai":["partial","*",18,2],"PerplexityBot":["partial","*",18,2],"Perplexity-User":["partial","*",18,2],"Google-Extended":["partial","*",18,2],"Applebot-Extended":["partial","*",18,2],"CCBot":["partial","*",18,2],"Bytespider":["partial","*",18,2],"Amazonbot":["partial","*",18,2],"meta-externalagent":["partial","*",18,2],"cohere-ai":["partial","*",18,2],"Diffbot":["partial","*",18,2],"ContextFetchBot":["partial","*",18,2]},"llms":{"status":"timeout","error":"timed out after 10 s"},"home":{"status":"ok","http":200,"url":"https://www.directv.com/","html_tokens":156253,"markdown_tokens":1184,"pruned_tokens":467}},
{"rank":7207,"domain":"telecinco.es","origin":"https://telecinco.es","robots":{"status":"ok","http":200,"url":"https://www.telecinco.es/robots.txt","bytes":723,"groups":3},"bots":{"GPTBot":["partial","*",16,0],"ChatGPT-User":["partial","*",16,0],"OAI-SearchBot":["partial","*",16,0],"ClaudeBot":["partial","*",16,0],"Claude-User":["partial","*",16,0],"Claude-SearchBot":["partial","*",16,0],"anthropic-ai":["partial","*",16,0],"PerplexityBot":["partial","*",16,0],"Perplexity-User":["partial","*",16,0],"Google-Extended":["partial","*",16,0],"Applebot-Extended":["partial","*",16,0],"CCBot":["partial","*",16,0],"Bytespider":["partial","*",16,0],"Amazonbot":["partial","*",16,0],"meta-externalagent":["partial","*",16,0],"cohere-ai":["partial","*",16,0],"Diffbot":["partial","*",16,0],"ContextFetchBot":["partial","*",16,0]},"llms":{"status":"missing","http":404,"bytes":307609},"home":{"status":"ok","http":200,"url":"https://www.telecinco.es/","html_tokens":167336,"markdown_tokens":86,"pruned_tokens":86}},
{"rank":7208,"domain":"siu.edu","origin":"https://siu.edu","robots":{"status":"timeout","http":null,"url":"https://siu.edu/robots.txt","bytes":0,"groups":0,"error":"timed out after 10 s"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":7209,"domain":"beforeitsnews.com","origin":"https://beforeitsnews.com","robots":{"status":"ok","http":200,"url":"https://beforeitsnews.com/robots.txt","bytes":462,"groups":1},"bots":{"GPTBot":["partial","*",18,1],"ChatGPT-User":["partial","*",18,1],"OAI-SearchBot":["partial","*",18,1],"ClaudeBot":["partial","*",18,1],"Claude-User":["partial","*",18,1],"Claude-SearchBot":["partial","*",18,1],"anthropic-ai":["partial","*",18,1],"PerplexityBot":["partial","*",18,1],"Perplexity-User":["partial","*",18,1],"Google-Extended":["partial","*",18,1],"Applebot-Extended":["partial","*",18,1],"CCBot":["partial","*",18,1],"Bytespider":["partial","*",18,1],"Amazonbot":["partial","*",18,1],"meta-externalagent":["partial","*",18,1],"cohere-ai":["partial","*",18,1],"Diffbot":["partial","*",18,1],"ContextFetchBot":["partial","*",18,1]},"llms":{"status":"missing","http":404,"bytes":15162},"home":{"status":"ok","http":200,"url":"https://beforeitsnews.com/","html_tokens":21791,"markdown_tokens":104,"pruned_tokens":104}},
{"rank":7210,"domain":"espressif.com","origin":"https://espressif.com","robots":{"status":"ok","http":200,"url":"https://www.espressif.com/robots.txt","bytes":1622,"groups":1},"bots":{"GPTBot":["partial","*",41,0],"ChatGPT-User":["partial","*",41,0],"OAI-SearchBot":["partial","*",41,0],"ClaudeBot":["partial","*",41,0],"Claude-User":["partial","*",41,0],"Claude-SearchBot":["partial","*",41,0],"anthropic-ai":["partial","*",41,0],"PerplexityBot":["partial","*",41,0],"Perplexity-User":["partial","*",41,0],"Google-Extended":["partial","*",41,0],"Applebot-Extended":["partial","*",41,0],"CCBot":["partial","*",41,0],"Bytespider":["partial","*",41,0],"Amazonbot":["partial","*",41,0],"meta-externalagent":["partial","*",41,0],"cohere-ai":["partial","*",41,0],"Diffbot":["partial","*",41,0],"ContextFetchBot":["partial","*",41,0]},"llms":{"status":"missing","http":404,"bytes":291},"home":{"status":"ok","http":200,"url":"https://www.espressif.com/","html_tokens":43992,"markdown_tokens":822,"pruned_tokens":68}},
{"rank":7211,"domain":"www.gov.wales","origin":"https://www.gov.wales","robots":{"status":"ok","http":200,"url":"https://www.gov.wales/robots.txt","bytes":190,"groups":1},"bots":{"GPTBot":["partial","*",5,0],"ChatGPT-User":["partial","*",5,0],"OAI-SearchBot":["partial","*",5,0],"ClaudeBot":["partial","*",5,0],"Claude-User":["partial","*",5,0],"Claude-SearchBot":["partial","*",5,0],"anthropic-ai":["partial","*",5,0],"PerplexityBot":["partial","*",5,0],"Perplexity-User":["partial","*",5,0],"Google-Extended":["partial","*",5,0],"Applebot-Extended":["partial","*",5,0],"CCBot":["partial","*",5,0],"Bytespider":["partial","*",5,0],"Amazonbot":["partial","*",5,0],"meta-externalagent":["partial","*",5,0],"cohere-ai":["partial","*",5,0],"Diffbot":["partial","*",5,0],"ContextFetchBot":["partial","*",5,0]},"llms":{"status":"missing","http":404,"bytes":207},"home":{"status":"ok","http":200,"url":"https://www.gov.wales/","html_tokens":7496,"markdown_tokens":502,"pruned_tokens":286}},
{"rank":7212,"domain":"dailymaverick.co.za","origin":"https://dailymaverick.co.za","robots":{"status":"ok","http":200,"url":"https://www.dailymaverick.co.za/robots.txt","bytes":1875,"groups":3},"bots":{"GPTBot":["partial","named",11,1],"ChatGPT-User":["partial","named",11,1],"OAI-SearchBot":["partial","named",11,1],"ClaudeBot":["partial","named",11,1],"Claude-User":["partial","named",11,1],"Claude-SearchBot":["partial","named",11,1],"anthropic-ai":["partial","*",11,0],"PerplexityBot":["partial","named",11,1],"Perplexity-User":["partial","named",11,1],"Google-Extended":["partial","named",11,1],"Applebot-Extended":["partial","named",11,1],"CCBot":["partial","named",11,1],"Bytespider":["partial","named",11,1],"Amazonbot":["partial","named",11,1],"meta-externalagent":["partial","named",11,1],"cohere-ai":["partial","*",11,0],"Diffbot":["partial","*",11,0],"ContextFetchBot":["partial","*",11,0]},"llms":{"status":"present","http":200,"bytes":1498},"home":{"status":"ok","http":200,"url":"https://www.dailymaverick.co.za/","html_tokens":81675,"markdown_tokens":591,"pruned_tokens":57}},
{"rank":7213,"domain":"hongkiat.com","origin":"https://hongkiat.com","robots":{"status":"ok","http":200,"url":"https://www.hongkiat.com/robots.txt","bytes":123,"groups":1},"bots":{"GPTBot":["partial","*",2,0],"ChatGPT-User":["partial","*",2,0],"OAI-SearchBot":["partial","*",2,0],"ClaudeBot":["partial","*",2,0],"Claude-User":["partial","*",2,0],"Claude-SearchBot":["partial","*",2,0],"anthropic-ai":["partial","*",2,0],"PerplexityBot":["partial","*",2,0],"Perplexity-User":["partial","*",2,0],"Google-Extended":["partial","*",2,0],"Applebot-Extended":["partial","*",2,0],"CCBot":["partial","*",2,0],"Bytespider":["partial","*",2,0],"Amazonbot":["partial","*",2,0],"meta-externalagent":["partial","*",2,0],"cohere-ai":["partial","*",2,0],"Diffbot":["partial","*",2,0],"ContextFetchBot":["partial","*",2,0]},"llms":{"status":"http_error","http":403,"bytes":162},"home":{"status":"ok","http":200,"url":"https://www.hongkiat.com/blog/","html_tokens":18695,"markdown_tokens":43,"pruned_tokens":43}},
{"rank":7214,"domain":"redcrossblood.org","origin":"https://redcrossblood.org","robots":{"status":"refused","http":403,"url":"https://www.redcrossblood.org/robots.txt","bytes":389,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":7215,"domain":"usb.org","origin":"https://usb.org","robots":{"status":"ok","http":200,"url":"https://usb.org/robots.txt","bytes":1596,"groups":1},"bots":{"GPTBot":["partial","*",22,18],"ChatGPT-User":["partial","*",22,18],"OAI-SearchBot":["partial","*",22,18],"ClaudeBot":["partial","*",22,18],"Claude-User":["partial","*",22,18],"Claude-SearchBot":["partial","*",22,18],"anthropic-ai":["partial","*",22,18],"PerplexityBot":["partial","*",22,18],"Perplexity-User":["partial","*",22,18],"Google-Extended":["partial","*",22,18],"Applebot-Extended":["partial","*",22,18],"CCBot":["partial","*",22,18],"Bytespider":["partial","*",22,18],"Amazonbot":["partial","*",22,18],"meta-externalagent":["partial","*",22,18],"cohere-ai":["partial","*",22,18],"Diffbot":["partial","*",22,18],"ContextFetchBot":["partial","*",22,18]},"llms":{"status":"missing","http":404,"bytes":181},"home":{"status":"ok","http":200,"url":"https://usb.org/","html_tokens":11504,"markdown_tokens":5,"pruned_tokens":5}},
{"rank":7216,"domain":"goope.jp","origin":"https://goope.jp","robots":{"status":"ok","http":200,"url":"https://goope.jp/robots.txt","bytes":136,"groups":2},"bots":{"GPTBot":["partial","*",1,0],"ChatGPT-User":["partial","*",1,0],"OAI-SearchBot":["partial","*",1,0],"ClaudeBot":["partial","*",1,0],"Claude-User":["partial","*",1,0],"Claude-SearchBot":["partial","*",1,0],"anthropic-ai":["partial","*",1,0],"PerplexityBot":["partial","*",1,0],"Perplexity-User":["partial","*",1,0],"Google-Extended":["partial","*",1,0],"Applebot-Extended":["partial","*",1,0],"CCBot":["partial","*",1,0],"Bytespider":["partial","*",1,0],"Amazonbot":["partial","*",1,0],"meta-externalagent":["partial","*",1,0],"cohere-ai":["partial","*",1,0],"Diffbot":["partial","*",1,0],"ContextFetchBot":["partial","*",1,0]},"llms":{"status":"present","http":200,"bytes":1933},"home":{"status":"ok","http":200,"url":"https://goope.jp/","html_tokens":24921,"markdown_tokens":96,"pruned_tokens":96}},
{"rank":7217,"domain":"chicagobooth.edu","origin":"https://chicagobooth.edu","robots":{"status":"ok","http":200,"url":"https://www.chicagobooth.edu/robots.txt","bytes":2123,"groups":4},"bots":{"GPTBot":["partial","*",71,0],"ChatGPT-User":["partial","*",71,0],"OAI-SearchBot":["partial","*",71,0],"ClaudeBot":["partial","*",71,0],"Claude-User":["partial","*",71,0],"Claude-SearchBot":["partial","*",71,0],"anthropic-ai":["partial","*",71,0],"PerplexityBot":["partial","*",71,0],"Perplexity-User":["partial","*",71,0],"Google-Extended":["partial","*",71,0],"Applebot-Extended":["partial","*",71,0],"CCBot":["partial","*",71,0],"Bytespider":["partial","*",71,0],"Amazonbot":["partial","*",71,0],"meta-externalagent":["partial","*",71,0],"cohere-ai":["partial","*",71,0],"Diffbot":["partial","*",71,0],"ContextFetchBot":["partial","*",71,0]},"llms":{"status":"missing","http":404,"bytes":1245},"home":{"status":"ok","http":200,"url":"https://www.chicagobooth.edu/","html_tokens":43237,"markdown_tokens":604,"pruned_tokens":100}},
{"rank":7218,"domain":"yourtango.com","origin":"https://yourtango.com","robots":{"status":"ok","http":200,"url":"https://www.yourtango.com/robots.txt","bytes":4420,"groups":2},"bots":{"GPTBot":["partial","*",108,22],"ChatGPT-User":["partial","*",108,22],"OAI-SearchBot":["partial","*",108,22],"ClaudeBot":["partial","*",108,22],"Claude-User":["partial","*",108,22],"Claude-SearchBot":["partial","*",108,22],"anthropic-ai":["partial","*",108,22],"PerplexityBot":["partial","*",108,22],"Perplexity-User":["partial","*",108,22],"Google-Extended":["partial","*",108,22],"Applebot-Extended":["partial","*",108,22],"CCBot":["partial","*",108,22],"Bytespider":["partial","*",108,22],"Amazonbot":["partial","*",108,22],"meta-externalagent":["partial","*",108,22],"cohere-ai":["partial","*",108,22],"Diffbot":["partial","*",108,22],"ContextFetchBot":["partial","*",108,22]},"llms":{"status":"http_error","http":403,"bytes":146},"home":{"status":"ok","http":200,"url":"https://www.yourtango.com/","html_tokens":41346,"markdown_tokens":49,"pruned_tokens":49}},
{"rank":7219,"domain":"zmescience.com","origin":"https://zmescience.com","robots":{"status":"ok","http":200,"url":"https://www.zmescience.com/robots.txt","bytes":118,"groups":1},"bots":{"GPTBot":["partial","*",1,1],"ChatGPT-User":["partial","*",1,1],"OAI-SearchBot":["partial","*",1,1],"ClaudeBot":["partial","*",1,1],"Claude-User":["partial","*",1,1],"Claude-SearchBot":["partial","*",1,1],"anthropic-ai":["partial","*",1,1],"PerplexityBot":["partial","*",1,1],"Perplexity-User":["partial","*",1,1],"Google-Extended":["partial","*",1,1],"Applebot-Extended":["partial","*",1,1],"CCBot":["partial","*",1,1],"Bytespider":["partial","*",1,1],"Amazonbot":["partial","*",1,1],"meta-externalagent":["partial","*",1,1],"cohere-ai":["partial","*",1,1],"Diffbot":["partial","*",1,1],"ContextFetchBot":["partial","*",1,1]},"llms":{"status":"missing","http":404,"bytes":226357},"home":{"status":"ok","http":200,"url":"https://www.zmescience.com/","html_tokens":122229,"markdown_tokens":48,"pruned_tokens":48}},
{"rank":7220,"domain":"google.com.sv","origin":"https://google.com.sv","robots":{"status":"ok","http":200,"url":"https://www.google.com.sv/robots.txt","bytes":6624,"groups":4},"bots":{"GPTBot":["partial","*",169,64],"ChatGPT-User":["partial","*",169,64],"OAI-SearchBot":["partial","*",169,64],"ClaudeBot":["partial","*",169,64],"Claude-User":["partial","*",169,64],"Claude-SearchBot":["partial","*",169,64],"anthropic-ai":["partial","*",169,64],"PerplexityBot":["partial","*",169,64],"Perplexity-User":["partial","*",169,64],"Google-Extended":["partial","*",169,64],"Applebot-Extended":["partial","*",169,64],"CCBot":["partial","*",169,64],"Bytespider":["partial","*",169,64],"Amazonbot":["partial","*",169,64],"meta-externalagent":["partial","*",169,64],"cohere-ai":["partial","*",169,64],"Diffbot":["partial","*",169,64],"ContextFetchBot":["partial","*",169,64]},"llms":{"status":"missing","http":404,"bytes":1569},"home":{"status":"ok","http":200,"url":"https://www.google.com/","html_tokens":21383,"markdown_tokens":30,"pruned_tokens":30}},
{"rank":7221,"domain":"mundiario.com","origin":"https://mundiario.com","robots":{"status":"ok","http":200,"url":"https://www.mundiario.com/robots.txt","bytes":294,"groups":1},"bots":{"GPTBot":["partial","*",8,0],"ChatGPT-User":["partial","*",8,0],"OAI-SearchBot":["partial","*",8,0],"ClaudeBot":["partial","*",8,0],"Claude-User":["partial","*",8,0],"Claude-SearchBot":["partial","*",8,0],"anthropic-ai":["partial","*",8,0],"PerplexityBot":["partial","*",8,0],"Perplexity-User":["partial","*",8,0],"Google-Extended":["partial","*",8,0],"Applebot-Extended":["partial","*",8,0],"CCBot":["partial","*",8,0],"Bytespider":["partial","*",8,0],"Amazonbot":["partial","*",8,0],"meta-externalagent":["partial","*",8,0],"cohere-ai":["partial","*",8,0],"Diffbot":["partial","*",8,0],"ContextFetchBot":["partial","*",8,0]},"llms":{"status":"missing","http":404,"bytes":431737},"home":{"status":"ok","http":200,"url":"https://www.mundiario.com/","html_tokens":234134,"markdown_tokens":109,"pruned_tokens":81}},
{"rank":7222,"domain":"leanpub.com","origin":"https://leanpub.com","robots":{"status":"ok","http":200,"url":"https://leanpub.com/robots.txt","bytes":648,"groups":2},"bots":{"GPTBot":["partial","*",9,0],"ChatGPT-User":["partial","*",9,0],"OAI-SearchBot":["partial","*",9,0],"ClaudeBot":["partial","*",9,0],"Claude-User":["partial","*",9,0],"Claude-SearchBot":["partial","*",9,0],"anthropic-ai":["partial","*",9,0],"PerplexityBot":["partial","*",9,0],"Perplexity-User":["partial","*",9,0],"Google-Extended":["partial","*",9,0],"Applebot-Extended":["partial","*",9,0],"CCBot":["partial","*",9,0],"Bytespider":["partial","*",9,0],"Amazonbot":["partial","*",9,0],"meta-externalagent":["partial","*",9,0],"cohere-ai":["partial","*",9,0],"Diffbot":["partial","*",9,0],"ContextFetchBot":["partial","*",9,0]},"llms":{"status":"missing","http":404,"bytes":66198},"home":{"status":"ok","http":200,"url":"https://leanpub.com/","html_tokens":135481,"markdown_tokens":3876,"pruned_tokens":913}},
{"rank":7223,"domain":"gobankingrates.com","origin":"https://gobankingrates.com","robots":{"status":"ok","http":200,"url":"https://www.gobankingrates.com/robots.txt","bytes":571,"groups":1},"bots":{"GPTBot":["partial","*",20,1],"ChatGPT-User":["partial","*",20,1],"OAI-SearchBot":["partial","*",20,1],"ClaudeBot":["partial","*",20,1],"Claude-User":["partial","*",20,1],"Claude-SearchBot":["partial","*",20,1],"anthropic-ai":["partial","*",20,1],"PerplexityBot":["partial","*",20,1],"Perplexity-User":["partial","*",20,1],"Google-Extended":["partial","*",20,1],"Applebot-Extended":["partial","*",20,1],"CCBot":["partial","*",20,1],"Bytespider":["partial","*",20,1],"Amazonbot":["partial","*",20,1],"meta-externalagent":["partial","*",20,1],"cohere-ai":["partial","*",20,1],"Diffbot":["partial","*",20,1],"ContextFetchBot":["partial","*",20,1]},"llms":{"status":"missing","http":404,"bytes":265195},"home":{"status":"ok","http":200,"url":"https://www.gobankingrates.com/","html_tokens":80013,"markdown_tokens":262,"pruned_tokens":35}},
{"rank":7224,"domain":"jnto.go.jp","origin":"https://www.jnto.go.jp","robots":{"status":"missing","http":404,"url":"https://www.jnto.go.jp/robots.txt","bytes":26858,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":26858},"home":{"status":"ok","http":200,"url":"https://www.jnto.go.jp/","html_tokens":11735,"markdown_tokens":77,"pruned_tokens":77}},
{"rank":7225,"domain":"maritime-executive.com","origin":"https://maritime-executive.com","robots":{"status":"ok","http":200,"url":"https://maritime-executive.com/robots.txt","bytes":727,"groups":1},"bots":{"GPTBot":["partial","*",16,11],"ChatGPT-User":["partial","*",16,11],"OAI-SearchBot":["partial","*",16,11],"ClaudeBot":["partial","*",16,11],"Claude-User":["partial","*",16,11],"Claude-SearchBot":["partial","*",16,11],"anthropic-ai":["partial","*",16,11],"PerplexityBot":["partial","*",16,11],"Perplexity-User":["partial","*",16,11],"Google-Extended":["partial","*",16,11],"Applebot-Extended":["partial","*",16,11],"CCBot":["partial","*",16,11],"Bytespider":["partial","*",16,11],"Amazonbot":["partial","*",16,11],"meta-externalagent":["partial","*",16,11],"cohere-ai":["partial","*",16,11],"Diffbot":["partial","*",16,11],"ContextFetchBot":["partial","*",16,11]},"llms":{"status":"missing","http":404,"bytes":73207},"home":{"status":"ok","http":200,"url":"https://maritime-executive.com/","html_tokens":53474,"markdown_tokens":149,"pruned_tokens":21}},
{"rank":7226,"domain":"google.co.bw","origin":"https://google.co.bw","robots":{"status":"ok","http":200,"url":"https://www.google.co.bw/robots.txt","bytes":6624,"groups":4},"bots":{"GPTBot":["partial","*",169,64],"ChatGPT-User":["partial","*",169,64],"OAI-SearchBot":["partial","*",169,64],"ClaudeBot":["partial","*",169,64],"Claude-User":["partial","*",169,64],"Claude-SearchBot":["partial","*",169,64],"anthropic-ai":["partial","*",169,64],"PerplexityBot":["partial","*",169,64],"Perplexity-User":["partial","*",169,64],"Google-Extended":["partial","*",169,64],"Applebot-Extended":["partial","*",169,64],"CCBot":["partial","*",169,64],"Bytespider":["partial","*",169,64],"Amazonbot":["partial","*",169,64],"meta-externalagent":["partial","*",169,64],"cohere-ai":["partial","*",169,64],"Diffbot":["partial","*",169,64],"ContextFetchBot":["partial","*",169,64]},"llms":{"status":"missing","http":404,"bytes":1569},"home":{"status":"ok","http":200,"url":"https://www.google.com/","html_tokens":21329,"markdown_tokens":30,"pruned_tokens":30}},
{"rank":7227,"domain":"bcm.edu","origin":"https://bcm.edu","robots":{"status":"ok","http":200,"url":"https://www.bcm.edu/robots.txt","bytes":1724,"groups":2},"bots":{"GPTBot":["partial","*",23,18],"ChatGPT-User":["partial","*",23,18],"OAI-SearchBot":["partial","*",23,18],"ClaudeBot":["partial","*",23,18],"Claude-User":["partial","*",23,18],"Claude-SearchBot":["partial","*",23,18],"anthropic-ai":["partial","*",23,18],"PerplexityBot":["partial","*",23,18],"Perplexity-User":["partial","*",23,18],"Google-Extended":["partial","*",23,18],"Applebot-Extended":["partial","*",23,18],"CCBot":["partial","*",23,18],"Bytespider":["partial","*",23,18],"Amazonbot":["partial","*",23,18],"meta-externalagent":["partial","*",23,18],"cohere-ai":["partial","*",23,18],"Diffbot":["partial","*",23,18],"ContextFetchBot":["partial","*",23,18]},"llms":{"status":"missing","http":404,"bytes":502},"home":{"status":"ok","http":200,"url":"https://www.bcm.edu/","html_tokens":38281,"markdown_tokens":271,"pruned_tokens":102}},
{"rank":7228,"domain":"steinberg.net","origin":"https://steinberg.net","robots":{"status":"ok","http":200,"url":"https://www.steinberg.net/robots.txt","bytes":46,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"error","error":"more than 5 redirects"},"home":{"status":"ok","http":200,"url":"https://www.steinberg.net/","html_tokens":30911,"markdown_tokens":19,"pruned_tokens":19}},
{"rank":7229,"domain":"caixin.com","origin":"https://caixin.com","robots":{"status":"ok","http":200,"url":"https://www.caixin.com/robots.txt","bytes":232,"groups":4},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["partial","*",5,0],"OAI-SearchBot":["partial","*",5,0],"ClaudeBot":["partial","*",5,0],"Claude-User":["partial","*",5,0],"Claude-SearchBot":["partial","*",5,0],"anthropic-ai":["partial","*",5,0],"PerplexityBot":["partial","*",5,0],"Perplexity-User":["partial","*",5,0],"Google-Extended":["partial","*",5,0],"Applebot-Extended":["partial","*",5,0],"CCBot":["blocked","named",1,0],"Bytespider":["partial","*",5,0],"Amazonbot":["partial","*",5,0],"meta-externalagent":["partial","*",5,0],"cohere-ai":["partial","*",5,0],"Diffbot":["partial","*",5,0],"ContextFetchBot":["partial","*",5,0]},"llms":{"status":"disallowed","error":"robots.txt of other.caixin.com not readable (html)"},"home":{"status":"ok","http":200,"url":"https://www.caixin.com/","html_tokens":21874,"markdown_tokens":344,"pruned_tokens":172}},
{"rank":7230,"domain":"bvsalud.org","origin":"https://bvsalud.org","robots":{"status":"timeout","http":null,"url":"https://bvsalud.org/robots.txt","bytes":0,"groups":0,"error":"ConnectTimeout"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":7231,"domain":"securityheaders.com","origin":"https://securityheaders.com","robots":{"status":"ok","http":200,"url":"https://securityheaders.com/robots.txt","bytes":22,"groups":1},"bots":{"GPTBot":["allowed","*",0,1],"ChatGPT-User":["allowed","*",0,1],"OAI-SearchBot":["allowed","*",0,1],"ClaudeBot":["allowed","*",0,1],"Claude-User":["allowed","*",0,1],"Claude-SearchBot":["allowed","*",0,1],"anthropic-ai":["allowed","*",0,1],"PerplexityBot":["allowed","*",0,1],"Perplexity-User":["allowed","*",0,1],"Google-Extended":["allowed","*",0,1],"Applebot-Extended":["allowed","*",0,1],"CCBot":["allowed","*",0,1],"Bytespider":["allowed","*",0,1],"Amazonbot":["allowed","*",0,1],"meta-externalagent":["allowed","*",0,1],"cohere-ai":["allowed","*",0,1],"Diffbot":["allowed","*",0,1],"ContextFetchBot":["allowed","*",0,1]},"llms":{"status":"missing","http":404,"bytes":1491},"home":{"status":"http_error","http":403,"url":"https://securityheaders.com/"}},
{"rank":7232,"domain":"clients.site","origin":"https://www.clients.site","robots":{"status":"ok","http":200,"url":"https://www.clients.site/robots.txt","bytes":70,"groups":1},"bots":{"GPTBot":["allowed","*",0,1],"ChatGPT-User":["allowed","*",0,1],"OAI-SearchBot":["allowed","*",0,1],"ClaudeBot":["allowed","*",0,1],"Claude-User":["allowed","*",0,1],"Claude-SearchBot":["allowed","*",0,1],"anthropic-ai":["allowed","*",0,1],"PerplexityBot":["allowed","*",0,1],"Perplexity-User":["allowed","*",0,1],"Google-Extended":["allowed","*",0,1],"Applebot-Extended":["allowed","*",0,1],"CCBot":["allowed","*",0,1],"Bytespider":["allowed","*",0,1],"Amazonbot":["allowed","*",0,1],"meta-externalagent":["allowed","*",0,1],"cohere-ai":["allowed","*",0,1],"Diffbot":["allowed","*",0,1],"ContextFetchBot":["allowed","*",0,1]},"llms":{"status":"missing","http":404,"bytes":2573},"home":{"status":"ok","http":200,"url":"https://www.clients.site/","html_tokens":11429,"markdown_tokens":12,"pruned_tokens":12}},
{"rank":7233,"domain":"ah.gov.cn","origin":"https://www.ah.gov.cn","robots":{"status":"refused","http":403,"url":"https://www.ah.gov.cn/robots.txt","bytes":287,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":7234,"domain":"ettoday.net","origin":"https://ettoday.net","robots":{"status":"ok","http":200,"url":"https://www.ettoday.net/robots.txt","bytes":1472,"groups":8},"bots":{"GPTBot":["allowed","named",0,1],"ChatGPT-User":["allowed","named",0,1],"OAI-SearchBot":["allowed","named",0,1],"ClaudeBot":["allowed","named",0,1],"Claude-User":["partial","*",37,0],"Claude-SearchBot":["partial","*",37,0],"anthropic-ai":["partial","*",37,0],"PerplexityBot":["allowed","named",0,1],"Perplexity-User":["partial","*",37,0],"Google-Extended":["partial","*",37,0],"Applebot-Extended":["partial","*",37,0],"CCBot":["partial","*",37,0],"Bytespider":["partial","*",37,0],"Amazonbot":["allowed","named",0,1],"meta-externalagent":["partial","*",37,0],"cohere-ai":["partial","*",37,0],"Diffbot":["partial","*",37,0],"ContextFetchBot":["partial","*",37,0]},"llms":{"status":"html","http":200,"bytes":287329},"home":{"status":"ok","http":200,"url":"https://www.ettoday.net/","html_tokens":66682,"markdown_tokens":255,"pruned_tokens":4}},
{"rank":7235,"domain":"geoscienceworld.org","origin":"https://geoscienceworld.org","robots":{"status":"ok","http":200,"url":"https://geoscienceworld.org/robots.txt","bytes":2327,"groups":2},"bots":{"GPTBot":["partial","*",35,3],"ChatGPT-User":["partial","*",35,3],"OAI-SearchBot":["partial","*",35,3],"ClaudeBot":["partial","*",35,3],"Claude-User":["partial","*",35,3],"Claude-SearchBot":["partial","*",35,3],"anthropic-ai":["partial","*",35,3],"PerplexityBot":["partial","*",35,3],"Perplexity-User":["partial","*",35,3],"Google-Extended":["partial","*",35,3],"Applebot-Extended":["partial","*",35,3],"CCBot":["partial","*",35,3],"Bytespider":["partial","*",35,3],"Amazonbot":["partial","*",35,3],"meta-externalagent":["partial","*",35,3],"cohere-ai":["partial","*",35,3],"Diffbot":["partial","*",35,3],"ContextFetchBot":["partial","*",35,3]},"llms":{"status":"http_error","http":403,"bytes":5535},"home":{"status":"http_error","http":403,"url":"https://pubs.geoscienceworld.org/"}},
{"rank":7236,"domain":"2345.com","origin":"https://2345.com","robots":{"status":"ok","http":200,"url":"https://www.2345.com/robots.txt","bytes":44,"groups":1},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":3833},"home":{"status":"ok","http":200,"url":"https://www.2345.com/","html_tokens":41629,"markdown_tokens":29,"pruned_tokens":29}},
{"rank":7237,"domain":"guru99.com","origin":"https://guru99.com","robots":{"status":"ok","http":200,"url":"https://www.guru99.com/robots.txt","bytes":7061,"groups":3},"bots":{"GPTBot":["partial","*",4,0],"ChatGPT-User":["partial","*",4,0],"OAI-SearchBot":["partial","*",4,0],"ClaudeBot":["partial","*",4,0],"Claude-User":["partial","*",4,0],"Claude-SearchBot":["partial","*",4,0],"anthropic-ai":["partial","*",4,0],"PerplexityBot":["partial","*",4,0],"Perplexity-User":["partial","*",4,0],"Google-Extended":["partial","*",4,0],"Applebot-Extended":["partial","*",4,0],"CCBot":["partial","*",4,0],"Bytespider":["partial","*",4,0],"Amazonbot":["partial","*",4,0],"meta-externalagent":["partial","*",4,0],"cohere-ai":["partial","*",4,0],"Diffbot":["partial","*",4,0],"ContextFetchBot":["partial","*",4,0]},"llms":{"status":"other_text","http":200,"bytes":1196645},"home":{"status":"ok","http":200,"url":"https://www.guru99.com/","html_tokens":136976,"markdown_tokens":178,"pruned_tokens":17}},
{"rank":7238,"domain":"foxit.com","origin":"https://foxit.com","robots":{"status":"ok","http":200,"url":"https://www.foxit.com/robots.txt","bytes":555,"groups":7},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["partial","*",11,0],"OAI-SearchBot":["partial","*",11,0],"ClaudeBot":["partial","*",11,0],"Claude-User":["partial","*",11,0],"Claude-SearchBot":["partial","*",11,0],"anthropic-ai":["partial","*",11,0],"PerplexityBot":["partial","*",11,0],"Perplexity-User":["partial","*",11,0],"Google-Extended":["partial","*",11,0],"Applebot-Extended":["partial","*",11,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["partial","*",11,0],"cohere-ai":["partial","*",11,0],"Diffbot":["partial","*",11,0],"ContextFetchBot":["partial","*",11,0]},"llms":{"status":"present","http":200,"bytes":6170},"home":{"status":"ok","http":200,"url":"https://www.foxit.com/","html_tokens":360829,"markdown_tokens":261,"pruned_tokens":59}},
{"rank":7239,"domain":"mucha-mayana-slots.com","origin":"https://mucha-mayana-slots.com","robots":{"status":"ok","http":200,"url":"https://mucha-mayana-slots.com/robots.txt","bytes":548,"groups":1},"bots":{"GPTBot":["partial","*",19,0],"ChatGPT-User":["partial","*",19,0],"OAI-SearchBot":["partial","*",19,0],"ClaudeBot":["partial","*",19,0],"Claude-User":["partial","*",19,0],"Claude-SearchBot":["partial","*",19,0],"anthropic-ai":["partial","*",19,0],"PerplexityBot":["partial","*",19,0],"Perplexity-User":["partial","*",19,0],"Google-Extended":["partial","*",19,0],"Applebot-Extended":["partial","*",19,0],"CCBot":["partial","*",19,0],"Bytespider":["partial","*",19,0],"Amazonbot":["partial","*",19,0],"meta-externalagent":["partial","*",19,0],"cohere-ai":["partial","*",19,0],"Diffbot":["partial","*",19,0],"ContextFetchBot":["partial","*",19,0]},"llms":{"status":"missing","http":404,"bytes":42294},"home":{"status":"ok","http":200,"url":"https://mucha-mayana-slots.com/","html_tokens":25897,"markdown_tokens":1381,"pruned_tokens":334}},
{"rank":7240,"domain":"spiedigitallibrary.org","origin":"https://spiedigitallibrary.org","robots":{"status":"html","http":200,"url":"https://www.spiedigitallibrary.org/robots.txt","bytes":1164,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":7241,"domain":"zoosnet.net","origin":"https://zoosnet.net","robots":{"status":"ok","http":200,"url":"https://zoosnet.net/robots.txt","bytes":44,"groups":1},"bots":{"GPTBot":["partial","*",1,0],"ChatGPT-User":["partial","*",1,0],"OAI-SearchBot":["partial","*",1,0],"ClaudeBot":["partial","*",1,0],"Claude-User":["partial","*",1,0],"Claude-SearchBot":["partial","*",1,0],"anthropic-ai":["partial","*",1,0],"PerplexityBot":["partial","*",1,0],"Perplexity-User":["partial","*",1,0],"Google-Extended":["partial","*",1,0],"Applebot-Extended":["partial","*",1,0],"CCBot":["partial","*",1,0],"Bytespider":["partial","*",1,0],"Amazonbot":["partial","*",1,0],"meta-externalagent":["partial","*",1,0],"cohere-ai":["partial","*",1,0],"Diffbot":["partial","*",1,0],"ContextFetchBot":["partial","*",1,0]},"llms":{"status":"missing","http":404,"bytes":1163},"home":{"status":"ok","http":200,"url":"https://zoosnet.net/","html_tokens":7821,"markdown_tokens":117,"pruned_tokens":117}},
{"rank":7242,"domain":"bjlhlawyer.com","origin":"http://bjlhlawyer.com","robots":{"status":"missing","http":404,"url":"http://bjlhlawyer.com/robots.txt","bytes":5075,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":5071},"home":{"status":"ok","http":200,"url":"http://bjlhlawyer.com/","html_tokens":22343,"markdown_tokens":1373,"pruned_tokens":113}},
{"rank":7243,"domain":"lbc.co.uk","origin":"https://lbc.co.uk","robots":{"status":"ok","http":200,"url":"https://www.lbc.co.uk/robots.txt","bytes":614,"groups":14},"bots":{"GPTBot":["partial","named",1,0],"ChatGPT-User":["partial","named",1,0],"OAI-SearchBot":["partial","*",1,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",1,0],"Claude-SearchBot":["partial","*",1,0],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["partial","*",1,0],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["partial","*",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["partial","*",1,0],"meta-externalagent":["partial","*",1,0],"cohere-ai":["partial","*",1,0],"Diffbot":["partial","*",1,0],"ContextFetchBot":["partial","*",1,0]},"llms":{"status":"missing","http":404,"bytes":55642},"home":{"status":"ok","http":200,"url":"https://www.lbc.co.uk/","html_tokens":92685,"markdown_tokens":2185,"pruned_tokens":36}},
{"rank":7244,"domain":"weddingbee.com","origin":"https://weddingbee.com","robots":{"status":"ok","http":200,"url":"https://www.weddingbee.com/robots.txt","bytes":361,"groups":4},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["partial","*",1,0],"OAI-SearchBot":["partial","*",1,0],"ClaudeBot":["partial","*",1,0],"Claude-User":["partial","*",1,0],"Claude-SearchBot":["partial","*",1,0],"anthropic-ai":["partial","*",1,0],"PerplexityBot":["partial","*",1,0],"Perplexity-User":["partial","*",1,0],"Google-Extended":["partial","*",1,0],"Applebot-Extended":["partial","*",1,0],"CCBot":["partial","*",1,0],"Bytespider":["partial","*",1,0],"Amazonbot":["partial","*",1,0],"meta-externalagent":["partial","*",1,0],"cohere-ai":["partial","*",1,0],"Diffbot":["partial","*",1,0],"ContextFetchBot":["partial","*",1,0]},"llms":{"status":"missing","http":404,"bytes":79220},"home":{"status":"ok","http":200,"url":"https://www.weddingbee.com/","html_tokens":16392,"markdown_tokens":0,"pruned_tokens":0}},
{"rank":7245,"domain":"hermitagemuseum.org","origin":"https://hermitagemuseum.org","robots":{"status":"unreachable","http":null,"url":"https://hermitagemuseum.org/robots.txt","bytes":0,"groups":0,"error":"[SSL: CERTIFICATE_VERIFY_FAILED] certificate verify failed: unable to get local issuer certificate (_ssl.c:1028)"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":7246,"domain":"ipetitions.com","origin":"https://ipetitions.com","robots":{"status":"ok","http":200,"url":"https://www.ipetitions.com/robots.txt","bytes":851,"groups":1},"bots":{"GPTBot":["partial","*",30,0],"ChatGPT-User":["partial","*",30,0],"OAI-SearchBot":["partial","*",30,0],"ClaudeBot":["partial","*",30,0],"Claude-User":["partial","*",30,0],"Claude-SearchBot":["partial","*",30,0],"anthropic-ai":["partial","*",30,0],"PerplexityBot":["partial","*",30,0],"Perplexity-User":["partial","*",30,0],"Google-Extended":["partial","*",30,0],"Applebot-Extended":["partial","*",30,0],"CCBot":["partial","*",30,0],"Bytespider":["partial","*",30,0],"Amazonbot":["partial","*",30,0],"meta-externalagent":["partial","*",30,0],"cohere-ai":["partial","*",30,0],"Diffbot":["partial","*",30,0],"ContextFetchBot":["partial","*",30,0]},"llms":{"status":"present","http":200,"bytes":7819},"home":{"status":"http_error","http":403,"url":"https://www.ipetitions.com/"}},
{"rank":7247,"domain":"neea.edu.cn","origin":"https://www.neea.edu.cn","robots":{"status":"missing","http":404,"url":"https://www.neea.edu.cn/robots.txt","bytes":17074,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":17074},"home":{"status":"ok","http":200,"url":"https://www.neea.edu.cn/","html_tokens":15533,"markdown_tokens":71,"pruned_tokens":71}},
{"rank":7248,"domain":"latex-project.org","origin":"https://latex-project.org","robots":{"status":"ok","http":200,"url":"https://latex-project.org/robots.txt","bytes":207,"groups":2},"bots":{"GPTBot":["partial","*",3,0],"ChatGPT-User":["partial","*",3,0],"OAI-SearchBot":["partial","*",3,0],"ClaudeBot":["partial","*",3,0],"Claude-User":["partial","*",3,0],"Claude-SearchBot":["partial","*",3,0],"anthropic-ai":["partial","*",3,0],"PerplexityBot":["partial","*",3,0],"Perplexity-User":["partial","*",3,0],"Google-Extended":["partial","*",3,0],"Applebot-Extended":["partial","*",3,0],"CCBot":["partial","*",3,0],"Bytespider":["partial","*",3,0],"Amazonbot":["partial","*",3,0],"meta-externalagent":["partial","*",3,0],"cohere-ai":["partial","*",3,0],"Diffbot":["partial","*",3,0],"ContextFetchBot":["partial","*",3,0]},"llms":{"status":"missing","http":404,"bytes":146},"home":{"status":"ok","http":200,"url":"https://latex-project.org/","html_tokens":3039,"markdown_tokens":226,"pruned_tokens":68}},
{"rank":7249,"domain":"google.co.ck","origin":"https://google.co.ck","robots":{"status":"ok","http":200,"url":"https://www.google.co.ck/robots.txt","bytes":6624,"groups":4},"bots":{"GPTBot":["partial","*",169,64],"ChatGPT-User":["partial","*",169,64],"OAI-SearchBot":["partial","*",169,64],"ClaudeBot":["partial","*",169,64],"Claude-User":["partial","*",169,64],"Claude-SearchBot":["partial","*",169,64],"anthropic-ai":["partial","*",169,64],"PerplexityBot":["partial","*",169,64],"Perplexity-User":["partial","*",169,64],"Google-Extended":["partial","*",169,64],"Applebot-Extended":["partial","*",169,64],"CCBot":["partial","*",169,64],"Bytespider":["partial","*",169,64],"Amazonbot":["partial","*",169,64],"meta-externalagent":["partial","*",169,64],"cohere-ai":["partial","*",169,64],"Diffbot":["partial","*",169,64],"ContextFetchBot":["partial","*",169,64]},"llms":{"status":"missing","http":404,"bytes":1569},"home":{"status":"ok","http":200,"url":"https://www.google.com/","html_tokens":21312,"markdown_tokens":30,"pruned_tokens":30}},
{"rank":7250,"domain":"raceresult.com","origin":"https://raceresult.com","robots":{"status":"ok","http":200,"url":"https://www.raceresult.com/robots.txt","bytes":41,"groups":1},"bots":{"GPTBot":["partial","*",1,0],"ChatGPT-User":["partial","*",1,0],"OAI-SearchBot":["partial","*",1,0],"ClaudeBot":["partial","*",1,0],"Claude-User":["partial","*",1,0],"Claude-SearchBot":["partial","*",1,0],"anthropic-ai":["partial","*",1,0],"PerplexityBot":["partial","*",1,0],"Perplexity-User":["partial","*",1,0],"Google-Extended":["partial","*",1,0],"Applebot-Extended":["partial","*",1,0],"CCBot":["partial","*",1,0],"Bytespider":["partial","*",1,0],"Amazonbot":["partial","*",1,0],"meta-externalagent":["partial","*",1,0],"cohere-ai":["partial","*",1,0],"Diffbot":["partial","*",1,0],"ContextFetchBot":["partial","*",1,0]},"llms":{"status":"missing","http":404,"bytes":1024},"home":{"status":"ok","http":200,"url":"https://www.raceresult.com/en-us/home/index","html_tokens":18834,"markdown_tokens":532,"pruned_tokens":153}},
{"rank":7251,"domain":"geni.com","origin":"https://geni.com","robots":{"status":"ok","http":200,"url":"https://www.geni.com/robots.txt","bytes":1504,"groups":2},"bots":{"GPTBot":["partial","*",36,0],"ChatGPT-User":["partial","*",36,0],"OAI-SearchBot":["partial","*",36,0],"ClaudeBot":["partial","*",36,0],"Claude-User":["partial","*",36,0],"Claude-SearchBot":["partial","*",36,0],"anthropic-ai":["partial","*",36,0],"PerplexityBot":["partial","*",36,0],"Perplexity-User":["partial","*",36,0],"Google-Extended":["partial","*",36,0],"Applebot-Extended":["partial","*",36,0],"CCBot":["partial","*",36,0],"Bytespider":["partial","*",36,0],"Amazonbot":["partial","*",36,0],"meta-externalagent":["partial","*",36,0],"cohere-ai":["partial","*",36,0],"Diffbot":["partial","*",36,0],"ContextFetchBot":["partial","*",36,0]},"llms":{"status":"missing","http":404,"bytes":56824},"home":{"status":"ok","http":200,"url":"https://www.geni.com/","html_tokens":37340,"markdown_tokens":143,"pruned_tokens":143}},
{"rank":7252,"domain":"hy315.cc","origin":"http://hy315.cc","robots":{"status":"missing","http":404,"url":"http://hy315.cc/robots.txt","bytes":5055,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":5051},"home":{"status":"ok","http":200,"url":"http://hy315.cc/","html_tokens":995,"markdown_tokens":32,"pruned_tokens":32}},
{"rank":7253,"domain":"isap.sejm.gov.pl","origin":"https://isap.sejm.gov.pl","robots":{"status":"ok","http":200,"url":"https://isap.sejm.gov.pl/robots.txt","bytes":93,"groups":3},"bots":{"GPTBot":["blocked","*",1,0],"ChatGPT-User":["blocked","*",1,0],"OAI-SearchBot":["blocked","*",1,0],"ClaudeBot":["blocked","*",1,0],"Claude-User":["blocked","*",1,0],"Claude-SearchBot":["blocked","*",1,0],"anthropic-ai":["blocked","*",1,0],"PerplexityBot":["blocked","*",1,0],"Perplexity-User":["blocked","*",1,0],"Google-Extended":["blocked","*",1,0],"Applebot-Extended":["blocked","*",1,0],"CCBot":["blocked","*",1,0],"Bytespider":["blocked","*",1,0],"Amazonbot":["blocked","*",1,0],"meta-externalagent":["blocked","*",1,0],"cohere-ai":["blocked","*",1,0],"Diffbot":["blocked","*",1,0],"ContextFetchBot":["blocked","*",1,0]},"llms":{"status":"disallowed"},"home":{"status":"disallowed"}},
{"rank":7254,"domain":"about.youtube","origin":"https://about.youtube","robots":{"status":"ok","http":200,"url":"https://about.youtube/robots.txt","bytes":67,"groups":1},"bots":{"GPTBot":["allowed","*",0,1],"ChatGPT-User":["allowed","*",0,1],"OAI-SearchBot":["allowed","*",0,1],"ClaudeBot":["allowed","*",0,1],"Claude-User":["allowed","*",0,1],"Claude-SearchBot":["allowed","*",0,1],"anthropic-ai":["allowed","*",0,1],"PerplexityBot":["allowed","*",0,1],"Perplexity-User":["allowed","*",0,1],"Google-Extended":["allowed","*",0,1],"Applebot-Extended":["allowed","*",0,1],"CCBot":["allowed","*",0,1],"Bytespider":["allowed","*",0,1],"Amazonbot":["allowed","*",0,1],"meta-externalagent":["allowed","*",0,1],"cohere-ai":["allowed","*",0,1],"Diffbot":["allowed","*",0,1],"ContextFetchBot":["allowed","*",0,1]},"llms":{"status":"missing","http":404,"bytes":1569},"home":{"status":"ok","http":200,"url":"https://about.youtube/","html_tokens":24035,"markdown_tokens":60,"pruned_tokens":60}},
{"rank":7255,"domain":"ks.ua","origin":"http://ks.ua","robots":{"status":"ok","http":200,"url":"https://nic.ks.ua/robots.txt","bytes":14,"groups":1},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"missing","http":404,"bytes":146},"home":{"status":"ok","http":200,"url":"https://nic.ks.ua/","html_tokens":747,"markdown_tokens":113,"pruned_tokens":113}},
{"rank":7256,"domain":"prv.pl","origin":"https://prv.pl","robots":{"status":"ok","http":200,"url":"https://prv.pl/robots.txt","bytes":71,"groups":1},"bots":{"GPTBot":["partial","*",2,0],"ChatGPT-User":["partial","*",2,0],"OAI-SearchBot":["partial","*",2,0],"ClaudeBot":["partial","*",2,0],"Claude-User":["partial","*",2,0],"Claude-SearchBot":["partial","*",2,0],"anthropic-ai":["partial","*",2,0],"PerplexityBot":["partial","*",2,0],"Perplexity-User":["partial","*",2,0],"Google-Extended":["partial","*",2,0],"Applebot-Extended":["partial","*",2,0],"CCBot":["partial","*",2,0],"Bytespider":["partial","*",2,0],"Amazonbot":["partial","*",2,0],"meta-externalagent":["partial","*",2,0],"cohere-ai":["partial","*",2,0],"Diffbot":["partial","*",2,0],"ContextFetchBot":["partial","*",2,0]},"llms":{"status":"html","http":200,"bytes":12916},"home":{"status":"ok","http":200,"url":"https://www.prv.pl/","html_tokens":12282,"markdown_tokens":135,"pruned_tokens":135}},
{"rank":7257,"domain":"jiguang.cn","origin":"https://jiguang.cn","robots":{"status":"ok","http":200,"url":"https://www.jiguang.cn/robots.txt","bytes":172,"groups":1},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"missing","http":404,"bytes":146},"home":{"status":"ok","http":200,"url":"https://www.jiguang.cn/en","html_tokens":32835,"markdown_tokens":862,"pruned_tokens":145}},
{"rank":7258,"domain":"gameshub.com","origin":"https://gameshub.com","robots":{"status":"ok","http":200,"url":"https://www.gameshub.com/robots.txt","bytes":620,"groups":1},"bots":{"GPTBot":["partial","*",15,1],"ChatGPT-User":["partial","*",15,1],"OAI-SearchBot":["partial","*",15,1],"ClaudeBot":["partial","*",15,1],"Claude-User":["partial","*",15,1],"Claude-SearchBot":["partial","*",15,1],"anthropic-ai":["partial","*",15,1],"PerplexityBot":["partial","*",15,1],"Perplexity-User":["partial","*",15,1],"Google-Extended":["partial","*",15,1],"Applebot-Extended":["partial","*",15,1],"CCBot":["partial","*",15,1],"Bytespider":["partial","*",15,1],"Amazonbot":["partial","*",15,1],"meta-externalagent":["partial","*",15,1],"cohere-ai":["partial","*",15,1],"Diffbot":["partial","*",15,1],"ContextFetchBot":["partial","*",15,1]},"llms":{"status":"missing","http":404,"bytes":120254},"home":{"status":"http_error","http":403,"url":"https://www.gameshub.com/"}},
{"rank":7259,"domain":"espncricinfo.com","origin":"https://espncricinfo.com","robots":{"status":"ok","http":200,"url":"https://espncricinfo.com/robots.txt","bytes":1347,"groups":4},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["partial","*",42,1],"OAI-SearchBot":["partial","*",42,1],"ClaudeBot":["partial","*",42,1],"Claude-User":["partial","*",42,1],"Claude-SearchBot":["partial","*",42,1],"anthropic-ai":["partial","*",42,1],"PerplexityBot":["partial","*",42,1],"Perplexity-User":["partial","*",42,1],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["partial","*",42,1],"CCBot":["partial","*",42,1],"Bytespider":["partial","*",42,1],"Amazonbot":["partial","*",42,1],"meta-externalagent":["partial","*",42,1],"cohere-ai":["partial","*",42,1],"Diffbot":["partial","*",42,1],"ContextFetchBot":["partial","*",42,1]},"llms":{"status":"disallowed","error":"robots.txt of www.cricinfo.com not readable (refused)"},"home":{"status":"disallowed","error":"robots.txt of www.cricinfo.com not readable (refused)"}},
{"rank":7260,"domain":"knaw.nl","origin":"https://knaw.nl","robots":{"status":"refused","http":403,"url":"https://knaw.nl/robots.txt","bytes":118,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":7261,"domain":"magic.ly","origin":"https://magic.ly","robots":{"status":"ok","http":200,"url":"https://magic.ly/robots.txt","bytes":556,"groups":6},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["blocked","named",1,0],"OAI-SearchBot":["partial","*",15,1],"ClaudeBot":["partial","*",15,1],"Claude-User":["partial","*",15,1],"Claude-SearchBot":["partial","*",15,1],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["partial","*",15,1],"Perplexity-User":["partial","*",15,1],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["partial","*",15,1],"CCBot":["blocked","named",1,0],"Bytespider":["partial","*",15,1],"Amazonbot":["partial","*",15,1],"meta-externalagent":["partial","*",15,1],"cohere-ai":["partial","*",15,1],"Diffbot":["partial","*",15,1],"ContextFetchBot":["partial","*",15,1]},"llms":{"status":"present","http":200,"bytes":2696},"home":{"status":"ok","http":200,"url":"https://magicly.bio/","html_tokens":28431,"markdown_tokens":95,"pruned_tokens":95}},
{"rank":7262,"domain":"finder.com","origin":"https://finder.com","robots":{"status":"ok","http":200,"url":"https://www.finder.com/robots.txt","bytes":1569,"groups":1},"bots":{"GPTBot":["partial","*",24,1],"ChatGPT-User":["partial","*",24,1],"OAI-SearchBot":["partial","*",24,1],"ClaudeBot":["partial","*",24,1],"Claude-User":["partial","*",24,1],"Claude-SearchBot":["partial","*",24,1],"anthropic-ai":["partial","*",24,1],"PerplexityBot":["partial","*",24,1],"Perplexity-User":["partial","*",24,1],"Google-Extended":["partial","*",24,1],"Applebot-Extended":["partial","*",24,1],"CCBot":["partial","*",24,1],"Bytespider":["partial","*",24,1],"Amazonbot":["partial","*",24,1],"meta-externalagent":["partial","*",24,1],"cohere-ai":["partial","*",24,1],"Diffbot":["partial","*",24,1],"ContextFetchBot":["partial","*",24,1]},"llms":{"status":"missing","http":404,"bytes":373842},"home":{"status":"ok","http":200,"url":"https://www.finder.com/","html_tokens":162370,"markdown_tokens":72,"pruned_tokens":72}},
{"rank":7263,"domain":"kobe-u.ac.jp","origin":"https://www.kobe-u.ac.jp","robots":{"status":"ok","http":200,"url":"https://www.kobe-u.ac.jp/robots.txt","bytes":88,"groups":1},"bots":{"GPTBot":["partial","*",2,0],"ChatGPT-User":["partial","*",2,0],"OAI-SearchBot":["partial","*",2,0],"ClaudeBot":["partial","*",2,0],"Claude-User":["partial","*",2,0],"Claude-SearchBot":["partial","*",2,0],"anthropic-ai":["partial","*",2,0],"PerplexityBot":["partial","*",2,0],"Perplexity-User":["partial","*",2,0],"Google-Extended":["partial","*",2,0],"Applebot-Extended":["partial","*",2,0],"CCBot":["partial","*",2,0],"Bytespider":["partial","*",2,0],"Amazonbot":["partial","*",2,0],"meta-externalagent":["partial","*",2,0],"cohere-ai":["partial","*",2,0],"Diffbot":["partial","*",2,0],"ContextFetchBot":["partial","*",2,0]},"llms":{"status":"missing","http":404,"bytes":68430},"home":{"status":"unreachable","error":"Server disconnected without sending a response."}},
{"rank":7264,"domain":"mars.com","origin":"https://mars.com","robots":{"status":"refused","http":403,"url":"https://mars.com/robots.txt","bytes":919,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":7265,"domain":"consumerist.com","origin":"https://consumerist.com","robots":{"status":"missing","http":410,"url":"https://consumerist.com/robots.txt","bytes":0,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":410,"bytes":0},"home":{"status":"ok","http":200,"url":"https://www.consumerreports.org/","html_tokens":120014,"markdown_tokens":80,"pruned_tokens":80}},
{"rank":7266,"domain":"thepioneerwoman.com","origin":"https://thepioneerwoman.com","robots":{"status":"ok","http":200,"url":"https://thepioneerwoman.com/robots.txt","bytes":1012,"groups":4},"bots":{"GPTBot":["partial","named",32,0],"ChatGPT-User":["partial","named",32,0],"OAI-SearchBot":["partial","named",32,0],"ClaudeBot":["partial","*",19,0],"Claude-User":["partial","*",19,0],"Claude-SearchBot":["partial","*",19,0],"anthropic-ai":["partial","*",19,0],"PerplexityBot":["partial","*",19,0],"Perplexity-User":["partial","*",19,0],"Google-Extended":["partial","*",19,0],"Applebot-Extended":["partial","*",19,0],"CCBot":["partial","*",19,0],"Bytespider":["partial","*",19,0],"Amazonbot":["partial","*",19,0],"meta-externalagent":["partial","*",19,0],"cohere-ai":["partial","*",19,0],"Diffbot":["partial","*",19,0],"ContextFetchBot":["partial","*",19,0]},"llms":{"status":"missing","http":404,"bytes":84},"home":{"status":"ok","http":200,"url":"https://www.thepioneerwoman.com/","html_tokens":193672,"markdown_tokens":219,"pruned_tokens":113}},
{"rank":7267,"domain":"telefonica.com","origin":"https://telefonica.com","robots":{"status":"ok","http":200,"url":"https://www.telefonica.com/robots.txt","bytes":615,"groups":2},"bots":{"GPTBot":["partial","*",11,0],"ChatGPT-User":["partial","*",11,0],"OAI-SearchBot":["partial","*",11,0],"ClaudeBot":["partial","*",11,0],"Claude-User":["partial","*",11,0],"Claude-SearchBot":["partial","*",11,0],"anthropic-ai":["partial","*",11,0],"PerplexityBot":["partial","*",11,0],"Perplexity-User":["partial","*",11,0],"Google-Extended":["partial","*",11,0],"Applebot-Extended":["partial","*",11,0],"CCBot":["partial","*",11,0],"Bytespider":["partial","*",11,0],"Amazonbot":["partial","*",11,0],"meta-externalagent":["partial","*",11,0],"cohere-ai":["partial","*",11,0],"Diffbot":["partial","*",11,0],"ContextFetchBot":["partial","*",11,0]},"llms":{"status":"missing","http":404,"bytes":166678},"home":{"status":"ok","http":200,"url":"https://www.telefonica.com/es/","html_tokens":51664,"markdown_tokens":155,"pruned_tokens":155}},
{"rank":7268,"domain":"stir.ac.uk","origin":"https://stir.ac.uk","robots":{"status":"ok","http":200,"url":"https://www.stir.ac.uk/robots.txt","bytes":883,"groups":2},"bots":{"GPTBot":["partial","*",13,0],"ChatGPT-User":["partial","*",13,0],"OAI-SearchBot":["partial","*",13,0],"ClaudeBot":["partial","*",13,0],"Claude-User":["partial","*",13,0],"Claude-SearchBot":["partial","*",13,0],"anthropic-ai":["partial","*",13,0],"PerplexityBot":["partial","*",13,0],"Perplexity-User":["partial","*",13,0],"Google-Extended":["partial","*",13,0],"Applebot-Extended":["partial","*",13,0],"CCBot":["partial","*",13,0],"Bytespider":["partial","*",13,0],"Amazonbot":["partial","*",13,0],"meta-externalagent":["partial","*",13,0],"cohere-ai":["partial","*",13,0],"Diffbot":["partial","*",13,0],"ContextFetchBot":["partial","*",13,0]},"llms":{"status":"missing","http":404,"bytes":37788},"home":{"status":"ok","http":200,"url":"https://www.stir.ac.uk/","html_tokens":13246,"markdown_tokens":433,"pruned_tokens":30}},
{"rank":7269,"domain":"afsp.org","origin":"https://afsp.org","robots":{"status":"ok","http":200,"url":"https://afsp.org/robots.txt","bytes":67,"groups":1},"bots":{"GPTBot":["allowed","*",0,1],"ChatGPT-User":["allowed","*",0,1],"OAI-SearchBot":["allowed","*",0,1],"ClaudeBot":["allowed","*",0,1],"Claude-User":["allowed","*",0,1],"Claude-SearchBot":["allowed","*",0,1],"anthropic-ai":["allowed","*",0,1],"PerplexityBot":["allowed","*",0,1],"Perplexity-User":["allowed","*",0,1],"Google-Extended":["allowed","*",0,1],"Applebot-Extended":["allowed","*",0,1],"CCBot":["allowed","*",0,1],"Bytespider":["allowed","*",0,1],"Amazonbot":["allowed","*",0,1],"meta-externalagent":["allowed","*",0,1],"cohere-ai":["allowed","*",0,1],"Diffbot":["allowed","*",0,1],"ContextFetchBot":["allowed","*",0,1]},"llms":{"status":"missing","http":404,"bytes":304865},"home":{"status":"ok","http":200,"url":"https://afsp.org/","html_tokens":87053,"markdown_tokens":216,"pruned_tokens":106}},
{"rank":7270,"domain":"telerama.fr","origin":"https://telerama.fr","robots":{"status":"ok","http":200,"url":"https://www.telerama.fr/robots.txt","bytes":7992,"groups":3},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["blocked","named",1,0],"OAI-SearchBot":["partial","*",75,36],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",75,36],"Claude-SearchBot":["partial","*",75,36],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["partial","*",75,36],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["partial","*",75,36],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",2,0],"Amazonbot":["partial","*",75,36],"meta-externalagent":["partial","*",75,36],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",2,0],"ContextFetchBot":["partial","*",75,36]},"llms":{"status":"missing","http":404,"bytes":163810},"home":{"status":"ok","http":200,"url":"https://www.telerama.fr/","html_tokens":148537,"markdown_tokens":256,"pruned_tokens":128}},
{"rank":7271,"domain":"google.dz","origin":"https://google.dz","robots":{"status":"ok","http":200,"url":"https://www.google.dz/robots.txt","bytes":6624,"groups":4},"bots":{"GPTBot":["partial","*",169,64],"ChatGPT-User":["partial","*",169,64],"OAI-SearchBot":["partial","*",169,64],"ClaudeBot":["partial","*",169,64],"Claude-User":["partial","*",169,64],"Claude-SearchBot":["partial","*",169,64],"anthropic-ai":["partial","*",169,64],"PerplexityBot":["partial","*",169,64],"Perplexity-User":["partial","*",169,64],"Google-Extended":["partial","*",169,64],"Applebot-Extended":["partial","*",169,64],"CCBot":["partial","*",169,64],"Bytespider":["partial","*",169,64],"Amazonbot":["partial","*",169,64],"meta-externalagent":["partial","*",169,64],"cohere-ai":["partial","*",169,64],"Diffbot":["partial","*",169,64],"ContextFetchBot":["partial","*",169,64]},"llms":{"status":"missing","http":404,"bytes":1569},"home":{"status":"ok","http":200,"url":"https://www.google.com/","html_tokens":21456,"markdown_tokens":30,"pruned_tokens":30}},
{"rank":7272,"domain":"cba.pl","origin":"https://cba.pl","robots":{"status":"ok","http":200,"url":"https://www.cba.pl/robots.txt","bytes":560,"groups":1},"bots":{"GPTBot":["partial","*",17,0],"ChatGPT-User":["partial","*",17,0],"OAI-SearchBot":["partial","*",17,0],"ClaudeBot":["partial","*",17,0],"Claude-User":["partial","*",17,0],"Claude-SearchBot":["partial","*",17,0],"anthropic-ai":["partial","*",17,0],"PerplexityBot":["partial","*",17,0],"Perplexity-User":["partial","*",17,0],"Google-Extended":["partial","*",17,0],"Applebot-Extended":["partial","*",17,0],"CCBot":["partial","*",17,0],"Bytespider":["partial","*",17,0],"Amazonbot":["partial","*",17,0],"meta-externalagent":["partial","*",17,0],"cohere-ai":["partial","*",17,0],"Diffbot":["partial","*",17,0],"ContextFetchBot":["partial","*",17,0]},"llms":{"status":"missing","http":404,"bytes":25387},"home":{"status":"ok","http":200,"url":"https://www.cba.pl/","html_tokens":13105,"markdown_tokens":298,"pruned_tokens":62}},
{"rank":7273,"domain":"dagbladet.no","origin":"https://dagbladet.no","robots":{"status":"ok","http":200,"url":"https://www.dagbladet.no/robots.txt","bytes":4167,"groups":48},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["blocked","named",1,0],"OAI-SearchBot":["blocked","named",1,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["blocked","named",1,0],"Claude-SearchBot":["blocked","named",1,0],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["blocked","named",1,0],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",17,9]},"llms":{"status":"missing","http":404,"bytes":66765},"home":{"status":"ok","http":200,"url":"https://www.dagbladet.no/","html_tokens":198240,"markdown_tokens":92,"pruned_tokens":37}},
{"rank":7274,"domain":"123js.cn","origin":"https://123js.cn","robots":{"status":"missing","http":404,"url":"https://123js.cn/robots.txt","bytes":315,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":315},"home":{"status":"http_error","http":404,"url":"https://123js.cn/"}},
{"rank":7275,"domain":"boston.gov","origin":"https://boston.gov","robots":{"status":"ok","http":200,"url":"https://www.boston.gov/robots.txt","bytes":2027,"groups":1},"bots":{"GPTBot":["partial","*",34,18],"ChatGPT-User":["partial","*",34,18],"OAI-SearchBot":["partial","*",34,18],"ClaudeBot":["partial","*",34,18],"Claude-User":["partial","*",34,18],"Claude-SearchBot":["partial","*",34,18],"anthropic-ai":["partial","*",34,18],"PerplexityBot":["partial","*",34,18],"Perplexity-User":["partial","*",34,18],"Google-Extended":["partial","*",34,18],"Applebot-Extended":["partial","*",34,18],"CCBot":["partial","*",34,18],"Bytespider":["partial","*",34,18],"Amazonbot":["partial","*",34,18],"meta-externalagent":["partial","*",34,18],"cohere-ai":["partial","*",34,18],"Diffbot":["partial","*",34,18],"ContextFetchBot":["partial","*",34,18]},"llms":{"status":"missing","http":404,"bytes":322},"home":{"status":"ok","http":200,"url":"https://www.boston.gov/","html_tokens":32581,"markdown_tokens":156,"pruned_tokens":156}},
{"rank":7276,"domain":"floridahealth.gov","origin":"https://floridahealth.gov","robots":{"status":"ok","http":200,"url":"https://www.floridahealth.gov/robots.txt","bytes":140,"groups":1},"bots":{"GPTBot":["partial","*",1,1],"ChatGPT-User":["partial","*",1,1],"OAI-SearchBot":["partial","*",1,1],"ClaudeBot":["partial","*",1,1],"Claude-User":["partial","*",1,1],"Claude-SearchBot":["partial","*",1,1],"anthropic-ai":["partial","*",1,1],"PerplexityBot":["partial","*",1,1],"Perplexity-User":["partial","*",1,1],"Google-Extended":["partial","*",1,1],"Applebot-Extended":["partial","*",1,1],"CCBot":["partial","*",1,1],"Bytespider":["partial","*",1,1],"Amazonbot":["partial","*",1,1],"meta-externalagent":["partial","*",1,1],"cohere-ai":["partial","*",1,1],"Diffbot":["partial","*",1,1],"ContextFetchBot":["partial","*",1,1]},"llms":{"status":"http_error","http":403,"bytes":5596},"home":{"status":"http_error","http":403,"url":"https://www.floridahealth.gov/"}},
{"rank":7277,"domain":"coschedule.com","origin":"https://coschedule.com","robots":{"status":"missing","http":404,"url":"https://coschedule.com/robots.txt","bytes":41562,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"present","http":200,"bytes":4386},"home":{"status":"ok","http":200,"url":"https://coschedule.com/","html_tokens":39911,"markdown_tokens":228,"pruned_tokens":163}},
{"rank":7278,"domain":"marchofdimes.org","origin":"https://marchofdimes.org","robots":{"status":"ok","http":200,"url":"https://www.marchofdimes.org/robots.txt","bytes":2027,"groups":1},"bots":{"GPTBot":["partial","*",34,18],"ChatGPT-User":["partial","*",34,18],"OAI-SearchBot":["partial","*",34,18],"ClaudeBot":["partial","*",34,18],"Claude-User":["partial","*",34,18],"Claude-SearchBot":["partial","*",34,18],"anthropic-ai":["partial","*",34,18],"PerplexityBot":["partial","*",34,18],"Perplexity-User":["partial","*",34,18],"Google-Extended":["partial","*",34,18],"Applebot-Extended":["partial","*",34,18],"CCBot":["partial","*",34,18],"Bytespider":["partial","*",34,18],"Amazonbot":["partial","*",34,18],"meta-externalagent":["partial","*",34,18],"cohere-ai":["partial","*",34,18],"Diffbot":["partial","*",34,18],"ContextFetchBot":["partial","*",34,18]},"llms":{"status":"missing","http":404,"bytes":555},"home":{"status":"ok","http":200,"url":"https://www.marchofdimes.org/","html_tokens":21924,"markdown_tokens":118,"pruned_tokens":118}},
{"rank":7279,"domain":"earthweb.com","origin":"https://earthweb.com","robots":{"status":"refused","http":403,"url":"https://earthweb.com/robots.txt","bytes":4545,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":7280,"domain":"heraldtribune.com","origin":"https://heraldtribune.com","robots":{"status":"ok","http":200,"url":"https://www.heraldtribune.com/robots.txt","bytes":14077,"groups":288},"bots":{"GPTBot":["partial","named",1,4],"ChatGPT-User":["partial","named",1,4],"OAI-SearchBot":["partial","named",1,4],"ClaudeBot":["blocked","named",1,0],"Claude-User":["blocked","named",1,0],"Claude-SearchBot":["blocked","named",1,0],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["partial","named",1,4],"Perplexity-User":["blocked","named",1,0],"Google-Extended":["partial","named",1,4],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",2,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",88,0]},"llms":{"status":"http_error","http":402,"bytes":254},"home":{"status":"http_error","http":402,"url":"https://www.heraldtribune.com/"}},
{"rank":7281,"domain":"theadvocate.com","origin":"https://theadvocate.com","robots":{"status":"ok","http":200,"url":"https://theadvocate.com/robots.txt","bytes":6132,"groups":31},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["blocked","named",1,0],"OAI-SearchBot":["blocked","named",1,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",70,0],"Claude-SearchBot":["partial","*",70,0],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["partial","*",70,0],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["partial","*",70,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",70,0]},"llms":{"status":"http_error","http":429,"bytes":64},"home":{"status":"ok","http":200,"url":"https://www.theadvocate.com/","html_tokens":146738,"markdown_tokens":137,"pruned_tokens":137}},
{"rank":7282,"domain":"sharefile.com","origin":"https://sharefile.com","robots":{"status":"ok","http":200,"url":"https://www.sharefile.com/robots.txt","bytes":160,"groups":1},"bots":{"GPTBot":["partial","*",1,0],"ChatGPT-User":["partial","*",1,0],"OAI-SearchBot":["partial","*",1,0],"ClaudeBot":["partial","*",1,0],"Claude-User":["partial","*",1,0],"Claude-SearchBot":["partial","*",1,0],"anthropic-ai":["partial","*",1,0],"PerplexityBot":["partial","*",1,0],"Perplexity-User":["partial","*",1,0],"Google-Extended":["partial","*",1,0],"Applebot-Extended":["partial","*",1,0],"CCBot":["partial","*",1,0],"Bytespider":["partial","*",1,0],"Amazonbot":["partial","*",1,0],"meta-externalagent":["partial","*",1,0],"cohere-ai":["partial","*",1,0],"Diffbot":["partial","*",1,0],"ContextFetchBot":["partial","*",1,0]},"llms":{"status":"present","http":200,"bytes":6972},"home":{"status":"ok","http":200,"url":"https://www.sharefile.com","html_tokens":24781,"markdown_tokens":771,"pruned_tokens":197}},
{"rank":7283,"domain":"edge.org","origin":"https://edge.org","robots":{"status":"ok","http":200,"url":"https://edge.org/robots.txt","bytes":2189,"groups":1},"bots":{"GPTBot":["partial","*",36,32],"ChatGPT-User":["partial","*",36,32],"OAI-SearchBot":["partial","*",36,32],"ClaudeBot":["partial","*",36,32],"Claude-User":["partial","*",36,32],"Claude-SearchBot":["partial","*",36,32],"anthropic-ai":["partial","*",36,32],"PerplexityBot":["partial","*",36,32],"Perplexity-User":["partial","*",36,32],"Google-Extended":["partial","*",36,32],"Applebot-Extended":["partial","*",36,32],"CCBot":["partial","*",36,32],"Bytespider":["partial","*",36,32],"Amazonbot":["partial","*",36,32],"meta-externalagent":["partial","*",36,32],"cohere-ai":["partial","*",36,32],"Diffbot":["partial","*",36,32],"ContextFetchBot":["partial","*",36,32]},"llms":{"status":"missing","http":404,"bytes":291},"home":{"status":"ok","http":200,"url":"https://edge.org/","html_tokens":13532,"markdown_tokens":5797,"pruned_tokens":157}},
{"rank":7284,"domain":"pilotonline.com","origin":"https://pilotonline.com","robots":{"status":"ok","http":200,"url":"https://www.pilotonline.com/robots.txt","bytes":4484,"groups":68},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["blocked","named",1,0],"OAI-SearchBot":["blocked","named",1,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["blocked","named",1,0],"Claude-SearchBot":["blocked","named",1,0],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["blocked","named",1,0],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["partial","*",8,1],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",8,1]},"llms":{"status":"missing","http":404,"bytes":83362},"home":{"status":"ok","http":200,"url":"https://www.pilotonline.com/","html_tokens":66213,"markdown_tokens":391,"pruned_tokens":173}},
{"rank":7285,"domain":"madrid.org","origin":"https://www.madrid.org","robots":{"status":"refused","http":403,"url":"https://www.madrid.org/robots.txt","bytes":919,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":7286,"domain":"flinders.edu.au","origin":"https://flinders.edu.au","robots":{"status":"ok","http":200,"url":"https://www.flinders.edu.au/robots.txt","bytes":2608,"groups":14},"bots":{"GPTBot":["partial","*",67,1],"ChatGPT-User":["partial","*",67,1],"OAI-SearchBot":["partial","*",67,1],"ClaudeBot":["partial","*",67,1],"Claude-User":["partial","*",67,1],"Claude-SearchBot":["partial","*",67,1],"anthropic-ai":["partial","*",67,1],"PerplexityBot":["partial","*",67,1],"Perplexity-User":["partial","*",67,1],"Google-Extended":["partial","*",67,1],"Applebot-Extended":["partial","*",67,1],"CCBot":["partial","*",67,1],"Bytespider":["partial","*",67,1],"Amazonbot":["partial","*",67,1],"meta-externalagent":["partial","*",67,1],"cohere-ai":["partial","*",67,1],"Diffbot":["partial","*",67,1],"ContextFetchBot":["partial","*",67,1]},"llms":{"status":"missing","http":404,"bytes":114483},"home":{"status":"ok","http":200,"url":"https://www.flinders.edu.au/","html_tokens":76802,"markdown_tokens":797,"pruned_tokens":549}},
{"rank":7287,"domain":"nethouse.ru","origin":"https://nethouse.ru","robots":{"status":"ok","http":200,"url":"https://nethouse.ru/robots.txt","bytes":833,"groups":1},"bots":{"GPTBot":["partial","*",35,1],"ChatGPT-User":["partial","*",35,1],"OAI-SearchBot":["partial","*",35,1],"ClaudeBot":["partial","*",35,1],"Claude-User":["partial","*",35,1],"Claude-SearchBot":["partial","*",35,1],"anthropic-ai":["partial","*",35,1],"PerplexityBot":["partial","*",35,1],"Perplexity-User":["partial","*",35,1],"Google-Extended":["partial","*",35,1],"Applebot-Extended":["partial","*",35,1],"CCBot":["partial","*",35,1],"Bytespider":["partial","*",35,1],"Amazonbot":["partial","*",35,1],"meta-externalagent":["partial","*",35,1],"cohere-ai":["partial","*",35,1],"Diffbot":["partial","*",35,1],"ContextFetchBot":["partial","*",35,1]},"llms":{"status":"present","http":200,"bytes":8568},"home":{"status":"ok","http":200,"url":"https://nethouse.ru/","html_tokens":70020,"markdown_tokens":72,"pruned_tokens":72}},
{"rank":7288,"domain":"processing.org","origin":"https://processing.org","robots":{"status":"missing","http":404,"url":"https://processing.org/robots.txt","bytes":0,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":0},"home":{"status":"ok","http":200,"url":"https://processing.org/","html_tokens":13667,"markdown_tokens":367,"pruned_tokens":247}},
{"rank":7289,"domain":"texy.info","origin":"https://texy.info","robots":{"status":"ok","http":200,"url":"https://texy.deegee.dev/robots.txt","bytes":58,"groups":1},"bots":{"GPTBot":["partial","*",2,0],"ChatGPT-User":["partial","*",2,0],"OAI-SearchBot":["partial","*",2,0],"ClaudeBot":["partial","*",2,0],"Claude-User":["partial","*",2,0],"Claude-SearchBot":["partial","*",2,0],"anthropic-ai":["partial","*",2,0],"PerplexityBot":["partial","*",2,0],"Perplexity-User":["partial","*",2,0],"Google-Extended":["partial","*",2,0],"Applebot-Extended":["partial","*",2,0],"CCBot":["partial","*",2,0],"Bytespider":["partial","*",2,0],"Amazonbot":["partial","*",2,0],"meta-externalagent":["partial","*",2,0],"cohere-ai":["partial","*",2,0],"Diffbot":["partial","*",2,0],"ContextFetchBot":["partial","*",2,0]},"llms":{"status":"present","http":200,"bytes":1159},"home":{"status":"ok","http":200,"url":"https://texy.deegee.dev/","html_tokens":3354,"markdown_tokens":1086,"pruned_tokens":790}},
{"rank":7290,"domain":"php-myadmin.ru","origin":"https://php-myadmin.ru","robots":{"status":"ok","http":200,"url":"https://php-myadmin.ru/robots.txt","bytes":108,"groups":2},"bots":{"GPTBot":["allowed","*",0,1],"ChatGPT-User":["allowed","*",0,1],"OAI-SearchBot":["allowed","*",0,1],"ClaudeBot":["allowed","*",0,1],"Claude-User":["allowed","*",0,1],"Claude-SearchBot":["allowed","*",0,1],"anthropic-ai":["allowed","*",0,1],"PerplexityBot":["allowed","*",0,1],"Perplexity-User":["allowed","*",0,1],"Google-Extended":["allowed","*",0,1],"Applebot-Extended":["allowed","*",0,1],"CCBot":["allowed","*",0,1],"Bytespider":["allowed","*",0,1],"Amazonbot":["allowed","*",0,1],"meta-externalagent":["allowed","*",0,1],"cohere-ai":["allowed","*",0,1],"Diffbot":["allowed","*",0,1],"ContextFetchBot":["allowed","*",0,1]},"llms":{"status":"missing","http":404,"bytes":43511},"home":{"status":"ok","http":200,"url":"https://php-myadmin.ru/","html_tokens":13834,"markdown_tokens":898,"pruned_tokens":477}},
{"rank":7291,"domain":"cat.com","origin":"https://cat.com","robots":{"status":"unreachable","http":null,"url":"https://cat.com/robots.txt","bytes":0,"groups":0,"error":"<StreamReset stream_id:1, error_code:2, remote_reset:True>"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":7292,"domain":"fanpop.com","origin":"https://www.fanpop.com","robots":{"status":"ok","http":200,"url":"https://www.fanpop.com/robots.txt","bytes":474,"groups":2},"bots":{"GPTBot":["partial","*",16,0],"ChatGPT-User":["partial","*",16,0],"OAI-SearchBot":["partial","*",16,0],"ClaudeBot":["partial","*",16,0],"Claude-User":["partial","*",16,0],"Claude-SearchBot":["partial","*",16,0],"anthropic-ai":["partial","*",16,0],"PerplexityBot":["partial","*",16,0],"Perplexity-User":["partial","*",16,0],"Google-Extended":["partial","*",16,0],"Applebot-Extended":["partial","*",16,0],"CCBot":["partial","*",16,0],"Bytespider":["partial","*",16,0],"Amazonbot":["partial","*",16,0],"meta-externalagent":["partial","*",16,0],"cohere-ai":["partial","*",16,0],"Diffbot":["partial","*",16,0],"ContextFetchBot":["partial","*",16,0]},"llms":{"status":"missing","http":404,"bytes":2690},"home":{"status":"http_error","http":403,"url":"https://www.fanpop.com/"}},
{"rank":7293,"domain":"legalzoom.com","origin":"https://legalzoom.com","robots":{"status":"ok","http":200,"url":"https://www.legalzoom.com/robots.txt","bytes":2428,"groups":6},"bots":{"GPTBot":["partial","named",1,0],"ChatGPT-User":["partial","named",1,0],"OAI-SearchBot":["partial","*",54,8],"ClaudeBot":["partial","named",1,0],"Claude-User":["partial","*",54,8],"Claude-SearchBot":["partial","*",54,8],"anthropic-ai":["partial","*",54,8],"PerplexityBot":["partial","named",1,0],"Perplexity-User":["partial","*",54,8],"Google-Extended":["partial","named",1,0],"Applebot-Extended":["partial","*",54,8],"CCBot":["partial","*",54,8],"Bytespider":["partial","*",54,8],"Amazonbot":["partial","*",54,8],"meta-externalagent":["partial","*",54,8],"cohere-ai":["partial","*",54,8],"Diffbot":["partial","*",54,8],"ContextFetchBot":["partial","*",54,8]},"llms":{"status":"present","http":200,"bytes":4869},"home":{"status":"ok","http":200,"url":"https://www.legalzoom.com/","html_tokens":79161,"markdown_tokens":404,"pruned_tokens":143}},
{"rank":7294,"domain":"education.com","origin":"https://education.com","robots":{"status":"ok","http":200,"url":"https://www.education.com/robots.txt","bytes":821,"groups":5},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["partial","*",15,0],"OAI-SearchBot":["partial","*",15,0],"ClaudeBot":["partial","*",15,0],"Claude-User":["partial","*",15,0],"Claude-SearchBot":["partial","*",15,0],"anthropic-ai":["partial","*",15,0],"PerplexityBot":["partial","*",15,0],"Perplexity-User":["partial","*",15,0],"Google-Extended":["partial","*",15,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["partial","*",15,0],"Amazonbot":["partial","*",15,0],"meta-externalagent":["partial","*",15,0],"cohere-ai":["partial","*",15,0],"Diffbot":["partial","*",15,0],"ContextFetchBot":["partial","*",15,0]},"llms":{"status":"missing","http":404,"bytes":90676},"home":{"status":"ok","http":200,"url":"https://www.education.com/","html_tokens":22899,"markdown_tokens":442,"pruned_tokens":80}},
{"rank":7295,"domain":"mfa.org","origin":"https://mfa.org","robots":{"status":"ok","http":200,"url":"https://www.mfa.org/robots.txt","bytes":2027,"groups":1},"bots":{"GPTBot":["partial","*",34,18],"ChatGPT-User":["partial","*",34,18],"OAI-SearchBot":["partial","*",34,18],"ClaudeBot":["partial","*",34,18],"Claude-User":["partial","*",34,18],"Claude-SearchBot":["partial","*",34,18],"anthropic-ai":["partial","*",34,18],"PerplexityBot":["partial","*",34,18],"Perplexity-User":["partial","*",34,18],"Google-Extended":["partial","*",34,18],"Applebot-Extended":["partial","*",34,18],"CCBot":["partial","*",34,18],"Bytespider":["partial","*",34,18],"Amazonbot":["partial","*",34,18],"meta-externalagent":["partial","*",34,18],"cohere-ai":["partial","*",34,18],"Diffbot":["partial","*",34,18],"ContextFetchBot":["partial","*",34,18]},"llms":{"status":"missing","http":404,"bytes":546},"home":{"status":"ok","http":200,"url":"https://www.mfa.org/","html_tokens":23684,"markdown_tokens":5,"pruned_tokens":5}},
{"rank":7296,"domain":"devex.com","origin":"https://devex.com","robots":{"status":"refused","http":403,"url":"https://devex.com/robots.txt","bytes":769,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":7297,"domain":"unherd.com","origin":"https://unherd.com","robots":{"status":"ok","http":200,"url":"https://unherd.com/robots.txt","bytes":1228,"groups":30},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["blocked","named",1,0],"OAI-SearchBot":["partial","*",3,0],"ClaudeBot":["partial","*",3,0],"Claude-User":["partial","*",3,0],"Claude-SearchBot":["partial","*",3,0],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["partial","*",3,0],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",3,0]},"llms":{"status":"missing","http":404,"bytes":74917},"home":{"status":"ok","http":200,"url":"https://unherd.com/?edition=us","html_tokens":88917,"markdown_tokens":28,"pruned_tokens":28}},
{"rank":7298,"domain":"thisiscolossal.com","origin":"https://thisiscolossal.com","robots":{"status":"ok","http":200,"url":"https://www.thisiscolossal.com/robots.txt","bytes":306,"groups":4},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["partial","*",1,0],"OAI-SearchBot":["partial","*",1,0],"ClaudeBot":["partial","*",1,0],"Claude-User":["partial","*",1,0],"Claude-SearchBot":["partial","*",1,0],"anthropic-ai":["partial","*",1,0],"PerplexityBot":["partial","*",1,0],"Perplexity-User":["partial","*",1,0],"Google-Extended":["partial","*",1,0],"Applebot-Extended":["partial","*",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["partial","*",1,0],"Amazonbot":["partial","*",1,0],"meta-externalagent":["partial","*",1,0],"cohere-ai":["partial","*",1,0],"Diffbot":["partial","*",1,0],"ContextFetchBot":["partial","*",1,0]},"llms":{"status":"missing","http":404,"bytes":177398},"home":{"status":"ok","http":200,"url":"https://www.thisiscolossal.com/","html_tokens":61249,"markdown_tokens":41,"pruned_tokens":41}},
{"rank":7299,"domain":"ras.ru","origin":"https://ras.ru","robots":{"status":"ok","http":200,"url":"https://www.ras.ru/robots.txt","bytes":158,"groups":1},"bots":{"GPTBot":["partial","*",6,0],"ChatGPT-User":["partial","*",6,0],"OAI-SearchBot":["partial","*",6,0],"ClaudeBot":["partial","*",6,0],"Claude-User":["partial","*",6,0],"Claude-SearchBot":["partial","*",6,0],"anthropic-ai":["partial","*",6,0],"PerplexityBot":["partial","*",6,0],"Perplexity-User":["partial","*",6,0],"Google-Extended":["partial","*",6,0],"Applebot-Extended":["partial","*",6,0],"CCBot":["partial","*",6,0],"Bytespider":["partial","*",6,0],"Amazonbot":["partial","*",6,0],"meta-externalagent":["partial","*",6,0],"cohere-ai":["partial","*",6,0],"Diffbot":["partial","*",6,0],"ContextFetchBot":["partial","*",6,0]},"llms":{"status":"missing","http":404,"bytes":1245},"home":{"status":"ok","http":200,"url":"https://www.ras.ru/","html_tokens":4337,"markdown_tokens":13,"pruned_tokens":13}},
{"rank":7300,"domain":"google.com.bo","origin":"https://google.com.bo","robots":{"status":"ok","http":200,"url":"https://www.google.com.bo/robots.txt","bytes":6624,"groups":4},"bots":{"GPTBot":["partial","*",169,64],"ChatGPT-User":["partial","*",169,64],"OAI-SearchBot":["partial","*",169,64],"ClaudeBot":["partial","*",169,64],"Claude-User":["partial","*",169,64],"Claude-SearchBot":["partial","*",169,64],"anthropic-ai":["partial","*",169,64],"PerplexityBot":["partial","*",169,64],"Perplexity-User":["partial","*",169,64],"Google-Extended":["partial","*",169,64],"Applebot-Extended":["partial","*",169,64],"CCBot":["partial","*",169,64],"Bytespider":["partial","*",169,64],"Amazonbot":["partial","*",169,64],"meta-externalagent":["partial","*",169,64],"cohere-ai":["partial","*",169,64],"Diffbot":["partial","*",169,64],"ContextFetchBot":["partial","*",169,64]},"llms":{"status":"missing","http":404,"bytes":1569},"home":{"status":"ok","http":200,"url":"https://www.google.com/","html_tokens":21298,"markdown_tokens":30,"pruned_tokens":30}},
{"rank":7301,"domain":"scio.gov.cn","origin":"https://scio.gov.cn","robots":{"status":"unreachable","http":null,"url":"https://scio.gov.cn/robots.txt","bytes":0,"groups":0,"error":"cannot resolve scio.gov.cn"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":7302,"domain":"zakupki.gov.ru","origin":"https://zakupki.gov.ru","robots":{"status":"timeout","http":null,"url":"https://zakupki.gov.ru/robots.txt","bytes":0,"groups":0,"error":"ConnectTimeout"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":7303,"domain":"oricon.co.jp","origin":"https://oricon.co.jp","robots":{"status":"ok","http":200,"url":"https://www.oricon.co.jp/robots.txt","bytes":1854,"groups":9},"bots":{"GPTBot":["partial","*",9,10],"ChatGPT-User":["partial","*",9,10],"OAI-SearchBot":["partial","*",9,10],"ClaudeBot":["partial","*",9,10],"Claude-User":["partial","*",9,10],"Claude-SearchBot":["partial","*",9,10],"anthropic-ai":["partial","*",9,10],"PerplexityBot":["partial","*",9,10],"Perplexity-User":["partial","*",9,10],"Google-Extended":["partial","*",9,10],"Applebot-Extended":["partial","*",9,10],"CCBot":["partial","*",9,10],"Bytespider":["partial","*",9,10],"Amazonbot":["partial","*",9,10],"meta-externalagent":["partial","*",9,10],"cohere-ai":["partial","*",9,10],"Diffbot":["partial","*",9,10],"ContextFetchBot":["partial","*",9,10]},"llms":{"status":"missing","http":404,"bytes":32422},"home":{"status":"ok","http":200,"url":"https://www.oricon.co.jp/","html_tokens":84419,"markdown_tokens":6147,"pruned_tokens":118}},
{"rank":7304,"domain":"metu.edu.tr","origin":"https://metu.edu.tr","robots":{"status":"ok","http":200,"url":"https://www.metu.edu.tr/robots.txt","bytes":2189,"groups":1},"bots":{"GPTBot":["partial","*",36,32],"ChatGPT-User":["partial","*",36,32],"OAI-SearchBot":["partial","*",36,32],"ClaudeBot":["partial","*",36,32],"Claude-User":["partial","*",36,32],"Claude-SearchBot":["partial","*",36,32],"anthropic-ai":["partial","*",36,32],"PerplexityBot":["partial","*",36,32],"Perplexity-User":["partial","*",36,32],"Google-Extended":["partial","*",36,32],"Applebot-Extended":["partial","*",36,32],"CCBot":["partial","*",36,32],"Bytespider":["partial","*",36,32],"Amazonbot":["partial","*",36,32],"meta-externalagent":["partial","*",36,32],"cohere-ai":["partial","*",36,32],"Diffbot":["partial","*",36,32],"ContextFetchBot":["partial","*",36,32]},"llms":{"status":"timeout","error":"timed out after 10 s"},"home":{"status":"disallowed","error":"robots.txt of www.metu.edu.tr not readable (timeout)"}},
{"rank":7305,"domain":"lareviewofbooks.org","origin":"https://lareviewofbooks.org","robots":{"status":"ok","http":200,"url":"https://lareviewofbooks.org/robots.txt","bytes":1014,"groups":26},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["blocked","named",1,0],"OAI-SearchBot":["partial","*",3,2],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",3,2],"Claude-SearchBot":["partial","*",3,2],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["partial","*",3,2],"Google-Extended":["partial","*",3,2],"Applebot-Extended":["partial","*",3,2],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["partial","*",3,2],"cohere-ai":["partial","*",3,2],"Diffbot":["partial","*",3,2],"ContextFetchBot":["partial","*",3,2]},"llms":{"status":"missing","http":404,"bytes":15750},"home":{"status":"ok","http":200,"url":"https://lareviewofbooks.org/","html_tokens":99466,"markdown_tokens":1178,"pruned_tokens":134}},
{"rank":7306,"domain":"rogers.com","origin":"https://www.rogers.com","robots":{"status":"ok","http":200,"url":"https://www.rogers.com/robots.txt","bytes":2494,"groups":1},"bots":{"GPTBot":["partial","*",9,1],"ChatGPT-User":["partial","*",9,1],"OAI-SearchBot":["partial","*",9,1],"ClaudeBot":["partial","*",9,1],"Claude-User":["partial","*",9,1],"Claude-SearchBot":["partial","*",9,1],"anthropic-ai":["partial","*",9,1],"PerplexityBot":["partial","*",9,1],"Perplexity-User":["partial","*",9,1],"Google-Extended":["partial","*",9,1],"Applebot-Extended":["partial","*",9,1],"CCBot":["partial","*",9,1],"Bytespider":["partial","*",9,1],"Amazonbot":["partial","*",9,1],"meta-externalagent":["partial","*",9,1],"cohere-ai":["partial","*",9,1],"Diffbot":["partial","*",9,1],"ContextFetchBot":["partial","*",9,1]},"llms":{"status":"missing","http":404,"bytes":2339},"home":{"status":"ok","http":200,"url":"https://www.rogers.com/","html_tokens":63151,"markdown_tokens":244,"pruned_tokens":30}},
{"rank":7307,"domain":"rzd.ru","origin":"https://rzd.ru","robots":{"status":"timeout","http":null,"url":"https://rzd.ru/robots.txt","bytes":0,"groups":0,"error":"ConnectTimeout"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":7308,"domain":"research.net","origin":"https://research.net","robots":{"status":"ok","http":200,"url":"https://www.research.net/robots.txt","bytes":90,"groups":1},"bots":{"GPTBot":["partial","*",4,0],"ChatGPT-User":["partial","*",4,0],"OAI-SearchBot":["partial","*",4,0],"ClaudeBot":["partial","*",4,0],"Claude-User":["partial","*",4,0],"Claude-SearchBot":["partial","*",4,0],"anthropic-ai":["partial","*",4,0],"PerplexityBot":["partial","*",4,0],"Perplexity-User":["partial","*",4,0],"Google-Extended":["partial","*",4,0],"Applebot-Extended":["partial","*",4,0],"CCBot":["partial","*",4,0],"Bytespider":["partial","*",4,0],"Amazonbot":["partial","*",4,0],"meta-externalagent":["partial","*",4,0],"cohere-ai":["partial","*",4,0],"Diffbot":["partial","*",4,0],"ContextFetchBot":["partial","*",4,0]},"llms":{"status":"present","http":200,"bytes":13693},"home":{"status":"ok","http":200,"url":"https://www.research.net/","html_tokens":50985,"markdown_tokens":220,"pruned_tokens":102}},
{"rank":7309,"domain":"deegee.dev","origin":"https://deegee.dev","robots":{"status":"ok","http":200,"url":"https://deegee.dev/robots.txt","bytes":58,"groups":1},"bots":{"GPTBot":["partial","*",2,0],"ChatGPT-User":["partial","*",2,0],"OAI-SearchBot":["partial","*",2,0],"ClaudeBot":["partial","*",2,0],"Claude-User":["partial","*",2,0],"Claude-SearchBot":["partial","*",2,0],"anthropic-ai":["partial","*",2,0],"PerplexityBot":["partial","*",2,0],"Perplexity-User":["partial","*",2,0],"Google-Extended":["partial","*",2,0],"Applebot-Extended":["partial","*",2,0],"CCBot":["partial","*",2,0],"Bytespider":["partial","*",2,0],"Amazonbot":["partial","*",2,0],"meta-externalagent":["partial","*",2,0],"cohere-ai":["partial","*",2,0],"Diffbot":["partial","*",2,0],"ContextFetchBot":["partial","*",2,0]},"llms":{"status":"missing","http":404,"bytes":2563},"home":{"status":"ok","http":200,"url":"https://deegee.dev/","html_tokens":1035,"markdown_tokens":26,"pruned_tokens":26}},
{"rank":7310,"domain":"mexc.com","origin":"https://mexc.com","robots":{"status":"ok","http":200,"url":"https://www.mexc.com/robots.txt","bytes":2096,"groups":2},"bots":{"GPTBot":["partial","*",26,12],"ChatGPT-User":["partial","*",26,12],"OAI-SearchBot":["partial","*",26,12],"ClaudeBot":["partial","*",26,12],"Claude-User":["partial","*",26,12],"Claude-SearchBot":["partial","*",26,12],"anthropic-ai":["partial","*",26,12],"PerplexityBot":["partial","*",26,12],"Perplexity-User":["partial","*",26,12],"Google-Extended":["partial","*",26,12],"Applebot-Extended":["partial","*",26,12],"CCBot":["partial","*",26,12],"Bytespider":["partial","*",26,12],"Amazonbot":["partial","*",26,12],"meta-externalagent":["partial","*",26,12],"cohere-ai":["partial","*",26,12],"Diffbot":["partial","*",26,12],"ContextFetchBot":["partial","*",26,12]},"llms":{"status":"present","http":200,"bytes":16808},"home":{"status":"ok","http":200,"url":"https://www.mexc.com/","html_tokens":315028,"markdown_tokens":343,"pruned_tokens":67}},
{"rank":7311,"domain":"servimg.com","origin":"https://servimg.com","robots":{"status":"ok","http":200,"url":"https://servimg.com/robots.txt","bytes":3460,"groups":5},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"missing","http":404,"bytes":427},"home":{"status":"ok","http":200,"url":"https://servimg.com/","html_tokens":3452,"markdown_tokens":150,"pruned_tokens":62}},
{"rank":7312,"domain":"solana.com","origin":"https://solana.com","robots":{"status":"ok","http":200,"url":"https://solana.com/robots.txt","bytes":163,"groups":1},"bots":{"GPTBot":["allowed","*",0,1],"ChatGPT-User":["allowed","*",0,1],"OAI-SearchBot":["allowed","*",0,1],"ClaudeBot":["allowed","*",0,1],"Claude-User":["allowed","*",0,1],"Claude-SearchBot":["allowed","*",0,1],"anthropic-ai":["allowed","*",0,1],"PerplexityBot":["allowed","*",0,1],"Perplexity-User":["allowed","*",0,1],"Google-Extended":["allowed","*",0,1],"Applebot-Extended":["allowed","*",0,1],"CCBot":["allowed","*",0,1],"Bytespider":["allowed","*",0,1],"Amazonbot":["allowed","*",0,1],"meta-externalagent":["allowed","*",0,1],"cohere-ai":["allowed","*",0,1],"Diffbot":["allowed","*",0,1],"ContextFetchBot":["allowed","*",0,1]},"llms":{"status":"present","http":200,"bytes":104154},"home":{"status":"ok","http":200,"url":"https://solana.com/","html_tokens":233357,"markdown_tokens":554,"pruned_tokens":136}},
{"rank":7313,"domain":"lightspeedhq.com","origin":"https://lightspeedhq.com","robots":{"status":"ok","http":200,"url":"https://www.lightspeedhq.com/robots.txt","bytes":764,"groups":2},"bots":{"GPTBot":["partial","*",4,2],"ChatGPT-User":["partial","*",4,2],"OAI-SearchBot":["partial","*",4,2],"ClaudeBot":["partial","*",4,2],"Claude-User":["partial","*",4,2],"Claude-SearchBot":["partial","*",4,2],"anthropic-ai":["partial","*",4,2],"PerplexityBot":["partial","*",4,2],"Perplexity-User":["partial","*",4,2],"Google-Extended":["partial","*",4,2],"Applebot-Extended":["partial","*",4,2],"CCBot":["partial","*",4,2],"Bytespider":["partial","*",4,2],"Amazonbot":["partial","*",4,2],"meta-externalagent":["partial","*",4,2],"cohere-ai":["partial","*",4,2],"Diffbot":["partial","*",4,2],"ContextFetchBot":["partial","*",4,2]},"llms":{"status":"http_error","http":403,"bytes":4549},"home":{"status":"ok","http":200,"url":"https://www.lightspeedhq.com/","html_tokens":80042,"markdown_tokens":428,"pruned_tokens":123}},
{"rank":7314,"domain":"usrfiles.com","origin":"https://www.usrfiles.com","robots":{"status":"refused","http":403,"url":"https://www.usrfiles.com/robots.txt","bytes":9,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":7315,"domain":"theroot.com","origin":"https://theroot.com","robots":{"status":"ok","http":200,"url":"https://www.theroot.com/robots.txt","bytes":116,"groups":1},"bots":{"GPTBot":["partial","*",1,1],"ChatGPT-User":["partial","*",1,1],"OAI-SearchBot":["partial","*",1,1],"ClaudeBot":["partial","*",1,1],"Claude-User":["partial","*",1,1],"Claude-SearchBot":["partial","*",1,1],"anthropic-ai":["partial","*",1,1],"PerplexityBot":["partial","*",1,1],"Perplexity-User":["partial","*",1,1],"Google-Extended":["partial","*",1,1],"Applebot-Extended":["partial","*",1,1],"CCBot":["partial","*",1,1],"Bytespider":["partial","*",1,1],"Amazonbot":["partial","*",1,1],"meta-externalagent":["partial","*",1,1],"cohere-ai":["partial","*",1,1],"Diffbot":["partial","*",1,1],"ContextFetchBot":["partial","*",1,1]},"llms":{"status":"missing","http":404,"bytes":155307},"home":{"status":"ok","http":202,"url":"https://www.theroot.com/","html_tokens":602,"markdown_tokens":40,"pruned_tokens":40}},
{"rank":7316,"domain":"ymaws.com","origin":"https://ymaws.com","robots":{"status":"unreachable","http":null,"url":"https://ymaws.com/robots.txt","bytes":0,"groups":0,"error":"cannot resolve ymaws.com"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":7317,"domain":"yn.gov.cn","origin":"https://yn.gov.cn","robots":{"status":"unreachable","http":null,"url":"https://yn.gov.cn/robots.txt","bytes":0,"groups":0,"error":"All connection attempts failed"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":7318,"domain":"share.flipboard.com","origin":"https://share.flipboard.com","robots":{"status":"server_error","http":503,"url":"https://share.flipboard.com/robots.txt","bytes":108,"groups":0,"error":"HTTP 503"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":7319,"domain":"redirectingat.com","origin":"https://redirectingat.com","robots":{"status":"ok","http":200,"url":"https://redirectingat.com/robots.txt","bytes":25,"groups":1},"bots":{"GPTBot":["blocked","*",1,0],"ChatGPT-User":["blocked","*",1,0],"OAI-SearchBot":["blocked","*",1,0],"ClaudeBot":["blocked","*",1,0],"Claude-User":["blocked","*",1,0],"Claude-SearchBot":["blocked","*",1,0],"anthropic-ai":["blocked","*",1,0],"PerplexityBot":["blocked","*",1,0],"Perplexity-User":["blocked","*",1,0],"Google-Extended":["blocked","*",1,0],"Applebot-Extended":["blocked","*",1,0],"CCBot":["blocked","*",1,0],"Bytespider":["blocked","*",1,0],"Amazonbot":["blocked","*",1,0],"meta-externalagent":["blocked","*",1,0],"cohere-ai":["blocked","*",1,0],"Diffbot":["blocked","*",1,0],"ContextFetchBot":["blocked","*",1,0]},"llms":{"status":"disallowed"},"home":{"status":"disallowed"}},
{"rank":7320,"domain":"youzan.com","origin":"https://youzan.com","robots":{"status":"ok","http":200,"url":"https://www.youzan.com/robots.txt","bytes":981,"groups":3},"bots":{"GPTBot":["partial","*",37,1],"ChatGPT-User":["partial","*",37,1],"OAI-SearchBot":["partial","*",37,1],"ClaudeBot":["partial","*",37,1],"Claude-User":["partial","*",37,1],"Claude-SearchBot":["partial","*",37,1],"anthropic-ai":["partial","*",37,1],"PerplexityBot":["partial","*",37,1],"Perplexity-User":["partial","*",37,1],"Google-Extended":["partial","*",37,1],"Applebot-Extended":["partial","*",37,1],"CCBot":["partial","*",37,1],"Bytespider":["partial","*",37,1],"Amazonbot":["partial","*",37,1],"meta-externalagent":["partial","*",37,1],"cohere-ai":["partial","*",37,1],"Diffbot":["partial","*",37,1],"ContextFetchBot":["partial","*",37,1]},"llms":{"status":"disallowed"},"home":{"status":"ok","http":200,"url":"https://www.youzan.com/","html_tokens":40731,"markdown_tokens":101,"pruned_tokens":101}},
{"rank":7321,"domain":"sc.gov.br","origin":"https://sc.gov.br","robots":{"status":"html","http":200,"url":"https://www.sc.gov.br/robots.txt","bytes":5051,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":7322,"domain":"stck.me","origin":"https://stck.me","robots":{"status":"ok","http":200,"url":"https://stck.me/robots.txt","bytes":61,"groups":1},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"present","http":200,"bytes":15635},"home":{"status":"ok","http":200,"url":"https://stck.me/","html_tokens":26526,"markdown_tokens":851,"pruned_tokens":160}},
{"rank":7323,"domain":"marktplaats.nl","origin":"https://marktplaats.nl","robots":{"status":"ok","http":200,"url":"https://www.marktplaats.nl/robots.txt","bytes":2940,"groups":1},"bots":{"GPTBot":["partial","*",84,0],"ChatGPT-User":["partial","*",84,0],"OAI-SearchBot":["partial","*",84,0],"ClaudeBot":["partial","*",84,0],"Claude-User":["partial","*",84,0],"Claude-SearchBot":["partial","*",84,0],"anthropic-ai":["partial","*",84,0],"PerplexityBot":["partial","*",84,0],"Perplexity-User":["partial","*",84,0],"Google-Extended":["partial","*",84,0],"Applebot-Extended":["partial","*",84,0],"CCBot":["partial","*",84,0],"Bytespider":["partial","*",84,0],"Amazonbot":["partial","*",84,0],"meta-externalagent":["partial","*",84,0],"cohere-ai":["partial","*",84,0],"Diffbot":["partial","*",84,0],"ContextFetchBot":["partial","*",84,0]},"llms":{"status":"present","http":200,"bytes":9485},"home":{"status":"ok","http":200,"url":"https://www.marktplaats.nl/","html_tokens":34756,"markdown_tokens":11,"pruned_tokens":11}},
{"rank":7324,"domain":"tianya.cn","origin":"https://tianya.cn","robots":{"status":"timeout","http":null,"url":"https://tianya.cn/robots.txt","bytes":0,"groups":0,"error":"timed out after 10 s"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":7325,"domain":"rtp.pt","origin":"https://rtp.pt","robots":{"status":"ok","http":200,"url":"https://www.rtp.pt/robots.txt","bytes":364,"groups":6},"bots":{"GPTBot":["partial","*",1,0],"ChatGPT-User":["partial","*",1,0],"OAI-SearchBot":["partial","*",1,0],"ClaudeBot":["partial","*",1,0],"Claude-User":["partial","*",1,0],"Claude-SearchBot":["partial","*",1,0],"anthropic-ai":["partial","*",1,0],"PerplexityBot":["partial","*",1,0],"Perplexity-User":["partial","*",1,0],"Google-Extended":["partial","*",1,0],"Applebot-Extended":["partial","*",1,0],"CCBot":["partial","*",1,0],"Bytespider":["partial","*",1,0],"Amazonbot":["partial","*",1,0],"meta-externalagent":["partial","*",1,0],"cohere-ai":["partial","*",1,0],"Diffbot":["partial","*",1,0],"ContextFetchBot":["partial","*",1,0]},"llms":{"status":"missing","http":404,"bytes":20489},"home":{"status":"ok","http":200,"url":"https://www.rtp.pt/homepage-internacional/","html_tokens":19363,"markdown_tokens":118,"pruned_tokens":80}},
{"rank":7326,"domain":"google.com.jm","origin":"https://google.com.jm","robots":{"status":"ok","http":200,"url":"https://www.google.com.jm/robots.txt","bytes":6624,"groups":4},"bots":{"GPTBot":["partial","*",169,64],"ChatGPT-User":["partial","*",169,64],"OAI-SearchBot":["partial","*",169,64],"ClaudeBot":["partial","*",169,64],"Claude-User":["partial","*",169,64],"Claude-SearchBot":["partial","*",169,64],"anthropic-ai":["partial","*",169,64],"PerplexityBot":["partial","*",169,64],"Perplexity-User":["partial","*",169,64],"Google-Extended":["partial","*",169,64],"Applebot-Extended":["partial","*",169,64],"CCBot":["partial","*",169,64],"Bytespider":["partial","*",169,64],"Amazonbot":["partial","*",169,64],"meta-externalagent":["partial","*",169,64],"cohere-ai":["partial","*",169,64],"Diffbot":["partial","*",169,64],"ContextFetchBot":["partial","*",169,64]},"llms":{"status":"missing","http":404,"bytes":1569},"home":{"status":"ok","http":200,"url":"https://www.google.com/","html_tokens":21344,"markdown_tokens":30,"pruned_tokens":30}},
{"rank":7327,"domain":"jabra.com","origin":"https://jabra.com","robots":{"status":"ok","http":200,"url":"https://www.jabra.com/robots.txt","bytes":3785,"groups":1},"bots":{"GPTBot":["partial","*",1,1],"ChatGPT-User":["partial","*",1,1],"OAI-SearchBot":["partial","*",1,1],"ClaudeBot":["partial","*",1,1],"Claude-User":["partial","*",1,1],"Claude-SearchBot":["partial","*",1,1],"anthropic-ai":["partial","*",1,1],"PerplexityBot":["partial","*",1,1],"Perplexity-User":["partial","*",1,1],"Google-Extended":["partial","*",1,1],"Applebot-Extended":["partial","*",1,1],"CCBot":["partial","*",1,1],"Bytespider":["partial","*",1,1],"Amazonbot":["partial","*",1,1],"meta-externalagent":["partial","*",1,1],"cohere-ai":["partial","*",1,1],"Diffbot":["partial","*",1,1],"ContextFetchBot":["partial","*",1,1]},"llms":{"status":"missing","http":404,"bytes":461451},"home":{"status":"ok","http":200,"url":"https://www.jabra.com/","html_tokens":84117,"markdown_tokens":341,"pruned_tokens":232}},
{"rank":7328,"domain":"mmu.ac.uk","origin":"https://mmu.ac.uk","robots":{"status":"ok","http":200,"url":"https://www.mmu.ac.uk/robots.txt","bytes":2132,"groups":1},"bots":{"GPTBot":["partial","*",38,18],"ChatGPT-User":["partial","*",38,18],"OAI-SearchBot":["partial","*",38,18],"ClaudeBot":["partial","*",38,18],"Claude-User":["partial","*",38,18],"Claude-SearchBot":["partial","*",38,18],"anthropic-ai":["partial","*",38,18],"PerplexityBot":["partial","*",38,18],"Perplexity-User":["partial","*",38,18],"Google-Extended":["partial","*",38,18],"Applebot-Extended":["partial","*",38,18],"CCBot":["partial","*",38,18],"Bytespider":["partial","*",38,18],"Amazonbot":["partial","*",38,18],"meta-externalagent":["partial","*",38,18],"cohere-ai":["partial","*",38,18],"Diffbot":["partial","*",38,18],"ContextFetchBot":["partial","*",38,18]},"llms":{"status":"missing","http":404,"bytes":146},"home":{"status":"ok","http":200,"url":"https://www.mmu.ac.uk/","html_tokens":25912,"markdown_tokens":160,"pruned_tokens":160}},
{"rank":7329,"domain":"gbif.org","origin":"https://gbif.org","robots":{"status":"refused","http":403,"url":"https://www.gbif.org/robots.txt","bytes":5422,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":7330,"domain":"garv.in","origin":"https://garv.in","robots":{"status":"ok","http":200,"url":"https://garv.in/robots.txt","bytes":179,"groups":5},"bots":{"GPTBot":["partial","*",1,0],"ChatGPT-User":["partial","*",1,0],"OAI-SearchBot":["partial","*",1,0],"ClaudeBot":["partial","*",1,0],"Claude-User":["partial","*",1,0],"Claude-SearchBot":["partial","*",1,0],"anthropic-ai":["partial","*",1,0],"PerplexityBot":["partial","*",1,0],"Perplexity-User":["partial","*",1,0],"Google-Extended":["partial","*",1,0],"Applebot-Extended":["partial","*",1,0],"CCBot":["partial","*",1,0],"Bytespider":["partial","*",1,0],"Amazonbot":["partial","*",1,0],"meta-externalagent":["partial","*",1,0],"cohere-ai":["partial","*",1,0],"Diffbot":["partial","*",1,0],"ContextFetchBot":["partial","*",1,0]},"llms":{"status":"missing","http":404,"bytes":6945},"home":{"status":"ok","http":200,"url":"https://garv.in/","html_tokens":1642,"markdown_tokens":8,"pruned_tokens":8}},
{"rank":7331,"domain":"arup.com","origin":"https://arup.com","robots":{"status":"ok","http":200,"url":"https://www.arup.com/robots.txt","bytes":266,"groups":1},"bots":{"GPTBot":["partial","*",8,1],"ChatGPT-User":["partial","*",8,1],"OAI-SearchBot":["partial","*",8,1],"ClaudeBot":["partial","*",8,1],"Claude-User":["partial","*",8,1],"Claude-SearchBot":["partial","*",8,1],"anthropic-ai":["partial","*",8,1],"PerplexityBot":["partial","*",8,1],"Perplexity-User":["partial","*",8,1],"Google-Extended":["partial","*",8,1],"Applebot-Extended":["partial","*",8,1],"CCBot":["partial","*",8,1],"Bytespider":["partial","*",8,1],"Amazonbot":["partial","*",8,1],"meta-externalagent":["partial","*",8,1],"cohere-ai":["partial","*",8,1],"Diffbot":["partial","*",8,1],"ContextFetchBot":["partial","*",8,1]},"llms":{"status":"missing","http":404,"bytes":101477},"home":{"status":"ok","http":200,"url":"https://www.arup.com/","html_tokens":37428,"markdown_tokens":206,"pruned_tokens":42}},
{"rank":7332,"domain":"ratgeberrecht.eu","origin":"https://ratgeberrecht.eu","robots":{"status":"ok","http":200,"url":"https://www.ratgeberrecht.eu/robots.txt","bytes":666,"groups":7},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["partial","*",6,8],"OAI-SearchBot":["partial","*",6,8],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",6,8],"Claude-SearchBot":["partial","*",6,8],"anthropic-ai":["partial","*",6,8],"PerplexityBot":["partial","*",6,8],"Perplexity-User":["partial","*",6,8],"Google-Extended":["partial","*",6,8],"Applebot-Extended":["partial","*",6,8],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["partial","*",6,8],"meta-externalagent":["partial","*",6,8],"cohere-ai":["partial","*",6,8],"Diffbot":["partial","*",6,8],"ContextFetchBot":["partial","*",6,8]},"llms":{"status":"missing","http":404,"bytes":146},"home":{"status":"ok","http":200,"url":"https://www.ratgeberrecht.eu/","html_tokens":22036,"markdown_tokens":425,"pruned_tokens":102}},
{"rank":7333,"domain":"phila.gov","origin":"https://phila.gov","robots":{"status":"missing","http":404,"url":"https://www.phila.gov/robots.txt","bytes":40025,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":40025},"home":{"status":"ok","http":200,"url":"https://www.phila.gov/","html_tokens":9322,"markdown_tokens":11,"pruned_tokens":11}},
{"rank":7334,"domain":"blackboard.com","origin":"https://blackboard.com","robots":{"status":"ok","http":200,"url":"https://www.blackboard.com/robots.txt","bytes":197,"groups":1},"bots":{"GPTBot":["partial","*",5,1],"ChatGPT-User":["partial","*",5,1],"OAI-SearchBot":["partial","*",5,1],"ClaudeBot":["partial","*",5,1],"Claude-User":["partial","*",5,1],"Claude-SearchBot":["partial","*",5,1],"anthropic-ai":["partial","*",5,1],"PerplexityBot":["partial","*",5,1],"Perplexity-User":["partial","*",5,1],"Google-Extended":["partial","*",5,1],"Applebot-Extended":["partial","*",5,1],"CCBot":["partial","*",5,1],"Bytespider":["partial","*",5,1],"Amazonbot":["partial","*",5,1],"meta-externalagent":["partial","*",5,1],"cohere-ai":["partial","*",5,1],"Diffbot":["partial","*",5,1],"ContextFetchBot":["partial","*",5,1]},"llms":{"status":"present","http":200,"bytes":3251},"home":{"status":"ok","http":200,"url":"https://www.blackboard.com/","html_tokens":168364,"markdown_tokens":82,"pruned_tokens":82}},
{"rank":7335,"domain":"thinkphp.cn","origin":"https://www.thinkphp.cn","robots":{"status":"missing","http":404,"url":"https://www.thinkphp.cn/robots.txt","bytes":6570,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"present","http":200,"bytes":3899},"home":{"status":"ok","http":200,"url":"https://www.thinkphp.cn/","html_tokens":7954,"markdown_tokens":80,"pruned_tokens":80}},
{"rank":7336,"domain":"madrid.es","origin":"https://madrid.es","robots":{"status":"refused","http":403,"url":"https://madrid.es/robots.txt","bytes":373,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":7337,"domain":"ndsu.edu","origin":"https://ndsu.edu","robots":{"status":"refused","http":403,"url":"https://ndsu.edu/robots.txt","bytes":560,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":7338,"domain":"thesmokinggun.com","origin":"https://thesmokinggun.com","robots":{"status":"ok","http":200,"url":"https://thesmokinggun.com/robots.txt","bytes":1510,"groups":1},"bots":{"GPTBot":["partial","*",35,0],"ChatGPT-User":["partial","*",35,0],"OAI-SearchBot":["partial","*",35,0],"ClaudeBot":["partial","*",35,0],"Claude-User":["partial","*",35,0],"Claude-SearchBot":["partial","*",35,0],"anthropic-ai":["partial","*",35,0],"PerplexityBot":["partial","*",35,0],"Perplexity-User":["partial","*",35,0],"Google-Extended":["partial","*",35,0],"Applebot-Extended":["partial","*",35,0],"CCBot":["partial","*",35,0],"Bytespider":["partial","*",35,0],"Amazonbot":["partial","*",35,0],"meta-externalagent":["partial","*",35,0],"cohere-ai":["partial","*",35,0],"Diffbot":["partial","*",35,0],"ContextFetchBot":["partial","*",35,0]},"llms":{"status":"missing","http":404,"bytes":549},"home":{"status":"http_error","http":403,"url":"https://thesmokinggun.com/"}},
{"rank":7339,"domain":"google.co.zm","origin":"https://google.co.zm","robots":{"status":"ok","http":200,"url":"https://www.google.co.zm/robots.txt","bytes":6624,"groups":4},"bots":{"GPTBot":["partial","*",169,64],"ChatGPT-User":["partial","*",169,64],"OAI-SearchBot":["partial","*",169,64],"ClaudeBot":["partial","*",169,64],"Claude-User":["partial","*",169,64],"Claude-SearchBot":["partial","*",169,64],"anthropic-ai":["partial","*",169,64],"PerplexityBot":["partial","*",169,64],"Perplexity-User":["partial","*",169,64],"Google-Extended":["partial","*",169,64],"Applebot-Extended":["partial","*",169,64],"CCBot":["partial","*",169,64],"Bytespider":["partial","*",169,64],"Amazonbot":["partial","*",169,64],"meta-externalagent":["partial","*",169,64],"cohere-ai":["partial","*",169,64],"Diffbot":["partial","*",169,64],"ContextFetchBot":["partial","*",169,64]},"llms":{"status":"missing","http":404,"bytes":1569},"home":{"status":"ok","http":200,"url":"https://www.google.com/","html_tokens":21302,"markdown_tokens":30,"pruned_tokens":30}},
{"rank":7340,"domain":"insead.edu","origin":"https://insead.edu","robots":{"status":"refused","http":403,"url":"https://www.insead.edu/robots.txt","bytes":5509,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":7341,"domain":"id.pinterest.com","origin":"https://id.pinterest.com","robots":{"status":"ok","http":200,"url":"https://id.pinterest.com/robots.txt","bytes":32150,"groups":8},"bots":{"GPTBot":["blocked","*",1,0],"ChatGPT-User":["blocked","*",1,0],"OAI-SearchBot":["blocked","*",1,0],"ClaudeBot":["blocked","*",1,0],"Claude-User":["blocked","*",1,0],"Claude-SearchBot":["blocked","*",1,0],"anthropic-ai":["blocked","*",1,0],"PerplexityBot":["blocked","*",1,0],"Perplexity-User":["blocked","*",1,0],"Google-Extended":["blocked","*",1,0],"Applebot-Extended":["blocked","*",1,0],"CCBot":["blocked","*",1,0],"Bytespider":["blocked","*",1,0],"Amazonbot":["blocked","*",1,0],"meta-externalagent":["blocked","*",1,0],"cohere-ai":["blocked","*",1,0],"Diffbot":["blocked","*",1,0],"ContextFetchBot":["blocked","*",1,0]},"llms":{"status":"disallowed"},"home":{"status":"disallowed"}},
{"rank":7342,"domain":"wtatennis.com","origin":"https://wtatennis.com","robots":{"status":"ok","http":200,"url":"https://www.wtatennis.com/robots.txt","bytes":78,"groups":1},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"present","http":200,"bytes":14043},"home":{"status":"ok","http":200,"url":"https://www.wtatennis.com/","html_tokens":108228,"markdown_tokens":10,"pruned_tokens":10}},
{"rank":7343,"domain":"yousendit.com","origin":"https://yousendit.com","robots":{"status":"refused","http":400,"url":"https://yousendit.com/robots.txt","bytes":26,"groups":0,"error":"HTTP 400"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":7344,"domain":"app.com","origin":"https://app.com","robots":{"status":"ok","http":200,"url":"https://www.app.com/robots.txt","bytes":12760,"groups":288},"bots":{"GPTBot":["partial","named",1,4],"ChatGPT-User":["partial","named",1,4],"OAI-SearchBot":["partial","named",1,4],"ClaudeBot":["blocked","named",1,0],"Claude-User":["blocked","named",1,0],"Claude-SearchBot":["blocked","named",1,0],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["partial","named",1,4],"Perplexity-User":["blocked","named",1,0],"Google-Extended":["partial","named",1,4],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",2,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",33,0]},"llms":{"status":"http_error","http":402,"bytes":254},"home":{"status":"http_error","http":402,"url":"https://www.app.com/"}},
{"rank":7345,"domain":"robertsspaceindustries.com","origin":"https://robertsspaceindustries.com","robots":{"status":"ok","http":200,"url":"https://robertsspaceindustries.com/robots.txt","bytes":89,"groups":1},"bots":{"GPTBot":["partial","*",1,0],"ChatGPT-User":["partial","*",1,0],"OAI-SearchBot":["partial","*",1,0],"ClaudeBot":["partial","*",1,0],"Claude-User":["partial","*",1,0],"Claude-SearchBot":["partial","*",1,0],"anthropic-ai":["partial","*",1,0],"PerplexityBot":["partial","*",1,0],"Perplexity-User":["partial","*",1,0],"Google-Extended":["partial","*",1,0],"Applebot-Extended":["partial","*",1,0],"CCBot":["partial","*",1,0],"Bytespider":["partial","*",1,0],"Amazonbot":["partial","*",1,0],"meta-externalagent":["partial","*",1,0],"cohere-ai":["partial","*",1,0],"Diffbot":["partial","*",1,0],"ContextFetchBot":["partial","*",1,0]},"llms":{"status":"missing","http":404,"bytes":32154},"home":{"status":"ok","http":200,"url":"https://robertsspaceindustries.com/en/","html_tokens":8827,"markdown_tokens":0,"pruned_tokens":0}},
{"rank":7346,"domain":"ifla.org","origin":"https://ifla.org","robots":{"status":"ok","http":200,"url":"https://www.ifla.org/robots.txt","bytes":113,"groups":1},"bots":{"GPTBot":["partial","*",1,1],"ChatGPT-User":["partial","*",1,1],"OAI-SearchBot":["partial","*",1,1],"ClaudeBot":["partial","*",1,1],"Claude-User":["partial","*",1,1],"Claude-SearchBot":["partial","*",1,1],"anthropic-ai":["partial","*",1,1],"PerplexityBot":["partial","*",1,1],"Perplexity-User":["partial","*",1,1],"Google-Extended":["partial","*",1,1],"Applebot-Extended":["partial","*",1,1],"CCBot":["partial","*",1,1],"Bytespider":["partial","*",1,1],"Amazonbot":["partial","*",1,1],"meta-externalagent":["partial","*",1,1],"cohere-ai":["partial","*",1,1],"Diffbot":["partial","*",1,1],"ContextFetchBot":["partial","*",1,1]},"llms":{"status":"html","http":200,"bytes":87164},"home":{"status":"http_error","http":403,"url":"https://ifla.org/"}},
{"rank":7347,"domain":"verywellfamily.com","origin":"http://verywellfamily.com","robots":{"status":"html","http":200,"url":"http://comingsoon.markmonitor.com/","bytes":812,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":7348,"domain":"scaleway.com","origin":"https://scaleway.com","robots":{"status":"ok","http":200,"url":"https://www.scaleway.com/robots.txt","bytes":517,"groups":1},"bots":{"GPTBot":["partial","*",24,0],"ChatGPT-User":["partial","*",24,0],"OAI-SearchBot":["partial","*",24,0],"ClaudeBot":["partial","*",24,0],"Claude-User":["partial","*",24,0],"Claude-SearchBot":["partial","*",24,0],"anthropic-ai":["partial","*",24,0],"PerplexityBot":["partial","*",24,0],"Perplexity-User":["partial","*",24,0],"Google-Extended":["partial","*",24,0],"Applebot-Extended":["partial","*",24,0],"CCBot":["partial","*",24,0],"Bytespider":["partial","*",24,0],"Amazonbot":["partial","*",24,0],"meta-externalagent":["partial","*",24,0],"cohere-ai":["partial","*",24,0],"Diffbot":["partial","*",24,0],"ContextFetchBot":["partial","*",24,0]},"llms":{"status":"missing","http":404,"bytes":405025},"home":{"status":"ok","http":200,"url":"https://www.scaleway.com/en/","html_tokens":120929,"markdown_tokens":1124,"pruned_tokens":231}},
{"rank":7349,"domain":"bbci.co.uk","origin":"http://bbci.co.uk","robots":{"status":"missing","http":404,"url":"http://bbci.co.uk/robots.txt","bytes":32408,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":32408},"home":{"status":"ok","http":200,"url":"https://www.bbc.co.uk/","html_tokens":179118,"markdown_tokens":56,"pruned_tokens":56}},
{"rank":7350,"domain":"verde-casino-spielen.com","origin":"https://verde-casino-spielen.com","robots":{"status":"html","http":200,"url":"https://verde-casino-spielen.com/robots.txt","bytes":1307,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":7351,"domain":"realvnc.com","origin":"https://realvnc.com","robots":{"status":"refused","http":403,"url":"https://realvnc.com/robots.txt","bytes":103,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":7352,"domain":"electronjs.org","origin":"https://electronjs.org","robots":{"status":"missing","http":404,"url":"https://www.electronjs.org/robots.txt","bytes":17288,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":17288},"home":{"status":"ok","http":200,"url":"https://www.electronjs.org/","html_tokens":12307,"markdown_tokens":774,"pruned_tokens":298}},
{"rank":7353,"domain":"infusionsoft.com","origin":"https://infusionsoft.com","robots":{"status":"ok","http":200,"url":"https://keap.com/robots.txt","bytes":369,"groups":3},"bots":{"GPTBot":["partial","*",6,0],"ChatGPT-User":["partial","*",6,0],"OAI-SearchBot":["partial","*",6,0],"ClaudeBot":["partial","*",6,0],"Claude-User":["partial","*",6,0],"Claude-SearchBot":["partial","*",6,0],"anthropic-ai":["partial","*",6,0],"PerplexityBot":["partial","*",6,0],"Perplexity-User":["partial","*",6,0],"Google-Extended":["partial","*",6,0],"Applebot-Extended":["partial","*",6,0],"CCBot":["partial","*",6,0],"Bytespider":["partial","*",6,0],"Amazonbot":["partial","*",6,0],"meta-externalagent":["partial","*",6,0],"cohere-ai":["partial","*",6,0],"Diffbot":["partial","*",6,0],"ContextFetchBot":["partial","*",6,0]},"llms":{"status":"missing","http":404,"bytes":200689},"home":{"status":"ok","http":200,"url":"https://keap.com","html_tokens":72496,"markdown_tokens":109,"pruned_tokens":109}},
{"rank":7354,"domain":"asq.org","origin":"https://asq.org","robots":{"status":"ok","http":200,"url":"https://asq.org/robots.txt","bytes":2020,"groups":1},"bots":{"GPTBot":["partial","*",50,0],"ChatGPT-User":["partial","*",50,0],"OAI-SearchBot":["partial","*",50,0],"ClaudeBot":["partial","*",50,0],"Claude-User":["partial","*",50,0],"Claude-SearchBot":["partial","*",50,0],"anthropic-ai":["partial","*",50,0],"PerplexityBot":["partial","*",50,0],"Perplexity-User":["partial","*",50,0],"Google-Extended":["partial","*",50,0],"Applebot-Extended":["partial","*",50,0],"CCBot":["partial","*",50,0],"Bytespider":["partial","*",50,0],"Amazonbot":["partial","*",50,0],"meta-externalagent":["partial","*",50,0],"cohere-ai":["partial","*",50,0],"Diffbot":["partial","*",50,0],"ContextFetchBot":["partial","*",50,0]},"llms":{"status":"missing","http":404,"bytes":28069},"home":{"status":"http_error","http":403,"url":"https://asq.org/"}},
{"rank":7355,"domain":"onlinemobilecasinos.de","origin":"https://onlinemobilecasinos.de","robots":{"status":"ok","http":200,"url":"https://onlinemobilecasinos.de/robots.txt","bytes":549,"groups":1},"bots":{"GPTBot":["partial","*",19,0],"ChatGPT-User":["partial","*",19,0],"OAI-SearchBot":["partial","*",19,0],"ClaudeBot":["partial","*",19,0],"Claude-User":["partial","*",19,0],"Claude-SearchBot":["partial","*",19,0],"anthropic-ai":["partial","*",19,0],"PerplexityBot":["partial","*",19,0],"Perplexity-User":["partial","*",19,0],"Google-Extended":["partial","*",19,0],"Applebot-Extended":["partial","*",19,0],"CCBot":["partial","*",19,0],"Bytespider":["partial","*",19,0],"Amazonbot":["partial","*",19,0],"meta-externalagent":["partial","*",19,0],"cohere-ai":["partial","*",19,0],"Diffbot":["partial","*",19,0],"ContextFetchBot":["partial","*",19,0]},"llms":{"status":"timeout","error":"timed out after 10 s"},"home":{"status":"ok","http":200,"url":"https://onlinemobilecasinos.de/","html_tokens":26011,"markdown_tokens":2200,"pruned_tokens":156}},
{"rank":7356,"domain":"arcelormittal.com","origin":"https://arcelormittal.com","robots":{"status":"refused","http":403,"url":"https://corporate.arcelormittal.com","bytes":5535,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":7357,"domain":"cshlp.org","origin":"https://cshlp.org","robots":{"status":"refused","http":403,"url":"https://cshlp.org/robots.txt","bytes":5546,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":7358,"domain":"skimresources.com","origin":"https://skimresources.com","robots":{"status":"ok","http":200,"url":"https://skimresources.com/robots.txt","bytes":25,"groups":1},"bots":{"GPTBot":["blocked","*",1,0],"ChatGPT-User":["blocked","*",1,0],"OAI-SearchBot":["blocked","*",1,0],"ClaudeBot":["blocked","*",1,0],"Claude-User":["blocked","*",1,0],"Claude-SearchBot":["blocked","*",1,0],"anthropic-ai":["blocked","*",1,0],"PerplexityBot":["blocked","*",1,0],"Perplexity-User":["blocked","*",1,0],"Google-Extended":["blocked","*",1,0],"Applebot-Extended":["blocked","*",1,0],"CCBot":["blocked","*",1,0],"Bytespider":["blocked","*",1,0],"Amazonbot":["blocked","*",1,0],"meta-externalagent":["blocked","*",1,0],"cohere-ai":["blocked","*",1,0],"Diffbot":["blocked","*",1,0],"ContextFetchBot":["blocked","*",1,0]},"llms":{"status":"disallowed"},"home":{"status":"disallowed"}},
{"rank":7359,"domain":"kenpai.com","origin":"https://kenpai.com","robots":{"status":"ok","http":200,"url":"https://www.kenpai.com/robots.txt","bytes":24,"groups":1},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"missing","http":404,"bytes":34547},"home":{"status":"ok","http":200,"url":"https://www.kenpai.com/","html_tokens":17085,"markdown_tokens":335,"pruned_tokens":175}},
{"rank":7360,"domain":"defenseurdesdroits.fr","origin":"https://defenseurdesdroits.fr","robots":{"status":"ok","http":200,"url":"https://www.defenseurdesdroits.fr/robots.txt","bytes":2489,"groups":1},"bots":{"GPTBot":["partial","*",48,18],"ChatGPT-User":["partial","*",48,18],"OAI-SearchBot":["partial","*",48,18],"ClaudeBot":["partial","*",48,18],"Claude-User":["partial","*",48,18],"Claude-SearchBot":["partial","*",48,18],"anthropic-ai":["partial","*",48,18],"PerplexityBot":["partial","*",48,18],"Perplexity-User":["partial","*",48,18],"Google-Extended":["partial","*",48,18],"Applebot-Extended":["partial","*",48,18],"CCBot":["partial","*",48,18],"Bytespider":["partial","*",48,18],"Amazonbot":["partial","*",48,18],"meta-externalagent":["partial","*",48,18],"cohere-ai":["partial","*",48,18],"Diffbot":["partial","*",48,18],"ContextFetchBot":["partial","*",48,18]},"llms":{"status":"missing","http":404,"bytes":199},"home":{"status":"ok","http":200,"url":"https://www.defenseurdesdroits.fr/","html_tokens":16744,"markdown_tokens":1131,"pruned_tokens":13}},
{"rank":7361,"domain":"laptopmag.com","origin":"https://laptopmag.com","robots":{"status":"ok","http":200,"url":"https://www.laptopmag.com/robots.txt","bytes":1882,"groups":5},"bots":{"GPTBot":["partial","*",19,0],"ChatGPT-User":["partial","*",19,0],"OAI-SearchBot":["partial","*",19,0],"ClaudeBot":["partial","*",19,0],"Claude-User":["partial","*",19,0],"Claude-SearchBot":["partial","*",19,0],"anthropic-ai":["partial","*",19,0],"PerplexityBot":["partial","*",19,0],"Perplexity-User":["partial","*",19,0],"Google-Extended":["partial","*",19,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["partial","*",19,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["partial","*",19,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",19,0]},"llms":{"status":"missing","http":404,"bytes":438452},"home":{"status":"ok","http":200,"url":"https://www.laptopmag.com/","html_tokens":241112,"markdown_tokens":17,"pruned_tokens":17}},
{"rank":7362,"domain":"sectigo.com","origin":"https://sectigo.com","robots":{"status":"ok","http":200,"url":"https://www.sectigo.com/robots.txt","bytes":1145,"groups":1},"bots":{"GPTBot":["partial","*",22,0],"ChatGPT-User":["partial","*",22,0],"OAI-SearchBot":["partial","*",22,0],"ClaudeBot":["partial","*",22,0],"Claude-User":["partial","*",22,0],"Claude-SearchBot":["partial","*",22,0],"anthropic-ai":["partial","*",22,0],"PerplexityBot":["partial","*",22,0],"Perplexity-User":["partial","*",22,0],"Google-Extended":["partial","*",22,0],"Applebot-Extended":["partial","*",22,0],"CCBot":["partial","*",22,0],"Bytespider":["partial","*",22,0],"Amazonbot":["partial","*",22,0],"meta-externalagent":["partial","*",22,0],"cohere-ai":["partial","*",22,0],"Diffbot":["partial","*",22,0],"ContextFetchBot":["partial","*",22,0]},"llms":{"status":"missing","http":404,"bytes":493778},"home":{"status":"ok","http":200,"url":"https://www.sectigo.com/","html_tokens":174481,"markdown_tokens":426,"pruned_tokens":8}},
{"rank":7363,"domain":"sports.ru","origin":"https://sports.ru","robots":{"status":"ok","http":200,"url":"https://www.sports.ru/robots.txt","bytes":4013,"groups":2},"bots":{"GPTBot":["partial","*",53,2],"ChatGPT-User":["partial","*",53,2],"OAI-SearchBot":["partial","*",53,2],"ClaudeBot":["partial","*",53,2],"Claude-User":["partial","*",53,2],"Claude-SearchBot":["partial","*",53,2],"anthropic-ai":["partial","*",53,2],"PerplexityBot":["partial","*",53,2],"Perplexity-User":["partial","*",53,2],"Google-Extended":["partial","*",53,2],"Applebot-Extended":["partial","*",53,2],"CCBot":["partial","*",53,2],"Bytespider":["partial","*",53,2],"Amazonbot":["partial","*",53,2],"meta-externalagent":["partial","*",53,2],"cohere-ai":["partial","*",53,2],"Diffbot":["partial","*",53,2],"ContextFetchBot":["partial","*",53,2]},"llms":{"status":"missing","http":404,"bytes":37154},"home":{"status":"ok","http":200,"url":"https://www.sports.ru/","html_tokens":225133,"markdown_tokens":22,"pruned_tokens":22}},
{"rank":7364,"domain":"spinmagazine.com","origin":"https://spinmagazine.com","robots":{"status":"ok","http":200,"url":"https://www.spinmagazine.com/robots.txt","bytes":446,"groups":5},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["partial","*",6,1],"OAI-SearchBot":["partial","*",6,1],"ClaudeBot":["partial","*",6,1],"Claude-User":["partial","*",6,1],"Claude-SearchBot":["partial","*",6,1],"anthropic-ai":["partial","*",6,1],"PerplexityBot":["partial","*",6,1],"Perplexity-User":["partial","*",6,1],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["partial","*",6,1],"CCBot":["blocked","named",1,0],"Bytespider":["partial","*",6,1],"Amazonbot":["partial","*",6,1],"meta-externalagent":["partial","*",6,1],"cohere-ai":["partial","*",6,1],"Diffbot":["partial","*",6,1],"ContextFetchBot":["partial","*",6,1]},"llms":{"status":"other_text","http":200,"bytes":4728},"home":{"status":"ok","http":200,"url":"https://www.spinmagazine.com/","html_tokens":91555,"markdown_tokens":8,"pruned_tokens":8}},
{"rank":7365,"domain":"livecasinoau.com","origin":"http://livecasinoau.com","robots":{"status":"html","http":200,"url":"http://livecasinoau.com/robots.txt","bytes":3086,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":7366,"domain":"doc-exam.com","origin":"https://www.doc-exam.com","robots":{"status":"missing","http":404,"url":"https://www.doc-exam.com/robots.txt","bytes":246,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":244},"home":{"status":"ok","http":200,"url":"https://www.doc-exam.com/","html_tokens":1760,"markdown_tokens":150,"pruned_tokens":150}},
{"rank":7367,"domain":"autoweek.com","origin":"https://autoweek.com","robots":{"status":"ok","http":200,"url":"https://autoweek.com/robots.txt","bytes":961,"groups":3},"bots":{"GPTBot":["partial","named",32,0],"ChatGPT-User":["partial","named",32,0],"OAI-SearchBot":["partial","named",32,0],"ClaudeBot":["partial","*",19,0],"Claude-User":["partial","*",19,0],"Claude-SearchBot":["partial","*",19,0],"anthropic-ai":["partial","*",19,0],"PerplexityBot":["partial","*",19,0],"Perplexity-User":["partial","*",19,0],"Google-Extended":["partial","*",19,0],"Applebot-Extended":["partial","*",19,0],"CCBot":["partial","*",19,0],"Bytespider":["partial","*",19,0],"Amazonbot":["partial","*",19,0],"meta-externalagent":["partial","*",19,0],"cohere-ai":["partial","*",19,0],"Diffbot":["partial","*",19,0],"ContextFetchBot":["partial","*",19,0]},"llms":{"status":"missing","http":404,"bytes":84},"home":{"status":"ok","http":200,"url":"https://www.autoweek.com/","html_tokens":156811,"markdown_tokens":176,"pruned_tokens":74}},
{"rank":7368,"domain":"roland.com","origin":"https://roland.com","robots":{"status":"ok","http":200,"url":"https://www.roland.com/robots.txt","bytes":2379,"groups":25},"bots":{"GPTBot":["partial","*",45,1],"ChatGPT-User":["partial","*",45,1],"OAI-SearchBot":["partial","*",45,1],"ClaudeBot":["partial","*",45,1],"Claude-User":["partial","*",45,1],"Claude-SearchBot":["partial","*",45,1],"anthropic-ai":["partial","*",45,1],"PerplexityBot":["partial","*",45,1],"Perplexity-User":["partial","*",45,1],"Google-Extended":["partial","*",45,1],"Applebot-Extended":["partial","*",45,1],"CCBot":["blocked","named",1,0],"Bytespider":["partial","*",45,1],"Amazonbot":["partial","*",45,1],"meta-externalagent":["partial","*",45,1],"cohere-ai":["partial","*",45,1],"Diffbot":["partial","*",45,1],"ContextFetchBot":["partial","*",45,1]},"llms":{"status":"missing","http":404,"bytes":5284},"home":{"status":"ok","http":200,"url":"https://www.roland.com/global/","html_tokens":17689,"markdown_tokens":72,"pruned_tokens":72}},
{"rank":7369,"domain":"tut.by","origin":"https://tut.by","robots":{"status":"unreachable","http":null,"url":"https://tut.by/robots.txt","bytes":0,"groups":0,"error":"cannot resolve tut.by"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":7370,"domain":"trustedshops.de","origin":"https://trustedshops.de","robots":{"status":"ok","http":200,"url":"https://www.trustedshops.de/robots.txt","bytes":2537,"groups":4},"bots":{"GPTBot":["partial","*",21,0],"ChatGPT-User":["partial","*",21,0],"OAI-SearchBot":["partial","*",21,0],"ClaudeBot":["partial","*",21,0],"Claude-User":["partial","*",21,0],"Claude-SearchBot":["partial","*",21,0],"anthropic-ai":["partial","*",21,0],"PerplexityBot":["partial","*",21,0],"Perplexity-User":["partial","*",21,0],"Google-Extended":["partial","*",21,0],"Applebot-Extended":["partial","*",21,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["partial","*",21,0],"meta-externalagent":["partial","*",21,0],"cohere-ai":["partial","*",21,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",21,0]},"llms":{"status":"missing","http":404,"bytes":83778},"home":{"status":"ok","http":200,"url":"https://www.trustedshops.de/","html_tokens":64654,"markdown_tokens":252,"pruned_tokens":84}},
{"rank":7371,"domain":"negocio.site","origin":"https://negocio.site","robots":{"status":"missing","http":404,"url":"https://negocio.site/robots.txt","bytes":1571,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":1569},"home":{"status":"ok","http":200,"url":"https://business.google.com/us/business-profile/","html_tokens":75339,"markdown_tokens":66,"pruned_tokens":66}},
{"rank":7372,"domain":"registrar-transfers.com","origin":"https://registrar-transfers.com","robots":{"status":"missing","http":404,"url":"https://registrar-transfers.com/robots.txt","bytes":778,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":776},"home":{"status":"http_error","http":500,"url":"https://registrar-transfers.com/"}},
{"rank":7373,"domain":"utorrent.com","origin":"https://utorrent.com","robots":{"status":"ok","http":200,"url":"https://www.utorrent.com/robots.txt","bytes":455,"groups":8},"bots":{"GPTBot":["allowed","named",0,1],"ChatGPT-User":["partial","*",6,0],"OAI-SearchBot":["allowed","named",0,1],"ClaudeBot":["allowed","named",0,1],"Claude-User":["partial","*",6,0],"Claude-SearchBot":["partial","*",6,0],"anthropic-ai":["allowed","named",0,1],"PerplexityBot":["allowed","named",0,1],"Perplexity-User":["partial","*",6,0],"Google-Extended":["allowed","named",0,1],"Applebot-Extended":["partial","*",6,0],"CCBot":["partial","*",6,0],"Bytespider":["partial","*",6,0],"Amazonbot":["partial","*",6,0],"meta-externalagent":["partial","*",6,0],"cohere-ai":["partial","*",6,0],"Diffbot":["partial","*",6,0],"ContextFetchBot":["partial","*",6,0]},"llms":{"status":"present","http":200,"bytes":3515},"home":{"status":"ok","http":200,"url":"https://www.utorrent.com/","html_tokens":81538,"markdown_tokens":292,"pruned_tokens":109}},
{"rank":7374,"domain":"newsru.com","origin":"https://newsru.com","robots":{"status":"ok","http":200,"url":"https://www.newsru.com/robots.txt","bytes":557,"groups":3},"bots":{"GPTBot":["partial","*",7,0],"ChatGPT-User":["partial","*",7,0],"OAI-SearchBot":["partial","*",7,0],"ClaudeBot":["partial","*",7,0],"Claude-User":["partial","*",7,0],"Claude-SearchBot":["partial","*",7,0],"anthropic-ai":["partial","*",7,0],"PerplexityBot":["partial","*",7,0],"Perplexity-User":["partial","*",7,0],"Google-Extended":["partial","*",7,0],"Applebot-Extended":["partial","*",7,0],"CCBot":["partial","*",7,0],"Bytespider":["partial","*",7,0],"Amazonbot":["partial","*",7,0],"meta-externalagent":["partial","*",7,0],"cohere-ai":["partial","*",7,0],"Diffbot":["partial","*",7,0],"ContextFetchBot":["partial","*",7,0]},"llms":{"status":"missing","http":404,"bytes":42260},"home":{"status":"ok","http":200,"url":"https://www.newsru.com/","html_tokens":14690,"markdown_tokens":264,"pruned_tokens":100}},
{"rank":7375,"domain":"kmp.im","origin":"https://kmp.im","robots":{"status":"missing","http":404,"url":"https://kmp.im/robots.txt","bytes":34,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":34},"home":{"status":"ok","http":200,"url":"https://www.kompas.com","html_tokens":121042,"markdown_tokens":16,"pruned_tokens":16}},
{"rank":7376,"domain":"ntnu.edu","origin":"https://ntnu.edu","robots":{"status":"ok","http":200,"url":"https://www.ntnu.edu/robots.txt","bytes":3759,"groups":1},"bots":{"GPTBot":["partial","*",69,0],"ChatGPT-User":["partial","*",69,0],"OAI-SearchBot":["partial","*",69,0],"ClaudeBot":["partial","*",69,0],"Claude-User":["partial","*",69,0],"Claude-SearchBot":["partial","*",69,0],"anthropic-ai":["partial","*",69,0],"PerplexityBot":["partial","*",69,0],"Perplexity-User":["partial","*",69,0],"Google-Extended":["partial","*",69,0],"Applebot-Extended":["partial","*",69,0],"CCBot":["partial","*",69,0],"Bytespider":["partial","*",69,0],"Amazonbot":["partial","*",69,0],"meta-externalagent":["partial","*",69,0],"cohere-ai":["partial","*",69,0],"Diffbot":["partial","*",69,0],"ContextFetchBot":["partial","*",69,0]},"llms":{"status":"missing","http":404,"bytes":4461},"home":{"status":"ok","http":200,"url":"https://www.ntnu.edu/","html_tokens":29712,"markdown_tokens":41,"pruned_tokens":41}},
{"rank":7377,"domain":"b2clogin.com","origin":"https://www.b2clogin.com","robots":{"status":"missing","http":404,"url":"https://www.b2clogin.com/robots.txt","bytes":103,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":103},"home":{"status":"http_error","http":404,"url":"https://www.b2clogin.com/"}},
{"rank":7378,"domain":"fosdem.org","origin":"https://fosdem.org","robots":{"status":"missing","http":404,"url":"https://fosdem.org/2027/robots.txt","bytes":3268,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":3268},"home":{"status":"ok","http":200,"url":"https://fosdem.org/2027/","html_tokens":1063,"markdown_tokens":13,"pruned_tokens":13}},
{"rank":7379,"domain":"jrc.or.jp","origin":"https://jrc.or.jp","robots":{"status":"missing","http":404,"url":"https://www.jrc.or.jp/robots.txt","bytes":76712,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":76711},"home":{"status":"ok","http":200,"url":"https://www.jrc.or.jp/","html_tokens":23906,"markdown_tokens":34,"pruned_tokens":34}},
{"rank":7380,"domain":"usafacts.org","origin":"https://usafacts.org","robots":{"status":"ok","http":200,"url":"https://usafacts.org/robots.txt","bytes":735,"groups":2},"bots":{"GPTBot":["partial","named",5,1],"ChatGPT-User":["partial","named",5,1],"OAI-SearchBot":["partial","named",5,1],"ClaudeBot":["partial","named",5,1],"Claude-User":["partial","*",5,1],"Claude-SearchBot":["partial","*",5,1],"anthropic-ai":["partial","*",5,1],"PerplexityBot":["partial","named",5,1],"Perplexity-User":["partial","*",5,1],"Google-Extended":["partial","named",5,1],"Applebot-Extended":["partial","named",5,1],"CCBot":["partial","named",5,1],"Bytespider":["partial","named",5,1],"Amazonbot":["partial","*",5,1],"meta-externalagent":["partial","*",5,1],"cohere-ai":["partial","*",5,1],"Diffbot":["partial","named",5,1],"ContextFetchBot":["partial","*",5,1]},"llms":{"status":"missing","http":404,"bytes":100220},"home":{"status":"ok","http":200,"url":"https://usafacts.org/","html_tokens":389811,"markdown_tokens":157,"pruned_tokens":157}},
{"rank":7381,"domain":"ntnu.edu.tw","origin":"https://www.ntnu.edu.tw","robots":{"status":"missing","http":404,"url":"https://www.ntnu.edu.tw/robots.txt","bytes":196,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":196},"home":{"status":"ok","http":200,"url":"https://www.ntnu.edu.tw/","html_tokens":8748,"markdown_tokens":9,"pruned_tokens":9}},
{"rank":7382,"domain":"sizzling-hot-deluxe-slot.com","origin":"https://sizzling-hot-deluxe-slot.com","robots":{"status":"ok","http":200,"url":"https://sizzling-hot-deluxe-slot.com/robots.txt","bytes":624,"groups":1},"bots":{"GPTBot":["partial","*",19,0],"ChatGPT-User":["partial","*",19,0],"OAI-SearchBot":["partial","*",19,0],"ClaudeBot":["partial","*",19,0],"Claude-User":["partial","*",19,0],"Claude-SearchBot":["partial","*",19,0],"anthropic-ai":["partial","*",19,0],"PerplexityBot":["partial","*",19,0],"Perplexity-User":["partial","*",19,0],"Google-Extended":["partial","*",19,0],"Applebot-Extended":["partial","*",19,0],"CCBot":["partial","*",19,0],"Bytespider":["partial","*",19,0],"Amazonbot":["partial","*",19,0],"meta-externalagent":["partial","*",19,0],"cohere-ai":["partial","*",19,0],"Diffbot":["partial","*",19,0],"ContextFetchBot":["partial","*",19,0]},"llms":{"status":"missing","http":404,"bytes":25870},"home":{"status":"ok","http":200,"url":"https://sizzling-hot-deluxe-slot.com/","html_tokens":24940,"markdown_tokens":4896,"pruned_tokens":993}},
{"rank":7383,"domain":"mtv.fr","origin":"https://mtv.fr","robots":{"status":"html","http":200,"url":"https://www.youtube.com/channel/UCjMXe_Tsgi3JBiDslqnqhOQ","bytes":1924968,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":7384,"domain":"element.io","origin":"https://element.io","robots":{"status":"ok","http":200,"url":"https://element.io/robots.txt","bytes":267,"groups":1},"bots":{"GPTBot":["partial","*",4,1],"ChatGPT-User":["partial","*",4,1],"OAI-SearchBot":["partial","*",4,1],"ClaudeBot":["partial","*",4,1],"Claude-User":["partial","*",4,1],"Claude-SearchBot":["partial","*",4,1],"anthropic-ai":["partial","*",4,1],"PerplexityBot":["partial","*",4,1],"Perplexity-User":["partial","*",4,1],"Google-Extended":["partial","*",4,1],"Applebot-Extended":["partial","*",4,1],"CCBot":["partial","*",4,1],"Bytespider":["partial","*",4,1],"Amazonbot":["partial","*",4,1],"meta-externalagent":["partial","*",4,1],"cohere-ai":["partial","*",4,1],"Diffbot":["partial","*",4,1],"ContextFetchBot":["partial","*",4,1]},"llms":{"status":"missing","http":404,"bytes":3558},"home":{"status":"ok","http":200,"url":"https://element.io/en","html_tokens":80060,"markdown_tokens":841,"pruned_tokens":256}},
{"rank":7385,"domain":"justjared.com","origin":"https://justjared.com","robots":{"status":"ok","http":200,"url":"https://www.justjared.com/robots.txt","bytes":511,"groups":4},"bots":{"GPTBot":["partial","*",11,1],"ChatGPT-User":["partial","*",11,1],"OAI-SearchBot":["partial","*",11,1],"ClaudeBot":["partial","*",11,1],"Claude-User":["partial","*",11,1],"Claude-SearchBot":["partial","*",11,1],"anthropic-ai":["partial","*",11,1],"PerplexityBot":["partial","*",11,1],"Perplexity-User":["partial","*",11,1],"Google-Extended":["partial","*",11,1],"Applebot-Extended":["partial","*",11,1],"CCBot":["partial","*",11,1],"Bytespider":["partial","*",11,1],"Amazonbot":["partial","*",11,1],"meta-externalagent":["partial","*",11,1],"cohere-ai":["partial","*",11,1],"Diffbot":["partial","*",11,1],"ContextFetchBot":["partial","*",11,1]},"llms":{"status":"http_error","http":403,"bytes":5566},"home":{"status":"http_error","http":403,"url":"https://justjared.com/"}},
{"rank":7386,"domain":"uofmhealth.org","origin":"https://uofmhealth.org","robots":{"status":"ok","http":200,"url":"https://www.uofmhealth.org/robots.txt","bytes":1812,"groups":1},"bots":{"GPTBot":["partial","*",27,18],"ChatGPT-User":["partial","*",27,18],"OAI-SearchBot":["partial","*",27,18],"ClaudeBot":["partial","*",27,18],"Claude-User":["partial","*",27,18],"Claude-SearchBot":["partial","*",27,18],"anthropic-ai":["partial","*",27,18],"PerplexityBot":["partial","*",27,18],"Perplexity-User":["partial","*",27,18],"Google-Extended":["partial","*",27,18],"Applebot-Extended":["partial","*",27,18],"CCBot":["partial","*",27,18],"Bytespider":["partial","*",27,18],"Amazonbot":["partial","*",27,18],"meta-externalagent":["partial","*",27,18],"cohere-ai":["partial","*",27,18],"Diffbot":["partial","*",27,18],"ContextFetchBot":["partial","*",27,18]},"llms":{"status":"missing","http":404,"bytes":192},"home":{"status":"http_error","http":403,"url":"https://uofmhealth.org/"}},
{"rank":7387,"domain":"c-and-a.com","origin":"https://c-and-a.com","robots":{"status":"html","http":200,"url":"https://www.c-and-a.com/eu/en/shop?showCountrySelector=true","bytes":614937,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":7388,"domain":"thelocal.de","origin":"https://thelocal.de","robots":{"status":"ok","http":200,"url":"https://www.thelocal.de/robots.txt","bytes":1370,"groups":32},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["blocked","named",1,0],"OAI-SearchBot":["partial","*",6,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",6,0],"Claude-SearchBot":["partial","*",6,0],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["partial","*",6,0],"Google-Extended":["partial","*",6,0],"Applebot-Extended":["partial","*",6,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["partial","*",6,0],"cohere-ai":["partial","*",6,0],"Diffbot":["partial","*",6,0],"ContextFetchBot":["partial","*",6,0]},"llms":{"status":"missing","http":404,"bytes":113835},"home":{"status":"ok","http":200,"url":"https://www.thelocal.de/","html_tokens":62438,"markdown_tokens":51,"pruned_tokens":51}},
{"rank":7389,"domain":"gitbooks.io","origin":"https://gitbooks.io","robots":{"status":"error","http":null,"url":"https://gitbooks.io/robots.txt","bytes":0,"groups":0,"error":"'idna' codec can't encode character '\\x2e' in position 0: label empty"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":7390,"domain":"jusbrasil.com.br","origin":"https://jusbrasil.com.br","robots":{"status":"ok","http":200,"url":"https://www.jusbrasil.com.br/robots.txt","bytes":38827,"groups":6},"bots":{"GPTBot":["partial","named",104,233],"ChatGPT-User":["partial","named",106,236],"OAI-SearchBot":["partial","named",121,34],"ClaudeBot":["partial","named",104,233],"Claude-User":["partial","named",106,236],"Claude-SearchBot":["partial","named",121,34],"anthropic-ai":["blocked","*",1,0],"PerplexityBot":["partial","named",121,34],"Perplexity-User":["partial","named",106,236],"Google-Extended":["partial","named",104,233],"Applebot-Extended":["partial","named",104,233],"CCBot":["blocked","*",1,0],"Bytespider":["blocked","*",1,0],"Amazonbot":["partial","named",104,233],"meta-externalagent":["partial","named",104,233],"cohere-ai":["blocked","*",1,0],"Diffbot":["blocked","*",1,0],"ContextFetchBot":["blocked","*",1,0]},"llms":{"status":"disallowed"},"home":{"status":"disallowed"}},
{"rank":7391,"domain":"cognizant.com","origin":"https://cognizant.com","robots":{"status":"ok","http":200,"url":"https://www.cognizant.com/robots.txt","bytes":90,"groups":1},"bots":{"GPTBot":["partial","*",1,0],"ChatGPT-User":["partial","*",1,0],"OAI-SearchBot":["partial","*",1,0],"ClaudeBot":["partial","*",1,0],"Claude-User":["partial","*",1,0],"Claude-SearchBot":["partial","*",1,0],"anthropic-ai":["partial","*",1,0],"PerplexityBot":["partial","*",1,0],"Perplexity-User":["partial","*",1,0],"Google-Extended":["partial","*",1,0],"Applebot-Extended":["partial","*",1,0],"CCBot":["partial","*",1,0],"Bytespider":["partial","*",1,0],"Amazonbot":["partial","*",1,0],"meta-externalagent":["partial","*",1,0],"cohere-ai":["partial","*",1,0],"Diffbot":["partial","*",1,0],"ContextFetchBot":["partial","*",1,0]},"llms":{"status":"present","http":200,"bytes":88397},"home":{"status":"ok","http":200,"url":"https://www.cognizant.com/us/en","html_tokens":8439,"markdown_tokens":86,"pruned_tokens":86}},
{"rank":7392,"domain":"weightwatchers.com","origin":"https://weightwatchers.com","robots":{"status":"ok","http":200,"url":"https://www.weightwatchers.com/robots.txt","bytes":5740,"groups":1},"bots":{"GPTBot":["partial","*",152,0],"ChatGPT-User":["partial","*",152,0],"OAI-SearchBot":["partial","*",152,0],"ClaudeBot":["partial","*",152,0],"Claude-User":["partial","*",152,0],"Claude-SearchBot":["partial","*",152,0],"anthropic-ai":["partial","*",152,0],"PerplexityBot":["partial","*",152,0],"Perplexity-User":["partial","*",152,0],"Google-Extended":["partial","*",152,0],"Applebot-Extended":["partial","*",152,0],"CCBot":["partial","*",152,0],"Bytespider":["partial","*",152,0],"Amazonbot":["partial","*",152,0],"meta-externalagent":["partial","*",152,0],"cohere-ai":["partial","*",152,0],"Diffbot":["partial","*",152,0],"ContextFetchBot":["partial","*",152,0]},"llms":{"status":"present","http":200,"bytes":3096},"home":{"status":"timeout","error":"timed out after 15 s"}},
{"rank":7393,"domain":"core77.com","origin":"https://core77.com","robots":{"status":"ok","http":200,"url":"http://www.core77.com/robots.txt","bytes":1109,"groups":28},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["blocked","named",1,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":1612},"home":{"status":"ok","http":200,"url":"https://www.core77.com/","html_tokens":17597,"markdown_tokens":52,"pruned_tokens":52}},
{"rank":7394,"domain":"nopcommerce.com","origin":"https://nopcommerce.com","robots":{"status":"refused","http":403,"url":"https://www.nopcommerce.com/robots.txt","bytes":5578,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":7395,"domain":"penny-arcade.com","origin":"https://penny-arcade.com","robots":{"status":"ok","http":200,"url":"https://www.penny-arcade.com/robots.txt","bytes":55,"groups":2},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"missing","http":404,"bytes":50082},"home":{"status":"ok","http":200,"url":"https://www.penny-arcade.com/","html_tokens":16102,"markdown_tokens":338,"pruned_tokens":58}},
{"rank":7396,"domain":"algolia.com","origin":"https://algolia.com","robots":{"status":"refused","http":403,"url":"https://algolia.com/robots.txt","bytes":5514,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":7397,"domain":"raiplay.it","origin":"https://raiplay.it","robots":{"status":"ok","http":200,"url":"https://www.raiplay.it/robots.txt","bytes":2598,"groups":26},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["blocked","named",1,0],"OAI-SearchBot":["partial","*",23,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",23,0],"Claude-SearchBot":["partial","*",23,0],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["partial","*",23,0],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["partial","*",23,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["partial","*",23,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",23,0]},"llms":{"status":"missing","http":404,"bytes":56316},"home":{"status":"ok","http":200,"url":"https://www.raiplay.it/","html_tokens":364736,"markdown_tokens":36,"pruned_tokens":36}},
{"rank":7398,"domain":"baesystems.com","origin":"https://baesystems.com","robots":{"status":"ok","http":200,"url":"https://www.baesystems.com/robots.txt","bytes":698,"groups":1},"bots":{"GPTBot":["partial","*",25,0],"ChatGPT-User":["partial","*",25,0],"OAI-SearchBot":["partial","*",25,0],"ClaudeBot":["partial","*",25,0],"Claude-User":["partial","*",25,0],"Claude-SearchBot":["partial","*",25,0],"anthropic-ai":["partial","*",25,0],"PerplexityBot":["partial","*",25,0],"Perplexity-User":["partial","*",25,0],"Google-Extended":["partial","*",25,0],"Applebot-Extended":["partial","*",25,0],"CCBot":["partial","*",25,0],"Bytespider":["partial","*",25,0],"Amazonbot":["partial","*",25,0],"meta-externalagent":["partial","*",25,0],"cohere-ai":["partial","*",25,0],"Diffbot":["partial","*",25,0],"ContextFetchBot":["partial","*",25,0]},"llms":{"status":"html","http":200,"bytes":1042},"home":{"status":"ok","http":200,"url":"https://www.baesystems.com/","html_tokens":259,"markdown_tokens":21,"pruned_tokens":21}},
{"rank":7399,"domain":"edupage.org","origin":"https://edupage.org","robots":{"status":"ok","http":200,"url":"https://www.edupage.org/robots.txt","bytes":1006,"groups":1},"bots":{"GPTBot":["partial","*",33,0],"ChatGPT-User":["partial","*",33,0],"OAI-SearchBot":["partial","*",33,0],"ClaudeBot":["partial","*",33,0],"Claude-User":["partial","*",33,0],"Claude-SearchBot":["partial","*",33,0],"anthropic-ai":["partial","*",33,0],"PerplexityBot":["partial","*",33,0],"Perplexity-User":["partial","*",33,0],"Google-Extended":["partial","*",33,0],"Applebot-Extended":["partial","*",33,0],"CCBot":["partial","*",33,0],"Bytespider":["partial","*",33,0],"Amazonbot":["partial","*",33,0],"meta-externalagent":["partial","*",33,0],"cohere-ai":["partial","*",33,0],"Diffbot":["partial","*",33,0],"ContextFetchBot":["partial","*",33,0]},"llms":{"status":"html","http":200,"bytes":0},"home":{"status":"ok","http":200,"url":"https://www.edupage.org/","html_tokens":72315,"markdown_tokens":5811,"pruned_tokens":143}},
{"rank":7400,"domain":"ip138.com","origin":"https://ip138.com","robots":{"status":"ok","http":200,"url":"https://ip138.com/robots.txt","bytes":23,"groups":1},"bots":{"GPTBot":["allowed","*",0,1],"ChatGPT-User":["allowed","*",0,1],"OAI-SearchBot":["allowed","*",0,1],"ClaudeBot":["allowed","*",0,1],"Claude-User":["allowed","*",0,1],"Claude-SearchBot":["allowed","*",0,1],"anthropic-ai":["allowed","*",0,1],"PerplexityBot":["allowed","*",0,1],"Perplexity-User":["allowed","*",0,1],"Google-Extended":["allowed","*",0,1],"Applebot-Extended":["allowed","*",0,1],"CCBot":["allowed","*",0,1],"Bytespider":["allowed","*",0,1],"Amazonbot":["allowed","*",0,1],"meta-externalagent":["allowed","*",0,1],"cohere-ai":["allowed","*",0,1],"Diffbot":["allowed","*",0,1],"ContextFetchBot":["allowed","*",0,1]},"llms":{"status":"missing","http":404,"bytes":2995},"home":{"status":"ok","http":200,"url":"https://ip138.com/","html_tokens":4344,"markdown_tokens":1,"pruned_tokens":1}},
{"rank":7401,"domain":"verdecasinoseite.com","origin":"https://www.verdecasinoseite.com","robots":{"status":"ok","http":200,"url":"https://verdecasinoseite.com/robots.txt","bytes":67,"groups":1},"bots":{"GPTBot":["partial","*",1,1],"ChatGPT-User":["partial","*",1,1],"OAI-SearchBot":["partial","*",1,1],"ClaudeBot":["partial","*",1,1],"Claude-User":["partial","*",1,1],"Claude-SearchBot":["partial","*",1,1],"anthropic-ai":["partial","*",1,1],"PerplexityBot":["partial","*",1,1],"Perplexity-User":["partial","*",1,1],"Google-Extended":["partial","*",1,1],"Applebot-Extended":["partial","*",1,1],"CCBot":["partial","*",1,1],"Bytespider":["partial","*",1,1],"Amazonbot":["partial","*",1,1],"meta-externalagent":["partial","*",1,1],"cohere-ai":["partial","*",1,1],"Diffbot":["partial","*",1,1],"ContextFetchBot":["partial","*",1,1]},"llms":{"status":"disallowed","error":"robots.txt of verdecasinoseite.com not readable (timeout)"},"home":{"status":"disallowed","error":"robots.txt of verdecasinoseite.com not readable (timeout)"}},
{"rank":7402,"domain":"cybersecurityventures.com","origin":"https://cybersecurityventures.com","robots":{"status":"ok","http":200,"url":"https://cybersecurityventures.com/robots.txt","bytes":251,"groups":1},"bots":{"GPTBot":["partial","*",1,1],"ChatGPT-User":["partial","*",1,1],"OAI-SearchBot":["partial","*",1,1],"ClaudeBot":["partial","*",1,1],"Claude-User":["partial","*",1,1],"Claude-SearchBot":["partial","*",1,1],"anthropic-ai":["partial","*",1,1],"PerplexityBot":["partial","*",1,1],"Perplexity-User":["partial","*",1,1],"Google-Extended":["partial","*",1,1],"Applebot-Extended":["partial","*",1,1],"CCBot":["partial","*",1,1],"Bytespider":["partial","*",1,1],"Amazonbot":["partial","*",1,1],"meta-externalagent":["partial","*",1,1],"cohere-ai":["partial","*",1,1],"Diffbot":["partial","*",1,1],"ContextFetchBot":["partial","*",1,1]},"llms":{"status":"http_error","http":403,"bytes":5600},"home":{"status":"http_error","http":403,"url":"https://cybersecurityventures.com/"}},
{"rank":7403,"domain":"hisense.com","origin":"https://hisense.com","robots":{"status":"ok","http":200,"url":"https://www.hisense.com/robots.txt","bytes":302,"groups":2},"bots":{"GPTBot":["allowed","*",0,1],"ChatGPT-User":["allowed","*",0,1],"OAI-SearchBot":["allowed","*",0,1],"ClaudeBot":["allowed","*",0,1],"Claude-User":["allowed","*",0,1],"Claude-SearchBot":["allowed","*",0,1],"anthropic-ai":["allowed","*",0,1],"PerplexityBot":["allowed","*",0,1],"Perplexity-User":["allowed","*",0,1],"Google-Extended":["allowed","*",0,1],"Applebot-Extended":["allowed","*",0,1],"CCBot":["allowed","*",0,1],"Bytespider":["allowed","*",0,1],"Amazonbot":["allowed","*",0,1],"meta-externalagent":["allowed","*",0,1],"cohere-ai":["allowed","*",0,1],"Diffbot":["allowed","*",0,1],"ContextFetchBot":["allowed","*",0,1]},"llms":{"status":"html","http":200,"bytes":2429345},"home":{"status":"ok","http":200,"url":"https://www.hisense-usa.com/","html_tokens":607278,"markdown_tokens":24,"pruned_tokens":24}},
{"rank":7404,"domain":"dailysignal.com","origin":"https://dailysignal.com","robots":{"status":"ok","http":200,"url":"https://www.dailysignal.com/robots.txt","bytes":157,"groups":1},"bots":{"GPTBot":["partial","*",2,1],"ChatGPT-User":["partial","*",2,1],"OAI-SearchBot":["partial","*",2,1],"ClaudeBot":["partial","*",2,1],"Claude-User":["partial","*",2,1],"Claude-SearchBot":["partial","*",2,1],"anthropic-ai":["partial","*",2,1],"PerplexityBot":["partial","*",2,1],"Perplexity-User":["partial","*",2,1],"Google-Extended":["partial","*",2,1],"Applebot-Extended":["partial","*",2,1],"CCBot":["partial","*",2,1],"Bytespider":["partial","*",2,1],"Amazonbot":["partial","*",2,1],"meta-externalagent":["partial","*",2,1],"cohere-ai":["partial","*",2,1],"Diffbot":["partial","*",2,1],"ContextFetchBot":["partial","*",2,1]},"llms":{"status":"missing","http":404,"bytes":186303},"home":{"status":"http_error","http":403,"url":"https://dailysignal.com/"}},
{"rank":7405,"domain":"westlaw.com","origin":"https://westlaw.com","robots":{"status":"refused","http":403,"url":"https://www.westlaw.com/robots.txt","bytes":4564,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":7406,"domain":"statuspage.io","origin":"https://statuspage.io","robots":{"status":"html","http":200,"url":"https://www.atlassian.com/software/statuspage","bytes":1081124,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":7407,"domain":"um.dk","origin":"https://um.dk","robots":{"status":"ok","http":200,"url":"https://um.dk/robots.txt","bytes":357,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":0},"home":{"status":"ok","http":200,"url":"https://um.dk/","html_tokens":24467,"markdown_tokens":63,"pruned_tokens":63}},
{"rank":7408,"domain":"reference-global.com","origin":"https://reference-global.com","robots":{"status":"ok","http":200,"url":"https://reference-global.com/robots.txt","bytes":387,"groups":6},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["partial","*",9,0],"OAI-SearchBot":["partial","*",9,0],"ClaudeBot":["partial","*",9,0],"Claude-User":["partial","*",9,0],"Claude-SearchBot":["partial","*",9,0],"anthropic-ai":["partial","*",9,0],"PerplexityBot":["partial","*",9,0],"Perplexity-User":["partial","*",9,0],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["partial","*",9,0],"CCBot":["blocked","named",1,0],"Bytespider":["partial","*",9,0],"Amazonbot":["partial","*",9,0],"meta-externalagent":["partial","*",9,0],"cohere-ai":["partial","*",9,0],"Diffbot":["partial","*",9,0],"ContextFetchBot":["partial","*",9,0]},"llms":{"status":"missing","http":404,"bytes":205315},"home":{"status":"ok","http":200,"url":"https://reference-global.com/","html_tokens":127100,"markdown_tokens":56,"pruned_tokens":56}},
{"rank":7409,"domain":"abbott.com","origin":"https://abbott.com","robots":{"status":"ok","http":200,"url":"https://www.abbott.com/content/dam/corp/abbottglobal/wwwroot/robots.txt","bytes":2508,"groups":2},"bots":{"GPTBot":["partial","*",36,0],"ChatGPT-User":["partial","*",36,0],"OAI-SearchBot":["partial","*",36,0],"ClaudeBot":["partial","*",36,0],"Claude-User":["partial","*",36,0],"Claude-SearchBot":["partial","*",36,0],"anthropic-ai":["partial","*",36,0],"PerplexityBot":["partial","*",36,0],"Perplexity-User":["partial","*",36,0],"Google-Extended":["partial","*",36,0],"Applebot-Extended":["partial","*",36,0],"CCBot":["partial","*",36,0],"Bytespider":["partial","*",36,0],"Amazonbot":["partial","*",36,0],"meta-externalagent":["partial","*",36,0],"cohere-ai":["partial","*",36,0],"Diffbot":["partial","*",36,0],"ContextFetchBot":["partial","*",36,0]},"llms":{"status":"missing","http":404,"bytes":69917},"home":{"status":"ok","http":200,"url":"https://www.abbott.com/en-us/homepage","html_tokens":21217,"markdown_tokens":117,"pruned_tokens":117}},
{"rank":7410,"domain":"yonhapnews.co.kr","origin":"http://yonhapnews.co.kr","robots":{"status":"html","http":200,"url":"https://www.yna.co.kr","bytes":188264,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":7411,"domain":"doxygen.nl","origin":"https://doxygen.nl","robots":{"status":"ok","http":200,"url":"https://doxygen.nl/robots.txt","bytes":1248,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":1618},"home":{"status":"ok","http":200,"url":"https://doxygen.nl/","html_tokens":3690,"markdown_tokens":533,"pruned_tokens":353}},
{"rank":7412,"domain":"google.co.ls","origin":"https://google.co.ls","robots":{"status":"ok","http":200,"url":"https://www.google.co.ls/robots.txt","bytes":6624,"groups":4},"bots":{"GPTBot":["partial","*",169,64],"ChatGPT-User":["partial","*",169,64],"OAI-SearchBot":["partial","*",169,64],"ClaudeBot":["partial","*",169,64],"Claude-User":["partial","*",169,64],"Claude-SearchBot":["partial","*",169,64],"anthropic-ai":["partial","*",169,64],"PerplexityBot":["partial","*",169,64],"Perplexity-User":["partial","*",169,64],"Google-Extended":["partial","*",169,64],"Applebot-Extended":["partial","*",169,64],"CCBot":["partial","*",169,64],"Bytespider":["partial","*",169,64],"Amazonbot":["partial","*",169,64],"meta-externalagent":["partial","*",169,64],"cohere-ai":["partial","*",169,64],"Diffbot":["partial","*",169,64],"ContextFetchBot":["partial","*",169,64]},"llms":{"status":"missing","http":404,"bytes":1569},"home":{"status":"ok","http":200,"url":"https://www.google.com/","html_tokens":21261,"markdown_tokens":30,"pruned_tokens":30}},
{"rank":7413,"domain":"usmint.gov","origin":"https://usmint.gov","robots":{"status":"refused","http":403,"url":"https://usmint.gov/robots.txt","bytes":5505,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":7414,"domain":"uantwerpen.be","origin":"https://uantwerpen.be","robots":{"status":"ok","http":200,"url":"https://www.uantwerpen.be/robots.txt","bytes":24,"groups":1},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"missing","http":404,"bytes":9065},"home":{"status":"ok","http":200,"url":"https://www.uantwerpen.be/nl/","html_tokens":9025,"markdown_tokens":522,"pruned_tokens":124}},
{"rank":7415,"domain":"schleswig-holstein.de","origin":"https://schleswig-holstein.de","robots":{"status":"ok","http":200,"url":"https://www.schleswig-holstein.de/robots.txt","bytes":4255,"groups":2},"bots":{"GPTBot":["partial","*",50,5],"ChatGPT-User":["partial","*",50,5],"OAI-SearchBot":["partial","*",50,5],"ClaudeBot":["partial","*",50,5],"Claude-User":["partial","*",50,5],"Claude-SearchBot":["partial","*",50,5],"anthropic-ai":["partial","*",50,5],"PerplexityBot":["partial","*",50,5],"Perplexity-User":["partial","*",50,5],"Google-Extended":["partial","*",50,5],"Applebot-Extended":["partial","*",50,5],"CCBot":["partial","*",50,5],"Bytespider":["partial","*",50,5],"Amazonbot":["partial","*",50,5],"meta-externalagent":["partial","*",50,5],"cohere-ai":["partial","*",50,5],"Diffbot":["partial","*",50,5],"ContextFetchBot":["partial","*",50,5]},"llms":{"status":"crawl_delay","error":"Crawl-delay 180 s"},"home":{"status":"crawl_delay","error":"Crawl-delay 180 s"}},
{"rank":7416,"domain":"time.is","origin":"https://time.is","robots":{"status":"ok","http":200,"url":"https://time.is/robots.txt","bytes":1066,"groups":2},"bots":{"GPTBot":["partial","*",20,1],"ChatGPT-User":["partial","*",20,1],"OAI-SearchBot":["partial","*",20,1],"ClaudeBot":["partial","*",20,1],"Claude-User":["partial","*",20,1],"Claude-SearchBot":["partial","*",20,1],"anthropic-ai":["partial","*",20,1],"PerplexityBot":["partial","*",20,1],"Perplexity-User":["partial","*",20,1],"Google-Extended":["partial","*",20,1],"Applebot-Extended":["partial","*",20,1],"CCBot":["partial","*",20,1],"Bytespider":["partial","*",20,1],"Amazonbot":["partial","*",20,1],"meta-externalagent":["partial","*",20,1],"cohere-ai":["partial","*",20,1],"Diffbot":["partial","*",20,1],"ContextFetchBot":["partial","*",20,1]},"llms":{"status":"missing","http":404,"bytes":507},"home":{"status":"ok","http":200,"url":"https://time.is/","html_tokens":22868,"markdown_tokens":35,"pruned_tokens":35}},
{"rank":7417,"domain":"cheatsheet.com","origin":"https://cheatsheet.com","robots":{"status":"ok","http":200,"url":"https://www.cheatsheet.com/robots.txt","bytes":866,"groups":3},"bots":{"GPTBot":["partial","*",17,4],"ChatGPT-User":["partial","*",17,4],"OAI-SearchBot":["partial","*",17,4],"ClaudeBot":["partial","*",17,4],"Claude-User":["partial","*",17,4],"Claude-SearchBot":["partial","*",17,4],"anthropic-ai":["partial","*",17,4],"PerplexityBot":["partial","*",17,4],"Perplexity-User":["partial","*",17,4],"Google-Extended":["partial","*",17,4],"Applebot-Extended":["partial","*",17,4],"CCBot":["partial","*",17,4],"Bytespider":["partial","*",17,4],"Amazonbot":["partial","*",17,4],"meta-externalagent":["partial","*",17,4],"cohere-ai":["partial","*",17,4],"Diffbot":["partial","*",17,4],"ContextFetchBot":["partial","*",17,4]},"llms":{"status":"missing","http":404,"bytes":144799},"home":{"status":"ok","http":200,"url":"https://www.cheatsheet.com/","html_tokens":44980,"markdown_tokens":89,"pruned_tokens":89}},
{"rank":7418,"domain":"kingcounty.gov","origin":"https://kingcounty.gov","robots":{"status":"ok","http":200,"url":"https://kingcounty.gov/robots.txt","bytes":828,"groups":1},"bots":{"GPTBot":["partial","*",23,0],"ChatGPT-User":["partial","*",23,0],"OAI-SearchBot":["partial","*",23,0],"ClaudeBot":["partial","*",23,0],"Claude-User":["partial","*",23,0],"Claude-SearchBot":["partial","*",23,0],"anthropic-ai":["partial","*",23,0],"PerplexityBot":["partial","*",23,0],"Perplexity-User":["partial","*",23,0],"Google-Extended":["partial","*",23,0],"Applebot-Extended":["partial","*",23,0],"CCBot":["partial","*",23,0],"Bytespider":["partial","*",23,0],"Amazonbot":["partial","*",23,0],"meta-externalagent":["partial","*",23,0],"cohere-ai":["partial","*",23,0],"Diffbot":["partial","*",23,0],"ContextFetchBot":["partial","*",23,0]},"llms":{"status":"error","error":"bad compressed body: Error -3 while decompressing data: incorrect header check"},"home":{"status":"error","error":"bad compressed body: Error -3 while decompressing data: incorrect header check"}},
{"rank":7419,"domain":"macmillan.org.uk","origin":"https://macmillan.org.uk","robots":{"status":"ok","http":200,"url":"https://www.macmillan.org.uk/robots.txt","bytes":2631,"groups":1},"bots":{"GPTBot":["partial","*",31,1],"ChatGPT-User":["partial","*",31,1],"OAI-SearchBot":["partial","*",31,1],"ClaudeBot":["partial","*",31,1],"Claude-User":["partial","*",31,1],"Claude-SearchBot":["partial","*",31,1],"anthropic-ai":["partial","*",31,1],"PerplexityBot":["partial","*",31,1],"Perplexity-User":["partial","*",31,1],"Google-Extended":["partial","*",31,1],"Applebot-Extended":["partial","*",31,1],"CCBot":["partial","*",31,1],"Bytespider":["partial","*",31,1],"Amazonbot":["partial","*",31,1],"meta-externalagent":["partial","*",31,1],"cohere-ai":["partial","*",31,1],"Diffbot":["partial","*",31,1],"ContextFetchBot":["partial","*",31,1]},"llms":{"status":"missing","http":404,"bytes":289256},"home":{"status":"ok","http":200,"url":"https://www.macmillan.org.uk/","html_tokens":84605,"markdown_tokens":194,"pruned_tokens":58}},
{"rank":7420,"domain":"riksdagen.se","origin":"https://riksdagen.se","robots":{"status":"ok","http":200,"url":"https://www.riksdagen.se/robots.txt","bytes":175,"groups":1},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","named",0,0],"Claude-User":["allowed","named",0,0],"Claude-SearchBot":["allowed","named",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":206329},"home":{"status":"ok","http":200,"url":"https://www.riksdagen.se/sv/","html_tokens":85917,"markdown_tokens":443,"pruned_tokens":155}},
{"rank":7421,"domain":"bg.ac.rs","origin":"https://bg.ac.rs","robots":{"status":"ok","http":200,"url":"https://www.bg.ac.rs/robots.txt","bytes":113,"groups":1},"bots":{"GPTBot":["partial","*",1,1],"ChatGPT-User":["partial","*",1,1],"OAI-SearchBot":["partial","*",1,1],"ClaudeBot":["partial","*",1,1],"Claude-User":["partial","*",1,1],"Claude-SearchBot":["partial","*",1,1],"anthropic-ai":["partial","*",1,1],"PerplexityBot":["partial","*",1,1],"Perplexity-User":["partial","*",1,1],"Google-Extended":["partial","*",1,1],"Applebot-Extended":["partial","*",1,1],"CCBot":["partial","*",1,1],"Bytespider":["partial","*",1,1],"Amazonbot":["partial","*",1,1],"meta-externalagent":["partial","*",1,1],"cohere-ai":["partial","*",1,1],"Diffbot":["partial","*",1,1],"ContextFetchBot":["partial","*",1,1]},"llms":{"status":"missing","http":404,"bytes":57648},"home":{"status":"ok","http":200,"url":"https://www.bg.ac.rs/","html_tokens":50603,"markdown_tokens":224,"pruned_tokens":68}},
{"rank":7422,"domain":"zocdoc.com","origin":"https://zocdoc.com","robots":{"status":"refused","http":403,"url":"https://www.zocdoc.com/robots.txt","bytes":778,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":7423,"domain":"scworld.com","origin":"https://scworld.com","robots":{"status":"ok","http":200,"url":"https://www.scworld.com/robots.txt","bytes":434,"groups":1},"bots":{"GPTBot":["partial","*",10,1],"ChatGPT-User":["partial","*",10,1],"OAI-SearchBot":["partial","*",10,1],"ClaudeBot":["partial","*",10,1],"Claude-User":["partial","*",10,1],"Claude-SearchBot":["partial","*",10,1],"anthropic-ai":["partial","*",10,1],"PerplexityBot":["partial","*",10,1],"Perplexity-User":["partial","*",10,1],"Google-Extended":["partial","*",10,1],"Applebot-Extended":["partial","*",10,1],"CCBot":["partial","*",10,1],"Bytespider":["partial","*",10,1],"Amazonbot":["partial","*",10,1],"meta-externalagent":["partial","*",10,1],"cohere-ai":["partial","*",10,1],"Diffbot":["partial","*",10,1],"ContextFetchBot":["partial","*",10,1]},"llms":{"status":"missing","http":404,"bytes":23615},"home":{"status":"http_error","http":403,"url":"https://scworld.com/"}},
{"rank":7424,"domain":"plymouth.ac.uk","origin":"https://plymouth.ac.uk","robots":{"status":"ok","http":200,"url":"https://www.plymouth.ac.uk/robots.txt","bytes":515,"groups":1},"bots":{"GPTBot":["partial","*",10,0],"ChatGPT-User":["partial","*",10,0],"OAI-SearchBot":["partial","*",10,0],"ClaudeBot":["partial","*",10,0],"Claude-User":["partial","*",10,0],"Claude-SearchBot":["partial","*",10,0],"anthropic-ai":["partial","*",10,0],"PerplexityBot":["partial","*",10,0],"Perplexity-User":["partial","*",10,0],"Google-Extended":["partial","*",10,0],"Applebot-Extended":["partial","*",10,0],"CCBot":["partial","*",10,0],"Bytespider":["partial","*",10,0],"Amazonbot":["partial","*",10,0],"meta-externalagent":["partial","*",10,0],"cohere-ai":["partial","*",10,0],"Diffbot":["partial","*",10,0],"ContextFetchBot":["partial","*",10,0]},"llms":{"status":"missing","http":404,"bytes":0},"home":{"status":"ok","http":200,"url":"https://www.plymouth.ac.uk/","html_tokens":18554,"markdown_tokens":208,"pruned_tokens":138}},
{"rank":7425,"domain":"oauth.net","origin":"https://oauth.net","robots":{"status":"ok","http":200,"url":"https://oauth.net/robots.txt","bytes":71,"groups":1},"bots":{"GPTBot":["partial","*",1,0],"ChatGPT-User":["partial","*",1,0],"OAI-SearchBot":["partial","*",1,0],"ClaudeBot":["partial","*",1,0],"Claude-User":["partial","*",1,0],"Claude-SearchBot":["partial","*",1,0],"anthropic-ai":["partial","*",1,0],"PerplexityBot":["partial","*",1,0],"Perplexity-User":["partial","*",1,0],"Google-Extended":["partial","*",1,0],"Applebot-Extended":["partial","*",1,0],"CCBot":["partial","*",1,0],"Bytespider":["partial","*",1,0],"Amazonbot":["partial","*",1,0],"meta-externalagent":["partial","*",1,0],"cohere-ai":["partial","*",1,0],"Diffbot":["partial","*",1,0],"ContextFetchBot":["partial","*",1,0]},"llms":{"status":"missing","http":404,"bytes":536},"home":{"status":"ok","http":200,"url":"https://oauth.net/","html_tokens":4944,"markdown_tokens":23,"pruned_tokens":23}},
{"rank":7426,"domain":"bio.link","origin":"https://bio.link","robots":{"status":"ok","http":200,"url":"https://bio.link/robots.txt","bytes":54,"groups":1},"bots":{"GPTBot":["partial","*",1,2],"ChatGPT-User":["partial","*",1,2],"OAI-SearchBot":["partial","*",1,2],"ClaudeBot":["partial","*",1,2],"Claude-User":["partial","*",1,2],"Claude-SearchBot":["partial","*",1,2],"anthropic-ai":["partial","*",1,2],"PerplexityBot":["partial","*",1,2],"Perplexity-User":["partial","*",1,2],"Google-Extended":["partial","*",1,2],"Applebot-Extended":["partial","*",1,2],"CCBot":["partial","*",1,2],"Bytespider":["partial","*",1,2],"Amazonbot":["partial","*",1,2],"meta-externalagent":["partial","*",1,2],"cohere-ai":["partial","*",1,2],"Diffbot":["partial","*",1,2],"ContextFetchBot":["partial","*",1,2]},"llms":{"status":"missing","http":404,"bytes":199124},"home":{"status":"http_error","http":403,"url":"https://bio.link/"}},
{"rank":7427,"domain":"berklee.edu","origin":"https://berklee.edu","robots":{"status":"ok","http":200,"url":"https://www.berklee.edu/robots.txt","bytes":3256,"groups":1},"bots":{"GPTBot":["partial","*",86,21],"ChatGPT-User":["partial","*",86,21],"OAI-SearchBot":["partial","*",86,21],"ClaudeBot":["partial","*",86,21],"Claude-User":["partial","*",86,21],"Claude-SearchBot":["partial","*",86,21],"anthropic-ai":["partial","*",86,21],"PerplexityBot":["partial","*",86,21],"Perplexity-User":["partial","*",86,21],"Google-Extended":["partial","*",86,21],"Applebot-Extended":["partial","*",86,21],"CCBot":["partial","*",86,21],"Bytespider":["partial","*",86,21],"Amazonbot":["partial","*",86,21],"meta-externalagent":["partial","*",86,21],"cohere-ai":["partial","*",86,21],"Diffbot":["partial","*",86,21],"ContextFetchBot":["partial","*",86,21]},"llms":{"status":"present","http":200,"bytes":7480},"home":{"status":"ok","http":200,"url":"https://www.berklee.edu/","html_tokens":23391,"markdown_tokens":354,"pruned_tokens":313}},
{"rank":7428,"domain":"google.hn","origin":"https://google.hn","robots":{"status":"ok","http":200,"url":"https://www.google.hn/robots.txt","bytes":6624,"groups":4},"bots":{"GPTBot":["partial","*",169,64],"ChatGPT-User":["partial","*",169,64],"OAI-SearchBot":["partial","*",169,64],"ClaudeBot":["partial","*",169,64],"Claude-User":["partial","*",169,64],"Claude-SearchBot":["partial","*",169,64],"anthropic-ai":["partial","*",169,64],"PerplexityBot":["partial","*",169,64],"Perplexity-User":["partial","*",169,64],"Google-Extended":["partial","*",169,64],"Applebot-Extended":["partial","*",169,64],"CCBot":["partial","*",169,64],"Bytespider":["partial","*",169,64],"Amazonbot":["partial","*",169,64],"meta-externalagent":["partial","*",169,64],"cohere-ai":["partial","*",169,64],"Diffbot":["partial","*",169,64],"ContextFetchBot":["partial","*",169,64]},"llms":{"status":"missing","http":404,"bytes":1569},"home":{"status":"ok","http":200,"url":"https://www.google.com/","html_tokens":21270,"markdown_tokens":30,"pruned_tokens":30}},
{"rank":7429,"domain":"joinpeertube.org","origin":"https://joinpeertube.org","robots":{"status":"missing","http":404,"url":"https://joinpeertube.org/robots.txt","bytes":146,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":36739},"home":{"status":"ok","http":200,"url":"https://joinpeertube.org/","html_tokens":27060,"markdown_tokens":1417,"pruned_tokens":171}},
{"rank":7430,"domain":"ripple.com","origin":"https://ripple.com","robots":{"status":"ok","http":200,"url":"https://ripple.com/robots.txt","bytes":173,"groups":1},"bots":{"GPTBot":["partial","*",3,1],"ChatGPT-User":["partial","*",3,1],"OAI-SearchBot":["partial","*",3,1],"ClaudeBot":["partial","*",3,1],"Claude-User":["partial","*",3,1],"Claude-SearchBot":["partial","*",3,1],"anthropic-ai":["partial","*",3,1],"PerplexityBot":["partial","*",3,1],"Perplexity-User":["partial","*",3,1],"Google-Extended":["partial","*",3,1],"Applebot-Extended":["partial","*",3,1],"CCBot":["partial","*",3,1],"Bytespider":["partial","*",3,1],"Amazonbot":["partial","*",3,1],"meta-externalagent":["partial","*",3,1],"cohere-ai":["partial","*",3,1],"Diffbot":["partial","*",3,1],"ContextFetchBot":["partial","*",3,1]},"llms":{"status":"missing","http":404,"bytes":48645},"home":{"status":"ok","http":200,"url":"https://ripple.com/","html_tokens":44390,"markdown_tokens":680,"pruned_tokens":397}},
{"rank":7431,"domain":"memrise.com","origin":"https://memrise.com","robots":{"status":"ok","http":200,"url":"https://www.memrise.com/robots.txt","bytes":517,"groups":1},"bots":{"GPTBot":["partial","*",17,0],"ChatGPT-User":["partial","*",17,0],"OAI-SearchBot":["partial","*",17,0],"ClaudeBot":["partial","*",17,0],"Claude-User":["partial","*",17,0],"Claude-SearchBot":["partial","*",17,0],"anthropic-ai":["partial","*",17,0],"PerplexityBot":["partial","*",17,0],"Perplexity-User":["partial","*",17,0],"Google-Extended":["partial","*",17,0],"Applebot-Extended":["partial","*",17,0],"CCBot":["partial","*",17,0],"Bytespider":["partial","*",17,0],"Amazonbot":["partial","*",17,0],"meta-externalagent":["partial","*",17,0],"cohere-ai":["partial","*",17,0],"Diffbot":["partial","*",17,0],"ContextFetchBot":["partial","*",17,0]},"llms":{"status":"present","http":200,"bytes":7094},"home":{"status":"ok","http":200,"url":"https://www.memrise.com/","html_tokens":49841,"markdown_tokens":706,"pruned_tokens":430}},
{"rank":7432,"domain":"nuernberg.de","origin":"https://nuernberg.de","robots":{"status":"ok","http":200,"url":"https://www.nuernberg.de/robots.txt","bytes":2261,"groups":2},"bots":{"GPTBot":["partial","*",40,0],"ChatGPT-User":["partial","*",40,0],"OAI-SearchBot":["partial","*",40,0],"ClaudeBot":["partial","*",40,0],"Claude-User":["partial","*",40,0],"Claude-SearchBot":["partial","*",40,0],"anthropic-ai":["partial","*",40,0],"PerplexityBot":["partial","*",40,0],"Perplexity-User":["partial","*",40,0],"Google-Extended":["partial","*",40,0],"Applebot-Extended":["partial","*",40,0],"CCBot":["partial","*",40,0],"Bytespider":["partial","*",40,0],"Amazonbot":["partial","*",40,0],"meta-externalagent":["partial","*",40,0],"cohere-ai":["partial","*",40,0],"Diffbot":["partial","*",40,0],"ContextFetchBot":["partial","*",40,0]},"llms":{"status":"missing","http":404,"bytes":82658},"home":{"status":"ok","http":200,"url":"https://www.nuernberg.de/internet/stadtportal/index.html","html_tokens":38142,"markdown_tokens":1375,"pruned_tokens":150}},
{"rank":7433,"domain":"americanthinker.com","origin":"https://americanthinker.com","robots":{"status":"missing","http":404,"url":"https://www.americanthinker.com/robots.txt","bytes":153,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":153},"home":{"status":"ok","http":200,"url":"https://www.americanthinker.com/","html_tokens":22665,"markdown_tokens":240,"pruned_tokens":165}},
{"rank":7434,"domain":"mayo.edu","origin":"https://mayo.edu","robots":{"status":"refused","http":403,"url":"https://www.mayo.edu/robots.txt","bytes":378,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":7435,"domain":"eventscloud.com","origin":"https://eventscloud.com","robots":{"status":"ok","http":200,"url":"https://eventscloud.com/robots.txt","bytes":22,"groups":1},"bots":{"GPTBot":["allowed","*",0,1],"ChatGPT-User":["allowed","*",0,1],"OAI-SearchBot":["allowed","*",0,1],"ClaudeBot":["allowed","*",0,1],"Claude-User":["allowed","*",0,1],"Claude-SearchBot":["allowed","*",0,1],"anthropic-ai":["allowed","*",0,1],"PerplexityBot":["allowed","*",0,1],"Perplexity-User":["allowed","*",0,1],"Google-Extended":["allowed","*",0,1],"Applebot-Extended":["allowed","*",0,1],"CCBot":["allowed","*",0,1],"Bytespider":["allowed","*",0,1],"Amazonbot":["allowed","*",0,1],"meta-externalagent":["allowed","*",0,1],"cohere-ai":["allowed","*",0,1],"Diffbot":["allowed","*",0,1],"ContextFetchBot":["allowed","*",0,1]},"llms":{"status":"present","http":200,"bytes":3503},"home":{"status":"ok","http":200,"url":"https://eventscloud.com/login.php","html_tokens":1033,"markdown_tokens":4,"pruned_tokens":4}},
{"rank":7436,"domain":"bertelsmann-stiftung.de","origin":"https://bertelsmann-stiftung.de","robots":{"status":"ok","http":200,"url":"https://www.bertelsmann-stiftung.de/de/robots.txt","bytes":519,"groups":2},"bots":{"GPTBot":["partial","*",7,0],"ChatGPT-User":["partial","*",7,0],"OAI-SearchBot":["partial","*",7,0],"ClaudeBot":["partial","*",7,0],"Claude-User":["partial","*",7,0],"Claude-SearchBot":["partial","*",7,0],"anthropic-ai":["partial","*",7,0],"PerplexityBot":["partial","*",7,0],"Perplexity-User":["partial","*",7,0],"Google-Extended":["partial","*",7,0],"Applebot-Extended":["partial","*",7,0],"CCBot":["partial","*",7,0],"Bytespider":["partial","*",7,0],"Amazonbot":["partial","*",7,0],"meta-externalagent":["partial","*",7,0],"cohere-ai":["partial","*",7,0],"Diffbot":["partial","*",7,0],"ContextFetchBot":["partial","*",7,0]},"llms":{"status":"missing","http":404,"bytes":58072},"home":{"status":"ok","http":200,"url":"https://www.bertelsmann-stiftung.de/de/startseite","html_tokens":38553,"markdown_tokens":101,"pruned_tokens":101}},
{"rank":7437,"domain":"beliefnet.com","origin":"https://beliefnet.com","robots":{"status":"ok","http":200,"url":"https://www.beliefnet.com/robots.txt","bytes":1197,"groups":3},"bots":{"GPTBot":["partial","*",40,0],"ChatGPT-User":["partial","*",40,0],"OAI-SearchBot":["partial","*",40,0],"ClaudeBot":["partial","*",40,0],"Claude-User":["partial","*",40,0],"Claude-SearchBot":["partial","*",40,0],"anthropic-ai":["partial","*",40,0],"PerplexityBot":["partial","*",40,0],"Perplexity-User":["partial","*",40,0],"Google-Extended":["partial","*",40,0],"Applebot-Extended":["partial","*",40,0],"CCBot":["partial","*",40,0],"Bytespider":["partial","*",40,0],"Amazonbot":["partial","*",40,0],"meta-externalagent":["partial","*",40,0],"cohere-ai":["partial","*",40,0],"Diffbot":["partial","*",40,0],"ContextFetchBot":["partial","*",40,0]},"llms":{"status":"missing","http":404,"bytes":76516},"home":{"status":"http_error","http":403,"url":"https://www.beliefnet.com/"}},
{"rank":7438,"domain":"hulkshare.com","origin":"https://hulkshare.com","robots":{"status":"refused","http":403,"url":"https://www.hulkshare.com/robots.txt","bytes":5555,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":7439,"domain":"banki.ru","origin":"https://banki.ru","robots":{"status":"html","http":200,"url":"https://banki.ru/robots.txt","bytes":13602,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":7440,"domain":"zscaler.com","origin":"https://zscaler.com","robots":{"status":"ok","http":200,"url":"https://www.zscaler.com/robots.txt","bytes":4700,"groups":2},"bots":{"GPTBot":["partial","named",66,46],"ChatGPT-User":["partial","named",66,46],"OAI-SearchBot":["partial","named",66,46],"ClaudeBot":["partial","named",66,46],"Claude-User":["partial","named",66,46],"Claude-SearchBot":["partial","named",66,46],"anthropic-ai":["allowed","*",0,1],"PerplexityBot":["partial","named",66,46],"Perplexity-User":["allowed","*",0,1],"Google-Extended":["partial","named",66,46],"Applebot-Extended":["partial","named",66,46],"CCBot":["allowed","*",0,1],"Bytespider":["allowed","*",0,1],"Amazonbot":["allowed","*",0,1],"meta-externalagent":["allowed","*",0,1],"cohere-ai":["partial","named",66,46],"Diffbot":["allowed","*",0,1],"ContextFetchBot":["allowed","*",0,1]},"llms":{"status":"present","http":200,"bytes":98420},"home":{"status":"ok","http":200,"url":"https://www.zscaler.com/","html_tokens":133285,"markdown_tokens":218,"pruned_tokens":198}},
{"rank":7441,"domain":"meta.ai","origin":"https://meta.ai","robots":{"status":"ok","http":200,"url":"https://meta.ai/robots.txt","bytes":1523,"groups":3},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["blocked","*",1,0],"OAI-SearchBot":["blocked","*",1,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["blocked","*",1,0],"Claude-SearchBot":["blocked","*",1,0],"anthropic-ai":["blocked","*",1,0],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["blocked","*",1,0],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","*",1,0],"Bytespider":["blocked","*",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","*",1,0],"cohere-ai":["blocked","*",1,0],"Diffbot":["blocked","*",1,0],"ContextFetchBot":["blocked","*",1,0]},"llms":{"status":"disallowed"},"home":{"status":"disallowed"}},
{"rank":7442,"domain":"123formbuilder.com","origin":"https://123formbuilder.com","robots":{"status":"ok","http":200,"url":"https://www.123formbuilder.com/robots.txt","bytes":837,"groups":1},"bots":{"GPTBot":["partial","*",9,2],"ChatGPT-User":["partial","*",9,2],"OAI-SearchBot":["partial","*",9,2],"ClaudeBot":["partial","*",9,2],"Claude-User":["partial","*",9,2],"Claude-SearchBot":["partial","*",9,2],"anthropic-ai":["partial","*",9,2],"PerplexityBot":["partial","*",9,2],"Perplexity-User":["partial","*",9,2],"Google-Extended":["partial","*",9,2],"Applebot-Extended":["partial","*",9,2],"CCBot":["partial","*",9,2],"Bytespider":["partial","*",9,2],"Amazonbot":["partial","*",9,2],"meta-externalagent":["partial","*",9,2],"cohere-ai":["partial","*",9,2],"Diffbot":["partial","*",9,2],"ContextFetchBot":["partial","*",9,2]},"llms":{"status":"missing","http":404,"bytes":434759},"home":{"status":"ok","http":200,"url":"https://www.123formbuilder.com/","html_tokens":89454,"markdown_tokens":1865,"pruned_tokens":216}},
{"rank":7443,"domain":"cloudbeds.com","origin":"https://cloudbeds.com","robots":{"status":"ok","http":200,"url":"https://www.cloudbeds.com/robots.txt","bytes":794,"groups":1},"bots":{"GPTBot":["partial","*",12,0],"ChatGPT-User":["partial","*",12,0],"OAI-SearchBot":["partial","*",12,0],"ClaudeBot":["partial","*",12,0],"Claude-User":["partial","*",12,0],"Claude-SearchBot":["partial","*",12,0],"anthropic-ai":["partial","*",12,0],"PerplexityBot":["partial","*",12,0],"Perplexity-User":["partial","*",12,0],"Google-Extended":["partial","*",12,0],"Applebot-Extended":["partial","*",12,0],"CCBot":["partial","*",12,0],"Bytespider":["partial","*",12,0],"Amazonbot":["partial","*",12,0],"meta-externalagent":["partial","*",12,0],"cohere-ai":["partial","*",12,0],"Diffbot":["partial","*",12,0],"ContextFetchBot":["partial","*",12,0]},"llms":{"status":"missing","http":404,"bytes":448510},"home":{"status":"ok","http":200,"url":"https://www.cloudbeds.com/","html_tokens":132043,"markdown_tokens":3688,"pruned_tokens":672}},
{"rank":7444,"domain":"chinaadec.com","origin":"https://chinaadec.com","robots":{"status":"missing","http":404,"url":"https://chinaadec.com/robots.txt","bytes":971,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":967},"home":{"status":"ok","http":200,"url":"https://chinaadec.com/","html_tokens":16938,"markdown_tokens":60,"pruned_tokens":60}},
{"rank":7445,"domain":"gemini.google","origin":"https://gemini.google","robots":{"status":"ok","http":200,"url":"https://gemini.google/robots.txt","bytes":66,"groups":1},"bots":{"GPTBot":["allowed","*",0,1],"ChatGPT-User":["allowed","*",0,1],"OAI-SearchBot":["allowed","*",0,1],"ClaudeBot":["allowed","*",0,1],"Claude-User":["allowed","*",0,1],"Claude-SearchBot":["allowed","*",0,1],"anthropic-ai":["allowed","*",0,1],"PerplexityBot":["allowed","*",0,1],"Perplexity-User":["allowed","*",0,1],"Google-Extended":["allowed","*",0,1],"Applebot-Extended":["allowed","*",0,1],"CCBot":["allowed","*",0,1],"Bytespider":["allowed","*",0,1],"Amazonbot":["allowed","*",0,1],"meta-externalagent":["allowed","*",0,1],"cohere-ai":["allowed","*",0,1],"Diffbot":["allowed","*",0,1],"ContextFetchBot":["allowed","*",0,1]},"llms":{"status":"missing","http":404,"bytes":3},"home":{"status":"ok","http":200,"url":"https://gemini.google.com/","html_tokens":200293,"markdown_tokens":2,"pruned_tokens":2}},
{"rank":7446,"domain":"domestika.org","origin":"https://domestika.org","robots":{"status":"ok","http":200,"url":"https://www.domestika.org/robots.txt","bytes":7440,"groups":4},"bots":{"GPTBot":["partial","*",70,0],"ChatGPT-User":["partial","named",70,0],"OAI-SearchBot":["partial","named",70,0],"ClaudeBot":["partial","named",70,0],"Claude-User":["partial","named",70,0],"Claude-SearchBot":["partial","named",70,0],"anthropic-ai":["partial","*",70,0],"PerplexityBot":["partial","named",70,0],"Perplexity-User":["partial","*",70,0],"Google-Extended":["partial","*",70,0],"Applebot-Extended":["partial","*",70,0],"CCBot":["partial","*",70,0],"Bytespider":["partial","*",70,0],"Amazonbot":["partial","*",70,0],"meta-externalagent":["partial","*",70,0],"cohere-ai":["partial","*",70,0],"Diffbot":["partial","*",70,0],"ContextFetchBot":["partial","*",70,0]},"llms":{"status":"present","http":200,"bytes":7362},"home":{"status":"ok","http":200,"url":"https://www.domestika.org/en","html_tokens":184367,"markdown_tokens":443,"pruned_tokens":142}},
{"rank":7447,"domain":"careeronestop.org","origin":"https://careeronestop.org","robots":{"status":"refused","http":403,"url":"https://www.careeronestop.org/robots.txt","bytes":919,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":7448,"domain":"newzoo.com","origin":"https://newzoo.com","robots":{"status":"refused","http":403,"url":"https://newzoo.com/robots.txt","bytes":5505,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":7449,"domain":"alchemer.com","origin":"https://alchemer.com","robots":{"status":"ok","http":200,"url":"https://alchemer.com/robots.txt","bytes":437,"groups":1},"bots":{"GPTBot":["partial","*",5,6],"ChatGPT-User":["partial","*",5,6],"OAI-SearchBot":["partial","*",5,6],"ClaudeBot":["partial","*",5,6],"Claude-User":["partial","*",5,6],"Claude-SearchBot":["partial","*",5,6],"anthropic-ai":["partial","*",5,6],"PerplexityBot":["partial","*",5,6],"Perplexity-User":["partial","*",5,6],"Google-Extended":["partial","*",5,6],"Applebot-Extended":["partial","*",5,6],"CCBot":["partial","*",5,6],"Bytespider":["partial","*",5,6],"Amazonbot":["partial","*",5,6],"meta-externalagent":["partial","*",5,6],"cohere-ai":["partial","*",5,6],"Diffbot":["partial","*",5,6],"ContextFetchBot":["partial","*",5,6]},"llms":{"status":"http_error","http":403,"bytes":103},"home":{"status":"http_error","http":403,"url":"https://alchemer.com/"}},
{"rank":7450,"domain":"tommy.com","origin":"https://tommy.com","robots":{"status":"refused","http":403,"url":"https://usa.tommy.com","bytes":1674,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":7451,"domain":"wesleyan.edu","origin":"https://wesleyan.edu","robots":{"status":"missing","http":404,"url":"https://www.wesleyan.edu/robots.txt","bytes":197817,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":197817},"home":{"status":"ok","http":200,"url":"https://www.wesleyan.edu/","html_tokens":98837,"markdown_tokens":417,"pruned_tokens":170}},
{"rank":7452,"domain":"gamepedia.com","origin":"https://gamepedia.com","robots":{"status":"refused","http":403,"url":"https://www.fandom.com/robots.txt","bytes":5573,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":7453,"domain":"realpython.com","origin":"https://realpython.com","robots":{"status":"ok","http":200,"url":"https://realpython.com/robots.txt","bytes":356,"groups":1},"bots":{"GPTBot":["partial","*",12,0],"ChatGPT-User":["partial","*",12,0],"OAI-SearchBot":["partial","*",12,0],"ClaudeBot":["partial","*",12,0],"Claude-User":["partial","*",12,0],"Claude-SearchBot":["partial","*",12,0],"anthropic-ai":["partial","*",12,0],"PerplexityBot":["partial","*",12,0],"Perplexity-User":["partial","*",12,0],"Google-Extended":["partial","*",12,0],"Applebot-Extended":["partial","*",12,0],"CCBot":["partial","*",12,0],"Bytespider":["partial","*",12,0],"Amazonbot":["partial","*",12,0],"meta-externalagent":["partial","*",12,0],"cohere-ai":["partial","*",12,0],"Diffbot":["partial","*",12,0],"ContextFetchBot":["partial","*",12,0]},"llms":{"status":"missing","http":404,"bytes":32021},"home":{"status":"http_error","http":403,"url":"https://realpython.com/"}},
{"rank":7454,"domain":"massey.ac.nz","origin":"https://massey.ac.nz","robots":{"status":"ok","http":200,"url":"https://www.massey.ac.nz/robots.txt","bytes":83209,"groups":1},"bots":{"GPTBot":["partial","*",1217,1],"ChatGPT-User":["partial","*",1217,1],"OAI-SearchBot":["partial","*",1217,1],"ClaudeBot":["partial","*",1217,1],"Claude-User":["partial","*",1217,1],"Claude-SearchBot":["partial","*",1217,1],"anthropic-ai":["partial","*",1217,1],"PerplexityBot":["partial","*",1217,1],"Perplexity-User":["partial","*",1217,1],"Google-Extended":["partial","*",1217,1],"Applebot-Extended":["partial","*",1217,1],"CCBot":["partial","*",1217,1],"Bytespider":["partial","*",1217,1],"Amazonbot":["partial","*",1217,1],"meta-externalagent":["partial","*",1217,1],"cohere-ai":["partial","*",1217,1],"Diffbot":["partial","*",1217,1],"ContextFetchBot":["partial","*",1217,1]},"llms":{"status":"missing","http":404,"bytes":4850},"home":{"status":"ok","http":200,"url":"https://www.massey.ac.nz/","html_tokens":15962,"markdown_tokens":531,"pruned_tokens":126}},
{"rank":7455,"domain":"yourwebsite.com","origin":"https://yourwebsite.com","robots":{"status":"unreachable","http":null,"url":"https://yourwebsite.com/robots.txt","bytes":0,"groups":0,"error":"All connection attempts failed"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":7456,"domain":"asiasociety.org","origin":"https://asiasociety.org","robots":{"status":"ok","http":200,"url":"https://asiasociety.org/robots.txt","bytes":3740,"groups":18},"bots":{"GPTBot":["partial","*",22,19],"ChatGPT-User":["blocked","named",1,0],"OAI-SearchBot":["partial","*",22,19],"ClaudeBot":["partial","*",22,19],"Claude-User":["blocked","named",1,0],"Claude-SearchBot":["partial","*",22,19],"anthropic-ai":["partial","*",22,19],"PerplexityBot":["partial","*",22,19],"Perplexity-User":["blocked","named",1,0],"Google-Extended":["partial","*",22,19],"Applebot-Extended":["partial","*",22,19],"CCBot":["partial","*",22,19],"Bytespider":["partial","*",22,19],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["partial","*",22,19],"cohere-ai":["partial","*",22,19],"Diffbot":["partial","*",22,19],"ContextFetchBot":["partial","*",22,19]},"llms":{"status":"missing","http":404,"bytes":550},"home":{"status":"http_error","http":403,"url":"https://asiasociety.org/"}},
{"rank":7457,"domain":"kornferry.com","origin":"https://kornferry.com","robots":{"status":"ok","http":200,"url":"https://www.kornferry.com/robots.txt","bytes":502,"groups":13},"bots":{"GPTBot":["allowed","named",0,1],"ChatGPT-User":["partial","*",2,0],"OAI-SearchBot":["allowed","named",0,1],"ClaudeBot":["allowed","named",0,1],"Claude-User":["partial","*",2,0],"Claude-SearchBot":["partial","*",2,0],"anthropic-ai":["allowed","named",0,1],"PerplexityBot":["allowed","named",0,1],"Perplexity-User":["partial","*",2,0],"Google-Extended":["allowed","named",0,1],"Applebot-Extended":["partial","*",2,0],"CCBot":["allowed","named",0,1],"Bytespider":["blocked","named",1,0],"Amazonbot":["partial","*",2,0],"meta-externalagent":["partial","*",2,0],"cohere-ai":["partial","*",2,0],"Diffbot":["partial","*",2,0],"ContextFetchBot":["partial","*",2,0]},"llms":{"status":"present","http":200,"bytes":17712},"home":{"status":"ok","http":200,"url":"https://www.kornferry.com/","html_tokens":60520,"markdown_tokens":218,"pruned_tokens":170}},
{"rank":7458,"domain":"utoledo.edu","origin":"https://www.utoledo.edu","robots":{"status":"ok","http":200,"url":"https://www.utoledo.edu/robots.txt","bytes":65,"groups":1},"bots":{"GPTBot":["partial","*",1,0],"ChatGPT-User":["partial","*",1,0],"OAI-SearchBot":["partial","*",1,0],"ClaudeBot":["partial","*",1,0],"Claude-User":["partial","*",1,0],"Claude-SearchBot":["partial","*",1,0],"anthropic-ai":["partial","*",1,0],"PerplexityBot":["partial","*",1,0],"Perplexity-User":["partial","*",1,0],"Google-Extended":["partial","*",1,0],"Applebot-Extended":["partial","*",1,0],"CCBot":["partial","*",1,0],"Bytespider":["partial","*",1,0],"Amazonbot":["partial","*",1,0],"meta-externalagent":["partial","*",1,0],"cohere-ai":["partial","*",1,0],"Diffbot":["partial","*",1,0],"ContextFetchBot":["partial","*",1,0]},"llms":{"status":"html","http":200,"bytes":67972},"home":{"status":"ok","http":200,"url":"https://www.utoledo.edu/","html_tokens":12176,"markdown_tokens":136,"pruned_tokens":113}},
{"rank":7459,"domain":"birdlife.org","origin":"https://birdlife.org","robots":{"status":"ok","http":200,"url":"https://www.birdlife.org/robots.txt","bytes":261,"groups":1},"bots":{"GPTBot":["partial","*",1,1],"ChatGPT-User":["partial","*",1,1],"OAI-SearchBot":["partial","*",1,1],"ClaudeBot":["partial","*",1,1],"Claude-User":["partial","*",1,1],"Claude-SearchBot":["partial","*",1,1],"anthropic-ai":["partial","*",1,1],"PerplexityBot":["partial","*",1,1],"Perplexity-User":["partial","*",1,1],"Google-Extended":["partial","*",1,1],"Applebot-Extended":["partial","*",1,1],"CCBot":["partial","*",1,1],"Bytespider":["partial","*",1,1],"Amazonbot":["partial","*",1,1],"meta-externalagent":["partial","*",1,1],"cohere-ai":["partial","*",1,1],"Diffbot":["partial","*",1,1],"ContextFetchBot":["partial","*",1,1]},"llms":{"status":"missing","http":404,"bytes":92408},"home":{"status":"ok","http":200,"url":"https://www.birdlife.org/","html_tokens":44141,"markdown_tokens":69,"pruned_tokens":69}},
{"rank":7460,"domain":"jigsawplanet.com","origin":"https://jigsawplanet.com","robots":{"status":"ok","http":200,"url":"https://www.jigsawplanet.com/robots.txt","bytes":974,"groups":4},"bots":{"GPTBot":["partial","*",11,0],"ChatGPT-User":["partial","*",11,0],"OAI-SearchBot":["partial","*",11,0],"ClaudeBot":["partial","*",11,0],"Claude-User":["partial","*",11,0],"Claude-SearchBot":["partial","*",11,0],"anthropic-ai":["partial","*",11,0],"PerplexityBot":["partial","*",11,0],"Perplexity-User":["partial","*",11,0],"Google-Extended":["partial","*",11,0],"Applebot-Extended":["partial","*",11,0],"CCBot":["partial","*",11,0],"Bytespider":["partial","*",11,0],"Amazonbot":["partial","*",11,0],"meta-externalagent":["partial","*",11,0],"cohere-ai":["partial","*",11,0],"Diffbot":["partial","*",11,0],"ContextFetchBot":["partial","*",11,0]},"llms":{"status":"missing","http":404,"bytes":4646},"home":{"status":"ok","http":200,"url":"https://www.jigsawplanet.com/","html_tokens":12158,"markdown_tokens":24,"pruned_tokens":24}},
{"rank":7461,"domain":"medianewsgroup.com","origin":"https://medianewsgroup.com","robots":{"status":"ok","http":200,"url":"https://www.medianewsgroup.com/robots.txt","bytes":67,"groups":1},"bots":{"GPTBot":["partial","*",1,1],"ChatGPT-User":["partial","*",1,1],"OAI-SearchBot":["partial","*",1,1],"ClaudeBot":["partial","*",1,1],"Claude-User":["partial","*",1,1],"Claude-SearchBot":["partial","*",1,1],"anthropic-ai":["partial","*",1,1],"PerplexityBot":["partial","*",1,1],"Perplexity-User":["partial","*",1,1],"Google-Extended":["partial","*",1,1],"Applebot-Extended":["partial","*",1,1],"CCBot":["partial","*",1,1],"Bytespider":["partial","*",1,1],"Amazonbot":["partial","*",1,1],"meta-externalagent":["partial","*",1,1],"cohere-ai":["partial","*",1,1],"Diffbot":["partial","*",1,1],"ContextFetchBot":["partial","*",1,1]},"llms":{"status":"missing","http":404,"bytes":25328},"home":{"status":"ok","http":200,"url":"https://www.medianewsgroup.com/","html_tokens":18488,"markdown_tokens":296,"pruned_tokens":163}},
{"rank":7462,"domain":"ozemail.com.au","origin":"http://ozemail.com.au","robots":{"status":"ok","http":200,"url":"https://www.iinet.net.au/robots.txt","bytes":2208,"groups":1},"bots":{"GPTBot":["partial","*",34,18],"ChatGPT-User":["partial","*",34,18],"OAI-SearchBot":["partial","*",34,18],"ClaudeBot":["partial","*",34,18],"Claude-User":["partial","*",34,18],"Claude-SearchBot":["partial","*",34,18],"anthropic-ai":["partial","*",34,18],"PerplexityBot":["partial","*",34,18],"Perplexity-User":["partial","*",34,18],"Google-Extended":["partial","*",34,18],"Applebot-Extended":["partial","*",34,18],"CCBot":["partial","*",34,18],"Bytespider":["partial","*",34,18],"Amazonbot":["partial","*",34,18],"meta-externalagent":["partial","*",34,18],"cohere-ai":["partial","*",34,18],"Diffbot":["partial","*",34,18],"ContextFetchBot":["partial","*",34,18]},"llms":{"status":"missing","http":404,"bytes":191},"home":{"status":"ok","http":200,"url":"https://www.iinet.net.au/home","html_tokens":53740,"markdown_tokens":5388,"pruned_tokens":152}},
{"rank":7463,"domain":"digistore24.com","origin":"https://digistore24.com","robots":{"status":"ok","http":200,"url":"https://www.digistore24.com/robots.txt","bytes":673,"groups":2},"bots":{"GPTBot":["partial","*",22,1],"ChatGPT-User":["partial","*",22,1],"OAI-SearchBot":["partial","*",22,1],"ClaudeBot":["partial","*",22,1],"Claude-User":["partial","*",22,1],"Claude-SearchBot":["partial","*",22,1],"anthropic-ai":["partial","*",22,1],"PerplexityBot":["partial","*",22,1],"Perplexity-User":["partial","*",22,1],"Google-Extended":["partial","*",22,1],"Applebot-Extended":["partial","*",22,1],"CCBot":["partial","*",22,1],"Bytespider":["partial","*",22,1],"Amazonbot":["partial","*",22,1],"meta-externalagent":["partial","*",22,1],"cohere-ai":["partial","*",22,1],"Diffbot":["partial","*",22,1],"ContextFetchBot":["partial","*",22,1]},"llms":{"status":"missing","http":404,"bytes":5198},"home":{"status":"ok","http":200,"url":"https://www.digistore24.com/","html_tokens":43051,"markdown_tokens":799,"pruned_tokens":482}},
{"rank":7464,"domain":"ubersuggest.com","origin":"https://ubersuggest.com","robots":{"status":"ok","http":200,"url":"https://ubersuggest.com/robots.txt","bytes":23981,"groups":2},"bots":{"GPTBot":["partial","named",320,1],"ChatGPT-User":["partial","named",320,1],"OAI-SearchBot":["partial","named",320,1],"ClaudeBot":["partial","named",320,1],"Claude-User":["partial","*",320,1],"Claude-SearchBot":["partial","*",320,1],"anthropic-ai":["partial","named",320,1],"PerplexityBot":["partial","named",320,1],"Perplexity-User":["partial","*",320,1],"Google-Extended":["partial","named",320,1],"Applebot-Extended":["partial","named",320,1],"CCBot":["partial","named",320,1],"Bytespider":["partial","*",320,1],"Amazonbot":["partial","*",320,1],"meta-externalagent":["partial","*",320,1],"cohere-ai":["partial","*",320,1],"Diffbot":["partial","*",320,1],"ContextFetchBot":["partial","*",320,1]},"llms":{"status":"present","http":200,"bytes":5567},"home":{"status":"ok","http":200,"url":"https://ubersuggest.com/en","html_tokens":291956,"markdown_tokens":3490,"pruned_tokens":550}},
{"rank":7465,"domain":"visitberlin.de","origin":"https://visitberlin.de","robots":{"status":"ok","http":200,"url":"https://www.visitberlin.de/robots.txt","bytes":5210,"groups":1},"bots":{"GPTBot":["partial","*",88,36],"ChatGPT-User":["partial","*",88,36],"OAI-SearchBot":["partial","*",88,36],"ClaudeBot":["partial","*",88,36],"Claude-User":["partial","*",88,36],"Claude-SearchBot":["partial","*",88,36],"anthropic-ai":["partial","*",88,36],"PerplexityBot":["partial","*",88,36],"Perplexity-User":["partial","*",88,36],"Google-Extended":["partial","*",88,36],"Applebot-Extended":["partial","*",88,36],"CCBot":["partial","*",88,36],"Bytespider":["partial","*",88,36],"Amazonbot":["partial","*",88,36],"meta-externalagent":["partial","*",88,36],"cohere-ai":["partial","*",88,36],"Diffbot":["partial","*",88,36],"ContextFetchBot":["partial","*",88,36]},"llms":{"status":"missing","http":404,"bytes":192},"home":{"status":"ok","http":200,"url":"https://www.visitberlin.de/de","html_tokens":132215,"markdown_tokens":253,"pruned_tokens":29}},
{"rank":7466,"domain":"delfi.lv","origin":"https://delfi.lv","robots":{"status":"ok","http":200,"url":"https://www.delfi.lv/robots.txt","bytes":1318,"groups":28},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["blocked","named",1,0],"OAI-SearchBot":["blocked","named",1,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",10,1],"Claude-SearchBot":["partial","*",10,1],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["partial","*",10,1],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",10,1]},"llms":{"status":"missing","http":404,"bytes":103463},"home":{"status":"ok","http":200,"url":"https://www.delfi.lv/","html_tokens":145357,"markdown_tokens":366,"pruned_tokens":55}},
{"rank":7467,"domain":"zt.ua","origin":"http://zt.ua","robots":{"status":"missing","http":404,"url":"http://zt.ua/robots.txt","bytes":162,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":162},"home":{"status":"ok","http":200,"url":"http://zt.ua/","html_tokens":20144,"markdown_tokens":8178,"pruned_tokens":164}},
{"rank":7468,"domain":"deutschepost.de","origin":"https://deutschepost.de","robots":{"status":"ok","http":200,"url":"https://www.deutschepost.de/robots.txt","bytes":938,"groups":1},"bots":{"GPTBot":["partial","*",13,0],"ChatGPT-User":["partial","*",13,0],"OAI-SearchBot":["partial","*",13,0],"ClaudeBot":["partial","*",13,0],"Claude-User":["partial","*",13,0],"Claude-SearchBot":["partial","*",13,0],"anthropic-ai":["partial","*",13,0],"PerplexityBot":["partial","*",13,0],"Perplexity-User":["partial","*",13,0],"Google-Extended":["partial","*",13,0],"Applebot-Extended":["partial","*",13,0],"CCBot":["partial","*",13,0],"Bytespider":["partial","*",13,0],"Amazonbot":["partial","*",13,0],"meta-externalagent":["partial","*",13,0],"cohere-ai":["partial","*",13,0],"Diffbot":["partial","*",13,0],"ContextFetchBot":["partial","*",13,0]},"llms":{"status":"missing","http":404,"bytes":46056},"home":{"status":"ok","http":200,"url":"https://www.deutschepost.de/de.html","html_tokens":43415,"markdown_tokens":25,"pruned_tokens":25}},
{"rank":7469,"domain":"vancouver.ca","origin":"https://vancouver.ca","robots":{"status":"refused","http":403,"url":"https://vancouver.ca/robots.txt","bytes":4545,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":7470,"domain":"formsite.com","origin":"https://formsite.com","robots":{"status":"refused","http":403,"url":"https://formsite.com/robots.txt","bytes":560,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":7471,"domain":"hkbu.edu.hk","origin":"https://hkbu.edu.hk","robots":{"status":"ok","http":200,"url":"https://www.hkbu.edu.hk/robots.txt","bytes":75,"groups":1},"bots":{"GPTBot":["partial","*",1,1],"ChatGPT-User":["partial","*",1,1],"OAI-SearchBot":["partial","*",1,1],"ClaudeBot":["partial","*",1,1],"Claude-User":["partial","*",1,1],"Claude-SearchBot":["partial","*",1,1],"anthropic-ai":["partial","*",1,1],"PerplexityBot":["partial","*",1,1],"Perplexity-User":["partial","*",1,1],"Google-Extended":["partial","*",1,1],"Applebot-Extended":["partial","*",1,1],"CCBot":["partial","*",1,1],"Bytespider":["partial","*",1,1],"Amazonbot":["partial","*",1,1],"meta-externalagent":["partial","*",1,1],"cohere-ai":["partial","*",1,1],"Diffbot":["partial","*",1,1],"ContextFetchBot":["partial","*",1,1]},"llms":{"status":"missing","http":404,"bytes":38007},"home":{"status":"ok","http":200,"url":"https://www.hkbu.edu.hk/","html_tokens":39847,"markdown_tokens":174,"pruned_tokens":174}},
{"rank":7472,"domain":"asda.com","origin":"https://asda.com","robots":{"status":"ok","http":200,"url":"https://asda.com/robots.txt","bytes":2676,"groups":32},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["allowed","*",0,1],"OAI-SearchBot":["allowed","*",0,1],"ClaudeBot":["blocked","named",1,0],"Claude-User":["allowed","*",0,1],"Claude-SearchBot":["allowed","*",0,1],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["allowed","*",0,1],"Perplexity-User":["allowed","*",0,1],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["allowed","*",0,1]},"llms":{"status":"disallowed","error":"robots.txt of www.asda.com not readable (refused)"},"home":{"status":"disallowed","error":"robots.txt of www.asda.com not readable (refused)"}},
{"rank":7473,"domain":"janes.com","origin":"https://janes.com","robots":{"status":"ok","http":200,"url":"https://www.janes.com/robots.txt","bytes":4838,"groups":32},"bots":{"GPTBot":["partial","named",11,0],"ChatGPT-User":["blocked","*",1,0],"OAI-SearchBot":["partial","named",11,0],"ClaudeBot":["partial","named",11,0],"Claude-User":["blocked","*",1,0],"Claude-SearchBot":["partial","named",11,0],"anthropic-ai":["partial","named",11,0],"PerplexityBot":["partial","named",11,0],"Perplexity-User":["partial","named",11,0],"Google-Extended":["partial","named",11,0],"Applebot-Extended":["blocked","*",1,0],"CCBot":["blocked","*",1,0],"Bytespider":["blocked","*",1,0],"Amazonbot":["blocked","*",1,0],"meta-externalagent":["blocked","*",1,0],"cohere-ai":["blocked","*",1,0],"Diffbot":["blocked","*",1,0],"ContextFetchBot":["blocked","*",1,0]},"llms":{"status":"disallowed"},"home":{"status":"disallowed"}},
{"rank":7474,"domain":"boost.org","origin":"https://boost.org","robots":{"status":"ok","http":200,"url":"https://www.boost.org/robots.txt","bytes":24,"groups":1},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"missing","http":404,"bytes":115417},"home":{"status":"ok","http":200,"url":"https://www.boost.org/","html_tokens":66518,"markdown_tokens":2531,"pruned_tokens":524}},
{"rank":7475,"domain":"dguv.de","origin":"https://dguv.de","robots":{"status":"missing","http":404,"url":"https://dguv.de/robots.txt","bytes":777,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":775},"home":{"status":"ok","http":200,"url":"https://www.dguv.de/de/index.jsp","html_tokens":15366,"markdown_tokens":204,"pruned_tokens":59}},
{"rank":7476,"domain":"interno.gov.it","origin":"https://interno.gov.it","robots":{"status":"html","http":200,"url":"https://www.interno.gov.it/it","bytes":125851,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":7477,"domain":"passion-games.com","origin":"https://passion-games.com","robots":{"status":"missing","http":404,"url":"https://passion-games.com/robots.txt","bytes":31827,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":31827},"home":{"status":"http_error","http":404,"url":"https://passion-games.com/"}},
{"rank":7478,"domain":"ak.us","origin":"https://ak.us","robots":{"status":"unreachable","http":null,"url":"https://ak.us/robots.txt","bytes":0,"groups":0,"error":"cannot resolve ak.us"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":7479,"domain":"npmjs.org","origin":"https://npmjs.org","robots":{"status":"refused","http":403,"url":"https://www.npmjs.com/robots.txt","bytes":5594,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":7480,"domain":"iconfont.cn","origin":"https://iconfont.cn","robots":{"status":"html","http":200,"url":"https://www.iconfont.cn/","bytes":2801,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":7481,"domain":"lightreading.com","origin":"https://lightreading.com","robots":{"status":"ok","http":200,"url":"https://www.lightreading.com/robots.txt","bytes":1546,"groups":2},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["partial","*",6,0],"OAI-SearchBot":["partial","*",6,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",6,0],"Claude-SearchBot":["partial","*",6,0],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["partial","*",6,0],"Perplexity-User":["partial","*",6,0],"Google-Extended":["partial","*",6,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",6,0]},"llms":{"status":"missing","http":404,"bytes":369138},"home":{"status":"ok","http":200,"url":"https://www.lightreading.com/","html_tokens":168134,"markdown_tokens":96,"pruned_tokens":38}},
{"rank":7482,"domain":"tor.com","origin":"https://tor.com","robots":{"status":"ok","http":200,"url":"https://reactormag.com/robots.txt","bytes":73,"groups":2},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"http_error","http":403,"bytes":5773},"home":{"status":"http_error","http":403,"url":"https://tor.com/"}},
{"rank":7483,"domain":"useit.com","origin":"https://useit.com","robots":{"status":"ok","http":200,"url":"https://www.nngroup.com/robots.txt","bytes":690,"groups":15},"bots":{"GPTBot":["partial","*",6,0],"ChatGPT-User":["partial","*",6,0],"OAI-SearchBot":["partial","*",6,0],"ClaudeBot":["partial","*",6,0],"Claude-User":["partial","*",6,0],"Claude-SearchBot":["partial","*",6,0],"anthropic-ai":["partial","*",6,0],"PerplexityBot":["partial","*",6,0],"Perplexity-User":["partial","*",6,0],"Google-Extended":["allowed","named",0,1],"Applebot-Extended":["partial","*",6,0],"CCBot":["blocked","named",1,0],"Bytespider":["partial","*",6,0],"Amazonbot":["partial","*",6,0],"meta-externalagent":["partial","*",6,0],"cohere-ai":["partial","*",6,0],"Diffbot":["partial","*",6,0],"ContextFetchBot":["partial","*",6,0]},"llms":{"status":"timeout","error":"timed out after 10 s"},"home":{"status":"timeout","error":"timed out after 15 s"}},
{"rank":7484,"domain":"livewallpapers.com","origin":"https://livewallpapers.com","robots":{"status":"ok","http":200,"url":"https://livewallpapers.com/robots.txt","bytes":271,"groups":1},"bots":{"GPTBot":["partial","*",2,1],"ChatGPT-User":["partial","*",2,1],"OAI-SearchBot":["partial","*",2,1],"ClaudeBot":["partial","*",2,1],"Claude-User":["partial","*",2,1],"Claude-SearchBot":["partial","*",2,1],"anthropic-ai":["partial","*",2,1],"PerplexityBot":["partial","*",2,1],"Perplexity-User":["partial","*",2,1],"Google-Extended":["partial","*",2,1],"Applebot-Extended":["partial","*",2,1],"CCBot":["partial","*",2,1],"Bytespider":["partial","*",2,1],"Amazonbot":["partial","*",2,1],"meta-externalagent":["partial","*",2,1],"cohere-ai":["partial","*",2,1],"Diffbot":["partial","*",2,1],"ContextFetchBot":["partial","*",2,1]},"llms":{"status":"html","http":200,"bytes":84926},"home":{"status":"ok","http":200,"url":"https://livewallpapers.com/","html_tokens":99532,"markdown_tokens":474,"pruned_tokens":171}},
{"rank":7485,"domain":"brynmawr.edu","origin":"https://brynmawr.edu","robots":{"status":"refused","http":403,"url":"https://brynmawr.edu/robots.txt","bytes":560,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":7486,"domain":"oeko-tex.com","origin":"https://oeko-tex.com","robots":{"status":"ok","http":200,"url":"https://www.oeko-tex.com/robots.txt","bytes":280,"groups":1},"bots":{"GPTBot":["partial","*",2,0],"ChatGPT-User":["partial","*",2,0],"OAI-SearchBot":["partial","*",2,0],"ClaudeBot":["partial","*",2,0],"Claude-User":["partial","*",2,0],"Claude-SearchBot":["partial","*",2,0],"anthropic-ai":["partial","*",2,0],"PerplexityBot":["partial","*",2,0],"Perplexity-User":["partial","*",2,0],"Google-Extended":["partial","*",2,0],"Applebot-Extended":["partial","*",2,0],"CCBot":["partial","*",2,0],"Bytespider":["partial","*",2,0],"Amazonbot":["partial","*",2,0],"meta-externalagent":["partial","*",2,0],"cohere-ai":["partial","*",2,0],"Diffbot":["partial","*",2,0],"ContextFetchBot":["partial","*",2,0]},"llms":{"status":"missing","http":404,"bytes":45494},"home":{"status":"ok","http":200,"url":"https://www.oeko-tex.com/en/","html_tokens":27853,"markdown_tokens":69,"pruned_tokens":69}},
{"rank":7487,"domain":"google.az","origin":"https://google.az","robots":{"status":"ok","http":200,"url":"https://www.google.az/robots.txt","bytes":6624,"groups":4},"bots":{"GPTBot":["partial","*",169,64],"ChatGPT-User":["partial","*",169,64],"OAI-SearchBot":["partial","*",169,64],"ClaudeBot":["partial","*",169,64],"Claude-User":["partial","*",169,64],"Claude-SearchBot":["partial","*",169,64],"anthropic-ai":["partial","*",169,64],"PerplexityBot":["partial","*",169,64],"Perplexity-User":["partial","*",169,64],"Google-Extended":["partial","*",169,64],"Applebot-Extended":["partial","*",169,64],"CCBot":["partial","*",169,64],"Bytespider":["partial","*",169,64],"Amazonbot":["partial","*",169,64],"meta-externalagent":["partial","*",169,64],"cohere-ai":["partial","*",169,64],"Diffbot":["partial","*",169,64],"ContextFetchBot":["partial","*",169,64]},"llms":{"status":"missing","http":404,"bytes":1569},"home":{"status":"ok","http":200,"url":"https://www.google.com/","html_tokens":21345,"markdown_tokens":30,"pruned_tokens":30}},
{"rank":7488,"domain":"costar.com","origin":"https://costar.com","robots":{"status":"refused","http":403,"url":"https://costar.com/robots.txt","bytes":372,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":7489,"domain":"fapaiwang.cn","origin":"http://fapaiwang.cn","robots":{"status":"missing","http":404,"url":"http://fapaiwang.cn/robots.txt","bytes":146,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":146},"home":{"status":"ok","http":200,"url":"http://fapaiwang.cn/","html_tokens":21474,"markdown_tokens":10,"pruned_tokens":10}},
{"rank":7490,"domain":"lpages.co","origin":"https://www.lpages.co","robots":{"status":"ok","http":200,"url":"https://www.lpages.co/robots.txt","bytes":0,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":100141},"home":{"status":"http_error","http":404,"url":"https://www.lpages.co/"}},
{"rank":7491,"domain":"informaconnect.com","origin":"https://informaconnect.com","robots":{"status":"ok","http":200,"url":"https://informaconnect.com/robots.txt","bytes":62,"groups":1},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"missing","http":404,"bytes":855671},"home":{"status":"ok","http":200,"url":"https://informaconnect.com/","html_tokens":726156,"markdown_tokens":166,"pruned_tokens":166}},
{"rank":7492,"domain":"download.it","origin":"https://download.it","robots":{"status":"ok","http":200,"url":"https://download.it/robots.txt","bytes":5722,"groups":138},"bots":{"GPTBot":["allowed","named",0,0],"ChatGPT-User":["partial","*",2,0],"OAI-SearchBot":["partial","*",2,0],"ClaudeBot":["partial","*",2,0],"Claude-User":["partial","*",2,0],"Claude-SearchBot":["partial","*",2,0],"anthropic-ai":["partial","*",2,0],"PerplexityBot":["partial","*",2,0],"Perplexity-User":["partial","*",2,0],"Google-Extended":["partial","*",2,0],"Applebot-Extended":["partial","*",2,0],"CCBot":["partial","*",2,0],"Bytespider":["partial","*",2,0],"Amazonbot":["partial","*",2,0],"meta-externalagent":["partial","*",2,0],"cohere-ai":["partial","*",2,0],"Diffbot":["partial","*",2,0],"ContextFetchBot":["partial","*",2,0]},"llms":{"status":"missing","http":404,"bytes":635},"home":{"status":"http_error","http":403,"url":"https://download.it/"}},
{"rank":7493,"domain":"johnsoncontrols.com","origin":"https://johnsoncontrols.com","robots":{"status":"ok","http":200,"url":"https://www.johnsoncontrols.com/robots.txt","bytes":802,"groups":1},"bots":{"GPTBot":["partial","named",16,2],"ChatGPT-User":["partial","named",16,2],"OAI-SearchBot":["partial","named",16,2],"ClaudeBot":["partial","named",16,2],"Claude-User":["partial","named",16,2],"Claude-SearchBot":["partial","named",16,2],"anthropic-ai":["partial","*",16,2],"PerplexityBot":["partial","named",16,2],"Perplexity-User":["partial","named",16,2],"Google-Extended":["partial","named",16,2],"Applebot-Extended":["partial","named",16,2],"CCBot":["partial","named",16,2],"Bytespider":["partial","*",16,2],"Amazonbot":["partial","named",16,2],"meta-externalagent":["partial","named",16,2],"cohere-ai":["partial","*",16,2],"Diffbot":["partial","*",16,2],"ContextFetchBot":["partial","*",16,2]},"llms":{"status":"present","http":200,"bytes":3810},"home":{"status":"ok","http":200,"url":"https://www.johnsoncontrols.com/","html_tokens":69376,"markdown_tokens":282,"pruned_tokens":75}},
{"rank":7494,"domain":"yahooinc.com","origin":"https://yahooinc.com","robots":{"status":"ok","http":200,"url":"https://www.yahooinc.com/robots.txt","bytes":70,"groups":1},"bots":{"GPTBot":["allowed","*",0,1],"ChatGPT-User":["allowed","*",0,1],"OAI-SearchBot":["allowed","*",0,1],"ClaudeBot":["allowed","*",0,1],"Claude-User":["allowed","*",0,1],"Claude-SearchBot":["allowed","*",0,1],"anthropic-ai":["allowed","*",0,1],"PerplexityBot":["allowed","*",0,1],"Perplexity-User":["allowed","*",0,1],"Google-Extended":["allowed","*",0,1],"Applebot-Extended":["allowed","*",0,1],"CCBot":["allowed","*",0,1],"Bytespider":["allowed","*",0,1],"Amazonbot":["allowed","*",0,1],"meta-externalagent":["allowed","*",0,1],"cohere-ai":["allowed","*",0,1],"Diffbot":["allowed","*",0,1],"ContextFetchBot":["allowed","*",0,1]},"llms":{"status":"missing","http":404,"bytes":27509},"home":{"status":"ok","http":200,"url":"https://www.yahooinc.com/","html_tokens":14451,"markdown_tokens":156,"pruned_tokens":107}},
{"rank":7495,"domain":"huya.com","origin":"https://huya.com","robots":{"status":"ok","http":200,"url":"https://www.huya.com/robots.txt","bytes":1172,"groups":12},"bots":{"GPTBot":["blocked","*",1,0],"ChatGPT-User":["blocked","*",1,0],"OAI-SearchBot":["blocked","*",1,0],"ClaudeBot":["blocked","*",1,0],"Claude-User":["blocked","*",1,0],"Claude-SearchBot":["blocked","*",1,0],"anthropic-ai":["blocked","*",1,0],"PerplexityBot":["blocked","*",1,0],"Perplexity-User":["blocked","*",1,0],"Google-Extended":["blocked","*",1,0],"Applebot-Extended":["blocked","*",1,0],"CCBot":["blocked","*",1,0],"Bytespider":["blocked","*",1,0],"Amazonbot":["blocked","*",1,0],"meta-externalagent":["blocked","*",1,0],"cohere-ai":["blocked","*",1,0],"Diffbot":["blocked","*",1,0],"ContextFetchBot":["blocked","*",1,0]},"llms":{"status":"disallowed"},"home":{"status":"disallowed"}},
{"rank":7496,"domain":"klingai.com","origin":"https://klingai.com","robots":{"status":"ok","http":200,"url":"https://klingai.com/robots.txt","bytes":408,"groups":1},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["blocked","named",1,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"present","http":200,"bytes":1729},"home":{"status":"ok","http":200,"url":"https://kling.ai/","html_tokens":25657,"markdown_tokens":938,"pruned_tokens":121}},
{"rank":7497,"domain":"ybetscasino.net","origin":"https://ybetscasino.net","robots":{"status":"ok","http":200,"url":"https://ybetscasino.net/robots.txt","bytes":67,"groups":1},"bots":{"GPTBot":["partial","*",1,1],"ChatGPT-User":["partial","*",1,1],"OAI-SearchBot":["partial","*",1,1],"ClaudeBot":["partial","*",1,1],"Claude-User":["partial","*",1,1],"Claude-SearchBot":["partial","*",1,1],"anthropic-ai":["partial","*",1,1],"PerplexityBot":["partial","*",1,1],"Perplexity-User":["partial","*",1,1],"Google-Extended":["partial","*",1,1],"Applebot-Extended":["partial","*",1,1],"CCBot":["partial","*",1,1],"Bytespider":["partial","*",1,1],"Amazonbot":["partial","*",1,1],"meta-externalagent":["partial","*",1,1],"cohere-ai":["partial","*",1,1],"Diffbot":["partial","*",1,1],"ContextFetchBot":["partial","*",1,1]},"llms":{"status":"missing","http":404,"bytes":43026},"home":{"status":"ok","http":200,"url":"https://ybetscasino.net/","html_tokens":15051,"markdown_tokens":1948,"pruned_tokens":114}},
{"rank":7498,"domain":"frappe.io","origin":"https://frappe.io","robots":{"status":"ok","http":200,"url":"https://frappe.io/robots.txt","bytes":279,"groups":2},"bots":{"GPTBot":["partial","*",5,5],"ChatGPT-User":["partial","*",5,5],"OAI-SearchBot":["partial","*",5,5],"ClaudeBot":["partial","*",5,5],"Claude-User":["partial","*",5,5],"Claude-SearchBot":["partial","*",5,5],"anthropic-ai":["partial","*",5,5],"PerplexityBot":["partial","*",5,5],"Perplexity-User":["partial","*",5,5],"Google-Extended":["partial","*",5,5],"Applebot-Extended":["partial","*",5,5],"CCBot":["partial","*",5,5],"Bytespider":["partial","*",5,5],"Amazonbot":["partial","*",5,5],"meta-externalagent":["partial","*",5,5],"cohere-ai":["partial","*",5,5],"Diffbot":["partial","*",5,5],"ContextFetchBot":["partial","*",5,5]},"llms":{"status":"present","http":200,"bytes":340},"home":{"status":"ok","http":200,"url":"https://frappe.io/","html_tokens":56703,"markdown_tokens":7,"pruned_tokens":7}},
{"rank":7499,"domain":"law-gov.cn","origin":"http://law-gov.cn","robots":{"status":"ok","http":200,"url":"http://law-gov.cn/robots.txt","bytes":2245,"groups":62},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["blocked","*",1,0],"OAI-SearchBot":["blocked","*",1,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["blocked","*",1,0],"Claude-SearchBot":["blocked","*",1,0],"anthropic-ai":["blocked","*",1,0],"PerplexityBot":["blocked","*",1,0],"Perplexity-User":["blocked","*",1,0],"Google-Extended":["blocked","*",1,0],"Applebot-Extended":["blocked","*",1,0],"CCBot":["blocked","*",1,0],"Bytespider":["blocked","*",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","*",1,0],"cohere-ai":["blocked","*",1,0],"Diffbot":["blocked","*",1,0],"ContextFetchBot":["blocked","*",1,0]},"llms":{"status":"disallowed"},"home":{"status":"disallowed"}},
{"rank":7500,"domain":"disneylandparis.com","origin":"https://disneylandparis.com","robots":{"status":"refused","http":403,"url":"https://www.disneylandparis.com/robots.txt","bytes":391,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":7501,"domain":"digital.gov","origin":"https://digital.gov","robots":{"status":"ok","http":200,"url":"https://digital.gov/robots.txt","bytes":1283,"groups":1},"bots":{"GPTBot":["partial","*",3,29],"ChatGPT-User":["partial","*",3,29],"OAI-SearchBot":["partial","*",3,29],"ClaudeBot":["partial","*",3,29],"Claude-User":["partial","*",3,29],"Claude-SearchBot":["partial","*",3,29],"anthropic-ai":["partial","*",3,29],"PerplexityBot":["partial","*",3,29],"Perplexity-User":["partial","*",3,29],"Google-Extended":["partial","*",3,29],"Applebot-Extended":["partial","*",3,29],"CCBot":["partial","*",3,29],"Bytespider":["partial","*",3,29],"Amazonbot":["partial","*",3,29],"meta-externalagent":["partial","*",3,29],"cohere-ai":["partial","*",3,29],"Diffbot":["partial","*",3,29],"ContextFetchBot":["partial","*",3,29]},"llms":{"status":"missing","http":404,"bytes":22043},"home":{"status":"ok","http":200,"url":"https://digital.gov/","html_tokens":8684,"markdown_tokens":199,"pruned_tokens":21}},
{"rank":7502,"domain":"elitedomains.de","origin":"https://elitedomains.de","robots":{"status":"ok","http":200,"url":"https://elitedomains.de/robots.txt","bytes":70,"groups":1},"bots":{"GPTBot":["allowed","*",0,1],"ChatGPT-User":["allowed","*",0,1],"OAI-SearchBot":["allowed","*",0,1],"ClaudeBot":["allowed","*",0,1],"Claude-User":["allowed","*",0,1],"Claude-SearchBot":["allowed","*",0,1],"anthropic-ai":["allowed","*",0,1],"PerplexityBot":["allowed","*",0,1],"Perplexity-User":["allowed","*",0,1],"Google-Extended":["allowed","*",0,1],"Applebot-Extended":["allowed","*",0,1],"CCBot":["allowed","*",0,1],"Bytespider":["allowed","*",0,1],"Amazonbot":["allowed","*",0,1],"meta-externalagent":["allowed","*",0,1],"cohere-ai":["allowed","*",0,1],"Diffbot":["allowed","*",0,1],"ContextFetchBot":["allowed","*",0,1]},"llms":{"status":"missing","http":404,"bytes":20095},"home":{"status":"ok","http":200,"url":"https://elitedomains.de/","html_tokens":20622,"markdown_tokens":1302,"pruned_tokens":86}},
{"rank":7503,"domain":"oddschecker.com","origin":"https://oddschecker.com","robots":{"status":"ok","http":200,"url":"https://www.oddschecker.com/robots.txt","bytes":1577,"groups":1},"bots":{"GPTBot":["partial","*",26,0],"ChatGPT-User":["partial","*",26,0],"OAI-SearchBot":["partial","*",26,0],"ClaudeBot":["partial","*",26,0],"Claude-User":["partial","*",26,0],"Claude-SearchBot":["partial","*",26,0],"anthropic-ai":["partial","*",26,0],"PerplexityBot":["partial","*",26,0],"Perplexity-User":["partial","*",26,0],"Google-Extended":["partial","*",26,0],"Applebot-Extended":["partial","*",26,0],"CCBot":["partial","*",26,0],"Bytespider":["partial","*",26,0],"Amazonbot":["partial","*",26,0],"meta-externalagent":["partial","*",26,0],"cohere-ai":["partial","*",26,0],"Diffbot":["partial","*",26,0],"ContextFetchBot":["partial","*",26,0]},"llms":{"status":"missing","http":404,"bytes":321372},"home":{"status":"http_error","http":403,"url":"https://www.oddschecker.com/"}},
{"rank":7504,"domain":"next.co.uk","origin":"https://next.co.uk","robots":{"status":"refused","http":403,"url":"https://www.next.co.uk/robots.txt","bytes":4317,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":7505,"domain":"golfdigest.com","origin":"https://golfdigest.com","robots":{"status":"ok","http":200,"url":"https://www.golfdigest.com/robots.txt","bytes":590,"groups":7},"bots":{"GPTBot":["partial","*",6,5],"ChatGPT-User":["partial","*",6,5],"OAI-SearchBot":["partial","*",6,5],"ClaudeBot":["partial","*",6,5],"Claude-User":["partial","*",6,5],"Claude-SearchBot":["partial","*",6,5],"anthropic-ai":["partial","*",6,5],"PerplexityBot":["partial","*",6,5],"Perplexity-User":["partial","*",6,5],"Google-Extended":["partial","*",6,5],"Applebot-Extended":["partial","*",6,5],"CCBot":["partial","*",6,5],"Bytespider":["partial","*",6,5],"Amazonbot":["partial","*",6,5],"meta-externalagent":["partial","*",6,5],"cohere-ai":["partial","*",6,5],"Diffbot":["partial","*",6,5],"ContextFetchBot":["partial","*",6,5]},"llms":{"status":"missing","http":404,"bytes":243755},"home":{"status":"ok","http":200,"url":"https://www.golfdigest.com/","html_tokens":99976,"markdown_tokens":194,"pruned_tokens":148}},
{"rank":7506,"domain":"municode.com","origin":"https://municode.com","robots":{"status":"refused","http":403,"url":"https://municode.com/robots.txt","bytes":5549,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":7507,"domain":"google.iq","origin":"https://google.iq","robots":{"status":"ok","http":200,"url":"https://www.google.iq/robots.txt","bytes":6624,"groups":4},"bots":{"GPTBot":["partial","*",169,64],"ChatGPT-User":["partial","*",169,64],"OAI-SearchBot":["partial","*",169,64],"ClaudeBot":["partial","*",169,64],"Claude-User":["partial","*",169,64],"Claude-SearchBot":["partial","*",169,64],"anthropic-ai":["partial","*",169,64],"PerplexityBot":["partial","*",169,64],"Perplexity-User":["partial","*",169,64],"Google-Extended":["partial","*",169,64],"Applebot-Extended":["partial","*",169,64],"CCBot":["partial","*",169,64],"Bytespider":["partial","*",169,64],"Amazonbot":["partial","*",169,64],"meta-externalagent":["partial","*",169,64],"cohere-ai":["partial","*",169,64],"Diffbot":["partial","*",169,64],"ContextFetchBot":["partial","*",169,64]},"llms":{"status":"missing","http":404,"bytes":1569},"home":{"status":"ok","http":200,"url":"https://www.google.com/","html_tokens":21278,"markdown_tokens":30,"pruned_tokens":30}},
{"rank":7508,"domain":"globalreporting.org","origin":"https://globalreporting.org","robots":{"status":"missing","http":404,"url":"https://globalreporting.org/robots.txt","bytes":0,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":0},"home":{"status":"ok","http":200,"url":"https://globalreporting.org/","html_tokens":22058,"markdown_tokens":107,"pruned_tokens":107}},
{"rank":7509,"domain":"debian.net","origin":"http://debian.net","robots":{"status":"missing","http":404,"url":"https://www.debian.org/robots.txt","bytes":7468,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":7468},"home":{"status":"ok","http":200,"url":"https://www.debian.org/","html_tokens":3966,"markdown_tokens":13,"pruned_tokens":13}},
{"rank":7510,"domain":"kinsta.cloud","origin":"https://kinsta.cloud","robots":{"status":"html","http":200,"url":"https://my.kinsta.com","bytes":3878,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":7511,"domain":"google.ki","origin":"https://google.ki","robots":{"status":"ok","http":200,"url":"https://www.google.ki/robots.txt","bytes":6624,"groups":4},"bots":{"GPTBot":["partial","*",169,64],"ChatGPT-User":["partial","*",169,64],"OAI-SearchBot":["partial","*",169,64],"ClaudeBot":["partial","*",169,64],"Claude-User":["partial","*",169,64],"Claude-SearchBot":["partial","*",169,64],"anthropic-ai":["partial","*",169,64],"PerplexityBot":["partial","*",169,64],"Perplexity-User":["partial","*",169,64],"Google-Extended":["partial","*",169,64],"Applebot-Extended":["partial","*",169,64],"CCBot":["partial","*",169,64],"Bytespider":["partial","*",169,64],"Amazonbot":["partial","*",169,64],"meta-externalagent":["partial","*",169,64],"cohere-ai":["partial","*",169,64],"Diffbot":["partial","*",169,64],"ContextFetchBot":["partial","*",169,64]},"llms":{"status":"missing","http":404,"bytes":1569},"home":{"status":"ok","http":200,"url":"https://www.google.com/","html_tokens":21350,"markdown_tokens":30,"pruned_tokens":30}},
{"rank":7512,"domain":"11alive.com","origin":"https://11alive.com","robots":{"status":"ok","http":200,"url":"https://www.11alive.com/robots.txt","bytes":453,"groups":2},"bots":{"GPTBot":["partial","*",10,0],"ChatGPT-User":["partial","*",10,0],"OAI-SearchBot":["partial","*",10,0],"ClaudeBot":["partial","*",10,0],"Claude-User":["partial","*",10,0],"Claude-SearchBot":["partial","*",10,0],"anthropic-ai":["partial","*",10,0],"PerplexityBot":["partial","*",10,0],"Perplexity-User":["partial","*",10,0],"Google-Extended":["partial","*",10,0],"Applebot-Extended":["partial","*",10,0],"CCBot":["partial","*",10,0],"Bytespider":["partial","*",10,0],"Amazonbot":["partial","*",10,0],"meta-externalagent":["partial","*",10,0],"cohere-ai":["partial","*",10,0],"Diffbot":["partial","*",10,0],"ContextFetchBot":["partial","*",10,0]},"llms":{"status":"missing","http":404,"bytes":6850},"home":{"status":"ok","http":200,"url":"https://www.11alive.com/","html_tokens":100580,"markdown_tokens":1644,"pruned_tokens":168}},
{"rank":7513,"domain":"webpronews.com","origin":"https://webpronews.com","robots":{"status":"refused","http":403,"url":"https://webpronews.com/robots.txt","bytes":5530,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":7514,"domain":"refractor.io","origin":"https://refractor.io","robots":{"status":"ok","http":200,"url":"https://refractor.io/robots.txt","bytes":203,"groups":2},"bots":{"GPTBot":["partial","*",3,0],"ChatGPT-User":["partial","*",3,0],"OAI-SearchBot":["partial","*",3,0],"ClaudeBot":["partial","*",3,0],"Claude-User":["partial","*",3,0],"Claude-SearchBot":["partial","*",3,0],"anthropic-ai":["partial","*",3,0],"PerplexityBot":["partial","*",3,0],"Perplexity-User":["partial","*",3,0],"Google-Extended":["partial","*",3,0],"Applebot-Extended":["partial","*",3,0],"CCBot":["partial","*",3,0],"Bytespider":["partial","*",3,0],"Amazonbot":["partial","*",3,0],"meta-externalagent":["partial","*",3,0],"cohere-ai":["partial","*",3,0],"Diffbot":["partial","*",3,0],"ContextFetchBot":["partial","*",3,0]},"llms":{"status":"missing","http":404,"bytes":64028},"home":{"status":"ok","http":200,"url":"https://refractor.io/","html_tokens":37854,"markdown_tokens":495,"pruned_tokens":137}},
{"rank":7515,"domain":"techadvisor.com","origin":"https://techadvisor.com","robots":{"status":"ok","http":200,"url":"https://www.techadvisor.com/robots.txt","bytes":2400,"groups":37},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["partial","*",1,0],"OAI-SearchBot":["partial","*",1,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",1,0],"Claude-SearchBot":["partial","*",1,0],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["partial","*",1,0],"Perplexity-User":["partial","*",1,0],"Google-Extended":["partial","*",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["partial","*",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",1,0]},"llms":{"status":"missing","http":404,"bytes":208759},"home":{"status":"ok","http":200,"url":"https://www.techadvisor.com","html_tokens":65436,"markdown_tokens":509,"pruned_tokens":169}},
{"rank":7516,"domain":"unizg.hr","origin":"https://www.unizg.hr","robots":{"status":"ok","http":200,"url":"https://www.unizg.hr/robots.txt","bytes":131,"groups":1},"bots":{"GPTBot":["partial","*",3,0],"ChatGPT-User":["partial","*",3,0],"OAI-SearchBot":["partial","*",3,0],"ClaudeBot":["partial","*",3,0],"Claude-User":["partial","*",3,0],"Claude-SearchBot":["partial","*",3,0],"anthropic-ai":["partial","*",3,0],"PerplexityBot":["partial","*",3,0],"Perplexity-User":["partial","*",3,0],"Google-Extended":["partial","*",3,0],"Applebot-Extended":["partial","*",3,0],"CCBot":["partial","*",3,0],"Bytespider":["partial","*",3,0],"Amazonbot":["partial","*",3,0],"meta-externalagent":["partial","*",3,0],"cohere-ai":["partial","*",3,0],"Diffbot":["partial","*",3,0],"ContextFetchBot":["partial","*",3,0]},"llms":{"status":"missing","http":404,"bytes":1414},"home":{"status":"ok","http":200,"url":"https://www.unizg.hr/","html_tokens":49296,"markdown_tokens":634,"pruned_tokens":112}},
{"rank":7517,"domain":"discoverhongkong.com","origin":"https://discoverhongkong.com","robots":{"status":"refused","http":403,"url":"https://www.discoverhongkong.com/robots.txt","bytes":5605,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":7518,"domain":"startrek.com","origin":"https://startrek.com","robots":{"status":"ok","http":200,"url":"https://www.startrek.com/robots.txt","bytes":142,"groups":1},"bots":{"GPTBot":["partial","*",1,1],"ChatGPT-User":["partial","*",1,1],"OAI-SearchBot":["partial","*",1,1],"ClaudeBot":["partial","*",1,1],"Claude-User":["partial","*",1,1],"Claude-SearchBot":["partial","*",1,1],"anthropic-ai":["partial","*",1,1],"PerplexityBot":["partial","*",1,1],"Perplexity-User":["partial","*",1,1],"Google-Extended":["partial","*",1,1],"Applebot-Extended":["partial","*",1,1],"CCBot":["partial","*",1,1],"Bytespider":["partial","*",1,1],"Amazonbot":["partial","*",1,1],"meta-externalagent":["partial","*",1,1],"cohere-ai":["partial","*",1,1],"Diffbot":["partial","*",1,1],"ContextFetchBot":["partial","*",1,1]},"llms":{"status":"missing","http":404,"bytes":883502},"home":{"status":"ok","http":200,"url":"https://www.startrek.com/","html_tokens":441924,"markdown_tokens":44,"pruned_tokens":44}},
{"rank":7519,"domain":"la.us","origin":"https://la.us","robots":{"status":"unreachable","http":null,"url":"https://la.us/robots.txt","bytes":0,"groups":0,"error":"cannot resolve la.us"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":7520,"domain":"xn--b1aew.xn--p1ai","origin":"http://xn--b1aew.xn--p1ai","robots":{"status":"html","http":200,"url":"http://xn--b1aew.xn--p1ai/robots.txt","bytes":389,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":7521,"domain":"jaguar.com","origin":"https://jaguar.com","robots":{"status":"ok","http":200,"url":"https://www.jaguar.com/robots.txt","bytes":193,"groups":3},"bots":{"GPTBot":["partial","*",1,0],"ChatGPT-User":["partial","*",1,0],"OAI-SearchBot":["partial","*",1,0],"ClaudeBot":["partial","*",1,0],"Claude-User":["partial","*",1,0],"Claude-SearchBot":["partial","*",1,0],"anthropic-ai":["partial","*",1,0],"PerplexityBot":["partial","*",1,0],"Perplexity-User":["partial","*",1,0],"Google-Extended":["partial","*",1,0],"Applebot-Extended":["partial","*",1,0],"CCBot":["partial","*",1,0],"Bytespider":["partial","*",1,0],"Amazonbot":["partial","*",1,0],"meta-externalagent":["partial","*",1,0],"cohere-ai":["partial","*",1,0],"Diffbot":["partial","*",1,0],"ContextFetchBot":["partial","*",1,0]},"llms":{"status":"missing","http":404,"bytes":196},"home":{"status":"ok","http":200,"url":"https://www.jaguar.com/en-us/index.html?akam_redirect=true","html_tokens":7473,"markdown_tokens":46,"pruned_tokens":46}},
{"rank":7522,"domain":"reedsy.com","origin":"https://reedsy.com","robots":{"status":"ok","http":200,"url":"https://reedsy.com/robots.txt","bytes":32,"groups":1},"bots":{"GPTBot":["partial","*",1,0],"ChatGPT-User":["partial","*",1,0],"OAI-SearchBot":["partial","*",1,0],"ClaudeBot":["partial","*",1,0],"Claude-User":["partial","*",1,0],"Claude-SearchBot":["partial","*",1,0],"anthropic-ai":["partial","*",1,0],"PerplexityBot":["partial","*",1,0],"Perplexity-User":["partial","*",1,0],"Google-Extended":["partial","*",1,0],"Applebot-Extended":["partial","*",1,0],"CCBot":["partial","*",1,0],"Bytespider":["partial","*",1,0],"Amazonbot":["partial","*",1,0],"meta-externalagent":["partial","*",1,0],"cohere-ai":["partial","*",1,0],"Diffbot":["partial","*",1,0],"ContextFetchBot":["partial","*",1,0]},"llms":{"status":"missing","http":404,"bytes":47027},"home":{"status":"ok","http":200,"url":"https://reedsy.com/","html_tokens":36566,"markdown_tokens":2512,"pruned_tokens":219}},
{"rank":7523,"domain":"dhl.de","origin":"https://dhl.de","robots":{"status":"ok","http":200,"url":"https://www.dhl.de/robots.txt","bytes":1144,"groups":11},"bots":{"GPTBot":["allowed","named",0,1],"ChatGPT-User":["allowed","named",0,1],"OAI-SearchBot":["partial","*",11,0],"ClaudeBot":["allowed","named",0,1],"Claude-User":["partial","*",11,0],"Claude-SearchBot":["partial","*",11,0],"anthropic-ai":["partial","*",11,0],"PerplexityBot":["allowed","named",0,1],"Perplexity-User":["partial","*",11,0],"Google-Extended":["allowed","named",0,1],"Applebot-Extended":["partial","*",11,0],"CCBot":["allowed","named",0,1],"Bytespider":["partial","*",11,0],"Amazonbot":["allowed","named",0,1],"meta-externalagent":["partial","*",11,0],"cohere-ai":["partial","*",11,0],"Diffbot":["partial","*",11,0],"ContextFetchBot":["partial","*",11,0]},"llms":{"status":"missing","http":404,"bytes":14248},"home":{"status":"ok","http":200,"url":"https://www.dhl.de/de/privatkunden.html","html_tokens":50376,"markdown_tokens":196,"pruned_tokens":175}},
{"rank":7524,"domain":"google.st","origin":"https://google.st","robots":{"status":"ok","http":200,"url":"https://www.google.st/robots.txt","bytes":6624,"groups":4},"bots":{"GPTBot":["partial","*",169,64],"ChatGPT-User":["partial","*",169,64],"OAI-SearchBot":["partial","*",169,64],"ClaudeBot":["partial","*",169,64],"Claude-User":["partial","*",169,64],"Claude-SearchBot":["partial","*",169,64],"anthropic-ai":["partial","*",169,64],"PerplexityBot":["partial","*",169,64],"Perplexity-User":["partial","*",169,64],"Google-Extended":["partial","*",169,64],"Applebot-Extended":["partial","*",169,64],"CCBot":["partial","*",169,64],"Bytespider":["partial","*",169,64],"Amazonbot":["partial","*",169,64],"meta-externalagent":["partial","*",169,64],"cohere-ai":["partial","*",169,64],"Diffbot":["partial","*",169,64],"ContextFetchBot":["partial","*",169,64]},"llms":{"status":"missing","http":404,"bytes":1569},"home":{"status":"ok","http":200,"url":"https://www.google.com/","html_tokens":21283,"markdown_tokens":30,"pruned_tokens":30}},
{"rank":7525,"domain":"pace.edu","origin":"https://pace.edu","robots":{"status":"ok","http":200,"url":"https://www.pace.edu/robots.txt","bytes":1832,"groups":1},"bots":{"GPTBot":["partial","*",27,18],"ChatGPT-User":["partial","*",27,18],"OAI-SearchBot":["partial","*",27,18],"ClaudeBot":["partial","*",27,18],"Claude-User":["partial","*",27,18],"Claude-SearchBot":["partial","*",27,18],"anthropic-ai":["partial","*",27,18],"PerplexityBot":["partial","*",27,18],"Perplexity-User":["partial","*",27,18],"Google-Extended":["partial","*",27,18],"Applebot-Extended":["partial","*",27,18],"CCBot":["partial","*",27,18],"Bytespider":["partial","*",27,18],"Amazonbot":["partial","*",27,18],"meta-externalagent":["partial","*",27,18],"cohere-ai":["partial","*",27,18],"Diffbot":["partial","*",27,18],"ContextFetchBot":["partial","*",27,18]},"llms":{"status":"present","http":200,"bytes":10597},"home":{"status":"ok","http":200,"url":"https://www.pace.edu/","html_tokens":51078,"markdown_tokens":431,"pruned_tokens":73}},
{"rank":7526,"domain":"hmzyet.com","origin":"http://hmzyet.com","robots":{"status":"missing","http":404,"url":"http://hmzyet.com/robots.txt","bytes":1231,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":1231},"home":{"status":"ok","http":200,"url":"http://hmzyet.com/","html_tokens":3393,"markdown_tokens":104,"pruned_tokens":33}},
{"rank":7527,"domain":"brisbanetimes.com.au","origin":"https://brisbanetimes.com.au","robots":{"status":"ok","http":200,"url":"https://www.brisbanetimes.com.au/robots.txt","bytes":4127,"groups":52},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["blocked","named",1,0],"OAI-SearchBot":["blocked","named",1,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",5,1],"Claude-SearchBot":["partial","*",5,1],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["partial","*",5,1],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",5,1]},"llms":{"status":"missing","http":404,"bytes":167059},"home":{"status":"ok","http":200,"url":"https://www.brisbanetimes.com.au/","html_tokens":279930,"markdown_tokens":605,"pruned_tokens":115}},
{"rank":7528,"domain":"ruc.edu.cn","origin":"https://www.ruc.edu.cn","robots":{"status":"missing","http":404,"url":"https://www.ruc.edu.cn/robots.txt","bytes":2493,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":2493},"home":{"status":"ok","http":200,"url":"https://www.ruc.edu.cn/","html_tokens":129243,"markdown_tokens":946,"pruned_tokens":53}},
{"rank":7529,"domain":"northropgrumman.com","origin":"https://northropgrumman.com","robots":{"status":"ok","http":200,"url":"https://www.northropgrumman.com/robots.txt","bytes":863,"groups":3},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["allowed","named",0,1],"OAI-SearchBot":["allowed","named",0,1],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",4,1],"Claude-SearchBot":["partial","*",4,1],"anthropic-ai":["partial","*",4,1],"PerplexityBot":["allowed","named",0,1],"Perplexity-User":["partial","*",4,1],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["partial","*",4,1],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["partial","*",4,1],"cohere-ai":["blocked","named",1,0],"Diffbot":["partial","*",4,1],"ContextFetchBot":["partial","*",4,1]},"llms":{"status":"missing","http":404,"bytes":34528},"home":{"status":"ok","http":200,"url":"https://www.northropgrumman.com/","html_tokens":29337,"markdown_tokens":29,"pruned_tokens":29}},
{"rank":7530,"domain":"storeboard.com","origin":"https://storeboard.com","robots":{"status":"ok","http":200,"url":"https://storeboard.com/robots.txt","bytes":3182,"groups":45},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["allowed","*",0,1],"OAI-SearchBot":["blocked","named",1,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["allowed","*",0,1],"Claude-SearchBot":["blocked","named",1,0],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["allowed","*",0,1],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["allowed","*",0,1]},"llms":{"status":"http_error","http":403,"bytes":5507},"home":{"status":"http_error","http":403,"url":"https://www.storeboard.com/"}},
{"rank":7531,"domain":"france.fr","origin":"https://france.fr","robots":{"status":"ok","http":200,"url":"https://www.france.fr/robots.txt","bytes":107,"groups":1},"bots":{"GPTBot":["partial","*",2,0],"ChatGPT-User":["partial","*",2,0],"OAI-SearchBot":["partial","*",2,0],"ClaudeBot":["partial","*",2,0],"Claude-User":["partial","*",2,0],"Claude-SearchBot":["partial","*",2,0],"anthropic-ai":["partial","*",2,0],"PerplexityBot":["partial","*",2,0],"Perplexity-User":["partial","*",2,0],"Google-Extended":["partial","*",2,0],"Applebot-Extended":["partial","*",2,0],"CCBot":["partial","*",2,0],"Bytespider":["partial","*",2,0],"Amazonbot":["partial","*",2,0],"meta-externalagent":["partial","*",2,0],"cohere-ai":["partial","*",2,0],"Diffbot":["partial","*",2,0],"ContextFetchBot":["partial","*",2,0]},"llms":{"status":"missing","http":404,"bytes":159697},"home":{"status":"ok","http":200,"url":"https://www.france.fr/en/","html_tokens":254610,"markdown_tokens":93,"pruned_tokens":59}},
{"rank":7532,"domain":"jwt.io","origin":"https://jwt.io","robots":{"status":"ok","http":200,"url":"https://www.jwt.io/robots.txt","bytes":60,"groups":1},"bots":{"GPTBot":["allowed","*",0,1],"ChatGPT-User":["allowed","*",0,1],"OAI-SearchBot":["allowed","*",0,1],"ClaudeBot":["allowed","*",0,1],"Claude-User":["allowed","*",0,1],"Claude-SearchBot":["allowed","*",0,1],"anthropic-ai":["allowed","*",0,1],"PerplexityBot":["allowed","*",0,1],"Perplexity-User":["allowed","*",0,1],"Google-Extended":["allowed","*",0,1],"Applebot-Extended":["allowed","*",0,1],"CCBot":["allowed","*",0,1],"Bytespider":["allowed","*",0,1],"Amazonbot":["allowed","*",0,1],"meta-externalagent":["allowed","*",0,1],"cohere-ai":["allowed","*",0,1],"Diffbot":["allowed","*",0,1],"ContextFetchBot":["allowed","*",0,1]},"llms":{"status":"present","http":200,"bytes":1163},"home":{"status":"ok","http":200,"url":"https://www.jwt.io/","html_tokens":28496,"markdown_tokens":65,"pruned_tokens":17}},
{"rank":7533,"domain":"joomag.com","origin":"https://joomag.com","robots":{"status":"ok","http":200,"url":"https://www.joomag.com/robots.txt","bytes":302,"groups":1},"bots":{"GPTBot":["partial","*",3,0],"ChatGPT-User":["partial","*",3,0],"OAI-SearchBot":["partial","*",3,0],"ClaudeBot":["partial","*",3,0],"Claude-User":["partial","*",3,0],"Claude-SearchBot":["partial","*",3,0],"anthropic-ai":["partial","*",3,0],"PerplexityBot":["partial","*",3,0],"Perplexity-User":["partial","*",3,0],"Google-Extended":["partial","*",3,0],"Applebot-Extended":["partial","*",3,0],"CCBot":["partial","*",3,0],"Bytespider":["partial","*",3,0],"Amazonbot":["partial","*",3,0],"meta-externalagent":["partial","*",3,0],"cohere-ai":["partial","*",3,0],"Diffbot":["partial","*",3,0],"ContextFetchBot":["partial","*",3,0]},"llms":{"status":"present","http":200,"bytes":7187},"home":{"status":"ok","http":200,"url":"https://www.joomag.com/","html_tokens":15023,"markdown_tokens":1742,"pruned_tokens":373}},
{"rank":7534,"domain":"ifrance.com","origin":"https://ifrance.com","robots":{"status":"unreachable","http":null,"url":"https://ifrance.com/robots.txt","bytes":0,"groups":0,"error":"cannot resolve ifrance.com"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":7535,"domain":"tsite.jp","origin":"https://tsite.jp","robots":{"status":"missing","http":404,"url":"https://tsite.jp/robots.txt","bytes":18384,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":18384},"home":{"status":"ok","http":200,"url":"https://tsite.jp/","html_tokens":9536,"markdown_tokens":68,"pruned_tokens":68}},
{"rank":7536,"domain":"autotrader.com","origin":"https://autotrader.com","robots":{"status":"ok","http":200,"url":"https://www.autotrader.com/robots.txt","bytes":19675,"groups":18},"bots":{"GPTBot":["partial","named",12,1],"ChatGPT-User":["partial","named",12,1],"OAI-SearchBot":["partial","named",12,1],"ClaudeBot":["partial","named",7,0],"Claude-User":["partial","named",11,0],"Claude-SearchBot":["partial","named",11,0],"anthropic-ai":["partial","*",161,1],"PerplexityBot":["partial","named",11,0],"Perplexity-User":["partial","named",11,0],"Google-Extended":["partial","*",161,1],"Applebot-Extended":["partial","*",161,1],"CCBot":["partial","named",11,0],"Bytespider":["partial","*",161,1],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["partial","*",161,1],"cohere-ai":["partial","*",161,1],"Diffbot":["partial","*",161,1],"ContextFetchBot":["partial","*",161,1]},"llms":{"status":"http_error","http":403,"bytes":376},"home":{"status":"http_error","http":403,"url":"https://autotrader.com/"}},
{"rank":7537,"domain":"kommune.no","origin":"https://kommune.no","robots":{"status":"unreachable","http":null,"url":"https://kommune.no/robots.txt","bytes":0,"groups":0,"error":"cannot resolve kommune.no"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":7538,"domain":"csod.com","origin":"https://csod.com","robots":{"status":"ok","http":200,"url":"https://www.cornerstoneondemand.com/robots.txt","bytes":80,"groups":1},"bots":{"GPTBot":["allowed","*",0,1],"ChatGPT-User":["allowed","*",0,1],"OAI-SearchBot":["allowed","*",0,1],"ClaudeBot":["allowed","*",0,1],"Claude-User":["allowed","*",0,1],"Claude-SearchBot":["allowed","*",0,1],"anthropic-ai":["allowed","*",0,1],"PerplexityBot":["allowed","*",0,1],"Perplexity-User":["allowed","*",0,1],"Google-Extended":["allowed","*",0,1],"Applebot-Extended":["allowed","*",0,1],"CCBot":["allowed","*",0,1],"Bytespider":["allowed","*",0,1],"Amazonbot":["allowed","*",0,1],"meta-externalagent":["allowed","*",0,1],"cohere-ai":["allowed","*",0,1],"Diffbot":["allowed","*",0,1],"ContextFetchBot":["allowed","*",0,1]},"llms":{"status":"missing","http":404,"bytes":138176},"home":{"status":"ok","http":200,"url":"https://www.cornerstoneondemand.com/","html_tokens":122549,"markdown_tokens":236,"pruned_tokens":115}},
{"rank":7539,"domain":"vagrantup.com","origin":"https://vagrantup.com","robots":{"status":"html","http":200,"url":"https://developer.hashicorp.com/vagrant","bytes":103634,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":7540,"domain":"howtoforge.com","origin":"https://howtoforge.com","robots":{"status":"ok","http":200,"url":"https://www.howtoforge.com/robots.txt","bytes":79,"groups":1},"bots":{"GPTBot":["allowed","*",0,1],"ChatGPT-User":["allowed","*",0,1],"OAI-SearchBot":["allowed","*",0,1],"ClaudeBot":["allowed","*",0,1],"Claude-User":["allowed","*",0,1],"Claude-SearchBot":["allowed","*",0,1],"anthropic-ai":["allowed","*",0,1],"PerplexityBot":["allowed","*",0,1],"Perplexity-User":["allowed","*",0,1],"Google-Extended":["allowed","*",0,1],"Applebot-Extended":["allowed","*",0,1],"CCBot":["allowed","*",0,1],"Bytespider":["allowed","*",0,1],"Amazonbot":["allowed","*",0,1],"meta-externalagent":["allowed","*",0,1],"cohere-ai":["allowed","*",0,1],"Diffbot":["allowed","*",0,1],"ContextFetchBot":["allowed","*",0,1]},"llms":{"status":"missing","http":404,"bytes":16091},"home":{"status":"http_error","http":403,"url":"https://howtoforge.com/"}},
{"rank":7541,"domain":"crazygames.com","origin":"https://crazygames.com","robots":{"status":"ok","http":200,"url":"https://www.crazygames.com/robots.txt","bytes":255,"groups":2},"bots":{"GPTBot":["partial","*",8,0],"ChatGPT-User":["partial","*",8,0],"OAI-SearchBot":["partial","*",8,0],"ClaudeBot":["partial","*",8,0],"Claude-User":["partial","*",8,0],"Claude-SearchBot":["partial","*",8,0],"anthropic-ai":["partial","*",8,0],"PerplexityBot":["partial","*",8,0],"Perplexity-User":["partial","*",8,0],"Google-Extended":["partial","*",8,0],"Applebot-Extended":["partial","*",8,0],"CCBot":["partial","*",8,0],"Bytespider":["partial","*",8,0],"Amazonbot":["partial","*",8,0],"meta-externalagent":["partial","*",8,0],"cohere-ai":["partial","*",8,0],"Diffbot":["partial","*",8,0],"ContextFetchBot":["partial","*",8,0]},"llms":{"status":"missing","http":404,"bytes":108442},"home":{"status":"ok","http":200,"url":"https://www.crazygames.com/","html_tokens":72555,"markdown_tokens":547,"pruned_tokens":354}},
{"rank":7542,"domain":"keywordtool.io","origin":"https://keywordtool.io","robots":{"status":"ok","http":200,"url":"https://keywordtool.io/robots.txt","bytes":802,"groups":1},"bots":{"GPTBot":["partial","*",11,7],"ChatGPT-User":["partial","*",11,7],"OAI-SearchBot":["partial","*",11,7],"ClaudeBot":["partial","*",11,7],"Claude-User":["partial","*",11,7],"Claude-SearchBot":["partial","*",11,7],"anthropic-ai":["partial","*",11,7],"PerplexityBot":["partial","*",11,7],"Perplexity-User":["partial","*",11,7],"Google-Extended":["partial","*",11,7],"Applebot-Extended":["partial","*",11,7],"CCBot":["partial","*",11,7],"Bytespider":["partial","*",11,7],"Amazonbot":["partial","*",11,7],"meta-externalagent":["partial","*",11,7],"cohere-ai":["partial","*",11,7],"Diffbot":["partial","*",11,7],"ContextFetchBot":["partial","*",11,7]},"llms":{"status":"missing","http":404,"bytes":40177},"home":{"status":"ok","http":200,"url":"https://keywordtool.io/","html_tokens":31913,"markdown_tokens":3571,"pruned_tokens":272}},
{"rank":7543,"domain":"gehealthcare.com","origin":"https://gehealthcare.com","robots":{"status":"ok","http":200,"url":"https://www.gehealthcare.com/robots.txt","bytes":1697,"groups":1},"bots":{"GPTBot":["partial","*",6,0],"ChatGPT-User":["partial","*",6,0],"OAI-SearchBot":["partial","*",6,0],"ClaudeBot":["partial","*",6,0],"Claude-User":["partial","*",6,0],"Claude-SearchBot":["partial","*",6,0],"anthropic-ai":["partial","*",6,0],"PerplexityBot":["partial","*",6,0],"Perplexity-User":["partial","*",6,0],"Google-Extended":["partial","*",6,0],"Applebot-Extended":["partial","*",6,0],"CCBot":["partial","*",6,0],"Bytespider":["partial","*",6,0],"Amazonbot":["partial","*",6,0],"meta-externalagent":["partial","*",6,0],"cohere-ai":["partial","*",6,0],"Diffbot":["partial","*",6,0],"ContextFetchBot":["partial","*",6,0]},"llms":{"status":"http_error","http":403,"bytes":386},"home":{"status":"ok","http":200,"url":"https://www.gehealthcare.com/en-us","html_tokens":121098,"markdown_tokens":96,"pruned_tokens":96}},
{"rank":7544,"domain":"republika.pl","origin":"http://republika.pl","robots":{"status":"html","http":200,"url":"https://www.onet.pl/?utm_source=republika&utm_medium=przekierowanie&utm_campaign=republika_info#zamkniecie-republika/robots.txt","bytes":338908,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":7545,"domain":"mine.nu","origin":"http://mine.nu","robots":{"status":"refused","http":403,"url":"https://www.oracle.com/cloud/networking/dns/","bytes":1449,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":7546,"domain":"caa.co.uk","origin":"https://caa.co.uk","robots":{"status":"missing","http":404,"url":"https://www.caa.co.uk/robots.txt","bytes":20408,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"other_text","http":200,"bytes":8201},"home":{"status":"ok","http":200,"url":"https://www.caa.co.uk/","html_tokens":9087,"markdown_tokens":44,"pruned_tokens":44}},
{"rank":7547,"domain":"thestranger.com","origin":"https://thestranger.com","robots":{"status":"ok","http":200,"url":"https://www.thestranger.com/robots.txt","bytes":550,"groups":2},"bots":{"GPTBot":["partial","*",6,1],"ChatGPT-User":["partial","*",6,1],"OAI-SearchBot":["partial","*",6,1],"ClaudeBot":["partial","*",6,1],"Claude-User":["partial","*",6,1],"Claude-SearchBot":["partial","*",6,1],"anthropic-ai":["partial","*",6,1],"PerplexityBot":["partial","*",6,1],"Perplexity-User":["partial","*",6,1],"Google-Extended":["partial","*",6,1],"Applebot-Extended":["partial","*",6,1],"CCBot":["partial","*",6,1],"Bytespider":["partial","*",6,1],"Amazonbot":["partial","*",6,1],"meta-externalagent":["partial","*",6,1],"cohere-ai":["partial","*",6,1],"Diffbot":["partial","*",6,1],"ContextFetchBot":["partial","*",6,1]},"llms":{"status":"missing","http":404,"bytes":207413},"home":{"status":"ok","http":200,"url":"https://www.thestranger.com/","html_tokens":94953,"markdown_tokens":32,"pruned_tokens":32}},
{"rank":7548,"domain":"berlinale.de","origin":"https://berlinale.de","robots":{"status":"ok","http":200,"url":"https://www.berlinale.de/robots.txt","bytes":1953,"groups":2},"bots":{"GPTBot":["partial","*",21,0],"ChatGPT-User":["partial","*",21,0],"OAI-SearchBot":["partial","*",21,0],"ClaudeBot":["partial","*",21,0],"Claude-User":["partial","*",21,0],"Claude-SearchBot":["partial","*",21,0],"anthropic-ai":["partial","*",21,0],"PerplexityBot":["partial","*",21,0],"Perplexity-User":["partial","*",21,0],"Google-Extended":["partial","*",21,0],"Applebot-Extended":["partial","*",21,0],"CCBot":["partial","*",21,0],"Bytespider":["partial","*",21,0],"Amazonbot":["partial","*",21,0],"meta-externalagent":["partial","*",21,0],"cohere-ai":["partial","*",21,0],"Diffbot":["partial","*",21,0],"ContextFetchBot":["partial","*",21,0]},"llms":{"status":"missing","http":404,"bytes":189082},"home":{"status":"ok","http":200,"url":"https://www.berlinale.de/en/home.html","html_tokens":50522,"markdown_tokens":95,"pruned_tokens":88}},
{"rank":7549,"domain":"greatplacetowork.com","origin":"https://greatplacetowork.com","robots":{"status":"ok","http":200,"url":"https://www.greatplacetowork.com/robots.txt","bytes":955,"groups":2},"bots":{"GPTBot":["partial","*",18,0],"ChatGPT-User":["partial","*",18,0],"OAI-SearchBot":["partial","*",18,0],"ClaudeBot":["partial","*",18,0],"Claude-User":["partial","*",18,0],"Claude-SearchBot":["partial","*",18,0],"anthropic-ai":["partial","*",18,0],"PerplexityBot":["partial","*",18,0],"Perplexity-User":["partial","*",18,0],"Google-Extended":["partial","*",18,0],"Applebot-Extended":["partial","*",18,0],"CCBot":["partial","*",18,0],"Bytespider":["partial","*",18,0],"Amazonbot":["partial","*",18,0],"meta-externalagent":["partial","*",18,0],"cohere-ai":["partial","*",18,0],"Diffbot":["partial","*",18,0],"ContextFetchBot":["partial","*",18,0]},"llms":{"status":"present","http":200,"bytes":23},"home":{"status":"ok","http":200,"url":"https://www.greatplacetowork.com/","html_tokens":30851,"markdown_tokens":109,"pruned_tokens":109}},
{"rank":7550,"domain":"data.gov","origin":"https://data.gov","robots":{"status":"ok","http":200,"url":"https://data.gov/robots.txt","bytes":61,"groups":1},"bots":{"GPTBot":["allowed","*",0,1],"ChatGPT-User":["allowed","*",0,1],"OAI-SearchBot":["allowed","*",0,1],"ClaudeBot":["allowed","*",0,1],"Claude-User":["allowed","*",0,1],"Claude-SearchBot":["allowed","*",0,1],"anthropic-ai":["allowed","*",0,1],"PerplexityBot":["allowed","*",0,1],"Perplexity-User":["allowed","*",0,1],"Google-Extended":["allowed","*",0,1],"Applebot-Extended":["allowed","*",0,1],"CCBot":["allowed","*",0,1],"Bytespider":["allowed","*",0,1],"Amazonbot":["allowed","*",0,1],"meta-externalagent":["allowed","*",0,1],"cohere-ai":["allowed","*",0,1],"Diffbot":["allowed","*",0,1],"ContextFetchBot":["allowed","*",0,1]},"llms":{"status":"missing","http":404,"bytes":111561},"home":{"status":"ok","http":200,"url":"https://data.gov/","html_tokens":30462,"markdown_tokens":163,"pruned_tokens":89}},
{"rank":7551,"domain":"mcsweeneys.net","origin":"https://mcsweeneys.net","robots":{"status":"ok","http":200,"url":"https://mcsweeneys.net/robots.txt","bytes":3300,"groups":46},"bots":{"GPTBot":["blocked","named",2,0],"ChatGPT-User":["partial","*",3,1],"OAI-SearchBot":["partial","*",3,1],"ClaudeBot":["blocked","named",2,0],"Claude-User":["partial","*",3,1],"Claude-SearchBot":["partial","*",3,1],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["partial","*",3,1],"Perplexity-User":["partial","*",3,1],"Google-Extended":["blocked","named",2,0],"Applebot-Extended":["blocked","named",2,0],"CCBot":["blocked","named",2,0],"Bytespider":["blocked","named",2,0],"Amazonbot":["blocked","named",2,0],"meta-externalagent":["blocked","named",2,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",3,1]},"llms":{"status":"missing","http":404,"bytes":20488},"home":{"status":"ok","http":200,"url":"https://www.mcsweeneys.net/","html_tokens":32312,"markdown_tokens":115,"pruned_tokens":115}},
{"rank":7552,"domain":"soas.ac.uk","origin":"https://soas.ac.uk","robots":{"status":"ok","http":200,"url":"https://www.soas.ac.uk/robots.txt","bytes":2027,"groups":1},"bots":{"GPTBot":["partial","*",34,18],"ChatGPT-User":["partial","*",34,18],"OAI-SearchBot":["partial","*",34,18],"ClaudeBot":["partial","*",34,18],"Claude-User":["partial","*",34,18],"Claude-SearchBot":["partial","*",34,18],"anthropic-ai":["partial","*",34,18],"PerplexityBot":["partial","*",34,18],"Perplexity-User":["partial","*",34,18],"Google-Extended":["partial","*",34,18],"Applebot-Extended":["partial","*",34,18],"CCBot":["partial","*",34,18],"Bytespider":["partial","*",34,18],"Amazonbot":["partial","*",34,18],"meta-externalagent":["partial","*",34,18],"cohere-ai":["partial","*",34,18],"Diffbot":["partial","*",34,18],"ContextFetchBot":["partial","*",34,18]},"llms":{"status":"missing","http":404,"bytes":188},"home":{"status":"ok","http":200,"url":"https://www.soas.ac.uk/","html_tokens":51122,"markdown_tokens":91,"pruned_tokens":91}},
{"rank":7553,"domain":"bentley.com","origin":"https://bentley.com","robots":{"status":"ok","http":200,"url":"https://www.bentley.com/robots.txt","bytes":742,"groups":1},"bots":{"GPTBot":["partial","*",28,1],"ChatGPT-User":["partial","*",28,1],"OAI-SearchBot":["partial","*",28,1],"ClaudeBot":["partial","*",28,1],"Claude-User":["partial","*",28,1],"Claude-SearchBot":["partial","*",28,1],"anthropic-ai":["partial","*",28,1],"PerplexityBot":["partial","*",28,1],"Perplexity-User":["partial","*",28,1],"Google-Extended":["partial","*",28,1],"Applebot-Extended":["partial","*",28,1],"CCBot":["partial","*",28,1],"Bytespider":["partial","*",28,1],"Amazonbot":["partial","*",28,1],"meta-externalagent":["partial","*",28,1],"cohere-ai":["partial","*",28,1],"Diffbot":["partial","*",28,1],"ContextFetchBot":["partial","*",28,1]},"llms":{"status":"missing","http":404,"bytes":2707},"home":{"status":"ok","http":200,"url":"https://www.bentley.com/","html_tokens":107436,"markdown_tokens":106,"pruned_tokens":48}},
{"rank":7554,"domain":"sostav.ru","origin":"https://sostav.ru","robots":{"status":"ok","http":200,"url":"https://www.sostav.ru/robots.txt","bytes":2587,"groups":5},"bots":{"GPTBot":["partial","*",50,0],"ChatGPT-User":["partial","*",50,0],"OAI-SearchBot":["partial","*",50,0],"ClaudeBot":["partial","*",50,0],"Claude-User":["partial","*",50,0],"Claude-SearchBot":["partial","*",50,0],"anthropic-ai":["partial","*",50,0],"PerplexityBot":["partial","*",50,0],"Perplexity-User":["partial","*",50,0],"Google-Extended":["partial","*",50,0],"Applebot-Extended":["partial","*",50,0],"CCBot":["partial","*",50,0],"Bytespider":["partial","*",50,0],"Amazonbot":["partial","*",50,0],"meta-externalagent":["partial","*",50,0],"cohere-ai":["partial","*",50,0],"Diffbot":["partial","*",50,0],"ContextFetchBot":["partial","*",50,0]},"llms":{"status":"missing","http":404,"bytes":36431},"home":{"status":"ok","http":200,"url":"https://www.sostav.ru/","html_tokens":37114,"markdown_tokens":50,"pruned_tokens":50}},
{"rank":7555,"domain":"all-inkl.com","origin":"https://all-inkl.com","robots":{"status":"ok","http":200,"url":"https://all-inkl.com/robots.txt","bytes":368,"groups":1},"bots":{"GPTBot":["partial","*",13,0],"ChatGPT-User":["partial","*",13,0],"OAI-SearchBot":["partial","*",13,0],"ClaudeBot":["partial","*",13,0],"Claude-User":["partial","*",13,0],"Claude-SearchBot":["partial","*",13,0],"anthropic-ai":["partial","*",13,0],"PerplexityBot":["partial","*",13,0],"Perplexity-User":["partial","*",13,0],"Google-Extended":["partial","*",13,0],"Applebot-Extended":["partial","*",13,0],"CCBot":["partial","*",13,0],"Bytespider":["partial","*",13,0],"Amazonbot":["partial","*",13,0],"meta-externalagent":["partial","*",13,0],"cohere-ai":["partial","*",13,0],"Diffbot":["partial","*",13,0],"ContextFetchBot":["partial","*",13,0]},"llms":{"status":"missing","http":404,"bytes":33997},"home":{"status":"ok","http":200,"url":"https://all-inkl.com/","html_tokens":15336,"markdown_tokens":42,"pruned_tokens":42}},
{"rank":7556,"domain":"gobiernodecanarias.org","origin":"https://gobiernodecanarias.org","robots":{"status":"ok","http":200,"url":"https://www.gobiernodecanarias.org/robots.txt","bytes":33910,"groups":1},"bots":{"GPTBot":["partial","*",718,0],"ChatGPT-User":["partial","*",718,0],"OAI-SearchBot":["partial","*",718,0],"ClaudeBot":["partial","*",718,0],"Claude-User":["partial","*",718,0],"Claude-SearchBot":["partial","*",718,0],"anthropic-ai":["partial","*",718,0],"PerplexityBot":["partial","*",718,0],"Perplexity-User":["partial","*",718,0],"Google-Extended":["partial","*",718,0],"Applebot-Extended":["partial","*",718,0],"CCBot":["partial","*",718,0],"Bytespider":["partial","*",718,0],"Amazonbot":["partial","*",718,0],"meta-externalagent":["partial","*",718,0],"cohere-ai":["partial","*",718,0],"Diffbot":["partial","*",718,0],"ContextFetchBot":["partial","*",718,0]},"llms":{"status":"missing","http":404,"bytes":9236},"home":{"status":"ok","http":200,"url":"https://www.gobiernodecanarias.org/principal/","html_tokens":16799,"markdown_tokens":162,"pruned_tokens":162}},
{"rank":7557,"domain":"dowjones.com","origin":"https://dowjones.com","robots":{"status":"ok","http":200,"url":"https://www.dowjones.com/robots.txt","bytes":174,"groups":1},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"missing","http":404,"bytes":165588},"home":{"status":"ok","http":200,"url":"https://www.dowjones.com/","html_tokens":52534,"markdown_tokens":994,"pruned_tokens":36}},
{"rank":7558,"domain":"annunciogratis.net","origin":"https://annunciogratis.net","robots":{"status":"ok","http":200,"url":"https://www.annunciogratis.net/robots.txt","bytes":484,"groups":1},"bots":{"GPTBot":["partial","*",5,4],"ChatGPT-User":["partial","*",5,4],"OAI-SearchBot":["partial","*",5,4],"ClaudeBot":["partial","*",5,4],"Claude-User":["partial","*",5,4],"Claude-SearchBot":["partial","*",5,4],"anthropic-ai":["partial","*",5,4],"PerplexityBot":["partial","*",5,4],"Perplexity-User":["partial","*",5,4],"Google-Extended":["partial","*",5,4],"Applebot-Extended":["partial","*",5,4],"CCBot":["partial","*",5,4],"Bytespider":["partial","*",5,4],"Amazonbot":["partial","*",5,4],"meta-externalagent":["partial","*",5,4],"cohere-ai":["partial","*",5,4],"Diffbot":["partial","*",5,4],"ContextFetchBot":["partial","*",5,4]},"llms":{"status":"html","http":200,"bytes":107352},"home":{"status":"ok","http":200,"url":"https://www.annunciogratis.net/","html_tokens":26929,"markdown_tokens":434,"pruned_tokens":53}},
{"rank":7559,"domain":"sizzlejs.com","origin":"https://sizzlejs.com","robots":{"status":"html","http":200,"url":"https://github.com/jquery/sizzle/wiki","bytes":247452,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":7560,"domain":"chambers.com","origin":"https://chambers.com","robots":{"status":"ok","http":200,"url":"https://chambers.com/robots.txt","bytes":5417,"groups":142},"bots":{"GPTBot":["partial","*",5,1],"ChatGPT-User":["partial","*",5,1],"OAI-SearchBot":["partial","*",5,1],"ClaudeBot":["partial","*",5,1],"Claude-User":["partial","*",5,1],"Claude-SearchBot":["partial","*",5,1],"anthropic-ai":["partial","*",5,1],"PerplexityBot":["partial","*",5,1],"Perplexity-User":["partial","*",5,1],"Google-Extended":["partial","*",5,1],"Applebot-Extended":["partial","*",5,1],"CCBot":["partial","*",5,1],"Bytespider":["partial","*",5,1],"Amazonbot":["partial","*",5,1],"meta-externalagent":["partial","*",5,1],"cohere-ai":["partial","*",5,1],"Diffbot":["partial","*",5,1],"ContextFetchBot":["partial","*",5,1]},"llms":{"status":"missing","http":404,"bytes":121377},"home":{"status":"ok","http":200,"url":"https://chambers.com/","html_tokens":111768,"markdown_tokens":118,"pruned_tokens":67}},
{"rank":7561,"domain":"boisestate.edu","origin":"https://boisestate.edu","robots":{"status":"refused","http":403,"url":"https://www.boisestate.edu/robots.txt","bytes":560,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":7562,"domain":"whitney.org","origin":"https://whitney.org","robots":{"status":"ok","http":200,"url":"https://whitney.org/robots.txt","bytes":314,"groups":1},"bots":{"GPTBot":["partial","*",6,0],"ChatGPT-User":["partial","*",6,0],"OAI-SearchBot":["partial","*",6,0],"ClaudeBot":["partial","*",6,0],"Claude-User":["partial","*",6,0],"Claude-SearchBot":["partial","*",6,0],"anthropic-ai":["partial","*",6,0],"PerplexityBot":["partial","*",6,0],"Perplexity-User":["partial","*",6,0],"Google-Extended":["partial","*",6,0],"Applebot-Extended":["partial","*",6,0],"CCBot":["partial","*",6,0],"Bytespider":["partial","*",6,0],"Amazonbot":["partial","*",6,0],"meta-externalagent":["partial","*",6,0],"cohere-ai":["partial","*",6,0],"Diffbot":["partial","*",6,0],"ContextFetchBot":["partial","*",6,0]},"llms":{"status":"missing","http":404,"bytes":52092},"home":{"status":"ok","http":200,"url":"https://whitney.org/","html_tokens":22687,"markdown_tokens":179,"pruned_tokens":50}},
{"rank":7563,"domain":"blogspot.ch","origin":"https://blogspot.ch","robots":{"status":"ok","http":200,"url":"https://blogspot.ch/robots.txt","bytes":6624,"groups":4},"bots":{"GPTBot":["partial","*",169,64],"ChatGPT-User":["partial","*",169,64],"OAI-SearchBot":["partial","*",169,64],"ClaudeBot":["partial","*",169,64],"Claude-User":["partial","*",169,64],"Claude-SearchBot":["partial","*",169,64],"anthropic-ai":["partial","*",169,64],"PerplexityBot":["partial","*",169,64],"Perplexity-User":["partial","*",169,64],"Google-Extended":["partial","*",169,64],"Applebot-Extended":["partial","*",169,64],"CCBot":["partial","*",169,64],"Bytespider":["partial","*",169,64],"Amazonbot":["partial","*",169,64],"meta-externalagent":["partial","*",169,64],"cohere-ai":["partial","*",169,64],"Diffbot":["partial","*",169,64],"ContextFetchBot":["partial","*",169,64]},"llms":{"status":"missing","http":404,"bytes":1569},"home":{"status":"ok","http":200,"url":"https://www.google.com/","html_tokens":21319,"markdown_tokens":30,"pruned_tokens":30}},
{"rank":7564,"domain":"api.org","origin":"https://api.org","robots":{"status":"ok","http":200,"url":"https://www.api.org/robots.txt","bytes":1025,"groups":13},"bots":{"GPTBot":["allowed","named",0,1],"ChatGPT-User":["allowed","named",0,1],"OAI-SearchBot":["allowed","*",0,1],"ClaudeBot":["allowed","named",0,1],"Claude-User":["allowed","*",0,1],"Claude-SearchBot":["allowed","*",0,1],"anthropic-ai":["allowed","*",0,1],"PerplexityBot":["allowed","named",0,1],"Perplexity-User":["allowed","*",0,1],"Google-Extended":["allowed","named",0,1],"Applebot-Extended":["allowed","named",0,1],"CCBot":["partial","named",11,1],"Bytespider":["blocked","named",1,0],"Amazonbot":["allowed","named",0,1],"meta-externalagent":["allowed","*",0,1],"cohere-ai":["allowed","*",0,1],"Diffbot":["allowed","*",0,1],"ContextFetchBot":["allowed","*",0,1]},"llms":{"status":"present","http":200,"bytes":2950},"home":{"status":"ok","http":200,"url":"https://www.api.org/","html_tokens":14654,"markdown_tokens":475,"pruned_tokens":114}},
{"rank":7565,"domain":"softether.org","origin":"https://www.softether.org","robots":{"status":"ok","http":200,"url":"https://www.softether.org/robots.txt","bytes":27,"groups":1},"bots":{"GPTBot":["allowed","*",0,1],"ChatGPT-User":["allowed","*",0,1],"OAI-SearchBot":["allowed","*",0,1],"ClaudeBot":["allowed","*",0,1],"Claude-User":["allowed","*",0,1],"Claude-SearchBot":["allowed","*",0,1],"anthropic-ai":["allowed","*",0,1],"PerplexityBot":["allowed","*",0,1],"Perplexity-User":["allowed","*",0,1],"Google-Extended":["allowed","*",0,1],"Applebot-Extended":["allowed","*",0,1],"CCBot":["allowed","*",0,1],"Bytespider":["allowed","*",0,1],"Amazonbot":["allowed","*",0,1],"meta-externalagent":["allowed","*",0,1],"cohere-ai":["allowed","*",0,1],"Diffbot":["allowed","*",0,1],"ContextFetchBot":["allowed","*",0,1]},"llms":{"status":"html","http":200,"bytes":73349},"home":{"status":"ok","http":200,"url":"https://www.softether.org/","html_tokens":18333,"markdown_tokens":3866,"pruned_tokens":519}},
{"rank":7566,"domain":"elle.fr","origin":"https://elle.fr","robots":{"status":"ok","http":200,"url":"https://www.elle.fr/robots.txt","bytes":4974,"groups":4},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["partial","*",58,0],"OAI-SearchBot":["partial","*",58,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",58,0],"Claude-SearchBot":["partial","*",58,0],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["partial","*",58,0],"Perplexity-User":["partial","*",58,0],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",58,0]},"llms":{"status":"http_error","http":403,"bytes":919},"home":{"status":"http_error","http":403,"url":"https://elle.fr/"}},
{"rank":7567,"domain":"cntraveller.com","origin":"https://cntraveller.com","robots":{"status":"ok","http":200,"url":"https://www.cntraveller.com/robots.txt","bytes":2084,"groups":3},"bots":{"GPTBot":["partial","*",13,12],"ChatGPT-User":["partial","*",13,12],"OAI-SearchBot":["partial","*",13,12],"ClaudeBot":["blocked","named",1,0],"Claude-User":["blocked","named",1,0],"Claude-SearchBot":["blocked","named",1,0],"anthropic-ai":["partial","*",13,12],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["blocked","named",1,0],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",13,12]},"llms":{"status":"missing","http":404,"bytes":745225},"home":{"status":"ok","http":200,"url":"https://www.cntraveller.com/","html_tokens":483348,"markdown_tokens":1927,"pruned_tokens":174}},
{"rank":7568,"domain":"mailgun.com","origin":"https://mailgun.com","robots":{"status":"ok","http":200,"url":"https://www.mailgun.com/robots.txt","bytes":189,"groups":1},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"http_error","http":403,"bytes":5542},"home":{"status":"http_error","http":403,"url":"https://mailgun.com/"}},
{"rank":7569,"domain":"google.cf","origin":"https://google.cf","robots":{"status":"ok","http":200,"url":"https://www.google.cf/robots.txt","bytes":6624,"groups":4},"bots":{"GPTBot":["partial","*",169,64],"ChatGPT-User":["partial","*",169,64],"OAI-SearchBot":["partial","*",169,64],"ClaudeBot":["partial","*",169,64],"Claude-User":["partial","*",169,64],"Claude-SearchBot":["partial","*",169,64],"anthropic-ai":["partial","*",169,64],"PerplexityBot":["partial","*",169,64],"Perplexity-User":["partial","*",169,64],"Google-Extended":["partial","*",169,64],"Applebot-Extended":["partial","*",169,64],"CCBot":["partial","*",169,64],"Bytespider":["partial","*",169,64],"Amazonbot":["partial","*",169,64],"meta-externalagent":["partial","*",169,64],"cohere-ai":["partial","*",169,64],"Diffbot":["partial","*",169,64],"ContextFetchBot":["partial","*",169,64]},"llms":{"status":"missing","http":404,"bytes":1569},"home":{"status":"ok","http":200,"url":"https://www.google.com/","html_tokens":21283,"markdown_tokens":30,"pruned_tokens":30}},
{"rank":7570,"domain":"ksat.com","origin":"https://ksat.com","robots":{"status":"ok","http":200,"url":"https://www.ksat.com/robots.txt","bytes":1139,"groups":9},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["partial","*",5,0],"OAI-SearchBot":["partial","*",5,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",5,0],"Claude-SearchBot":["partial","*",5,0],"anthropic-ai":["partial","*",5,0],"PerplexityBot":["partial","*",5,0],"Perplexity-User":["partial","*",5,0],"Google-Extended":["partial","*",5,0],"Applebot-Extended":["partial","*",5,0],"CCBot":["partial","*",5,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["partial","*",5,0],"cohere-ai":["partial","*",5,0],"Diffbot":["partial","*",5,0],"ContextFetchBot":["partial","*",5,0]},"llms":{"status":"missing","http":404,"bytes":164770},"home":{"status":"ok","http":200,"url":"https://www.ksat.com/","html_tokens":211101,"markdown_tokens":291,"pruned_tokens":70}},
{"rank":7571,"domain":"leprogres.fr","origin":"https://www.leprogres.fr","robots":{"status":"ok","http":200,"url":"https://www.leprogres.fr/robots.txt","bytes":4420,"groups":11},"bots":{"GPTBot":["partial","named",1,2],"ChatGPT-User":["partial","named",1,2],"OAI-SearchBot":["partial","*",33,0],"ClaudeBot":["partial","*",33,0],"Claude-User":["partial","*",33,0],"Claude-SearchBot":["partial","*",33,0],"anthropic-ai":["partial","named",1,2],"PerplexityBot":["partial","*",33,0],"Perplexity-User":["partial","*",33,0],"Google-Extended":["partial","named",1,2],"Applebot-Extended":["partial","*",33,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["partial","named",1,2],"meta-externalagent":["partial","*",33,0],"cohere-ai":["partial","*",33,0],"Diffbot":["partial","*",33,0],"ContextFetchBot":["partial","*",33,0]},"llms":{"status":"missing","http":404,"bytes":4983},"home":{"status":"ok","http":200,"url":"https://www.leprogres.fr/","html_tokens":231948,"markdown_tokens":251,"pruned_tokens":120}},
{"rank":7572,"domain":"expatica.com","origin":"https://expatica.com","robots":{"status":"ok","http":200,"url":"https://www.expatica.com/robots.txt","bytes":1115,"groups":1},"bots":{"GPTBot":["partial","*",29,21],"ChatGPT-User":["partial","*",29,21],"OAI-SearchBot":["partial","*",29,21],"ClaudeBot":["partial","*",29,21],"Claude-User":["partial","*",29,21],"Claude-SearchBot":["partial","*",29,21],"anthropic-ai":["partial","*",29,21],"PerplexityBot":["partial","*",29,21],"Perplexity-User":["partial","*",29,21],"Google-Extended":["partial","*",29,21],"Applebot-Extended":["partial","*",29,21],"CCBot":["partial","*",29,21],"Bytespider":["partial","*",29,21],"Amazonbot":["partial","*",29,21],"meta-externalagent":["partial","*",29,21],"cohere-ai":["partial","*",29,21],"Diffbot":["partial","*",29,21],"ContextFetchBot":["partial","*",29,21]},"llms":{"status":"missing","http":404,"bytes":277035},"home":{"status":"ok","http":200,"url":"https://www.expatica.com/","html_tokens":87892,"markdown_tokens":24,"pruned_tokens":24}},
{"rank":7573,"domain":"a1.net","origin":"https://a1.net","robots":{"status":"ok","http":200,"url":"https://www.a1.net/robots.txt","bytes":3913,"groups":44},"bots":{"GPTBot":["allowed","named",0,1],"ChatGPT-User":["allowed","named",0,1],"OAI-SearchBot":["allowed","named",0,1],"ClaudeBot":["partial","*",56,0],"Claude-User":["partial","*",56,0],"Claude-SearchBot":["allowed","named",0,1],"anthropic-ai":["partial","*",56,0],"PerplexityBot":["allowed","named",0,1],"Perplexity-User":["partial","*",56,0],"Google-Extended":["partial","*",56,0],"Applebot-Extended":["partial","*",56,0],"CCBot":["allowed","named",0,1],"Bytespider":["allowed","named",0,1],"Amazonbot":["allowed","named",0,1],"meta-externalagent":["partial","*",56,0],"cohere-ai":["partial","*",56,0],"Diffbot":["partial","*",56,0],"ContextFetchBot":["partial","*",56,0]},"llms":{"status":"missing","http":404,"bytes":179764},"home":{"status":"ok","http":200,"url":"https://www.a1.net/","html_tokens":59531,"markdown_tokens":369,"pruned_tokens":170}},
{"rank":7574,"domain":"liv.ac.uk","origin":"https://liv.ac.uk","robots":{"status":"ok","http":200,"url":"https://www.liverpool.ac.uk/robots.txt","bytes":769,"groups":3},"bots":{"GPTBot":["partial","*",10,0],"ChatGPT-User":["partial","*",10,0],"OAI-SearchBot":["partial","*",10,0],"ClaudeBot":["partial","*",10,0],"Claude-User":["partial","*",10,0],"Claude-SearchBot":["partial","*",10,0],"anthropic-ai":["partial","*",10,0],"PerplexityBot":["partial","*",10,0],"Perplexity-User":["partial","*",10,0],"Google-Extended":["partial","*",10,0],"Applebot-Extended":["partial","*",10,0],"CCBot":["partial","*",10,0],"Bytespider":["partial","*",10,0],"Amazonbot":["partial","*",10,0],"meta-externalagent":["partial","*",10,0],"cohere-ai":["partial","*",10,0],"Diffbot":["partial","*",10,0],"ContextFetchBot":["partial","*",10,0]},"llms":{"status":"missing","http":404,"bytes":28235},"home":{"status":"ok","http":200,"url":"https://www.liverpool.ac.uk/","html_tokens":25258,"markdown_tokens":128,"pruned_tokens":128}},
{"rank":7575,"domain":"cqc.org.uk","origin":"https://cqc.org.uk","robots":{"status":"ok","http":200,"url":"https://www.cqc.org.uk/robots.txt","bytes":1594,"groups":1},"bots":{"GPTBot":["partial","*",22,18],"ChatGPT-User":["partial","*",22,18],"OAI-SearchBot":["partial","*",22,18],"ClaudeBot":["partial","*",22,18],"Claude-User":["partial","*",22,18],"Claude-SearchBot":["partial","*",22,18],"anthropic-ai":["partial","*",22,18],"PerplexityBot":["partial","*",22,18],"Perplexity-User":["partial","*",22,18],"Google-Extended":["partial","*",22,18],"Applebot-Extended":["partial","*",22,18],"CCBot":["partial","*",22,18],"Bytespider":["partial","*",22,18],"Amazonbot":["partial","*",22,18],"meta-externalagent":["partial","*",22,18],"cohere-ai":["partial","*",22,18],"Diffbot":["partial","*",22,18],"ContextFetchBot":["partial","*",22,18]},"llms":{"status":"missing","http":404,"bytes":63004},"home":{"status":"ok","http":200,"url":"https://www.cqc.org.uk/","html_tokens":24182,"markdown_tokens":180,"pruned_tokens":38}},
{"rank":7576,"domain":"google.com.gh","origin":"https://google.com.gh","robots":{"status":"ok","http":200,"url":"https://www.google.com.gh/robots.txt","bytes":6624,"groups":4},"bots":{"GPTBot":["partial","*",169,64],"ChatGPT-User":["partial","*",169,64],"OAI-SearchBot":["partial","*",169,64],"ClaudeBot":["partial","*",169,64],"Claude-User":["partial","*",169,64],"Claude-SearchBot":["partial","*",169,64],"anthropic-ai":["partial","*",169,64],"PerplexityBot":["partial","*",169,64],"Perplexity-User":["partial","*",169,64],"Google-Extended":["partial","*",169,64],"Applebot-Extended":["partial","*",169,64],"CCBot":["partial","*",169,64],"Bytespider":["partial","*",169,64],"Amazonbot":["partial","*",169,64],"meta-externalagent":["partial","*",169,64],"cohere-ai":["partial","*",169,64],"Diffbot":["partial","*",169,64],"ContextFetchBot":["partial","*",169,64]},"llms":{"status":"missing","http":404,"bytes":1569},"home":{"status":"ok","http":200,"url":"https://www.google.com/","html_tokens":21358,"markdown_tokens":30,"pruned_tokens":30}},
{"rank":7577,"domain":"google.ml","origin":"https://google.ml","robots":{"status":"ok","http":200,"url":"https://www.google.ml/robots.txt","bytes":6624,"groups":4},"bots":{"GPTBot":["partial","*",169,64],"ChatGPT-User":["partial","*",169,64],"OAI-SearchBot":["partial","*",169,64],"ClaudeBot":["partial","*",169,64],"Claude-User":["partial","*",169,64],"Claude-SearchBot":["partial","*",169,64],"anthropic-ai":["partial","*",169,64],"PerplexityBot":["partial","*",169,64],"Perplexity-User":["partial","*",169,64],"Google-Extended":["partial","*",169,64],"Applebot-Extended":["partial","*",169,64],"CCBot":["partial","*",169,64],"Bytespider":["partial","*",169,64],"Amazonbot":["partial","*",169,64],"meta-externalagent":["partial","*",169,64],"cohere-ai":["partial","*",169,64],"Diffbot":["partial","*",169,64],"ContextFetchBot":["partial","*",169,64]},"llms":{"status":"missing","http":404,"bytes":1569},"home":{"status":"ok","http":200,"url":"https://www.google.com/","html_tokens":21328,"markdown_tokens":30,"pruned_tokens":30}},
{"rank":7578,"domain":"nationmaster.com","origin":"https://nationmaster.com","robots":{"status":"ok","http":200,"url":"https://www.nationmaster.com/robots.txt","bytes":56,"groups":1},"bots":{"GPTBot":["partial","*",2,0],"ChatGPT-User":["partial","*",2,0],"OAI-SearchBot":["partial","*",2,0],"ClaudeBot":["partial","*",2,0],"Claude-User":["partial","*",2,0],"Claude-SearchBot":["partial","*",2,0],"anthropic-ai":["partial","*",2,0],"PerplexityBot":["partial","*",2,0],"Perplexity-User":["partial","*",2,0],"Google-Extended":["partial","*",2,0],"Applebot-Extended":["partial","*",2,0],"CCBot":["partial","*",2,0],"Bytespider":["partial","*",2,0],"Amazonbot":["partial","*",2,0],"meta-externalagent":["partial","*",2,0],"cohere-ai":["partial","*",2,0],"Diffbot":["partial","*",2,0],"ContextFetchBot":["partial","*",2,0]},"llms":{"status":"missing","http":404,"bytes":46864},"home":{"status":"ok","http":200,"url":"https://www.nationmaster.com/","html_tokens":10092,"markdown_tokens":167,"pruned_tokens":44}},
{"rank":7579,"domain":"whois.com","origin":"https://whois.com","robots":{"status":"ok","http":200,"url":"https://www.whois.com/robots.txt","bytes":363,"groups":6},"bots":{"GPTBot":["partial","*",6,0],"ChatGPT-User":["partial","*",6,0],"OAI-SearchBot":["partial","*",6,0],"ClaudeBot":["partial","*",6,0],"Claude-User":["partial","*",6,0],"Claude-SearchBot":["partial","*",6,0],"anthropic-ai":["partial","*",6,0],"PerplexityBot":["partial","*",6,0],"Perplexity-User":["partial","*",6,0],"Google-Extended":["partial","*",6,0],"Applebot-Extended":["partial","*",6,0],"CCBot":["partial","*",6,0],"Bytespider":["partial","*",6,0],"Amazonbot":["partial","*",6,0],"meta-externalagent":["partial","*",6,0],"cohere-ai":["partial","*",6,0],"Diffbot":["partial","*",6,0],"ContextFetchBot":["partial","*",6,0]},"llms":{"status":"missing","http":404,"bytes":1359},"home":{"status":"ok","http":200,"url":"https://www.whois.com/","html_tokens":6826,"markdown_tokens":50,"pruned_tokens":50}},
{"rank":7580,"domain":"swift.com","origin":"https://swift.com","robots":{"status":"refused","http":403,"url":"https://swift.com/robots.txt","bytes":5254,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":7581,"domain":"imweb.me","origin":"https://imweb.me","robots":{"status":"ok","http":200,"url":"https://imweb.me/robots.txt","bytes":198,"groups":1},"bots":{"GPTBot":["allowed","*",0,1],"ChatGPT-User":["allowed","*",0,1],"OAI-SearchBot":["allowed","*",0,1],"ClaudeBot":["allowed","*",0,1],"Claude-User":["allowed","*",0,1],"Claude-SearchBot":["allowed","*",0,1],"anthropic-ai":["allowed","*",0,1],"PerplexityBot":["allowed","*",0,1],"Perplexity-User":["allowed","*",0,1],"Google-Extended":["allowed","*",0,1],"Applebot-Extended":["allowed","*",0,1],"CCBot":["allowed","*",0,1],"Bytespider":["allowed","*",0,1],"Amazonbot":["allowed","*",0,1],"meta-externalagent":["allowed","*",0,1],"cohere-ai":["allowed","*",0,1],"Diffbot":["allowed","*",0,1],"ContextFetchBot":["allowed","*",0,1]},"llms":{"status":"missing","http":404,"bytes":2874},"home":{"status":"ok","http":200,"url":"https://imweb.me/","html_tokens":38933,"markdown_tokens":155,"pruned_tokens":155}},
{"rank":7582,"domain":"google.tn","origin":"https://google.tn","robots":{"status":"ok","http":200,"url":"https://www.google.tn/robots.txt","bytes":6624,"groups":4},"bots":{"GPTBot":["partial","*",169,64],"ChatGPT-User":["partial","*",169,64],"OAI-SearchBot":["partial","*",169,64],"ClaudeBot":["partial","*",169,64],"Claude-User":["partial","*",169,64],"Claude-SearchBot":["partial","*",169,64],"anthropic-ai":["partial","*",169,64],"PerplexityBot":["partial","*",169,64],"Perplexity-User":["partial","*",169,64],"Google-Extended":["partial","*",169,64],"Applebot-Extended":["partial","*",169,64],"CCBot":["partial","*",169,64],"Bytespider":["partial","*",169,64],"Amazonbot":["partial","*",169,64],"meta-externalagent":["partial","*",169,64],"cohere-ai":["partial","*",169,64],"Diffbot":["partial","*",169,64],"ContextFetchBot":["partial","*",169,64]},"llms":{"status":"missing","http":404,"bytes":1569},"home":{"status":"ok","http":200,"url":"https://www.google.com/","html_tokens":21442,"markdown_tokens":30,"pruned_tokens":30}},
{"rank":7583,"domain":"sildemedx.com","origin":"https://sildemedx.com","robots":{"status":"ok","http":200,"url":"https://sildemedx.com/robots.txt","bytes":76,"groups":1},"bots":{"GPTBot":["partial","*",1,0],"ChatGPT-User":["partial","*",1,0],"OAI-SearchBot":["partial","*",1,0],"ClaudeBot":["partial","*",1,0],"Claude-User":["partial","*",1,0],"Claude-SearchBot":["partial","*",1,0],"anthropic-ai":["partial","*",1,0],"PerplexityBot":["partial","*",1,0],"Perplexity-User":["partial","*",1,0],"Google-Extended":["partial","*",1,0],"Applebot-Extended":["partial","*",1,0],"CCBot":["partial","*",1,0],"Bytespider":["partial","*",1,0],"Amazonbot":["partial","*",1,0],"meta-externalagent":["partial","*",1,0],"cohere-ai":["partial","*",1,0],"Diffbot":["partial","*",1,0],"ContextFetchBot":["partial","*",1,0]},"llms":{"status":"missing","http":404,"bytes":642},"home":{"status":"ok","http":200,"url":"https://sildemedx.com/","html_tokens":11880,"markdown_tokens":3660,"pruned_tokens":158}},
{"rank":7584,"domain":"bravehost.com","origin":"https://bravehost.com","robots":{"status":"missing","http":404,"url":"https://bravehost.com/robots.txt","bytes":8839,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":8839},"home":{"status":"ok","http":200,"url":"https://www.bravenet.com/webhosting/","html_tokens":10551,"markdown_tokens":542,"pruned_tokens":148}},
{"rank":7585,"domain":"google.pn","origin":"https://google.pn","robots":{"status":"ok","http":200,"url":"https://www.google.pn/robots.txt","bytes":6624,"groups":4},"bots":{"GPTBot":["partial","*",169,64],"ChatGPT-User":["partial","*",169,64],"OAI-SearchBot":["partial","*",169,64],"ClaudeBot":["partial","*",169,64],"Claude-User":["partial","*",169,64],"Claude-SearchBot":["partial","*",169,64],"anthropic-ai":["partial","*",169,64],"PerplexityBot":["partial","*",169,64],"Perplexity-User":["partial","*",169,64],"Google-Extended":["partial","*",169,64],"Applebot-Extended":["partial","*",169,64],"CCBot":["partial","*",169,64],"Bytespider":["partial","*",169,64],"Amazonbot":["partial","*",169,64],"meta-externalagent":["partial","*",169,64],"cohere-ai":["partial","*",169,64],"Diffbot":["partial","*",169,64],"ContextFetchBot":["partial","*",169,64]},"llms":{"status":"missing","http":404,"bytes":1569},"home":{"status":"ok","http":200,"url":"https://www.google.com/","html_tokens":21308,"markdown_tokens":30,"pruned_tokens":30}},
{"rank":7586,"domain":"vulkanvegaswin.net","origin":"https://vulkanvegaswin.net","robots":{"status":"ok","http":200,"url":"https://vulkanvegaswin.net/robots.txt","bytes":72,"groups":1},"bots":{"GPTBot":["allowed","*",0,1],"ChatGPT-User":["allowed","*",0,1],"OAI-SearchBot":["allowed","*",0,1],"ClaudeBot":["allowed","*",0,1],"Claude-User":["allowed","*",0,1],"Claude-SearchBot":["allowed","*",0,1],"anthropic-ai":["allowed","*",0,1],"PerplexityBot":["allowed","*",0,1],"Perplexity-User":["allowed","*",0,1],"Google-Extended":["allowed","*",0,1],"Applebot-Extended":["allowed","*",0,1],"CCBot":["allowed","*",0,1],"Bytespider":["allowed","*",0,1],"Amazonbot":["allowed","*",0,1],"meta-externalagent":["allowed","*",0,1],"cohere-ai":["allowed","*",0,1],"Diffbot":["allowed","*",0,1],"ContextFetchBot":["allowed","*",0,1]},"llms":{"status":"missing","http":404,"bytes":6903},"home":{"status":"ok","http":200,"url":"https://vulkanvegaswin.net/","html_tokens":18519,"markdown_tokens":3825,"pruned_tokens":112}},
{"rank":7587,"domain":"atmel.com","origin":"https://atmel.com","robots":{"status":"unreachable","http":null,"url":"https://atmel.com/robots.txt","bytes":0,"groups":0,"error":"cannot resolve atmel.com"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":7588,"domain":"cargo.site","origin":"https://cargo.site","robots":{"status":"ok","http":200,"url":"https://cargo.site/robots.txt","bytes":505,"groups":13},"bots":{"GPTBot":["allowed","*",0,1],"ChatGPT-User":["allowed","*",0,1],"OAI-SearchBot":["allowed","*",0,1],"ClaudeBot":["allowed","*",0,1],"Claude-User":["allowed","*",0,1],"Claude-SearchBot":["allowed","*",0,1],"anthropic-ai":["allowed","*",0,1],"PerplexityBot":["allowed","*",0,1],"Perplexity-User":["allowed","*",0,1],"Google-Extended":["allowed","*",0,1],"Applebot-Extended":["allowed","*",0,1],"CCBot":["allowed","*",0,1],"Bytespider":["allowed","*",0,1],"Amazonbot":["allowed","*",0,1],"meta-externalagent":["allowed","*",0,1],"cohere-ai":["allowed","*",0,1],"Diffbot":["allowed","*",0,1],"ContextFetchBot":["allowed","*",0,1]},"llms":{"status":"present","http":200,"bytes":4774},"home":{"status":"ok","http":200,"url":"https://cargo.site/","html_tokens":2067,"markdown_tokens":75,"pruned_tokens":75}},
{"rank":7589,"domain":"computerbild.de","origin":"https://computerbild.de","robots":{"status":"ok","http":200,"url":"https://www.computerbild.de/robots.txt","bytes":2944,"groups":2},"bots":{"GPTBot":["partial","*",29,0],"ChatGPT-User":["partial","*",29,0],"OAI-SearchBot":["partial","*",29,0],"ClaudeBot":["partial","*",29,0],"Claude-User":["partial","*",29,0],"Claude-SearchBot":["partial","*",29,0],"anthropic-ai":["partial","*",29,0],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["blocked","named",1,0],"Google-Extended":["partial","*",29,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["partial","*",29,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",29,0]},"llms":{"status":"http_error","http":403,"bytes":385},"home":{"status":"ok","http":200,"url":"https://www.computerbild.de/","html_tokens":159587,"markdown_tokens":28,"pruned_tokens":28}},
{"rank":7590,"domain":"numerama.com","origin":"https://numerama.com","robots":{"status":"ok","http":200,"url":"https://www.numerama.com/robots.txt","bytes":1018,"groups":9},"bots":{"GPTBot":["allowed","named",0,1],"ChatGPT-User":["allowed","named",0,1],"OAI-SearchBot":["allowed","named",0,1],"ClaudeBot":["partial","*",17,2],"Claude-User":["partial","*",17,2],"Claude-SearchBot":["partial","*",17,2],"anthropic-ai":["partial","*",17,2],"PerplexityBot":["allowed","named",0,1],"Perplexity-User":["partial","*",17,2],"Google-Extended":["partial","*",17,2],"Applebot-Extended":["partial","*",17,2],"CCBot":["blocked","named",1,0],"Bytespider":["partial","*",17,2],"Amazonbot":["partial","*",17,2],"meta-externalagent":["partial","*",17,2],"cohere-ai":["partial","*",17,2],"Diffbot":["partial","*",17,2],"ContextFetchBot":["partial","*",17,2]},"llms":{"status":"missing","http":404,"bytes":262837},"home":{"status":"ok","http":200,"url":"https://www.numerama.com/","html_tokens":153504,"markdown_tokens":15,"pruned_tokens":15}},
{"rank":7591,"domain":"3dnews.ru","origin":"https://3dnews.ru","robots":{"status":"ok","http":200,"url":"https://3dnews.ru/robots.txt","bytes":433,"groups":2},"bots":{"GPTBot":["partial","*",4,1],"ChatGPT-User":["partial","*",4,1],"OAI-SearchBot":["partial","*",4,1],"ClaudeBot":["partial","*",4,1],"Claude-User":["partial","*",4,1],"Claude-SearchBot":["partial","*",4,1],"anthropic-ai":["partial","*",4,1],"PerplexityBot":["partial","*",4,1],"Perplexity-User":["partial","*",4,1],"Google-Extended":["partial","*",4,1],"Applebot-Extended":["partial","*",4,1],"CCBot":["partial","*",4,1],"Bytespider":["partial","*",4,1],"Amazonbot":["partial","*",4,1],"meta-externalagent":["partial","*",4,1],"cohere-ai":["partial","*",4,1],"Diffbot":["partial","*",4,1],"ContextFetchBot":["partial","*",4,1]},"llms":{"status":"missing","http":404,"bytes":161483},"home":{"status":"ok","http":200,"url":"https://3dnews.ru/","html_tokens":60739,"markdown_tokens":2119,"pruned_tokens":130}},
{"rank":7592,"domain":"www12.senado.leg.br","origin":"https://www12.senado.leg.br","robots":{"status":"ok","http":200,"url":"https://www12.senado.leg.br/robots.txt","bytes":162,"groups":2},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"http_error","http":403,"bytes":93},"home":{"status":"ok","http":200,"url":"https://www12.senado.leg.br/hpsenado","html_tokens":28462,"markdown_tokens":152,"pruned_tokens":152}},
{"rank":7593,"domain":"skynews.com.au","origin":"https://skynews.com.au","robots":{"status":"ok","http":200,"url":"https://www.news24.com.au/robots.txt","bytes":3039,"groups":4},"bots":{"GPTBot":["partial","*",1,2],"ChatGPT-User":["partial","*",1,2],"OAI-SearchBot":["partial","*",1,2],"ClaudeBot":["partial","*",1,2],"Claude-User":["partial","*",1,2],"Claude-SearchBot":["partial","*",1,2],"anthropic-ai":["partial","*",1,2],"PerplexityBot":["partial","*",1,2],"Perplexity-User":["partial","*",1,2],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["partial","*",1,2],"CCBot":["partial","*",1,2],"Bytespider":["partial","*",1,2],"Amazonbot":["partial","*",1,2],"meta-externalagent":["partial","*",1,2],"cohere-ai":["partial","*",1,2],"Diffbot":["partial","*",1,2],"ContextFetchBot":["partial","*",1,2]},"llms":{"status":"disallowed"},"home":{"status":"disallowed"}},
{"rank":7594,"domain":"google.cv","origin":"https://google.cv","robots":{"status":"ok","http":200,"url":"https://www.google.cv/robots.txt","bytes":6624,"groups":4},"bots":{"GPTBot":["partial","*",169,64],"ChatGPT-User":["partial","*",169,64],"OAI-SearchBot":["partial","*",169,64],"ClaudeBot":["partial","*",169,64],"Claude-User":["partial","*",169,64],"Claude-SearchBot":["partial","*",169,64],"anthropic-ai":["partial","*",169,64],"PerplexityBot":["partial","*",169,64],"Perplexity-User":["partial","*",169,64],"Google-Extended":["partial","*",169,64],"Applebot-Extended":["partial","*",169,64],"CCBot":["partial","*",169,64],"Bytespider":["partial","*",169,64],"Amazonbot":["partial","*",169,64],"meta-externalagent":["partial","*",169,64],"cohere-ai":["partial","*",169,64],"Diffbot":["partial","*",169,64],"ContextFetchBot":["partial","*",169,64]},"llms":{"status":"missing","http":404,"bytes":1569},"home":{"status":"ok","http":200,"url":"https://www.google.com/","html_tokens":21278,"markdown_tokens":30,"pruned_tokens":30}},
{"rank":7595,"domain":"sainsburys.co.uk","origin":"https://sainsburys.co.uk","robots":{"status":"refused","http":403,"url":"https://sainsburys.co.uk/robots.txt","bytes":384,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":7596,"domain":"mediabistro.com","origin":"https://mediabistro.com","robots":{"status":"ok","http":200,"url":"https://www.mediabistro.com/robots.txt","bytes":851,"groups":2},"bots":{"GPTBot":["partial","*",6,2],"ChatGPT-User":["partial","*",6,2],"OAI-SearchBot":["partial","*",6,2],"ClaudeBot":["partial","*",6,2],"Claude-User":["partial","*",6,2],"Claude-SearchBot":["partial","*",6,2],"anthropic-ai":["partial","*",6,2],"PerplexityBot":["partial","*",6,2],"Perplexity-User":["partial","*",6,2],"Google-Extended":["partial","*",6,2],"Applebot-Extended":["partial","*",6,2],"CCBot":["partial","named",10,0],"Bytespider":["partial","*",6,2],"Amazonbot":["partial","*",6,2],"meta-externalagent":["partial","named",10,0],"cohere-ai":["partial","*",6,2],"Diffbot":["partial","*",6,2],"ContextFetchBot":["partial","*",6,2]},"llms":{"status":"present","http":200,"bytes":2343},"home":{"status":"ok","http":200,"url":"https://www.mediabistro.com/","html_tokens":55090,"markdown_tokens":362,"pruned_tokens":164}},
{"rank":7597,"domain":"dwell.com","origin":"https://dwell.com","robots":{"status":"ok","http":200,"url":"https://www.dwell.com/robots.txt","bytes":115,"groups":1},"bots":{"GPTBot":["allowed","*",0,1],"ChatGPT-User":["allowed","*",0,1],"OAI-SearchBot":["allowed","*",0,1],"ClaudeBot":["allowed","*",0,1],"Claude-User":["allowed","*",0,1],"Claude-SearchBot":["allowed","*",0,1],"anthropic-ai":["allowed","*",0,1],"PerplexityBot":["allowed","*",0,1],"Perplexity-User":["allowed","*",0,1],"Google-Extended":["allowed","*",0,1],"Applebot-Extended":["allowed","*",0,1],"CCBot":["allowed","*",0,1],"Bytespider":["allowed","*",0,1],"Amazonbot":["allowed","*",0,1],"meta-externalagent":["allowed","*",0,1],"cohere-ai":["allowed","*",0,1],"Diffbot":["allowed","*",0,1],"ContextFetchBot":["allowed","*",0,1]},"llms":{"status":"html","http":200,"bytes":98245},"home":{"status":"ok","http":200,"url":"https://www.dwell.com/","html_tokens":133743,"markdown_tokens":213,"pruned_tokens":169}},
{"rank":7598,"domain":"prusa3d.com","origin":"https://prusa3d.com","robots":{"status":"ok","http":200,"url":"https://www.prusa3d.com/robots.txt","bytes":977,"groups":1},"bots":{"GPTBot":["partial","*",24,0],"ChatGPT-User":["partial","*",24,0],"OAI-SearchBot":["partial","*",24,0],"ClaudeBot":["partial","*",24,0],"Claude-User":["partial","*",24,0],"Claude-SearchBot":["partial","*",24,0],"anthropic-ai":["partial","*",24,0],"PerplexityBot":["partial","*",24,0],"Perplexity-User":["partial","*",24,0],"Google-Extended":["partial","*",24,0],"Applebot-Extended":["partial","*",24,0],"CCBot":["partial","*",24,0],"Bytespider":["partial","*",24,0],"Amazonbot":["partial","*",24,0],"meta-externalagent":["partial","*",24,0],"cohere-ai":["partial","*",24,0],"Diffbot":["partial","*",24,0],"ContextFetchBot":["partial","*",24,0]},"llms":{"status":"missing","http":404,"bytes":362882},"home":{"status":"ok","http":200,"url":"https://www.prusa3d.com","html_tokens":436376,"markdown_tokens":946,"pruned_tokens":67}},
{"rank":7599,"domain":"books.com.tw","origin":"https://www.books.com.tw","robots":{"status":"refused","http":403,"url":"https://www.books.com.tw/robots.txt","bytes":1780,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":7600,"domain":"themerex.net","origin":"https://themerex.net","robots":{"status":"ok","http":200,"url":"https://themerex.net/robots.txt","bytes":70,"groups":1},"bots":{"GPTBot":["partial","*",1,1],"ChatGPT-User":["partial","*",1,1],"OAI-SearchBot":["partial","*",1,1],"ClaudeBot":["partial","*",1,1],"Claude-User":["partial","*",1,1],"Claude-SearchBot":["partial","*",1,1],"anthropic-ai":["partial","*",1,1],"PerplexityBot":["partial","*",1,1],"Perplexity-User":["partial","*",1,1],"Google-Extended":["partial","*",1,1],"Applebot-Extended":["partial","*",1,1],"CCBot":["partial","*",1,1],"Bytespider":["partial","*",1,1],"Amazonbot":["partial","*",1,1],"meta-externalagent":["partial","*",1,1],"cohere-ai":["partial","*",1,1],"Diffbot":["partial","*",1,1],"ContextFetchBot":["partial","*",1,1]},"llms":{"status":"other_text","http":200,"bytes":6260},"home":{"status":"ok","http":200,"url":"https://themerex.net/","html_tokens":75673,"markdown_tokens":595,"pruned_tokens":143}},
{"rank":7601,"domain":"mondediplo.com","origin":"https://mondediplo.com","robots":{"status":"ok","http":200,"url":"https://mondediplo.com/robots.txt","bytes":435,"groups":1},"bots":{"GPTBot":["partial","*",7,4],"ChatGPT-User":["partial","*",7,4],"OAI-SearchBot":["partial","*",7,4],"ClaudeBot":["partial","*",7,4],"Claude-User":["partial","*",7,4],"Claude-SearchBot":["partial","*",7,4],"anthropic-ai":["partial","*",7,4],"PerplexityBot":["partial","*",7,4],"Perplexity-User":["partial","*",7,4],"Google-Extended":["partial","*",7,4],"Applebot-Extended":["partial","*",7,4],"CCBot":["partial","*",7,4],"Bytespider":["partial","*",7,4],"Amazonbot":["partial","*",7,4],"meta-externalagent":["partial","*",7,4],"cohere-ai":["partial","*",7,4],"Diffbot":["partial","*",7,4],"ContextFetchBot":["partial","*",7,4]},"llms":{"status":"missing","http":404,"bytes":986},"home":{"status":"ok","http":200,"url":"https://mondediplo.com/","html_tokens":13181,"markdown_tokens":97,"pruned_tokens":97}},
{"rank":7602,"domain":"voxeu.org","origin":"https://voxeu.org","robots":{"status":"refused","http":403,"url":"https://cepr.org/robots.txt","bytes":4541,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":7603,"domain":"whowhatwear.com","origin":"https://whowhatwear.com","robots":{"status":"ok","http":200,"url":"https://www.whowhatwear.com/robots.txt","bytes":1990,"groups":5},"bots":{"GPTBot":["partial","*",26,0],"ChatGPT-User":["partial","*",26,0],"OAI-SearchBot":["partial","*",26,0],"ClaudeBot":["partial","*",26,0],"Claude-User":["partial","*",26,0],"Claude-SearchBot":["partial","*",26,0],"anthropic-ai":["partial","*",26,0],"PerplexityBot":["partial","*",26,0],"Perplexity-User":["partial","*",26,0],"Google-Extended":["partial","*",26,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["partial","*",26,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["partial","*",26,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",26,0]},"llms":{"status":"missing","http":404,"bytes":831262},"home":{"status":"ok","http":200,"url":"https://www.whowhatwear.com/","html_tokens":390882,"markdown_tokens":6534,"pruned_tokens":145}},
{"rank":7604,"domain":"news4jax.com","origin":"https://news4jax.com","robots":{"status":"ok","http":200,"url":"https://www.news4jax.com/robots.txt","bytes":1159,"groups":9},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["partial","*",5,0],"OAI-SearchBot":["partial","*",5,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",5,0],"Claude-SearchBot":["partial","*",5,0],"anthropic-ai":["partial","*",5,0],"PerplexityBot":["partial","*",5,0],"Perplexity-User":["partial","*",5,0],"Google-Extended":["partial","*",5,0],"Applebot-Extended":["partial","*",5,0],"CCBot":["partial","*",5,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["partial","*",5,0],"cohere-ai":["partial","*",5,0],"Diffbot":["partial","*",5,0],"ContextFetchBot":["partial","*",5,0]},"llms":{"status":"missing","http":404,"bytes":191747},"home":{"status":"ok","http":200,"url":"https://www.news4jax.com/","html_tokens":234759,"markdown_tokens":5726,"pruned_tokens":162}},
{"rank":7605,"domain":"wpr.org","origin":"https://wpr.org","robots":{"status":"refused","http":403,"url":"https://wpr.org/robots.txt","bytes":560,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":7606,"domain":"thepinknews.com","origin":"https://thepinknews.com","robots":{"status":"ok","http":200,"url":"https://www.thepinknews.com/robots.txt","bytes":864,"groups":6},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["blocked","named",1,0],"OAI-SearchBot":["partial","*",3,0],"ClaudeBot":["partial","*",3,0],"Claude-User":["partial","*",3,0],"Claude-SearchBot":["partial","*",3,0],"anthropic-ai":["partial","*",3,0],"PerplexityBot":["partial","*",3,0],"Perplexity-User":["partial","*",3,0],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["partial","*",3,0],"CCBot":["blocked","named",1,0],"Bytespider":["partial","*",3,0],"Amazonbot":["partial","*",3,0],"meta-externalagent":["partial","*",3,0],"cohere-ai":["partial","*",3,0],"Diffbot":["partial","*",3,0],"ContextFetchBot":["partial","*",3,0]},"llms":{"status":"missing","http":404,"bytes":87814},"home":{"status":"ok","http":200,"url":"https://www.thepinknews.com/","html_tokens":59676,"markdown_tokens":173,"pruned_tokens":173}},
{"rank":7607,"domain":"scania.com","origin":"https://scania.com","robots":{"status":"ok","http":200,"url":"https://www.scania.com/robots.txt","bytes":328,"groups":1},"bots":{"GPTBot":["partial","*",7,1],"ChatGPT-User":["partial","*",7,1],"OAI-SearchBot":["partial","*",7,1],"ClaudeBot":["partial","*",7,1],"Claude-User":["partial","*",7,1],"Claude-SearchBot":["partial","*",7,1],"anthropic-ai":["partial","*",7,1],"PerplexityBot":["partial","*",7,1],"Perplexity-User":["partial","*",7,1],"Google-Extended":["partial","*",7,1],"Applebot-Extended":["partial","*",7,1],"CCBot":["partial","*",7,1],"Bytespider":["partial","*",7,1],"Amazonbot":["partial","*",7,1],"meta-externalagent":["partial","*",7,1],"cohere-ai":["partial","*",7,1],"Diffbot":["partial","*",7,1],"ContextFetchBot":["partial","*",7,1]},"llms":{"status":"missing","http":404,"bytes":69969},"home":{"status":"ok","http":200,"url":"https://www.scania.com/","html_tokens":33439,"markdown_tokens":682,"pruned_tokens":213}},
{"rank":7608,"domain":"puc-rio.br","origin":"https://puc-rio.br","robots":{"status":"missing","http":404,"url":"https://puc-rio.br/robots.txt","bytes":103,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":103},"home":{"status":"ok","http":200,"url":"https://puc-rio.br/","html_tokens":10832,"markdown_tokens":365,"pruned_tokens":216}},
{"rank":7609,"domain":"gvsu.edu","origin":"https://gvsu.edu","robots":{"status":"ok","http":200,"url":"https://www.gvsu.edu/robots.txt","bytes":379,"groups":5},"bots":{"GPTBot":["partial","*",5,0],"ChatGPT-User":["partial","*",5,0],"OAI-SearchBot":["partial","*",5,0],"ClaudeBot":["partial","*",5,0],"Claude-User":["partial","*",5,0],"Claude-SearchBot":["partial","*",5,0],"anthropic-ai":["partial","*",5,0],"PerplexityBot":["partial","*",5,0],"Perplexity-User":["partial","*",5,0],"Google-Extended":["partial","*",5,0],"Applebot-Extended":["partial","*",5,0],"CCBot":["partial","*",5,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["partial","*",5,0],"meta-externalagent":["partial","*",5,0],"cohere-ai":["partial","*",5,0],"Diffbot":["partial","*",5,0],"ContextFetchBot":["partial","*",5,0]},"llms":{"status":"missing","http":404,"bytes":7740},"home":{"status":"ok","http":200,"url":"https://www.gvsu.edu/","html_tokens":18458,"markdown_tokens":97,"pruned_tokens":97}},
{"rank":7610,"domain":"siemens-healthineers.com","origin":"https://siemens-healthineers.com","robots":{"status":"ok","http":200,"url":"https://www.siemens-healthineers.com/robots.txt","bytes":207,"groups":1},"bots":{"GPTBot":["partial","*",2,0],"ChatGPT-User":["partial","*",2,0],"OAI-SearchBot":["partial","*",2,0],"ClaudeBot":["partial","*",2,0],"Claude-User":["partial","*",2,0],"Claude-SearchBot":["partial","*",2,0],"anthropic-ai":["partial","*",2,0],"PerplexityBot":["partial","*",2,0],"Perplexity-User":["partial","*",2,0],"Google-Extended":["partial","*",2,0],"Applebot-Extended":["partial","*",2,0],"CCBot":["partial","*",2,0],"Bytespider":["partial","*",2,0],"Amazonbot":["partial","*",2,0],"meta-externalagent":["partial","*",2,0],"cohere-ai":["partial","*",2,0],"Diffbot":["partial","*",2,0],"ContextFetchBot":["partial","*",2,0]},"llms":{"status":"missing","http":404,"bytes":527},"home":{"status":"ok","http":200,"url":"https://www.siemens-healthineers.com","html_tokens":22980,"markdown_tokens":85,"pruned_tokens":85}},
{"rank":7611,"domain":"bshare.cn","origin":"https://bshare.cn","robots":{"status":"unreachable","http":null,"url":"https://bshare.cn/robots.txt","bytes":0,"groups":0,"error":"cannot resolve bshare.cn"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":7612,"domain":"greatschools.org","origin":"https://www.greatschools.org","robots":{"status":"ok","http":200,"url":"https://www.greatschools.org/robots.txt","bytes":11669,"groups":19},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["partial","*",85,0],"OAI-SearchBot":["partial","*",85,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",85,0],"Claude-SearchBot":["partial","*",85,0],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["partial","*",85,0],"Perplexity-User":["partial","*",85,0],"Google-Extended":["partial","named",1,8],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["partial","*",85,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",85,0]},"llms":{"status":"missing","http":404,"bytes":60773},"home":{"status":"ok","http":200,"url":"https://www.greatschools.org/","html_tokens":19379,"markdown_tokens":46,"pruned_tokens":46}},
{"rank":7613,"domain":"submittable.com","origin":"https://submittable.com","robots":{"status":"ok","http":200,"url":"https://www.submittable.com/robots.txt","bytes":703,"groups":15},"bots":{"GPTBot":["allowed","named",0,1],"ChatGPT-User":["allowed","named",0,1],"OAI-SearchBot":["allowed","named",0,1],"ClaudeBot":["allowed","named",0,1],"Claude-User":["allowed","*",0,1],"Claude-SearchBot":["allowed","*",0,1],"anthropic-ai":["allowed","named",0,1],"PerplexityBot":["allowed","named",0,1],"Perplexity-User":["allowed","*",0,1],"Google-Extended":["allowed","named",0,1],"Applebot-Extended":["allowed","*",0,1],"CCBot":["allowed","*",0,1],"Bytespider":["allowed","*",0,1],"Amazonbot":["allowed","*",0,1],"meta-externalagent":["allowed","*",0,1],"cohere-ai":["allowed","*",0,1],"Diffbot":["allowed","*",0,1],"ContextFetchBot":["allowed","*",0,1]},"llms":{"status":"missing","http":404,"bytes":148189},"home":{"status":"ok","http":200,"url":"https://www.submittable.com/","html_tokens":72335,"markdown_tokens":191,"pruned_tokens":104}},
{"rank":7614,"domain":"srce.hr","origin":"https://srce.hr","robots":{"status":"ok","http":200,"url":"https://www.srce.unizg.hr/robots.txt","bytes":1731,"groups":1},"bots":{"GPTBot":["partial","*",27,18],"ChatGPT-User":["partial","*",27,18],"OAI-SearchBot":["partial","*",27,18],"ClaudeBot":["partial","*",27,18],"Claude-User":["partial","*",27,18],"Claude-SearchBot":["partial","*",27,18],"anthropic-ai":["partial","*",27,18],"PerplexityBot":["partial","*",27,18],"Perplexity-User":["partial","*",27,18],"Google-Extended":["partial","*",27,18],"Applebot-Extended":["partial","*",27,18],"CCBot":["partial","*",27,18],"Bytespider":["partial","*",27,18],"Amazonbot":["partial","*",27,18],"meta-externalagent":["partial","*",27,18],"cohere-ai":["partial","*",27,18],"Diffbot":["partial","*",27,18],"ContextFetchBot":["partial","*",27,18]},"llms":{"status":"missing","http":404,"bytes":191},"home":{"status":"ok","http":200,"url":"https://www.srce.unizg.hr/","html_tokens":34619,"markdown_tokens":661,"pruned_tokens":124}},
{"rank":7615,"domain":"asianetnews.com","origin":"https://asianetnews.com","robots":{"status":"ok","http":200,"url":"https://www.asianetnews.com/robots.txt","bytes":1014,"groups":5},"bots":{"GPTBot":["partial","*",5,1],"ChatGPT-User":["partial","*",5,1],"OAI-SearchBot":["partial","*",5,1],"ClaudeBot":["partial","*",5,1],"Claude-User":["partial","*",5,1],"Claude-SearchBot":["partial","*",5,1],"anthropic-ai":["partial","*",5,1],"PerplexityBot":["partial","*",5,1],"Perplexity-User":["partial","*",5,1],"Google-Extended":["allowed","named",0,1],"Applebot-Extended":["partial","*",5,1],"CCBot":["partial","*",5,1],"Bytespider":["partial","*",5,1],"Amazonbot":["partial","*",5,1],"meta-externalagent":["partial","*",5,1],"cohere-ai":["partial","*",5,1],"Diffbot":["partial","*",5,1],"ContextFetchBot":["partial","*",5,1]},"llms":{"status":"present","http":200,"bytes":13815},"home":{"status":"ok","http":200,"url":"https://www.asianetnews.com/","html_tokens":141273,"markdown_tokens":190,"pruned_tokens":26}},
{"rank":7616,"domain":"hkex.com.hk","origin":"https://www.hkex.com.hk","robots":{"status":"ok","http":200,"url":"https://www.hkex.com.hk/robots.txt","bytes":348,"groups":1},"bots":{"GPTBot":["partial","*",4,0],"ChatGPT-User":["partial","*",4,0],"OAI-SearchBot":["partial","*",4,0],"ClaudeBot":["partial","*",4,0],"Claude-User":["partial","*",4,0],"Claude-SearchBot":["partial","*",4,0],"anthropic-ai":["partial","*",4,0],"PerplexityBot":["partial","*",4,0],"Perplexity-User":["partial","*",4,0],"Google-Extended":["partial","*",4,0],"Applebot-Extended":["partial","*",4,0],"CCBot":["partial","*",4,0],"Bytespider":["partial","*",4,0],"Amazonbot":["partial","*",4,0],"meta-externalagent":["partial","*",4,0],"cohere-ai":["partial","*",4,0],"Diffbot":["partial","*",4,0],"ContextFetchBot":["partial","*",4,0]},"llms":{"status":"missing","http":404,"bytes":2947},"home":{"status":"ok","http":200,"url":"https://www.hkex.com.hk/?sc_lang=en","html_tokens":84096,"markdown_tokens":94,"pruned_tokens":94}},
{"rank":7617,"domain":"ebay-kleinanzeigen.de","origin":"https://ebay-kleinanzeigen.de","robots":{"status":"missing","http":410,"url":"https://ebay-kleinanzeigen.de/robots.txt","bytes":298,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":410,"bytes":298},"home":{"status":"http_error","http":410,"url":"https://ebay-kleinanzeigen.de/"}},
{"rank":7618,"domain":"csu.edu.au","origin":"https://csu.edu.au","robots":{"status":"refused","http":403,"url":"https://www.csu.edu.au/robots.txt","bytes":5552,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":7619,"domain":"tweaktown.com","origin":"https://tweaktown.com","robots":{"status":"ok","http":200,"url":"https://www.tweaktown.com/robots.txt","bytes":2131,"groups":4},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["partial","*",8,3],"OAI-SearchBot":["partial","*",8,3],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",8,3],"Claude-SearchBot":["partial","*",8,3],"anthropic-ai":["partial","*",8,3],"PerplexityBot":["partial","*",8,3],"Perplexity-User":["partial","*",8,3],"Google-Extended":["partial","*",8,3],"Applebot-Extended":["partial","*",8,3],"CCBot":["blocked","named",1,0],"Bytespider":["partial","*",8,3],"Amazonbot":["partial","*",8,3],"meta-externalagent":["partial","*",8,3],"cohere-ai":["partial","*",8,3],"Diffbot":["partial","*",8,3],"ContextFetchBot":["partial","*",8,3]},"llms":{"status":"present","http":200,"bytes":5346},"home":{"status":"http_error","http":403,"url":"https://www.tweaktown.com/"}},
{"rank":7620,"domain":"google.mu","origin":"https://google.mu","robots":{"status":"ok","http":200,"url":"https://www.google.mu/robots.txt","bytes":6624,"groups":4},"bots":{"GPTBot":["partial","*",169,64],"ChatGPT-User":["partial","*",169,64],"OAI-SearchBot":["partial","*",169,64],"ClaudeBot":["partial","*",169,64],"Claude-User":["partial","*",169,64],"Claude-SearchBot":["partial","*",169,64],"anthropic-ai":["partial","*",169,64],"PerplexityBot":["partial","*",169,64],"Perplexity-User":["partial","*",169,64],"Google-Extended":["partial","*",169,64],"Applebot-Extended":["partial","*",169,64],"CCBot":["partial","*",169,64],"Bytespider":["partial","*",169,64],"Amazonbot":["partial","*",169,64],"meta-externalagent":["partial","*",169,64],"cohere-ai":["partial","*",169,64],"Diffbot":["partial","*",169,64],"ContextFetchBot":["partial","*",169,64]},"llms":{"status":"missing","http":404,"bytes":1569},"home":{"status":"ok","http":200,"url":"https://www.google.com/","html_tokens":21359,"markdown_tokens":30,"pruned_tokens":30}},
{"rank":7621,"domain":"zhandian.cn","origin":"https://zhandian.cn","robots":{"status":"ok","http":200,"url":"https://zhandian.cn/robots.txt","bytes":253,"groups":5},"bots":{"GPTBot":["partial","*",2,1],"ChatGPT-User":["partial","*",2,1],"OAI-SearchBot":["partial","*",2,1],"ClaudeBot":["partial","*",2,1],"Claude-User":["partial","*",2,1],"Claude-SearchBot":["partial","*",2,1],"anthropic-ai":["partial","*",2,1],"PerplexityBot":["partial","*",2,1],"Perplexity-User":["partial","*",2,1],"Google-Extended":["partial","*",2,1],"Applebot-Extended":["partial","*",2,1],"CCBot":["partial","*",2,1],"Bytespider":["partial","*",2,1],"Amazonbot":["partial","*",2,1],"meta-externalagent":["partial","*",2,1],"cohere-ai":["partial","*",2,1],"Diffbot":["partial","*",2,1],"ContextFetchBot":["partial","*",2,1]},"llms":{"status":"present","http":200,"bytes":12154},"home":{"status":"ok","http":200,"url":"https://zhandian.cn/","html_tokens":43759,"markdown_tokens":14,"pruned_tokens":14}},
{"rank":7622,"domain":"eurovision.com","origin":"https://eurovision.com","robots":{"status":"ok","http":200,"url":"https://www.eurovision.com/robots.txt","bytes":569,"groups":1},"bots":{"GPTBot":["partial","*",10,0],"ChatGPT-User":["partial","*",10,0],"OAI-SearchBot":["partial","*",10,0],"ClaudeBot":["partial","*",10,0],"Claude-User":["partial","*",10,0],"Claude-SearchBot":["partial","*",10,0],"anthropic-ai":["partial","*",10,0],"PerplexityBot":["partial","*",10,0],"Perplexity-User":["partial","*",10,0],"Google-Extended":["partial","*",10,0],"Applebot-Extended":["partial","*",10,0],"CCBot":["partial","*",10,0],"Bytespider":["partial","*",10,0],"Amazonbot":["partial","*",10,0],"meta-externalagent":["partial","*",10,0],"cohere-ai":["partial","*",10,0],"Diffbot":["partial","*",10,0],"ContextFetchBot":["partial","*",10,0]},"llms":{"status":"missing","http":404,"bytes":64675},"home":{"status":"ok","http":200,"url":"https://www.eurovision.com/","html_tokens":34732,"markdown_tokens":200,"pruned_tokens":83}},
{"rank":7623,"domain":"seek.com.au","origin":"https://seek.com.au","robots":{"status":"ok","http":200,"url":"https://au.seek.com/robots.txt","bytes":853,"groups":6},"bots":{"GPTBot":["partial","named",2,0],"ChatGPT-User":["partial","*",6,4],"OAI-SearchBot":["partial","*",6,4],"ClaudeBot":["partial","*",6,4],"Claude-User":["partial","*",6,4],"Claude-SearchBot":["partial","*",6,4],"anthropic-ai":["partial","named",2,0],"PerplexityBot":["partial","*",6,4],"Perplexity-User":["partial","*",6,4],"Google-Extended":["partial","named",2,0],"Applebot-Extended":["partial","*",6,4],"CCBot":["partial","named",2,0],"Bytespider":["partial","named",2,0],"Amazonbot":["partial","*",6,4],"meta-externalagent":["partial","*",6,4],"cohere-ai":["partial","*",6,4],"Diffbot":["partial","named",2,0],"ContextFetchBot":["partial","*",6,4]},"llms":{"status":"missing","http":404,"bytes":10218},"home":{"status":"http_error","http":403,"url":"https://www.seek.com.au/"}},
{"rank":7624,"domain":"alstom.com","origin":"https://alstom.com","robots":{"status":"ok","http":200,"url":"https://www.alstom.com/robots.txt","bytes":2762,"groups":3},"bots":{"GPTBot":["partial","*",58,19],"ChatGPT-User":["partial","*",58,19],"OAI-SearchBot":["partial","*",58,19],"ClaudeBot":["partial","*",58,19],"Claude-User":["partial","*",58,19],"Claude-SearchBot":["partial","*",58,19],"anthropic-ai":["partial","*",58,19],"PerplexityBot":["partial","*",58,19],"Perplexity-User":["partial","*",58,19],"Google-Extended":["partial","*",58,19],"Applebot-Extended":["partial","*",58,19],"CCBot":["partial","*",58,19],"Bytespider":["partial","*",58,19],"Amazonbot":["partial","*",58,19],"meta-externalagent":["partial","*",58,19],"cohere-ai":["partial","*",58,19],"Diffbot":["partial","*",58,19],"ContextFetchBot":["partial","*",58,19]},"llms":{"status":"missing","http":404,"bytes":549},"home":{"status":"ok","http":200,"url":"https://www.alstom.com/","html_tokens":70815,"markdown_tokens":156,"pruned_tokens":148}},
{"rank":7625,"domain":"aeroflot.ru","origin":"https://aeroflot.ru","robots":{"status":"server_error","http":503,"url":"https://www.aeroflot.ru/robots.txt","bytes":8388,"groups":0,"error":"HTTP 503"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":7626,"domain":"nvaccess.org","origin":"https://nvaccess.org","robots":{"status":"ok","http":200,"url":"https://www.nvaccess.org/robots.txt","bytes":319,"groups":1},"bots":{"GPTBot":["partial","*",6,1],"ChatGPT-User":["partial","*",6,1],"OAI-SearchBot":["partial","*",6,1],"ClaudeBot":["partial","*",6,1],"Claude-User":["partial","*",6,1],"Claude-SearchBot":["partial","*",6,1],"anthropic-ai":["partial","*",6,1],"PerplexityBot":["partial","*",6,1],"Perplexity-User":["partial","*",6,1],"Google-Extended":["partial","*",6,1],"Applebot-Extended":["partial","*",6,1],"CCBot":["partial","*",6,1],"Bytespider":["partial","*",6,1],"Amazonbot":["partial","*",6,1],"meta-externalagent":["partial","*",6,1],"cohere-ai":["partial","*",6,1],"Diffbot":["partial","*",6,1],"ContextFetchBot":["partial","*",6,1]},"llms":{"status":"missing","http":404,"bytes":36897},"home":{"status":"ok","http":200,"url":"https://www.nvaccess.org/","html_tokens":12674,"markdown_tokens":515,"pruned_tokens":58}},
{"rank":7627,"domain":"isi.edu","origin":"https://isi.edu","robots":{"status":"ok","http":200,"url":"https://isi.edu/robots.txt","bytes":444,"groups":2},"bots":{"GPTBot":["partial","*",3,0],"ChatGPT-User":["partial","*",3,0],"OAI-SearchBot":["partial","*",3,0],"ClaudeBot":["partial","*",3,0],"Claude-User":["partial","*",3,0],"Claude-SearchBot":["partial","*",3,0],"anthropic-ai":["partial","*",3,0],"PerplexityBot":["partial","*",3,0],"Perplexity-User":["partial","*",3,0],"Google-Extended":["partial","*",3,0],"Applebot-Extended":["partial","*",3,0],"CCBot":["partial","*",3,0],"Bytespider":["partial","*",3,0],"Amazonbot":["partial","*",3,0],"meta-externalagent":["partial","*",3,0],"cohere-ai":["partial","*",3,0],"Diffbot":["partial","*",3,0],"ContextFetchBot":["partial","*",3,0]},"llms":{"status":"http_error","http":403,"bytes":5538},"home":{"status":"http_error","http":403,"url":"https://isi.edu/"}},
{"rank":7628,"domain":"cnsnews.com","origin":"https://cnsnews.com","robots":{"status":"refused","http":403,"url":"https://cnsnews.com/robots.txt","bytes":560,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":7629,"domain":"threejs.org","origin":"https://threejs.org","robots":{"status":"missing","http":404,"url":"https://threejs.org/robots.txt","bytes":1057,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"present","http":200,"bytes":274},"home":{"status":"ok","http":200,"url":"https://threejs.org/","html_tokens":15708,"markdown_tokens":17,"pruned_tokens":17}},
{"rank":7630,"domain":"cedars-sinai.org","origin":"https://cedars-sinai.org","robots":{"status":"ok","http":200,"url":"https://www.cedars-sinai.org/robots.txt","bytes":614,"groups":1},"bots":{"GPTBot":["partial","*",14,0],"ChatGPT-User":["partial","*",14,0],"OAI-SearchBot":["partial","*",14,0],"ClaudeBot":["partial","*",14,0],"Claude-User":["partial","*",14,0],"Claude-SearchBot":["partial","*",14,0],"anthropic-ai":["partial","*",14,0],"PerplexityBot":["partial","*",14,0],"Perplexity-User":["partial","*",14,0],"Google-Extended":["partial","*",14,0],"Applebot-Extended":["partial","*",14,0],"CCBot":["partial","*",14,0],"Bytespider":["partial","*",14,0],"Amazonbot":["partial","*",14,0],"meta-externalagent":["partial","*",14,0],"cohere-ai":["partial","*",14,0],"Diffbot":["partial","*",14,0],"ContextFetchBot":["partial","*",14,0]},"llms":{"status":"present","http":200,"bytes":6542},"home":{"status":"ok","http":200,"url":"https://www.cedars-sinai.org/home.html","html_tokens":3221,"markdown_tokens":12,"pruned_tokens":12}},
{"rank":7631,"domain":"tga.gov.au","origin":"https://tga.gov.au","robots":{"status":"unreachable","http":null,"url":"https://tga.gov.au/robots.txt","bytes":0,"groups":0,"error":"<StreamReset stream_id:1, error_code:2, remote_reset:True>"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":7632,"domain":"nationalgeographic.com.es","origin":"https://nationalgeographic.com.es","robots":{"status":"refused","http":403,"url":"https://www.nationalgeographic.com.es/robots.txt","bytes":401,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":7633,"domain":"wired.it","origin":"https://wired.it","robots":{"status":"ok","http":200,"url":"https://www.wired.it/robots.txt","bytes":2084,"groups":3},"bots":{"GPTBot":["partial","*",13,12],"ChatGPT-User":["partial","*",13,12],"OAI-SearchBot":["partial","*",13,12],"ClaudeBot":["blocked","named",1,0],"Claude-User":["blocked","named",1,0],"Claude-SearchBot":["blocked","named",1,0],"anthropic-ai":["partial","*",13,12],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["blocked","named",1,0],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",13,12]},"llms":{"status":"missing","http":404,"bytes":437404},"home":{"status":"ok","http":200,"url":"https://www.wired.it/","html_tokens":303865,"markdown_tokens":2464,"pruned_tokens":85}},
{"rank":7634,"domain":"reverb.com","origin":"https://reverb.com","robots":{"status":"ok","http":200,"url":"https://reverb.com/robots.txt","bytes":40563,"groups":2},"bots":{"GPTBot":["partial","*",1165,0],"ChatGPT-User":["partial","*",1165,0],"OAI-SearchBot":["partial","*",1165,0],"ClaudeBot":["partial","*",1165,0],"Claude-User":["partial","*",1165,0],"Claude-SearchBot":["partial","*",1165,0],"anthropic-ai":["partial","*",1165,0],"PerplexityBot":["partial","*",1165,0],"Perplexity-User":["partial","*",1165,0],"Google-Extended":["partial","*",1165,0],"Applebot-Extended":["partial","*",1165,0],"CCBot":["partial","*",1165,0],"Bytespider":["partial","*",1165,0],"Amazonbot":["partial","*",1165,0],"meta-externalagent":["partial","*",1165,0],"cohere-ai":["partial","*",1165,0],"Diffbot":["partial","*",1165,0],"ContextFetchBot":["partial","*",1165,0]},"llms":{"status":"missing","http":404,"bytes":625156},"home":{"status":"http_error","http":403,"url":"https://reverb.com/"}},
{"rank":7635,"domain":"parool.nl","origin":"https://parool.nl","robots":{"status":"ok","http":200,"url":"https://www.parool.nl/robots.txt","bytes":2985,"groups":41},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["blocked","named",1,0],"OAI-SearchBot":["blocked","named",1,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",14,2],"Claude-SearchBot":["blocked","named",1,0],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["blocked","named",1,0],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["partial","*",14,2],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",14,2]},"llms":{"status":"missing","http":404,"bytes":115195},"home":{"status":"ok","http":200,"url":"https://www.parool.nl/","html_tokens":323166,"markdown_tokens":35,"pruned_tokens":35}},
{"rank":7636,"domain":"thefa.com","origin":"https://thefa.com","robots":{"status":"ok","http":200,"url":"https://thefa.com/robots.txt","bytes":1513,"groups":13},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"html","http":200,"bytes":140743},"home":{"status":"ok","http":200,"url":"https://thefa.com/","html_tokens":52869,"markdown_tokens":20,"pruned_tokens":20}},
{"rank":7637,"domain":"virginmedia.com","origin":"https://virginmedia.com","robots":{"status":"ok","http":200,"url":"https://www.virginmedia.com/robots.txt","bytes":452,"groups":1},"bots":{"GPTBot":["partial","*",12,1],"ChatGPT-User":["partial","*",12,1],"OAI-SearchBot":["partial","*",12,1],"ClaudeBot":["partial","*",12,1],"Claude-User":["partial","*",12,1],"Claude-SearchBot":["partial","*",12,1],"anthropic-ai":["partial","*",12,1],"PerplexityBot":["partial","*",12,1],"Perplexity-User":["partial","*",12,1],"Google-Extended":["partial","*",12,1],"Applebot-Extended":["partial","*",12,1],"CCBot":["partial","*",12,1],"Bytespider":["partial","*",12,1],"Amazonbot":["partial","*",12,1],"meta-externalagent":["partial","*",12,1],"cohere-ai":["partial","*",12,1],"Diffbot":["partial","*",12,1],"ContextFetchBot":["partial","*",12,1]},"llms":{"status":"missing","http":404,"bytes":463332},"home":{"status":"ok","http":200,"url":"https://www.virginmedia.com/","html_tokens":172270,"markdown_tokens":10523,"pruned_tokens":457}},
{"rank":7638,"domain":"immerioncasino.net","origin":"https://immerioncasino.net","robots":{"status":"ok","http":200,"url":"https://immerioncasino.net/robots.txt","bytes":67,"groups":1},"bots":{"GPTBot":["partial","*",1,1],"ChatGPT-User":["partial","*",1,1],"OAI-SearchBot":["partial","*",1,1],"ClaudeBot":["partial","*",1,1],"Claude-User":["partial","*",1,1],"Claude-SearchBot":["partial","*",1,1],"anthropic-ai":["partial","*",1,1],"PerplexityBot":["partial","*",1,1],"Perplexity-User":["partial","*",1,1],"Google-Extended":["partial","*",1,1],"Applebot-Extended":["partial","*",1,1],"CCBot":["partial","*",1,1],"Bytespider":["partial","*",1,1],"Amazonbot":["partial","*",1,1],"meta-externalagent":["partial","*",1,1],"cohere-ai":["partial","*",1,1],"Diffbot":["partial","*",1,1],"ContextFetchBot":["partial","*",1,1]},"llms":{"status":"missing","http":404,"bytes":45685},"home":{"status":"ok","http":200,"url":"https://immerioncasino.net/","html_tokens":15838,"markdown_tokens":2003,"pruned_tokens":169}},
{"rank":7639,"domain":"ruanyifeng.com","origin":"https://ruanyifeng.com","robots":{"status":"ok","http":200,"url":"https://ruanyifeng.com/robots.txt","bytes":0,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":643},"home":{"status":"ok","http":200,"url":"https://ruanyifeng.com/","html_tokens":2944,"markdown_tokens":19,"pruned_tokens":19}},
{"rank":7640,"domain":"official.ec","origin":"https://www.official.ec","robots":{"status":"ok","http":200,"url":"https://www.official.ec/robots.txt","bytes":809,"groups":3},"bots":{"GPTBot":["partial","*",6,1],"ChatGPT-User":["partial","*",6,1],"OAI-SearchBot":["partial","*",6,1],"ClaudeBot":["partial","*",6,1],"Claude-User":["partial","*",6,1],"Claude-SearchBot":["partial","*",6,1],"anthropic-ai":["partial","*",6,1],"PerplexityBot":["partial","*",6,1],"Perplexity-User":["partial","*",6,1],"Google-Extended":["partial","*",6,1],"Applebot-Extended":["partial","*",6,1],"CCBot":["partial","*",6,1],"Bytespider":["partial","*",6,1],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["partial","*",6,1],"cohere-ai":["partial","*",6,1],"Diffbot":["partial","*",6,1],"ContextFetchBot":["partial","*",6,1]},"llms":{"status":"html","http":200,"bytes":149241},"home":{"status":"ok","http":200,"url":"https://thebase.com/","html_tokens":35899,"markdown_tokens":151,"pruned_tokens":151}},
{"rank":7641,"domain":"nationbuilder.com","origin":"https://nationbuilder.com","robots":{"status":"ok","http":200,"url":"https://nationbuilder.com/robots.txt","bytes":156,"groups":1},"bots":{"GPTBot":["partial","*",5,0],"ChatGPT-User":["partial","*",5,0],"OAI-SearchBot":["partial","*",5,0],"ClaudeBot":["partial","*",5,0],"Claude-User":["partial","*",5,0],"Claude-SearchBot":["partial","*",5,0],"anthropic-ai":["partial","*",5,0],"PerplexityBot":["partial","*",5,0],"Perplexity-User":["partial","*",5,0],"Google-Extended":["partial","*",5,0],"Applebot-Extended":["partial","*",5,0],"CCBot":["partial","*",5,0],"Bytespider":["partial","*",5,0],"Amazonbot":["partial","*",5,0],"meta-externalagent":["partial","*",5,0],"cohere-ai":["partial","*",5,0],"Diffbot":["partial","*",5,0],"ContextFetchBot":["partial","*",5,0]},"llms":{"status":"present","http":200,"bytes":8291},"home":{"status":"ok","http":200,"url":"https://nationbuilder.com/","html_tokens":23022,"markdown_tokens":18,"pruned_tokens":18}},
{"rank":7642,"domain":"labs.google","origin":"https://labs.google","robots":{"status":"ok","http":200,"url":"https://labs.google/robots.txt","bytes":293,"groups":1},"bots":{"GPTBot":["partial","*",1,0],"ChatGPT-User":["partial","*",1,0],"OAI-SearchBot":["partial","*",1,0],"ClaudeBot":["partial","*",1,0],"Claude-User":["partial","*",1,0],"Claude-SearchBot":["partial","*",1,0],"anthropic-ai":["partial","*",1,0],"PerplexityBot":["partial","*",1,0],"Perplexity-User":["partial","*",1,0],"Google-Extended":["partial","*",1,0],"Applebot-Extended":["partial","*",1,0],"CCBot":["partial","*",1,0],"Bytespider":["partial","*",1,0],"Amazonbot":["partial","*",1,0],"meta-externalagent":["partial","*",1,0],"cohere-ai":["partial","*",1,0],"Diffbot":["partial","*",1,0],"ContextFetchBot":["partial","*",1,0]},"llms":{"status":"disallowed","error":"robots.txt of labs.google disallows /404"},"home":{"status":"ok","http":200,"url":"https://labs.google/","html_tokens":74338,"markdown_tokens":279,"pruned_tokens":102}},
{"rank":7643,"domain":"cqvip.com","origin":"https://cqvip.com","robots":{"status":"ok","http":200,"url":"https://www.cqvip.com/robots.txt","bytes":711,"groups":19},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"missing","http":404,"bytes":0},"home":{"status":"ok","http":200,"url":"https://www.cqvip.com/","html_tokens":32833,"markdown_tokens":30,"pruned_tokens":30}},
{"rank":7644,"domain":"acefitness.org","origin":"https://acefitness.org","robots":{"status":"ok","http":200,"url":"https://www.acefitness.org/robots.txt","bytes":291,"groups":1},"bots":{"GPTBot":["partial","*",5,1],"ChatGPT-User":["partial","*",5,1],"OAI-SearchBot":["partial","*",5,1],"ClaudeBot":["partial","*",5,1],"Claude-User":["partial","*",5,1],"Claude-SearchBot":["partial","*",5,1],"anthropic-ai":["partial","*",5,1],"PerplexityBot":["partial","*",5,1],"Perplexity-User":["partial","*",5,1],"Google-Extended":["partial","*",5,1],"Applebot-Extended":["partial","*",5,1],"CCBot":["partial","*",5,1],"Bytespider":["partial","*",5,1],"Amazonbot":["partial","*",5,1],"meta-externalagent":["partial","*",5,1],"cohere-ai":["partial","*",5,1],"Diffbot":["partial","*",5,1],"ContextFetchBot":["partial","*",5,1]},"llms":{"status":"missing","http":404,"bytes":103409},"home":{"status":"ok","http":200,"url":"https://www.acefitness.org/","html_tokens":34902,"markdown_tokens":339,"pruned_tokens":40}},
{"rank":7645,"domain":"rspca.org.uk","origin":"https://rspca.org.uk","robots":{"status":"ok","http":200,"url":"https://www.rspca.org.uk/robots.txt","bytes":4103,"groups":68},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["partial","*",64,0],"OAI-SearchBot":["partial","*",64,0],"ClaudeBot":["partial","*",64,0],"Claude-User":["partial","*",64,0],"Claude-SearchBot":["partial","*",64,0],"anthropic-ai":["partial","*",64,0],"PerplexityBot":["partial","*",64,0],"Perplexity-User":["partial","*",64,0],"Google-Extended":["partial","*",64,0],"Applebot-Extended":["partial","*",64,0],"CCBot":["partial","*",64,0],"Bytespider":["partial","*",64,0],"Amazonbot":["partial","*",64,0],"meta-externalagent":["partial","*",64,0],"cohere-ai":["partial","*",64,0],"Diffbot":["partial","*",64,0],"ContextFetchBot":["partial","*",64,0]},"llms":{"status":"missing","http":404,"bytes":195539},"home":{"status":"ok","http":200,"url":"https://www.rspca.org.uk/","html_tokens":93853,"markdown_tokens":737,"pruned_tokens":148}},
{"rank":7646,"domain":"research.com","origin":"https://research.com","robots":{"status":"ok","http":200,"url":"https://research.com/robots.txt","bytes":1106,"groups":27},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["blocked","named",1,0],"OAI-SearchBot":["partial","*",5,0],"ClaudeBot":["partial","*",5,0],"Claude-User":["partial","*",5,0],"Claude-SearchBot":["partial","*",5,0],"anthropic-ai":["partial","*",5,0],"PerplexityBot":["partial","*",5,0],"Perplexity-User":["partial","*",5,0],"Google-Extended":["partial","*",5,0],"Applebot-Extended":["partial","*",5,0],"CCBot":["blocked","named",1,0],"Bytespider":["partial","*",5,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["partial","*",5,0],"cohere-ai":["partial","*",5,0],"Diffbot":["partial","*",5,0],"ContextFetchBot":["partial","*",5,0]},"llms":{"status":"missing","http":404,"bytes":78285},"home":{"status":"http_error","http":403,"url":"https://research.com/"}},
{"rank":7647,"domain":"marketo.com","origin":"https://marketo.com","robots":{"status":"unreachable","http":null,"url":"https://marketo.com/robots.txt","bytes":0,"groups":0,"error":"<StreamReset stream_id:1, error_code:2, remote_reset:True>"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":7648,"domain":"aptoide.com","origin":"https://aptoide.com","robots":{"status":"ok","http":200,"url":"https://aptoide.com/robots.txt","bytes":39,"groups":1},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"html","http":200,"bytes":499751},"home":{"status":"ok","http":200,"url":"https://en.aptoide.com/","html_tokens":124760,"markdown_tokens":220,"pruned_tokens":141}},
{"rank":7649,"domain":"short.gy","origin":"https://short.gy","robots":{"status":"ok","http":200,"url":"https://short.gy/robots.txt","bytes":23,"groups":1},"bots":{"GPTBot":["allowed","*",0,1],"ChatGPT-User":["allowed","*",0,1],"OAI-SearchBot":["allowed","*",0,1],"ClaudeBot":["allowed","*",0,1],"Claude-User":["allowed","*",0,1],"Claude-SearchBot":["allowed","*",0,1],"anthropic-ai":["allowed","*",0,1],"PerplexityBot":["allowed","*",0,1],"Perplexity-User":["allowed","*",0,1],"Google-Extended":["allowed","*",0,1],"Applebot-Extended":["allowed","*",0,1],"CCBot":["allowed","*",0,1],"Bytespider":["allowed","*",0,1],"Amazonbot":["allowed","*",0,1],"meta-externalagent":["allowed","*",0,1],"cohere-ai":["allowed","*",0,1],"Diffbot":["allowed","*",0,1],"ContextFetchBot":["allowed","*",0,1]},"llms":{"status":"missing","http":404,"bytes":6215},"home":{"status":"ok","http":200,"url":"https://short.io/","html_tokens":44257,"markdown_tokens":1058,"pruned_tokens":112}},
{"rank":7650,"domain":"newswire.com","origin":"https://newswire.com","robots":{"status":"ok","http":200,"url":"https://www.newswire.com/robots.txt","bytes":4696,"groups":111},"bots":{"GPTBot":["partial","*",10,0],"ChatGPT-User":["partial","*",10,0],"OAI-SearchBot":["partial","*",10,0],"ClaudeBot":["partial","*",10,0],"Claude-User":["partial","*",10,0],"Claude-SearchBot":["partial","*",10,0],"anthropic-ai":["partial","*",10,0],"PerplexityBot":["partial","*",10,0],"Perplexity-User":["partial","*",10,0],"Google-Extended":["partial","*",10,0],"Applebot-Extended":["partial","*",10,0],"CCBot":["partial","*",10,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["partial","*",10,0],"meta-externalagent":["partial","*",10,0],"cohere-ai":["partial","*",10,0],"Diffbot":["partial","*",10,0],"ContextFetchBot":["partial","*",10,0]},"llms":{"status":"disallowed","error":"robots.txt of www.newswire.com not readable (refused)"},"home":{"status":"disallowed","error":"robots.txt of www.newswire.com not readable (refused)"}},
{"rank":7651,"domain":"bundesverfassungsgericht.de","origin":"https://bundesverfassungsgericht.de","robots":{"status":"ok","http":200,"url":"https://bundesverfassungsgericht.de/robots.txt","bytes":260,"groups":1},"bots":{"GPTBot":["partial","*",3,3],"ChatGPT-User":["partial","*",3,3],"OAI-SearchBot":["partial","*",3,3],"ClaudeBot":["partial","*",3,3],"Claude-User":["partial","*",3,3],"Claude-SearchBot":["partial","*",3,3],"anthropic-ai":["partial","*",3,3],"PerplexityBot":["partial","*",3,3],"Perplexity-User":["partial","*",3,3],"Google-Extended":["partial","*",3,3],"Applebot-Extended":["partial","*",3,3],"CCBot":["partial","*",3,3],"Bytespider":["partial","*",3,3],"Amazonbot":["partial","*",3,3],"meta-externalagent":["partial","*",3,3],"cohere-ai":["partial","*",3,3],"Diffbot":["partial","*",3,3],"ContextFetchBot":["partial","*",3,3]},"llms":{"status":"missing","http":404,"bytes":97134},"home":{"status":"ok","http":200,"url":"https://www.bundesverfassungsgericht.de/DE/Home/home_node.html","html_tokens":28910,"markdown_tokens":137,"pruned_tokens":137}},
{"rank":7652,"domain":"angularjs.org","origin":"https://angularjs.org","robots":{"status":"missing","http":404,"url":"https://angularjs.org/robots.txt","bytes":21376,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":21376},"home":{"status":"ok","http":200,"url":"https://angularjs.org/","html_tokens":8614,"markdown_tokens":223,"pruned_tokens":155}},
{"rank":7653,"domain":"qidigroup.com","origin":"http://qidigroup.com","robots":{"status":"missing","http":404,"url":"http://www.qidigroup.com/robots.txt","bytes":208,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":206},"home":{"status":"ok","http":200,"url":"http://www.qidigroup.com/","html_tokens":15377,"markdown_tokens":64,"pruned_tokens":64}},
{"rank":7654,"domain":"analyticsinsight.net","origin":"https://analyticsinsight.net","robots":{"status":"ok","http":200,"url":"https://www.analyticsinsight.net/robots.txt","bytes":135,"groups":1},"bots":{"GPTBot":["allowed","*",0,1],"ChatGPT-User":["allowed","*",0,1],"OAI-SearchBot":["allowed","*",0,1],"ClaudeBot":["allowed","*",0,1],"Claude-User":["allowed","*",0,1],"Claude-SearchBot":["allowed","*",0,1],"anthropic-ai":["allowed","*",0,1],"PerplexityBot":["allowed","*",0,1],"Perplexity-User":["allowed","*",0,1],"Google-Extended":["allowed","*",0,1],"Applebot-Extended":["allowed","*",0,1],"CCBot":["allowed","*",0,1],"Bytespider":["allowed","*",0,1],"Amazonbot":["allowed","*",0,1],"meta-externalagent":["allowed","*",0,1],"cohere-ai":["allowed","*",0,1],"Diffbot":["allowed","*",0,1],"ContextFetchBot":["allowed","*",0,1]},"llms":{"status":"present","http":200,"bytes":16795},"home":{"status":"ok","http":200,"url":"https://www.analyticsinsight.net/","html_tokens":409352,"markdown_tokens":266,"pruned_tokens":112}},
{"rank":7655,"domain":"hawaiinewsnow.com","origin":"https://hawaiinewsnow.com","robots":{"status":"ok","http":200,"url":"https://www.hawaiinewsnow.com/robots.txt","bytes":1225,"groups":2},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["blocked","named",1,0],"OAI-SearchBot":["partial","*",19,3],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",19,3],"Claude-SearchBot":["partial","*",19,3],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["partial","*",19,3],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["partial","*",19,3],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["partial","*",19,3],"cohere-ai":["partial","*",19,3],"Diffbot":["partial","*",19,3],"ContextFetchBot":["partial","*",19,3]},"llms":{"status":"missing","http":404,"bytes":96137},"home":{"status":"ok","http":200,"url":"https://www.hawaiinewsnow.com/","html_tokens":153315,"markdown_tokens":194,"pruned_tokens":133}},
{"rank":7656,"domain":"cilalisez.com","origin":"https://cilalisez.com","robots":{"status":"ok","http":200,"url":"https://cilalisez.com/robots.txt","bytes":76,"groups":1},"bots":{"GPTBot":["partial","*",1,0],"ChatGPT-User":["partial","*",1,0],"OAI-SearchBot":["partial","*",1,0],"ClaudeBot":["partial","*",1,0],"Claude-User":["partial","*",1,0],"Claude-SearchBot":["partial","*",1,0],"anthropic-ai":["partial","*",1,0],"PerplexityBot":["partial","*",1,0],"Perplexity-User":["partial","*",1,0],"Google-Extended":["partial","*",1,0],"Applebot-Extended":["partial","*",1,0],"CCBot":["partial","*",1,0],"Bytespider":["partial","*",1,0],"Amazonbot":["partial","*",1,0],"meta-externalagent":["partial","*",1,0],"cohere-ai":["partial","*",1,0],"Diffbot":["partial","*",1,0],"ContextFetchBot":["partial","*",1,0]},"llms":{"status":"missing","http":404,"bytes":642},"home":{"status":"ok","http":200,"url":"https://cilalisez.com/","html_tokens":8559,"markdown_tokens":2139,"pruned_tokens":130}},
{"rank":7657,"domain":"dailyhive.com","origin":"https://dailyhive.com","robots":{"status":"ok","http":200,"url":"https://dailyhive.com/robots.txt","bytes":1854,"groups":3},"bots":{"GPTBot":["allowed","named",0,1],"ChatGPT-User":["allowed","named",0,1],"OAI-SearchBot":["allowed","named",0,1],"ClaudeBot":["allowed","named",0,1],"Claude-User":["allowed","named",0,1],"Claude-SearchBot":["allowed","named",0,1],"anthropic-ai":["allowed","named",0,1],"PerplexityBot":["allowed","named",0,1],"Perplexity-User":["partial","*",1,0],"Google-Extended":["allowed","named",0,1],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["partial","*",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",1,0]},"llms":{"status":"missing","http":404,"bytes":18473},"home":{"status":"ok","http":200,"url":"https://dailyhive.com/","html_tokens":111267,"markdown_tokens":106,"pruned_tokens":106}},
{"rank":7658,"domain":"sasac.gov.cn","origin":"https://sasac.gov.cn","robots":{"status":"unreachable","http":null,"url":"https://sasac.gov.cn/robots.txt","bytes":0,"groups":0,"error":"cannot resolve sasac.gov.cn"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":7659,"domain":"spinsfest.com","origin":"https://spinsfest.com","robots":{"status":"ok","http":200,"url":"https://spinsfest.com/robots.txt","bytes":67,"groups":1},"bots":{"GPTBot":["partial","*",1,1],"ChatGPT-User":["partial","*",1,1],"OAI-SearchBot":["partial","*",1,1],"ClaudeBot":["partial","*",1,1],"Claude-User":["partial","*",1,1],"Claude-SearchBot":["partial","*",1,1],"anthropic-ai":["partial","*",1,1],"PerplexityBot":["partial","*",1,1],"Perplexity-User":["partial","*",1,1],"Google-Extended":["partial","*",1,1],"Applebot-Extended":["partial","*",1,1],"CCBot":["partial","*",1,1],"Bytespider":["partial","*",1,1],"Amazonbot":["partial","*",1,1],"meta-externalagent":["partial","*",1,1],"cohere-ai":["partial","*",1,1],"Diffbot":["partial","*",1,1],"ContextFetchBot":["partial","*",1,1]},"llms":{"status":"missing","http":404,"bytes":44212},"home":{"status":"ok","http":200,"url":"https://spinsfest.com/","html_tokens":15212,"markdown_tokens":2080,"pruned_tokens":632}},
{"rank":7660,"domain":"mydomain.org","origin":"https://mydomain.org","robots":{"status":"html","http":200,"url":"https://mydomain.org/robots.txt","bytes":777,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":7661,"domain":"google.com.kh","origin":"https://google.com.kh","robots":{"status":"ok","http":200,"url":"https://www.google.com.kh/robots.txt","bytes":6624,"groups":4},"bots":{"GPTBot":["partial","*",169,64],"ChatGPT-User":["partial","*",169,64],"OAI-SearchBot":["partial","*",169,64],"ClaudeBot":["partial","*",169,64],"Claude-User":["partial","*",169,64],"Claude-SearchBot":["partial","*",169,64],"anthropic-ai":["partial","*",169,64],"PerplexityBot":["partial","*",169,64],"Perplexity-User":["partial","*",169,64],"Google-Extended":["partial","*",169,64],"Applebot-Extended":["partial","*",169,64],"CCBot":["partial","*",169,64],"Bytespider":["partial","*",169,64],"Amazonbot":["partial","*",169,64],"meta-externalagent":["partial","*",169,64],"cohere-ai":["partial","*",169,64],"Diffbot":["partial","*",169,64],"ContextFetchBot":["partial","*",169,64]},"llms":{"status":"missing","http":404,"bytes":1569},"home":{"status":"ok","http":200,"url":"https://www.google.com/","html_tokens":21382,"markdown_tokens":30,"pruned_tokens":30}},
{"rank":7662,"domain":"aia.org","origin":"https://aia.org","robots":{"status":"refused","http":403,"url":"https://aia.org/robots.txt","bytes":560,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":7663,"domain":"greenbiz.com","origin":"https://greenbiz.com","robots":{"status":"html","http":200,"url":"https://trellis.net/","bytes":126213,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":7664,"domain":"clarku.edu","origin":"https://clarku.edu","robots":{"status":"ok","http":200,"url":"https://www.clarku.edu/robots.txt","bytes":6699,"groups":1},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"missing","http":404,"bytes":127217},"home":{"status":"ok","http":200,"url":"https://www.clarku.edu/","html_tokens":40469,"markdown_tokens":445,"pruned_tokens":236}},
{"rank":7665,"domain":"110mb.com","origin":"https://110mb.com","robots":{"status":"unreachable","http":null,"url":"https://110mb.com/robots.txt","bytes":0,"groups":0,"error":"All connection attempts failed"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":7666,"domain":"zcool.com.cn","origin":"https://zcool.com.cn","robots":{"status":"ok","http":200,"url":"https://www.zcool.com.cn/robots.txt","bytes":578,"groups":1},"bots":{"GPTBot":["partial","*",25,0],"ChatGPT-User":["partial","*",25,0],"OAI-SearchBot":["partial","*",25,0],"ClaudeBot":["partial","*",25,0],"Claude-User":["partial","*",25,0],"Claude-SearchBot":["partial","*",25,0],"anthropic-ai":["partial","*",25,0],"PerplexityBot":["partial","*",25,0],"Perplexity-User":["partial","*",25,0],"Google-Extended":["partial","*",25,0],"Applebot-Extended":["partial","*",25,0],"CCBot":["partial","*",25,0],"Bytespider":["partial","*",25,0],"Amazonbot":["partial","*",25,0],"meta-externalagent":["partial","*",25,0],"cohere-ai":["partial","*",25,0],"Diffbot":["partial","*",25,0],"ContextFetchBot":["partial","*",25,0]},"llms":{"status":"missing","http":404,"bytes":304},"home":{"status":"ok","http":200,"url":"https://www.zcool.com.cn/","html_tokens":4003,"markdown_tokens":0,"pruned_tokens":0}},
{"rank":7667,"domain":"humanite.fr","origin":"https://humanite.fr","robots":{"status":"ok","http":200,"url":"https://humanite.fr/robots.txt","bytes":1307,"groups":24},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["blocked","named",1,0],"OAI-SearchBot":["blocked","named",1,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["blocked","named",1,0],"Claude-SearchBot":["blocked","named",1,0],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["blocked","named",1,0],"Google-Extended":["blocked","named",2,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["partial","*",8,1],"ContextFetchBot":["partial","*",8,1]},"llms":{"status":"missing","http":404,"bytes":247307},"home":{"status":"ok","http":200,"url":"https://www.humanite.fr/","html_tokens":93871,"markdown_tokens":80,"pruned_tokens":80}},
{"rank":7668,"domain":"pensoft.net","origin":"https://pensoft.net","robots":{"status":"ok","http":200,"url":"https://pensoft.net/robots.txt","bytes":14,"groups":1},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"missing","http":404,"bytes":196},"home":{"status":"ok","http":200,"url":"https://pensoft.net/","html_tokens":4383,"markdown_tokens":175,"pruned_tokens":175}},
{"rank":7669,"domain":"colliers.com","origin":"https://colliers.com","robots":{"status":"refused","http":403,"url":"https://www.colliers.com/robots.txt","bytes":5597,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":7670,"domain":"clan.su","origin":"https://clan.su","robots":{"status":"ok","http":200,"url":"https://www.ucoz.ru/robots.txt","bytes":121,"groups":1},"bots":{"GPTBot":["partial","*",5,0],"ChatGPT-User":["partial","*",5,0],"OAI-SearchBot":["partial","*",5,0],"ClaudeBot":["partial","*",5,0],"Claude-User":["partial","*",5,0],"Claude-SearchBot":["partial","*",5,0],"anthropic-ai":["partial","*",5,0],"PerplexityBot":["partial","*",5,0],"Perplexity-User":["partial","*",5,0],"Google-Extended":["partial","*",5,0],"Applebot-Extended":["partial","*",5,0],"CCBot":["partial","*",5,0],"Bytespider":["partial","*",5,0],"Amazonbot":["partial","*",5,0],"meta-externalagent":["partial","*",5,0],"cohere-ai":["partial","*",5,0],"Diffbot":["partial","*",5,0],"ContextFetchBot":["partial","*",5,0]},"llms":{"status":"present","http":200,"bytes":33987},"home":{"status":"ok","http":200,"url":"https://www.ucoz.ru/","html_tokens":13583,"markdown_tokens":183,"pruned_tokens":133}},
{"rank":7671,"domain":"opec.org","origin":"https://opec.org","robots":{"status":"refused","http":403,"url":"https://www.opec.org/404.html","bytes":5775,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":7672,"domain":"hypersecu.com","origin":"https://hypersecu.com","robots":{"status":"ok","http":200,"url":"https://hypersecu.com/robots.txt","bytes":491,"groups":4},"bots":{"GPTBot":["partial","*",1,1],"ChatGPT-User":["partial","*",1,1],"OAI-SearchBot":["partial","*",1,1],"ClaudeBot":["partial","*",1,1],"Claude-User":["partial","*",1,1],"Claude-SearchBot":["partial","*",1,1],"anthropic-ai":["partial","*",1,1],"PerplexityBot":["partial","*",1,1],"Perplexity-User":["partial","*",1,1],"Google-Extended":["partial","*",1,1],"Applebot-Extended":["partial","*",1,1],"CCBot":["partial","*",1,1],"Bytespider":["partial","*",1,1],"Amazonbot":["partial","*",1,1],"meta-externalagent":["partial","*",1,1],"cohere-ai":["partial","*",1,1],"Diffbot":["partial","*",1,1],"ContextFetchBot":["partial","*",1,1]},"llms":{"status":"present","http":200,"bytes":3305},"home":{"status":"ok","http":200,"url":"https://www.hypersecu.com/","html_tokens":188049,"markdown_tokens":819,"pruned_tokens":276}},
{"rank":7673,"domain":"litek.cn","origin":"http://litek.cn","robots":{"status":"missing","http":404,"url":"http://litek.cn/robots.txt","bytes":1231,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":1231},"home":{"status":"ok","http":200,"url":"http://litek.cn/","html_tokens":176,"markdown_tokens":0,"pruned_tokens":0}},
{"rank":7674,"domain":"lasexta.com","origin":"https://lasexta.com","robots":{"status":"ok","http":200,"url":"https://www.lasexta.com/robots.txt","bytes":2173,"groups":2},"bots":{"GPTBot":["partial","*",14,0],"ChatGPT-User":["partial","*",14,0],"OAI-SearchBot":["partial","*",14,0],"ClaudeBot":["partial","*",14,0],"Claude-User":["partial","*",14,0],"Claude-SearchBot":["partial","*",14,0],"anthropic-ai":["partial","*",14,0],"PerplexityBot":["partial","*",14,0],"Perplexity-User":["partial","*",14,0],"Google-Extended":["partial","*",14,0],"Applebot-Extended":["partial","*",14,0],"CCBot":["partial","*",14,0],"Bytespider":["partial","*",14,0],"Amazonbot":["partial","*",14,0],"meta-externalagent":["partial","*",14,0],"cohere-ai":["partial","*",14,0],"Diffbot":["partial","*",14,0],"ContextFetchBot":["partial","*",14,0]},"llms":{"status":"missing","http":404,"bytes":232926},"home":{"status":"ok","http":200,"url":"https://www.lasexta.com/","html_tokens":139919,"markdown_tokens":1002,"pruned_tokens":165}},
{"rank":7675,"domain":"digital.nhs.uk","origin":"https://digital.nhs.uk","robots":{"status":"refused","http":403,"url":"https://digital.nhs.uk/robots.txt","bytes":5577,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":7676,"domain":"saksfifthavenue.com","origin":"https://saksfifthavenue.com","robots":{"status":"refused","http":403,"url":"https://www.saksfifthavenue.com/robots.txt","bytes":779,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":7677,"domain":"udayton.edu","origin":"https://udayton.edu","robots":{"status":"ok","http":200,"url":"https://udayton.edu/robots.txt","bytes":9234,"groups":3},"bots":{"GPTBot":["partial","named",1,42],"ChatGPT-User":["partial","named",1,42],"OAI-SearchBot":["partial","*",177,0],"ClaudeBot":["partial","named",1,42],"Claude-User":["partial","*",177,0],"Claude-SearchBot":["partial","*",177,0],"anthropic-ai":["partial","named",1,42],"PerplexityBot":["partial","named",1,42],"Perplexity-User":["partial","*",177,0],"Google-Extended":["partial","named",1,42],"Applebot-Extended":["partial","named",1,0],"CCBot":["partial","*",177,0],"Bytespider":["partial","*",177,0],"Amazonbot":["partial","*",177,0],"meta-externalagent":["partial","*",177,0],"cohere-ai":["partial","*",177,0],"Diffbot":["partial","*",177,0],"ContextFetchBot":["partial","*",177,0]},"llms":{"status":"html","http":200,"bytes":27967},"home":{"status":"ok","http":200,"url":"https://udayton.edu/","html_tokens":26079,"markdown_tokens":489,"pruned_tokens":153}},
{"rank":7678,"domain":"wits.ac.za","origin":"https://wits.ac.za","robots":{"status":"html","http":200,"url":"https://www.wits.ac.za","bytes":33290,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":7679,"domain":"1xslot-casino.net","origin":"https://1xslot-casino.net","robots":{"status":"unreachable","http":null,"url":"https://1xslot-casino.net/robots.txt","bytes":0,"groups":0,"error":"ConnectError"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":7680,"domain":"informit.com","origin":"https://informit.com","robots":{"status":"ok","http":200,"url":"https://informit.com/robots.txt","bytes":719,"groups":1},"bots":{"GPTBot":["partial","*",22,0],"ChatGPT-User":["partial","*",22,0],"OAI-SearchBot":["partial","*",22,0],"ClaudeBot":["partial","*",22,0],"Claude-User":["partial","*",22,0],"Claude-SearchBot":["partial","*",22,0],"anthropic-ai":["partial","*",22,0],"PerplexityBot":["partial","*",22,0],"Perplexity-User":["partial","*",22,0],"Google-Extended":["partial","*",22,0],"Applebot-Extended":["partial","*",22,0],"CCBot":["partial","*",22,0],"Bytespider":["partial","*",22,0],"Amazonbot":["partial","*",22,0],"meta-externalagent":["partial","*",22,0],"cohere-ai":["partial","*",22,0],"Diffbot":["partial","*",22,0],"ContextFetchBot":["partial","*",22,0]},"llms":{"status":"html","http":200,"bytes":95240},"home":{"status":"ok","http":200,"url":"https://www.informit.com/","html_tokens":27256,"markdown_tokens":203,"pruned_tokens":61}},
{"rank":7681,"domain":"rover.com","origin":"https://rover.com","robots":{"status":"ok","http":200,"url":"https://www.rover.com/robots.txt","bytes":958,"groups":2},"bots":{"GPTBot":["partial","*",25,0],"ChatGPT-User":["partial","*",25,0],"OAI-SearchBot":["partial","*",25,0],"ClaudeBot":["partial","*",25,0],"Claude-User":["partial","*",25,0],"Claude-SearchBot":["partial","*",25,0],"anthropic-ai":["partial","*",25,0],"PerplexityBot":["partial","*",25,0],"Perplexity-User":["partial","*",25,0],"Google-Extended":["partial","*",25,0],"Applebot-Extended":["partial","*",25,0],"CCBot":["partial","*",25,0],"Bytespider":["partial","*",25,0],"Amazonbot":["partial","*",25,0],"meta-externalagent":["partial","*",25,0],"cohere-ai":["partial","*",25,0],"Diffbot":["partial","*",25,0],"ContextFetchBot":["partial","*",25,0]},"llms":{"status":"present","http":200,"bytes":3072},"home":{"status":"http_error","http":403,"url":"https://www.rover.com/"}},
{"rank":7682,"domain":"asiae.co.kr","origin":"https://asiae.co.kr","robots":{"status":"ok","http":200,"url":"https://www.asiae.co.kr/robots.txt","bytes":861,"groups":15},"bots":{"GPTBot":["blocked","*",4,0],"ChatGPT-User":["blocked","*",4,0],"OAI-SearchBot":["blocked","*",4,0],"ClaudeBot":["blocked","*",4,0],"Claude-User":["blocked","*",4,0],"Claude-SearchBot":["blocked","*",4,0],"anthropic-ai":["blocked","*",4,0],"PerplexityBot":["blocked","*",4,0],"Perplexity-User":["blocked","*",4,0],"Google-Extended":["blocked","*",4,0],"Applebot-Extended":["blocked","*",4,0],"CCBot":["blocked","*",4,0],"Bytespider":["blocked","*",4,0],"Amazonbot":["blocked","*",4,0],"meta-externalagent":["blocked","*",4,0],"cohere-ai":["blocked","*",4,0],"Diffbot":["blocked","*",4,0],"ContextFetchBot":["blocked","*",4,0]},"llms":{"status":"disallowed"},"home":{"status":"disallowed"}},
{"rank":7683,"domain":"comsenz.com","origin":"https://comsenz.com","robots":{"status":"unreachable","http":null,"url":"https://comsenz.com/robots.txt","bytes":0,"groups":0,"error":"comsenz.com is not a public address"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":7684,"domain":"kodansha.co.jp","origin":"https://kodansha.co.jp","robots":{"status":"missing","http":404,"url":"https://kodansha.co.jp/robots.txt","bytes":82,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":80},"home":{"status":"ok","http":200,"url":"https://www.kodansha.co.jp/","html_tokens":92047,"markdown_tokens":654,"pruned_tokens":135}},
{"rank":7685,"domain":"understood.org","origin":"https://understood.org","robots":{"status":"ok","http":200,"url":"https://www.understood.org/robots.txt","bytes":272,"groups":1},"bots":{"GPTBot":["partial","*",10,0],"ChatGPT-User":["partial","*",10,0],"OAI-SearchBot":["partial","*",10,0],"ClaudeBot":["partial","*",10,0],"Claude-User":["partial","*",10,0],"Claude-SearchBot":["partial","*",10,0],"anthropic-ai":["partial","*",10,0],"PerplexityBot":["partial","*",10,0],"Perplexity-User":["partial","*",10,0],"Google-Extended":["partial","*",10,0],"Applebot-Extended":["partial","*",10,0],"CCBot":["partial","*",10,0],"Bytespider":["partial","*",10,0],"Amazonbot":["partial","*",10,0],"meta-externalagent":["partial","*",10,0],"cohere-ai":["partial","*",10,0],"Diffbot":["partial","*",10,0],"ContextFetchBot":["partial","*",10,0]},"llms":{"status":"present","http":200,"bytes":5695},"home":{"status":"ok","http":200,"url":"https://www.understood.org/","html_tokens":99149,"markdown_tokens":248,"pruned_tokens":88}},
{"rank":7686,"domain":"3dn.ru","origin":"https://3dn.ru","robots":{"status":"ok","http":200,"url":"https://www.ucoz.ru/robots.txt","bytes":121,"groups":1},"bots":{"GPTBot":["partial","*",5,0],"ChatGPT-User":["partial","*",5,0],"OAI-SearchBot":["partial","*",5,0],"ClaudeBot":["partial","*",5,0],"Claude-User":["partial","*",5,0],"Claude-SearchBot":["partial","*",5,0],"anthropic-ai":["partial","*",5,0],"PerplexityBot":["partial","*",5,0],"Perplexity-User":["partial","*",5,0],"Google-Extended":["partial","*",5,0],"Applebot-Extended":["partial","*",5,0],"CCBot":["partial","*",5,0],"Bytespider":["partial","*",5,0],"Amazonbot":["partial","*",5,0],"meta-externalagent":["partial","*",5,0],"cohere-ai":["partial","*",5,0],"Diffbot":["partial","*",5,0],"ContextFetchBot":["partial","*",5,0]},"llms":{"status":"present","http":200,"bytes":33987},"home":{"status":"ok","http":200,"url":"https://www.ucoz.ru/","html_tokens":13583,"markdown_tokens":183,"pruned_tokens":133}},
{"rank":7687,"domain":"hzgjp.com","origin":"https://hzgjp.com","robots":{"status":"ok","http":200,"url":"https://hzgjp.com/robots.txt","bytes":129,"groups":1},"bots":{"GPTBot":["partial","*",4,0],"ChatGPT-User":["partial","*",4,0],"OAI-SearchBot":["partial","*",4,0],"ClaudeBot":["partial","*",4,0],"Claude-User":["partial","*",4,0],"Claude-SearchBot":["partial","*",4,0],"anthropic-ai":["partial","*",4,0],"PerplexityBot":["partial","*",4,0],"Perplexity-User":["partial","*",4,0],"Google-Extended":["partial","*",4,0],"Applebot-Extended":["partial","*",4,0],"CCBot":["partial","*",4,0],"Bytespider":["partial","*",4,0],"Amazonbot":["partial","*",4,0],"meta-externalagent":["partial","*",4,0],"cohere-ai":["partial","*",4,0],"Diffbot":["partial","*",4,0],"ContextFetchBot":["partial","*",4,0]},"llms":{"status":"missing","http":404,"bytes":0},"home":{"status":"ok","http":200,"url":"https://hzgjp.com/","html_tokens":21116,"markdown_tokens":19,"pruned_tokens":19}},
{"rank":7688,"domain":"mlssoccer.com","origin":"https://mlssoccer.com","robots":{"status":"ok","http":200,"url":"https://www.mlssoccer.com/robots.txt","bytes":296,"groups":1},"bots":{"GPTBot":["partial","*",13,1],"ChatGPT-User":["partial","*",13,1],"OAI-SearchBot":["partial","*",13,1],"ClaudeBot":["partial","*",13,1],"Claude-User":["partial","*",13,1],"Claude-SearchBot":["partial","*",13,1],"anthropic-ai":["partial","*",13,1],"PerplexityBot":["partial","*",13,1],"Perplexity-User":["partial","*",13,1],"Google-Extended":["partial","*",13,1],"Applebot-Extended":["partial","*",13,1],"CCBot":["partial","*",13,1],"Bytespider":["partial","*",13,1],"Amazonbot":["partial","*",13,1],"meta-externalagent":["partial","*",13,1],"cohere-ai":["partial","*",13,1],"Diffbot":["partial","*",13,1],"ContextFetchBot":["partial","*",13,1]},"llms":{"status":"missing","http":404,"bytes":170865},"home":{"status":"ok","http":200,"url":"https://www.mlssoccer.com/","html_tokens":93632,"markdown_tokens":92,"pruned_tokens":92}},
{"rank":7689,"domain":"google.com.sb","origin":"https://google.com.sb","robots":{"status":"ok","http":200,"url":"https://www.google.com.sb/robots.txt","bytes":6624,"groups":4},"bots":{"GPTBot":["partial","*",169,64],"ChatGPT-User":["partial","*",169,64],"OAI-SearchBot":["partial","*",169,64],"ClaudeBot":["partial","*",169,64],"Claude-User":["partial","*",169,64],"Claude-SearchBot":["partial","*",169,64],"anthropic-ai":["partial","*",169,64],"PerplexityBot":["partial","*",169,64],"Perplexity-User":["partial","*",169,64],"Google-Extended":["partial","*",169,64],"Applebot-Extended":["partial","*",169,64],"CCBot":["partial","*",169,64],"Bytespider":["partial","*",169,64],"Amazonbot":["partial","*",169,64],"meta-externalagent":["partial","*",169,64],"cohere-ai":["partial","*",169,64],"Diffbot":["partial","*",169,64],"ContextFetchBot":["partial","*",169,64]},"llms":{"status":"missing","http":404,"bytes":1569},"home":{"status":"ok","http":200,"url":"https://www.google.com/","html_tokens":21290,"markdown_tokens":30,"pruned_tokens":30}},
{"rank":7690,"domain":"mfa.gov.ua","origin":"https://mfa.gov.ua","robots":{"status":"refused","http":403,"url":"https://mfa.gov.ua/robots.txt","bytes":4543,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":7691,"domain":"pardot.com","origin":"https://pardot.com","robots":{"status":"html","http":200,"url":"https://www.salesforce.com/marketing/b2b-automation/?redirect=pardot.com&bc=DB","bytes":696355,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":7692,"domain":"univ-lorraine.fr","origin":"https://univ-lorraine.fr","robots":{"status":"ok","http":200,"url":"https://univ-lorraine.fr/robots.txt","bytes":178,"groups":2},"bots":{"GPTBot":["partial","*",2,1],"ChatGPT-User":["partial","*",2,1],"OAI-SearchBot":["partial","*",2,1],"ClaudeBot":["partial","*",2,1],"Claude-User":["partial","*",2,1],"Claude-SearchBot":["partial","*",2,1],"anthropic-ai":["partial","*",2,1],"PerplexityBot":["partial","*",2,1],"Perplexity-User":["partial","*",2,1],"Google-Extended":["partial","*",2,1],"Applebot-Extended":["partial","*",2,1],"CCBot":["partial","*",2,1],"Bytespider":["partial","*",2,1],"Amazonbot":["partial","*",2,1],"meta-externalagent":["partial","*",2,1],"cohere-ai":["partial","*",2,1],"Diffbot":["partial","*",2,1],"ContextFetchBot":["partial","*",2,1]},"llms":{"status":"html","http":200,"bytes":347079},"home":{"status":"ok","http":200,"url":"https://www.univ-lorraine.fr/","html_tokens":86564,"markdown_tokens":814,"pruned_tokens":116}},
{"rank":7693,"domain":"line.biz","origin":"https://line.biz","robots":{"status":"ok","http":204,"url":"https://line.biz/robots.txt","bytes":0,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"empty","http":204,"bytes":0},"home":{"status":"error","http":204,"url":"https://line.biz/","error":"unparseable HTML"}},
{"rank":7694,"domain":"sport.gov.cn","origin":"https://www.sport.gov.cn","robots":{"status":"missing","http":404,"url":"https://www.sport.gov.cn/robots.txt","bytes":560,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":558},"home":{"status":"error","http":200,"url":"https://www.sport.gov.cn/","error":"unparseable HTML"}},
{"rank":7695,"domain":"anwb.nl","origin":"https://anwb.nl","robots":{"status":"ok","http":200,"url":"https://www.anwb.nl/robots.txt","bytes":2543,"groups":2},"bots":{"GPTBot":["partial","*",51,0],"ChatGPT-User":["partial","*",51,0],"OAI-SearchBot":["partial","*",51,0],"ClaudeBot":["partial","*",51,0],"Claude-User":["partial","*",51,0],"Claude-SearchBot":["partial","*",51,0],"anthropic-ai":["partial","*",51,0],"PerplexityBot":["partial","*",51,0],"Perplexity-User":["partial","*",51,0],"Google-Extended":["partial","*",51,0],"Applebot-Extended":["partial","*",51,0],"CCBot":["partial","*",51,0],"Bytespider":["partial","*",51,0],"Amazonbot":["partial","*",51,0],"meta-externalagent":["partial","named",1,1],"cohere-ai":["partial","*",51,0],"Diffbot":["partial","*",51,0],"ContextFetchBot":["partial","*",51,0]},"llms":{"status":"missing","http":404,"bytes":410155},"home":{"status":"ok","http":200,"url":"https://www.anwb.nl","html_tokens":155855,"markdown_tokens":81,"pruned_tokens":43}},
{"rank":7696,"domain":"warontherocks.com","origin":"https://warontherocks.com","robots":{"status":"ok","http":200,"url":"https://warontherocks.com/robots.txt","bytes":176,"groups":1},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"other_text","http":200,"bytes":2437},"home":{"status":"ok","http":200,"url":"https://warontherocks.com/","html_tokens":33468,"markdown_tokens":32,"pruned_tokens":32}},
{"rank":7697,"domain":"adultswim.com","origin":"https://adultswim.com","robots":{"status":"refused","http":403,"url":"https://adultswim.com/robots.txt","bytes":118,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":7698,"domain":"trustarc.com","origin":"https://trustarc.com","robots":{"status":"ok","http":200,"url":"https://trustarc.com/robots.txt","bytes":213,"groups":1},"bots":{"GPTBot":["partial","*",4,0],"ChatGPT-User":["partial","*",4,0],"OAI-SearchBot":["partial","*",4,0],"ClaudeBot":["partial","*",4,0],"Claude-User":["partial","*",4,0],"Claude-SearchBot":["partial","*",4,0],"anthropic-ai":["partial","*",4,0],"PerplexityBot":["partial","*",4,0],"Perplexity-User":["partial","*",4,0],"Google-Extended":["partial","*",4,0],"Applebot-Extended":["partial","*",4,0],"CCBot":["partial","*",4,0],"Bytespider":["partial","*",4,0],"Amazonbot":["partial","*",4,0],"meta-externalagent":["partial","*",4,0],"cohere-ai":["partial","*",4,0],"Diffbot":["partial","*",4,0],"ContextFetchBot":["partial","*",4,0]},"llms":{"status":"http_error","http":403,"bytes":5565},"home":{"status":"http_error","http":403,"url":"https://trustarc.com/"}},
{"rank":7699,"domain":"bd.com","origin":"https://bd.com","robots":{"status":"ok","http":200,"url":"https://www.bd.com/robots.txt","bytes":2415,"groups":1},"bots":{"GPTBot":["partial","*",12,0],"ChatGPT-User":["partial","*",12,0],"OAI-SearchBot":["partial","*",12,0],"ClaudeBot":["partial","*",12,0],"Claude-User":["partial","*",12,0],"Claude-SearchBot":["partial","*",12,0],"anthropic-ai":["partial","*",12,0],"PerplexityBot":["partial","*",12,0],"Perplexity-User":["partial","*",12,0],"Google-Extended":["partial","*",12,0],"Applebot-Extended":["partial","*",12,0],"CCBot":["partial","*",12,0],"Bytespider":["partial","*",12,0],"Amazonbot":["partial","*",12,0],"meta-externalagent":["partial","*",12,0],"cohere-ai":["partial","*",12,0],"Diffbot":["partial","*",12,0],"ContextFetchBot":["partial","*",12,0]},"llms":{"status":"present","http":200,"bytes":2366},"home":{"status":"ok","http":200,"url":"https://www.bd.com/en-us","html_tokens":32348,"markdown_tokens":144,"pruned_tokens":118}},
{"rank":7700,"domain":"google.com.et","origin":"https://google.com.et","robots":{"status":"ok","http":200,"url":"https://www.google.com.et/robots.txt","bytes":6624,"groups":4},"bots":{"GPTBot":["partial","*",169,64],"ChatGPT-User":["partial","*",169,64],"OAI-SearchBot":["partial","*",169,64],"ClaudeBot":["partial","*",169,64],"Claude-User":["partial","*",169,64],"Claude-SearchBot":["partial","*",169,64],"anthropic-ai":["partial","*",169,64],"PerplexityBot":["partial","*",169,64],"Perplexity-User":["partial","*",169,64],"Google-Extended":["partial","*",169,64],"Applebot-Extended":["partial","*",169,64],"CCBot":["partial","*",169,64],"Bytespider":["partial","*",169,64],"Amazonbot":["partial","*",169,64],"meta-externalagent":["partial","*",169,64],"cohere-ai":["partial","*",169,64],"Diffbot":["partial","*",169,64],"ContextFetchBot":["partial","*",169,64]},"llms":{"status":"missing","http":404,"bytes":1569},"home":{"status":"ok","http":200,"url":"https://www.google.com/","html_tokens":21295,"markdown_tokens":30,"pruned_tokens":30}},
{"rank":7701,"domain":"southparkstudios.com","origin":"https://southparkstudios.com","robots":{"status":"ok","http":200,"url":"https://southpark.cc.com/robots.txt","bytes":523,"groups":2},"bots":{"GPTBot":["partial","*",4,0],"ChatGPT-User":["partial","*",4,0],"OAI-SearchBot":["partial","*",4,0],"ClaudeBot":["partial","*",4,0],"Claude-User":["partial","*",4,0],"Claude-SearchBot":["partial","*",4,0],"anthropic-ai":["partial","*",4,0],"PerplexityBot":["partial","*",4,0],"Perplexity-User":["partial","*",4,0],"Google-Extended":["partial","*",4,0],"Applebot-Extended":["partial","*",4,0],"CCBot":["partial","*",4,0],"Bytespider":["partial","*",4,0],"Amazonbot":["partial","*",4,0],"meta-externalagent":["partial","*",4,0],"cohere-ai":["partial","*",4,0],"Diffbot":["partial","*",4,0],"ContextFetchBot":["partial","*",4,0]},"llms":{"status":"missing","http":404,"bytes":244954},"home":{"status":"ok","http":200,"url":"https://southpark.cc.com/","html_tokens":91211,"markdown_tokens":87,"pruned_tokens":87}},
{"rank":7702,"domain":"ireland.com","origin":"https://ireland.com","robots":{"status":"ok","http":200,"url":"https://www.ireland.com/robots.txt","bytes":1589,"groups":2},"bots":{"GPTBot":["partial","*",21,1],"ChatGPT-User":["partial","*",21,1],"OAI-SearchBot":["partial","*",21,1],"ClaudeBot":["partial","*",21,1],"Claude-User":["partial","*",21,1],"Claude-SearchBot":["partial","*",21,1],"anthropic-ai":["partial","*",21,1],"PerplexityBot":["partial","*",21,1],"Perplexity-User":["partial","*",21,1],"Google-Extended":["partial","*",21,1],"Applebot-Extended":["partial","*",21,1],"CCBot":["partial","*",21,1],"Bytespider":["partial","*",21,1],"Amazonbot":["partial","*",21,1],"meta-externalagent":["partial","*",21,1],"cohere-ai":["partial","*",21,1],"Diffbot":["partial","*",21,1],"ContextFetchBot":["partial","*",21,1]},"llms":{"status":"missing","http":404,"bytes":613648},"home":{"status":"ok","http":200,"url":"https://www.ireland.com/en-us/","html_tokens":247910,"markdown_tokens":753,"pruned_tokens":114}},
{"rank":7703,"domain":"ecmwf.int","origin":"https://ecmwf.int","robots":{"status":"ok","http":200,"url":"https://www.ecmwf.int/robots.txt","bytes":2027,"groups":1},"bots":{"GPTBot":["partial","*",34,18],"ChatGPT-User":["partial","*",34,18],"OAI-SearchBot":["partial","*",34,18],"ClaudeBot":["partial","*",34,18],"Claude-User":["partial","*",34,18],"Claude-SearchBot":["partial","*",34,18],"anthropic-ai":["partial","*",34,18],"PerplexityBot":["partial","*",34,18],"Perplexity-User":["partial","*",34,18],"Google-Extended":["partial","*",34,18],"Applebot-Extended":["partial","*",34,18],"CCBot":["partial","*",34,18],"Bytespider":["partial","*",34,18],"Amazonbot":["partial","*",34,18],"meta-externalagent":["partial","*",34,18],"cohere-ai":["partial","*",34,18],"Diffbot":["partial","*",34,18],"ContextFetchBot":["partial","*",34,18]},"llms":{"status":"http_error","http":403,"bytes":146},"home":{"status":"ok","http":200,"url":"https://www.ecmwf.int/","html_tokens":193132,"markdown_tokens":364,"pruned_tokens":175}},
{"rank":7704,"domain":"saitama.lg.jp","origin":"https://saitama.lg.jp","robots":{"status":"unreachable","http":null,"url":"https://saitama.lg.jp/robots.txt","bytes":0,"groups":0,"error":"cannot resolve saitama.lg.jp"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":7705,"domain":"dai.ly","origin":"https://dai.ly","robots":{"status":"missing","http":404,"url":"https://www.dailymotion.com/urlshortener?path=robots.txt","bytes":22688,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":22674},"home":{"status":"ok","http":200,"url":"https://www.dailymotion.com/","html_tokens":15154,"markdown_tokens":0,"pruned_tokens":0}},
{"rank":7706,"domain":"govexec.com","origin":"https://govexec.com","robots":{"status":"ok","http":200,"url":"https://www.govexec.com/robots.txt","bytes":204,"groups":3},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["partial","*",5,0],"OAI-SearchBot":["partial","*",5,0],"ClaudeBot":["partial","*",5,0],"Claude-User":["partial","*",5,0],"Claude-SearchBot":["partial","*",5,0],"anthropic-ai":["partial","*",5,0],"PerplexityBot":["partial","*",5,0],"Perplexity-User":["partial","*",5,0],"Google-Extended":["partial","*",5,0],"Applebot-Extended":["partial","*",5,0],"CCBot":["partial","*",5,0],"Bytespider":["partial","*",5,0],"Amazonbot":["partial","*",5,0],"meta-externalagent":["partial","*",5,0],"cohere-ai":["partial","*",5,0],"Diffbot":["partial","*",5,0],"ContextFetchBot":["partial","*",5,0]},"llms":{"status":"missing","http":404,"bytes":153},"home":{"status":"ok","http":200,"url":"https://www.govexec.com/","html_tokens":58881,"markdown_tokens":791,"pruned_tokens":151}},
{"rank":7707,"domain":"juniperresearch.com","origin":"https://juniperresearch.com","robots":{"status":"ok","http":200,"url":"https://www.juniperresearch.com/robots.txt","bytes":803,"groups":10},"bots":{"GPTBot":["partial","*",9,1],"ChatGPT-User":["partial","*",9,1],"OAI-SearchBot":["partial","*",9,1],"ClaudeBot":["partial","*",9,1],"Claude-User":["partial","*",9,1],"Claude-SearchBot":["partial","*",9,1],"anthropic-ai":["partial","*",9,1],"PerplexityBot":["partial","*",9,1],"Perplexity-User":["partial","*",9,1],"Google-Extended":["partial","*",9,1],"Applebot-Extended":["partial","*",9,1],"CCBot":["partial","*",9,1],"Bytespider":["partial","*",9,1],"Amazonbot":["partial","*",9,1],"meta-externalagent":["partial","*",9,1],"cohere-ai":["partial","*",9,1],"Diffbot":["partial","*",9,1],"ContextFetchBot":["partial","*",9,1]},"llms":{"status":"missing","http":404,"bytes":0},"home":{"status":"ok","http":200,"url":"https://www.juniperresearch.com/","html_tokens":76471,"markdown_tokens":3962,"pruned_tokens":164}},
{"rank":7708,"domain":"lacity.org","origin":"https://lacity.org","robots":{"status":"refused","http":403,"url":"https://lacity.org/robots.txt","bytes":372,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":7709,"domain":"weber.com","origin":"https://weber.com","robots":{"status":"ok","http":200,"url":"https://www.weber.com/robots.txt","bytes":2289,"groups":1},"bots":{"GPTBot":["partial","*",81,1],"ChatGPT-User":["partial","*",81,1],"OAI-SearchBot":["partial","*",81,1],"ClaudeBot":["partial","*",81,1],"Claude-User":["partial","*",81,1],"Claude-SearchBot":["partial","*",81,1],"anthropic-ai":["partial","*",81,1],"PerplexityBot":["partial","*",81,1],"Perplexity-User":["partial","*",81,1],"Google-Extended":["partial","*",81,1],"Applebot-Extended":["partial","*",81,1],"CCBot":["partial","*",81,1],"Bytespider":["partial","*",81,1],"Amazonbot":["partial","*",81,1],"meta-externalagent":["partial","*",81,1],"cohere-ai":["partial","*",81,1],"Diffbot":["partial","*",81,1],"ContextFetchBot":["partial","*",81,1]},"llms":{"status":"html","http":200,"bytes":1486},"home":{"status":"ok","http":200,"url":"https://amer.weber.com/en-us","html_tokens":168428,"markdown_tokens":233,"pruned_tokens":111}},
{"rank":7710,"domain":"bard.edu","origin":"https://bard.edu","robots":{"status":"ok","http":200,"url":"https://www.bard.edu/robots.txt","bytes":523,"groups":1},"bots":{"GPTBot":["partial","*",16,0],"ChatGPT-User":["partial","*",16,0],"OAI-SearchBot":["partial","*",16,0],"ClaudeBot":["partial","*",16,0],"Claude-User":["partial","*",16,0],"Claude-SearchBot":["partial","*",16,0],"anthropic-ai":["partial","*",16,0],"PerplexityBot":["partial","*",16,0],"Perplexity-User":["partial","*",16,0],"Google-Extended":["partial","*",16,0],"Applebot-Extended":["partial","*",16,0],"CCBot":["partial","*",16,0],"Bytespider":["partial","*",16,0],"Amazonbot":["partial","*",16,0],"meta-externalagent":["partial","*",16,0],"cohere-ai":["partial","*",16,0],"Diffbot":["partial","*",16,0],"ContextFetchBot":["partial","*",16,0]},"llms":{"status":"html","http":200,"bytes":33068},"home":{"status":"ok","http":200,"url":"https://www.bard.edu/","html_tokens":13715,"markdown_tokens":236,"pruned_tokens":201}},
{"rank":7711,"domain":"naeyc.org","origin":"https://naeyc.org","robots":{"status":"refused","http":403,"url":"https://naeyc.org/robots.txt","bytes":5525,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":7712,"domain":"vulkanvegas777.org","origin":"https://vulkanvegas777.org","robots":{"status":"ok","http":200,"url":"https://vulkanvegas777.org/robots.txt","bytes":72,"groups":1},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"html","http":200,"bytes":68408},"home":{"status":"ok","http":200,"url":"https://vulkanvegas777.org/","html_tokens":16768,"markdown_tokens":1989,"pruned_tokens":160}},
{"rank":7713,"domain":"rense.com","origin":"https://rense.com","robots":{"status":"ok","http":200,"url":"https://rense.com/robots.txt","bytes":507,"groups":2},"bots":{"GPTBot":["partial","*",10,0],"ChatGPT-User":["partial","*",10,0],"OAI-SearchBot":["partial","*",10,0],"ClaudeBot":["partial","*",10,0],"Claude-User":["partial","*",10,0],"Claude-SearchBot":["partial","*",10,0],"anthropic-ai":["partial","*",10,0],"PerplexityBot":["partial","*",10,0],"Perplexity-User":["partial","*",10,0],"Google-Extended":["partial","*",10,0],"Applebot-Extended":["partial","*",10,0],"CCBot":["partial","*",10,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["partial","*",10,0],"cohere-ai":["partial","*",10,0],"Diffbot":["partial","*",10,0],"ContextFetchBot":["partial","*",10,0]},"llms":{"status":"missing","http":404,"bytes":1245},"home":{"status":"ok","http":200,"url":"https://rense.com/","html_tokens":367801,"markdown_tokens":84270,"pruned_tokens":172}},
{"rank":7714,"domain":"digital.go.jp","origin":"https://digital.go.jp","robots":{"status":"ok","http":200,"url":"https://www.digital.go.jp/robots.txt","bytes":2035,"groups":2},"bots":{"GPTBot":["partial","*",33,18],"ChatGPT-User":["partial","*",33,18],"OAI-SearchBot":["partial","*",33,18],"ClaudeBot":["partial","*",33,18],"Claude-User":["partial","*",33,18],"Claude-SearchBot":["partial","*",33,18],"anthropic-ai":["partial","*",33,18],"PerplexityBot":["partial","*",33,18],"Perplexity-User":["partial","*",33,18],"Google-Extended":["partial","*",33,18],"Applebot-Extended":["partial","*",33,18],"CCBot":["partial","*",33,18],"Bytespider":["partial","*",33,18],"Amazonbot":["partial","*",33,18],"meta-externalagent":["partial","*",33,18],"cohere-ai":["partial","*",33,18],"Diffbot":["partial","*",33,18],"ContextFetchBot":["partial","*",33,18]},"llms":{"status":"missing","http":404,"bytes":2828},"home":{"status":"ok","http":200,"url":"https://www.digital.go.jp/","html_tokens":22410,"markdown_tokens":183,"pruned_tokens":173}},
{"rank":7715,"domain":"lumenlearning.com","origin":"https://lumenlearning.com","robots":{"status":"ok","http":200,"url":"https://lumenlearning.com/robots.txt","bytes":156,"groups":1},"bots":{"GPTBot":["partial","*",3,2],"ChatGPT-User":["partial","*",3,2],"OAI-SearchBot":["partial","*",3,2],"ClaudeBot":["partial","*",3,2],"Claude-User":["partial","*",3,2],"Claude-SearchBot":["partial","*",3,2],"anthropic-ai":["partial","*",3,2],"PerplexityBot":["partial","*",3,2],"Perplexity-User":["partial","*",3,2],"Google-Extended":["partial","*",3,2],"Applebot-Extended":["partial","*",3,2],"CCBot":["partial","*",3,2],"Bytespider":["partial","*",3,2],"Amazonbot":["partial","*",3,2],"meta-externalagent":["partial","*",3,2],"cohere-ai":["partial","*",3,2],"Diffbot":["partial","*",3,2],"ContextFetchBot":["partial","*",3,2]},"llms":{"status":"missing","http":404,"bytes":1208519},"home":{"status":"ok","http":200,"url":"https://lumenlearning.com/","html_tokens":312275,"markdown_tokens":107,"pruned_tokens":107}},
{"rank":7716,"domain":"nyulangone.org","origin":"https://nyulangone.org","robots":{"status":"html","http":200,"url":"https://nyulangone.org/robots.txt","bytes":2830,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":7717,"domain":"gaiaonline.com","origin":"https://gaiaonline.com","robots":{"status":"ok","http":200,"url":"https://gaiaonline.com/robots.txt","bytes":1633,"groups":4},"bots":{"GPTBot":["partial","*",57,0],"ChatGPT-User":["partial","*",57,0],"OAI-SearchBot":["partial","*",57,0],"ClaudeBot":["partial","*",57,0],"Claude-User":["partial","*",57,0],"Claude-SearchBot":["partial","*",57,0],"anthropic-ai":["partial","*",57,0],"PerplexityBot":["partial","*",57,0],"Perplexity-User":["partial","*",57,0],"Google-Extended":["partial","*",57,0],"Applebot-Extended":["partial","*",57,0],"CCBot":["partial","*",57,0],"Bytespider":["partial","*",57,0],"Amazonbot":["partial","*",57,0],"meta-externalagent":["partial","*",57,0],"cohere-ai":["partial","*",57,0],"Diffbot":["partial","*",57,0],"ContextFetchBot":["partial","*",57,0]},"llms":{"status":"present","http":200,"bytes":1651},"home":{"status":"ok","http":200,"url":"https://gaiaonline.com/","html_tokens":40659,"markdown_tokens":17,"pruned_tokens":17}},
{"rank":7718,"domain":"bbk.ac.uk","origin":"https://bbk.ac.uk","robots":{"status":"ok","http":200,"url":"https://www.bbk.ac.uk/robots.txt","bytes":1996,"groups":4},"bots":{"GPTBot":["partial","*",63,0],"ChatGPT-User":["partial","*",63,0],"OAI-SearchBot":["partial","*",63,0],"ClaudeBot":["partial","*",63,0],"Claude-User":["partial","*",63,0],"Claude-SearchBot":["partial","*",63,0],"anthropic-ai":["partial","*",63,0],"PerplexityBot":["partial","*",63,0],"Perplexity-User":["partial","*",63,0],"Google-Extended":["partial","*",63,0],"Applebot-Extended":["partial","*",63,0],"CCBot":["partial","*",63,0],"Bytespider":["partial","*",63,0],"Amazonbot":["partial","*",63,0],"meta-externalagent":["partial","*",63,0],"cohere-ai":["partial","*",63,0],"Diffbot":["partial","*",63,0],"ContextFetchBot":["partial","*",63,0]},"llms":{"status":"missing","http":404,"bytes":43478},"home":{"status":"ok","http":200,"url":"https://www.bbk.ac.uk/","html_tokens":22303,"markdown_tokens":56,"pruned_tokens":56}},
{"rank":7719,"domain":"jfrog.com","origin":"https://jfrog.com","robots":{"status":"ok","http":200,"url":"https://jfrog.com/robots.txt","bytes":938,"groups":1},"bots":{"GPTBot":["partial","*",19,22],"ChatGPT-User":["partial","*",19,22],"OAI-SearchBot":["partial","*",19,22],"ClaudeBot":["partial","*",19,22],"Claude-User":["partial","*",19,22],"Claude-SearchBot":["partial","*",19,22],"anthropic-ai":["partial","*",19,22],"PerplexityBot":["partial","*",19,22],"Perplexity-User":["partial","*",19,22],"Google-Extended":["partial","*",19,22],"Applebot-Extended":["partial","*",19,22],"CCBot":["partial","*",19,22],"Bytespider":["partial","*",19,22],"Amazonbot":["partial","*",19,22],"meta-externalagent":["partial","*",19,22],"cohere-ai":["partial","*",19,22],"Diffbot":["partial","*",19,22],"ContextFetchBot":["partial","*",19,22]},"llms":{"status":"present","http":200,"bytes":4198},"home":{"status":"ok","http":200,"url":"https://jfrog.com/","html_tokens":90104,"markdown_tokens":330,"pruned_tokens":101}},
{"rank":7720,"domain":"bitcoin.it","origin":"https://bitcoin.it","robots":{"status":"missing","http":404,"url":"https://bitcoin.it/robots.txt","bytes":146,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":13938},"home":{"status":"ok","http":200,"url":"https://en.bitcoin.it/wiki/Main_Page","html_tokens":7381,"markdown_tokens":763,"pruned_tokens":750}},
{"rank":7721,"domain":"passagen.se","origin":"https://passagen.se","robots":{"status":"ok","http":200,"url":"https://passagen.se/robots.txt","bytes":115,"groups":1},"bots":{"GPTBot":["partial","*",1,1],"ChatGPT-User":["partial","*",1,1],"OAI-SearchBot":["partial","*",1,1],"ClaudeBot":["partial","*",1,1],"Claude-User":["partial","*",1,1],"Claude-SearchBot":["partial","*",1,1],"anthropic-ai":["partial","*",1,1],"PerplexityBot":["partial","*",1,1],"Perplexity-User":["partial","*",1,1],"Google-Extended":["partial","*",1,1],"Applebot-Extended":["partial","*",1,1],"CCBot":["partial","*",1,1],"Bytespider":["partial","*",1,1],"Amazonbot":["partial","*",1,1],"meta-externalagent":["partial","*",1,1],"cohere-ai":["partial","*",1,1],"Diffbot":["partial","*",1,1],"ContextFetchBot":["partial","*",1,1]},"llms":{"status":"missing","http":404,"bytes":64025},"home":{"status":"ok","http":200,"url":"https://passagen.se/","html_tokens":60613,"markdown_tokens":8592,"pruned_tokens":165}},
{"rank":7722,"domain":"qunar.com","origin":"https://qunar.com","robots":{"status":"ok","http":200,"url":"https://www.qunar.com/robots.txt","bytes":502,"groups":1},"bots":{"GPTBot":["partial","*",19,0],"ChatGPT-User":["partial","*",19,0],"OAI-SearchBot":["partial","*",19,0],"ClaudeBot":["partial","*",19,0],"Claude-User":["partial","*",19,0],"Claude-SearchBot":["partial","*",19,0],"anthropic-ai":["partial","*",19,0],"PerplexityBot":["partial","*",19,0],"Perplexity-User":["partial","*",19,0],"Google-Extended":["partial","*",19,0],"Applebot-Extended":["partial","*",19,0],"CCBot":["partial","*",19,0],"Bytespider":["partial","*",19,0],"Amazonbot":["partial","*",19,0],"meta-externalagent":["partial","*",19,0],"cohere-ai":["partial","*",19,0],"Diffbot":["partial","*",19,0],"ContextFetchBot":["partial","*",19,0]},"llms":{"status":"missing","http":404,"bytes":1893},"home":{"status":"ok","http":200,"url":"https://www.qunar.com/","html_tokens":700,"markdown_tokens":0,"pruned_tokens":0}},
{"rank":7723,"domain":"arkansasonline.com","origin":"https://arkansasonline.com","robots":{"status":"ok","http":200,"url":"https://www.arkansasonline.com/robots.txt","bytes":9679,"groups":249},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["blocked","named",1,0],"OAI-SearchBot":["blocked","named",1,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["blocked","named",1,0],"Claude-SearchBot":["blocked","named",1,0],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["blocked","named",1,0],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["allowed","*",0,1]},"llms":{"status":"missing","http":404,"bytes":112671},"home":{"status":"ok","http":200,"url":"https://www.arkansasonline.com/","html_tokens":55133,"markdown_tokens":47,"pruned_tokens":47}},
{"rank":7724,"domain":"theskimm.com","origin":"https://theskimm.com","robots":{"status":"ok","http":200,"url":"https://www.theskimm.com/robots.txt","bytes":288,"groups":1},"bots":{"GPTBot":["partial","*",2,1],"ChatGPT-User":["partial","*",2,1],"OAI-SearchBot":["partial","*",2,1],"ClaudeBot":["partial","*",2,1],"Claude-User":["partial","*",2,1],"Claude-SearchBot":["partial","*",2,1],"anthropic-ai":["partial","*",2,1],"PerplexityBot":["partial","*",2,1],"Perplexity-User":["partial","*",2,1],"Google-Extended":["partial","*",2,1],"Applebot-Extended":["partial","*",2,1],"CCBot":["partial","*",2,1],"Bytespider":["partial","*",2,1],"Amazonbot":["partial","*",2,1],"meta-externalagent":["partial","*",2,1],"cohere-ai":["partial","*",2,1],"Diffbot":["partial","*",2,1],"ContextFetchBot":["partial","*",2,1]},"llms":{"status":"missing","http":404,"bytes":161655},"home":{"status":"ok","http":200,"url":"https://www.theskimm.com/","html_tokens":662709,"markdown_tokens":80,"pruned_tokens":80}},
{"rank":7725,"domain":"elfinanciero.com.mx","origin":"https://elfinanciero.com.mx","robots":{"status":"ok","http":200,"url":"https://www.elfinanciero.com.mx/robots.txt","bytes":523,"groups":3},"bots":{"GPTBot":["allowed","*",0,1],"ChatGPT-User":["allowed","*",0,1],"OAI-SearchBot":["allowed","*",0,1],"ClaudeBot":["allowed","*",0,1],"Claude-User":["allowed","*",0,1],"Claude-SearchBot":["allowed","*",0,1],"anthropic-ai":["allowed","*",0,1],"PerplexityBot":["allowed","*",0,1],"Perplexity-User":["allowed","*",0,1],"Google-Extended":["allowed","*",0,1],"Applebot-Extended":["allowed","*",0,1],"CCBot":["allowed","*",0,1],"Bytespider":["allowed","*",0,1],"Amazonbot":["allowed","*",0,1],"meta-externalagent":["allowed","*",0,1],"cohere-ai":["allowed","*",0,1],"Diffbot":["allowed","*",0,1],"ContextFetchBot":["allowed","*",0,1]},"llms":{"status":"missing","http":404,"bytes":412335},"home":{"status":"ok","http":200,"url":"https://www.elfinanciero.com.mx/","html_tokens":160967,"markdown_tokens":16,"pruned_tokens":16}},
{"rank":7726,"domain":"prismic.io","origin":"https://prismic.io","robots":{"status":"ok","http":200,"url":"https://prismic.io/robots.txt","bytes":309,"groups":1},"bots":{"GPTBot":["partial","*",4,1],"ChatGPT-User":["partial","*",4,1],"OAI-SearchBot":["partial","*",4,1],"ClaudeBot":["partial","*",4,1],"Claude-User":["partial","*",4,1],"Claude-SearchBot":["partial","*",4,1],"anthropic-ai":["partial","*",4,1],"PerplexityBot":["partial","*",4,1],"Perplexity-User":["partial","*",4,1],"Google-Extended":["partial","*",4,1],"Applebot-Extended":["partial","*",4,1],"CCBot":["partial","*",4,1],"Bytespider":["partial","*",4,1],"Amazonbot":["partial","*",4,1],"meta-externalagent":["partial","*",4,1],"cohere-ai":["partial","*",4,1],"Diffbot":["partial","*",4,1],"ContextFetchBot":["partial","*",4,1]},"llms":{"status":"present","http":200,"bytes":5533},"home":{"status":"ok","http":200,"url":"https://prismic.io/","html_tokens":110917,"markdown_tokens":783,"pruned_tokens":139}},
{"rank":7727,"domain":"malaymail.com","origin":"https://malaymail.com","robots":{"status":"ok","http":200,"url":"https://www.malaymail.com/robots.txt","bytes":451,"groups":2},"bots":{"GPTBot":["partial","*",16,0],"ChatGPT-User":["partial","*",16,0],"OAI-SearchBot":["partial","*",16,0],"ClaudeBot":["partial","*",16,0],"Claude-User":["partial","*",16,0],"Claude-SearchBot":["partial","*",16,0],"anthropic-ai":["partial","*",16,0],"PerplexityBot":["partial","*",16,0],"Perplexity-User":["partial","*",16,0],"Google-Extended":["partial","*",16,0],"Applebot-Extended":["partial","*",16,0],"CCBot":["partial","*",16,0],"Bytespider":["partial","*",16,0],"Amazonbot":["partial","*",16,0],"meta-externalagent":["partial","*",16,0],"cohere-ai":["partial","*",16,0],"Diffbot":["partial","*",16,0],"ContextFetchBot":["partial","*",16,0]},"llms":{"status":"present","http":200,"bytes":2608},"home":{"status":"http_error","http":403,"url":"https://www.malaymail.com/"}},
{"rank":7728,"domain":"amny.com","origin":"https://amny.com","robots":{"status":"ok","http":200,"url":"https://www.amny.com/robots.txt","bytes":3582,"groups":3},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["partial","*",1,1],"OAI-SearchBot":["partial","*",1,1],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",1,1],"Claude-SearchBot":["partial","*",1,1],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["partial","*",1,1],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",1,1]},"llms":{"status":"missing","http":404,"bytes":106979},"home":{"status":"ok","http":200,"url":"https://www.amny.com/","html_tokens":53607,"markdown_tokens":1636,"pruned_tokens":58}},
{"rank":7729,"domain":"google.co.ao","origin":"https://google.co.ao","robots":{"status":"ok","http":200,"url":"https://www.google.co.ao/robots.txt","bytes":6624,"groups":4},"bots":{"GPTBot":["partial","*",169,64],"ChatGPT-User":["partial","*",169,64],"OAI-SearchBot":["partial","*",169,64],"ClaudeBot":["partial","*",169,64],"Claude-User":["partial","*",169,64],"Claude-SearchBot":["partial","*",169,64],"anthropic-ai":["partial","*",169,64],"PerplexityBot":["partial","*",169,64],"Perplexity-User":["partial","*",169,64],"Google-Extended":["partial","*",169,64],"Applebot-Extended":["partial","*",169,64],"CCBot":["partial","*",169,64],"Bytespider":["partial","*",169,64],"Amazonbot":["partial","*",169,64],"meta-externalagent":["partial","*",169,64],"cohere-ai":["partial","*",169,64],"Diffbot":["partial","*",169,64],"ContextFetchBot":["partial","*",169,64]},"llms":{"status":"missing","http":404,"bytes":1569},"home":{"status":"ok","http":200,"url":"https://www.google.com/","html_tokens":21299,"markdown_tokens":30,"pruned_tokens":30}},
{"rank":7730,"domain":"monde-diplomatique.fr","origin":"https://monde-diplomatique.fr","robots":{"status":"ok","http":200,"url":"https://www.monde-diplomatique.fr/robots.txt","bytes":454,"groups":1},"bots":{"GPTBot":["partial","*",7,4],"ChatGPT-User":["partial","*",7,4],"OAI-SearchBot":["partial","*",7,4],"ClaudeBot":["partial","*",7,4],"Claude-User":["partial","*",7,4],"Claude-SearchBot":["partial","*",7,4],"anthropic-ai":["partial","*",7,4],"PerplexityBot":["partial","*",7,4],"Perplexity-User":["partial","*",7,4],"Google-Extended":["partial","*",7,4],"Applebot-Extended":["partial","*",7,4],"CCBot":["partial","*",7,4],"Bytespider":["partial","*",7,4],"Amazonbot":["partial","*",7,4],"meta-externalagent":["partial","*",7,4],"cohere-ai":["partial","*",7,4],"Diffbot":["partial","*",7,4],"ContextFetchBot":["partial","*",7,4]},"llms":{"status":"missing","http":404,"bytes":993},"home":{"status":"ok","http":200,"url":"https://www.monde-diplomatique.fr/","html_tokens":19991,"markdown_tokens":159,"pruned_tokens":55}},
{"rank":7731,"domain":"eskimo.com","origin":"https://eskimo.com","robots":{"status":"ok","http":200,"url":"https://eskimo.com/robots.txt","bytes":231,"groups":2},"bots":{"GPTBot":["partial","*",7,1],"ChatGPT-User":["partial","*",7,1],"OAI-SearchBot":["partial","*",7,1],"ClaudeBot":["partial","*",7,1],"Claude-User":["partial","*",7,1],"Claude-SearchBot":["partial","*",7,1],"anthropic-ai":["partial","*",7,1],"PerplexityBot":["partial","*",7,1],"Perplexity-User":["partial","*",7,1],"Google-Extended":["partial","*",7,1],"Applebot-Extended":["partial","*",7,1],"CCBot":["partial","*",7,1],"Bytespider":["partial","*",7,1],"Amazonbot":["partial","*",7,1],"meta-externalagent":["partial","*",7,1],"cohere-ai":["partial","*",7,1],"Diffbot":["partial","*",7,1],"ContextFetchBot":["partial","*",7,1]},"llms":{"status":"other_text","http":200,"bytes":351228},"home":{"status":"ok","http":200,"url":"https://www.eskimo.com/","html_tokens":13785,"markdown_tokens":598,"pruned_tokens":243}},
{"rank":7732,"domain":"frebsite.nl","origin":"https://frebsite.nl","robots":{"status":"missing","http":404,"url":"https://frebsite.nl/robots.txt","bytes":2419,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":2419},"home":{"status":"ok","http":200,"url":"https://frebsite.nl/","html_tokens":1029,"markdown_tokens":22,"pruned_tokens":22}},
{"rank":7733,"domain":"sport.es","origin":"https://www.sport.es","robots":{"status":"refused","http":406,"url":"https://www.sport.es/robots.txt","bytes":435,"groups":0,"error":"HTTP 406"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":7734,"domain":"thegatewaypundit.com","origin":"https://thegatewaypundit.com","robots":{"status":"ok","http":200,"url":"https://www.thegatewaypundit.com/robots.txt","bytes":3328,"groups":38},"bots":{"GPTBot":["partial","*",58,3],"ChatGPT-User":["partial","*",58,3],"OAI-SearchBot":["partial","*",58,3],"ClaudeBot":["partial","*",58,3],"Claude-User":["partial","*",58,3],"Claude-SearchBot":["partial","*",58,3],"anthropic-ai":["partial","*",58,3],"PerplexityBot":["partial","*",58,3],"Perplexity-User":["partial","*",58,3],"Google-Extended":["partial","*",58,3],"Applebot-Extended":["partial","*",58,3],"CCBot":["blocked","named",2,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",2,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["partial","*",58,3],"Diffbot":["partial","*",58,3],"ContextFetchBot":["partial","*",58,3]},"llms":{"status":"crawl_delay","error":"Crawl-delay 120 s"},"home":{"status":"crawl_delay","error":"Crawl-delay 120 s"}},
{"rank":7735,"domain":"rtlnieuws.nl","origin":"https://rtlnieuws.nl","robots":{"status":"ok","http":200,"url":"https://www.rtl.nl/robots.txt","bytes":3757,"groups":3},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["blocked","named",1,0],"OAI-SearchBot":["blocked","named",1,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["blocked","named",1,0],"Claude-SearchBot":["blocked","named",1,0],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["blocked","named",1,0],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",10,0]},"llms":{"status":"missing","http":404,"bytes":153},"home":{"status":"ok","http":200,"url":"https://www.rtl.nl/","html_tokens":128031,"markdown_tokens":19,"pruned_tokens":19}},
{"rank":7736,"domain":"schoolido.lu","origin":"https://schoolido.lu","robots":{"status":"ok","http":200,"url":"https://schoolido.lu/robots.txt","bytes":1248,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":11403},"home":{"status":"ok","http":200,"url":"https://schoolido.lu/","html_tokens":4941,"markdown_tokens":245,"pruned_tokens":174}},
{"rank":7737,"domain":"inra.fr","origin":"https://inra.fr","robots":{"status":"ok","http":200,"url":"https://www.inrae.fr/robots.txt","bytes":1917,"groups":1},"bots":{"GPTBot":["partial","*",27,18],"ChatGPT-User":["partial","*",27,18],"OAI-SearchBot":["partial","*",27,18],"ClaudeBot":["partial","*",27,18],"Claude-User":["partial","*",27,18],"Claude-SearchBot":["partial","*",27,18],"anthropic-ai":["partial","*",27,18],"PerplexityBot":["partial","*",27,18],"Perplexity-User":["partial","*",27,18],"Google-Extended":["partial","*",27,18],"Applebot-Extended":["partial","*",27,18],"CCBot":["partial","*",27,18],"Bytespider":["partial","*",27,18],"Amazonbot":["partial","*",27,18],"meta-externalagent":["partial","*",27,18],"cohere-ai":["partial","*",27,18],"Diffbot":["partial","*",27,18],"ContextFetchBot":["partial","*",27,18]},"llms":{"status":"html","http":200,"bytes":272616},"home":{"status":"ok","http":200,"url":"https://www.inrae.fr/","html_tokens":67997,"markdown_tokens":243,"pruned_tokens":152}},
{"rank":7738,"domain":"musicbusinessworldwide.com","origin":"https://musicbusinessworldwide.com","robots":{"status":"ok","http":200,"url":"https://www.musicbusinessworldwide.com/robots.txt","bytes":193,"groups":1},"bots":{"GPTBot":["partial","*",1,1],"ChatGPT-User":["partial","*",1,1],"OAI-SearchBot":["partial","*",1,1],"ClaudeBot":["partial","*",1,1],"Claude-User":["partial","*",1,1],"Claude-SearchBot":["partial","*",1,1],"anthropic-ai":["partial","*",1,1],"PerplexityBot":["partial","*",1,1],"Perplexity-User":["partial","*",1,1],"Google-Extended":["partial","*",1,1],"Applebot-Extended":["partial","*",1,1],"CCBot":["partial","*",1,1],"Bytespider":["partial","*",1,1],"Amazonbot":["partial","*",1,1],"meta-externalagent":["partial","*",1,1],"cohere-ai":["partial","*",1,1],"Diffbot":["partial","*",1,1],"ContextFetchBot":["partial","*",1,1]},"llms":{"status":"missing","http":404,"bytes":62765},"home":{"status":"ok","http":200,"url":"https://www.musicbusinessworldwide.com/","html_tokens":22999,"markdown_tokens":207,"pruned_tokens":43}},
{"rank":7739,"domain":"eurostar.com","origin":"https://eurostar.com","robots":{"status":"unreachable","http":null,"url":"https://eurostar.com/robots.txt","bytes":0,"groups":0,"error":"<ConnectionTerminated error_code:0, last_stream_id:2147483647, additional_data:677261636566756c2073687574646f776e>"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":7740,"domain":"cervantesvirtual.com","origin":"https://cervantesvirtual.com","robots":{"status":"ok","http":200,"url":"https://cervantesvirtual.com/robots.txt","bytes":291,"groups":1},"bots":{"GPTBot":["partial","*",10,0],"ChatGPT-User":["partial","*",10,0],"OAI-SearchBot":["partial","*",10,0],"ClaudeBot":["partial","*",10,0],"Claude-User":["partial","*",10,0],"Claude-SearchBot":["partial","*",10,0],"anthropic-ai":["partial","*",10,0],"PerplexityBot":["partial","*",10,0],"Perplexity-User":["partial","*",10,0],"Google-Extended":["partial","*",10,0],"Applebot-Extended":["partial","*",10,0],"CCBot":["partial","*",10,0],"Bytespider":["partial","*",10,0],"Amazonbot":["partial","*",10,0],"meta-externalagent":["partial","*",10,0],"cohere-ai":["partial","*",10,0],"Diffbot":["partial","*",10,0],"ContextFetchBot":["partial","*",10,0]},"llms":{"status":"missing","http":404,"bytes":17105},"home":{"status":"ok","http":200,"url":"https://cervantesvirtual.com/","html_tokens":9889,"markdown_tokens":449,"pruned_tokens":167}},
{"rank":7741,"domain":"asco.org","origin":"https://asco.org","robots":{"status":"refused","http":403,"url":"https://asco.org/robots.txt","bytes":919,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":7742,"domain":"friendfeed.com","origin":"http://friendfeed.com","robots":{"status":"ok","http":200,"url":"https://www.facebook.com/robots.txt","bytes":41641,"groups":36},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["blocked","*",1,0],"OAI-SearchBot":["blocked","*",1,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["blocked","*",1,0],"Claude-SearchBot":["blocked","*",1,0],"anthropic-ai":["blocked","*",1,0],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["blocked","*",1,0],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","*",1,0],"Bytespider":["blocked","*",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","*",1,0],"cohere-ai":["blocked","*",1,0],"Diffbot":["blocked","*",1,0],"ContextFetchBot":["blocked","*",1,0]},"llms":{"status":"disallowed"},"home":{"status":"disallowed"}},
{"rank":7743,"domain":"lolipop.jp","origin":"https://lolipop.jp","robots":{"status":"ok","http":200,"url":"https://lolipop.jp/robots.txt","bytes":271,"groups":2},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["partial","named",1,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"disallowed","error":"robots.txt of err.lolipop.jp not readable (timeout)"},"home":{"status":"ok","http":200,"url":"https://lolipop.jp/","html_tokens":32170,"markdown_tokens":482,"pruned_tokens":143}},
{"rank":7744,"domain":"chatgpt.site","origin":"https://chatgpt.site","robots":{"status":"refused","http":403,"url":"https://chatgpt.com/features/sites/","bytes":8646,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":7745,"domain":"governing.com","origin":"https://governing.com","robots":{"status":"ok","http":200,"url":"https://www.governing.com/robots.txt","bytes":825,"groups":17},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["allowed","named",0,1],"OAI-SearchBot":["allowed","named",0,1],"ClaudeBot":["blocked","named",1,0],"Claude-User":["allowed","named",0,1],"Claude-SearchBot":["allowed","named",0,1],"anthropic-ai":["allowed","*",0,1],"PerplexityBot":["allowed","named",0,1],"Perplexity-User":["allowed","named",0,1],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["allowed","*",0,1],"ContextFetchBot":["allowed","*",0,1]},"llms":{"status":"present","http":200,"bytes":1348},"home":{"status":"ok","http":200,"url":"https://www.governing.com/","html_tokens":43642,"markdown_tokens":775,"pruned_tokens":146}},
{"rank":7746,"domain":"understandingwar.org","origin":"https://understandingwar.org","robots":{"status":"refused","http":403,"url":"https://understandingwar.org/robots.txt","bytes":5558,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":7747,"domain":"thestate.com","origin":"https://thestate.com","robots":{"status":"unreachable","http":null,"url":"https://thestate.com/robots.txt","bytes":0,"groups":0,"error":"<StreamReset stream_id:1, error_code:2, remote_reset:True>"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":7748,"domain":"priceline.com","origin":"https://priceline.com","robots":{"status":"ok","http":200,"url":"https://priceline.com/robots.txt","bytes":4163,"groups":10},"bots":{"GPTBot":["partial","*",121,1],"ChatGPT-User":["partial","*",121,1],"OAI-SearchBot":["allowed","named",0,1],"ClaudeBot":["partial","*",121,1],"Claude-User":["partial","*",121,1],"Claude-SearchBot":["partial","*",121,1],"anthropic-ai":["partial","*",121,1],"PerplexityBot":["partial","*",121,1],"Perplexity-User":["partial","*",121,1],"Google-Extended":["partial","*",121,1],"Applebot-Extended":["partial","*",121,1],"CCBot":["partial","*",121,1],"Bytespider":["partial","*",121,1],"Amazonbot":["partial","*",121,1],"meta-externalagent":["partial","*",121,1],"cohere-ai":["partial","*",121,1],"Diffbot":["partial","*",121,1],"ContextFetchBot":["partial","*",121,1]},"llms":{"status":"missing","http":404,"bytes":2617},"home":{"status":"ok","http":200,"url":"https://www.priceline.com/","html_tokens":160129,"markdown_tokens":411,"pruned_tokens":80}},
{"rank":7749,"domain":"xmu.edu.cn","origin":"https://xmu.edu.cn","robots":{"status":"missing","http":404,"url":"https://www.xmu.edu.cn/robots.txt","bytes":1693,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":1693},"home":{"status":"ok","http":200,"url":"https://www.xmu.edu.cn/","html_tokens":25891,"markdown_tokens":850,"pruned_tokens":167}},
{"rank":7750,"domain":"huduser.gov","origin":"https://huduser.gov","robots":{"status":"html","http":202,"url":"https://huduser.gov/robots.txt","bytes":2407,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":7751,"domain":"menafn.com","origin":"https://menafn.com","robots":{"status":"ok","http":200,"url":"https://menafn.com/robots.txt","bytes":993,"groups":8},"bots":{"GPTBot":["allowed","named",0,1],"ChatGPT-User":["partial","*",13,1],"OAI-SearchBot":["allowed","named",0,1],"ClaudeBot":["partial","*",13,1],"Claude-User":["partial","*",13,1],"Claude-SearchBot":["partial","*",13,1],"anthropic-ai":["partial","*",13,1],"PerplexityBot":["allowed","named",0,1],"Perplexity-User":["partial","*",13,1],"Google-Extended":["allowed","named",0,1],"Applebot-Extended":["partial","*",13,1],"CCBot":["partial","*",13,1],"Bytespider":["partial","*",13,1],"Amazonbot":["partial","*",13,1],"meta-externalagent":["partial","*",13,1],"cohere-ai":["partial","*",13,1],"Diffbot":["partial","*",13,1],"ContextFetchBot":["partial","*",13,1]},"llms":{"status":"present","http":200,"bytes":993},"home":{"status":"ok","http":200,"url":"https://menafn.com/","html_tokens":9878,"markdown_tokens":47,"pruned_tokens":47}},
{"rank":7752,"domain":"iinet.net.au","origin":"https://iinet.net.au","robots":{"status":"ok","http":200,"url":"https://www.iinet.net.au/robots.txt","bytes":2208,"groups":1},"bots":{"GPTBot":["partial","*",34,18],"ChatGPT-User":["partial","*",34,18],"OAI-SearchBot":["partial","*",34,18],"ClaudeBot":["partial","*",34,18],"Claude-User":["partial","*",34,18],"Claude-SearchBot":["partial","*",34,18],"anthropic-ai":["partial","*",34,18],"PerplexityBot":["partial","*",34,18],"Perplexity-User":["partial","*",34,18],"Google-Extended":["partial","*",34,18],"Applebot-Extended":["partial","*",34,18],"CCBot":["partial","*",34,18],"Bytespider":["partial","*",34,18],"Amazonbot":["partial","*",34,18],"meta-externalagent":["partial","*",34,18],"cohere-ai":["partial","*",34,18],"Diffbot":["partial","*",34,18],"ContextFetchBot":["partial","*",34,18]},"llms":{"status":"missing","http":404,"bytes":191},"home":{"status":"ok","http":200,"url":"https://www.iinet.net.au/home","html_tokens":53740,"markdown_tokens":5388,"pruned_tokens":68}},
{"rank":7753,"domain":"pp.ru","origin":"https://pp.ru","robots":{"status":"missing","http":404,"url":"https://pp.ru/robots.txt","bytes":5152,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":5152},"home":{"status":"ok","http":200,"url":"https://pp.ru/","html_tokens":1527,"markdown_tokens":29,"pruned_tokens":29}},
{"rank":7754,"domain":"templatemo.com","origin":"https://templatemo.com","robots":{"status":"ok","http":200,"url":"https://templatemo.com/robots.txt","bytes":454,"groups":2},"bots":{"GPTBot":["partial","*",4,6],"ChatGPT-User":["partial","*",4,6],"OAI-SearchBot":["partial","*",4,6],"ClaudeBot":["partial","*",4,6],"Claude-User":["partial","*",4,6],"Claude-SearchBot":["partial","*",4,6],"anthropic-ai":["partial","*",4,6],"PerplexityBot":["partial","*",4,6],"Perplexity-User":["partial","*",4,6],"Google-Extended":["partial","*",4,6],"Applebot-Extended":["partial","*",4,6],"CCBot":["partial","*",4,6],"Bytespider":["partial","*",4,6],"Amazonbot":["partial","*",4,6],"meta-externalagent":["partial","*",4,6],"cohere-ai":["partial","*",4,6],"Diffbot":["partial","*",4,6],"ContextFetchBot":["partial","*",4,6]},"llms":{"status":"missing","http":404,"bytes":3943},"home":{"status":"ok","http":200,"url":"https://templatemo.com/","html_tokens":4683,"markdown_tokens":671,"pruned_tokens":172}},
{"rank":7755,"domain":"acmethemes.com","origin":"https://acmethemes.com","robots":{"status":"ok","http":200,"url":"https://www.acmethemes.com/robots.txt","bytes":176,"groups":1},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"missing","http":404,"bytes":36976},"home":{"status":"ok","http":200,"url":"https://www.acmethemes.com/","html_tokens":19288,"markdown_tokens":680,"pruned_tokens":105}},
{"rank":7756,"domain":"omnicalculator.com","origin":"https://omnicalculator.com","robots":{"status":"ok","http":200,"url":"https://www.omnicalculator.com/robots.txt","bytes":382,"groups":1},"bots":{"GPTBot":["partial","*",7,0],"ChatGPT-User":["partial","*",7,0],"OAI-SearchBot":["partial","*",7,0],"ClaudeBot":["partial","*",7,0],"Claude-User":["partial","*",7,0],"Claude-SearchBot":["partial","*",7,0],"anthropic-ai":["partial","*",7,0],"PerplexityBot":["partial","*",7,0],"Perplexity-User":["partial","*",7,0],"Google-Extended":["partial","*",7,0],"Applebot-Extended":["partial","*",7,0],"CCBot":["partial","*",7,0],"Bytespider":["partial","*",7,0],"Amazonbot":["partial","*",7,0],"meta-externalagent":["partial","*",7,0],"cohere-ai":["partial","*",7,0],"Diffbot":["partial","*",7,0],"ContextFetchBot":["partial","*",7,0]},"llms":{"status":"html","http":200,"bytes":10932},"home":{"status":"ok","http":200,"url":"https://www.omnicalculator.com/","html_tokens":57967,"markdown_tokens":626,"pruned_tokens":168}},
{"rank":7757,"domain":"democratandchronicle.com","origin":"https://democratandchronicle.com","robots":{"status":"ok","http":200,"url":"https://www.democratandchronicle.com/robots.txt","bytes":12707,"groups":288},"bots":{"GPTBot":["partial","named",1,4],"ChatGPT-User":["partial","named",1,4],"OAI-SearchBot":["partial","named",1,4],"ClaudeBot":["blocked","named",1,0],"Claude-User":["blocked","named",1,0],"Claude-SearchBot":["blocked","named",1,0],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["partial","named",1,4],"Perplexity-User":["blocked","named",1,0],"Google-Extended":["partial","named",1,4],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",2,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",32,0]},"llms":{"status":"http_error","http":402,"bytes":254},"home":{"status":"http_error","http":402,"url":"https://www.democratandchronicle.com/"}},
{"rank":7758,"domain":"gardeningknowhow.com","origin":"https://gardeningknowhow.com","robots":{"status":"ok","http":200,"url":"https://www.gardeningknowhow.com/robots.txt","bytes":1862,"groups":5},"bots":{"GPTBot":["partial","*",25,0],"ChatGPT-User":["partial","*",25,0],"OAI-SearchBot":["partial","*",25,0],"ClaudeBot":["partial","*",25,0],"Claude-User":["partial","*",25,0],"Claude-SearchBot":["partial","*",25,0],"anthropic-ai":["partial","*",25,0],"PerplexityBot":["partial","*",25,0],"Perplexity-User":["partial","*",25,0],"Google-Extended":["partial","*",25,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["partial","*",25,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["partial","*",25,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",25,0]},"llms":{"status":"missing","http":404,"bytes":1292063},"home":{"status":"ok","http":200,"url":"https://www.gardeningknowhow.com/","html_tokens":433163,"markdown_tokens":719,"pruned_tokens":31}},
{"rank":7759,"domain":"xunta.gal","origin":"https://xunta.gal","robots":{"status":"server_error","http":500,"url":"https://www.xunta.gal/robots.txt","bytes":341,"groups":0,"error":"HTTP 500"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":7760,"domain":"amoxihub.com","origin":"https://amoxihub.com","robots":{"status":"ok","http":200,"url":"https://amoxihub.com/robots.txt","bytes":144,"groups":1},"bots":{"GPTBot":["partial","*",5,0],"ChatGPT-User":["partial","*",5,0],"OAI-SearchBot":["partial","*",5,0],"ClaudeBot":["partial","*",5,0],"Claude-User":["partial","*",5,0],"Claude-SearchBot":["partial","*",5,0],"anthropic-ai":["partial","*",5,0],"PerplexityBot":["partial","*",5,0],"Perplexity-User":["partial","*",5,0],"Google-Extended":["partial","*",5,0],"Applebot-Extended":["partial","*",5,0],"CCBot":["partial","*",5,0],"Bytespider":["partial","*",5,0],"Amazonbot":["partial","*",5,0],"meta-externalagent":["partial","*",5,0],"cohere-ai":["partial","*",5,0],"Diffbot":["partial","*",5,0],"ContextFetchBot":["partial","*",5,0]},"llms":{"status":"missing","http":404,"bytes":641},"home":{"status":"ok","http":200,"url":"https://amoxihub.com/","html_tokens":12486,"markdown_tokens":4504,"pruned_tokens":114}},
{"rank":7761,"domain":"rosewoodhotels.com","origin":"https://rosewoodhotels.com","robots":{"status":"ok","http":200,"url":"https://www.rosewoodhotels.com/robots.txt","bytes":989,"groups":1},"bots":{"GPTBot":["partial","*",8,0],"ChatGPT-User":["partial","*",8,0],"OAI-SearchBot":["partial","*",8,0],"ClaudeBot":["partial","*",8,0],"Claude-User":["partial","*",8,0],"Claude-SearchBot":["partial","*",8,0],"anthropic-ai":["partial","*",8,0],"PerplexityBot":["partial","*",8,0],"Perplexity-User":["partial","*",8,0],"Google-Extended":["partial","*",8,0],"Applebot-Extended":["partial","*",8,0],"CCBot":["partial","*",8,0],"Bytespider":["partial","*",8,0],"Amazonbot":["partial","*",8,0],"meta-externalagent":["partial","*",8,0],"cohere-ai":["partial","*",8,0],"Diffbot":["partial","*",8,0],"ContextFetchBot":["partial","*",8,0]},"llms":{"status":"http_error","http":403,"bytes":243},"home":{"status":"ok","http":200,"url":"https://www.rosewoodhotels.com/en/default","html_tokens":46236,"markdown_tokens":87,"pruned_tokens":87}},
{"rank":7762,"domain":"optimole.com","origin":"https://optimole.com","robots":{"status":"ok","http":200,"url":"https://optimole.com/robots.txt","bytes":244,"groups":2},"bots":{"GPTBot":["partial","*",1,1],"ChatGPT-User":["partial","*",1,1],"OAI-SearchBot":["partial","*",1,1],"ClaudeBot":["partial","*",1,1],"Claude-User":["partial","*",1,1],"Claude-SearchBot":["partial","*",1,1],"anthropic-ai":["partial","*",1,1],"PerplexityBot":["partial","*",1,1],"Perplexity-User":["partial","*",1,1],"Google-Extended":["partial","*",1,1],"Applebot-Extended":["partial","*",1,1],"CCBot":["partial","*",1,1],"Bytespider":["partial","*",1,1],"Amazonbot":["partial","*",1,1],"meta-externalagent":["partial","*",1,1],"cohere-ai":["partial","*",1,1],"Diffbot":["partial","*",1,1],"ContextFetchBot":["partial","*",1,1]},"llms":{"status":"missing","http":404,"bytes":135132},"home":{"status":"ok","http":200,"url":"https://optimole.com/","html_tokens":82166,"markdown_tokens":824,"pruned_tokens":75}},
{"rank":7763,"domain":"aspeninstitute.org","origin":"https://aspeninstitute.org","robots":{"status":"ok","http":200,"url":"https://www.aspeninstitute.org/robots.txt","bytes":180,"groups":1},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"missing","http":404,"bytes":192422},"home":{"status":"http_error","http":403,"url":"https://aspeninstitute.org/"}},
{"rank":7764,"domain":"prb.org","origin":"https://prb.org","robots":{"status":"ok","http":200,"url":"https://www.prb.org/robots.txt","bytes":243,"groups":2},"bots":{"GPTBot":["partial","*",1,1],"ChatGPT-User":["partial","*",1,1],"OAI-SearchBot":["partial","*",1,1],"ClaudeBot":["partial","*",1,1],"Claude-User":["partial","*",1,1],"Claude-SearchBot":["partial","*",1,1],"anthropic-ai":["partial","*",1,1],"PerplexityBot":["partial","*",1,1],"Perplexity-User":["partial","*",1,1],"Google-Extended":["partial","*",1,1],"Applebot-Extended":["partial","*",1,1],"CCBot":["partial","*",1,1],"Bytespider":["partial","*",1,1],"Amazonbot":["partial","*",1,1],"meta-externalagent":["partial","*",1,1],"cohere-ai":["partial","*",1,1],"Diffbot":["partial","*",1,1],"ContextFetchBot":["partial","*",1,1]},"llms":{"status":"missing","http":404,"bytes":74409},"home":{"status":"ok","http":200,"url":"https://www.prb.org/","html_tokens":30251,"markdown_tokens":322,"pruned_tokens":159}},
{"rank":7765,"domain":"x.org","origin":"https://x.org","robots":{"status":"missing","http":404,"url":"https://www.x.org/robots.txt","bytes":3725,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":3725},"home":{"status":"ok","http":200,"url":"https://www.x.org/","html_tokens":2345,"markdown_tokens":867,"pruned_tokens":319}},
{"rank":7766,"domain":"jfklibrary.org","origin":"https://jfklibrary.org","robots":{"status":"ok","http":200,"url":"https://www.jfklibrary.org/robots.txt","bytes":5105,"groups":27},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["partial","*",44,18],"OAI-SearchBot":["partial","*",44,18],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",44,18],"Claude-SearchBot":["partial","*",44,18],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["partial","*",44,18],"Perplexity-User":["partial","*",44,18],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["partial","*",44,18],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",44,18]},"llms":{"status":"missing","http":404,"bytes":662},"home":{"status":"http_error","http":403,"url":"https://www.jfklibrary.org/"}},
{"rank":7767,"domain":"metropoles.com","origin":"https://metropoles.com","robots":{"status":"ok","http":200,"url":"https://www.metropoles.com/robots.txt","bytes":455,"groups":1},"bots":{"GPTBot":["partial","*",16,2],"ChatGPT-User":["partial","*",16,2],"OAI-SearchBot":["partial","*",16,2],"ClaudeBot":["partial","*",16,2],"Claude-User":["partial","*",16,2],"Claude-SearchBot":["partial","*",16,2],"anthropic-ai":["partial","*",16,2],"PerplexityBot":["partial","*",16,2],"Perplexity-User":["partial","*",16,2],"Google-Extended":["partial","*",16,2],"Applebot-Extended":["partial","*",16,2],"CCBot":["partial","*",16,2],"Bytespider":["partial","*",16,2],"Amazonbot":["partial","*",16,2],"meta-externalagent":["partial","*",16,2],"cohere-ai":["partial","*",16,2],"Diffbot":["partial","*",16,2],"ContextFetchBot":["partial","*",16,2]},"llms":{"status":"missing","http":404,"bytes":40207},"home":{"status":"ok","http":200,"url":"https://www.metropoles.com/","html_tokens":247610,"markdown_tokens":381,"pruned_tokens":92}},
{"rank":7768,"domain":"thrivecart.com","origin":"https://thrivecart.com","robots":{"status":"ok","http":200,"url":"https://thrivecart.com/robots.txt","bytes":302,"groups":1},"bots":{"GPTBot":["partial","*",9,0],"ChatGPT-User":["partial","*",9,0],"OAI-SearchBot":["partial","*",9,0],"ClaudeBot":["partial","*",9,0],"Claude-User":["partial","*",9,0],"Claude-SearchBot":["partial","*",9,0],"anthropic-ai":["partial","*",9,0],"PerplexityBot":["partial","*",9,0],"Perplexity-User":["partial","*",9,0],"Google-Extended":["partial","*",9,0],"Applebot-Extended":["partial","*",9,0],"CCBot":["partial","*",9,0],"Bytespider":["partial","*",9,0],"Amazonbot":["partial","*",9,0],"meta-externalagent":["partial","*",9,0],"cohere-ai":["partial","*",9,0],"Diffbot":["partial","*",9,0],"ContextFetchBot":["partial","*",9,0]},"llms":{"status":"other_text","http":200,"bytes":2634},"home":{"status":"ok","http":200,"url":"https://thrivecart.com/","html_tokens":64031,"markdown_tokens":1343,"pruned_tokens":419}},
{"rank":7769,"domain":"novascotia.ca","origin":"https://novascotia.ca","robots":{"status":"ok","http":200,"url":"https://novascotia.ca/robots.txt","bytes":456,"groups":1},"bots":{"GPTBot":["partial","*",10,0],"ChatGPT-User":["partial","*",10,0],"OAI-SearchBot":["partial","*",10,0],"ClaudeBot":["partial","*",10,0],"Claude-User":["partial","*",10,0],"Claude-SearchBot":["partial","*",10,0],"anthropic-ai":["partial","*",10,0],"PerplexityBot":["partial","*",10,0],"Perplexity-User":["partial","*",10,0],"Google-Extended":["partial","*",10,0],"Applebot-Extended":["partial","*",10,0],"CCBot":["partial","*",10,0],"Bytespider":["partial","*",10,0],"Amazonbot":["partial","*",10,0],"meta-externalagent":["partial","*",10,0],"cohere-ai":["partial","*",10,0],"Diffbot":["partial","*",10,0],"ContextFetchBot":["partial","*",10,0]},"llms":{"status":"missing","http":404,"bytes":4222},"home":{"status":"ok","http":200,"url":"https://www.novascotia.ca/","html_tokens":3489,"markdown_tokens":162,"pruned_tokens":162}},
{"rank":7770,"domain":"nikonusa.com","origin":"https://nikonusa.com","robots":{"status":"ok","http":200,"url":"https://www.nikonusa.com/robots.txt","bytes":336,"groups":1},"bots":{"GPTBot":["partial","*",10,0],"ChatGPT-User":["partial","*",10,0],"OAI-SearchBot":["partial","*",10,0],"ClaudeBot":["partial","*",10,0],"Claude-User":["partial","*",10,0],"Claude-SearchBot":["partial","*",10,0],"anthropic-ai":["partial","*",10,0],"PerplexityBot":["partial","*",10,0],"Perplexity-User":["partial","*",10,0],"Google-Extended":["partial","*",10,0],"Applebot-Extended":["partial","*",10,0],"CCBot":["partial","*",10,0],"Bytespider":["partial","*",10,0],"Amazonbot":["partial","*",10,0],"meta-externalagent":["partial","*",10,0],"cohere-ai":["partial","*",10,0],"Diffbot":["partial","*",10,0],"ContextFetchBot":["partial","*",10,0]},"llms":{"status":"missing","http":404,"bytes":620520},"home":{"status":"ok","http":200,"url":"https://www.nikonusa.com/","html_tokens":114142,"markdown_tokens":102,"pruned_tokens":87}},
{"rank":7771,"domain":"koreajoongangdaily.com","origin":"https://koreajoongangdaily.com","robots":{"status":"ok","http":200,"url":"https://www.koreajoongangdaily.com/robots.txt","bytes":339,"groups":2},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["allowed","*",0,1],"OAI-SearchBot":["allowed","*",0,1],"ClaudeBot":["blocked","named",1,0],"Claude-User":["allowed","*",0,1],"Claude-SearchBot":["allowed","*",0,1],"anthropic-ai":["allowed","*",0,1],"PerplexityBot":["allowed","*",0,1],"Perplexity-User":["allowed","*",0,1],"Google-Extended":["allowed","*",0,1],"Applebot-Extended":["allowed","*",0,1],"CCBot":["allowed","*",0,1],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["allowed","*",0,1],"cohere-ai":["allowed","*",0,1],"Diffbot":["allowed","*",0,1],"ContextFetchBot":["allowed","*",0,1]},"llms":{"status":"missing","http":404,"bytes":100871},"home":{"status":"ok","http":200,"url":"https://www.koreajoongangdaily.com/","html_tokens":91044,"markdown_tokens":154,"pruned_tokens":108}},
{"rank":7772,"domain":"consumentenbond.nl","origin":"https://consumentenbond.nl","robots":{"status":"ok","http":200,"url":"https://www.consumentenbond.nl/robots.txt","bytes":271,"groups":1},"bots":{"GPTBot":["partial","*",8,0],"ChatGPT-User":["partial","*",8,0],"OAI-SearchBot":["partial","*",8,0],"ClaudeBot":["partial","*",8,0],"Claude-User":["partial","*",8,0],"Claude-SearchBot":["partial","*",8,0],"anthropic-ai":["partial","*",8,0],"PerplexityBot":["partial","*",8,0],"Perplexity-User":["partial","*",8,0],"Google-Extended":["partial","*",8,0],"Applebot-Extended":["partial","*",8,0],"CCBot":["partial","*",8,0],"Bytespider":["partial","*",8,0],"Amazonbot":["partial","*",8,0],"meta-externalagent":["partial","*",8,0],"cohere-ai":["partial","*",8,0],"Diffbot":["partial","*",8,0],"ContextFetchBot":["partial","*",8,0]},"llms":{"status":"missing","http":404,"bytes":756574},"home":{"status":"ok","http":200,"url":"https://www.consumentenbond.nl/","html_tokens":323045,"markdown_tokens":282,"pruned_tokens":46}},
{"rank":7773,"domain":"gazette.com","origin":"https://gazette.com","robots":{"status":"ok","http":200,"url":"https://gazette.com/robots.txt","bytes":615,"groups":13},"bots":{"GPTBot":["partial","*",5,1],"ChatGPT-User":["partial","*",5,1],"OAI-SearchBot":["partial","*",5,1],"ClaudeBot":["partial","*",5,1],"Claude-User":["partial","*",5,1],"Claude-SearchBot":["partial","*",5,1],"anthropic-ai":["partial","*",5,1],"PerplexityBot":["partial","*",5,1],"Perplexity-User":["partial","*",5,1],"Google-Extended":["partial","*",5,1],"Applebot-Extended":["partial","*",5,1],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["partial","*",5,1],"meta-externalagent":["partial","*",5,1],"cohere-ai":["partial","*",5,1],"Diffbot":["partial","*",5,1],"ContextFetchBot":["partial","*",5,1]},"llms":{"status":"missing","http":404,"bytes":128636},"home":{"status":"ok","http":200,"url":"https://gazette.com/","html_tokens":49070,"markdown_tokens":77,"pruned_tokens":77}},
{"rank":7774,"domain":"pen.org","origin":"https://pen.org","robots":{"status":"ok","http":200,"url":"https://pen.org/robots.txt","bytes":165,"groups":1},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"missing","http":404,"bytes":443809},"home":{"status":"http_error","http":403,"url":"https://pen.org/"}},
{"rank":7775,"domain":"google.com.kw","origin":"https://google.com.kw","robots":{"status":"ok","http":200,"url":"https://www.google.com.kw/robots.txt","bytes":6624,"groups":4},"bots":{"GPTBot":["partial","*",169,64],"ChatGPT-User":["partial","*",169,64],"OAI-SearchBot":["partial","*",169,64],"ClaudeBot":["partial","*",169,64],"Claude-User":["partial","*",169,64],"Claude-SearchBot":["partial","*",169,64],"anthropic-ai":["partial","*",169,64],"PerplexityBot":["partial","*",169,64],"Perplexity-User":["partial","*",169,64],"Google-Extended":["partial","*",169,64],"Applebot-Extended":["partial","*",169,64],"CCBot":["partial","*",169,64],"Bytespider":["partial","*",169,64],"Amazonbot":["partial","*",169,64],"meta-externalagent":["partial","*",169,64],"cohere-ai":["partial","*",169,64],"Diffbot":["partial","*",169,64],"ContextFetchBot":["partial","*",169,64]},"llms":{"status":"missing","http":404,"bytes":1569},"home":{"status":"ok","http":200,"url":"https://www.google.com/","html_tokens":21368,"markdown_tokens":30,"pruned_tokens":30}},
{"rank":7776,"domain":"idw-online.de","origin":"https://idw-online.de","robots":{"status":"ok","http":200,"url":"https://idw-online.de/robots.txt","bytes":240,"groups":2},"bots":{"GPTBot":["partial","*",8,0],"ChatGPT-User":["partial","*",8,0],"OAI-SearchBot":["partial","*",8,0],"ClaudeBot":["partial","*",8,0],"Claude-User":["partial","*",8,0],"Claude-SearchBot":["partial","*",8,0],"anthropic-ai":["partial","*",8,0],"PerplexityBot":["partial","*",8,0],"Perplexity-User":["partial","*",8,0],"Google-Extended":["partial","*",8,0],"Applebot-Extended":["partial","*",8,0],"CCBot":["partial","*",8,0],"Bytespider":["partial","*",8,0],"Amazonbot":["partial","*",8,0],"meta-externalagent":["partial","*",8,0],"cohere-ai":["partial","*",8,0],"Diffbot":["partial","*",8,0],"ContextFetchBot":["partial","*",8,0]},"llms":{"status":"missing","http":404,"bytes":513},"home":{"status":"ok","http":200,"url":"https://idw-online.de/en/","html_tokens":30104,"markdown_tokens":649,"pruned_tokens":170}},
{"rank":7777,"domain":"azquotes.com","origin":"https://azquotes.com","robots":{"status":"ok","http":200,"url":"https://azquotes.com/robots.txt","bytes":294,"groups":3},"bots":{"GPTBot":["partial","*",9,1],"ChatGPT-User":["partial","*",9,1],"OAI-SearchBot":["partial","*",9,1],"ClaudeBot":["partial","*",9,1],"Claude-User":["partial","*",9,1],"Claude-SearchBot":["partial","*",9,1],"anthropic-ai":["partial","*",9,1],"PerplexityBot":["partial","*",9,1],"Perplexity-User":["partial","*",9,1],"Google-Extended":["partial","*",9,1],"Applebot-Extended":["partial","*",9,1],"CCBot":["partial","*",9,1],"Bytespider":["partial","*",9,1],"Amazonbot":["partial","*",9,1],"meta-externalagent":["partial","*",9,1],"cohere-ai":["partial","*",9,1],"Diffbot":["partial","*",9,1],"ContextFetchBot":["partial","*",9,1]},"llms":{"status":"missing","http":404,"bytes":196},"home":{"status":"ok","http":200,"url":"https://www.azquotes.com/","html_tokens":32434,"markdown_tokens":39,"pruned_tokens":39}},
{"rank":7778,"domain":"inthesetimes.com","origin":"https://inthesetimes.com","robots":{"status":"ok","http":200,"url":"https://inthesetimes.com/robots.txt","bytes":1248,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":37677},"home":{"status":"ok","http":200,"url":"https://inthesetimes.com/","html_tokens":43667,"markdown_tokens":1014,"pruned_tokens":166}},
{"rank":7779,"domain":"manning.com","origin":"https://manning.com","robots":{"status":"ok","http":200,"url":"https://www.manning.com/robots.txt","bytes":242,"groups":1},"bots":{"GPTBot":["partial","*",8,0],"ChatGPT-User":["partial","*",8,0],"OAI-SearchBot":["partial","*",8,0],"ClaudeBot":["partial","*",8,0],"Claude-User":["partial","*",8,0],"Claude-SearchBot":["partial","*",8,0],"anthropic-ai":["partial","*",8,0],"PerplexityBot":["partial","*",8,0],"Perplexity-User":["partial","*",8,0],"Google-Extended":["partial","*",8,0],"Applebot-Extended":["partial","*",8,0],"CCBot":["partial","*",8,0],"Bytespider":["partial","*",8,0],"Amazonbot":["partial","*",8,0],"meta-externalagent":["partial","*",8,0],"cohere-ai":["partial","*",8,0],"Diffbot":["partial","*",8,0],"ContextFetchBot":["partial","*",8,0]},"llms":{"status":"missing","http":404,"bytes":5094},"home":{"status":"ok","http":200,"url":"https://www.manning.com/","html_tokens":92470,"markdown_tokens":7,"pruned_tokens":7}},
{"rank":7780,"domain":"majorgeeks.com","origin":"https://majorgeeks.com","robots":{"status":"ok","http":200,"url":"https://majorgeeks.com/robots.txt","bytes":56,"groups":1},"bots":{"GPTBot":["partial","*",1,0],"ChatGPT-User":["partial","*",1,0],"OAI-SearchBot":["partial","*",1,0],"ClaudeBot":["partial","*",1,0],"Claude-User":["partial","*",1,0],"Claude-SearchBot":["partial","*",1,0],"anthropic-ai":["partial","*",1,0],"PerplexityBot":["partial","*",1,0],"Perplexity-User":["partial","*",1,0],"Google-Extended":["partial","*",1,0],"Applebot-Extended":["partial","*",1,0],"CCBot":["partial","*",1,0],"Bytespider":["partial","*",1,0],"Amazonbot":["partial","*",1,0],"meta-externalagent":["partial","*",1,0],"cohere-ai":["partial","*",1,0],"Diffbot":["partial","*",1,0],"ContextFetchBot":["partial","*",1,0]},"llms":{"status":"html","http":200,"bytes":96681},"home":{"status":"ok","http":200,"url":"https://www.majorgeeks.com/","html_tokens":24162,"markdown_tokens":2315,"pruned_tokens":102}},
{"rank":7781,"domain":"easyslimnow.com","origin":"https://easyslimnow.com","robots":{"status":"ok","http":200,"url":"https://easyslimnow.com/robots.txt","bytes":78,"groups":1},"bots":{"GPTBot":["partial","*",1,0],"ChatGPT-User":["partial","*",1,0],"OAI-SearchBot":["partial","*",1,0],"ClaudeBot":["partial","*",1,0],"Claude-User":["partial","*",1,0],"Claude-SearchBot":["partial","*",1,0],"anthropic-ai":["partial","*",1,0],"PerplexityBot":["partial","*",1,0],"Perplexity-User":["partial","*",1,0],"Google-Extended":["partial","*",1,0],"Applebot-Extended":["partial","*",1,0],"CCBot":["partial","*",1,0],"Bytespider":["partial","*",1,0],"Amazonbot":["partial","*",1,0],"meta-externalagent":["partial","*",1,0],"cohere-ai":["partial","*",1,0],"Diffbot":["partial","*",1,0],"ContextFetchBot":["partial","*",1,0]},"llms":{"status":"missing","http":404,"bytes":644},"home":{"status":"ok","http":200,"url":"https://easyslimnow.com/","html_tokens":2835,"markdown_tokens":114,"pruned_tokens":114}},
{"rank":7782,"domain":"ccel.org","origin":"https://ccel.org","robots":{"status":"ok","http":200,"url":"https://ccel.org/robots.txt","bytes":122,"groups":2},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"missing","http":404,"bytes":13272},"home":{"status":"ok","http":200,"url":"https://ccel.org/","html_tokens":8540,"markdown_tokens":443,"pruned_tokens":109}},
{"rank":7783,"domain":"360doc.com","origin":"https://360doc.com","robots":{"status":"unreachable","http":null,"url":"https://360doc.com/robots.txt","bytes":0,"groups":0,"error":"cannot resolve 360doc.com"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":7784,"domain":"njgjp.com","origin":"https://njgjp.com","robots":{"status":"missing","http":404,"url":"https://njgjp.com/robots.txt","bytes":0,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":0},"home":{"status":"ok","http":200,"url":"https://njgjp.com/","html_tokens":20040,"markdown_tokens":19,"pruned_tokens":19}},
{"rank":7785,"domain":"syracuse.edu","origin":"https://syracuse.edu","robots":{"status":"ok","http":200,"url":"https://www.syracuse.edu/robots.txt","bytes":159,"groups":1},"bots":{"GPTBot":["partial","*",5,0],"ChatGPT-User":["partial","*",5,0],"OAI-SearchBot":["partial","*",5,0],"ClaudeBot":["partial","*",5,0],"Claude-User":["partial","*",5,0],"Claude-SearchBot":["partial","*",5,0],"anthropic-ai":["partial","*",5,0],"PerplexityBot":["partial","*",5,0],"Perplexity-User":["partial","*",5,0],"Google-Extended":["partial","*",5,0],"Applebot-Extended":["partial","*",5,0],"CCBot":["partial","*",5,0],"Bytespider":["partial","*",5,0],"Amazonbot":["partial","*",5,0],"meta-externalagent":["partial","*",5,0],"cohere-ai":["partial","*",5,0],"Diffbot":["partial","*",5,0],"ContextFetchBot":["partial","*",5,0]},"llms":{"status":"present","http":200,"bytes":6308},"home":{"status":"ok","http":200,"url":"https://www.syracuse.edu/","html_tokens":56244,"markdown_tokens":77,"pruned_tokens":77}},
{"rank":7786,"domain":"qgiv.com","origin":"https://qgiv.com","robots":{"status":"ok","http":200,"url":"https://www.qgiv.com/robots.txt","bytes":251,"groups":1},"bots":{"GPTBot":["partial","*",6,0],"ChatGPT-User":["partial","*",6,0],"OAI-SearchBot":["partial","*",6,0],"ClaudeBot":["partial","*",6,0],"Claude-User":["partial","*",6,0],"Claude-SearchBot":["partial","*",6,0],"anthropic-ai":["partial","*",6,0],"PerplexityBot":["partial","*",6,0],"Perplexity-User":["partial","*",6,0],"Google-Extended":["partial","*",6,0],"Applebot-Extended":["partial","*",6,0],"CCBot":["partial","*",6,0],"Bytespider":["partial","*",6,0],"Amazonbot":["partial","*",6,0],"meta-externalagent":["partial","*",6,0],"cohere-ai":["partial","*",6,0],"Diffbot":["partial","*",6,0],"ContextFetchBot":["partial","*",6,0]},"llms":{"status":"missing","http":404,"bytes":93591},"home":{"status":"ok","http":200,"url":"https://www.qgiv.com/","html_tokens":26312,"markdown_tokens":42,"pruned_tokens":42}},
{"rank":7787,"domain":"miniclip.com","origin":"https://miniclip.com","robots":{"status":"ok","http":200,"url":"https://miniclip.com/robots.txt","bytes":23,"groups":1},"bots":{"GPTBot":["allowed","*",0,1],"ChatGPT-User":["allowed","*",0,1],"OAI-SearchBot":["allowed","*",0,1],"ClaudeBot":["allowed","*",0,1],"Claude-User":["allowed","*",0,1],"Claude-SearchBot":["allowed","*",0,1],"anthropic-ai":["allowed","*",0,1],"PerplexityBot":["allowed","*",0,1],"Perplexity-User":["allowed","*",0,1],"Google-Extended":["allowed","*",0,1],"Applebot-Extended":["allowed","*",0,1],"CCBot":["allowed","*",0,1],"Bytespider":["allowed","*",0,1],"Amazonbot":["allowed","*",0,1],"meta-externalagent":["allowed","*",0,1],"cohere-ai":["allowed","*",0,1],"Diffbot":["allowed","*",0,1],"ContextFetchBot":["allowed","*",0,1]},"llms":{"status":"html","http":200,"bytes":143332},"home":{"status":"ok","http":200,"url":"https://miniclip.com/","html_tokens":35827,"markdown_tokens":116,"pruned_tokens":116}},
{"rank":7788,"domain":"sr.ht","origin":"https://sr.ht","robots":{"status":"ok","http":200,"url":"https://sr.ht/robots.txt","bytes":2624,"groups":31},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["blocked","named",1,0],"OAI-SearchBot":["blocked","named",1,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",11,0],"Claude-SearchBot":["partial","*",11,0],"anthropic-ai":["partial","*",11,0],"PerplexityBot":["partial","*",11,0],"Perplexity-User":["partial","*",11,0],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["partial","*",11,0],"CCBot":["partial","*",11,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["partial","*",11,0],"Diffbot":["partial","*",11,0],"ContextFetchBot":["partial","*",11,0]},"llms":{"status":"missing","http":404,"bytes":1446},"home":{"status":"ok","http":200,"url":"https://sr.ht/","html_tokens":1778,"markdown_tokens":110,"pruned_tokens":109}},
{"rank":7789,"domain":"smrj.go.jp","origin":"https://www.smrj.go.jp","robots":{"status":"ok","http":200,"url":"https://www.smrj.go.jp/robots.txt","bytes":1950,"groups":8},"bots":{"GPTBot":["partial","*",6,0],"ChatGPT-User":["partial","*",6,0],"OAI-SearchBot":["partial","*",6,0],"ClaudeBot":["partial","*",6,0],"Claude-User":["partial","*",6,0],"Claude-SearchBot":["partial","*",6,0],"anthropic-ai":["partial","*",6,0],"PerplexityBot":["partial","*",6,0],"Perplexity-User":["partial","*",6,0],"Google-Extended":["partial","*",6,0],"Applebot-Extended":["partial","*",6,0],"CCBot":["partial","*",6,0],"Bytespider":["partial","*",6,0],"Amazonbot":["partial","*",6,0],"meta-externalagent":["partial","*",6,0],"cohere-ai":["partial","*",6,0],"Diffbot":["partial","*",6,0],"ContextFetchBot":["partial","*",6,0]},"llms":{"status":"missing","http":404,"bytes":51715},"home":{"status":"ok","http":200,"url":"https://www.smrj.go.jp/","html_tokens":29737,"markdown_tokens":26,"pruned_tokens":26}},
{"rank":7790,"domain":"correiobraziliense.com.br","origin":"https://correiobraziliense.com.br","robots":{"status":"ok","http":200,"url":"https://www.correiobraziliense.com.br/robots.txt","bytes":504,"groups":5},"bots":{"GPTBot":["partial","*",1,1],"ChatGPT-User":["partial","*",1,1],"OAI-SearchBot":["partial","*",1,1],"ClaudeBot":["partial","*",1,1],"Claude-User":["partial","*",1,1],"Claude-SearchBot":["partial","*",1,1],"anthropic-ai":["partial","*",1,1],"PerplexityBot":["partial","*",1,1],"Perplexity-User":["partial","*",1,1],"Google-Extended":["allowed","named",0,1],"Applebot-Extended":["partial","*",1,1],"CCBot":["partial","*",1,1],"Bytespider":["partial","*",1,1],"Amazonbot":["partial","*",1,1],"meta-externalagent":["partial","*",1,1],"cohere-ai":["partial","*",1,1],"Diffbot":["partial","*",1,1],"ContextFetchBot":["partial","*",1,1]},"llms":{"status":"missing","http":404,"bytes":764},"home":{"status":"ok","http":200,"url":"https://www.correiobraziliense.com.br/","html_tokens":87865,"markdown_tokens":30,"pruned_tokens":30}},
{"rank":7791,"domain":"wiz.io","origin":"https://wiz.io","robots":{"status":"ok","http":200,"url":"https://www.wiz.io/robots.txt","bytes":101,"groups":1},"bots":{"GPTBot":["allowed","*",0,1],"ChatGPT-User":["allowed","*",0,1],"OAI-SearchBot":["allowed","*",0,1],"ClaudeBot":["allowed","*",0,1],"Claude-User":["allowed","*",0,1],"Claude-SearchBot":["allowed","*",0,1],"anthropic-ai":["allowed","*",0,1],"PerplexityBot":["allowed","*",0,1],"Perplexity-User":["allowed","*",0,1],"Google-Extended":["allowed","*",0,1],"Applebot-Extended":["allowed","*",0,1],"CCBot":["allowed","*",0,1],"Bytespider":["allowed","*",0,1],"Amazonbot":["allowed","*",0,1],"meta-externalagent":["allowed","*",0,1],"cohere-ai":["allowed","*",0,1],"Diffbot":["allowed","*",0,1],"ContextFetchBot":["allowed","*",0,1]},"llms":{"status":"present","http":200,"bytes":11332},"home":{"status":"ok","http":200,"url":"https://www.wiz.io/","html_tokens":143690,"markdown_tokens":1386,"pruned_tokens":567}},
{"rank":7792,"domain":"wfla.com","origin":"https://wfla.com","robots":{"status":"ok","http":200,"url":"https://www.wfla.com/robots.txt","bytes":2305,"groups":40},"bots":{"GPTBot":["blocked","named",2,0],"ChatGPT-User":["blocked","named",1,0],"OAI-SearchBot":["blocked","named",1,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",12,1],"Claude-SearchBot":["partial","*",12,1],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["blocked","named",2,0],"Perplexity-User":["partial","*",12,1],"Google-Extended":["blocked","named",2,0],"Applebot-Extended":["blocked","named",2,0],"CCBot":["blocked","named",2,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["partial","*",12,1],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",12,1]},"llms":{"status":"http_error","http":403,"bytes":6514},"home":{"status":"http_error","http":403,"url":"https://wfla.com/"}},
{"rank":7793,"domain":"tricentis.com","origin":"https://tricentis.com","robots":{"status":"ok","http":200,"url":"https://www.tricentis.com/robots.txt","bytes":352,"groups":4},"bots":{"GPTBot":["partial","*",9,1],"ChatGPT-User":["partial","*",9,1],"OAI-SearchBot":["partial","*",9,1],"ClaudeBot":["partial","*",9,1],"Claude-User":["partial","*",9,1],"Claude-SearchBot":["partial","*",9,1],"anthropic-ai":["partial","*",9,1],"PerplexityBot":["partial","*",9,1],"Perplexity-User":["partial","*",9,1],"Google-Extended":["partial","*",9,1],"Applebot-Extended":["partial","*",9,1],"CCBot":["partial","*",9,1],"Bytespider":["partial","*",9,1],"Amazonbot":["partial","*",9,1],"meta-externalagent":["partial","*",9,1],"cohere-ai":["partial","*",9,1],"Diffbot":["partial","*",9,1],"ContextFetchBot":["partial","*",9,1]},"llms":{"status":"other_text","http":200,"bytes":13842},"home":{"status":"ok","http":200,"url":"https://www.tricentis.com/","html_tokens":84199,"markdown_tokens":351,"pruned_tokens":138}},
{"rank":7794,"domain":"squareblogs.net","origin":"https://squareblogs.net","robots":{"status":"missing","http":404,"url":"https://squareblogs.net/robots.txt","bytes":19,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"other_text","http":200,"bytes":15},"home":{"status":"ok","http":200,"url":"https://squareblogs.net/","html_tokens":3611,"markdown_tokens":94,"pruned_tokens":33}},
{"rank":7795,"domain":"115.com","origin":"https://115.com","robots":{"status":"ok","http":200,"url":"https://115.com/robots.txt","bytes":79,"groups":1},"bots":{"GPTBot":["partial","*",3,0],"ChatGPT-User":["partial","*",3,0],"OAI-SearchBot":["partial","*",3,0],"ClaudeBot":["partial","*",3,0],"Claude-User":["partial","*",3,0],"Claude-SearchBot":["partial","*",3,0],"anthropic-ai":["partial","*",3,0],"PerplexityBot":["partial","*",3,0],"Perplexity-User":["partial","*",3,0],"Google-Extended":["partial","*",3,0],"Applebot-Extended":["partial","*",3,0],"CCBot":["partial","*",3,0],"Bytespider":["partial","*",3,0],"Amazonbot":["partial","*",3,0],"meta-externalagent":["partial","*",3,0],"cohere-ai":["partial","*",3,0],"Diffbot":["partial","*",3,0],"ContextFetchBot":["partial","*",3,0]},"llms":{"status":"missing","http":404,"bytes":21153},"home":{"status":"ok","http":200,"url":"https://115.com/","html_tokens":9760,"markdown_tokens":132,"pruned_tokens":132}},
{"rank":7796,"domain":"mingsenlawyer.com","origin":"http://mingsenlawyer.com","robots":{"status":"missing","http":404,"url":"http://mingsenlawyer.com/robots.txt","bytes":5078,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":5074},"home":{"status":"ok","http":200,"url":"http://mingsenlawyer.com/","html_tokens":3098,"markdown_tokens":87,"pruned_tokens":87}},
{"rank":7797,"domain":"vagaro.com","origin":"https://vagaro.com","robots":{"status":"ok","http":200,"url":"https://www.vagaro.com/robots.txt","bytes":5408,"groups":5},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["partial","named",44,0],"OAI-SearchBot":["partial","named",44,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","named",44,0],"Claude-SearchBot":["partial","named",44,0],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["partial","named",44,0],"Perplexity-User":["partial","named",44,0],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["partial","named",44,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["blocked","*",1,0]},"llms":{"status":"disallowed"},"home":{"status":"disallowed"}},
{"rank":7798,"domain":"cake.me","origin":"https://cake.me","robots":{"status":"ok","http":200,"url":"https://www.cake.me/robots.txt","bytes":55,"groups":1},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"missing","http":404,"bytes":164196},"home":{"status":"ok","http":200,"url":"https://www.cake.me/","html_tokens":292316,"markdown_tokens":1929,"pruned_tokens":232}},
{"rank":7799,"domain":"widen.net","origin":"https://widen.net","robots":{"status":"ok","http":200,"url":"https://widen.net/robots.txt","bytes":115,"groups":2},"bots":{"GPTBot":["blocked","*",1,0],"ChatGPT-User":["blocked","*",1,0],"OAI-SearchBot":["blocked","*",1,0],"ClaudeBot":["blocked","*",1,0],"Claude-User":["blocked","*",1,0],"Claude-SearchBot":["blocked","*",1,0],"anthropic-ai":["blocked","*",1,0],"PerplexityBot":["blocked","*",1,0],"Perplexity-User":["blocked","*",1,0],"Google-Extended":["blocked","*",1,0],"Applebot-Extended":["blocked","*",1,0],"CCBot":["blocked","*",1,0],"Bytespider":["blocked","*",1,0],"Amazonbot":["blocked","*",1,0],"meta-externalagent":["blocked","*",1,0],"cohere-ai":["blocked","*",1,0],"Diffbot":["blocked","*",1,0],"ContextFetchBot":["blocked","*",1,0]},"llms":{"status":"disallowed"},"home":{"status":"disallowed"}},
{"rank":7800,"domain":"agro.pl","origin":"https://agro.pl","robots":{"status":"unreachable","http":null,"url":"https://agro.pl/robots.txt","bytes":0,"groups":0,"error":"cannot resolve agro.pl"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":7801,"domain":"rabbitmq.com","origin":"https://rabbitmq.com","robots":{"status":"missing","http":404,"url":"https://www.rabbitmq.com/robots.txt","bytes":12964,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":12964},"home":{"status":"ok","http":200,"url":"https://www.rabbitmq.com/","html_tokens":6776,"markdown_tokens":1360,"pruned_tokens":725}},
{"rank":7802,"domain":"clustrmaps.com","origin":"https://clustrmaps.com","robots":{"status":"unreachable","http":null,"url":"https://clustrmaps.com/robots.txt","bytes":0,"groups":0,"error":"clustrmaps.com is not a public address"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":7803,"domain":"haihuizx.com","origin":"http://haihuizx.com","robots":{"status":"ok","http":200,"url":"http://www.haihuizx.com/robots.txt","bytes":160,"groups":1},"bots":{"GPTBot":["partial","*",7,0],"ChatGPT-User":["partial","*",7,0],"OAI-SearchBot":["partial","*",7,0],"ClaudeBot":["partial","*",7,0],"Claude-User":["partial","*",7,0],"Claude-SearchBot":["partial","*",7,0],"anthropic-ai":["partial","*",7,0],"PerplexityBot":["partial","*",7,0],"Perplexity-User":["partial","*",7,0],"Google-Extended":["partial","*",7,0],"Applebot-Extended":["partial","*",7,0],"CCBot":["partial","*",7,0],"Bytespider":["partial","*",7,0],"Amazonbot":["partial","*",7,0],"meta-externalagent":["partial","*",7,0],"cohere-ai":["partial","*",7,0],"Diffbot":["partial","*",7,0],"ContextFetchBot":["partial","*",7,0]},"llms":{"status":"missing","http":404,"bytes":2665},"home":{"status":"ok","http":200,"url":"http://www.haihuizx.com/","html_tokens":4740,"markdown_tokens":157,"pruned_tokens":157}},
{"rank":7804,"domain":"megaphone.fm","origin":"https://megaphone.fm","robots":{"status":"ok","http":200,"url":"https://megaphone.spotify.com/robots.txt","bytes":25,"groups":1},"bots":{"GPTBot":["blocked","*",1,0],"ChatGPT-User":["blocked","*",1,0],"OAI-SearchBot":["blocked","*",1,0],"ClaudeBot":["blocked","*",1,0],"Claude-User":["blocked","*",1,0],"Claude-SearchBot":["blocked","*",1,0],"anthropic-ai":["blocked","*",1,0],"PerplexityBot":["blocked","*",1,0],"Perplexity-User":["blocked","*",1,0],"Google-Extended":["blocked","*",1,0],"Applebot-Extended":["blocked","*",1,0],"CCBot":["blocked","*",1,0],"Bytespider":["blocked","*",1,0],"Amazonbot":["blocked","*",1,0],"meta-externalagent":["blocked","*",1,0],"cohere-ai":["blocked","*",1,0],"Diffbot":["blocked","*",1,0],"ContextFetchBot":["blocked","*",1,0]},"llms":{"status":"disallowed"},"home":{"status":"disallowed"}},
{"rank":7805,"domain":"specialized.com","origin":"https://specialized.com","robots":{"status":"ok","http":200,"url":"https://www.specialized.com/robots.txt","bytes":532,"groups":5},"bots":{"GPTBot":["partial","*",2,0],"ChatGPT-User":["partial","*",2,0],"OAI-SearchBot":["partial","*",2,0],"ClaudeBot":["partial","*",2,0],"Claude-User":["partial","*",2,0],"Claude-SearchBot":["partial","*",2,0],"anthropic-ai":["partial","*",2,0],"PerplexityBot":["partial","*",2,0],"Perplexity-User":["partial","*",2,0],"Google-Extended":["partial","*",2,0],"Applebot-Extended":["partial","*",2,0],"CCBot":["partial","*",2,0],"Bytespider":["partial","*",2,0],"Amazonbot":["partial","*",2,0],"meta-externalagent":["partial","*",2,0],"cohere-ai":["partial","*",2,0],"Diffbot":["partial","*",2,0],"ContextFetchBot":["partial","*",2,0]},"llms":{"status":"missing","http":404,"bytes":8966},"home":{"status":"ok","http":200,"url":"https://www.specialized.com/us/en","html_tokens":74009,"markdown_tokens":61,"pruned_tokens":61}},
{"rank":7806,"domain":"kaspersky.ru","origin":"https://kaspersky.ru","robots":{"status":"ok","http":200,"url":"https://www.kaspersky.ru/robots.txt","bytes":3026,"groups":18},"bots":{"GPTBot":["allowed","named",0,1],"ChatGPT-User":["allowed","named",0,1],"OAI-SearchBot":["allowed","named",0,1],"ClaudeBot":["allowed","named",0,1],"Claude-User":["allowed","named",0,1],"Claude-SearchBot":["partial","*",51,9],"anthropic-ai":["partial","*",51,9],"PerplexityBot":["allowed","named",0,1],"Perplexity-User":["allowed","named",0,1],"Google-Extended":["allowed","named",0,1],"Applebot-Extended":["allowed","named",0,1],"CCBot":["allowed","named",0,1],"Bytespider":["partial","*",51,9],"Amazonbot":["partial","*",51,9],"meta-externalagent":["partial","*",51,9],"cohere-ai":["partial","*",51,9],"Diffbot":["allowed","named",0,1],"ContextFetchBot":["partial","*",51,9]},"llms":{"status":"present","http":200,"bytes":83679},"home":{"status":"ok","http":200,"url":"https://www.kaspersky.ru/","html_tokens":260914,"markdown_tokens":112,"pruned_tokens":112}},
{"rank":7807,"domain":"s.weibo.com","origin":"https://s.weibo.com","robots":{"status":"html","http":200,"url":"https://passport.weibo.com/visitor/visitor?entry=miniblog&a=enter&url=https%3A%2F%2Fweibo.com%2Fsorry%3Fpagenotfound%26&domain=.weibo.com&ua=php-sso_sdk_client-0.6.36&_rand=1792374892.4368","bytes":9600,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":7808,"domain":"mod.go.jp","origin":"https://www.mod.go.jp","robots":{"status":"ok","http":200,"url":"https://www.mod.go.jp/robots.txt","bytes":48,"groups":1},"bots":{"GPTBot":["partial","*",2,0],"ChatGPT-User":["partial","*",2,0],"OAI-SearchBot":["partial","*",2,0],"ClaudeBot":["partial","*",2,0],"Claude-User":["partial","*",2,0],"Claude-SearchBot":["partial","*",2,0],"anthropic-ai":["partial","*",2,0],"PerplexityBot":["partial","*",2,0],"Perplexity-User":["partial","*",2,0],"Google-Extended":["partial","*",2,0],"Applebot-Extended":["partial","*",2,0],"CCBot":["partial","*",2,0],"Bytespider":["partial","*",2,0],"Amazonbot":["partial","*",2,0],"meta-externalagent":["partial","*",2,0],"cohere-ai":["partial","*",2,0],"Diffbot":["partial","*",2,0],"ContextFetchBot":["partial","*",2,0]},"llms":{"status":"missing","http":404,"bytes":206},"home":{"status":"http_error","http":403,"url":"https://www.mod.go.jp/"}},
{"rank":7809,"domain":"amnesty.org.uk","origin":"https://amnesty.org.uk","robots":{"status":"ok","http":200,"url":"https://www.amnesty.org.uk/robots.txt","bytes":42,"groups":1},"bots":{"GPTBot":["partial","*",1,1],"ChatGPT-User":["partial","*",1,1],"OAI-SearchBot":["partial","*",1,1],"ClaudeBot":["partial","*",1,1],"Claude-User":["partial","*",1,1],"Claude-SearchBot":["partial","*",1,1],"anthropic-ai":["partial","*",1,1],"PerplexityBot":["partial","*",1,1],"Perplexity-User":["partial","*",1,1],"Google-Extended":["partial","*",1,1],"Applebot-Extended":["partial","*",1,1],"CCBot":["partial","*",1,1],"Bytespider":["partial","*",1,1],"Amazonbot":["partial","*",1,1],"meta-externalagent":["partial","*",1,1],"cohere-ai":["partial","*",1,1],"Diffbot":["partial","*",1,1],"ContextFetchBot":["partial","*",1,1]},"llms":{"status":"missing","http":404,"bytes":163649},"home":{"status":"ok","http":200,"url":"https://www.amnesty.org.uk","html_tokens":54943,"markdown_tokens":262,"pruned_tokens":160}},
{"rank":7810,"domain":"und.edu","origin":"https://und.edu","robots":{"status":"ok","http":200,"url":"https://und.edu/robots.txt","bytes":1526,"groups":3},"bots":{"GPTBot":["partial","*",14,7],"ChatGPT-User":["partial","*",14,7],"OAI-SearchBot":["partial","*",14,7],"ClaudeBot":["partial","*",14,7],"Claude-User":["partial","*",14,7],"Claude-SearchBot":["partial","*",14,7],"anthropic-ai":["partial","*",14,7],"PerplexityBot":["partial","*",14,7],"Perplexity-User":["partial","*",14,7],"Google-Extended":["partial","*",14,7],"Applebot-Extended":["partial","*",14,7],"CCBot":["partial","*",14,7],"Bytespider":["partial","*",14,7],"Amazonbot":["partial","*",14,7],"meta-externalagent":["partial","*",14,7],"cohere-ai":["partial","*",14,7],"Diffbot":["partial","*",14,7],"ContextFetchBot":["partial","*",14,7]},"llms":{"status":"missing","http":404,"bytes":28109},"home":{"status":"ok","http":200,"url":"https://und.edu/","html_tokens":12638,"markdown_tokens":76,"pruned_tokens":76}},
{"rank":7811,"domain":"icecasinopl.org","origin":"https://icecasinopl.org","robots":{"status":"timeout","http":null,"url":"https://icecasinopl.org/robots.txt","bytes":0,"groups":0,"error":"ConnectTimeout"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":7812,"domain":"news12.com","origin":"https://news12.com","robots":{"status":"ok","http":200,"url":"https://news12.com/api/robots","bytes":145,"groups":1},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"missing","http":404,"bytes":117259},"home":{"status":"ok","http":200,"url":"https://longisland.news12.com","html_tokens":133711,"markdown_tokens":15,"pruned_tokens":15}},
{"rank":7813,"domain":"binance.info","origin":"https://binance.info","robots":{"status":"html","http":200,"url":"https://www.binance.info/en","bytes":419376,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":7814,"domain":"rosettastone.com","origin":"https://rosettastone.com","robots":{"status":"ok","http":200,"url":"https://www.rosettastone.com/robots.txt","bytes":193,"groups":4},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["partial","*",1,0],"OAI-SearchBot":["partial","*",1,0],"ClaudeBot":["partial","*",1,0],"Claude-User":["partial","*",1,0],"Claude-SearchBot":["partial","*",1,0],"anthropic-ai":["partial","*",1,0],"PerplexityBot":["partial","*",1,0],"Perplexity-User":["partial","*",1,0],"Google-Extended":["partial","*",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["partial","*",1,0],"Amazonbot":["partial","*",1,0],"meta-externalagent":["partial","*",1,0],"cohere-ai":["partial","*",1,0],"Diffbot":["partial","*",1,0],"ContextFetchBot":["partial","*",1,0]},"llms":{"status":"missing","http":404,"bytes":416625},"home":{"status":"ok","http":200,"url":"https://www.rosettastone.com/","html_tokens":326923,"markdown_tokens":237,"pruned_tokens":75}},
{"rank":7815,"domain":"appstate.edu","origin":"https://appstate.edu","robots":{"status":"ok","http":200,"url":"https://www.appstate.edu/robots.txt","bytes":15,"groups":1},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"missing","http":404,"bytes":14533},"home":{"status":"ok","http":200,"url":"https://www.appstate.edu/","html_tokens":8336,"markdown_tokens":68,"pruned_tokens":68}},
{"rank":7816,"domain":"shaanxi.gov.cn","origin":"https://shaanxi.gov.cn","robots":{"status":"unreachable","http":null,"url":"https://shaanxi.gov.cn/robots.txt","bytes":0,"groups":0,"error":"cannot resolve shaanxi.gov.cn"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":7817,"domain":"kdvr.com","origin":"https://kdvr.com","robots":{"status":"ok","http":200,"url":"https://kdvr.com/robots.txt","bytes":2294,"groups":40},"bots":{"GPTBot":["blocked","named",2,0],"ChatGPT-User":["blocked","named",1,0],"OAI-SearchBot":["blocked","named",1,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",12,1],"Claude-SearchBot":["partial","*",12,1],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["blocked","named",2,0],"Perplexity-User":["partial","*",12,1],"Google-Extended":["blocked","named",2,0],"Applebot-Extended":["blocked","named",2,0],"CCBot":["blocked","named",2,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["partial","*",12,1],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",12,1]},"llms":{"status":"http_error","http":403,"bytes":6514},"home":{"status":"http_error","http":403,"url":"https://kdvr.com/"}},
{"rank":7818,"domain":"instagr.am","origin":"https://instagr.am","robots":{"status":"ok","http":200,"url":"https://www.instagram.com/robots.txt?short_redirect=1","bytes":6184,"groups":36},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["blocked","*",1,0],"OAI-SearchBot":["blocked","*",1,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["blocked","*",1,0],"Claude-SearchBot":["blocked","*",1,0],"anthropic-ai":["blocked","*",1,0],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["blocked","*",1,0],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","*",1,0],"Bytespider":["blocked","*",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","*",1,0],"cohere-ai":["blocked","*",1,0],"Diffbot":["blocked","*",1,0],"ContextFetchBot":["blocked","*",1,0]},"llms":{"status":"disallowed"},"home":{"status":"disallowed"}},
{"rank":7819,"domain":"vassar.edu","origin":"https://vassar.edu","robots":{"status":"ok","http":200,"url":"https://www.vassar.edu/robots.txt","bytes":2141,"groups":1},"bots":{"GPTBot":["partial","*",35,22],"ChatGPT-User":["partial","*",35,22],"OAI-SearchBot":["partial","*",35,22],"ClaudeBot":["partial","*",35,22],"Claude-User":["partial","*",35,22],"Claude-SearchBot":["partial","*",35,22],"anthropic-ai":["partial","*",35,22],"PerplexityBot":["partial","*",35,22],"Perplexity-User":["partial","*",35,22],"Google-Extended":["partial","*",35,22],"Applebot-Extended":["partial","*",35,22],"CCBot":["partial","*",35,22],"Bytespider":["partial","*",35,22],"Amazonbot":["partial","*",35,22],"meta-externalagent":["partial","*",35,22],"cohere-ai":["partial","*",35,22],"Diffbot":["partial","*",35,22],"ContextFetchBot":["partial","*",35,22]},"llms":{"status":"missing","http":404,"bytes":188},"home":{"status":"ok","http":200,"url":"https://www.vassar.edu/","html_tokens":22424,"markdown_tokens":288,"pruned_tokens":210}},
{"rank":7820,"domain":"carfax.com","origin":"https://carfax.com","robots":{"status":"ok","http":200,"url":"https://www.carfax.com/robots.txt","bytes":1344,"groups":1},"bots":{"GPTBot":["partial","*",55,0],"ChatGPT-User":["partial","*",55,0],"OAI-SearchBot":["partial","*",55,0],"ClaudeBot":["partial","*",55,0],"Claude-User":["partial","*",55,0],"Claude-SearchBot":["partial","*",55,0],"anthropic-ai":["partial","*",55,0],"PerplexityBot":["partial","*",55,0],"Perplexity-User":["partial","*",55,0],"Google-Extended":["partial","*",55,0],"Applebot-Extended":["partial","*",55,0],"CCBot":["partial","*",55,0],"Bytespider":["partial","*",55,0],"Amazonbot":["partial","*",55,0],"meta-externalagent":["partial","*",55,0],"cohere-ai":["partial","*",55,0],"Diffbot":["partial","*",55,0],"ContextFetchBot":["partial","*",55,0]},"llms":{"status":"missing","http":404,"bytes":64948},"home":{"status":"ok","http":200,"url":"https://www.carfax.com/","html_tokens":22038,"markdown_tokens":29,"pruned_tokens":29}},
{"rank":7821,"domain":"23andme.com","origin":"https://23andme.com","robots":{"status":"ok","http":200,"url":"https://www.23andme.org/robots.txt","bytes":3030,"groups":4},"bots":{"GPTBot":["partial","*",14,1],"ChatGPT-User":["partial","named",8,1],"OAI-SearchBot":["partial","named",8,1],"ClaudeBot":["partial","*",14,1],"Claude-User":["partial","named",8,1],"Claude-SearchBot":["partial","named",8,1],"anthropic-ai":["partial","*",14,1],"PerplexityBot":["partial","named",8,1],"Perplexity-User":["partial","named",8,1],"Google-Extended":["partial","*",14,1],"Applebot-Extended":["partial","*",14,1],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["partial","*",14,1],"meta-externalagent":["partial","*",14,1],"cohere-ai":["partial","*",14,1],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",14,1]},"llms":{"status":"present","http":200,"bytes":6639},"home":{"status":"ok","http":200,"url":"https://www.23andme.org/","html_tokens":32016,"markdown_tokens":2970,"pruned_tokens":391}},
{"rank":7822,"domain":"ascd.org","origin":"https://ascd.org","robots":{"status":"ok","http":200,"url":"https://ascd.org/robots.txt","bytes":188,"groups":1},"bots":{"GPTBot":["partial","*",2,0],"ChatGPT-User":["partial","*",2,0],"OAI-SearchBot":["partial","*",2,0],"ClaudeBot":["partial","*",2,0],"Claude-User":["partial","*",2,0],"Claude-SearchBot":["partial","*",2,0],"anthropic-ai":["partial","*",2,0],"PerplexityBot":["partial","*",2,0],"Perplexity-User":["partial","*",2,0],"Google-Extended":["partial","*",2,0],"Applebot-Extended":["partial","*",2,0],"CCBot":["partial","*",2,0],"Bytespider":["partial","*",2,0],"Amazonbot":["partial","*",2,0],"meta-externalagent":["partial","*",2,0],"cohere-ai":["partial","*",2,0],"Diffbot":["partial","*",2,0],"ContextFetchBot":["partial","*",2,0]},"llms":{"status":"missing","http":404,"bytes":143061},"home":{"status":"ok","http":200,"url":"https://ascd.org/","html_tokens":91730,"markdown_tokens":57,"pruned_tokens":43}},
{"rank":7823,"domain":"modernatx.com","origin":"https://modernatx.com","robots":{"status":"ok","http":200,"url":"https://www.modernatx.com/robots.txt","bytes":437,"groups":1},"bots":{"GPTBot":["partial","*",18,1],"ChatGPT-User":["partial","*",18,1],"OAI-SearchBot":["partial","*",18,1],"ClaudeBot":["partial","*",18,1],"Claude-User":["partial","*",18,1],"Claude-SearchBot":["partial","*",18,1],"anthropic-ai":["partial","*",18,1],"PerplexityBot":["partial","*",18,1],"Perplexity-User":["partial","*",18,1],"Google-Extended":["partial","*",18,1],"Applebot-Extended":["partial","*",18,1],"CCBot":["partial","*",18,1],"Bytespider":["partial","*",18,1],"Amazonbot":["partial","*",18,1],"meta-externalagent":["partial","*",18,1],"cohere-ai":["partial","*",18,1],"Diffbot":["partial","*",18,1],"ContextFetchBot":["partial","*",18,1]},"llms":{"status":"missing","http":404,"bytes":9202},"home":{"status":"ok","http":200,"url":"https://www.modernatx.com/","html_tokens":304344,"markdown_tokens":182,"pruned_tokens":133}},
{"rank":7824,"domain":"artforum.com","origin":"https://artforum.com","robots":{"status":"ok","http":200,"url":"https://www.artforum.com/robots.txt","bytes":1256,"groups":26},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["blocked","named",1,0],"OAI-SearchBot":["blocked","named",1,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",9,1],"Claude-SearchBot":["partial","*",9,1],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["partial","*",9,1],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["partial","*",9,1],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",9,1]},"llms":{"status":"missing","http":404,"bytes":325531},"home":{"status":"ok","http":200,"url":"https://www.artforum.com/","html_tokens":122558,"markdown_tokens":125,"pruned_tokens":83}},
{"rank":7825,"domain":"nfb.ca","origin":"https://nfb.ca","robots":{"status":"ok","http":200,"url":"https://www.nfb.ca/robots.txt","bytes":181,"groups":1},"bots":{"GPTBot":["partial","*",4,0],"ChatGPT-User":["partial","*",4,0],"OAI-SearchBot":["partial","*",4,0],"ClaudeBot":["partial","*",4,0],"Claude-User":["partial","*",4,0],"Claude-SearchBot":["partial","*",4,0],"anthropic-ai":["partial","*",4,0],"PerplexityBot":["partial","*",4,0],"Perplexity-User":["partial","*",4,0],"Google-Extended":["partial","*",4,0],"Applebot-Extended":["partial","*",4,0],"CCBot":["partial","*",4,0],"Bytespider":["partial","*",4,0],"Amazonbot":["partial","*",4,0],"meta-externalagent":["partial","*",4,0],"cohere-ai":["partial","*",4,0],"Diffbot":["partial","*",4,0],"ContextFetchBot":["partial","*",4,0]},"llms":{"status":"missing","http":404,"bytes":0},"home":{"status":"ok","http":200,"url":"https://www.nfb.ca/","html_tokens":15141,"markdown_tokens":70,"pruned_tokens":67}},
{"rank":7826,"domain":"clamav.net","origin":"https://clamav.net","robots":{"status":"ok","http":200,"url":"https://www.clamav.net/robots.txt","bytes":204,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"timeout","error":"ConnectTimeout"},"home":{"status":"timeout","error":"ConnectTimeout"}},
{"rank":7827,"domain":"webtoons.com","origin":"https://webtoons.com","robots":{"status":"ok","http":200,"url":"https://www.webtoons.com/robots.txt","bytes":572,"groups":3},"bots":{"GPTBot":["partial","*",12,2],"ChatGPT-User":["partial","*",12,2],"OAI-SearchBot":["partial","*",12,2],"ClaudeBot":["partial","*",12,2],"Claude-User":["partial","*",12,2],"Claude-SearchBot":["partial","*",12,2],"anthropic-ai":["partial","*",12,2],"PerplexityBot":["partial","*",12,2],"Perplexity-User":["partial","*",12,2],"Google-Extended":["partial","*",12,2],"Applebot-Extended":["partial","*",12,2],"CCBot":["partial","*",12,2],"Bytespider":["partial","*",12,2],"Amazonbot":["partial","*",12,2],"meta-externalagent":["partial","*",12,2],"cohere-ai":["partial","*",12,2],"Diffbot":["partial","*",12,2],"ContextFetchBot":["partial","*",12,2]},"llms":{"status":"missing","http":404,"bytes":5555},"home":{"status":"ok","http":200,"url":"https://www.webtoons.com/en/","html_tokens":60621,"markdown_tokens":30,"pruned_tokens":29}},
{"rank":7828,"domain":"huopint.com","origin":"https://huopint.com","robots":{"status":"ok","http":200,"url":"https://huopint.com/robots.txt","bytes":555,"groups":1},"bots":{"GPTBot":["partial","*",24,0],"ChatGPT-User":["partial","*",24,0],"OAI-SearchBot":["partial","*",24,0],"ClaudeBot":["partial","*",24,0],"Claude-User":["partial","*",24,0],"Claude-SearchBot":["partial","*",24,0],"anthropic-ai":["partial","*",24,0],"PerplexityBot":["partial","*",24,0],"Perplexity-User":["partial","*",24,0],"Google-Extended":["partial","*",24,0],"Applebot-Extended":["partial","*",24,0],"CCBot":["partial","*",24,0],"Bytespider":["partial","*",24,0],"Amazonbot":["partial","*",24,0],"meta-externalagent":["partial","*",24,0],"cohere-ai":["partial","*",24,0],"Diffbot":["partial","*",24,0],"ContextFetchBot":["partial","*",24,0]},"llms":{"status":"disallowed"},"home":{"status":"ok","http":200,"url":"https://huopint.com/","html_tokens":51,"markdown_tokens":0,"pruned_tokens":0}},
{"rank":7829,"domain":"infosys.com","origin":"https://www.infosys.com","robots":{"status":"refused","http":403,"url":"https://www.infosys.com/robots.txt","bytes":381,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":7830,"domain":"virginatlantic.com","origin":"https://virginatlantic.com","robots":{"status":"ok","http":200,"url":"https://www.virginatlantic.com/robots.txt","bytes":669,"groups":1},"bots":{"GPTBot":["partial","*",15,1],"ChatGPT-User":["partial","*",15,1],"OAI-SearchBot":["partial","*",15,1],"ClaudeBot":["partial","*",15,1],"Claude-User":["partial","*",15,1],"Claude-SearchBot":["partial","*",15,1],"anthropic-ai":["partial","*",15,1],"PerplexityBot":["partial","*",15,1],"Perplexity-User":["partial","*",15,1],"Google-Extended":["partial","*",15,1],"Applebot-Extended":["partial","*",15,1],"CCBot":["partial","*",15,1],"Bytespider":["partial","*",15,1],"Amazonbot":["partial","*",15,1],"meta-externalagent":["partial","*",15,1],"cohere-ai":["partial","*",15,1],"Diffbot":["partial","*",15,1],"ContextFetchBot":["partial","*",15,1]},"llms":{"status":"missing","http":404,"bytes":30496},"home":{"status":"ok","http":200,"url":"https://www.virginatlantic.com/en-US","html_tokens":86481,"markdown_tokens":130,"pruned_tokens":129}},
{"rank":7831,"domain":"scribehow.com","origin":"https://scribehow.com","robots":{"status":"ok","http":200,"url":"https://scribe.com/robots.txt","bytes":202,"groups":1},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"missing","http":404,"bytes":91250},"home":{"status":"ok","http":200,"url":"https://scribe.com/","html_tokens":93896,"markdown_tokens":631,"pruned_tokens":57}},
{"rank":7832,"domain":"jcyl.es","origin":"https://jcyl.es","robots":{"status":"ok","http":200,"url":"https://www.jcyl.es/robots.txt","bytes":1899,"groups":2},"bots":{"GPTBot":["partial","*",39,0],"ChatGPT-User":["partial","*",39,0],"OAI-SearchBot":["partial","*",39,0],"ClaudeBot":["partial","*",39,0],"Claude-User":["partial","*",39,0],"Claude-SearchBot":["partial","*",39,0],"anthropic-ai":["partial","*",39,0],"PerplexityBot":["partial","*",39,0],"Perplexity-User":["partial","*",39,0],"Google-Extended":["partial","*",39,0],"Applebot-Extended":["partial","*",39,0],"CCBot":["partial","*",39,0],"Bytespider":["partial","*",39,0],"Amazonbot":["partial","*",39,0],"meta-externalagent":["partial","*",39,0],"cohere-ai":["partial","*",39,0],"Diffbot":["partial","*",39,0],"ContextFetchBot":["partial","*",39,0]},"llms":{"status":"missing","http":404,"bytes":984},"home":{"status":"ok","http":200,"url":"https://www.jcyl.es/web/jcyl/Portada/es/Home/1246890364336/_/_/_","html_tokens":8905,"markdown_tokens":113,"pruned_tokens":113}},
{"rank":7833,"domain":"z.ai","origin":"https://z.ai","robots":{"status":"ok","http":200,"url":"https://z.ai/robots.txt","bytes":56,"groups":1},"bots":{"GPTBot":["allowed","*",0,1],"ChatGPT-User":["allowed","*",0,1],"OAI-SearchBot":["allowed","*",0,1],"ClaudeBot":["allowed","*",0,1],"Claude-User":["allowed","*",0,1],"Claude-SearchBot":["allowed","*",0,1],"anthropic-ai":["allowed","*",0,1],"PerplexityBot":["allowed","*",0,1],"Perplexity-User":["allowed","*",0,1],"Google-Extended":["allowed","*",0,1],"Applebot-Extended":["allowed","*",0,1],"CCBot":["allowed","*",0,1],"Bytespider":["allowed","*",0,1],"Amazonbot":["allowed","*",0,1],"meta-externalagent":["allowed","*",0,1],"cohere-ai":["allowed","*",0,1],"Diffbot":["allowed","*",0,1],"ContextFetchBot":["allowed","*",0,1]},"llms":{"status":"missing","http":404,"bytes":21678},"home":{"status":"ok","http":200,"url":"https://chat.z.ai/","html_tokens":3653,"markdown_tokens":0,"pruned_tokens":0}},
{"rank":7834,"domain":"google.co.ug","origin":"https://google.co.ug","robots":{"status":"ok","http":200,"url":"https://www.google.co.ug/robots.txt","bytes":6624,"groups":4},"bots":{"GPTBot":["partial","*",169,64],"ChatGPT-User":["partial","*",169,64],"OAI-SearchBot":["partial","*",169,64],"ClaudeBot":["partial","*",169,64],"Claude-User":["partial","*",169,64],"Claude-SearchBot":["partial","*",169,64],"anthropic-ai":["partial","*",169,64],"PerplexityBot":["partial","*",169,64],"Perplexity-User":["partial","*",169,64],"Google-Extended":["partial","*",169,64],"Applebot-Extended":["partial","*",169,64],"CCBot":["partial","*",169,64],"Bytespider":["partial","*",169,64],"Amazonbot":["partial","*",169,64],"meta-externalagent":["partial","*",169,64],"cohere-ai":["partial","*",169,64],"Diffbot":["partial","*",169,64],"ContextFetchBot":["partial","*",169,64]},"llms":{"status":"missing","http":404,"bytes":1569},"home":{"status":"ok","http":200,"url":"https://www.google.com/","html_tokens":21337,"markdown_tokens":30,"pruned_tokens":30}},
{"rank":7835,"domain":"alpinelinux.org","origin":"https://alpinelinux.org","robots":{"status":"missing","http":404,"url":"https://alpinelinux.org/robots.txt","bytes":146,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":146},"home":{"status":"ok","http":200,"url":"https://alpinelinux.org/","html_tokens":2289,"markdown_tokens":121,"pruned_tokens":121}},
{"rank":7836,"domain":"resend.com","origin":"https://resend.com","robots":{"status":"ok","http":200,"url":"https://resend.com/robots.txt","bytes":143,"groups":1},"bots":{"GPTBot":["allowed","*",0,1],"ChatGPT-User":["allowed","*",0,1],"OAI-SearchBot":["allowed","*",0,1],"ClaudeBot":["allowed","*",0,1],"Claude-User":["allowed","*",0,1],"Claude-SearchBot":["allowed","*",0,1],"anthropic-ai":["allowed","*",0,1],"PerplexityBot":["allowed","*",0,1],"Perplexity-User":["allowed","*",0,1],"Google-Extended":["allowed","*",0,1],"Applebot-Extended":["allowed","*",0,1],"CCBot":["allowed","*",0,1],"Bytespider":["allowed","*",0,1],"Amazonbot":["allowed","*",0,1],"meta-externalagent":["allowed","*",0,1],"cohere-ai":["allowed","*",0,1],"Diffbot":["allowed","*",0,1],"ContextFetchBot":["allowed","*",0,1]},"llms":{"status":"present","http":200,"bytes":8340},"home":{"status":"not_html","error":"unsupported content type text/markdown"}},
{"rank":7837,"domain":"networkforgood.com","origin":"https://networkforgood.com","robots":{"status":"ok","http":200,"url":"https://www.bonterratech.com/robots.txt?utm_source=networkforgood&utm_medium=redirect","bytes":197,"groups":1},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"http_error","http":403,"bytes":5549},"home":{"status":"http_error","http":403,"url":"https://www.bonterratech.com/network-for-good?utm_source=networkforgood&utm_medium=redirect"}},
{"rank":7838,"domain":"ind.in","origin":"https://ind.in","robots":{"status":"unreachable","http":null,"url":"https://ind.in/robots.txt","bytes":0,"groups":0,"error":"cannot resolve ind.in"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":7839,"domain":"ecu.edu","origin":"https://ecu.edu","robots":{"status":"refused","http":403,"url":"https://ecu.edu/robots.txt","bytes":560,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":7840,"domain":"playpokiesfree.com","origin":"https://www.playpokiesfree.com","robots":{"status":"missing","http":404,"url":"https://www.playpokiesfree.com/robots.txt","bytes":162,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"timeout","error":"ConnectTimeout"},"home":{"status":"timeout","error":"ConnectTimeout"}},
{"rank":7841,"domain":"jccqiangbu.com","origin":"https://www.jccqiangbu.com","robots":{"status":"ok","http":200,"url":"https://www.jccqiangbu.com/robots.txt","bytes":921,"groups":20},"bots":{"GPTBot":["partial","*",3,0],"ChatGPT-User":["partial","*",3,0],"OAI-SearchBot":["partial","*",3,0],"ClaudeBot":["partial","*",3,0],"Claude-User":["partial","*",3,0],"Claude-SearchBot":["partial","*",3,0],"anthropic-ai":["partial","*",3,0],"PerplexityBot":["partial","*",3,0],"Perplexity-User":["partial","*",3,0],"Google-Extended":["partial","*",3,0],"Applebot-Extended":["partial","*",3,0],"CCBot":["partial","*",3,0],"Bytespider":["partial","*",3,0],"Amazonbot":["partial","*",3,0],"meta-externalagent":["partial","*",3,0],"cohere-ai":["partial","*",3,0],"Diffbot":["partial","*",3,0],"ContextFetchBot":["partial","*",3,0]},"llms":{"status":"missing","http":404,"bytes":102620},"home":{"status":"ok","http":200,"url":"https://www.jccqiangbu.com/","html_tokens":11080,"markdown_tokens":31,"pruned_tokens":31}},
{"rank":7842,"domain":"sofascore.com","origin":"https://sofascore.com","robots":{"status":"refused","http":403,"url":"https://sofascore.com/robots.txt","bytes":48,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":7843,"domain":"archiveofourown.org","origin":"https://archiveofourown.org","robots":{"status":"refused","http":403,"url":"https://archiveofourown.org/robots.txt","bytes":8948,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":7844,"domain":"marketingprofs.com","origin":"https://marketingprofs.com","robots":{"status":"ok","http":200,"url":"https://www.marketingprofs.com/robots.txt","bytes":1091,"groups":2},"bots":{"GPTBot":["partial","*",21,0],"ChatGPT-User":["partial","*",21,0],"OAI-SearchBot":["partial","*",21,0],"ClaudeBot":["partial","*",21,0],"Claude-User":["partial","*",21,0],"Claude-SearchBot":["partial","*",21,0],"anthropic-ai":["partial","*",21,0],"PerplexityBot":["partial","*",21,0],"Perplexity-User":["partial","*",21,0],"Google-Extended":["partial","*",21,0],"Applebot-Extended":["partial","*",21,0],"CCBot":["partial","*",21,0],"Bytespider":["partial","*",21,0],"Amazonbot":["partial","*",21,0],"meta-externalagent":["partial","*",21,0],"cohere-ai":["partial","*",21,0],"Diffbot":["partial","*",21,0],"ContextFetchBot":["partial","*",21,0]},"llms":{"status":"missing","http":404,"bytes":31499},"home":{"status":"http_error","http":403,"url":"https://marketingprofs.com/"}},
{"rank":7845,"domain":"villanova.edu","origin":"https://villanova.edu","robots":{"status":"ok","http":200,"url":"https://www.villanova.edu/robots.txt","bytes":377,"groups":1},"bots":{"GPTBot":["partial","*",15,0],"ChatGPT-User":["partial","*",15,0],"OAI-SearchBot":["partial","*",15,0],"ClaudeBot":["partial","*",15,0],"Claude-User":["partial","*",15,0],"Claude-SearchBot":["partial","*",15,0],"anthropic-ai":["partial","*",15,0],"PerplexityBot":["partial","*",15,0],"Perplexity-User":["partial","*",15,0],"Google-Extended":["partial","*",15,0],"Applebot-Extended":["partial","*",15,0],"CCBot":["partial","*",15,0],"Bytespider":["partial","*",15,0],"Amazonbot":["partial","*",15,0],"meta-externalagent":["partial","*",15,0],"cohere-ai":["partial","*",15,0],"Diffbot":["partial","*",15,0],"ContextFetchBot":["partial","*",15,0]},"llms":{"status":"missing","http":404,"bytes":34400},"home":{"status":"ok","http":200,"url":"https://www.villanova.edu/university.html","html_tokens":18268,"markdown_tokens":677,"pruned_tokens":423}},
{"rank":7846,"domain":"scene7.com","origin":"https://scene7.com","robots":{"status":"timeout","http":null,"url":"https://scene7.com/robots.txt","bytes":0,"groups":0,"error":"ConnectTimeout"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":7847,"domain":"histats.com","origin":"https://histats.com","robots":{"status":"ok","http":200,"url":"https://www.histats.com/robots.txt","bytes":192,"groups":1},"bots":{"GPTBot":["partial","*",5,0],"ChatGPT-User":["partial","*",5,0],"OAI-SearchBot":["partial","*",5,0],"ClaudeBot":["partial","*",5,0],"Claude-User":["partial","*",5,0],"Claude-SearchBot":["partial","*",5,0],"anthropic-ai":["partial","*",5,0],"PerplexityBot":["partial","*",5,0],"Perplexity-User":["partial","*",5,0],"Google-Extended":["partial","*",5,0],"Applebot-Extended":["partial","*",5,0],"CCBot":["partial","*",5,0],"Bytespider":["partial","*",5,0],"Amazonbot":["partial","*",5,0],"meta-externalagent":["partial","*",5,0],"cohere-ai":["partial","*",5,0],"Diffbot":["partial","*",5,0],"ContextFetchBot":["partial","*",5,0]},"llms":{"status":"html","http":200,"bytes":311},"home":{"status":"ok","http":200,"url":"https://www.histats.com/","html_tokens":8522,"markdown_tokens":276,"pruned_tokens":36}},
{"rank":7848,"domain":"g20.org","origin":"https://g20.org","robots":{"status":"refused","http":403,"url":"https://g20.org/robots.txt","bytes":919,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":7849,"domain":"hbchengjian.com","origin":"http://hbchengjian.com","robots":{"status":"missing","http":404,"url":"http://hbchengjian.com/robots.txt","bytes":4738,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":4734},"home":{"status":"ok","http":200,"url":"http://hbchengjian.com/","html_tokens":4956,"markdown_tokens":118,"pruned_tokens":118}},
{"rank":7850,"domain":"eni.com","origin":"https://eni.com","robots":{"status":"ok","http":200,"url":"https://www.eni.com/robots.txt","bytes":760,"groups":1},"bots":{"GPTBot":["partial","*",9,0],"ChatGPT-User":["partial","*",9,0],"OAI-SearchBot":["partial","*",9,0],"ClaudeBot":["partial","*",9,0],"Claude-User":["partial","*",9,0],"Claude-SearchBot":["partial","*",9,0],"anthropic-ai":["partial","*",9,0],"PerplexityBot":["partial","*",9,0],"Perplexity-User":["partial","*",9,0],"Google-Extended":["partial","*",9,0],"Applebot-Extended":["partial","*",9,0],"CCBot":["partial","*",9,0],"Bytespider":["partial","*",9,0],"Amazonbot":["partial","*",9,0],"meta-externalagent":["partial","*",9,0],"cohere-ai":["partial","*",9,0],"Diffbot":["partial","*",9,0],"ContextFetchBot":["partial","*",9,0]},"llms":{"status":"missing","http":404,"bytes":373947},"home":{"status":"ok","http":200,"url":"https://www.eni.com/en-IT/home.html","html_tokens":100963,"markdown_tokens":99,"pruned_tokens":99}},
{"rank":7851,"domain":"inegi.org.mx","origin":"https://inegi.org.mx","robots":{"status":"ok","http":200,"url":"https://inegi.org.mx/robots.txt","bytes":2657,"groups":1},"bots":{"GPTBot":["partial","*",44,23],"ChatGPT-User":["partial","*",44,23],"OAI-SearchBot":["partial","*",44,23],"ClaudeBot":["partial","*",44,23],"Claude-User":["partial","*",44,23],"Claude-SearchBot":["partial","*",44,23],"anthropic-ai":["partial","*",44,23],"PerplexityBot":["partial","*",44,23],"Perplexity-User":["partial","*",44,23],"Google-Extended":["partial","*",44,23],"Applebot-Extended":["partial","*",44,23],"CCBot":["partial","*",44,23],"Bytespider":["partial","*",44,23],"Amazonbot":["partial","*",44,23],"meta-externalagent":["partial","*",44,23],"cohere-ai":["partial","*",44,23],"Diffbot":["partial","*",44,23],"ContextFetchBot":["partial","*",44,23]},"llms":{"status":"html","http":200,"bytes":13370},"home":{"status":"ok","http":200,"url":"https://inegi.org.mx/","html_tokens":38424,"markdown_tokens":115,"pruned_tokens":115}},
{"rank":7852,"domain":"telstra.com.au","origin":"https://telstra.com.au","robots":{"status":"ok","http":200,"url":"https://www.telstra.com.au/robots.txt","bytes":3140,"groups":6},"bots":{"GPTBot":["partial","*",34,1],"ChatGPT-User":["partial","*",34,1],"OAI-SearchBot":["partial","*",34,1],"ClaudeBot":["partial","*",34,1],"Claude-User":["partial","*",34,1],"Claude-SearchBot":["partial","*",34,1],"anthropic-ai":["partial","*",34,1],"PerplexityBot":["partial","*",34,1],"Perplexity-User":["partial","*",34,1],"Google-Extended":["partial","*",34,1],"Applebot-Extended":["partial","*",34,1],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["partial","*",34,1],"cohere-ai":["partial","*",34,1],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",34,1]},"llms":{"status":"missing","http":404,"bytes":257055},"home":{"status":"ok","http":200,"url":"https://www.telstra.com.au/","html_tokens":77040,"markdown_tokens":253,"pruned_tokens":22}},
{"rank":7853,"domain":"econstor.eu","origin":"https://econstor.eu","robots":{"status":"ok","http":200,"url":"https://www.econstor.eu/robots.txt","bytes":3506,"groups":29},"bots":{"GPTBot":["partial","*",8,0],"ChatGPT-User":["partial","*",8,0],"OAI-SearchBot":["partial","*",8,0],"ClaudeBot":["partial","*",8,0],"Claude-User":["partial","*",8,0],"Claude-SearchBot":["partial","*",8,0],"anthropic-ai":["partial","*",8,0],"PerplexityBot":["partial","*",8,0],"Perplexity-User":["partial","*",8,0],"Google-Extended":["partial","*",8,0],"Applebot-Extended":["partial","*",8,0],"CCBot":["partial","*",8,0],"Bytespider":["partial","*",8,0],"Amazonbot":["partial","*",8,0],"meta-externalagent":["partial","*",8,0],"cohere-ai":["partial","*",8,0],"Diffbot":["partial","*",8,0],"ContextFetchBot":["partial","*",8,0]},"llms":{"status":"http_error","http":429,"bytes":2722},"home":{"status":"http_error","http":429,"url":"https://www.econstor.eu/"}},
{"rank":7854,"domain":"kcrw.com","origin":"https://kcrw.com","robots":{"status":"ok","http":200,"url":"https://www.kcrw.com/robots.txt","bytes":119,"groups":2},"bots":{"GPTBot":["allowed","*",0,1],"ChatGPT-User":["allowed","*",0,1],"OAI-SearchBot":["allowed","*",0,1],"ClaudeBot":["allowed","*",0,1],"Claude-User":["allowed","*",0,1],"Claude-SearchBot":["allowed","*",0,1],"anthropic-ai":["allowed","*",0,1],"PerplexityBot":["allowed","*",0,1],"Perplexity-User":["allowed","*",0,1],"Google-Extended":["allowed","*",0,1],"Applebot-Extended":["allowed","*",0,1],"CCBot":["allowed","*",0,1],"Bytespider":["allowed","*",0,1],"Amazonbot":["allowed","*",0,1],"meta-externalagent":["allowed","*",0,1],"cohere-ai":["allowed","*",0,1],"Diffbot":["allowed","*",0,1],"ContextFetchBot":["allowed","*",0,1]},"llms":{"status":"http_error","http":429,"bytes":34219},"home":{"status":"http_error","http":429,"url":"https://kcrw.com/"}},
{"rank":7855,"domain":"liberty.edu","origin":"https://liberty.edu","robots":{"status":"ok","http":200,"url":"https://www.liberty.edu/robots.txt","bytes":405,"groups":1},"bots":{"GPTBot":["partial","*",12,0],"ChatGPT-User":["partial","*",12,0],"OAI-SearchBot":["partial","*",12,0],"ClaudeBot":["partial","*",12,0],"Claude-User":["partial","*",12,0],"Claude-SearchBot":["partial","*",12,0],"anthropic-ai":["partial","*",12,0],"PerplexityBot":["partial","*",12,0],"Perplexity-User":["partial","*",12,0],"Google-Extended":["partial","*",12,0],"Applebot-Extended":["partial","*",12,0],"CCBot":["partial","*",12,0],"Bytespider":["partial","*",12,0],"Amazonbot":["partial","*",12,0],"meta-externalagent":["partial","*",12,0],"cohere-ai":["partial","*",12,0],"Diffbot":["partial","*",12,0],"ContextFetchBot":["partial","*",12,0]},"llms":{"status":"missing","http":404,"bytes":96742},"home":{"status":"ok","http":200,"url":"https://www.liberty.edu/","html_tokens":50133,"markdown_tokens":363,"pruned_tokens":183}},
{"rank":7856,"domain":"yelp.ca","origin":"https://yelp.ca","robots":{"status":"ok","http":200,"url":"https://www.yelp.ca/robots.txt","bytes":12697,"groups":9},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["partial","named",1,2],"OAI-SearchBot":["partial","named",1,3],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","named",1,2],"Claude-SearchBot":["partial","named",1,2],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["partial","named",1,2],"Perplexity-User":["partial","named",1,2],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","*",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","*",1,0],"Amazonbot":["blocked","*",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","*",1,0],"Diffbot":["blocked","*",1,0],"ContextFetchBot":["blocked","*",1,0]},"llms":{"status":"disallowed"},"home":{"status":"disallowed"}},
{"rank":7857,"domain":"google.td","origin":"https://google.td","robots":{"status":"ok","http":200,"url":"https://www.google.td/robots.txt","bytes":6624,"groups":4},"bots":{"GPTBot":["partial","*",169,64],"ChatGPT-User":["partial","*",169,64],"OAI-SearchBot":["partial","*",169,64],"ClaudeBot":["partial","*",169,64],"Claude-User":["partial","*",169,64],"Claude-SearchBot":["partial","*",169,64],"anthropic-ai":["partial","*",169,64],"PerplexityBot":["partial","*",169,64],"Perplexity-User":["partial","*",169,64],"Google-Extended":["partial","*",169,64],"Applebot-Extended":["partial","*",169,64],"CCBot":["partial","*",169,64],"Bytespider":["partial","*",169,64],"Amazonbot":["partial","*",169,64],"meta-externalagent":["partial","*",169,64],"cohere-ai":["partial","*",169,64],"Diffbot":["partial","*",169,64],"ContextFetchBot":["partial","*",169,64]},"llms":{"status":"missing","http":404,"bytes":1569},"home":{"status":"ok","http":200,"url":"https://www.google.com/","html_tokens":21301,"markdown_tokens":30,"pruned_tokens":30}},
{"rank":7858,"domain":"ice-casinos.org","origin":"https://ice-casinos.org","robots":{"status":"ok","http":200,"url":"https://ice-casinos.org/robots.txt","bytes":67,"groups":1},"bots":{"GPTBot":["partial","*",1,1],"ChatGPT-User":["partial","*",1,1],"OAI-SearchBot":["partial","*",1,1],"ClaudeBot":["partial","*",1,1],"Claude-User":["partial","*",1,1],"Claude-SearchBot":["partial","*",1,1],"anthropic-ai":["partial","*",1,1],"PerplexityBot":["partial","*",1,1],"Perplexity-User":["partial","*",1,1],"Google-Extended":["partial","*",1,1],"Applebot-Extended":["partial","*",1,1],"CCBot":["partial","*",1,1],"Bytespider":["partial","*",1,1],"Amazonbot":["partial","*",1,1],"meta-externalagent":["partial","*",1,1],"cohere-ai":["partial","*",1,1],"Diffbot":["partial","*",1,1],"ContextFetchBot":["partial","*",1,1]},"llms":{"status":"missing","http":404,"bytes":49069},"home":{"status":"ok","http":200,"url":"https://ice-casinos.org/","html_tokens":16965,"markdown_tokens":1675,"pruned_tokens":88}},
{"rank":7859,"domain":"nixos.org","origin":"https://nixos.org","robots":{"status":"ok","http":200,"url":"https://nixos.org/robots.txt","bytes":69,"groups":1},"bots":{"GPTBot":["allowed","*",0,1],"ChatGPT-User":["allowed","*",0,1],"OAI-SearchBot":["allowed","*",0,1],"ClaudeBot":["allowed","*",0,1],"Claude-User":["allowed","*",0,1],"Claude-SearchBot":["allowed","*",0,1],"anthropic-ai":["allowed","*",0,1],"PerplexityBot":["allowed","*",0,1],"Perplexity-User":["allowed","*",0,1],"Google-Extended":["allowed","*",0,1],"Applebot-Extended":["allowed","*",0,1],"CCBot":["allowed","*",0,1],"Bytespider":["allowed","*",0,1],"Amazonbot":["allowed","*",0,1],"meta-externalagent":["allowed","*",0,1],"cohere-ai":["allowed","*",0,1],"Diffbot":["allowed","*",0,1],"ContextFetchBot":["allowed","*",0,1]},"llms":{"status":"missing","http":404,"bytes":23057},"home":{"status":"ok","http":200,"url":"https://nixos.org/","html_tokens":15010,"markdown_tokens":100,"pruned_tokens":100}},
{"rank":7860,"domain":"turnkeylinux.org","origin":"https://turnkeylinux.org","robots":{"status":"ok","http":200,"url":"https://www.turnkeylinux.org/robots.txt","bytes":73,"groups":1},"bots":{"GPTBot":["allowed","*",0,1],"ChatGPT-User":["allowed","*",0,1],"OAI-SearchBot":["allowed","*",0,1],"ClaudeBot":["allowed","*",0,1],"Claude-User":["allowed","*",0,1],"Claude-SearchBot":["allowed","*",0,1],"anthropic-ai":["allowed","*",0,1],"PerplexityBot":["allowed","*",0,1],"Perplexity-User":["allowed","*",0,1],"Google-Extended":["allowed","*",0,1],"Applebot-Extended":["allowed","*",0,1],"CCBot":["allowed","*",0,1],"Bytespider":["allowed","*",0,1],"Amazonbot":["allowed","*",0,1],"meta-externalagent":["allowed","*",0,1],"cohere-ai":["allowed","*",0,1],"Diffbot":["allowed","*",0,1],"ContextFetchBot":["allowed","*",0,1]},"llms":{"status":"missing","http":404,"bytes":5481},"home":{"status":"ok","http":200,"url":"https://www.turnkeylinux.org/","html_tokens":13820,"markdown_tokens":535,"pruned_tokens":46}},
{"rank":7861,"domain":"owncloud.org","origin":"https://owncloud.org","robots":{"status":"ok","http":200,"url":"https://owncloud.com/robots.txt","bytes":334,"groups":3},"bots":{"GPTBot":["partial","*",11,0],"ChatGPT-User":["partial","*",11,0],"OAI-SearchBot":["partial","*",11,0],"ClaudeBot":["partial","*",11,0],"Claude-User":["partial","*",11,0],"Claude-SearchBot":["partial","*",11,0],"anthropic-ai":["partial","*",11,0],"PerplexityBot":["partial","*",11,0],"Perplexity-User":["partial","*",11,0],"Google-Extended":["partial","*",11,0],"Applebot-Extended":["partial","*",11,0],"CCBot":["partial","*",11,0],"Bytespider":["partial","*",11,0],"Amazonbot":["partial","*",11,0],"meta-externalagent":["partial","*",11,0],"cohere-ai":["partial","*",11,0],"Diffbot":["partial","*",11,0],"ContextFetchBot":["partial","*",11,0]},"llms":{"status":"http_error","http":403,"bytes":795},"home":{"status":"ok","http":200,"url":"https://owncloud.com/","html_tokens":45499,"markdown_tokens":827,"pruned_tokens":545}},
{"rank":7862,"domain":"esggi.com","origin":"http://esggi.com","robots":{"status":"missing","http":404,"url":"http://esggi.com/robots.txt","bytes":1163,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":1163},"home":{"status":"ok","http":200,"url":"http://www.esggi.com","html_tokens":7027,"markdown_tokens":359,"pruned_tokens":167}},
{"rank":7863,"domain":"cdsuojia.com","origin":"https://cdsuojia.com","robots":{"status":"ok","http":200,"url":"https://cdsuojia.com/robots.txt","bytes":113,"groups":1},"bots":{"GPTBot":["partial","*",2,1],"ChatGPT-User":["partial","*",2,1],"OAI-SearchBot":["partial","*",2,1],"ClaudeBot":["partial","*",2,1],"Claude-User":["partial","*",2,1],"Claude-SearchBot":["partial","*",2,1],"anthropic-ai":["partial","*",2,1],"PerplexityBot":["partial","*",2,1],"Perplexity-User":["partial","*",2,1],"Google-Extended":["partial","*",2,1],"Applebot-Extended":["partial","*",2,1],"CCBot":["partial","*",2,1],"Bytespider":["partial","*",2,1],"Amazonbot":["partial","*",2,1],"meta-externalagent":["partial","*",2,1],"cohere-ai":["partial","*",2,1],"Diffbot":["partial","*",2,1],"ContextFetchBot":["partial","*",2,1]},"llms":{"status":"missing","http":404,"bytes":0},"home":{"status":"ok","http":200,"url":"https://cdsuojia.com/","html_tokens":1992,"markdown_tokens":135,"pruned_tokens":135}},
{"rank":7864,"domain":"woerxny.com","origin":"https://woerxny.com","robots":{"status":"ok","http":200,"url":"https://woerxny.com/robots.txt","bytes":183,"groups":1},"bots":{"GPTBot":["partial","*",9,0],"ChatGPT-User":["partial","*",9,0],"OAI-SearchBot":["partial","*",9,0],"ClaudeBot":["partial","*",9,0],"Claude-User":["partial","*",9,0],"Claude-SearchBot":["partial","*",9,0],"anthropic-ai":["partial","*",9,0],"PerplexityBot":["partial","*",9,0],"Perplexity-User":["partial","*",9,0],"Google-Extended":["partial","*",9,0],"Applebot-Extended":["partial","*",9,0],"CCBot":["partial","*",9,0],"Bytespider":["partial","*",9,0],"Amazonbot":["partial","*",9,0],"meta-externalagent":["partial","*",9,0],"cohere-ai":["partial","*",9,0],"Diffbot":["partial","*",9,0],"ContextFetchBot":["partial","*",9,0]},"llms":{"status":"html","http":200,"bytes":3368},"home":{"status":"ok","http":200,"url":"https://woerxny.com/","html_tokens":4065,"markdown_tokens":16,"pruned_tokens":11}},
{"rank":7865,"domain":"jxgjp.com","origin":"https://jxgjp.com","robots":{"status":"ok","http":200,"url":"https://jxgjp.com/robots.txt","bytes":129,"groups":1},"bots":{"GPTBot":["partial","*",4,0],"ChatGPT-User":["partial","*",4,0],"OAI-SearchBot":["partial","*",4,0],"ClaudeBot":["partial","*",4,0],"Claude-User":["partial","*",4,0],"Claude-SearchBot":["partial","*",4,0],"anthropic-ai":["partial","*",4,0],"PerplexityBot":["partial","*",4,0],"Perplexity-User":["partial","*",4,0],"Google-Extended":["partial","*",4,0],"Applebot-Extended":["partial","*",4,0],"CCBot":["partial","*",4,0],"Bytespider":["partial","*",4,0],"Amazonbot":["partial","*",4,0],"meta-externalagent":["partial","*",4,0],"cohere-ai":["partial","*",4,0],"Diffbot":["partial","*",4,0],"ContextFetchBot":["partial","*",4,0]},"llms":{"status":"missing","http":404,"bytes":0},"home":{"status":"ok","http":200,"url":"https://jxgjp.com/","html_tokens":20912,"markdown_tokens":20,"pruned_tokens":20}},
{"rank":7866,"domain":"wallonie.be","origin":"https://wallonie.be","robots":{"status":"ok","http":200,"url":"https://wallonie.be/robots.txt","bytes":1672,"groups":1},"bots":{"GPTBot":["partial","*",23,18],"ChatGPT-User":["partial","*",23,18],"OAI-SearchBot":["partial","*",23,18],"ClaudeBot":["partial","*",23,18],"Claude-User":["partial","*",23,18],"Claude-SearchBot":["partial","*",23,18],"anthropic-ai":["partial","*",23,18],"PerplexityBot":["partial","*",23,18],"Perplexity-User":["partial","*",23,18],"Google-Extended":["partial","*",23,18],"Applebot-Extended":["partial","*",23,18],"CCBot":["partial","*",23,18],"Bytespider":["partial","*",23,18],"Amazonbot":["partial","*",23,18],"meta-externalagent":["partial","*",23,18],"cohere-ai":["partial","*",23,18],"Diffbot":["partial","*",23,18],"ContextFetchBot":["partial","*",23,18]},"llms":{"status":"missing","http":404,"bytes":185},"home":{"status":"ok","http":200,"url":"https://wallonie.be/fr","html_tokens":21001,"markdown_tokens":630,"pruned_tokens":121}},
{"rank":7867,"domain":"google.co.vi","origin":"https://google.co.vi","robots":{"status":"ok","http":200,"url":"https://www.google.co.vi/robots.txt","bytes":6624,"groups":4},"bots":{"GPTBot":["partial","*",169,64],"ChatGPT-User":["partial","*",169,64],"OAI-SearchBot":["partial","*",169,64],"ClaudeBot":["partial","*",169,64],"Claude-User":["partial","*",169,64],"Claude-SearchBot":["partial","*",169,64],"anthropic-ai":["partial","*",169,64],"PerplexityBot":["partial","*",169,64],"Perplexity-User":["partial","*",169,64],"Google-Extended":["partial","*",169,64],"Applebot-Extended":["partial","*",169,64],"CCBot":["partial","*",169,64],"Bytespider":["partial","*",169,64],"Amazonbot":["partial","*",169,64],"meta-externalagent":["partial","*",169,64],"cohere-ai":["partial","*",169,64],"Diffbot":["partial","*",169,64],"ContextFetchBot":["partial","*",169,64]},"llms":{"status":"missing","http":404,"bytes":1569},"home":{"status":"ok","http":200,"url":"https://www.google.com/","html_tokens":21273,"markdown_tokens":30,"pruned_tokens":30}},
{"rank":7868,"domain":"google.com.bz","origin":"https://google.com.bz","robots":{"status":"ok","http":200,"url":"https://www.google.com.bz/robots.txt","bytes":6624,"groups":4},"bots":{"GPTBot":["partial","*",169,64],"ChatGPT-User":["partial","*",169,64],"OAI-SearchBot":["partial","*",169,64],"ClaudeBot":["partial","*",169,64],"Claude-User":["partial","*",169,64],"Claude-SearchBot":["partial","*",169,64],"anthropic-ai":["partial","*",169,64],"PerplexityBot":["partial","*",169,64],"Perplexity-User":["partial","*",169,64],"Google-Extended":["partial","*",169,64],"Applebot-Extended":["partial","*",169,64],"CCBot":["partial","*",169,64],"Bytespider":["partial","*",169,64],"Amazonbot":["partial","*",169,64],"meta-externalagent":["partial","*",169,64],"cohere-ai":["partial","*",169,64],"Diffbot":["partial","*",169,64],"ContextFetchBot":["partial","*",169,64]},"llms":{"status":"missing","http":404,"bytes":1569},"home":{"status":"ok","http":200,"url":"https://www.google.com/","html_tokens":21340,"markdown_tokens":30,"pruned_tokens":30}},
{"rank":7869,"domain":"google.md","origin":"https://google.md","robots":{"status":"ok","http":200,"url":"https://www.google.md/robots.txt","bytes":6624,"groups":4},"bots":{"GPTBot":["partial","*",169,64],"ChatGPT-User":["partial","*",169,64],"OAI-SearchBot":["partial","*",169,64],"ClaudeBot":["partial","*",169,64],"Claude-User":["partial","*",169,64],"Claude-SearchBot":["partial","*",169,64],"anthropic-ai":["partial","*",169,64],"PerplexityBot":["partial","*",169,64],"Perplexity-User":["partial","*",169,64],"Google-Extended":["partial","*",169,64],"Applebot-Extended":["partial","*",169,64],"CCBot":["partial","*",169,64],"Bytespider":["partial","*",169,64],"Amazonbot":["partial","*",169,64],"meta-externalagent":["partial","*",169,64],"cohere-ai":["partial","*",169,64],"Diffbot":["partial","*",169,64],"ContextFetchBot":["partial","*",169,64]},"llms":{"status":"missing","http":404,"bytes":1569},"home":{"status":"ok","http":200,"url":"https://www.google.com/","html_tokens":21350,"markdown_tokens":30,"pruned_tokens":30}},
{"rank":7870,"domain":"pnnl.gov","origin":"https://pnnl.gov","robots":{"status":"html","http":200,"url":"https://www.pnnl.gov","bytes":87866,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":7871,"domain":"darksky.org","origin":"https://darksky.org","robots":{"status":"ok","http":200,"url":"https://darksky.org/robots.txt","bytes":115,"groups":1},"bots":{"GPTBot":["partial","*",1,1],"ChatGPT-User":["partial","*",1,1],"OAI-SearchBot":["partial","*",1,1],"ClaudeBot":["partial","*",1,1],"Claude-User":["partial","*",1,1],"Claude-SearchBot":["partial","*",1,1],"anthropic-ai":["partial","*",1,1],"PerplexityBot":["partial","*",1,1],"Perplexity-User":["partial","*",1,1],"Google-Extended":["partial","*",1,1],"Applebot-Extended":["partial","*",1,1],"CCBot":["partial","*",1,1],"Bytespider":["partial","*",1,1],"Amazonbot":["partial","*",1,1],"meta-externalagent":["partial","*",1,1],"cohere-ai":["partial","*",1,1],"Diffbot":["partial","*",1,1],"ContextFetchBot":["partial","*",1,1]},"llms":{"status":"missing","http":404,"bytes":308350},"home":{"status":"http_error","http":403,"url":"https://darksky.org/"}},
{"rank":7872,"domain":"ushistory.org","origin":"https://ushistory.org","robots":{"status":"refused","http":401,"url":"https://ushistory.org/robots.txt","bytes":5434,"groups":0,"error":"HTTP 401"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":7873,"domain":"300.cn","origin":"https://www.300.cn","robots":{"status":"ok","http":200,"url":"https://www.300.cn/robots.txt","bytes":440,"groups":1},"bots":{"GPTBot":["partial","*",3,1],"ChatGPT-User":["partial","*",3,1],"OAI-SearchBot":["partial","*",3,1],"ClaudeBot":["partial","*",3,1],"Claude-User":["partial","*",3,1],"Claude-SearchBot":["partial","*",3,1],"anthropic-ai":["partial","*",3,1],"PerplexityBot":["partial","*",3,1],"Perplexity-User":["partial","*",3,1],"Google-Extended":["partial","*",3,1],"Applebot-Extended":["partial","*",3,1],"CCBot":["partial","*",3,1],"Bytespider":["partial","*",3,1],"Amazonbot":["partial","*",3,1],"meta-externalagent":["partial","*",3,1],"cohere-ai":["partial","*",3,1],"Diffbot":["partial","*",3,1],"ContextFetchBot":["partial","*",3,1]},"llms":{"status":"present","http":200,"bytes":7409},"home":{"status":"ok","http":200,"url":"https://www.300.cn/","html_tokens":49999,"markdown_tokens":296,"pruned_tokens":70}},
{"rank":7874,"domain":"janeapp.com","origin":"https://janeapp.com","robots":{"status":"ok","http":200,"url":"https://jane.app/robots.txt","bytes":1563,"groups":1},"bots":{"GPTBot":["partial","*",23,1],"ChatGPT-User":["partial","*",23,1],"OAI-SearchBot":["partial","*",23,1],"ClaudeBot":["partial","*",23,1],"Claude-User":["partial","*",23,1],"Claude-SearchBot":["partial","*",23,1],"anthropic-ai":["partial","*",23,1],"PerplexityBot":["partial","*",23,1],"Perplexity-User":["partial","*",23,1],"Google-Extended":["partial","*",23,1],"Applebot-Extended":["partial","*",23,1],"CCBot":["partial","*",23,1],"Bytespider":["partial","*",23,1],"Amazonbot":["partial","*",23,1],"meta-externalagent":["partial","*",23,1],"cohere-ai":["partial","*",23,1],"Diffbot":["partial","*",23,1],"ContextFetchBot":["partial","*",23,1]},"llms":{"status":"missing","http":404,"bytes":9},"home":{"status":"ok","http":200,"url":"https://jane.app/","html_tokens":46674,"markdown_tokens":339,"pruned_tokens":202}},
{"rank":7875,"domain":"crucial.com","origin":"https://crucial.com","robots":{"status":"html","http":200,"url":"https://crucial.com/robots.txt","bytes":269,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":7876,"domain":"google.ps","origin":"https://google.ps","robots":{"status":"ok","http":200,"url":"https://www.google.ps/robots.txt","bytes":6624,"groups":4},"bots":{"GPTBot":["partial","*",169,64],"ChatGPT-User":["partial","*",169,64],"OAI-SearchBot":["partial","*",169,64],"ClaudeBot":["partial","*",169,64],"Claude-User":["partial","*",169,64],"Claude-SearchBot":["partial","*",169,64],"anthropic-ai":["partial","*",169,64],"PerplexityBot":["partial","*",169,64],"Perplexity-User":["partial","*",169,64],"Google-Extended":["partial","*",169,64],"Applebot-Extended":["partial","*",169,64],"CCBot":["partial","*",169,64],"Bytespider":["partial","*",169,64],"Amazonbot":["partial","*",169,64],"meta-externalagent":["partial","*",169,64],"cohere-ai":["partial","*",169,64],"Diffbot":["partial","*",169,64],"ContextFetchBot":["partial","*",169,64]},"llms":{"status":"missing","http":404,"bytes":1569},"home":{"status":"ok","http":200,"url":"https://www.google.com/","html_tokens":21259,"markdown_tokens":30,"pruned_tokens":30}},
{"rank":7877,"domain":"e-flux.com","origin":"https://e-flux.com","robots":{"status":"ok","http":200,"url":"https://www.e-flux.com/robots.txt","bytes":333,"groups":6},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["partial","*",5,0],"OAI-SearchBot":["partial","*",5,0],"ClaudeBot":["partial","*",5,0],"Claude-User":["partial","*",5,0],"Claude-SearchBot":["partial","*",5,0],"anthropic-ai":["partial","*",5,0],"PerplexityBot":["partial","*",5,0],"Perplexity-User":["partial","*",5,0],"Google-Extended":["partial","*",5,0],"Applebot-Extended":["partial","*",5,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["partial","*",5,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["partial","*",5,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",5,0]},"llms":{"status":"missing","http":404,"bytes":25650},"home":{"status":"ok","http":200,"url":"https://www.e-flux.com/","html_tokens":133767,"markdown_tokens":23,"pruned_tokens":23}},
{"rank":7878,"domain":"baeldung.com","origin":"https://baeldung.com","robots":{"status":"ok","http":200,"url":"https://www.baeldung.com/robots.txt","bytes":1080,"groups":1},"bots":{"GPTBot":["partial","*",11,11],"ChatGPT-User":["partial","*",11,11],"OAI-SearchBot":["partial","*",11,11],"ClaudeBot":["partial","*",11,11],"Claude-User":["partial","*",11,11],"Claude-SearchBot":["partial","*",11,11],"anthropic-ai":["partial","*",11,11],"PerplexityBot":["partial","*",11,11],"Perplexity-User":["partial","*",11,11],"Google-Extended":["partial","*",11,11],"Applebot-Extended":["partial","*",11,11],"CCBot":["partial","*",11,11],"Bytespider":["partial","*",11,11],"Amazonbot":["partial","*",11,11],"meta-externalagent":["partial","*",11,11],"cohere-ai":["partial","*",11,11],"Diffbot":["partial","*",11,11],"ContextFetchBot":["partial","*",11,11]},"llms":{"status":"other_text","http":200,"bytes":254707},"home":{"status":"http_error","http":403,"url":"https://www.baeldung.com/"}},
{"rank":7879,"domain":"cgi.com","origin":"https://cgi.com","robots":{"status":"ok","http":200,"url":"https://www.cgi.com/robots.txt","bytes":4286,"groups":2},"bots":{"GPTBot":["partial","*",68,18],"ChatGPT-User":["partial","*",68,18],"OAI-SearchBot":["partial","*",68,18],"ClaudeBot":["partial","*",68,18],"Claude-User":["partial","*",68,18],"Claude-SearchBot":["partial","*",68,18],"anthropic-ai":["partial","*",68,18],"PerplexityBot":["partial","*",68,18],"Perplexity-User":["partial","*",68,18],"Google-Extended":["partial","*",68,18],"Applebot-Extended":["partial","*",68,18],"CCBot":["partial","*",68,18],"Bytespider":["partial","*",68,18],"Amazonbot":["partial","*",68,18],"meta-externalagent":["partial","*",68,18],"cohere-ai":["partial","*",68,18],"Diffbot":["partial","*",68,18],"ContextFetchBot":["partial","*",68,18]},"llms":{"status":"present","http":200,"bytes":16251},"home":{"status":"ok","http":200,"url":"https://www.cgi.com/en","html_tokens":28074,"markdown_tokens":266,"pruned_tokens":110}},
{"rank":7880,"domain":"ncl.com","origin":"https://ncl.com","robots":{"status":"ok","http":200,"url":"https://www.ncl.com/robots.txt","bytes":2632,"groups":1},"bots":{"GPTBot":["partial","*",84,2],"ChatGPT-User":["partial","*",84,2],"OAI-SearchBot":["partial","*",84,2],"ClaudeBot":["partial","*",84,2],"Claude-User":["partial","*",84,2],"Claude-SearchBot":["partial","*",84,2],"anthropic-ai":["partial","*",84,2],"PerplexityBot":["partial","*",84,2],"Perplexity-User":["partial","*",84,2],"Google-Extended":["partial","*",84,2],"Applebot-Extended":["partial","*",84,2],"CCBot":["partial","*",84,2],"Bytespider":["partial","*",84,2],"Amazonbot":["partial","*",84,2],"meta-externalagent":["partial","*",84,2],"cohere-ai":["partial","*",84,2],"Diffbot":["partial","*",84,2],"ContextFetchBot":["partial","*",84,2]},"llms":{"status":"missing","http":404,"bytes":90885},"home":{"status":"ok","http":200,"url":"https://www.ncl.com/","html_tokens":104920,"markdown_tokens":110,"pruned_tokens":110}},
{"rank":7881,"domain":"uni.edu","origin":"https://uni.edu","robots":{"status":"ok","http":200,"url":"https://uni.edu/robots.txt","bytes":1920,"groups":1},"bots":{"GPTBot":["partial","*",30,18],"ChatGPT-User":["partial","*",30,18],"OAI-SearchBot":["partial","*",30,18],"ClaudeBot":["partial","*",30,18],"Claude-User":["partial","*",30,18],"Claude-SearchBot":["partial","*",30,18],"anthropic-ai":["partial","*",30,18],"PerplexityBot":["partial","*",30,18],"Perplexity-User":["partial","*",30,18],"Google-Extended":["partial","*",30,18],"Applebot-Extended":["partial","*",30,18],"CCBot":["partial","*",30,18],"Bytespider":["partial","*",30,18],"Amazonbot":["partial","*",30,18],"meta-externalagent":["partial","*",30,18],"cohere-ai":["partial","*",30,18],"Diffbot":["partial","*",30,18],"ContextFetchBot":["partial","*",30,18]},"llms":{"status":"missing","http":404,"bytes":196},"home":{"status":"ok","http":200,"url":"https://uni.edu/","html_tokens":54855,"markdown_tokens":6643,"pruned_tokens":175}},
{"rank":7882,"domain":"247sports.com","origin":"https://247sports.com","robots":{"status":"ok","http":200,"url":"https://247sports.com/robots.txt","bytes":661,"groups":1},"bots":{"GPTBot":["partial","*",24,0],"ChatGPT-User":["partial","*",24,0],"OAI-SearchBot":["partial","*",24,0],"ClaudeBot":["partial","*",24,0],"Claude-User":["partial","*",24,0],"Claude-SearchBot":["partial","*",24,0],"anthropic-ai":["partial","*",24,0],"PerplexityBot":["partial","*",24,0],"Perplexity-User":["partial","*",24,0],"Google-Extended":["partial","*",24,0],"Applebot-Extended":["partial","*",24,0],"CCBot":["partial","*",24,0],"Bytespider":["partial","*",24,0],"Amazonbot":["partial","*",24,0],"meta-externalagent":["partial","*",24,0],"cohere-ai":["partial","*",24,0],"Diffbot":["partial","*",24,0],"ContextFetchBot":["partial","*",24,0]},"llms":{"status":"http_error","http":403,"bytes":0},"home":{"status":"http_error","http":403,"url":"https://247sports.com/"}},
{"rank":7883,"domain":"anker.com","origin":"https://anker.com","robots":{"status":"ok","http":200,"url":"https://www.anker.com/robots.txt","bytes":1192,"groups":1},"bots":{"GPTBot":["partial","*",23,0],"ChatGPT-User":["partial","*",23,0],"OAI-SearchBot":["partial","*",23,0],"ClaudeBot":["partial","*",23,0],"Claude-User":["partial","*",23,0],"Claude-SearchBot":["partial","*",23,0],"anthropic-ai":["partial","*",23,0],"PerplexityBot":["partial","*",23,0],"Perplexity-User":["partial","*",23,0],"Google-Extended":["partial","*",23,0],"Applebot-Extended":["partial","*",23,0],"CCBot":["partial","*",23,0],"Bytespider":["partial","*",23,0],"Amazonbot":["partial","*",23,0],"meta-externalagent":["partial","*",23,0],"cohere-ai":["partial","*",23,0],"Diffbot":["partial","*",23,0],"ContextFetchBot":["partial","*",23,0]},"llms":{"status":"present","http":200,"bytes":7075},"home":{"status":"ok","http":200,"url":"https://www.anker.com/","html_tokens":501580,"markdown_tokens":8684,"pruned_tokens":357}},
{"rank":7884,"domain":"manus.im","origin":"https://manus.im","robots":{"status":"ok","http":200,"url":"https://manus.im/robots.txt","bytes":1045,"groups":1},"bots":{"GPTBot":["partial","*",1,1],"ChatGPT-User":["partial","*",1,1],"OAI-SearchBot":["partial","*",1,1],"ClaudeBot":["partial","*",1,1],"Claude-User":["partial","*",1,1],"Claude-SearchBot":["partial","*",1,1],"anthropic-ai":["partial","*",1,1],"PerplexityBot":["partial","*",1,1],"Perplexity-User":["partial","*",1,1],"Google-Extended":["partial","*",1,1],"Applebot-Extended":["partial","*",1,1],"CCBot":["partial","*",1,1],"Bytespider":["partial","*",1,1],"Amazonbot":["partial","*",1,1],"meta-externalagent":["partial","*",1,1],"cohere-ai":["partial","*",1,1],"Diffbot":["partial","*",1,1],"ContextFetchBot":["partial","*",1,1]},"llms":{"status":"missing","http":404,"bytes":99450},"home":{"status":"ok","http":200,"url":"https://manus.im/","html_tokens":59596,"markdown_tokens":194,"pruned_tokens":19}},
{"rank":7885,"domain":"erudit.org","origin":"https://erudit.org","robots":{"status":"ok","http":200,"url":"https://www.erudit.org/robots.txt","bytes":269,"groups":1},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"html","http":200,"bytes":7596},"home":{"status":"ok","http":200,"url":"https://www.erudit.org/en/","html_tokens":18612,"markdown_tokens":3074,"pruned_tokens":68}},
{"rank":7886,"domain":"tapas.io","origin":"https://tapas.io","robots":{"status":"ok","http":200,"url":"https://tapas.io/robots.txt","bytes":2967,"groups":11},"bots":{"GPTBot":["partial","*",13,1],"ChatGPT-User":["partial","*",13,1],"OAI-SearchBot":["partial","*",13,1],"ClaudeBot":["partial","*",13,1],"Claude-User":["partial","*",13,1],"Claude-SearchBot":["partial","*",13,1],"anthropic-ai":["partial","*",13,1],"PerplexityBot":["partial","*",13,1],"Perplexity-User":["partial","*",13,1],"Google-Extended":["partial","*",13,1],"Applebot-Extended":["partial","*",13,1],"CCBot":["partial","*",13,1],"Bytespider":["partial","*",13,1],"Amazonbot":["partial","*",13,1],"meta-externalagent":["partial","*",13,1],"cohere-ai":["partial","*",13,1],"Diffbot":["partial","*",13,1],"ContextFetchBot":["partial","*",13,1]},"llms":{"status":"missing","http":404,"bytes":3392},"home":{"status":"ok","http":200,"url":"https://tapas.io/","html_tokens":39935,"markdown_tokens":18,"pruned_tokens":18}},
{"rank":7887,"domain":"wusa9.com","origin":"https://wusa9.com","robots":{"status":"ok","http":200,"url":"https://www.wusa9.com/robots.txt","bytes":447,"groups":2},"bots":{"GPTBot":["partial","*",10,0],"ChatGPT-User":["partial","*",10,0],"OAI-SearchBot":["partial","*",10,0],"ClaudeBot":["partial","*",10,0],"Claude-User":["partial","*",10,0],"Claude-SearchBot":["partial","*",10,0],"anthropic-ai":["partial","*",10,0],"PerplexityBot":["partial","*",10,0],"Perplexity-User":["partial","*",10,0],"Google-Extended":["partial","*",10,0],"Applebot-Extended":["partial","*",10,0],"CCBot":["partial","*",10,0],"Bytespider":["partial","*",10,0],"Amazonbot":["partial","*",10,0],"meta-externalagent":["partial","*",10,0],"cohere-ai":["partial","*",10,0],"Diffbot":["partial","*",10,0],"ContextFetchBot":["partial","*",10,0]},"llms":{"status":"missing","http":404,"bytes":6850},"home":{"status":"ok","http":200,"url":"https://www.wusa9.com/","html_tokens":99266,"markdown_tokens":1638,"pruned_tokens":168}},
{"rank":7888,"domain":"westword.com","origin":"https://westword.com","robots":{"status":"ok","http":200,"url":"https://www.westword.com/robots.txt","bytes":1406,"groups":2},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["blocked","named",1,0],"OAI-SearchBot":["blocked","named",1,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["blocked","named",1,0],"Claude-SearchBot":["blocked","named",1,0],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["blocked","named",1,0],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",1,1]},"llms":{"status":"missing","http":404,"bytes":119429},"home":{"status":"ok","http":200,"url":"https://www.westword.com/","html_tokens":72097,"markdown_tokens":59,"pruned_tokens":59}},
{"rank":7889,"domain":"lucky88slotmachine.com","origin":"https://lucky88slotmachine.com","robots":{"status":"ok","http":200,"url":"https://lucky88slotmachine.com/robots.txt","bytes":548,"groups":1},"bots":{"GPTBot":["partial","*",19,0],"ChatGPT-User":["partial","*",19,0],"OAI-SearchBot":["partial","*",19,0],"ClaudeBot":["partial","*",19,0],"Claude-User":["partial","*",19,0],"Claude-SearchBot":["partial","*",19,0],"anthropic-ai":["partial","*",19,0],"PerplexityBot":["partial","*",19,0],"Perplexity-User":["partial","*",19,0],"Google-Extended":["partial","*",19,0],"Applebot-Extended":["partial","*",19,0],"CCBot":["partial","*",19,0],"Bytespider":["partial","*",19,0],"Amazonbot":["partial","*",19,0],"meta-externalagent":["partial","*",19,0],"cohere-ai":["partial","*",19,0],"Diffbot":["partial","*",19,0],"ContextFetchBot":["partial","*",19,0]},"llms":{"status":"missing","http":404,"bytes":15},"home":{"status":"http_error","http":404,"url":"https://lucky88slotmachine.com/"}},
{"rank":7890,"domain":"apnic.net","origin":"https://apnic.net","robots":{"status":"ok","http":200,"url":"https://www.apnic.net/robots.txt","bytes":145,"groups":1},"bots":{"GPTBot":["partial","*",2,1],"ChatGPT-User":["partial","*",2,1],"OAI-SearchBot":["partial","*",2,1],"ClaudeBot":["partial","*",2,1],"Claude-User":["partial","*",2,1],"Claude-SearchBot":["partial","*",2,1],"anthropic-ai":["partial","*",2,1],"PerplexityBot":["partial","*",2,1],"Perplexity-User":["partial","*",2,1],"Google-Extended":["partial","*",2,1],"Applebot-Extended":["partial","*",2,1],"CCBot":["partial","*",2,1],"Bytespider":["partial","*",2,1],"Amazonbot":["partial","*",2,1],"meta-externalagent":["partial","*",2,1],"cohere-ai":["partial","*",2,1],"Diffbot":["partial","*",2,1],"ContextFetchBot":["partial","*",2,1]},"llms":{"status":"missing","http":404,"bytes":67874},"home":{"status":"ok","http":200,"url":"https://www.apnic.net/","html_tokens":22026,"markdown_tokens":534,"pruned_tokens":35}},
{"rank":7891,"domain":"ncregister.com","origin":"https://ncregister.com","robots":{"status":"ok","http":200,"url":"https://www.ncregister.com/robots.txt","bytes":82,"groups":1},"bots":{"GPTBot":["allowed","*",0,1],"ChatGPT-User":["allowed","*",0,1],"OAI-SearchBot":["allowed","*",0,1],"ClaudeBot":["allowed","*",0,1],"Claude-User":["allowed","*",0,1],"Claude-SearchBot":["allowed","*",0,1],"anthropic-ai":["allowed","*",0,1],"PerplexityBot":["allowed","*",0,1],"Perplexity-User":["allowed","*",0,1],"Google-Extended":["allowed","*",0,1],"Applebot-Extended":["allowed","*",0,1],"CCBot":["allowed","*",0,1],"Bytespider":["allowed","*",0,1],"Amazonbot":["allowed","*",0,1],"meta-externalagent":["allowed","*",0,1],"cohere-ai":["allowed","*",0,1],"Diffbot":["allowed","*",0,1],"ContextFetchBot":["allowed","*",0,1]},"llms":{"status":"missing","http":404,"bytes":45265},"home":{"status":"ok","http":200,"url":"https://www.ncregister.com/","html_tokens":49136,"markdown_tokens":142,"pruned_tokens":142}},
{"rank":7892,"domain":"cabelas.com","origin":"https://cabelas.com","robots":{"status":"refused","http":403,"url":"https://cabelas.com/robots.txt","bytes":375,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":7893,"domain":"jyxlzs.com","origin":"https://jyxlzs.com","robots":{"status":"html","http":200,"url":"https://jyxlzs.com/robots.txt","bytes":65535,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":7894,"domain":"nbqfbj.com","origin":"https://nbqfbj.com","robots":{"status":"missing","http":404,"url":"https://nbqfbj.com/robots.txt","bytes":0,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":0},"home":{"status":"ok","http":200,"url":"https://nbqfbj.com/","html_tokens":17401,"markdown_tokens":22,"pruned_tokens":22}},
{"rank":7895,"domain":"donlim.com","origin":"https://www.donlim.com","robots":{"status":"missing","http":404,"url":"https://www.donlim.com/robots.txt","bytes":682,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":682},"home":{"status":"ok","http":200,"url":"https://www.donlim.com/","html_tokens":9000,"markdown_tokens":98,"pruned_tokens":98}},
{"rank":7896,"domain":"fashionnetwork.com","origin":"https://fashionnetwork.com","robots":{"status":"ok","http":200,"url":"https://uk.fashionnetwork.com/robots.txt#rl","bytes":3990,"groups":18},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["partial","named",19,1],"OAI-SearchBot":["partial","named",19,1],"ClaudeBot":["partial","named",19,1],"Claude-User":["partial","named",19,1],"Claude-SearchBot":["partial","named",19,1],"anthropic-ai":["partial","named",19,1],"PerplexityBot":["partial","named",19,1],"Perplexity-User":["partial","named",19,1],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["partial","*",56,0],"Diffbot":["partial","*",56,0],"ContextFetchBot":["partial","*",56,0]},"llms":{"status":"missing","http":404,"bytes":14},"home":{"status":"http_error","http":403,"url":"https://fashionnetwork.com/"}},
{"rank":7897,"domain":"knauf.com","origin":"https://knauf.com","robots":{"status":"refused","http":403,"url":"https://knauf.com/robots.txt","bytes":5546,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":7898,"domain":"grainger.com","origin":"https://grainger.com","robots":{"status":"html","http":200,"url":"https://www.grainger.com/robots.txt","bytes":18269,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":7899,"domain":"google.com.ni","origin":"https://google.com.ni","robots":{"status":"ok","http":200,"url":"https://www.google.com.ni/robots.txt","bytes":6624,"groups":4},"bots":{"GPTBot":["partial","*",169,64],"ChatGPT-User":["partial","*",169,64],"OAI-SearchBot":["partial","*",169,64],"ClaudeBot":["partial","*",169,64],"Claude-User":["partial","*",169,64],"Claude-SearchBot":["partial","*",169,64],"anthropic-ai":["partial","*",169,64],"PerplexityBot":["partial","*",169,64],"Perplexity-User":["partial","*",169,64],"Google-Extended":["partial","*",169,64],"Applebot-Extended":["partial","*",169,64],"CCBot":["partial","*",169,64],"Bytespider":["partial","*",169,64],"Amazonbot":["partial","*",169,64],"meta-externalagent":["partial","*",169,64],"cohere-ai":["partial","*",169,64],"Diffbot":["partial","*",169,64],"ContextFetchBot":["partial","*",169,64]},"llms":{"status":"missing","http":404,"bytes":1569},"home":{"status":"ok","http":200,"url":"https://www.google.com/","html_tokens":21310,"markdown_tokens":30,"pruned_tokens":30}},
{"rank":7900,"domain":"hqlf.cn","origin":"https://hqlf.cn","robots":{"status":"ok","http":200,"url":"https://hqlf.cn/robots.txt","bytes":485,"groups":1},"bots":{"GPTBot":["partial","*",21,0],"ChatGPT-User":["partial","*",21,0],"OAI-SearchBot":["partial","*",21,0],"ClaudeBot":["partial","*",21,0],"Claude-User":["partial","*",21,0],"Claude-SearchBot":["partial","*",21,0],"anthropic-ai":["partial","*",21,0],"PerplexityBot":["partial","*",21,0],"Perplexity-User":["partial","*",21,0],"Google-Extended":["partial","*",21,0],"Applebot-Extended":["partial","*",21,0],"CCBot":["partial","*",21,0],"Bytespider":["partial","*",21,0],"Amazonbot":["partial","*",21,0],"meta-externalagent":["partial","*",21,0],"cohere-ai":["partial","*",21,0],"Diffbot":["partial","*",21,0],"ContextFetchBot":["partial","*",21,0]},"llms":{"status":"missing","http":404,"bytes":4721},"home":{"status":"ok","http":200,"url":"https://hqlf.cn/","html_tokens":11219,"markdown_tokens":146,"pruned_tokens":146}},
{"rank":7901,"domain":"bundesliga.com","origin":"https://bundesliga.com","robots":{"status":"ok","http":200,"url":"https://www.bundesliga.com/robots.txt","bytes":11644,"groups":38},"bots":{"GPTBot":["allowed","named",0,1],"ChatGPT-User":["partial","*",3,0],"OAI-SearchBot":["partial","*",3,0],"ClaudeBot":["allowed","named",0,1],"Claude-User":["partial","*",3,0],"Claude-SearchBot":["allowed","named",0,1],"anthropic-ai":["allowed","named",0,1],"PerplexityBot":["allowed","named",0,1],"Perplexity-User":["allowed","named",0,1],"Google-Extended":["allowed","named",0,1],"Applebot-Extended":["allowed","named",0,1],"CCBot":["partial","*",3,0],"Bytespider":["partial","*",3,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["partial","*",3,0],"cohere-ai":["partial","*",3,0],"Diffbot":["partial","*",3,0],"ContextFetchBot":["partial","*",3,0]},"llms":{"status":"missing","http":404,"bytes":360729},"home":{"status":"ok","http":200,"url":"https://www.bundesliga.com/en/bundesliga","html_tokens":281596,"markdown_tokens":107,"pruned_tokens":78}},
{"rank":7902,"domain":"udec.cl","origin":"https://udec.cl","robots":{"status":"ok","http":200,"url":"https://www.udec.cl/robots.txt","bytes":243,"groups":2},"bots":{"GPTBot":["partial","*",1,1],"ChatGPT-User":["partial","*",1,1],"OAI-SearchBot":["partial","*",1,1],"ClaudeBot":["partial","*",1,1],"Claude-User":["partial","*",1,1],"Claude-SearchBot":["partial","*",1,1],"anthropic-ai":["partial","*",1,1],"PerplexityBot":["partial","*",1,1],"Perplexity-User":["partial","*",1,1],"Google-Extended":["partial","*",1,1],"Applebot-Extended":["partial","*",1,1],"CCBot":["partial","*",1,1],"Bytespider":["partial","*",1,1],"Amazonbot":["partial","*",1,1],"meta-externalagent":["partial","*",1,1],"cohere-ai":["partial","*",1,1],"Diffbot":["partial","*",1,1],"ContextFetchBot":["partial","*",1,1]},"llms":{"status":"missing","http":404,"bytes":59275},"home":{"status":"ok","http":200,"url":"https://www.udec.cl/","html_tokens":36615,"markdown_tokens":367,"pruned_tokens":100}},
{"rank":7903,"domain":"xnview.com","origin":"https://xnview.com","robots":{"status":"ok","http":200,"url":"https://www.xnview.com/robots.txt","bytes":790,"groups":14},"bots":{"GPTBot":["allowed","named",0,1],"ChatGPT-User":["allowed","named",0,1],"OAI-SearchBot":["allowed","named",0,1],"ClaudeBot":["allowed","named",0,1],"Claude-User":["partial","*",1,1],"Claude-SearchBot":["allowed","named",0,1],"anthropic-ai":["allowed","named",0,1],"PerplexityBot":["allowed","named",0,1],"Perplexity-User":["allowed","named",0,1],"Google-Extended":["allowed","named",0,1],"Applebot-Extended":["allowed","named",0,1],"CCBot":["allowed","named",0,1],"Bytespider":["partial","*",1,1],"Amazonbot":["partial","*",1,1],"meta-externalagent":["allowed","named",0,1],"cohere-ai":["partial","*",1,1],"Diffbot":["partial","*",1,1],"ContextFetchBot":["partial","*",1,1]},"llms":{"status":"missing","http":404,"bytes":14420},"home":{"status":"ok","http":200,"url":"https://www.xnview.com/en/","html_tokens":10446,"markdown_tokens":449,"pruned_tokens":260}},
{"rank":7904,"domain":"fzjiading.com","origin":"https://www.fzjiading.com","robots":{"status":"html","http":200,"url":"https://www.fzjiading.com/robots.txt","bytes":11722,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":7905,"domain":"home.saxo","origin":"https://home.saxo","robots":{"status":"ok","http":200,"url":"https://www.home.saxo/robots.txt","bytes":1014,"groups":7},"bots":{"GPTBot":["partial","*",7,16],"ChatGPT-User":["partial","*",7,16],"OAI-SearchBot":["partial","*",7,16],"ClaudeBot":["partial","*",7,16],"Claude-User":["partial","*",7,16],"Claude-SearchBot":["partial","*",7,16],"anthropic-ai":["partial","*",7,16],"PerplexityBot":["partial","*",7,16],"Perplexity-User":["partial","*",7,16],"Google-Extended":["partial","*",7,16],"Applebot-Extended":["partial","*",7,16],"CCBot":["partial","*",7,16],"Bytespider":["partial","*",7,16],"Amazonbot":["partial","*",7,16],"meta-externalagent":["partial","*",7,16],"cohere-ai":["partial","*",7,16],"Diffbot":["partial","*",7,16],"ContextFetchBot":["partial","*",7,16]},"llms":{"status":"present","http":200,"bytes":11380},"home":{"status":"ok","http":200,"url":"https://www.home.saxo/","html_tokens":70475,"markdown_tokens":122,"pruned_tokens":49}},
{"rank":7906,"domain":"askgamblers.com","origin":"https://askgamblers.com","robots":{"status":"ok","http":200,"url":"https://www.askgamblers.com/robots.txt","bytes":2433,"groups":1},"bots":{"GPTBot":["partial","*",103,0],"ChatGPT-User":["partial","*",103,0],"OAI-SearchBot":["partial","*",103,0],"ClaudeBot":["partial","*",103,0],"Claude-User":["partial","*",103,0],"Claude-SearchBot":["partial","*",103,0],"anthropic-ai":["partial","*",103,0],"PerplexityBot":["partial","*",103,0],"Perplexity-User":["partial","*",103,0],"Google-Extended":["partial","*",103,0],"Applebot-Extended":["partial","*",103,0],"CCBot":["partial","*",103,0],"Bytespider":["partial","*",103,0],"Amazonbot":["partial","*",103,0],"meta-externalagent":["partial","*",103,0],"cohere-ai":["partial","*",103,0],"Diffbot":["partial","*",103,0],"ContextFetchBot":["partial","*",103,0]},"llms":{"status":"http_error","http":403,"bytes":5572},"home":{"status":"http_error","http":403,"url":"https://www.askgamblers.com/"}},
{"rank":7907,"domain":"savethechildren.net","origin":"https://savethechildren.net","robots":{"status":"ok","http":200,"url":"https://www.savethechildren.net/robots.txt","bytes":1806,"groups":1},"bots":{"GPTBot":["partial","*",27,18],"ChatGPT-User":["partial","*",27,18],"OAI-SearchBot":["partial","*",27,18],"ClaudeBot":["partial","*",27,18],"Claude-User":["partial","*",27,18],"Claude-SearchBot":["partial","*",27,18],"anthropic-ai":["partial","*",27,18],"PerplexityBot":["partial","*",27,18],"Perplexity-User":["partial","*",27,18],"Google-Extended":["partial","*",27,18],"Applebot-Extended":["partial","*",27,18],"CCBot":["partial","*",27,18],"Bytespider":["partial","*",27,18],"Amazonbot":["partial","*",27,18],"meta-externalagent":["partial","*",27,18],"cohere-ai":["partial","*",27,18],"Diffbot":["partial","*",27,18],"ContextFetchBot":["partial","*",27,18]},"llms":{"status":"http_error","http":403,"bytes":17},"home":{"status":"http_error","http":403,"url":"https://www.savethechildren.net/"}},
{"rank":7908,"domain":"google.li","origin":"https://google.li","robots":{"status":"ok","http":200,"url":"https://www.google.li/robots.txt","bytes":6624,"groups":4},"bots":{"GPTBot":["partial","*",169,64],"ChatGPT-User":["partial","*",169,64],"OAI-SearchBot":["partial","*",169,64],"ClaudeBot":["partial","*",169,64],"Claude-User":["partial","*",169,64],"Claude-SearchBot":["partial","*",169,64],"anthropic-ai":["partial","*",169,64],"PerplexityBot":["partial","*",169,64],"Perplexity-User":["partial","*",169,64],"Google-Extended":["partial","*",169,64],"Applebot-Extended":["partial","*",169,64],"CCBot":["partial","*",169,64],"Bytespider":["partial","*",169,64],"Amazonbot":["partial","*",169,64],"meta-externalagent":["partial","*",169,64],"cohere-ai":["partial","*",169,64],"Diffbot":["partial","*",169,64],"ContextFetchBot":["partial","*",169,64]},"llms":{"status":"missing","http":404,"bytes":1569},"home":{"status":"ok","http":200,"url":"https://www.google.com/","html_tokens":21449,"markdown_tokens":30,"pruned_tokens":30}},
{"rank":7909,"domain":"earthcam.com","origin":"https://earthcam.com","robots":{"status":"ok","http":200,"url":"https://earthcam.com/robots.txt","bytes":61,"groups":1},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"missing","http":404,"bytes":42098},"home":{"status":"ok","http":200,"url":"https://www.earthcam.com/","html_tokens":23025,"markdown_tokens":11,"pruned_tokens":11}},
{"rank":7910,"domain":"donaldjtrump.com","origin":"https://donaldjtrump.com","robots":{"status":"missing","http":404,"url":"https://donaldjtrump.com/robots.txt","bytes":196,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":196},"home":{"status":"ok","http":200,"url":"https://donaldjtrump.com/","html_tokens":721,"markdown_tokens":7,"pruned_tokens":7}},
{"rank":7911,"domain":"androidheadlines.com","origin":"https://androidheadlines.com","robots":{"status":"ok","http":200,"url":"https://www.androidheadlines.com/robots.txt","bytes":2309,"groups":15},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["blocked","named",1,0],"OAI-SearchBot":["blocked","named",1,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["allowed","named",0,1],"Claude-SearchBot":["allowed","named",0,1],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["partial","*",15,1],"Google-Extended":["partial","*",15,1],"Applebot-Extended":["partial","*",15,1],"CCBot":["blocked","named",1,0],"Bytespider":["partial","*",15,1],"Amazonbot":["partial","*",15,1],"meta-externalagent":["partial","*",15,1],"cohere-ai":["partial","*",15,1],"Diffbot":["partial","*",15,1],"ContextFetchBot":["partial","*",15,1]},"llms":{"status":"missing","http":404,"bytes":546681},"home":{"status":"http_error","http":403,"url":"https://www.androidheadlines.com"}},
{"rank":7912,"domain":"sweetwater.com","origin":"https://sweetwater.com","robots":{"status":"ok","http":200,"url":"https://www.sweetwater.com/robots.txt","bytes":2365,"groups":2},"bots":{"GPTBot":["partial","*",22,9],"ChatGPT-User":["partial","*",22,9],"OAI-SearchBot":["partial","*",22,9],"ClaudeBot":["partial","*",22,9],"Claude-User":["partial","*",22,9],"Claude-SearchBot":["partial","*",22,9],"anthropic-ai":["partial","*",22,9],"PerplexityBot":["partial","*",22,9],"Perplexity-User":["partial","*",22,9],"Google-Extended":["partial","*",22,9],"Applebot-Extended":["partial","*",22,9],"CCBot":["partial","*",22,9],"Bytespider":["partial","*",22,9],"Amazonbot":["partial","*",22,9],"meta-externalagent":["partial","*",22,9],"cohere-ai":["partial","*",22,9],"Diffbot":["partial","*",22,9],"ContextFetchBot":["partial","*",22,9]},"llms":{"status":"http_error","http":403,"bytes":5691},"home":{"status":"http_error","http":403,"url":"https://sweetwater.com/"}},
{"rank":7913,"domain":"wdl.org","origin":"https://wdl.org","robots":{"status":"refused","http":403,"url":"https://www.loc.gov/collections/world-digital-library/about-this-collection/","bytes":5750,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":7914,"domain":"google.nr","origin":"https://google.nr","robots":{"status":"ok","http":200,"url":"https://www.google.nr/robots.txt","bytes":6624,"groups":4},"bots":{"GPTBot":["partial","*",169,64],"ChatGPT-User":["partial","*",169,64],"OAI-SearchBot":["partial","*",169,64],"ClaudeBot":["partial","*",169,64],"Claude-User":["partial","*",169,64],"Claude-SearchBot":["partial","*",169,64],"anthropic-ai":["partial","*",169,64],"PerplexityBot":["partial","*",169,64],"Perplexity-User":["partial","*",169,64],"Google-Extended":["partial","*",169,64],"Applebot-Extended":["partial","*",169,64],"CCBot":["partial","*",169,64],"Bytespider":["partial","*",169,64],"Amazonbot":["partial","*",169,64],"meta-externalagent":["partial","*",169,64],"cohere-ai":["partial","*",169,64],"Diffbot":["partial","*",169,64],"ContextFetchBot":["partial","*",169,64]},"llms":{"status":"missing","http":404,"bytes":1569},"home":{"status":"ok","http":200,"url":"https://www.google.com/","html_tokens":21325,"markdown_tokens":30,"pruned_tokens":30}},
{"rank":7915,"domain":"cargoplus.cn","origin":"https://cargoplus.cn","robots":{"status":"missing","http":404,"url":"https://cargoplus.cn/robots.txt","bytes":4730,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":4726},"home":{"status":"ok","http":200,"url":"https://cargoplus.cn/","html_tokens":2966,"markdown_tokens":25,"pruned_tokens":25}},
{"rank":7916,"domain":"gibson.com","origin":"https://gibson.com","robots":{"status":"ok","http":200,"url":"https://www.gibson.com/robots.txt","bytes":7021,"groups":6},"bots":{"GPTBot":["partial","*",48,0],"ChatGPT-User":["partial","*",48,0],"OAI-SearchBot":["partial","*",48,0],"ClaudeBot":["partial","*",48,0],"Claude-User":["partial","*",48,0],"Claude-SearchBot":["partial","*",48,0],"anthropic-ai":["partial","*",48,0],"PerplexityBot":["partial","*",48,0],"Perplexity-User":["partial","*",48,0],"Google-Extended":["partial","*",48,0],"Applebot-Extended":["partial","*",48,0],"CCBot":["partial","*",48,0],"Bytespider":["partial","*",48,0],"Amazonbot":["partial","*",48,0],"meta-externalagent":["partial","*",48,0],"cohere-ai":["partial","*",48,0],"Diffbot":["partial","*",48,0],"ContextFetchBot":["partial","*",48,0]},"llms":{"status":"present","http":200,"bytes":5775},"home":{"status":"ok","http":200,"url":"https://www.gibson.com/","html_tokens":780066,"markdown_tokens":38,"pruned_tokens":26}},
{"rank":7917,"domain":"dreamworks.com","origin":"https://dreamworks.com","robots":{"status":"ok","http":200,"url":"https://www.dreamworks.com/robots.txt","bytes":411,"groups":1},"bots":{"GPTBot":["partial","*",5,0],"ChatGPT-User":["partial","*",5,0],"OAI-SearchBot":["partial","*",5,0],"ClaudeBot":["partial","*",5,0],"Claude-User":["partial","*",5,0],"Claude-SearchBot":["partial","*",5,0],"anthropic-ai":["partial","*",5,0],"PerplexityBot":["partial","*",5,0],"Perplexity-User":["partial","*",5,0],"Google-Extended":["partial","*",5,0],"Applebot-Extended":["partial","*",5,0],"CCBot":["partial","*",5,0],"Bytespider":["partial","*",5,0],"Amazonbot":["partial","*",5,0],"meta-externalagent":["partial","*",5,0],"cohere-ai":["partial","*",5,0],"Diffbot":["partial","*",5,0],"ContextFetchBot":["partial","*",5,0]},"llms":{"status":"http_error","http":403,"bytes":4552},"home":{"status":"ok","http":200,"url":"https://www.dreamworks.com/","html_tokens":22075,"markdown_tokens":56,"pruned_tokens":56}},
{"rank":7918,"domain":"pole-emploi.fr","origin":"https://pole-emploi.fr","robots":{"status":"ok","http":200,"url":"https://www.francetravail.fr/robots.txt","bytes":396,"groups":1},"bots":{"GPTBot":["partial","*",13,0],"ChatGPT-User":["partial","*",13,0],"OAI-SearchBot":["partial","*",13,0],"ClaudeBot":["partial","*",13,0],"Claude-User":["partial","*",13,0],"Claude-SearchBot":["partial","*",13,0],"anthropic-ai":["partial","*",13,0],"PerplexityBot":["partial","*",13,0],"Perplexity-User":["partial","*",13,0],"Google-Extended":["partial","*",13,0],"Applebot-Extended":["partial","*",13,0],"CCBot":["partial","*",13,0],"Bytespider":["partial","*",13,0],"Amazonbot":["partial","*",13,0],"meta-externalagent":["partial","*",13,0],"cohere-ai":["partial","*",13,0],"Diffbot":["partial","*",13,0],"ContextFetchBot":["partial","*",13,0]},"llms":{"status":"missing","http":404,"bytes":2870},"home":{"status":"ok","http":200,"url":"https://www.francetravail.fr/accueil/","html_tokens":5817,"markdown_tokens":0,"pruned_tokens":0}},
{"rank":7919,"domain":"szgjp.com","origin":"https://szgjp.com","robots":{"status":"ok","http":200,"url":"https://szgjp.com/robots.txt","bytes":67,"groups":1},"bots":{"GPTBot":["allowed","*",0,1],"ChatGPT-User":["allowed","*",0,1],"OAI-SearchBot":["allowed","*",0,1],"ClaudeBot":["allowed","*",0,1],"Claude-User":["allowed","*",0,1],"Claude-SearchBot":["allowed","*",0,1],"anthropic-ai":["allowed","*",0,1],"PerplexityBot":["allowed","*",0,1],"Perplexity-User":["allowed","*",0,1],"Google-Extended":["allowed","*",0,1],"Applebot-Extended":["allowed","*",0,1],"CCBot":["allowed","*",0,1],"Bytespider":["allowed","*",0,1],"Amazonbot":["allowed","*",0,1],"meta-externalagent":["allowed","*",0,1],"cohere-ai":["allowed","*",0,1],"Diffbot":["allowed","*",0,1],"ContextFetchBot":["allowed","*",0,1]},"llms":{"status":"missing","http":404,"bytes":0},"home":{"status":"ok","http":200,"url":"https://szgjp.com/","html_tokens":20994,"markdown_tokens":20,"pruned_tokens":20}},
{"rank":7920,"domain":"topelec.com.cn","origin":"https://topelec.com.cn","robots":{"status":"ok","http":200,"url":"https://www.topelec.com.cn/robots.txt","bytes":292,"groups":1},"bots":{"GPTBot":["partial","*",9,1],"ChatGPT-User":["partial","*",9,1],"OAI-SearchBot":["partial","*",9,1],"ClaudeBot":["partial","*",9,1],"Claude-User":["partial","*",9,1],"Claude-SearchBot":["partial","*",9,1],"anthropic-ai":["partial","*",9,1],"PerplexityBot":["partial","*",9,1],"Perplexity-User":["partial","*",9,1],"Google-Extended":["partial","*",9,1],"Applebot-Extended":["partial","*",9,1],"CCBot":["partial","*",9,1],"Bytespider":["partial","*",9,1],"Amazonbot":["partial","*",9,1],"meta-externalagent":["partial","*",9,1],"cohere-ai":["partial","*",9,1],"Diffbot":["partial","*",9,1],"ContextFetchBot":["partial","*",9,1]},"llms":{"status":"missing","http":404,"bytes":24309},"home":{"status":"ok","http":200,"url":"https://www.topelec.com.cn/en","html_tokens":35124,"markdown_tokens":146,"pruned_tokens":146}},
{"rank":7921,"domain":"lrv.lt","origin":"https://lrv.lt","robots":{"status":"refused","http":403,"url":"https://lrv.lt/robots.txt","bytes":5509,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":7922,"domain":"wnba.com","origin":"https://wnba.com","robots":{"status":"ok","http":200,"url":"https://www.wnba.com/robots.txt","bytes":2460,"groups":14},"bots":{"GPTBot":["partial","named",1,7],"ChatGPT-User":["blocked","named",1,0],"OAI-SearchBot":["allowed","named",0,1],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",19,3],"Claude-SearchBot":["partial","*",19,3],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["partial","*",19,3],"Google-Extended":["partial","named",1,7],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["partial","*",19,3],"cohere-ai":["partial","*",19,3],"Diffbot":["partial","*",19,3],"ContextFetchBot":["partial","*",19,3]},"llms":{"status":"http_error","http":403,"bytes":263},"home":{"status":"ok","http":200,"url":"https://www.wnba.com/","html_tokens":67230,"markdown_tokens":89,"pruned_tokens":83}},
{"rank":7923,"domain":"sznews.com","origin":"https://www.sznews.com","robots":{"status":"missing","http":404,"url":"https://www.sznews.com/robots.txt","bytes":7543,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":7543},"home":{"status":"ok","http":200,"url":"https://www.sznews.com/","html_tokens":50909,"markdown_tokens":43,"pruned_tokens":43}},
{"rank":7924,"domain":"ofweek.com","origin":"https://www.ofweek.com","robots":{"status":"ok","http":200,"url":"https://www.ofweek.com/robots.txt","bytes":342,"groups":3},"bots":{"GPTBot":["partial","*",14,1],"ChatGPT-User":["partial","*",14,1],"OAI-SearchBot":["partial","*",14,1],"ClaudeBot":["partial","*",14,1],"Claude-User":["partial","*",14,1],"Claude-SearchBot":["partial","*",14,1],"anthropic-ai":["partial","*",14,1],"PerplexityBot":["partial","*",14,1],"Perplexity-User":["partial","*",14,1],"Google-Extended":["partial","*",14,1],"Applebot-Extended":["partial","*",14,1],"CCBot":["partial","*",14,1],"Bytespider":["partial","*",14,1],"Amazonbot":["partial","*",14,1],"meta-externalagent":["partial","*",14,1],"cohere-ai":["partial","*",14,1],"Diffbot":["partial","*",14,1],"ContextFetchBot":["partial","*",14,1]},"llms":{"status":"missing","http":404,"bytes":1549},"home":{"status":"ok","http":200,"url":"https://www.ofweek.com/","html_tokens":128474,"markdown_tokens":10,"pruned_tokens":10}},
{"rank":7925,"domain":"excelsecu.com.cn","origin":"https://excelsecu.com.cn","robots":{"status":"ok","http":200,"url":"https://excelsecu.com.cn/robots.txt","bytes":3611,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"html","http":200,"bytes":3611},"home":{"status":"ok","http":200,"url":"https://excelsecu.com.cn/","html_tokens":5944,"markdown_tokens":25,"pruned_tokens":25}},
{"rank":7926,"domain":"nidirect.gov.uk","origin":"https://nidirect.gov.uk","robots":{"status":"ok","http":200,"url":"https://www.nidirect.gov.uk/robots.txt","bytes":6990,"groups":11},"bots":{"GPTBot":["partial","*",49,18],"ChatGPT-User":["partial","*",49,18],"OAI-SearchBot":["partial","*",49,18],"ClaudeBot":["partial","*",49,18],"Claude-User":["partial","*",49,18],"Claude-SearchBot":["partial","*",49,18],"anthropic-ai":["partial","*",49,18],"PerplexityBot":["partial","*",49,18],"Perplexity-User":["partial","*",49,18],"Google-Extended":["partial","*",49,18],"Applebot-Extended":["partial","*",49,18],"CCBot":["partial","*",49,18],"Bytespider":["partial","*",49,18],"Amazonbot":["partial","*",49,18],"meta-externalagent":["partial","*",49,18],"cohere-ai":["partial","*",49,18],"Diffbot":["partial","*",49,18],"ContextFetchBot":["partial","*",49,18]},"llms":{"status":"missing","http":404,"bytes":193},"home":{"status":"ok","http":200,"url":"https://www.nidirect.gov.uk/","html_tokens":24325,"markdown_tokens":1187,"pruned_tokens":365}},
{"rank":7927,"domain":"yogajournal.com","origin":"https://yogajournal.com","robots":{"status":"ok","http":200,"url":"https://www.yogajournal.com/robots.txt","bytes":1077,"groups":2},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["partial","*",4,1],"OAI-SearchBot":["partial","*",4,1],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",4,1],"Claude-SearchBot":["partial","*",4,1],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["partial","*",4,1],"Perplexity-User":["partial","*",4,1],"Google-Extended":["partial","*",4,1],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",4,1]},"llms":{"status":"missing","http":404,"bytes":246353},"home":{"status":"ok","http":200,"url":"https://www.yogajournal.com/","html_tokens":94759,"markdown_tokens":73,"pruned_tokens":73}},
{"rank":7928,"domain":"slb.com","origin":"https://slb.com","robots":{"status":"ok","http":200,"url":"https://www.slb.com/robots.txt","bytes":240,"groups":1},"bots":{"GPTBot":["partial","*",4,2],"ChatGPT-User":["partial","*",4,2],"OAI-SearchBot":["partial","*",4,2],"ClaudeBot":["partial","*",4,2],"Claude-User":["partial","*",4,2],"Claude-SearchBot":["partial","*",4,2],"anthropic-ai":["partial","*",4,2],"PerplexityBot":["partial","*",4,2],"Perplexity-User":["partial","*",4,2],"Google-Extended":["partial","*",4,2],"Applebot-Extended":["partial","*",4,2],"CCBot":["partial","*",4,2],"Bytespider":["partial","*",4,2],"Amazonbot":["partial","*",4,2],"meta-externalagent":["partial","*",4,2],"cohere-ai":["partial","*",4,2],"Diffbot":["partial","*",4,2],"ContextFetchBot":["partial","*",4,2]},"llms":{"status":"missing","http":404,"bytes":1245},"home":{"status":"ok","http":200,"url":"https://www.slb.com/","html_tokens":234882,"markdown_tokens":719,"pruned_tokens":80}},
{"rank":7929,"domain":"osnews.com","origin":"https://osnews.com","robots":{"status":"ok","http":200,"url":"https://osnews.com/robots.txt","bytes":248,"groups":3},"bots":{"GPTBot":["partial","*",8,0],"ChatGPT-User":["partial","*",8,0],"OAI-SearchBot":["partial","*",8,0],"ClaudeBot":["partial","*",8,0],"Claude-User":["partial","*",8,0],"Claude-SearchBot":["partial","*",8,0],"anthropic-ai":["partial","*",8,0],"PerplexityBot":["partial","*",8,0],"Perplexity-User":["partial","*",8,0],"Google-Extended":["partial","*",8,0],"Applebot-Extended":["partial","*",8,0],"CCBot":["partial","*",8,0],"Bytespider":["partial","*",8,0],"Amazonbot":["partial","*",8,0],"meta-externalagent":["partial","*",8,0],"cohere-ai":["partial","*",8,0],"Diffbot":["partial","*",8,0],"ContextFetchBot":["partial","*",8,0]},"llms":{"status":"missing","http":404,"bytes":106877},"home":{"status":"ok","http":200,"url":"https://www.osnews.com/","html_tokens":50855,"markdown_tokens":10500,"pruned_tokens":77}},
{"rank":7930,"domain":"chick-fil-a.com","origin":"https://www.chick-fil-a.com","robots":{"status":"ok","http":200,"url":"https://www.chick-fil-a.com/robots.txt","bytes":142,"groups":1},"bots":{"GPTBot":["partial","*",2,1],"ChatGPT-User":["partial","*",2,1],"OAI-SearchBot":["partial","*",2,1],"ClaudeBot":["partial","*",2,1],"Claude-User":["partial","*",2,1],"Claude-SearchBot":["partial","*",2,1],"anthropic-ai":["partial","*",2,1],"PerplexityBot":["partial","*",2,1],"Perplexity-User":["partial","*",2,1],"Google-Extended":["partial","*",2,1],"Applebot-Extended":["partial","*",2,1],"CCBot":["partial","*",2,1],"Bytespider":["partial","*",2,1],"Amazonbot":["partial","*",2,1],"meta-externalagent":["partial","*",2,1],"cohere-ai":["partial","*",2,1],"Diffbot":["partial","*",2,1],"ContextFetchBot":["partial","*",2,1]},"llms":{"status":"missing","http":404,"bytes":225152},"home":{"status":"ok","http":200,"url":"https://www.chick-fil-a.com/","html_tokens":120639,"markdown_tokens":372,"pruned_tokens":264}},
{"rank":7931,"domain":"citizenlab.ca","origin":"https://citizenlab.ca","robots":{"status":"ok","http":200,"url":"https://citizenlab.ca/robots.txt","bytes":1248,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":72659},"home":{"status":"ok","http":200,"url":"https://citizenlab.ca/","html_tokens":32337,"markdown_tokens":199,"pruned_tokens":78}},
{"rank":7932,"domain":"nm.ru","origin":"http://nm.ru","robots":{"status":"missing","http":404,"url":"http://nm.ru/robots.txt","bytes":196,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":196},"home":{"status":"ok","http":200,"url":"http://nm.ru/","html_tokens":15323,"markdown_tokens":138,"pruned_tokens":114}},
{"rank":7933,"domain":"gdleibaoji.com","origin":"http://gdleibaoji.com","robots":{"status":"ok","http":200,"url":"http://www.gdleibaoji.com/robots.txt","bytes":188,"groups":1},"bots":{"GPTBot":["partial","*",8,1],"ChatGPT-User":["partial","*",8,1],"OAI-SearchBot":["partial","*",8,1],"ClaudeBot":["partial","*",8,1],"Claude-User":["partial","*",8,1],"Claude-SearchBot":["partial","*",8,1],"anthropic-ai":["partial","*",8,1],"PerplexityBot":["partial","*",8,1],"Perplexity-User":["partial","*",8,1],"Google-Extended":["partial","*",8,1],"Applebot-Extended":["partial","*",8,1],"CCBot":["partial","*",8,1],"Bytespider":["partial","*",8,1],"Amazonbot":["partial","*",8,1],"meta-externalagent":["partial","*",8,1],"cohere-ai":["partial","*",8,1],"Diffbot":["partial","*",8,1],"ContextFetchBot":["partial","*",8,1]},"llms":{"status":"missing","http":404,"bytes":5079},"home":{"status":"ok","http":200,"url":"http://www.gdleibaoji.com/","html_tokens":13050,"markdown_tokens":254,"pruned_tokens":94}},
{"rank":7934,"domain":"bignewsnetwork.com","origin":"https://bignewsnetwork.com","robots":{"status":"ok","http":200,"url":"https://www.bignewsnetwork.com/robots.txt","bytes":2843,"groups":26},"bots":{"GPTBot":["partial","*",2,1],"ChatGPT-User":["partial","*",2,1],"OAI-SearchBot":["partial","*",2,1],"ClaudeBot":["partial","*",2,1],"Claude-User":["partial","*",2,1],"Claude-SearchBot":["partial","*",2,1],"anthropic-ai":["partial","*",2,1],"PerplexityBot":["partial","*",2,1],"Perplexity-User":["partial","*",2,1],"Google-Extended":["partial","*",2,1],"Applebot-Extended":["partial","*",2,1],"CCBot":["partial","*",2,1],"Bytespider":["partial","*",2,1],"Amazonbot":["partial","*",2,1],"meta-externalagent":["partial","*",2,1],"cohere-ai":["partial","*",2,1],"Diffbot":["partial","*",2,1],"ContextFetchBot":["partial","*",2,1]},"llms":{"status":"missing","http":404,"bytes":49076},"home":{"status":"http_error","http":403,"url":"https://bignewsnetwork.com/"}},
{"rank":7935,"domain":"delawareonline.com","origin":"https://delawareonline.com","robots":{"status":"ok","http":200,"url":"https://www.delawareonline.com/robots.txt","bytes":12683,"groups":288},"bots":{"GPTBot":["partial","named",1,4],"ChatGPT-User":["partial","named",1,4],"OAI-SearchBot":["partial","named",1,4],"ClaudeBot":["blocked","named",1,0],"Claude-User":["blocked","named",1,0],"Claude-SearchBot":["blocked","named",1,0],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["partial","named",1,4],"Perplexity-User":["blocked","named",1,0],"Google-Extended":["partial","named",1,4],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",2,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",32,0]},"llms":{"status":"http_error","http":402,"bytes":254},"home":{"status":"http_error","http":402,"url":"https://www.delawareonline.com/"}},
{"rank":7936,"domain":"cummins.com","origin":"https://cummins.com","robots":{"status":"ok","http":200,"url":"https://www.cummins.com/robots.txt","bytes":2615,"groups":1},"bots":{"GPTBot":["partial","*",54,18],"ChatGPT-User":["partial","*",54,18],"OAI-SearchBot":["partial","*",54,18],"ClaudeBot":["partial","*",54,18],"Claude-User":["partial","*",54,18],"Claude-SearchBot":["partial","*",54,18],"anthropic-ai":["partial","*",54,18],"PerplexityBot":["partial","*",54,18],"Perplexity-User":["partial","*",54,18],"Google-Extended":["partial","*",54,18],"Applebot-Extended":["partial","*",54,18],"CCBot":["partial","*",54,18],"Bytespider":["partial","*",54,18],"Amazonbot":["partial","*",54,18],"meta-externalagent":["partial","*",54,18],"cohere-ai":["partial","*",54,18],"Diffbot":["partial","*",54,18],"ContextFetchBot":["partial","*",54,18]},"llms":{"status":"missing","http":404,"bytes":550},"home":{"status":"http_error","http":403,"url":"https://www.cummins.com/en-na"}},
{"rank":7937,"domain":"ask.fm","origin":"https://ask.fm","robots":{"status":"unreachable","http":null,"url":"https://ask.fm/robots.txt","bytes":0,"groups":0,"error":"cannot resolve ask.fm"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":7938,"domain":"mancity.com","origin":"https://mancity.com","robots":{"status":"ok","http":200,"url":"https://www.mancity.com/robots.txt","bytes":643,"groups":1},"bots":{"GPTBot":["partial","*",17,0],"ChatGPT-User":["partial","*",17,0],"OAI-SearchBot":["partial","*",17,0],"ClaudeBot":["partial","*",17,0],"Claude-User":["partial","*",17,0],"Claude-SearchBot":["partial","*",17,0],"anthropic-ai":["partial","*",17,0],"PerplexityBot":["partial","*",17,0],"Perplexity-User":["partial","*",17,0],"Google-Extended":["partial","*",17,0],"Applebot-Extended":["partial","*",17,0],"CCBot":["partial","*",17,0],"Bytespider":["partial","*",17,0],"Amazonbot":["partial","*",17,0],"meta-externalagent":["partial","*",17,0],"cohere-ai":["partial","*",17,0],"Diffbot":["partial","*",17,0],"ContextFetchBot":["partial","*",17,0]},"llms":{"status":"http_error","http":403,"bytes":910396},"home":{"status":"http_error","http":403,"url":"https://mancity.com/"}},
{"rank":7939,"domain":"khan.co.kr","origin":"https://khan.co.kr","robots":{"status":"ok","http":200,"url":"https://www.khan.co.kr/robots.txt","bytes":2958,"groups":4},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["partial","named",7,0],"OAI-SearchBot":["partial","named",7,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",8,0],"Claude-SearchBot":["partial","named",7,0],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["partial","named",7,0],"Perplexity-User":["partial","named",7,0],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["partial","*",8,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",8,0]},"llms":{"status":"missing","http":404,"bytes":95052},"home":{"status":"ok","http":200,"url":"https://www.khan.co.kr/","html_tokens":72123,"markdown_tokens":2587,"pruned_tokens":172}},
{"rank":7940,"domain":"agid.gov.it","origin":"https://agid.gov.it","robots":{"status":"refused","http":403,"url":"https://www.agid.gov.it//robots.txt","bytes":919,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":7941,"domain":"luofutang.com","origin":"http://luofutang.com","robots":{"status":"missing","http":404,"url":"http://luofutang.com/robots.txt","bytes":315,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":315},"home":{"status":"http_error","http":404,"url":"http://luofutang.com/"}},
{"rank":7942,"domain":"tvline.com","origin":"https://tvline.com","robots":{"status":"ok","http":200,"url":"https://tvline.com/robots.txt","bytes":303,"groups":2},"bots":{"GPTBot":["partial","*",4,0],"ChatGPT-User":["partial","*",4,0],"OAI-SearchBot":["partial","*",4,0],"ClaudeBot":["partial","*",4,0],"Claude-User":["partial","*",4,0],"Claude-SearchBot":["partial","*",4,0],"anthropic-ai":["partial","*",4,0],"PerplexityBot":["partial","*",4,0],"Perplexity-User":["partial","*",4,0],"Google-Extended":["partial","*",4,0],"Applebot-Extended":["partial","*",4,0],"CCBot":["partial","*",4,0],"Bytespider":["partial","*",4,0],"Amazonbot":["partial","*",4,0],"meta-externalagent":["partial","*",4,0],"cohere-ai":["partial","*",4,0],"Diffbot":["partial","*",4,0],"ContextFetchBot":["partial","*",4,0]},"llms":{"status":"html","http":200,"bytes":107561},"home":{"status":"ok","http":200,"url":"https://www.tvline.com/","html_tokens":26886,"markdown_tokens":1004,"pruned_tokens":175}},
{"rank":7943,"domain":"chroniclelive.co.uk","origin":"https://chroniclelive.co.uk","robots":{"status":"ok","http":200,"url":"https://www.chroniclelive.co.uk/robots.txt","bytes":937,"groups":13},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["partial","*",9,0],"OAI-SearchBot":["blocked","named",1,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",9,0],"Claude-SearchBot":["partial","*",9,0],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["partial","*",9,0],"Google-Extended":["partial","*",9,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["partial","*",9,0],"Amazonbot":["partial","*",9,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["partial","*",9,0],"Diffbot":["partial","*",9,0],"ContextFetchBot":["partial","*",9,0]},"llms":{"status":"missing","http":404,"bytes":807482},"home":{"status":"ok","http":200,"url":"https://www.chroniclelive.co.uk/","html_tokens":221235,"markdown_tokens":153,"pruned_tokens":153}},
{"rank":7944,"domain":"universalorlando.com","origin":"https://universalorlando.com","robots":{"status":"ok","http":200,"url":"https://www.universalorlando.com/robots.txt","bytes":222,"groups":1},"bots":{"GPTBot":["partial","*",4,0],"ChatGPT-User":["partial","*",4,0],"OAI-SearchBot":["partial","*",4,0],"ClaudeBot":["partial","*",4,0],"Claude-User":["partial","*",4,0],"Claude-SearchBot":["partial","*",4,0],"anthropic-ai":["partial","*",4,0],"PerplexityBot":["partial","*",4,0],"Perplexity-User":["partial","*",4,0],"Google-Extended":["partial","*",4,0],"Applebot-Extended":["partial","*",4,0],"CCBot":["partial","*",4,0],"Bytespider":["partial","*",4,0],"Amazonbot":["partial","*",4,0],"meta-externalagent":["partial","*",4,0],"cohere-ai":["partial","*",4,0],"Diffbot":["partial","*",4,0],"ContextFetchBot":["partial","*",4,0]},"llms":{"status":"html","http":200,"bytes":10892},"home":{"status":"ok","http":200,"url":"https://www.universalorlando.com/web/en/us","html_tokens":2723,"markdown_tokens":0,"pruned_tokens":0}},
{"rank":7945,"domain":"google.tt","origin":"https://google.tt","robots":{"status":"ok","http":200,"url":"https://www.google.tt/robots.txt","bytes":6624,"groups":4},"bots":{"GPTBot":["partial","*",169,64],"ChatGPT-User":["partial","*",169,64],"OAI-SearchBot":["partial","*",169,64],"ClaudeBot":["partial","*",169,64],"Claude-User":["partial","*",169,64],"Claude-SearchBot":["partial","*",169,64],"anthropic-ai":["partial","*",169,64],"PerplexityBot":["partial","*",169,64],"Perplexity-User":["partial","*",169,64],"Google-Extended":["partial","*",169,64],"Applebot-Extended":["partial","*",169,64],"CCBot":["partial","*",169,64],"Bytespider":["partial","*",169,64],"Amazonbot":["partial","*",169,64],"meta-externalagent":["partial","*",169,64],"cohere-ai":["partial","*",169,64],"Diffbot":["partial","*",169,64],"ContextFetchBot":["partial","*",169,64]},"llms":{"status":"missing","http":404,"bytes":1569},"home":{"status":"ok","http":200,"url":"https://www.google.com/","html_tokens":21340,"markdown_tokens":30,"pruned_tokens":30}},
{"rank":7946,"domain":"url.cn","origin":"https://url.cn","robots":{"status":"ok","http":200,"url":"https://url.cn/robots.txt","bytes":26,"groups":1},"bots":{"GPTBot":["blocked","*",1,0],"ChatGPT-User":["blocked","*",1,0],"OAI-SearchBot":["blocked","*",1,0],"ClaudeBot":["blocked","*",1,0],"Claude-User":["blocked","*",1,0],"Claude-SearchBot":["blocked","*",1,0],"anthropic-ai":["blocked","*",1,0],"PerplexityBot":["blocked","*",1,0],"Perplexity-User":["blocked","*",1,0],"Google-Extended":["blocked","*",1,0],"Applebot-Extended":["blocked","*",1,0],"CCBot":["blocked","*",1,0],"Bytespider":["blocked","*",1,0],"Amazonbot":["blocked","*",1,0],"meta-externalagent":["blocked","*",1,0],"cohere-ai":["blocked","*",1,0],"Diffbot":["blocked","*",1,0],"ContextFetchBot":["blocked","*",1,0]},"llms":{"status":"disallowed"},"home":{"status":"disallowed"}},
{"rank":7947,"domain":"physio-pedia.com","origin":"https://physio-pedia.com","robots":{"status":"ok","http":200,"url":"https://www.physio-pedia.com/robots.txt","bytes":465,"groups":1},"bots":{"GPTBot":["partial","*",9,6],"ChatGPT-User":["partial","*",9,6],"OAI-SearchBot":["partial","*",9,6],"ClaudeBot":["partial","*",9,6],"Claude-User":["partial","*",9,6],"Claude-SearchBot":["partial","*",9,6],"anthropic-ai":["partial","*",9,6],"PerplexityBot":["partial","*",9,6],"Perplexity-User":["partial","*",9,6],"Google-Extended":["partial","*",9,6],"Applebot-Extended":["partial","*",9,6],"CCBot":["partial","*",9,6],"Bytespider":["partial","*",9,6],"Amazonbot":["partial","*",9,6],"meta-externalagent":["partial","*",9,6],"cohere-ai":["partial","*",9,6],"Diffbot":["partial","*",9,6],"ContextFetchBot":["partial","*",9,6]},"llms":{"status":"missing","http":404,"bytes":32580},"home":{"status":"http_error","http":403,"url":"https://www.physio-pedia.com/"}},
{"rank":7948,"domain":"atavi.com","origin":"https://atavi.com","robots":{"status":"timeout","http":null,"url":"https://atavi.com/robots.txt","bytes":0,"groups":0,"error":"ConnectTimeout"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":7949,"domain":"cpuid.com","origin":"https://cpuid.com","robots":{"status":"missing","http":404,"url":"https://cpuid.com/robots.txt","bytes":13614,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":13612},"home":{"status":"ok","http":200,"url":"https://cpuid.com/","html_tokens":7672,"markdown_tokens":202,"pruned_tokens":149}},
{"rank":7950,"domain":"austrian.com","origin":"https://austrian.com","robots":{"status":"ok","http":200,"url":"https://www.austrian.com/robots.txt","bytes":241,"groups":1},"bots":{"GPTBot":["partial","*",2,1],"ChatGPT-User":["partial","*",2,1],"OAI-SearchBot":["partial","*",2,1],"ClaudeBot":["partial","*",2,1],"Claude-User":["partial","*",2,1],"Claude-SearchBot":["partial","*",2,1],"anthropic-ai":["partial","*",2,1],"PerplexityBot":["partial","*",2,1],"Perplexity-User":["partial","*",2,1],"Google-Extended":["partial","*",2,1],"Applebot-Extended":["partial","*",2,1],"CCBot":["partial","*",2,1],"Bytespider":["partial","*",2,1],"Amazonbot":["partial","*",2,1],"meta-externalagent":["partial","*",2,1],"cohere-ai":["partial","*",2,1],"Diffbot":["partial","*",2,1],"ContextFetchBot":["partial","*",2,1]},"llms":{"status":"missing","http":404,"bytes":27150},"home":{"status":"http_error","http":403,"url":"https://www.austrian.com/"}},
{"rank":7951,"domain":"valuecommerce.com","origin":"https://valuecommerce.com","robots":{"status":"unreachable","http":null,"url":"https://valuecommerce.com/robots.txt","bytes":0,"groups":0,"error":"cannot resolve valuecommerce.com"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":7952,"domain":"superlawyers.com","origin":"https://superlawyers.com","robots":{"status":"refused","http":403,"url":"https://superlawyers.com/robots.txt","bytes":5533,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":7953,"domain":"carto.com","origin":"https://carto.com","robots":{"status":"ok","http":200,"url":"https://carto.com/robots.txt","bytes":666,"groups":8},"bots":{"GPTBot":["allowed","named",0,1],"ChatGPT-User":["allowed","named",0,1],"OAI-SearchBot":["allowed","*",0,1],"ClaudeBot":["allowed","named",0,1],"Claude-User":["allowed","*",0,1],"Claude-SearchBot":["allowed","*",0,1],"anthropic-ai":["allowed","named",0,1],"PerplexityBot":["allowed","named",0,1],"Perplexity-User":["allowed","*",0,1],"Google-Extended":["allowed","named",0,1],"Applebot-Extended":["allowed","*",0,1],"CCBot":["allowed","*",0,1],"Bytespider":["allowed","named",0,1],"Amazonbot":["allowed","*",0,1],"meta-externalagent":["allowed","*",0,1],"cohere-ai":["allowed","*",0,1],"Diffbot":["allowed","*",0,1],"ContextFetchBot":["allowed","*",0,1]},"llms":{"status":"present","http":200,"bytes":26721},"home":{"status":"ok","http":200,"url":"https://carto.com/","html_tokens":49706,"markdown_tokens":391,"pruned_tokens":194}},
{"rank":7954,"domain":"museivaticani.va","origin":"https://www.museivaticani.va","robots":{"status":"ok","http":200,"url":"https://www.museivaticani.va/robots.txt","bytes":39,"groups":1},"bots":{"GPTBot":["allowed","*",0,1],"ChatGPT-User":["allowed","*",0,1],"OAI-SearchBot":["allowed","*",0,1],"ClaudeBot":["allowed","*",0,1],"Claude-User":["allowed","*",0,1],"Claude-SearchBot":["allowed","*",0,1],"anthropic-ai":["allowed","*",0,1],"PerplexityBot":["allowed","*",0,1],"Perplexity-User":["allowed","*",0,1],"Google-Extended":["allowed","*",0,1],"Applebot-Extended":["allowed","*",0,1],"CCBot":["allowed","*",0,1],"Bytespider":["allowed","*",0,1],"Amazonbot":["allowed","*",0,1],"meta-externalagent":["allowed","*",0,1],"cohere-ai":["allowed","*",0,1],"Diffbot":["allowed","*",0,1],"ContextFetchBot":["allowed","*",0,1]},"llms":{"status":"missing","http":404,"bytes":206},"home":{"status":"ok","http":200,"url":"https://www.museivaticani.va/content/museivaticani.html","html_tokens":658,"markdown_tokens":0,"pruned_tokens":0}},
{"rank":7955,"domain":"thule.com","origin":"https://thule.com","robots":{"status":"ok","http":200,"url":"https://www.thule.com/robots.txt","bytes":693,"groups":1},"bots":{"GPTBot":["partial","*",17,0],"ChatGPT-User":["partial","*",17,0],"OAI-SearchBot":["partial","*",17,0],"ClaudeBot":["partial","*",17,0],"Claude-User":["partial","*",17,0],"Claude-SearchBot":["partial","*",17,0],"anthropic-ai":["partial","*",17,0],"PerplexityBot":["partial","*",17,0],"Perplexity-User":["partial","*",17,0],"Google-Extended":["partial","*",17,0],"Applebot-Extended":["partial","*",17,0],"CCBot":["partial","*",17,0],"Bytespider":["partial","*",17,0],"Amazonbot":["partial","*",17,0],"meta-externalagent":["partial","*",17,0],"cohere-ai":["partial","*",17,0],"Diffbot":["partial","*",17,0],"ContextFetchBot":["partial","*",17,0]},"llms":{"status":"missing","http":404,"bytes":0},"home":{"status":"ok","http":200,"url":"https://www.thule.com/en-us/","html_tokens":42796,"markdown_tokens":182,"pruned_tokens":166}},
{"rank":7956,"domain":"regulation.gov.ru","origin":"https://regulation.gov.ru","robots":{"status":"timeout","http":null,"url":"https://regulation.gov.ru/robots.txt","bytes":0,"groups":0,"error":"timed out after 10 s"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":7957,"domain":"xueqiu.com","origin":"https://xueqiu.com","robots":{"status":"ok","http":200,"url":"https://www.xueqiu.com/robots.txt","bytes":1444,"groups":20},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["blocked","named",1,0],"OAI-SearchBot":["partial","*",7,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",7,0],"Claude-SearchBot":["partial","*",7,0],"anthropic-ai":["partial","*",7,0],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["partial","*",7,0],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["partial","*",7,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["partial","*",7,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",7,0]},"llms":{"status":"missing","http":404,"bytes":60864},"home":{"status":"ok","http":200,"url":"https://xueqiu.com/","html_tokens":27578,"markdown_tokens":0,"pruned_tokens":0}},
{"rank":7958,"domain":"visitdubai.com","origin":"https://visitdubai.com","robots":{"status":"ok","http":200,"url":"https://www.visitdubai.com/robots.txt","bytes":2067,"groups":2},"bots":{"GPTBot":["allowed","named",0,2],"ChatGPT-User":["allowed","named",0,2],"OAI-SearchBot":["partial","*",42,0],"ClaudeBot":["allowed","named",0,2],"Claude-User":["partial","*",42,0],"Claude-SearchBot":["partial","*",42,0],"anthropic-ai":["allowed","named",0,2],"PerplexityBot":["allowed","named",0,2],"Perplexity-User":["allowed","named",0,2],"Google-Extended":["allowed","named",0,2],"Applebot-Extended":["allowed","named",0,2],"CCBot":["allowed","named",0,2],"Bytespider":["allowed","named",0,2],"Amazonbot":["allowed","named",0,2],"meta-externalagent":["allowed","named",0,2],"cohere-ai":["allowed","named",0,2],"Diffbot":["allowed","named",0,2],"ContextFetchBot":["partial","*",42,0]},"llms":{"status":"present","http":200,"bytes":7668},"home":{"status":"ok","http":200,"url":"https://www.visitdubai.com/en","html_tokens":94567,"markdown_tokens":1135,"pruned_tokens":175}},
{"rank":7959,"domain":"finanzen.net","origin":"https://finanzen.net","robots":{"status":"ok","http":200,"url":"https://www.finanzen.net/robots.txt","bytes":3646,"groups":3},"bots":{"GPTBot":["partial","*",97,0],"ChatGPT-User":["partial","*",97,0],"OAI-SearchBot":["partial","*",97,0],"ClaudeBot":["partial","*",97,0],"Claude-User":["partial","*",97,0],"Claude-SearchBot":["partial","*",97,0],"anthropic-ai":["partial","*",97,0],"PerplexityBot":["partial","*",97,0],"Perplexity-User":["partial","*",97,0],"Google-Extended":["partial","*",97,0],"Applebot-Extended":["partial","*",97,0],"CCBot":["partial","*",97,0],"Bytespider":["partial","*",97,0],"Amazonbot":["partial","*",97,0],"meta-externalagent":["partial","*",97,0],"cohere-ai":["partial","*",97,0],"Diffbot":["partial","*",97,0],"ContextFetchBot":["partial","*",97,0]},"llms":{"status":"http_error","http":403,"bytes":382},"home":{"status":"http_error","http":403,"url":"https://www.finanzen.net/"}},
{"rank":7960,"domain":"conductor.com","origin":"https://conductor.com","robots":{"status":"ok","http":200,"url":"https://www.conductor.com/robots.txt","bytes":162,"groups":2},"bots":{"GPTBot":["partial","*",1,1],"ChatGPT-User":["partial","*",1,1],"OAI-SearchBot":["partial","*",1,1],"ClaudeBot":["partial","*",1,1],"Claude-User":["partial","*",1,1],"Claude-SearchBot":["partial","*",1,1],"anthropic-ai":["partial","*",1,1],"PerplexityBot":["partial","*",1,1],"Perplexity-User":["partial","*",1,1],"Google-Extended":["partial","*",1,1],"Applebot-Extended":["partial","*",1,1],"CCBot":["partial","*",1,1],"Bytespider":["partial","*",1,1],"Amazonbot":["partial","*",1,1],"meta-externalagent":["partial","*",1,1],"cohere-ai":["partial","*",1,1],"Diffbot":["partial","*",1,1],"ContextFetchBot":["partial","*",1,1]},"llms":{"status":"present","http":200,"bytes":47189},"home":{"status":"ok","http":200,"url":"https://www.conductor.com/","html_tokens":144503,"markdown_tokens":347,"pruned_tokens":81}},
{"rank":7961,"domain":"datenschutz.de","origin":"https://datenschutz.de","robots":{"status":"ok","http":200,"url":"https://www.datenschutz.de/robots.txt","bytes":119,"groups":1},"bots":{"GPTBot":["partial","*",1,1],"ChatGPT-User":["partial","*",1,1],"OAI-SearchBot":["partial","*",1,1],"ClaudeBot":["partial","*",1,1],"Claude-User":["partial","*",1,1],"Claude-SearchBot":["partial","*",1,1],"anthropic-ai":["partial","*",1,1],"PerplexityBot":["partial","*",1,1],"Perplexity-User":["partial","*",1,1],"Google-Extended":["partial","*",1,1],"Applebot-Extended":["partial","*",1,1],"CCBot":["partial","*",1,1],"Bytespider":["partial","*",1,1],"Amazonbot":["partial","*",1,1],"meta-externalagent":["partial","*",1,1],"cohere-ai":["partial","*",1,1],"Diffbot":["partial","*",1,1],"ContextFetchBot":["partial","*",1,1]},"llms":{"status":"missing","http":404,"bytes":46251},"home":{"status":"ok","http":200,"url":"https://www.datenschutz.de/","html_tokens":11845,"markdown_tokens":273,"pruned_tokens":141}},
{"rank":7962,"domain":"ehu.es","origin":"https://ehu.es","robots":{"status":"ok","http":200,"url":"https://www.ehu.eus/robots.txt","bytes":1054,"groups":3},"bots":{"GPTBot":["partial","*",14,0],"ChatGPT-User":["partial","*",14,0],"OAI-SearchBot":["partial","*",14,0],"ClaudeBot":["partial","*",14,0],"Claude-User":["partial","*",14,0],"Claude-SearchBot":["partial","*",14,0],"anthropic-ai":["partial","*",14,0],"PerplexityBot":["partial","*",14,0],"Perplexity-User":["partial","*",14,0],"Google-Extended":["partial","*",14,0],"Applebot-Extended":["partial","*",14,0],"CCBot":["partial","*",14,0],"Bytespider":["partial","*",14,0],"Amazonbot":["partial","named",3,0],"meta-externalagent":["partial","*",14,0],"cohere-ai":["partial","*",14,0],"Diffbot":["partial","*",14,0],"ContextFetchBot":["partial","*",14,0]},"llms":{"status":"missing","http":404,"bytes":3149},"home":{"status":"ok","http":200,"url":"https://www.ehu.eus/en/en-home","html_tokens":23521,"markdown_tokens":9,"pruned_tokens":9}},
{"rank":7963,"domain":"rodeointl.com","origin":"http://rodeointl.com","robots":{"status":"missing","http":404,"url":"http://rodeointl.com/robots.txt","bytes":1163,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":1163},"home":{"status":"ok","http":200,"url":"http://rodeointl.com/","html_tokens":19017,"markdown_tokens":44,"pruned_tokens":44}},
{"rank":7964,"domain":"marketingweek.com","origin":"https://marketingweek.com","robots":{"status":"ok","http":200,"url":"https://www.marketingweek.com/robots.txt","bytes":119,"groups":1},"bots":{"GPTBot":["partial","*",1,1],"ChatGPT-User":["partial","*",1,1],"OAI-SearchBot":["partial","*",1,1],"ClaudeBot":["partial","*",1,1],"Claude-User":["partial","*",1,1],"Claude-SearchBot":["partial","*",1,1],"anthropic-ai":["partial","*",1,1],"PerplexityBot":["partial","*",1,1],"Perplexity-User":["partial","*",1,1],"Google-Extended":["partial","*",1,1],"Applebot-Extended":["partial","*",1,1],"CCBot":["partial","*",1,1],"Bytespider":["partial","*",1,1],"Amazonbot":["partial","*",1,1],"meta-externalagent":["partial","*",1,1],"cohere-ai":["partial","*",1,1],"Diffbot":["partial","*",1,1],"ContextFetchBot":["partial","*",1,1]},"llms":{"status":"missing","http":404,"bytes":129370},"home":{"status":"http_error","http":403,"url":"https://marketingweek.com/"}},
{"rank":7965,"domain":"lnzjlss.com","origin":"https://lnzjlss.com","robots":{"status":"unreachable","http":null,"url":"https://lnzjlss.com/robots.txt","bytes":0,"groups":0,"error":"cannot resolve lnzjlss.com"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":7966,"domain":"giga.de","origin":"https://giga.de","robots":{"status":"ok","http":200,"url":"https://www.giga.de/robots.txt","bytes":1233,"groups":6},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["partial","*",15,0],"OAI-SearchBot":["partial","*",15,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",15,0],"Claude-SearchBot":["partial","*",15,0],"anthropic-ai":["partial","*",15,0],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["partial","*",15,0],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["partial","*",15,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["partial","*",15,0],"Diffbot":["partial","*",15,0],"ContextFetchBot":["partial","*",15,0]},"llms":{"status":"missing","http":404,"bytes":146},"home":{"status":"ok","http":200,"url":"https://www.giga.de/","html_tokens":105186,"markdown_tokens":24,"pruned_tokens":24}},
{"rank":7967,"domain":"8tracks.com","origin":"https://8tracks.com","robots":{"status":"refused","http":403,"url":"https://8tracks.com/robots.txt","bytes":5527,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":7968,"domain":"nnov.ru","origin":"https://nnov.ru","robots":{"status":"ok","http":200,"url":"https://nnov.ru/robots.txt","bytes":400,"groups":1},"bots":{"GPTBot":["partial","*",12,0],"ChatGPT-User":["partial","*",12,0],"OAI-SearchBot":["partial","*",12,0],"ClaudeBot":["partial","*",12,0],"Claude-User":["partial","*",12,0],"Claude-SearchBot":["partial","*",12,0],"anthropic-ai":["partial","*",12,0],"PerplexityBot":["partial","*",12,0],"Perplexity-User":["partial","*",12,0],"Google-Extended":["partial","*",12,0],"Applebot-Extended":["partial","*",12,0],"CCBot":["partial","*",12,0],"Bytespider":["partial","*",12,0],"Amazonbot":["partial","*",12,0],"meta-externalagent":["partial","*",12,0],"cohere-ai":["partial","*",12,0],"Diffbot":["partial","*",12,0],"ContextFetchBot":["partial","*",12,0]},"llms":{"status":"missing","http":404,"bytes":177993},"home":{"status":"ok","http":200,"url":"https://nnov.ru/","html_tokens":139432,"markdown_tokens":142,"pruned_tokens":141}},
{"rank":7969,"domain":"fukuoka.lg.jp","origin":"https://fukuoka.lg.jp","robots":{"status":"unreachable","http":null,"url":"https://fukuoka.lg.jp/robots.txt","bytes":0,"groups":0,"error":"cannot resolve fukuoka.lg.jp"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":7970,"domain":"uoou.gov.cz","origin":"https://uoou.gov.cz","robots":{"status":"ok","http":200,"url":"https://uoou.gov.cz/robots.txt","bytes":78,"groups":1},"bots":{"GPTBot":["partial","*",1,0],"ChatGPT-User":["partial","*",1,0],"OAI-SearchBot":["partial","*",1,0],"ClaudeBot":["partial","*",1,0],"Claude-User":["partial","*",1,0],"Claude-SearchBot":["partial","*",1,0],"anthropic-ai":["partial","*",1,0],"PerplexityBot":["partial","*",1,0],"Perplexity-User":["partial","*",1,0],"Google-Extended":["partial","*",1,0],"Applebot-Extended":["partial","*",1,0],"CCBot":["partial","*",1,0],"Bytespider":["partial","*",1,0],"Amazonbot":["partial","*",1,0],"meta-externalagent":["partial","*",1,0],"cohere-ai":["partial","*",1,0],"Diffbot":["partial","*",1,0],"ContextFetchBot":["partial","*",1,0]},"llms":{"status":"missing","http":404,"bytes":33013},"home":{"status":"ok","http":200,"url":"https://uoou.gov.cz/","html_tokens":11695,"markdown_tokens":9,"pruned_tokens":9}},
{"rank":7971,"domain":"azleg.gov","origin":"https://azleg.gov","robots":{"status":"ok","http":200,"url":"https://azleg.gov/robots.txt","bytes":793,"groups":2},"bots":{"GPTBot":["partial","*",15,0],"ChatGPT-User":["partial","*",15,0],"OAI-SearchBot":["partial","*",15,0],"ClaudeBot":["partial","*",15,0],"Claude-User":["partial","*",15,0],"Claude-SearchBot":["partial","*",15,0],"anthropic-ai":["partial","*",15,0],"PerplexityBot":["partial","*",15,0],"Perplexity-User":["partial","*",15,0],"Google-Extended":["partial","*",15,0],"Applebot-Extended":["partial","*",15,0],"CCBot":["partial","*",15,0],"Bytespider":["partial","*",15,0],"Amazonbot":["partial","*",15,0],"meta-externalagent":["partial","*",15,0],"cohere-ai":["partial","*",15,0],"Diffbot":["partial","*",15,0],"ContextFetchBot":["partial","*",15,0]},"llms":{"status":"crawl_delay","error":"Crawl-delay 120 s"},"home":{"status":"crawl_delay","error":"Crawl-delay 120 s"}},
{"rank":7972,"domain":"17track.net","origin":"https://www.17track.net","robots":{"status":"ok","http":200,"url":"https://www.17track.net/robots.txt","bytes":76,"groups":1},"bots":{"GPTBot":["allowed","*",0,1],"ChatGPT-User":["allowed","*",0,1],"OAI-SearchBot":["allowed","*",0,1],"ClaudeBot":["allowed","*",0,1],"Claude-User":["allowed","*",0,1],"Claude-SearchBot":["allowed","*",0,1],"anthropic-ai":["allowed","*",0,1],"PerplexityBot":["allowed","*",0,1],"Perplexity-User":["allowed","*",0,1],"Google-Extended":["allowed","*",0,1],"Applebot-Extended":["allowed","*",0,1],"CCBot":["allowed","*",0,1],"Bytespider":["allowed","*",0,1],"Amazonbot":["allowed","*",0,1],"meta-externalagent":["allowed","*",0,1],"cohere-ai":["allowed","*",0,1],"Diffbot":["allowed","*",0,1],"ContextFetchBot":["allowed","*",0,1]},"llms":{"status":"present","http":200,"bytes":10179},"home":{"status":"ok","http":200,"url":"https://www.17track.net/en","html_tokens":50201,"markdown_tokens":912,"pruned_tokens":260}},
{"rank":7973,"domain":"kgw.com","origin":"https://kgw.com","robots":{"status":"ok","http":200,"url":"https://www.kgw.com/robots.txt","bytes":441,"groups":2},"bots":{"GPTBot":["partial","*",10,0],"ChatGPT-User":["partial","*",10,0],"OAI-SearchBot":["partial","*",10,0],"ClaudeBot":["partial","*",10,0],"Claude-User":["partial","*",10,0],"Claude-SearchBot":["partial","*",10,0],"anthropic-ai":["partial","*",10,0],"PerplexityBot":["partial","*",10,0],"Perplexity-User":["partial","*",10,0],"Google-Extended":["partial","*",10,0],"Applebot-Extended":["partial","*",10,0],"CCBot":["partial","*",10,0],"Bytespider":["partial","*",10,0],"Amazonbot":["partial","*",10,0],"meta-externalagent":["partial","*",10,0],"cohere-ai":["partial","*",10,0],"Diffbot":["partial","*",10,0],"ContextFetchBot":["partial","*",10,0]},"llms":{"status":"missing","http":404,"bytes":6833},"home":{"status":"ok","http":200,"url":"https://www.kgw.com/","html_tokens":99934,"markdown_tokens":1638,"pruned_tokens":168}},
{"rank":7974,"domain":"sciencenet.cn","origin":"https://www.sciencenet.cn","robots":{"status":"missing","http":404,"url":"https://www.sciencenet.cn/robots.txt","bytes":0,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":0},"home":{"status":"ok","http":200,"url":"https://www.sciencenet.cn/","html_tokens":17552,"markdown_tokens":5,"pruned_tokens":5}},
{"rank":7975,"domain":"boku.ac.at","origin":"https://boku.ac.at","robots":{"status":"ok","http":200,"url":"https://boku.ac.at/robots.txt","bytes":360,"groups":1},"bots":{"GPTBot":["partial","*",6,3],"ChatGPT-User":["partial","*",6,3],"OAI-SearchBot":["partial","*",6,3],"ClaudeBot":["partial","*",6,3],"Claude-User":["partial","*",6,3],"Claude-SearchBot":["partial","*",6,3],"anthropic-ai":["partial","*",6,3],"PerplexityBot":["partial","*",6,3],"Perplexity-User":["partial","*",6,3],"Google-Extended":["partial","*",6,3],"Applebot-Extended":["partial","*",6,3],"CCBot":["partial","*",6,3],"Bytespider":["partial","*",6,3],"Amazonbot":["partial","*",6,3],"meta-externalagent":["partial","*",6,3],"cohere-ai":["partial","*",6,3],"Diffbot":["partial","*",6,3],"ContextFetchBot":["partial","*",6,3]},"llms":{"status":"missing","http":404,"bytes":32038},"home":{"status":"ok","http":200,"url":"https://boku.ac.at/","html_tokens":10697,"markdown_tokens":253,"pruned_tokens":106}},
{"rank":7976,"domain":"crosswalk.com","origin":"https://crosswalk.com","robots":{"status":"ok","http":200,"url":"https://www.crosswalk.com/robots.txt","bytes":380,"groups":2},"bots":{"GPTBot":["partial","*",1,1],"ChatGPT-User":["partial","*",1,1],"OAI-SearchBot":["partial","*",1,1],"ClaudeBot":["partial","*",1,1],"Claude-User":["partial","*",1,1],"Claude-SearchBot":["partial","*",1,1],"anthropic-ai":["partial","*",1,1],"PerplexityBot":["partial","*",1,1],"Perplexity-User":["partial","*",1,1],"Google-Extended":["partial","*",1,1],"Applebot-Extended":["partial","*",1,1],"CCBot":["partial","*",1,1],"Bytespider":["partial","*",1,1],"Amazonbot":["partial","*",1,1],"meta-externalagent":["partial","*",1,1],"cohere-ai":["partial","*",1,1],"Diffbot":["partial","*",1,1],"ContextFetchBot":["partial","*",1,1]},"llms":{"status":"missing","http":404,"bytes":54324},"home":{"status":"ok","http":200,"url":"https://www.crosswalk.com/","html_tokens":23025,"markdown_tokens":36,"pruned_tokens":36}},
{"rank":7977,"domain":"openpetition.de","origin":"https://openpetition.de","robots":{"status":"ok","http":200,"url":"https://www.openpetition.de/robots.txt","bytes":566,"groups":2},"bots":{"GPTBot":["partial","*",17,0],"ChatGPT-User":["partial","*",17,0],"OAI-SearchBot":["partial","*",17,0],"ClaudeBot":["partial","*",17,0],"Claude-User":["partial","*",17,0],"Claude-SearchBot":["partial","*",17,0],"anthropic-ai":["partial","*",17,0],"PerplexityBot":["partial","*",17,0],"Perplexity-User":["partial","*",17,0],"Google-Extended":["partial","*",17,0],"Applebot-Extended":["partial","*",17,0],"CCBot":["partial","*",17,0],"Bytespider":["partial","*",17,0],"Amazonbot":["partial","*",17,0],"meta-externalagent":["partial","*",17,0],"cohere-ai":["partial","*",17,0],"Diffbot":["partial","*",17,0],"ContextFetchBot":["partial","*",17,0]},"llms":{"status":"missing","http":404,"bytes":113233},"home":{"status":"ok","http":200,"url":"https://www.openpetition.de/","html_tokens":54983,"markdown_tokens":323,"pruned_tokens":147}},
{"rank":7978,"domain":"mapyourshow.com","origin":"https://mapyourshow.com","robots":{"status":"ok","http":200,"url":"https://www.mapyourshow.com/robots.txt?/","bytes":1890,"groups":8},"bots":{"GPTBot":["partial","named",5,1],"ChatGPT-User":["partial","*",5,3],"OAI-SearchBot":["partial","*",5,3],"ClaudeBot":["partial","named",5,1],"Claude-User":["partial","*",5,3],"Claude-SearchBot":["partial","*",5,3],"anthropic-ai":["partial","*",5,3],"PerplexityBot":["partial","named",5,1],"Perplexity-User":["partial","*",5,3],"Google-Extended":["partial","named",5,1],"Applebot-Extended":["partial","*",5,3],"CCBot":["partial","*",5,3],"Bytespider":["partial","*",5,3],"Amazonbot":["partial","*",5,3],"meta-externalagent":["partial","*",5,3],"cohere-ai":["partial","*",5,3],"Diffbot":["partial","*",5,3],"ContextFetchBot":["partial","*",5,3]},"llms":{"status":"missing","http":404,"bytes":121938},"home":{"status":"ok","http":200,"url":"https://www.mapyourshow.com/?/","html_tokens":53344,"markdown_tokens":473,"pruned_tokens":115}},
{"rank":7979,"domain":"hyey.cn","origin":"https://hyey.cn","robots":{"status":"missing","http":404,"url":"https://hyey.cn/robots.txt","bytes":0,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":0},"home":{"status":"http_error","http":404,"url":"https://hyey.cn/"}},
{"rank":7980,"domain":"suo-lang.com","origin":"https://suo-lang.com","robots":{"status":"timeout","http":null,"url":"https://suo-lang.com/robots.txt","bytes":0,"groups":0,"error":"ConnectTimeout"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":7981,"domain":"fiercehealthcare.com","origin":"https://fiercehealthcare.com","robots":{"status":"ok","http":200,"url":"https://www.fiercehealthcare.com/robots.txt","bytes":1620,"groups":1},"bots":{"GPTBot":["partial","*",23,18],"ChatGPT-User":["partial","*",23,18],"OAI-SearchBot":["partial","*",23,18],"ClaudeBot":["partial","*",23,18],"Claude-User":["partial","*",23,18],"Claude-SearchBot":["partial","*",23,18],"anthropic-ai":["partial","*",23,18],"PerplexityBot":["partial","*",23,18],"Perplexity-User":["partial","*",23,18],"Google-Extended":["partial","*",23,18],"Applebot-Extended":["partial","*",23,18],"CCBot":["partial","*",23,18],"Bytespider":["partial","*",23,18],"Amazonbot":["partial","*",23,18],"meta-externalagent":["partial","*",23,18],"cohere-ai":["partial","*",23,18],"Diffbot":["partial","*",23,18],"ContextFetchBot":["partial","*",23,18]},"llms":{"status":"http_error","http":403,"bytes":507},"home":{"status":"http_error","http":403,"url":"https://fiercehealthcare.com/"}},
{"rank":7982,"domain":"google.ht","origin":"https://google.ht","robots":{"status":"ok","http":200,"url":"https://www.google.ht/robots.txt","bytes":6624,"groups":4},"bots":{"GPTBot":["partial","*",169,64],"ChatGPT-User":["partial","*",169,64],"OAI-SearchBot":["partial","*",169,64],"ClaudeBot":["partial","*",169,64],"Claude-User":["partial","*",169,64],"Claude-SearchBot":["partial","*",169,64],"anthropic-ai":["partial","*",169,64],"PerplexityBot":["partial","*",169,64],"Perplexity-User":["partial","*",169,64],"Google-Extended":["partial","*",169,64],"Applebot-Extended":["partial","*",169,64],"CCBot":["partial","*",169,64],"Bytespider":["partial","*",169,64],"Amazonbot":["partial","*",169,64],"meta-externalagent":["partial","*",169,64],"cohere-ai":["partial","*",169,64],"Diffbot":["partial","*",169,64],"ContextFetchBot":["partial","*",169,64]},"llms":{"status":"missing","http":404,"bytes":1569},"home":{"status":"ok","http":200,"url":"https://www.google.com/","html_tokens":21457,"markdown_tokens":30,"pruned_tokens":30}},
{"rank":7983,"domain":"planetminecraft.com","origin":"https://planetminecraft.com","robots":{"status":"refused","http":403,"url":"https://planetminecraft.com/robots.txt","bytes":4838,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":7984,"domain":"ahcgyy120.com","origin":"http://ahcgyy120.com","robots":{"status":"missing","http":404,"url":"http://ahcgyy120.com/robots.txt","bytes":315,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":315},"home":{"status":"http_error","http":404,"url":"http://ahcgyy120.com/"}},
{"rank":7985,"domain":"shanghufang.com","origin":"http://shanghufang.com","robots":{"status":"missing","http":404,"url":"http://shanghufang.com/robots.txt","bytes":1163,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":1163},"home":{"status":"ok","http":200,"url":"http://shanghufang.com/","html_tokens":176,"markdown_tokens":0,"pruned_tokens":0}},
{"rank":7986,"domain":"tallahassee.com","origin":"https://tallahassee.com","robots":{"status":"ok","http":200,"url":"https://www.tallahassee.com/robots.txt","bytes":12671,"groups":288},"bots":{"GPTBot":["partial","named",1,4],"ChatGPT-User":["partial","named",1,4],"OAI-SearchBot":["partial","named",1,4],"ClaudeBot":["blocked","named",1,0],"Claude-User":["blocked","named",1,0],"Claude-SearchBot":["blocked","named",1,0],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["partial","named",1,4],"Perplexity-User":["blocked","named",1,0],"Google-Extended":["partial","named",1,4],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",2,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",32,0]},"llms":{"status":"http_error","http":402,"bytes":254},"home":{"status":"http_error","http":402,"url":"https://www.tallahassee.com/"}},
{"rank":7987,"domain":"traderjoes.com","origin":"https://traderjoes.com","robots":{"status":"refused","http":403,"url":"https://www.traderjoes.com/robots.txt","bytes":386,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":7988,"domain":"woah.org","origin":"https://woah.org","robots":{"status":"ok","http":200,"url":"https://www.woah.org/robots.txt","bytes":483,"groups":1},"bots":{"GPTBot":["partial","*",11,3],"ChatGPT-User":["partial","*",11,3],"OAI-SearchBot":["partial","*",11,3],"ClaudeBot":["partial","*",11,3],"Claude-User":["partial","*",11,3],"Claude-SearchBot":["partial","*",11,3],"anthropic-ai":["partial","*",11,3],"PerplexityBot":["partial","*",11,3],"Perplexity-User":["partial","*",11,3],"Google-Extended":["partial","*",11,3],"Applebot-Extended":["partial","*",11,3],"CCBot":["partial","*",11,3],"Bytespider":["partial","*",11,3],"Amazonbot":["partial","*",11,3],"meta-externalagent":["partial","*",11,3],"cohere-ai":["partial","*",11,3],"Diffbot":["partial","*",11,3],"ContextFetchBot":["partial","*",11,3]},"llms":{"status":"missing","http":404,"bytes":111087},"home":{"status":"ok","http":200,"url":"https://www.woah.org/en/home/","html_tokens":95166,"markdown_tokens":595,"pruned_tokens":149}},
{"rank":7989,"domain":"sapporosushikingston.com","origin":"https://sapporosushikingston.com","robots":{"status":"ok","http":200,"url":"https://www.sapporosushikingston.com/robots.txt","bytes":118,"groups":1},"bots":{"GPTBot":["partial","*",2,1],"ChatGPT-User":["partial","*",2,1],"OAI-SearchBot":["partial","*",2,1],"ClaudeBot":["partial","*",2,1],"Claude-User":["partial","*",2,1],"Claude-SearchBot":["partial","*",2,1],"anthropic-ai":["partial","*",2,1],"PerplexityBot":["partial","*",2,1],"Perplexity-User":["partial","*",2,1],"Google-Extended":["partial","*",2,1],"Applebot-Extended":["partial","*",2,1],"CCBot":["partial","*",2,1],"Bytespider":["partial","*",2,1],"Amazonbot":["partial","*",2,1],"meta-externalagent":["partial","*",2,1],"cohere-ai":["partial","*",2,1],"Diffbot":["partial","*",2,1],"ContextFetchBot":["partial","*",2,1]},"llms":{"status":"missing","http":404,"bytes":1183},"home":{"status":"ok","http":200,"url":"https://www.sapporosushikingston.com/","html_tokens":3092,"markdown_tokens":222,"pruned_tokens":119}},
{"rank":7990,"domain":"voc.com.cn","origin":"https://voc.com.cn","robots":{"status":"missing","http":404,"url":"http://www.voc.com.cn/robots.txt","bytes":665,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":665},"home":{"status":"ok","http":200,"url":"http://www.voc.com.cn/","html_tokens":32809,"markdown_tokens":8,"pruned_tokens":8}},
{"rank":7991,"domain":"wstfls.com","origin":"http://wstfls.com","robots":{"status":"missing","http":404,"url":"http://wstfls.com/robots.txt","bytes":1163,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":1163},"home":{"status":"ok","http":200,"url":"http://wstfls.com/","html_tokens":6788,"markdown_tokens":52,"pruned_tokens":51}},
{"rank":7992,"domain":"character.ai","origin":"https://character.ai","robots":{"status":"ok","http":200,"url":"https://character.ai/robots.txt","bytes":159,"groups":1},"bots":{"GPTBot":["partial","*",5,0],"ChatGPT-User":["partial","*",5,0],"OAI-SearchBot":["partial","*",5,0],"ClaudeBot":["partial","*",5,0],"Claude-User":["partial","*",5,0],"Claude-SearchBot":["partial","*",5,0],"anthropic-ai":["partial","*",5,0],"PerplexityBot":["partial","*",5,0],"Perplexity-User":["partial","*",5,0],"Google-Extended":["partial","*",5,0],"Applebot-Extended":["partial","*",5,0],"CCBot":["partial","*",5,0],"Bytespider":["partial","*",5,0],"Amazonbot":["partial","*",5,0],"meta-externalagent":["partial","*",5,0],"cohere-ai":["partial","*",5,0],"Diffbot":["partial","*",5,0],"ContextFetchBot":["partial","*",5,0]},"llms":{"status":"missing","http":404,"bytes":8173},"home":{"status":"http_error","http":403,"url":"https://character.ai/"}},
{"rank":7993,"domain":"www.rct.uk","origin":"https://www.rct.uk","robots":{"status":"refused","http":403,"url":"https://www.rct.uk/robots.txt","bytes":4900,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":7994,"domain":"bk.com","origin":"https://bk.com","robots":{"status":"html","http":200,"url":"https://www.bk.com/robots.txt","bytes":3381,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":7995,"domain":"czgjp.com","origin":"https://czgjp.com","robots":{"status":"missing","http":404,"url":"https://czgjp.com/robots.txt","bytes":0,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":0},"home":{"status":"ok","http":200,"url":"https://czgjp.com/","html_tokens":18856,"markdown_tokens":19,"pruned_tokens":19}},
{"rank":7996,"domain":"wxgrasp.com","origin":"https://wxgrasp.com","robots":{"status":"missing","http":404,"url":"https://wxgrasp.com/robots.txt","bytes":0,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":0},"home":{"status":"ok","http":200,"url":"https://wxgrasp.com/","html_tokens":17655,"markdown_tokens":19,"pruned_tokens":19}},
{"rank":7997,"domain":"gunma.jp","origin":"https://gunma.jp","robots":{"status":"unreachable","http":null,"url":"https://gunma.jp/robots.txt","bytes":0,"groups":0,"error":"cannot resolve gunma.jp"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":7998,"domain":"jcrew.com","origin":"https://jcrew.com","robots":{"status":"ok","http":200,"url":"https://www.jcrew.com/robots.txt","bytes":2495,"groups":3},"bots":{"GPTBot":["partial","*",103,4],"ChatGPT-User":["partial","*",103,4],"OAI-SearchBot":["partial","*",103,4],"ClaudeBot":["partial","*",103,4],"Claude-User":["partial","*",103,4],"Claude-SearchBot":["partial","*",103,4],"anthropic-ai":["partial","*",103,4],"PerplexityBot":["partial","*",103,4],"Perplexity-User":["partial","*",103,4],"Google-Extended":["partial","*",103,4],"Applebot-Extended":["partial","*",103,4],"CCBot":["partial","*",103,4],"Bytespider":["partial","*",103,4],"Amazonbot":["partial","*",103,4],"meta-externalagent":["partial","*",103,4],"cohere-ai":["partial","*",103,4],"Diffbot":["partial","*",103,4],"ContextFetchBot":["partial","*",103,4]},"llms":{"status":"missing","http":404,"bytes":963464},"home":{"status":"ok","http":200,"url":"https://www.jcrew.com/","html_tokens":307836,"markdown_tokens":159,"pruned_tokens":43}},
{"rank":7999,"domain":"tv2.dk","origin":"https://tv2.dk","robots":{"status":"ok","http":200,"url":"https://tv2.dk/robots.txt","bytes":3078,"groups":2},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["blocked","named",1,0],"OAI-SearchBot":["blocked","named",1,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["blocked","named",1,0],"Claude-SearchBot":["blocked","named",1,0],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["blocked","named",1,0],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["allowed","*",0,1]},"llms":{"status":"missing","http":404,"bytes":202306},"home":{"status":"ok","http":200,"url":"https://tv2.dk/","html_tokens":114915,"markdown_tokens":26,"pruned_tokens":26}},
{"rank":8000,"domain":"free-pokies.co.nz","origin":"https://free-pokies.co.nz","robots":{"status":"ok","http":200,"url":"https://free-pokies.co.nz/robots.txt","bytes":538,"groups":1},"bots":{"GPTBot":["partial","*",19,0],"ChatGPT-User":["partial","*",19,0],"OAI-SearchBot":["partial","*",19,0],"ClaudeBot":["partial","*",19,0],"Claude-User":["partial","*",19,0],"Claude-SearchBot":["partial","*",19,0],"anthropic-ai":["partial","*",19,0],"PerplexityBot":["partial","*",19,0],"Perplexity-User":["partial","*",19,0],"Google-Extended":["partial","*",19,0],"Applebot-Extended":["partial","*",19,0],"CCBot":["partial","*",19,0],"Bytespider":["partial","*",19,0],"Amazonbot":["partial","*",19,0],"meta-externalagent":["partial","*",19,0],"cohere-ai":["partial","*",19,0],"Diffbot":["partial","*",19,0],"ContextFetchBot":["partial","*",19,0]},"llms":{"status":"missing","http":404,"bytes":34376},"home":{"status":"ok","http":200,"url":"https://free-pokies.co.nz/","html_tokens":23912,"markdown_tokens":1257,"pruned_tokens":166}},
{"rank":8001,"domain":"chat.ru","origin":"http://chat.ru","robots":{"status":"ok","http":200,"url":"http://chat.ru/robots.txt","bytes":0,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":206},"home":{"status":"ok","http":200,"url":"http://chat.ru/","html_tokens":81248,"markdown_tokens":222,"pruned_tokens":149}},
{"rank":8002,"domain":"sxgjp.com","origin":"https://sxgjp.com","robots":{"status":"missing","http":404,"url":"https://sxgjp.com/robots.txt","bytes":0,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":0},"home":{"status":"ok","http":200,"url":"https://sxgjp.com/","html_tokens":20943,"markdown_tokens":20,"pruned_tokens":20}},
{"rank":8003,"domain":"ck.ua","origin":"https://www.ck.ua","robots":{"status":"ok","http":200,"url":"https://www.ck.ua/robots.txt","bytes":70,"groups":1},"bots":{"GPTBot":["partial","*",2,0],"ChatGPT-User":["partial","*",2,0],"OAI-SearchBot":["partial","*",2,0],"ClaudeBot":["partial","*",2,0],"Claude-User":["partial","*",2,0],"Claude-SearchBot":["partial","*",2,0],"anthropic-ai":["partial","*",2,0],"PerplexityBot":["partial","*",2,0],"Perplexity-User":["partial","*",2,0],"Google-Extended":["partial","*",2,0],"Applebot-Extended":["partial","*",2,0],"CCBot":["partial","*",2,0],"Bytespider":["partial","*",2,0],"Amazonbot":["partial","*",2,0],"meta-externalagent":["partial","*",2,0],"cohere-ai":["partial","*",2,0],"Diffbot":["partial","*",2,0],"ContextFetchBot":["partial","*",2,0]},"llms":{"status":"missing","http":404,"bytes":1402},"home":{"status":"ok","http":200,"url":"https://www.ck.ua/","html_tokens":805,"markdown_tokens":37,"pruned_tokens":37}},
{"rank":8004,"domain":"able2know.org","origin":"https://able2know.org","robots":{"status":"ok","http":200,"url":"https://able2know.org/robots.txt","bytes":631,"groups":1},"bots":{"GPTBot":["partial","*",23,0],"ChatGPT-User":["partial","*",23,0],"OAI-SearchBot":["partial","*",23,0],"ClaudeBot":["partial","*",23,0],"Claude-User":["partial","*",23,0],"Claude-SearchBot":["partial","*",23,0],"anthropic-ai":["partial","*",23,0],"PerplexityBot":["partial","*",23,0],"Perplexity-User":["partial","*",23,0],"Google-Extended":["partial","*",23,0],"Applebot-Extended":["partial","*",23,0],"CCBot":["partial","*",23,0],"Bytespider":["partial","*",23,0],"Amazonbot":["partial","*",23,0],"meta-externalagent":["partial","*",23,0],"cohere-ai":["partial","*",23,0],"Diffbot":["partial","*",23,0],"ContextFetchBot":["partial","*",23,0]},"llms":{"status":"missing","http":404,"bytes":284},"home":{"status":"ok","http":200,"url":"https://able2know.org/","html_tokens":4629,"markdown_tokens":1,"pruned_tokens":1}},
{"rank":8005,"domain":"csuchico.edu","origin":"https://www.csuchico.edu","robots":{"status":"ok","http":200,"url":"https://www.csuchico.edu/robots.txt","bytes":1125,"groups":5},"bots":{"GPTBot":["partial","*",34,0],"ChatGPT-User":["partial","*",34,0],"OAI-SearchBot":["partial","*",34,0],"ClaudeBot":["blocked","named",2,0],"Claude-User":["partial","*",34,0],"Claude-SearchBot":["partial","*",34,0],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["partial","*",34,0],"Perplexity-User":["partial","*",34,0],"Google-Extended":["partial","*",34,0],"Applebot-Extended":["partial","*",34,0],"CCBot":["partial","*",34,0],"Bytespider":["partial","*",34,0],"Amazonbot":["partial","*",34,0],"meta-externalagent":["partial","*",34,0],"cohere-ai":["partial","*",34,0],"Diffbot":["partial","*",34,0],"ContextFetchBot":["partial","*",34,0]},"llms":{"status":"missing","http":404,"bytes":176092},"home":{"status":"ok","http":200,"url":"https://www.csuchico.edu/","html_tokens":326156,"markdown_tokens":698,"pruned_tokens":153}},
{"rank":8006,"domain":"fwsyxt.com","origin":"http://fwsyxt.com","robots":{"status":"missing","http":404,"url":"http://fwsyxt.com/robots.txt","bytes":146,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":146},"home":{"status":"ok","http":200,"url":"http://fwsyxt.com/","html_tokens":35,"markdown_tokens":6,"pruned_tokens":6}},
{"rank":8007,"domain":"sdyuxi.com","origin":"http://sdyuxi.com","robots":{"status":"missing","http":404,"url":"http://sdyuxi.com/robots.txt","bytes":27250,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":27250},"home":{"status":"ok","http":200,"url":"http://sdyuxi.com/","html_tokens":10436,"markdown_tokens":23,"pruned_tokens":23}},
{"rank":8008,"domain":"transportenvironment.org","origin":"https://transportenvironment.org","robots":{"status":"ok","http":200,"url":"https://www.transportenvironment.org/robots.txt","bytes":1110,"groups":3},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["partial","*",4,0],"OAI-SearchBot":["partial","*",4,0],"ClaudeBot":["partial","*",4,0],"Claude-User":["partial","*",4,0],"Claude-SearchBot":["partial","*",4,0],"anthropic-ai":["partial","*",4,0],"PerplexityBot":["partial","*",4,0],"Perplexity-User":["partial","*",4,0],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["partial","*",4,0],"CCBot":["partial","*",4,0],"Bytespider":["partial","*",4,0],"Amazonbot":["partial","*",4,0],"meta-externalagent":["partial","*",4,0],"cohere-ai":["partial","*",4,0],"Diffbot":["partial","*",4,0],"ContextFetchBot":["partial","*",4,0]},"llms":{"status":"missing","http":404,"bytes":196181},"home":{"status":"ok","http":200,"url":"https://www.transportenvironment.org/","html_tokens":72095,"markdown_tokens":425,"pruned_tokens":108}},
{"rank":8009,"domain":"lord-of-the-ocean-slot.com","origin":"https://lord-of-the-ocean-slot.com","robots":{"status":"ok","http":200,"url":"https://lord-of-the-ocean-slot.com/robots.txt","bytes":557,"groups":1},"bots":{"GPTBot":["partial","*",19,0],"ChatGPT-User":["partial","*",19,0],"OAI-SearchBot":["partial","*",19,0],"ClaudeBot":["partial","*",19,0],"Claude-User":["partial","*",19,0],"Claude-SearchBot":["partial","*",19,0],"anthropic-ai":["partial","*",19,0],"PerplexityBot":["partial","*",19,0],"Perplexity-User":["partial","*",19,0],"Google-Extended":["partial","*",19,0],"Applebot-Extended":["partial","*",19,0],"CCBot":["partial","*",19,0],"Bytespider":["partial","*",19,0],"Amazonbot":["partial","*",19,0],"meta-externalagent":["partial","*",19,0],"cohere-ai":["partial","*",19,0],"Diffbot":["partial","*",19,0],"ContextFetchBot":["partial","*",19,0]},"llms":{"status":"missing","http":404,"bytes":26160},"home":{"status":"ok","http":200,"url":"https://lord-of-the-ocean-slot.com/","html_tokens":25324,"markdown_tokens":2785,"pruned_tokens":746}},
{"rank":8010,"domain":"hitwebcounter.com","origin":"https://hitwebcounter.com","robots":{"status":"ok","http":200,"url":"https://www.hitwebcounter.com/robots.txt","bytes":29,"groups":1},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"missing","http":404,"bytes":722},"home":{"status":"ok","http":200,"url":"https://www.hitwebcounter.com/","html_tokens":8562,"markdown_tokens":597,"pruned_tokens":152}},
{"rank":8011,"domain":"gratowin-casino.com","origin":"https://gratowin-casino.com","robots":{"status":"html","http":200,"url":"https://gratowin-casino.com/robots.txt","bytes":22697,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":8012,"domain":"jacc.org","origin":"https://jacc.org","robots":{"status":"ok","http":200,"url":"https://www.jacc.org/robots.txt","bytes":2092,"groups":3},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["partial","named",26,7],"OAI-SearchBot":["partial","named",26,7],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","named",26,7],"Claude-SearchBot":["partial","named",26,7],"anthropic-ai":["partial","*",26,7],"PerplexityBot":["partial","named",26,7],"Perplexity-User":["partial","*",26,7],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["partial","*",26,7],"CCBot":["blocked","named",1,0],"Bytespider":["partial","named",26,7],"Amazonbot":["partial","*",26,7],"meta-externalagent":["partial","*",26,7],"cohere-ai":["partial","*",26,7],"Diffbot":["partial","*",26,7],"ContextFetchBot":["partial","*",26,7]},"llms":{"status":"http_error","http":403,"bytes":5518},"home":{"status":"http_error","http":403,"url":"https://jacc.org/"}},
{"rank":8013,"domain":"heineken.com","origin":"https://heineken.com","robots":{"status":"ok","http":200,"url":"https://www.heineken.com/robots.txt","bytes":3574,"groups":1},"bots":{"GPTBot":["partial","*",1,0],"ChatGPT-User":["partial","*",1,0],"OAI-SearchBot":["partial","*",1,0],"ClaudeBot":["partial","*",1,0],"Claude-User":["partial","*",1,0],"Claude-SearchBot":["partial","*",1,0],"anthropic-ai":["partial","*",1,0],"PerplexityBot":["partial","*",1,0],"Perplexity-User":["partial","*",1,0],"Google-Extended":["partial","*",1,0],"Applebot-Extended":["partial","*",1,0],"CCBot":["partial","*",1,0],"Bytespider":["partial","*",1,0],"Amazonbot":["partial","*",1,0],"meta-externalagent":["partial","*",1,0],"cohere-ai":["partial","*",1,0],"Diffbot":["partial","*",1,0],"ContextFetchBot":["partial","*",1,0]},"llms":{"status":"missing","http":404,"bytes":0},"home":{"status":"ok","http":200,"url":"https://www.heineken.com/global/en/home/","html_tokens":151537,"markdown_tokens":304,"pruned_tokens":75}},
{"rank":8014,"domain":"airliners.net","origin":"https://airliners.net","robots":{"status":"ok","http":200,"url":"https://www.airliners.net/robots.txt","bytes":1296,"groups":3},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["blocked","named",1,0],"OAI-SearchBot":["blocked","named",1,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",2,0],"Claude-SearchBot":["partial","*",2,0],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["partial","*",2,0],"Google-Extended":["partial","*",2,0],"Applebot-Extended":["partial","*",2,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",2,0]},"llms":{"status":"missing","http":404,"bytes":471},"home":{"status":"ok","http":200,"url":"https://www.airliners.net/","html_tokens":24927,"markdown_tokens":195,"pruned_tokens":31}},
{"rank":8015,"domain":"arcade.now","origin":"https://arcade.now","robots":{"status":"ok","http":200,"url":"https://arcade.now/robots.txt","bytes":692,"groups":3},"bots":{"GPTBot":["partial","*",6,1],"ChatGPT-User":["partial","*",6,1],"OAI-SearchBot":["partial","*",6,1],"ClaudeBot":["partial","*",6,1],"Claude-User":["partial","*",6,1],"Claude-SearchBot":["partial","*",6,1],"anthropic-ai":["partial","*",6,1],"PerplexityBot":["partial","*",6,1],"Perplexity-User":["partial","*",6,1],"Google-Extended":["partial","*",6,1],"Applebot-Extended":["partial","*",6,1],"CCBot":["partial","*",6,1],"Bytespider":["partial","*",6,1],"Amazonbot":["partial","*",6,1],"meta-externalagent":["partial","*",6,1],"cohere-ai":["partial","*",6,1],"Diffbot":["partial","*",6,1],"ContextFetchBot":["partial","*",6,1]},"llms":{"status":"missing","http":404,"bytes":33943},"home":{"status":"ok","http":200,"url":"https://arcade.now/","html_tokens":19637,"markdown_tokens":1097,"pruned_tokens":280}},
{"rank":8016,"domain":"kingarthurbaking.com","origin":"https://kingarthurbaking.com","robots":{"status":"ok","http":200,"url":"https://www.kingarthurbaking.com/robots.txt","bytes":2646,"groups":2},"bots":{"GPTBot":["partial","*",50,18],"ChatGPT-User":["partial","*",50,18],"OAI-SearchBot":["partial","*",50,18],"ClaudeBot":["partial","*",50,18],"Claude-User":["partial","*",50,18],"Claude-SearchBot":["partial","*",50,18],"anthropic-ai":["partial","*",50,18],"PerplexityBot":["partial","*",50,18],"Perplexity-User":["partial","*",50,18],"Google-Extended":["partial","*",50,18],"Applebot-Extended":["partial","*",50,18],"CCBot":["partial","*",50,18],"Bytespider":["partial","*",50,18],"Amazonbot":["partial","*",50,18],"meta-externalagent":["partial","*",50,18],"cohere-ai":["partial","*",50,18],"Diffbot":["partial","*",50,18],"ContextFetchBot":["partial","*",50,18]},"llms":{"status":"missing","http":404,"bytes":198},"home":{"status":"ok","http":200,"url":"https://www.kingarthurbaking.com/","html_tokens":80559,"markdown_tokens":194,"pruned_tokens":104}},
{"rank":8017,"domain":"i24news.tv","origin":"https://i24news.tv","robots":{"status":"ok","http":200,"url":"https://www.i24news.tv/robots.txt","bytes":1527,"groups":2},"bots":{"GPTBot":["partial","*",3,1],"ChatGPT-User":["partial","*",3,1],"OAI-SearchBot":["partial","*",3,1],"ClaudeBot":["partial","*",3,1],"Claude-User":["partial","*",3,1],"Claude-SearchBot":["partial","*",3,1],"anthropic-ai":["partial","*",3,1],"PerplexityBot":["partial","*",3,1],"Perplexity-User":["partial","*",3,1],"Google-Extended":["partial","*",3,1],"Applebot-Extended":["partial","*",3,1],"CCBot":["partial","*",3,1],"Bytespider":["partial","*",3,1],"Amazonbot":["partial","*",3,1],"meta-externalagent":["partial","*",3,1],"cohere-ai":["partial","*",3,1],"Diffbot":["partial","*",3,1],"ContextFetchBot":["partial","*",3,1]},"llms":{"status":"html","http":200,"bytes":7310},"home":{"status":"ok","http":200,"url":"https://www.i24news.tv/en","html_tokens":60070,"markdown_tokens":220,"pruned_tokens":59}},
{"rank":8018,"domain":"google.bt","origin":"https://google.bt","robots":{"status":"ok","http":200,"url":"https://www.google.bt/robots.txt","bytes":6624,"groups":4},"bots":{"GPTBot":["partial","*",169,64],"ChatGPT-User":["partial","*",169,64],"OAI-SearchBot":["partial","*",169,64],"ClaudeBot":["partial","*",169,64],"Claude-User":["partial","*",169,64],"Claude-SearchBot":["partial","*",169,64],"anthropic-ai":["partial","*",169,64],"PerplexityBot":["partial","*",169,64],"Perplexity-User":["partial","*",169,64],"Google-Extended":["partial","*",169,64],"Applebot-Extended":["partial","*",169,64],"CCBot":["partial","*",169,64],"Bytespider":["partial","*",169,64],"Amazonbot":["partial","*",169,64],"meta-externalagent":["partial","*",169,64],"cohere-ai":["partial","*",169,64],"Diffbot":["partial","*",169,64],"ContextFetchBot":["partial","*",169,64]},"llms":{"status":"missing","http":404,"bytes":1569},"home":{"status":"ok","http":200,"url":"https://www.google.com/","html_tokens":21349,"markdown_tokens":30,"pruned_tokens":30}},
{"rank":8019,"domain":"linuxquestions.org","origin":"https://linuxquestions.org","robots":{"status":"refused","http":403,"url":"http://www.linuxquestions.org/robots.txt","bytes":5431,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":8020,"domain":"bobaocn.com","origin":"http://bobaocn.com","robots":{"status":"missing","http":404,"url":"http://bobaocn.com/robots.txt","bytes":1831,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":1829},"home":{"status":"ok","http":200,"url":"http://bobaocn.com/","html_tokens":13945,"markdown_tokens":54,"pruned_tokens":54}},
{"rank":8021,"domain":"mi.gov","origin":"https://mi.gov","robots":{"status":"refused","http":403,"url":"https://mi.gov/robots.txt","bytes":370,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":8022,"domain":"tsn.ua","origin":"https://tsn.ua","robots":{"status":"ok","http":200,"url":"https://tsn.ua/robots.txt","bytes":929,"groups":1},"bots":{"GPTBot":["partial","*",16,1],"ChatGPT-User":["partial","*",16,1],"OAI-SearchBot":["partial","*",16,1],"ClaudeBot":["partial","*",16,1],"Claude-User":["partial","*",16,1],"Claude-SearchBot":["partial","*",16,1],"anthropic-ai":["partial","*",16,1],"PerplexityBot":["partial","*",16,1],"Perplexity-User":["partial","*",16,1],"Google-Extended":["partial","*",16,1],"Applebot-Extended":["partial","*",16,1],"CCBot":["partial","*",16,1],"Bytespider":["partial","*",16,1],"Amazonbot":["partial","*",16,1],"meta-externalagent":["partial","*",16,1],"cohere-ai":["partial","*",16,1],"Diffbot":["partial","*",16,1],"ContextFetchBot":["partial","*",16,1]},"llms":{"status":"present","http":200,"bytes":2771},"home":{"status":"ok","http":200,"url":"https://tsn.ua/","html_tokens":48419,"markdown_tokens":1495,"pruned_tokens":171}},
{"rank":8023,"domain":"daytondailynews.com","origin":"https://daytondailynews.com","robots":{"status":"ok","http":200,"url":"https://daytondailynews.com/robots.txt","bytes":4253,"groups":31},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["blocked","named",1,0],"OAI-SearchBot":["blocked","named",1,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",34,0],"Claude-SearchBot":["partial","*",34,0],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["partial","*",34,0],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["partial","*",34,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",34,0]},"llms":{"status":"http_error","http":429,"bytes":64},"home":{"status":"ok","http":200,"url":"https://www.daytondailynews.com/","html_tokens":154740,"markdown_tokens":188,"pruned_tokens":174}},
{"rank":8024,"domain":"agh.edu.pl","origin":"https://agh.edu.pl","robots":{"status":"refused","http":403,"url":"https://agh.edu.pl/robots.txt","bytes":24956,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":8025,"domain":"gktsy.com","origin":"https://gktsy.com","robots":{"status":"unreachable","http":null,"url":"https://gktsy.com/robots.txt","bytes":0,"groups":0,"error":"cannot resolve gktsy.com"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":8026,"domain":"qplmall.com","origin":"https://qplmall.com","robots":{"status":"timeout","http":null,"url":"https://qplmall.com/robots.txt","bytes":0,"groups":0,"error":"ConnectTimeout"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":8027,"domain":"shengyitang.com","origin":"http://shengyitang.com","robots":{"status":"ok","http":200,"url":"http://shengyitang.com/robots.txt","bytes":520,"groups":1},"bots":{"GPTBot":["partial","*",18,0],"ChatGPT-User":["partial","*",18,0],"OAI-SearchBot":["partial","*",18,0],"ClaudeBot":["partial","*",18,0],"Claude-User":["partial","*",18,0],"Claude-SearchBot":["partial","*",18,0],"anthropic-ai":["partial","*",18,0],"PerplexityBot":["partial","*",18,0],"Perplexity-User":["partial","*",18,0],"Google-Extended":["partial","*",18,0],"Applebot-Extended":["partial","*",18,0],"CCBot":["partial","*",18,0],"Bytespider":["partial","*",18,0],"Amazonbot":["partial","*",18,0],"meta-externalagent":["partial","*",18,0],"cohere-ai":["partial","*",18,0],"Diffbot":["partial","*",18,0],"ContextFetchBot":["partial","*",18,0]},"llms":{"status":"missing","http":404,"bytes":1308},"home":{"status":"ok","http":200,"url":"http://shengyitang.com/","html_tokens":16365,"markdown_tokens":55,"pruned_tokens":55}},
{"rank":8028,"domain":"lastminute.com","origin":"https://lastminute.com","robots":{"status":"refused","http":403,"url":"https://www.lastminute.com/robots.txt","bytes":5556,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":8029,"domain":"dermnetnz.org","origin":"https://dermnetnz.org","robots":{"status":"ok","http":200,"url":"https://dermnetnz.org/robots.txt","bytes":143,"groups":1},"bots":{"GPTBot":["partial","*",3,0],"ChatGPT-User":["partial","*",3,0],"OAI-SearchBot":["partial","*",3,0],"ClaudeBot":["partial","*",3,0],"Claude-User":["partial","*",3,0],"Claude-SearchBot":["partial","*",3,0],"anthropic-ai":["partial","*",3,0],"PerplexityBot":["partial","*",3,0],"Perplexity-User":["partial","*",3,0],"Google-Extended":["partial","*",3,0],"Applebot-Extended":["partial","*",3,0],"CCBot":["partial","*",3,0],"Bytespider":["partial","*",3,0],"Amazonbot":["partial","*",3,0],"meta-externalagent":["partial","*",3,0],"cohere-ai":["partial","*",3,0],"Diffbot":["partial","*",3,0],"ContextFetchBot":["partial","*",3,0]},"llms":{"status":"missing","http":404,"bytes":37452},"home":{"status":"ok","http":200,"url":"https://dermnetnz.org/","html_tokens":20697,"markdown_tokens":52,"pruned_tokens":52}},
{"rank":8030,"domain":"kaizhenglaw.com","origin":"https://www.kaizhenglaw.com","robots":{"status":"missing","http":404,"url":"https://www.kaizhenglaw.com/robots.txt","bytes":5082,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":5078},"home":{"status":"ok","http":200,"url":"https://www.kaizhenglaw.com/","html_tokens":4014,"markdown_tokens":165,"pruned_tokens":165}},
{"rank":8031,"domain":"vw.com","origin":"https://vw.com","robots":{"status":"ok","http":200,"url":"https://www.vw.com/robots.txt","bytes":1286,"groups":6},"bots":{"GPTBot":["partial","*",20,0],"ChatGPT-User":["partial","*",20,0],"OAI-SearchBot":["partial","*",20,0],"ClaudeBot":["partial","*",20,0],"Claude-User":["partial","*",20,0],"Claude-SearchBot":["partial","*",20,0],"anthropic-ai":["partial","*",20,0],"PerplexityBot":["partial","*",20,0],"Perplexity-User":["partial","*",20,0],"Google-Extended":["partial","*",20,0],"Applebot-Extended":["partial","*",20,0],"CCBot":["partial","*",20,0],"Bytespider":["partial","*",20,0],"Amazonbot":["partial","*",20,0],"meta-externalagent":["partial","*",20,0],"cohere-ai":["partial","*",20,0],"Diffbot":["partial","*",20,0],"ContextFetchBot":["partial","*",20,0]},"llms":{"status":"present","http":200,"bytes":148562},"home":{"status":"ok","http":200,"url":"https://www.vw.com/en.html","html_tokens":163799,"markdown_tokens":754,"pruned_tokens":152}},
{"rank":8032,"domain":"cpc-sieton.cn","origin":"http://cpc-sieton.cn","robots":{"status":"missing","http":404,"url":"http://cpc-sieton.cn/robots.txt","bytes":1231,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":1231},"home":{"status":"ok","http":200,"url":"http://cpc-sieton.cn/login.aspx","html_tokens":636,"markdown_tokens":34,"pruned_tokens":34}},
{"rank":8033,"domain":"techbullion.com","origin":"https://techbullion.com","robots":{"status":"ok","http":200,"url":"https://techbullion.com/robots.txt","bytes":870,"groups":8},"bots":{"GPTBot":["partial","*",18,3],"ChatGPT-User":["partial","*",18,3],"OAI-SearchBot":["partial","*",18,3],"ClaudeBot":["partial","*",18,3],"Claude-User":["partial","*",18,3],"Claude-SearchBot":["partial","*",18,3],"anthropic-ai":["partial","*",18,3],"PerplexityBot":["allowed","named",0,1],"Perplexity-User":["partial","*",18,3],"Google-Extended":["allowed","named",0,1],"Applebot-Extended":["partial","*",18,3],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["allowed","named",0,1],"cohere-ai":["partial","*",18,3],"Diffbot":["partial","*",18,3],"ContextFetchBot":["partial","*",18,3]},"llms":{"status":"other_text","http":200,"bytes":2517},"home":{"status":"ok","http":200,"url":"https://techbullion.com/","html_tokens":47636,"markdown_tokens":552,"pruned_tokens":137}},
{"rank":8034,"domain":"moveon.org","origin":"https://moveon.org","robots":{"status":"ok","http":200,"url":"https://moveon.org/robots.txt","bytes":23,"groups":1},"bots":{"GPTBot":["allowed","*",0,1],"ChatGPT-User":["allowed","*",0,1],"OAI-SearchBot":["allowed","*",0,1],"ClaudeBot":["allowed","*",0,1],"Claude-User":["allowed","*",0,1],"Claude-SearchBot":["allowed","*",0,1],"anthropic-ai":["allowed","*",0,1],"PerplexityBot":["allowed","*",0,1],"Perplexity-User":["allowed","*",0,1],"Google-Extended":["allowed","*",0,1],"Applebot-Extended":["allowed","*",0,1],"CCBot":["allowed","*",0,1],"Bytespider":["allowed","*",0,1],"Amazonbot":["allowed","*",0,1],"meta-externalagent":["allowed","*",0,1],"cohere-ai":["allowed","*",0,1],"Diffbot":["allowed","*",0,1],"ContextFetchBot":["allowed","*",0,1]},"llms":{"status":"missing","http":404,"bytes":401060},"home":{"status":"ok","http":200,"url":"https://front.moveon.org/","html_tokens":177369,"markdown_tokens":179,"pruned_tokens":37}},
{"rank":8035,"domain":"google.dj","origin":"https://google.dj","robots":{"status":"ok","http":200,"url":"https://www.google.dj/robots.txt","bytes":6624,"groups":4},"bots":{"GPTBot":["partial","*",169,64],"ChatGPT-User":["partial","*",169,64],"OAI-SearchBot":["partial","*",169,64],"ClaudeBot":["partial","*",169,64],"Claude-User":["partial","*",169,64],"Claude-SearchBot":["partial","*",169,64],"anthropic-ai":["partial","*",169,64],"PerplexityBot":["partial","*",169,64],"Perplexity-User":["partial","*",169,64],"Google-Extended":["partial","*",169,64],"Applebot-Extended":["partial","*",169,64],"CCBot":["partial","*",169,64],"Bytespider":["partial","*",169,64],"Amazonbot":["partial","*",169,64],"meta-externalagent":["partial","*",169,64],"cohere-ai":["partial","*",169,64],"Diffbot":["partial","*",169,64],"ContextFetchBot":["partial","*",169,64]},"llms":{"status":"missing","http":404,"bytes":1569},"home":{"status":"ok","http":200,"url":"https://www.google.com/","html_tokens":21362,"markdown_tokens":30,"pruned_tokens":30}},
{"rank":8036,"domain":"uwe.ac.uk","origin":"https://uwe.ac.uk","robots":{"status":"ok","http":200,"url":"https://www.uwe.ac.uk/robots.txt","bytes":449,"groups":1},"bots":{"GPTBot":["partial","*",5,2],"ChatGPT-User":["partial","*",5,2],"OAI-SearchBot":["partial","*",5,2],"ClaudeBot":["partial","*",5,2],"Claude-User":["partial","*",5,2],"Claude-SearchBot":["partial","*",5,2],"anthropic-ai":["partial","*",5,2],"PerplexityBot":["partial","*",5,2],"Perplexity-User":["partial","*",5,2],"Google-Extended":["partial","*",5,2],"Applebot-Extended":["partial","*",5,2],"CCBot":["partial","*",5,2],"Bytespider":["partial","*",5,2],"Amazonbot":["partial","*",5,2],"meta-externalagent":["partial","*",5,2],"cohere-ai":["partial","*",5,2],"Diffbot":["partial","*",5,2],"ContextFetchBot":["partial","*",5,2]},"llms":{"status":"missing","http":404,"bytes":196},"home":{"status":"ok","http":200,"url":"https://www.uwe.ac.uk/","html_tokens":17091,"markdown_tokens":38,"pruned_tokens":38}},
{"rank":8037,"domain":"bupropion150mg.shop","origin":"https://bupropion150mg.shop","robots":{"status":"ok","http":200,"url":"https://bupropion150mg.shop/robots.txt","bytes":80,"groups":1},"bots":{"GPTBot":["partial","*",1,0],"ChatGPT-User":["partial","*",1,0],"OAI-SearchBot":["partial","*",1,0],"ClaudeBot":["partial","*",1,0],"Claude-User":["partial","*",1,0],"Claude-SearchBot":["partial","*",1,0],"anthropic-ai":["partial","*",1,0],"PerplexityBot":["partial","*",1,0],"Perplexity-User":["partial","*",1,0],"Google-Extended":["partial","*",1,0],"Applebot-Extended":["partial","*",1,0],"CCBot":["partial","*",1,0],"Bytespider":["partial","*",1,0],"Amazonbot":["partial","*",1,0],"meta-externalagent":["partial","*",1,0],"cohere-ai":["partial","*",1,0],"Diffbot":["partial","*",1,0],"ContextFetchBot":["partial","*",1,0]},"llms":{"status":"missing","http":404,"bytes":648},"home":{"status":"ok","http":200,"url":"https://bupropion150mg.shop/","html_tokens":18305,"markdown_tokens":2588,"pruned_tokens":389}},
{"rank":8038,"domain":"ncisoft.com","origin":"https://www.ncisoft.com","robots":{"status":"missing","http":404,"url":"https://www.ncisoft.com/robots.txt","bytes":1831,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":1829},"home":{"status":"ok","http":200,"url":"https://www.ncisoft.com/index.html","html_tokens":2902,"markdown_tokens":40,"pruned_tokens":7}},
{"rank":8039,"domain":"ahfplaw.com","origin":"http://ahfplaw.com","robots":{"status":"missing","http":404,"url":"http://ahfplaw.com/robots.txt","bytes":1163,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":1163},"home":{"status":"ok","http":200,"url":"http://ahfplaw.com/","html_tokens":4691,"markdown_tokens":18,"pruned_tokens":18}},
{"rank":8040,"domain":"google.gp","origin":"https://google.gp","robots":{"status":"ok","http":200,"url":"https://www.google.gp/robots.txt","bytes":6624,"groups":4},"bots":{"GPTBot":["partial","*",169,64],"ChatGPT-User":["partial","*",169,64],"OAI-SearchBot":["partial","*",169,64],"ClaudeBot":["partial","*",169,64],"Claude-User":["partial","*",169,64],"Claude-SearchBot":["partial","*",169,64],"anthropic-ai":["partial","*",169,64],"PerplexityBot":["partial","*",169,64],"Perplexity-User":["partial","*",169,64],"Google-Extended":["partial","*",169,64],"Applebot-Extended":["partial","*",169,64],"CCBot":["partial","*",169,64],"Bytespider":["partial","*",169,64],"Amazonbot":["partial","*",169,64],"meta-externalagent":["partial","*",169,64],"cohere-ai":["partial","*",169,64],"Diffbot":["partial","*",169,64],"ContextFetchBot":["partial","*",169,64]},"llms":{"status":"missing","http":404,"bytes":1569},"home":{"status":"ok","http":200,"url":"https://www.google.gp/","html_tokens":21374,"markdown_tokens":32,"pruned_tokens":32}},
{"rank":8041,"domain":"jsdelivr.com","origin":"https://jsdelivr.com","robots":{"status":"ok","http":200,"url":"https://www.jsdelivr.com/robots.txt","bytes":127,"groups":1},"bots":{"GPTBot":["allowed","*",0,1],"ChatGPT-User":["allowed","*",0,1],"OAI-SearchBot":["allowed","*",0,1],"ClaudeBot":["allowed","*",0,1],"Claude-User":["allowed","*",0,1],"Claude-SearchBot":["allowed","*",0,1],"anthropic-ai":["allowed","*",0,1],"PerplexityBot":["allowed","*",0,1],"Perplexity-User":["allowed","*",0,1],"Google-Extended":["allowed","*",0,1],"Applebot-Extended":["allowed","*",0,1],"CCBot":["allowed","*",0,1],"Bytespider":["allowed","*",0,1],"Amazonbot":["allowed","*",0,1],"meta-externalagent":["allowed","*",0,1],"cohere-ai":["allowed","*",0,1],"Diffbot":["allowed","*",0,1],"ContextFetchBot":["allowed","*",0,1]},"llms":{"status":"missing","http":404,"bytes":11599},"home":{"status":"ok","http":200,"url":"https://www.jsdelivr.com/","html_tokens":12207,"markdown_tokens":104,"pruned_tokens":92}},
{"rank":8042,"domain":"ntu.ac.uk","origin":"https://ntu.ac.uk","robots":{"status":"refused","http":403,"url":"https://www.ntu.ac.uk/robots.txt","bytes":5551,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":8043,"domain":"furocare.online","origin":"https://furocare.online","robots":{"status":"ok","http":200,"url":"https://furocare.online/robots.txt","bytes":76,"groups":1},"bots":{"GPTBot":["partial","*",1,0],"ChatGPT-User":["partial","*",1,0],"OAI-SearchBot":["partial","*",1,0],"ClaudeBot":["partial","*",1,0],"Claude-User":["partial","*",1,0],"Claude-SearchBot":["partial","*",1,0],"anthropic-ai":["partial","*",1,0],"PerplexityBot":["partial","*",1,0],"Perplexity-User":["partial","*",1,0],"Google-Extended":["partial","*",1,0],"Applebot-Extended":["partial","*",1,0],"CCBot":["partial","*",1,0],"Bytespider":["partial","*",1,0],"Amazonbot":["partial","*",1,0],"meta-externalagent":["partial","*",1,0],"cohere-ai":["partial","*",1,0],"Diffbot":["partial","*",1,0],"ContextFetchBot":["partial","*",1,0]},"llms":{"status":"missing","http":404,"bytes":644},"home":{"status":"ok","http":200,"url":"https://furocare.online/","html_tokens":21126,"markdown_tokens":1571,"pruned_tokens":156}},
{"rank":8044,"domain":"uni-rostock.de","origin":"https://uni-rostock.de","robots":{"status":"html","http":200,"url":"https://www.uni-rostock.de","bytes":122399,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":8045,"domain":"soutugo.com","origin":"http://soutugo.com","robots":{"status":"missing","http":404,"url":"http://soutugo.com/robots.txt","bytes":3422,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":3418},"home":{"status":"ok","http":200,"url":"http://soutugo.com/","html_tokens":3510,"markdown_tokens":22,"pruned_tokens":22}},
{"rank":8046,"domain":"sho.com","origin":"https://sho.com","robots":{"status":"ok","http":200,"url":"https://www.sho.com/robots.txt","bytes":299,"groups":3},"bots":{"GPTBot":["partial","*",8,0],"ChatGPT-User":["partial","*",8,0],"OAI-SearchBot":["partial","*",8,0],"ClaudeBot":["partial","*",8,0],"Claude-User":["partial","*",8,0],"Claude-SearchBot":["partial","*",8,0],"anthropic-ai":["partial","*",8,0],"PerplexityBot":["partial","*",8,0],"Perplexity-User":["partial","*",8,0],"Google-Extended":["partial","*",8,0],"Applebot-Extended":["partial","*",8,0],"CCBot":["partial","*",8,0],"Bytespider":["partial","*",8,0],"Amazonbot":["partial","*",8,0],"meta-externalagent":["partial","*",8,0],"cohere-ai":["partial","*",8,0],"Diffbot":["partial","*",8,0],"ContextFetchBot":["partial","*",8,0]},"llms":{"status":"disallowed","error":"robots.txt of www.paramountpluswithshowtime.com not readable (html)"},"home":{"status":"disallowed","error":"robots.txt of www.paramountpluswithshowtime.com not readable (html)"}},
{"rank":8047,"domain":"malegislature.gov","origin":"https://malegislature.gov","robots":{"status":"ok","http":200,"url":"https://malegislature.gov/robots.txt","bytes":27,"groups":1},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"missing","http":404,"bytes":68268},"home":{"status":"ok","http":200,"url":"https://malegislature.gov/","html_tokens":26489,"markdown_tokens":215,"pruned_tokens":9}},
{"rank":8048,"domain":"navarra.es","origin":"https://navarra.es","robots":{"status":"timeout","http":null,"url":"https://navarra.es/robots.txt","bytes":0,"groups":0,"error":"ConnectTimeout"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":8049,"domain":"google.co.mz","origin":"https://google.co.mz","robots":{"status":"ok","http":200,"url":"https://www.google.co.mz/robots.txt","bytes":6624,"groups":4},"bots":{"GPTBot":["partial","*",169,64],"ChatGPT-User":["partial","*",169,64],"OAI-SearchBot":["partial","*",169,64],"ClaudeBot":["partial","*",169,64],"Claude-User":["partial","*",169,64],"Claude-SearchBot":["partial","*",169,64],"anthropic-ai":["partial","*",169,64],"PerplexityBot":["partial","*",169,64],"Perplexity-User":["partial","*",169,64],"Google-Extended":["partial","*",169,64],"Applebot-Extended":["partial","*",169,64],"CCBot":["partial","*",169,64],"Bytespider":["partial","*",169,64],"Amazonbot":["partial","*",169,64],"meta-externalagent":["partial","*",169,64],"cohere-ai":["partial","*",169,64],"Diffbot":["partial","*",169,64],"ContextFetchBot":["partial","*",169,64]},"llms":{"status":"missing","http":404,"bytes":1569},"home":{"status":"ok","http":200,"url":"https://www.google.com/","html_tokens":21346,"markdown_tokens":30,"pruned_tokens":30}},
{"rank":8050,"domain":"cdt.org","origin":"https://cdt.org","robots":{"status":"refused","http":403,"url":"https://cdt.org/robots.txt","bytes":5187,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":8051,"domain":"castrol.com","origin":"https://castrol.com","robots":{"status":"ok","http":200,"url":"https://www.castrol.com/robots.txt","bytes":3550,"groups":1},"bots":{"GPTBot":["partial","*",10,30],"ChatGPT-User":["partial","*",10,30],"OAI-SearchBot":["partial","*",10,30],"ClaudeBot":["partial","*",10,30],"Claude-User":["partial","*",10,30],"Claude-SearchBot":["partial","*",10,30],"anthropic-ai":["partial","*",10,30],"PerplexityBot":["partial","*",10,30],"Perplexity-User":["partial","*",10,30],"Google-Extended":["partial","*",10,30],"Applebot-Extended":["partial","*",10,30],"CCBot":["partial","*",10,30],"Bytespider":["partial","*",10,30],"Amazonbot":["partial","*",10,30],"meta-externalagent":["partial","*",10,30],"cohere-ai":["partial","*",10,30],"Diffbot":["partial","*",10,30],"ContextFetchBot":["partial","*",10,30]},"llms":{"status":"missing","http":404,"bytes":15798},"home":{"status":"ok","http":200,"url":"https://www.castrol.com/","html_tokens":36450,"markdown_tokens":108,"pruned_tokens":108}},
{"rank":8052,"domain":"hbjqlaw.com","origin":"http://hbjqlaw.com","robots":{"status":"missing","http":404,"url":"http://hbjqlaw.com/robots.txt","bytes":5077,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":5073},"home":{"status":"ok","http":200,"url":"http://hbjqlaw.com/","html_tokens":71,"markdown_tokens":0,"pruned_tokens":0}},
{"rank":8053,"domain":"aixin119.cn","origin":"https://www.aixin119.cn","robots":{"status":"missing","http":404,"url":"https://www.aixin119.cn/robots.txt","bytes":1245,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":1245},"home":{"status":"ok","http":200,"url":"https://www.aixin119.cn/","html_tokens":12566,"markdown_tokens":225,"pruned_tokens":78}},
{"rank":8054,"domain":"scispace.com","origin":"https://scispace.com","robots":{"status":"ok","http":200,"url":"https://scispace.com/robots.txt","bytes":6400,"groups":12},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["partial","*",53,1],"OAI-SearchBot":["partial","*",53,1],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",53,1],"Claude-SearchBot":["partial","*",53,1],"anthropic-ai":["partial","*",53,1],"PerplexityBot":["partial","*",53,1],"Perplexity-User":["partial","*",53,1],"Google-Extended":["partial","*",53,1],"Applebot-Extended":["partial","*",53,1],"CCBot":["partial","*",53,1],"Bytespider":["partial","*",53,1],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["partial","*",53,1],"cohere-ai":["partial","*",53,1],"Diffbot":["partial","*",53,1],"ContextFetchBot":["partial","*",53,1]},"llms":{"status":"present","http":200,"bytes":5171},"home":{"status":"http_error","http":403,"url":"https://scispace.com/"}},
{"rank":8055,"domain":"book-of-ra-fixed.com","origin":"https://book-of-ra-fixed.com","robots":{"status":"ok","http":200,"url":"https://book-of-ra-fixed.com/robots.txt","bytes":545,"groups":1},"bots":{"GPTBot":["partial","*",19,0],"ChatGPT-User":["partial","*",19,0],"OAI-SearchBot":["partial","*",19,0],"ClaudeBot":["partial","*",19,0],"Claude-User":["partial","*",19,0],"Claude-SearchBot":["partial","*",19,0],"anthropic-ai":["partial","*",19,0],"PerplexityBot":["partial","*",19,0],"Perplexity-User":["partial","*",19,0],"Google-Extended":["partial","*",19,0],"Applebot-Extended":["partial","*",19,0],"CCBot":["partial","*",19,0],"Bytespider":["partial","*",19,0],"Amazonbot":["partial","*",19,0],"meta-externalagent":["partial","*",19,0],"cohere-ai":["partial","*",19,0],"Diffbot":["partial","*",19,0],"ContextFetchBot":["partial","*",19,0]},"llms":{"status":"missing","http":404,"bytes":19358},"home":{"status":"ok","http":200,"url":"https://book-of-ra-fixed.com/","html_tokens":15910,"markdown_tokens":895,"pruned_tokens":741}},
{"rank":8056,"domain":"finnair.com","origin":"https://finnair.com","robots":{"status":"ok","http":200,"url":"https://www.finnair.com/robots.txt","bytes":5081,"groups":5},"bots":{"GPTBot":["partial","*",7,7],"ChatGPT-User":["partial","*",7,7],"OAI-SearchBot":["partial","*",7,7],"ClaudeBot":["partial","*",7,7],"Claude-User":["partial","*",7,7],"Claude-SearchBot":["partial","*",7,7],"anthropic-ai":["partial","*",7,7],"PerplexityBot":["partial","*",7,7],"Perplexity-User":["partial","*",7,7],"Google-Extended":["partial","*",7,7],"Applebot-Extended":["partial","*",7,7],"CCBot":["partial","*",7,7],"Bytespider":["partial","*",7,7],"Amazonbot":["partial","*",7,7],"meta-externalagent":["partial","*",7,7],"cohere-ai":["partial","*",7,7],"Diffbot":["partial","*",7,7],"ContextFetchBot":["partial","*",7,7]},"llms":{"status":"html","http":200,"bytes":742212},"home":{"status":"ok","http":200,"url":"https://www.finnair.com/","html_tokens":186822,"markdown_tokens":169,"pruned_tokens":169}},
{"rank":8057,"domain":"google.cg","origin":"https://google.cg","robots":{"status":"ok","http":200,"url":"https://www.google.cg/robots.txt","bytes":6624,"groups":4},"bots":{"GPTBot":["partial","*",169,64],"ChatGPT-User":["partial","*",169,64],"OAI-SearchBot":["partial","*",169,64],"ClaudeBot":["partial","*",169,64],"Claude-User":["partial","*",169,64],"Claude-SearchBot":["partial","*",169,64],"anthropic-ai":["partial","*",169,64],"PerplexityBot":["partial","*",169,64],"Perplexity-User":["partial","*",169,64],"Google-Extended":["partial","*",169,64],"Applebot-Extended":["partial","*",169,64],"CCBot":["partial","*",169,64],"Bytespider":["partial","*",169,64],"Amazonbot":["partial","*",169,64],"meta-externalagent":["partial","*",169,64],"cohere-ai":["partial","*",169,64],"Diffbot":["partial","*",169,64],"ContextFetchBot":["partial","*",169,64]},"llms":{"status":"missing","http":404,"bytes":1569},"home":{"status":"ok","http":200,"url":"https://www.google.com/","html_tokens":21339,"markdown_tokens":30,"pruned_tokens":30}},
{"rank":8058,"domain":"yfmac.com","origin":"https://www.yfmac.com","robots":{"status":"ok","http":200,"url":"https://www.yfmac.com/robots.txt","bytes":252,"groups":1},"bots":{"GPTBot":["partial","*",12,0],"ChatGPT-User":["partial","*",12,0],"OAI-SearchBot":["partial","*",12,0],"ClaudeBot":["partial","*",12,0],"Claude-User":["partial","*",12,0],"Claude-SearchBot":["partial","*",12,0],"anthropic-ai":["partial","*",12,0],"PerplexityBot":["partial","*",12,0],"Perplexity-User":["partial","*",12,0],"Google-Extended":["partial","*",12,0],"Applebot-Extended":["partial","*",12,0],"CCBot":["partial","*",12,0],"Bytespider":["partial","*",12,0],"Amazonbot":["partial","*",12,0],"meta-externalagent":["partial","*",12,0],"cohere-ai":["partial","*",12,0],"Diffbot":["partial","*",12,0],"ContextFetchBot":["partial","*",12,0]},"llms":{"status":"missing","http":404,"bytes":1231},"home":{"status":"ok","http":200,"url":"https://www.yfmac.com/","html_tokens":8807,"markdown_tokens":2,"pruned_tokens":2}},
{"rank":8059,"domain":"ewuye.vip","origin":"https://ewuye.vip","robots":{"status":"missing","http":404,"url":"https://ewuye.vip/robots.txt","bytes":4711,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":4707},"home":{"status":"ok","http":200,"url":"https://ewuye.vip/","html_tokens":3820,"markdown_tokens":49,"pruned_tokens":49}},
{"rank":8060,"domain":"aaaai.org","origin":"https://aaaai.org","robots":{"status":"ok","http":200,"url":"http://www.aaaai.org/robots.txt","bytes":120,"groups":1},"bots":{"GPTBot":["partial","*",2,0],"ChatGPT-User":["partial","*",2,0],"OAI-SearchBot":["partial","*",2,0],"ClaudeBot":["partial","*",2,0],"Claude-User":["partial","*",2,0],"Claude-SearchBot":["partial","*",2,0],"anthropic-ai":["partial","*",2,0],"PerplexityBot":["partial","*",2,0],"Perplexity-User":["partial","*",2,0],"Google-Extended":["partial","*",2,0],"Applebot-Extended":["partial","*",2,0],"CCBot":["partial","*",2,0],"Bytespider":["partial","*",2,0],"Amazonbot":["partial","*",2,0],"meta-externalagent":["partial","*",2,0],"cohere-ai":["partial","*",2,0],"Diffbot":["partial","*",2,0],"ContextFetchBot":["partial","*",2,0]},"llms":{"status":"missing","http":404,"bytes":70971},"home":{"status":"error","error":"bad compressed body: Error -3 while decompressing data: incorrect header check"}},
{"rank":8061,"domain":"google.so","origin":"https://google.so","robots":{"status":"ok","http":200,"url":"https://www.google.so/robots.txt","bytes":6624,"groups":4},"bots":{"GPTBot":["partial","*",169,64],"ChatGPT-User":["partial","*",169,64],"OAI-SearchBot":["partial","*",169,64],"ClaudeBot":["partial","*",169,64],"Claude-User":["partial","*",169,64],"Claude-SearchBot":["partial","*",169,64],"anthropic-ai":["partial","*",169,64],"PerplexityBot":["partial","*",169,64],"Perplexity-User":["partial","*",169,64],"Google-Extended":["partial","*",169,64],"Applebot-Extended":["partial","*",169,64],"CCBot":["partial","*",169,64],"Bytespider":["partial","*",169,64],"Amazonbot":["partial","*",169,64],"meta-externalagent":["partial","*",169,64],"cohere-ai":["partial","*",169,64],"Diffbot":["partial","*",169,64],"ContextFetchBot":["partial","*",169,64]},"llms":{"status":"missing","http":404,"bytes":1569},"home":{"status":"ok","http":200,"url":"https://www.google.com/","html_tokens":21289,"markdown_tokens":30,"pruned_tokens":30}},
{"rank":8062,"domain":"ghgprotocol.org","origin":"https://ghgprotocol.org","robots":{"status":"ok","http":200,"url":"https://ghgprotocol.org/robots.txt","bytes":2005,"groups":1},"bots":{"GPTBot":["partial","*",33,18],"ChatGPT-User":["partial","*",33,18],"OAI-SearchBot":["partial","*",33,18],"ClaudeBot":["partial","*",33,18],"Claude-User":["partial","*",33,18],"Claude-SearchBot":["partial","*",33,18],"anthropic-ai":["partial","*",33,18],"PerplexityBot":["partial","*",33,18],"Perplexity-User":["partial","*",33,18],"Google-Extended":["partial","*",33,18],"Applebot-Extended":["partial","*",33,18],"CCBot":["partial","*",33,18],"Bytespider":["partial","*",33,18],"Amazonbot":["partial","*",33,18],"meta-externalagent":["partial","*",33,18],"cohere-ai":["partial","*",33,18],"Diffbot":["partial","*",33,18],"ContextFetchBot":["partial","*",33,18]},"llms":{"status":"missing","http":404,"bytes":556},"home":{"status":"ok","http":200,"url":"https://ghgprotocol.org/","html_tokens":14002,"markdown_tokens":183,"pruned_tokens":105}},
{"rank":8063,"domain":"waldenu.edu","origin":"https://waldenu.edu","robots":{"status":"ok","http":200,"url":"https://www.waldenu.edu/robots.txt","bytes":2562,"groups":6},"bots":{"GPTBot":["partial","*",26,18],"ChatGPT-User":["partial","*",26,18],"OAI-SearchBot":["partial","*",26,18],"ClaudeBot":["partial","*",26,18],"Claude-User":["partial","*",26,18],"Claude-SearchBot":["partial","*",26,18],"anthropic-ai":["partial","*",26,18],"PerplexityBot":["partial","*",26,18],"Perplexity-User":["partial","*",26,18],"Google-Extended":["partial","*",26,18],"Applebot-Extended":["partial","*",26,18],"CCBot":["partial","*",26,18],"Bytespider":["partial","*",26,18],"Amazonbot":["partial","*",26,18],"meta-externalagent":["partial","*",26,18],"cohere-ai":["partial","*",26,18],"Diffbot":["partial","*",26,18],"ContextFetchBot":["partial","*",26,18]},"llms":{"status":"missing","http":404,"bytes":189},"home":{"status":"ok","http":200,"url":"https://www.waldenu.edu/","html_tokens":192294,"markdown_tokens":2695,"pruned_tokens":369}},
{"rank":8064,"domain":"chordie.com","origin":"https://chordie.com","robots":{"status":"ok","http":200,"url":"https://www.chordie.com/robots.txt","bytes":266,"groups":5},"bots":{"GPTBot":["partial","*",4,0],"ChatGPT-User":["partial","*",4,0],"OAI-SearchBot":["partial","*",4,0],"ClaudeBot":["partial","*",4,0],"Claude-User":["partial","*",4,0],"Claude-SearchBot":["partial","*",4,0],"anthropic-ai":["partial","*",4,0],"PerplexityBot":["partial","*",4,0],"Perplexity-User":["partial","*",4,0],"Google-Extended":["partial","*",4,0],"Applebot-Extended":["partial","*",4,0],"CCBot":["partial","*",4,0],"Bytespider":["partial","*",4,0],"Amazonbot":["partial","*",4,0],"meta-externalagent":["partial","*",4,0],"cohere-ai":["partial","*",4,0],"Diffbot":["partial","*",4,0],"ContextFetchBot":["partial","*",4,0]},"llms":{"status":"missing","http":404,"bytes":19372},"home":{"status":"ok","http":200,"url":"https://www.chordie.com/","html_tokens":10959,"markdown_tokens":259,"pruned_tokens":248}},
{"rank":8065,"domain":"google.bj","origin":"https://google.bj","robots":{"status":"ok","http":200,"url":"https://www.google.bj/robots.txt","bytes":6624,"groups":4},"bots":{"GPTBot":["partial","*",169,64],"ChatGPT-User":["partial","*",169,64],"OAI-SearchBot":["partial","*",169,64],"ClaudeBot":["partial","*",169,64],"Claude-User":["partial","*",169,64],"Claude-SearchBot":["partial","*",169,64],"anthropic-ai":["partial","*",169,64],"PerplexityBot":["partial","*",169,64],"Perplexity-User":["partial","*",169,64],"Google-Extended":["partial","*",169,64],"Applebot-Extended":["partial","*",169,64],"CCBot":["partial","*",169,64],"Bytespider":["partial","*",169,64],"Amazonbot":["partial","*",169,64],"meta-externalagent":["partial","*",169,64],"cohere-ai":["partial","*",169,64],"Diffbot":["partial","*",169,64],"ContextFetchBot":["partial","*",169,64]},"llms":{"status":"missing","http":404,"bytes":1569},"home":{"status":"ok","http":200,"url":"https://www.google.com/","html_tokens":21333,"markdown_tokens":30,"pruned_tokens":30}},
{"rank":8066,"domain":"surl.li","origin":"https://surl.li","robots":{"status":"ok","http":200,"url":"https://surl.li/robots.txt","bytes":24,"groups":1},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"missing","http":404,"bytes":14133},"home":{"status":"ok","http":200,"url":"https://surl.li/en","html_tokens":19786,"markdown_tokens":359,"pruned_tokens":258}},
{"rank":8067,"domain":"glosbe.com","origin":"https://glosbe.com","robots":{"status":"ok","http":200,"url":"https://glosbe.com/robots.txt","bytes":528,"groups":10},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["partial","*",8,1],"OAI-SearchBot":["partial","*",8,1],"ClaudeBot":["partial","*",8,1],"Claude-User":["partial","*",8,1],"Claude-SearchBot":["partial","*",8,1],"anthropic-ai":["partial","*",8,1],"PerplexityBot":["partial","*",8,1],"Perplexity-User":["partial","*",8,1],"Google-Extended":["partial","*",8,1],"Applebot-Extended":["partial","*",8,1],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["partial","*",8,1],"meta-externalagent":["partial","*",8,1],"cohere-ai":["partial","*",8,1],"Diffbot":["partial","*",8,1],"ContextFetchBot":["partial","*",8,1]},"llms":{"status":"missing","http":404,"bytes":49135},"home":{"status":"ok","http":200,"url":"https://glosbe.com/","html_tokens":17093,"markdown_tokens":146,"pruned_tokens":146}},
{"rank":8068,"domain":"usherbrooke.ca","origin":"https://usherbrooke.ca","robots":{"status":"refused","http":403,"url":"https://www.usherbrooke.ca/robots.txt","bytes":229,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":8069,"domain":"brooklynmuseum.org","origin":"https://brooklynmuseum.org","robots":{"status":"refused","http":429,"url":"https://brooklynmuseum.org/robots.txt","bytes":34224,"groups":0,"error":"HTTP 429"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":8070,"domain":"harborfreight.com","origin":"https://harborfreight.com","robots":{"status":"ok","http":200,"url":"https://www.harborfreight.com/robots.txt","bytes":2684,"groups":16},"bots":{"GPTBot":["allowed","named",0,1],"ChatGPT-User":["allowed","named",0,1],"OAI-SearchBot":["allowed","named",0,1],"ClaudeBot":["allowed","named",0,1],"Claude-User":["partial","*",17,0],"Claude-SearchBot":["allowed","named",0,1],"anthropic-ai":["partial","*",17,0],"PerplexityBot":["allowed","named",0,1],"Perplexity-User":["partial","*",17,0],"Google-Extended":["partial","*",17,0],"Applebot-Extended":["partial","*",17,0],"CCBot":["allowed","named",0,1],"Bytespider":["partial","*",17,0],"Amazonbot":["partial","*",17,0],"meta-externalagent":["partial","*",17,0],"cohere-ai":["partial","*",17,0],"Diffbot":["partial","*",17,0],"ContextFetchBot":["partial","*",17,0]},"llms":{"status":"http_error","http":403,"bytes":5645},"home":{"status":"http_error","http":403,"url":"https://harborfreight.com/"}},
{"rank":8071,"domain":"challonge.com","origin":"https://challonge.com","robots":{"status":"ok","http":200,"url":"https://challonge.com/robots.txt","bytes":217,"groups":4},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":2122},"home":{"status":"http_error","http":403,"url":"https://challonge.com/"}},
{"rank":8072,"domain":"uni.lu","origin":"https://www.uni.lu","robots":{"status":"html","http":202,"url":"https://www.uni.lu/en/","bytes":2087,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":8073,"domain":"sammobile.com","origin":"https://sammobile.com","robots":{"status":"ok","http":200,"url":"https://www.sammobile.com/robots.txt","bytes":991,"groups":4},"bots":{"GPTBot":["partial","named",6,1],"ChatGPT-User":["partial","named",6,1],"OAI-SearchBot":["partial","named",6,1],"ClaudeBot":["partial","named",6,1],"Claude-User":["partial","named",6,1],"Claude-SearchBot":["partial","named",6,1],"anthropic-ai":["partial","*",6,0],"PerplexityBot":["partial","named",6,1],"Perplexity-User":["partial","named",6,1],"Google-Extended":["partial","named",6,1],"Applebot-Extended":["partial","*",6,0],"CCBot":["partial","*",6,0],"Bytespider":["partial","*",6,0],"Amazonbot":["partial","*",6,0],"meta-externalagent":["partial","named",1,0],"cohere-ai":["partial","*",6,0],"Diffbot":["partial","*",6,0],"ContextFetchBot":["partial","*",6,0]},"llms":{"status":"present","http":200,"bytes":1692},"home":{"status":"ok","http":200,"url":"https://www.sammobile.com/","html_tokens":81859,"markdown_tokens":46,"pruned_tokens":43}},
{"rank":8074,"domain":"canadahelps.org","origin":"https://canadahelps.org","robots":{"status":"ok","http":200,"url":"https://www.canadahelps.org/robots.txt","bytes":452,"groups":1},"bots":{"GPTBot":["partial","*",13,0],"ChatGPT-User":["partial","*",13,0],"OAI-SearchBot":["partial","*",13,0],"ClaudeBot":["partial","*",13,0],"Claude-User":["partial","*",13,0],"Claude-SearchBot":["partial","*",13,0],"anthropic-ai":["partial","*",13,0],"PerplexityBot":["partial","*",13,0],"Perplexity-User":["partial","*",13,0],"Google-Extended":["partial","*",13,0],"Applebot-Extended":["partial","*",13,0],"CCBot":["partial","*",13,0],"Bytespider":["partial","*",13,0],"Amazonbot":["partial","*",13,0],"meta-externalagent":["partial","*",13,0],"cohere-ai":["partial","*",13,0],"Diffbot":["partial","*",13,0],"ContextFetchBot":["partial","*",13,0]},"llms":{"status":"missing","http":404,"bytes":100865},"home":{"status":"ok","http":200,"url":"https://www.canadahelps.org/en/","html_tokens":31921,"markdown_tokens":70,"pruned_tokens":70}},
{"rank":8075,"domain":"bloombergquint.com","origin":"https://bloombergquint.com","robots":{"status":"unreachable","http":null,"url":"https://bloombergquint.com/robots.txt","bytes":0,"groups":0,"error":"cannot resolve bloombergquint.com"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":8076,"domain":"payoneer.com","origin":"https://payoneer.com","robots":{"status":"ok","http":200,"url":"https://www.payoneer.com/robots.txt","bytes":67,"groups":1},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"present","http":200,"bytes":22731},"home":{"status":"ok","http":200,"url":"https://www.payoneer.com/","html_tokens":116839,"markdown_tokens":576,"pruned_tokens":38}},
{"rank":8077,"domain":"cpr.org","origin":"https://cpr.org","robots":{"status":"ok","http":200,"url":"https://www.cpr.org/robots.txt","bytes":2175,"groups":14},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["allowed","named",0,1],"OAI-SearchBot":["partial","*",10,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",10,0],"Claude-SearchBot":["partial","*",10,0],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["partial","*",10,0],"Perplexity-User":["partial","*",10,0],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["partial","*",10,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["partial","*",10,0],"meta-externalagent":["partial","*",10,0],"cohere-ai":["partial","*",10,0],"Diffbot":["partial","*",10,0],"ContextFetchBot":["partial","*",10,0]},"llms":{"status":"missing","http":404,"bytes":110683},"home":{"status":"http_error","http":403,"url":"https://www.cpr.org/"}},
{"rank":8078,"domain":"google.mw","origin":"https://google.mw","robots":{"status":"ok","http":200,"url":"https://www.google.mw/robots.txt","bytes":6624,"groups":4},"bots":{"GPTBot":["partial","*",169,64],"ChatGPT-User":["partial","*",169,64],"OAI-SearchBot":["partial","*",169,64],"ClaudeBot":["partial","*",169,64],"Claude-User":["partial","*",169,64],"Claude-SearchBot":["partial","*",169,64],"anthropic-ai":["partial","*",169,64],"PerplexityBot":["partial","*",169,64],"Perplexity-User":["partial","*",169,64],"Google-Extended":["partial","*",169,64],"Applebot-Extended":["partial","*",169,64],"CCBot":["partial","*",169,64],"Bytespider":["partial","*",169,64],"Amazonbot":["partial","*",169,64],"meta-externalagent":["partial","*",169,64],"cohere-ai":["partial","*",169,64],"Diffbot":["partial","*",169,64],"ContextFetchBot":["partial","*",169,64]},"llms":{"status":"missing","http":404,"bytes":1569},"home":{"status":"ok","http":200,"url":"https://www.google.com/","html_tokens":21457,"markdown_tokens":30,"pruned_tokens":30}},
{"rank":8079,"domain":"worldofwarcraft.com","origin":"https://worldofwarcraft.com","robots":{"status":"ok","http":200,"url":"https://worldofwarcraft.blizzard.com/robots.txt","bytes":346,"groups":1},"bots":{"GPTBot":["partial","*",12,0],"ChatGPT-User":["partial","*",12,0],"OAI-SearchBot":["partial","*",12,0],"ClaudeBot":["partial","*",12,0],"Claude-User":["partial","*",12,0],"Claude-SearchBot":["partial","*",12,0],"anthropic-ai":["partial","*",12,0],"PerplexityBot":["partial","*",12,0],"Perplexity-User":["partial","*",12,0],"Google-Extended":["partial","*",12,0],"Applebot-Extended":["partial","*",12,0],"CCBot":["partial","*",12,0],"Bytespider":["partial","*",12,0],"Amazonbot":["partial","*",12,0],"meta-externalagent":["partial","*",12,0],"cohere-ai":["partial","*",12,0],"Diffbot":["partial","*",12,0],"ContextFetchBot":["partial","*",12,0]},"llms":{"status":"missing","http":404,"bytes":17586},"home":{"status":"ok","http":200,"url":"https://worldofwarcraft.blizzard.com/en-us/","html_tokens":9934,"markdown_tokens":1165,"pruned_tokens":90}},
{"rank":8080,"domain":"peacealu.com","origin":"https://peacealu.com","robots":{"status":"ok","http":200,"url":"https://www.peacealu.com/robots.txt","bytes":824,"groups":20},"bots":{"GPTBot":["partial","*",1,0],"ChatGPT-User":["partial","*",1,0],"OAI-SearchBot":["partial","*",1,0],"ClaudeBot":["partial","*",1,0],"Claude-User":["partial","*",1,0],"Claude-SearchBot":["partial","*",1,0],"anthropic-ai":["partial","*",1,0],"PerplexityBot":["partial","*",1,0],"Perplexity-User":["partial","*",1,0],"Google-Extended":["partial","*",1,0],"Applebot-Extended":["partial","*",1,0],"CCBot":["partial","*",1,0],"Bytespider":["partial","*",1,0],"Amazonbot":["partial","*",1,0],"meta-externalagent":["partial","*",1,0],"cohere-ai":["partial","*",1,0],"Diffbot":["partial","*",1,0],"ContextFetchBot":["partial","*",1,0]},"llms":{"status":"missing","http":404,"bytes":1163},"home":{"status":"ok","http":200,"url":"https://www.peacealu.com/","html_tokens":8561,"markdown_tokens":45,"pruned_tokens":45}},
{"rank":8081,"domain":"nationalmssociety.org","origin":"https://nationalmssociety.org","robots":{"status":"ok","http":200,"url":"https://www.nationalmssociety.org/robots.txt","bytes":244,"groups":1},"bots":{"GPTBot":["partial","*",1,1],"ChatGPT-User":["partial","*",1,1],"OAI-SearchBot":["partial","*",1,1],"ClaudeBot":["partial","*",1,1],"Claude-User":["partial","*",1,1],"Claude-SearchBot":["partial","*",1,1],"anthropic-ai":["partial","*",1,1],"PerplexityBot":["partial","*",1,1],"Perplexity-User":["partial","*",1,1],"Google-Extended":["partial","*",1,1],"Applebot-Extended":["partial","*",1,1],"CCBot":["partial","*",1,1],"Bytespider":["partial","*",1,1],"Amazonbot":["partial","*",1,1],"meta-externalagent":["partial","*",1,1],"cohere-ai":["partial","*",1,1],"Diffbot":["partial","*",1,1],"ContextFetchBot":["partial","*",1,1]},"llms":{"status":"missing","http":404,"bytes":2307457},"home":{"status":"ok","http":200,"url":"https://www.nationalmssociety.org/","html_tokens":576860,"markdown_tokens":488,"pruned_tokens":157}},
{"rank":8082,"domain":"radio.com","origin":"https://radio.com","robots":{"status":"ok","http":200,"url":"https://radio.com/robots.txt","bytes":211,"groups":1},"bots":{"GPTBot":["partial","*",4,1],"ChatGPT-User":["partial","*",4,1],"OAI-SearchBot":["partial","*",4,1],"ClaudeBot":["partial","*",4,1],"Claude-User":["partial","*",4,1],"Claude-SearchBot":["partial","*",4,1],"anthropic-ai":["partial","*",4,1],"PerplexityBot":["partial","*",4,1],"Perplexity-User":["partial","*",4,1],"Google-Extended":["partial","*",4,1],"Applebot-Extended":["partial","*",4,1],"CCBot":["partial","*",4,1],"Bytespider":["partial","*",4,1],"Amazonbot":["partial","*",4,1],"meta-externalagent":["partial","*",4,1],"cohere-ai":["partial","*",4,1],"Diffbot":["partial","*",4,1],"ContextFetchBot":["partial","*",4,1]},"llms":{"status":"present","http":200,"bytes":1129},"home":{"status":"ok","http":200,"url":"https://radio.com/","html_tokens":9072,"markdown_tokens":606,"pruned_tokens":34}},
{"rank":8083,"domain":"amzn.com","origin":"https://amzn.com","robots":{"status":"ok","http":200,"url":"https://www.amazon.com/robots.txt","bytes":7947,"groups":101},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["blocked","named",1,0],"OAI-SearchBot":["blocked","named",1,0],"ClaudeBot":["blocked","named",2,0],"Claude-User":["blocked","named",1,0],"Claude-SearchBot":["blocked","named",1,0],"anthropic-ai":["partial","*",120,17],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["blocked","named",1,0],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["partial","*",120,17],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["partial","*",120,17],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",120,17]},"llms":{"status":"missing","http":404,"bytes":2296},"home":{"status":"ok","http":200,"url":"https://www.amazon.com/","html_tokens":155322,"markdown_tokens":1138,"pruned_tokens":163}},
{"rank":8084,"domain":"borda.ru","origin":"https://borda.ru","robots":{"status":"ok","http":200,"url":"https://borda.ru/robots.txt","bytes":0,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"other_text","http":200,"bytes":43},"home":{"status":"ok","http":200,"url":"https://borda.ru/","html_tokens":3343,"markdown_tokens":201,"pruned_tokens":141}},
{"rank":8085,"domain":"finance.people.com.cn","origin":"http://finance.people.com.cn","robots":{"status":"missing","http":404,"url":"http://finance.people.com.cn/robots.txt","bytes":155,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":155},"home":{"status":"ok","http":200,"url":"http://finance.people.com.cn/","html_tokens":18963,"markdown_tokens":89,"pruned_tokens":89}},
{"rank":8086,"domain":"byjus.com","origin":"https://byjus.com","robots":{"status":"ok","http":200,"url":"https://byjus.com/robots.txt","bytes":12505,"groups":1},"bots":{"GPTBot":["partial","*",44,0],"ChatGPT-User":["partial","*",44,0],"OAI-SearchBot":["partial","*",44,0],"ClaudeBot":["partial","*",44,0],"Claude-User":["partial","*",44,0],"Claude-SearchBot":["partial","*",44,0],"anthropic-ai":["partial","*",44,0],"PerplexityBot":["partial","*",44,0],"Perplexity-User":["partial","*",44,0],"Google-Extended":["partial","*",44,0],"Applebot-Extended":["partial","*",44,0],"CCBot":["partial","*",44,0],"Bytespider":["partial","*",44,0],"Amazonbot":["partial","*",44,0],"meta-externalagent":["partial","*",44,0],"cohere-ai":["partial","*",44,0],"Diffbot":["partial","*",44,0],"ContextFetchBot":["partial","*",44,0]},"llms":{"status":"missing","http":404,"bytes":315404},"home":{"status":"ok","http":200,"url":"https://byjus.com/","html_tokens":90757,"markdown_tokens":77,"pruned_tokens":72}},
{"rank":8087,"domain":"visitnorway.com","origin":"https://visitnorway.com","robots":{"status":"ok","http":200,"url":"https://www.visitnorway.com/robots.txt","bytes":73,"groups":1},"bots":{"GPTBot":["allowed","*",0,1],"ChatGPT-User":["allowed","*",0,1],"OAI-SearchBot":["allowed","*",0,1],"ClaudeBot":["allowed","*",0,1],"Claude-User":["allowed","*",0,1],"Claude-SearchBot":["allowed","*",0,1],"anthropic-ai":["allowed","*",0,1],"PerplexityBot":["allowed","*",0,1],"Perplexity-User":["allowed","*",0,1],"Google-Extended":["allowed","*",0,1],"Applebot-Extended":["allowed","*",0,1],"CCBot":["allowed","*",0,1],"Bytespider":["allowed","*",0,1],"Amazonbot":["allowed","*",0,1],"meta-externalagent":["allowed","*",0,1],"cohere-ai":["allowed","*",0,1],"Diffbot":["allowed","*",0,1],"ContextFetchBot":["allowed","*",0,1]},"llms":{"status":"missing","http":404,"bytes":102926},"home":{"status":"ok","http":200,"url":"https://www.visitnorway.com/","html_tokens":350681,"markdown_tokens":560,"pruned_tokens":119}},
{"rank":8088,"domain":"harvardmagazine.com","origin":"https://harvardmagazine.com","robots":{"status":"ok","http":200,"url":"https://harvardmagazine.com/robots.txt","bytes":3109,"groups":1},"bots":{"GPTBot":["partial","*",45,19],"ChatGPT-User":["partial","*",45,19],"OAI-SearchBot":["partial","*",45,19],"ClaudeBot":["partial","*",45,19],"Claude-User":["partial","*",45,19],"Claude-SearchBot":["partial","*",45,19],"anthropic-ai":["partial","*",45,19],"PerplexityBot":["partial","*",45,19],"Perplexity-User":["partial","*",45,19],"Google-Extended":["partial","*",45,19],"Applebot-Extended":["partial","*",45,19],"CCBot":["partial","*",45,19],"Bytespider":["partial","*",45,19],"Amazonbot":["partial","*",45,19],"meta-externalagent":["partial","*",45,19],"cohere-ai":["partial","*",45,19],"Diffbot":["partial","*",45,19],"ContextFetchBot":["partial","*",45,19]},"llms":{"status":"disallowed","error":"robots.txt of www.harvardmagazine.com not readable (refused)"},"home":{"status":"http_error","http":403,"url":"https://harvardmagazine.com/"}},
{"rank":8089,"domain":"adorama.com","origin":"https://adorama.com","robots":{"status":"ok","http":200,"url":"https://www.adorama.com/robots.txt","bytes":5556,"groups":3},"bots":{"GPTBot":["partial","*",114,31],"ChatGPT-User":["partial","*",114,31],"OAI-SearchBot":["partial","*",114,31],"ClaudeBot":["partial","*",114,31],"Claude-User":["partial","*",114,31],"Claude-SearchBot":["partial","*",114,31],"anthropic-ai":["partial","*",114,31],"PerplexityBot":["partial","*",114,31],"Perplexity-User":["partial","*",114,31],"Google-Extended":["partial","*",114,31],"Applebot-Extended":["partial","*",114,31],"CCBot":["partial","*",114,31],"Bytespider":["partial","*",114,31],"Amazonbot":["partial","*",114,31],"meta-externalagent":["partial","*",114,31],"cohere-ai":["partial","*",114,31],"Diffbot":["partial","*",114,31],"ContextFetchBot":["partial","*",114,31]},"llms":{"status":"http_error","http":403,"bytes":771},"home":{"status":"http_error","http":403,"url":"https://www.adorama.com/"}},
{"rank":8090,"domain":"onsetrx.com","origin":"https://onsetrx.com","robots":{"status":"ok","http":200,"url":"https://onsetrx.com/robots.txt","bytes":74,"groups":1},"bots":{"GPTBot":["partial","*",1,0],"ChatGPT-User":["partial","*",1,0],"OAI-SearchBot":["partial","*",1,0],"ClaudeBot":["partial","*",1,0],"Claude-User":["partial","*",1,0],"Claude-SearchBot":["partial","*",1,0],"anthropic-ai":["partial","*",1,0],"PerplexityBot":["partial","*",1,0],"Perplexity-User":["partial","*",1,0],"Google-Extended":["partial","*",1,0],"Applebot-Extended":["partial","*",1,0],"CCBot":["partial","*",1,0],"Bytespider":["partial","*",1,0],"Amazonbot":["partial","*",1,0],"meta-externalagent":["partial","*",1,0],"cohere-ai":["partial","*",1,0],"Diffbot":["partial","*",1,0],"ContextFetchBot":["partial","*",1,0]},"llms":{"status":"missing","http":404,"bytes":640},"home":{"status":"ok","http":200,"url":"https://onsetrx.com/","html_tokens":7929,"markdown_tokens":2742,"pruned_tokens":101}},
{"rank":8091,"domain":"mr-bet.ca","origin":"https://mr-bet.ca","robots":{"status":"ok","http":200,"url":"https://mr-bet.ca/robots.txt","bytes":524,"groups":1},"bots":{"GPTBot":["partial","*",19,0],"ChatGPT-User":["partial","*",19,0],"OAI-SearchBot":["partial","*",19,0],"ClaudeBot":["partial","*",19,0],"Claude-User":["partial","*",19,0],"Claude-SearchBot":["partial","*",19,0],"anthropic-ai":["partial","*",19,0],"PerplexityBot":["partial","*",19,0],"Perplexity-User":["partial","*",19,0],"Google-Extended":["partial","*",19,0],"Applebot-Extended":["partial","*",19,0],"CCBot":["partial","*",19,0],"Bytespider":["partial","*",19,0],"Amazonbot":["partial","*",19,0],"meta-externalagent":["partial","*",19,0],"cohere-ai":["partial","*",19,0],"Diffbot":["partial","*",19,0],"ContextFetchBot":["partial","*",19,0]},"llms":{"status":"missing","http":404,"bytes":21101},"home":{"status":"ok","http":200,"url":"https://mr-bet.ca/","html_tokens":9246,"markdown_tokens":1278,"pruned_tokens":464}},
{"rank":8092,"domain":"backstage.com","origin":"https://backstage.com","robots":{"status":"refused","http":403,"url":"https://backstage.com/robots.txt","bytes":4546,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":8093,"domain":"virgin.net","origin":"https://virgin.net","robots":{"status":"unreachable","http":null,"url":"https://virgin.net/robots.txt","bytes":0,"groups":0,"error":"cannot resolve virgin.net"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":8094,"domain":"uff.br","origin":"https://uff.br","robots":{"status":"ok","http":200,"url":"https://uff.br/robots.txt","bytes":125,"groups":1},"bots":{"GPTBot":["partial","*",4,1],"ChatGPT-User":["partial","*",4,1],"OAI-SearchBot":["partial","*",4,1],"ClaudeBot":["partial","*",4,1],"Claude-User":["partial","*",4,1],"Claude-SearchBot":["partial","*",4,1],"anthropic-ai":["partial","*",4,1],"PerplexityBot":["partial","*",4,1],"Perplexity-User":["partial","*",4,1],"Google-Extended":["partial","*",4,1],"Applebot-Extended":["partial","*",4,1],"CCBot":["partial","*",4,1],"Bytespider":["partial","*",4,1],"Amazonbot":["partial","*",4,1],"meta-externalagent":["partial","*",4,1],"cohere-ai":["partial","*",4,1],"Diffbot":["partial","*",4,1],"ContextFetchBot":["partial","*",4,1]},"llms":{"status":"missing","http":404,"bytes":542929},"home":{"status":"ok","http":200,"url":"https://www.uff.br/","html_tokens":163448,"markdown_tokens":11,"pruned_tokens":11}},
{"rank":8095,"domain":"tv2.no","origin":"https://tv2.no","robots":{"status":"ok","http":200,"url":"https://www.tv2.no/robots.txt","bytes":2067,"groups":2},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["blocked","named",1,0],"OAI-SearchBot":["blocked","named",1,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",18,0],"Claude-SearchBot":["blocked","named",1,0],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["blocked","named",1,0],"Google-Extended":["partial","*",18,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",18,0]},"llms":{"status":"missing","http":404,"bytes":0},"home":{"status":"ok","http":200,"url":"https://www.tv2.no/","html_tokens":127767,"markdown_tokens":117,"pruned_tokens":116}},
{"rank":8096,"domain":"theaa.com","origin":"https://theaa.com","robots":{"status":"ok","http":200,"url":"https://www.theaa.com/robots.txt","bytes":660,"groups":1},"bots":{"GPTBot":["partial","*",15,0],"ChatGPT-User":["partial","*",15,0],"OAI-SearchBot":["partial","*",15,0],"ClaudeBot":["partial","*",15,0],"Claude-User":["partial","*",15,0],"Claude-SearchBot":["partial","*",15,0],"anthropic-ai":["partial","*",15,0],"PerplexityBot":["partial","*",15,0],"Perplexity-User":["partial","*",15,0],"Google-Extended":["partial","*",15,0],"Applebot-Extended":["partial","*",15,0],"CCBot":["partial","*",15,0],"Bytespider":["partial","*",15,0],"Amazonbot":["partial","*",15,0],"meta-externalagent":["partial","*",15,0],"cohere-ai":["partial","*",15,0],"Diffbot":["partial","*",15,0],"ContextFetchBot":["partial","*",15,0]},"llms":{"status":"missing","http":404,"bytes":2358},"home":{"status":"ok","http":200,"url":"https://www.theaa.com/","html_tokens":90905,"markdown_tokens":216,"pruned_tokens":129}},
{"rank":8097,"domain":"ccohs.ca","origin":"https://ccohs.ca","robots":{"status":"ok","http":200,"url":"https://www.ccohs.ca/robots.txt","bytes":137,"groups":1},"bots":{"GPTBot":["partial","*",3,0],"ChatGPT-User":["partial","*",3,0],"OAI-SearchBot":["partial","*",3,0],"ClaudeBot":["partial","*",3,0],"Claude-User":["partial","*",3,0],"Claude-SearchBot":["partial","*",3,0],"anthropic-ai":["partial","*",3,0],"PerplexityBot":["partial","*",3,0],"Perplexity-User":["partial","*",3,0],"Google-Extended":["partial","*",3,0],"Applebot-Extended":["partial","*",3,0],"CCBot":["partial","*",3,0],"Bytespider":["partial","*",3,0],"Amazonbot":["partial","*",3,0],"meta-externalagent":["partial","*",3,0],"cohere-ai":["partial","*",3,0],"Diffbot":["partial","*",3,0],"ContextFetchBot":["partial","*",3,0]},"llms":{"status":"missing","http":404,"bytes":25758},"home":{"status":"ok","http":200,"url":"https://www.ccohs.ca/","html_tokens":9845,"markdown_tokens":44,"pruned_tokens":44}},
{"rank":8098,"domain":"bubble.io","origin":"https://bubble.io","robots":{"status":"ok","http":200,"url":"https://bubble.io/robots.txt","bytes":1295,"groups":1},"bots":{"GPTBot":["partial","*",41,0],"ChatGPT-User":["partial","*",41,0],"OAI-SearchBot":["partial","*",41,0],"ClaudeBot":["partial","*",41,0],"Claude-User":["partial","*",41,0],"Claude-SearchBot":["partial","*",41,0],"anthropic-ai":["partial","*",41,0],"PerplexityBot":["partial","*",41,0],"Perplexity-User":["partial","*",41,0],"Google-Extended":["partial","*",41,0],"Applebot-Extended":["partial","*",41,0],"CCBot":["partial","*",41,0],"Bytespider":["partial","*",41,0],"Amazonbot":["partial","*",41,0],"meta-externalagent":["partial","*",41,0],"cohere-ai":["partial","*",41,0],"Diffbot":["partial","*",41,0],"ContextFetchBot":["partial","*",41,0]},"llms":{"status":"missing","http":404,"bytes":45147},"home":{"status":"ok","http":200,"url":"https://bubble.io/","html_tokens":13095,"markdown_tokens":4,"pruned_tokens":4}},
{"rank":8099,"domain":"hessenschau.de","origin":"https://hessenschau.de","robots":{"status":"ok","http":200,"url":"https://www.hessenschau.de/robots.txt","bytes":2686,"groups":21},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["partial","*",15,1],"OAI-SearchBot":["partial","*",15,1],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",15,1],"Claude-SearchBot":["partial","*",15,1],"anthropic-ai":["partial","*",15,1],"PerplexityBot":["partial","*",15,1],"Perplexity-User":["partial","*",15,1],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["partial","*",15,1],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["partial","*",15,1],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",15,1]},"llms":{"status":"missing","http":404,"bytes":217026},"home":{"status":"ok","http":200,"url":"https://www.hessenschau.de/index.html","html_tokens":187334,"markdown_tokens":2040,"pruned_tokens":60}},
{"rank":8100,"domain":"huifengjiazheng.cn","origin":"http://huifengjiazheng.cn","robots":{"status":"missing","http":404,"url":"http://huifengjiazheng.cn/robots.txt","bytes":208,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":206},"home":{"status":"ok","http":200,"url":"http://huifengjiazheng.cn/","html_tokens":8955,"markdown_tokens":33,"pruned_tokens":33}},
{"rank":8101,"domain":"ioplex.com","origin":"https://ioplex.com","robots":{"status":"ok","http":200,"url":"https://www.ioplex.com/robots.txt","bytes":28,"groups":1},"bots":{"GPTBot":["partial","*",1,0],"ChatGPT-User":["partial","*",1,0],"OAI-SearchBot":["partial","*",1,0],"ClaudeBot":["partial","*",1,0],"Claude-User":["partial","*",1,0],"Claude-SearchBot":["partial","*",1,0],"anthropic-ai":["partial","*",1,0],"PerplexityBot":["partial","*",1,0],"Perplexity-User":["partial","*",1,0],"Google-Extended":["partial","*",1,0],"Applebot-Extended":["partial","*",1,0],"CCBot":["partial","*",1,0],"Bytespider":["partial","*",1,0],"Amazonbot":["partial","*",1,0],"meta-externalagent":["partial","*",1,0],"cohere-ai":["partial","*",1,0],"Diffbot":["partial","*",1,0],"ContextFetchBot":["partial","*",1,0]},"llms":{"status":"missing","http":404,"bytes":19},"home":{"status":"ok","http":200,"url":"https://www.ioplex.com/","html_tokens":2275,"markdown_tokens":1001,"pruned_tokens":159}},
{"rank":8102,"domain":"lrt.lt","origin":"https://lrt.lt","robots":{"status":"ok","http":200,"url":"https://www.lrt.lt/robots.txt","bytes":601,"groups":13},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["partial","named",1,1],"OAI-SearchBot":["partial","named",1,1],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",2,0],"Claude-SearchBot":["partial","*",2,0],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["partial","named",1,1],"Perplexity-User":["partial","*",2,0],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["partial","*",2,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["partial","*",2,0],"Diffbot":["partial","*",2,0],"ContextFetchBot":["partial","*",2,0]},"llms":{"status":"missing","http":404,"bytes":64552},"home":{"status":"ok","http":200,"url":"https://www.lrt.lt/","html_tokens":91498,"markdown_tokens":197,"pruned_tokens":105}},
{"rank":8103,"domain":"dhnet.be","origin":"https://dhnet.be","robots":{"status":"ok","http":200,"url":"https://www.dhnet.be/robots.txt","bytes":8950,"groups":125},"bots":{"GPTBot":["partial","named",1,10],"ChatGPT-User":["partial","named",1,10],"OAI-SearchBot":["blocked","named",1,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",18,0],"Claude-SearchBot":["partial","*",18,0],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["partial","*",18,0],"Google-Extended":["partial","named",1,10],"Applebot-Extended":["blocked","named",1,0],"CCBot":["partial","named",1,10],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["partial","*",18,0],"ContextFetchBot":["partial","*",18,0]},"llms":{"status":"missing","http":404,"bytes":378003},"home":{"status":"ok","http":200,"url":"https://www.dhnet.be/","html_tokens":942180,"markdown_tokens":26,"pruned_tokens":26}},
{"rank":8104,"domain":"ticketmaster.co.uk","origin":"https://ticketmaster.co.uk","robots":{"status":"ok","http":200,"url":"https://www.ticketmaster.co.uk/robots.txt","bytes":20256,"groups":7},"bots":{"GPTBot":["partial","*",39,2],"ChatGPT-User":["partial","*",39,2],"OAI-SearchBot":["partial","*",39,2],"ClaudeBot":["partial","*",39,2],"Claude-User":["partial","*",39,2],"Claude-SearchBot":["partial","*",39,2],"anthropic-ai":["partial","*",39,2],"PerplexityBot":["partial","*",39,2],"Perplexity-User":["partial","*",39,2],"Google-Extended":["partial","*",39,2],"Applebot-Extended":["partial","*",39,2],"CCBot":["partial","*",39,2],"Bytespider":["partial","*",39,2],"Amazonbot":["partial","*",39,2],"meta-externalagent":["partial","*",39,2],"cohere-ai":["partial","*",39,2],"Diffbot":["partial","*",39,2],"ContextFetchBot":["partial","*",39,2]},"llms":{"status":"missing","http":404,"bytes":155128},"home":{"status":"ok","http":200,"url":"https://www.ticketmaster.co.uk/","html_tokens":130000,"markdown_tokens":16,"pruned_tokens":16}},
{"rank":8105,"domain":"sri.com","origin":"https://sri.com","robots":{"status":"ok","http":200,"url":"https://www.sri.com/robots.txt","bytes":176,"groups":1},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"missing","http":404,"bytes":117646},"home":{"status":"ok","http":200,"url":"https://www.sri.com/","html_tokens":51936,"markdown_tokens":458,"pruned_tokens":166}},
{"rank":8106,"domain":"cnytsj.com","origin":"https://cnytsj.com","robots":{"status":"unreachable","http":null,"url":"https://cnytsj.com/robots.txt","bytes":0,"groups":0,"error":"cannot resolve cnytsj.com"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":8107,"domain":"aslwls.com","origin":"http://aslwls.com","robots":{"status":"refused","http":403,"url":"http://aslwls.com/robots.txt","bytes":640,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":8108,"domain":"gwi.com","origin":"https://gwi.com","robots":{"status":"ok","http":200,"url":"https://www.gwi.com/robots.txt","bytes":243,"groups":1},"bots":{"GPTBot":["partial","*",6,0],"ChatGPT-User":["partial","*",6,0],"OAI-SearchBot":["partial","*",6,0],"ClaudeBot":["partial","*",6,0],"Claude-User":["partial","*",6,0],"Claude-SearchBot":["partial","*",6,0],"anthropic-ai":["partial","*",6,0],"PerplexityBot":["partial","*",6,0],"Perplexity-User":["partial","*",6,0],"Google-Extended":["partial","*",6,0],"Applebot-Extended":["partial","*",6,0],"CCBot":["partial","*",6,0],"Bytespider":["partial","*",6,0],"Amazonbot":["partial","*",6,0],"meta-externalagent":["partial","*",6,0],"cohere-ai":["partial","*",6,0],"Diffbot":["partial","*",6,0],"ContextFetchBot":["partial","*",6,0]},"llms":{"status":"missing","http":404,"bytes":89620},"home":{"status":"ok","http":200,"url":"https://www.gwi.com/","html_tokens":100572,"markdown_tokens":393,"pruned_tokens":229}},
{"rank":8109,"domain":"lseg.com","origin":"https://lseg.com","robots":{"status":"ok","http":200,"url":"https://www.lseg.com/robots.txt","bytes":2910,"groups":1},"bots":{"GPTBot":["partial","*",65,0],"ChatGPT-User":["partial","*",65,0],"OAI-SearchBot":["partial","*",65,0],"ClaudeBot":["partial","*",65,0],"Claude-User":["partial","*",65,0],"Claude-SearchBot":["partial","*",65,0],"anthropic-ai":["partial","*",65,0],"PerplexityBot":["partial","*",65,0],"Perplexity-User":["partial","*",65,0],"Google-Extended":["partial","*",65,0],"Applebot-Extended":["partial","*",65,0],"CCBot":["partial","*",65,0],"Bytespider":["partial","*",65,0],"Amazonbot":["partial","*",65,0],"meta-externalagent":["partial","*",65,0],"cohere-ai":["partial","*",65,0],"Diffbot":["partial","*",65,0],"ContextFetchBot":["partial","*",65,0]},"llms":{"status":"missing","http":404,"bytes":55267},"home":{"status":"ok","http":200,"url":"https://www.lseg.com/en","html_tokens":37694,"markdown_tokens":58,"pruned_tokens":58}},
{"rank":8110,"domain":"xiaomi.com","origin":"https://xiaomi.com","robots":{"status":"refused","http":403,"url":"https://www.mi.com/robots.txt","bytes":376,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":8111,"domain":"google.com.py","origin":"https://google.com.py","robots":{"status":"ok","http":200,"url":"https://www.google.com.py/robots.txt","bytes":6624,"groups":4},"bots":{"GPTBot":["partial","*",169,64],"ChatGPT-User":["partial","*",169,64],"OAI-SearchBot":["partial","*",169,64],"ClaudeBot":["partial","*",169,64],"Claude-User":["partial","*",169,64],"Claude-SearchBot":["partial","*",169,64],"anthropic-ai":["partial","*",169,64],"PerplexityBot":["partial","*",169,64],"Perplexity-User":["partial","*",169,64],"Google-Extended":["partial","*",169,64],"Applebot-Extended":["partial","*",169,64],"CCBot":["partial","*",169,64],"Bytespider":["partial","*",169,64],"Amazonbot":["partial","*",169,64],"meta-externalagent":["partial","*",169,64],"cohere-ai":["partial","*",169,64],"Diffbot":["partial","*",169,64],"ContextFetchBot":["partial","*",169,64]},"llms":{"status":"missing","http":404,"bytes":1569},"home":{"status":"ok","http":200,"url":"https://www.google.com/","html_tokens":21270,"markdown_tokens":30,"pruned_tokens":30}},
{"rank":8112,"domain":"micro.blog","origin":"https://micro.blog","robots":{"status":"missing","http":404,"url":"https://micro.blog/robots.txt","bytes":14,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":14},"home":{"status":"ok","http":200,"url":"https://micro.blog/","html_tokens":972,"markdown_tokens":83,"pruned_tokens":38}},
{"rank":8113,"domain":"gmwq.com.cn","origin":"http://gmwq.com.cn","robots":{"status":"missing","http":404,"url":"http://gmwq.com.cn/robots.txt","bytes":4842,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":4838},"home":{"status":"ok","http":200,"url":"http://gmwq.com.cn/","html_tokens":13960,"markdown_tokens":295,"pruned_tokens":78}},
{"rank":8114,"domain":"jujingsw.cn","origin":"http://jujingsw.cn","robots":{"status":"missing","http":404,"url":"http://jujingsw.cn/robots.txt","bytes":1163,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":1163},"home":{"status":"ok","http":200,"url":"http://jujingsw.cn/","html_tokens":28037,"markdown_tokens":450,"pruned_tokens":118}},
{"rank":8115,"domain":"tasteatlas.com","origin":"https://tasteatlas.com","robots":{"status":"ok","http":200,"url":"https://www.tasteatlas.com/robots.txt","bytes":13,"groups":1},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"missing","http":404,"bytes":5209},"home":{"status":"http_error","http":403,"url":"https://tasteatlas.com/"}},
{"rank":8116,"domain":"google.al","origin":"https://google.al","robots":{"status":"ok","http":200,"url":"https://www.google.al/robots.txt","bytes":6624,"groups":4},"bots":{"GPTBot":["partial","*",169,64],"ChatGPT-User":["partial","*",169,64],"OAI-SearchBot":["partial","*",169,64],"ClaudeBot":["partial","*",169,64],"Claude-User":["partial","*",169,64],"Claude-SearchBot":["partial","*",169,64],"anthropic-ai":["partial","*",169,64],"PerplexityBot":["partial","*",169,64],"Perplexity-User":["partial","*",169,64],"Google-Extended":["partial","*",169,64],"Applebot-Extended":["partial","*",169,64],"CCBot":["partial","*",169,64],"Bytespider":["partial","*",169,64],"Amazonbot":["partial","*",169,64],"meta-externalagent":["partial","*",169,64],"cohere-ai":["partial","*",169,64],"Diffbot":["partial","*",169,64],"ContextFetchBot":["partial","*",169,64]},"llms":{"status":"missing","http":404,"bytes":1569},"home":{"status":"ok","http":200,"url":"https://www.google.com/","html_tokens":21269,"markdown_tokens":30,"pruned_tokens":30}},
{"rank":8117,"domain":"scielo.org.mx","origin":"https://www.scielo.org.mx","robots":{"status":"missing","http":404,"url":"https://www.scielo.org.mx/robots.txt","bytes":290,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":288},"home":{"status":"ok","http":200,"url":"https://www.scielo.org.mx/scielo.php","html_tokens":3621,"markdown_tokens":1344,"pruned_tokens":17}},
{"rank":8118,"domain":"powerappsportals.com","origin":"https://powerappsportals.com","robots":{"status":"unreachable","http":null,"url":"https://powerappsportals.com/robots.txt","bytes":0,"groups":0,"error":"cannot resolve powerappsportals.com"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":8119,"domain":"catholic.org","origin":"https://catholic.org","robots":{"status":"ok","http":200,"url":"https://www.catholic.org/robots.txt","bytes":606,"groups":1},"bots":{"GPTBot":["partial","*",29,0],"ChatGPT-User":["partial","*",29,0],"OAI-SearchBot":["partial","*",29,0],"ClaudeBot":["partial","*",29,0],"Claude-User":["partial","*",29,0],"Claude-SearchBot":["partial","*",29,0],"anthropic-ai":["partial","*",29,0],"PerplexityBot":["partial","*",29,0],"Perplexity-User":["partial","*",29,0],"Google-Extended":["partial","*",29,0],"Applebot-Extended":["partial","*",29,0],"CCBot":["partial","*",29,0],"Bytespider":["partial","*",29,0],"Amazonbot":["partial","*",29,0],"meta-externalagent":["partial","*",29,0],"cohere-ai":["partial","*",29,0],"Diffbot":["partial","*",29,0],"ContextFetchBot":["partial","*",29,0]},"llms":{"status":"missing","http":404,"bytes":195356},"home":{"status":"ok","http":200,"url":"https://www.catholic.org/","html_tokens":54344,"markdown_tokens":329,"pruned_tokens":170}},
{"rank":8120,"domain":"creator-spring.com","origin":"https://creator-spring.com","robots":{"status":"ok","http":200,"url":"https://creator-spring.com/robots.txt","bytes":62,"groups":1},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"missing","http":404,"bytes":17621},"home":{"status":"ok","http":200,"url":"https://creator-spring.com/","html_tokens":47470,"markdown_tokens":18,"pruned_tokens":18}},
{"rank":8121,"domain":"run.app","origin":"https://run.app","robots":{"status":"missing","http":404,"url":"https://run.app/robots.txt","bytes":1571,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":1569},"home":{"status":"http_error","http":404,"url":"https://run.app/"}},
{"rank":8122,"domain":"kcra.com","origin":"https://kcra.com","robots":{"status":"ok","http":200,"url":"https://kcra.com/robots.txt","bytes":3763,"groups":48},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["blocked","named",1,0],"OAI-SearchBot":["partial","*",22,2],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",22,2],"Claude-SearchBot":["partial","*",22,2],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["partial","*",22,2],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",2,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",22,2]},"llms":{"status":"missing","http":404,"bytes":0},"home":{"status":"ok","http":200,"url":"https://www.kcra.com/","html_tokens":149628,"markdown_tokens":55,"pruned_tokens":55}},
{"rank":8123,"domain":"kicker.de","origin":"https://kicker.de","robots":{"status":"ok","http":200,"url":"https://www.kicker.de/robots.txt","bytes":2294,"groups":38},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["blocked","named",1,0],"OAI-SearchBot":["partial","*",13,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",13,0],"Claude-SearchBot":["blocked","named",1,0],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["blocked","named",1,0],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["partial","*",13,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",13,0]},"llms":{"status":"http_error","http":403,"bytes":769},"home":{"status":"http_error","http":403,"url":"https://www.kicker.de/"}},
{"rank":8124,"domain":"wallhaven.cc","origin":"https://wallhaven.cc","robots":{"status":"ok","http":200,"url":"https://wallhaven.cc/robots.txt","bytes":24,"groups":1},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"html","http":200,"bytes":23367},"home":{"status":"ok","http":200,"url":"https://wallhaven.cc/","html_tokens":5530,"markdown_tokens":85,"pruned_tokens":54}},
{"rank":8125,"domain":"hitnspin-casino.org","origin":"https://hitnspin-casino.org","robots":{"status":"ok","http":200,"url":"https://hitnspin-casino.org/robots.txt","bytes":67,"groups":1},"bots":{"GPTBot":["partial","*",1,1],"ChatGPT-User":["partial","*",1,1],"OAI-SearchBot":["partial","*",1,1],"ClaudeBot":["partial","*",1,1],"Claude-User":["partial","*",1,1],"Claude-SearchBot":["partial","*",1,1],"anthropic-ai":["partial","*",1,1],"PerplexityBot":["partial","*",1,1],"Perplexity-User":["partial","*",1,1],"Google-Extended":["partial","*",1,1],"Applebot-Extended":["partial","*",1,1],"CCBot":["partial","*",1,1],"Bytespider":["partial","*",1,1],"Amazonbot":["partial","*",1,1],"meta-externalagent":["partial","*",1,1],"cohere-ai":["partial","*",1,1],"Diffbot":["partial","*",1,1],"ContextFetchBot":["partial","*",1,1]},"llms":{"status":"missing","http":404,"bytes":47986},"home":{"status":"ok","http":200,"url":"https://hitnspin-casino.org/","html_tokens":21165,"markdown_tokens":5928,"pruned_tokens":138}},
{"rank":8126,"domain":"hcyt.tech","origin":"https://hcyt.tech","robots":{"status":"unreachable","http":null,"url":"https://hcyt.tech/robots.txt","bytes":0,"groups":0,"error":"cannot resolve hcyt.tech"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":8127,"domain":"tvnz.co.nz","origin":"https://tvnz.co.nz","robots":{"status":"ok","http":200,"url":"https://tvnz.co.nz/robots.txt","bytes":180,"groups":1},"bots":{"GPTBot":["partial","*",5,1],"ChatGPT-User":["partial","*",5,1],"OAI-SearchBot":["partial","*",5,1],"ClaudeBot":["partial","*",5,1],"Claude-User":["partial","*",5,1],"Claude-SearchBot":["partial","*",5,1],"anthropic-ai":["partial","*",5,1],"PerplexityBot":["partial","*",5,1],"Perplexity-User":["partial","*",5,1],"Google-Extended":["partial","*",5,1],"Applebot-Extended":["partial","*",5,1],"CCBot":["partial","*",5,1],"Bytespider":["partial","*",5,1],"Amazonbot":["partial","*",5,1],"meta-externalagent":["partial","*",5,1],"cohere-ai":["partial","*",5,1],"Diffbot":["partial","*",5,1],"ContextFetchBot":["partial","*",5,1]},"llms":{"status":"missing","http":404,"bytes":311233},"home":{"status":"ok","http":200,"url":"https://www.tvnz.co.nz/","html_tokens":122624,"markdown_tokens":684,"pruned_tokens":170}},
{"rank":8128,"domain":"tsmc.com","origin":"https://tsmc.com","robots":{"status":"refused","http":403,"url":"https://www.tsmc.com/robots.txt","bytes":5515,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":8129,"domain":"rbb-online.de","origin":"https://rbb-online.de","robots":{"status":"ok","http":200,"url":"https://www.rbb-online.de/robots.txt","bytes":3253,"groups":32},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["partial","*",4,0],"OAI-SearchBot":["partial","*",4,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",4,0],"Claude-SearchBot":["partial","*",4,0],"anthropic-ai":["partial","*",4,0],"PerplexityBot":["partial","*",4,0],"Perplexity-User":["partial","*",4,0],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["partial","*",4,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["partial","*",4,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",4,0]},"llms":{"status":"missing","http":404,"bytes":52703},"home":{"status":"ok","http":200,"url":"https://www.rbb24.de/","html_tokens":97182,"markdown_tokens":265,"pruned_tokens":96}},
{"rank":8130,"domain":"austinchronicle.com","origin":"https://austinchronicle.com","robots":{"status":"refused","http":403,"url":"https://www.austinchronicle.com/robots.txt","bytes":7355,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":8131,"domain":"gumtree.com","origin":"https://gumtree.com","robots":{"status":"ok","http":200,"url":"https://www.gumtree.com/robots.txt","bytes":13171,"groups":2},"bots":{"GPTBot":["partial","*",140,176],"ChatGPT-User":["partial","*",140,176],"OAI-SearchBot":["partial","*",140,176],"ClaudeBot":["partial","*",140,176],"Claude-User":["partial","*",140,176],"Claude-SearchBot":["partial","*",140,176],"anthropic-ai":["partial","*",140,176],"PerplexityBot":["partial","*",140,176],"Perplexity-User":["partial","*",140,176],"Google-Extended":["partial","*",140,176],"Applebot-Extended":["partial","*",140,176],"CCBot":["partial","*",140,176],"Bytespider":["partial","*",140,176],"Amazonbot":["partial","*",140,176],"meta-externalagent":["partial","*",140,176],"cohere-ai":["partial","*",140,176],"Diffbot":["partial","*",140,176],"ContextFetchBot":["partial","*",140,176]},"llms":{"status":"present","http":200,"bytes":25108},"home":{"status":"ok","http":200,"url":"https://www.gumtree.com/","html_tokens":148539,"markdown_tokens":28,"pruned_tokens":28}},
{"rank":8132,"domain":"dmu.ac.uk","origin":"https://dmu.ac.uk","robots":{"status":"ok","http":200,"url":"https://www.dmu.ac.uk/robots.txt","bytes":627,"groups":2},"bots":{"GPTBot":["partial","*",15,0],"ChatGPT-User":["partial","*",15,0],"OAI-SearchBot":["partial","*",15,0],"ClaudeBot":["partial","*",15,0],"Claude-User":["partial","*",15,0],"Claude-SearchBot":["partial","*",15,0],"anthropic-ai":["partial","*",15,0],"PerplexityBot":["partial","*",15,0],"Perplexity-User":["partial","*",15,0],"Google-Extended":["partial","*",15,0],"Applebot-Extended":["partial","*",15,0],"CCBot":["partial","*",15,0],"Bytespider":["partial","*",15,0],"Amazonbot":["partial","*",15,0],"meta-externalagent":["partial","*",15,0],"cohere-ai":["partial","*",15,0],"Diffbot":["partial","*",15,0],"ContextFetchBot":["partial","*",15,0]},"llms":{"status":"missing","http":404,"bytes":14},"home":{"status":"ok","http":200,"url":"https://www.dmu.ac.uk/home.aspx","html_tokens":20872,"markdown_tokens":91,"pruned_tokens":31}},
{"rank":8133,"domain":"bookofra-slot.fr","origin":"https://bookofra-slot.fr","robots":{"status":"ok","http":200,"url":"https://bookofra-slot.fr/robots.txt","bytes":1389,"groups":9},"bots":{"GPTBot":["partial","*",5,3],"ChatGPT-User":["partial","*",5,3],"OAI-SearchBot":["partial","*",5,3],"ClaudeBot":["partial","*",5,3],"Claude-User":["partial","*",5,3],"Claude-SearchBot":["partial","*",5,3],"anthropic-ai":["partial","*",5,3],"PerplexityBot":["partial","*",5,3],"Perplexity-User":["partial","*",5,3],"Google-Extended":["partial","*",5,3],"Applebot-Extended":["partial","*",5,3],"CCBot":["partial","*",5,3],"Bytespider":["partial","*",5,3],"Amazonbot":["partial","*",5,3],"meta-externalagent":["partial","*",5,3],"cohere-ai":["partial","*",5,3],"Diffbot":["partial","*",5,3],"ContextFetchBot":["partial","*",5,3]},"llms":{"status":"missing","http":404,"bytes":13114},"home":{"status":"ok","http":200,"url":"https://bookofra-slot.fr/","html_tokens":15787,"markdown_tokens":2231,"pruned_tokens":103}},
{"rank":8134,"domain":"zwmss.com","origin":"https://zwmss.com","robots":{"status":"missing","http":404,"url":"https://zwmss.com/robots.txt","bytes":315,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":315},"home":{"status":"http_error","http":404,"url":"https://zwmss.com/"}},
{"rank":8135,"domain":"quechoisir.org","origin":"https://quechoisir.org","robots":{"status":"ok","http":200,"url":"https://www.quechoisir.org/robots.txt","bytes":1235,"groups":1},"bots":{"GPTBot":["partial","*",34,14],"ChatGPT-User":["partial","*",34,14],"OAI-SearchBot":["partial","*",34,14],"ClaudeBot":["partial","*",34,14],"Claude-User":["partial","*",34,14],"Claude-SearchBot":["partial","*",34,14],"anthropic-ai":["partial","*",34,14],"PerplexityBot":["partial","*",34,14],"Perplexity-User":["partial","*",34,14],"Google-Extended":["partial","*",34,14],"Applebot-Extended":["partial","*",34,14],"CCBot":["partial","*",34,14],"Bytespider":["partial","*",34,14],"Amazonbot":["partial","*",34,14],"meta-externalagent":["partial","*",34,14],"cohere-ai":["partial","*",34,14],"Diffbot":["partial","*",34,14],"ContextFetchBot":["partial","*",34,14]},"llms":{"status":"missing","http":404,"bytes":587439},"home":{"status":"ok","http":200,"url":"https://www.quechoisir.org/","html_tokens":163762,"markdown_tokens":66,"pruned_tokens":66}},
{"rank":8136,"domain":"finanznachrichten.de","origin":"https://finanznachrichten.de","robots":{"status":"ok","http":200,"url":"https://www.finanznachrichten.de/robots.txt","bytes":1050,"groups":7},"bots":{"GPTBot":["partial","*",33,0],"ChatGPT-User":["partial","*",33,0],"OAI-SearchBot":["partial","*",33,0],"ClaudeBot":["partial","*",33,0],"Claude-User":["partial","*",33,0],"Claude-SearchBot":["partial","*",33,0],"anthropic-ai":["partial","*",33,0],"PerplexityBot":["partial","*",33,0],"Perplexity-User":["partial","*",33,0],"Google-Extended":["partial","*",33,0],"Applebot-Extended":["partial","*",33,0],"CCBot":["partial","*",33,0],"Bytespider":["partial","*",33,0],"Amazonbot":["partial","*",33,0],"meta-externalagent":["partial","*",33,0],"cohere-ai":["partial","*",33,0],"Diffbot":["partial","*",33,0],"ContextFetchBot":["partial","*",33,0]},"llms":{"status":"timeout","error":"ConnectTimeout"},"home":{"status":"timeout","error":"ConnectTimeout"}},
{"rank":8137,"domain":"guru3d.com","origin":"https://guru3d.com","robots":{"status":"refused","http":403,"url":"https://guru3d.com/robots.txt","bytes":5505,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":8138,"domain":"book-of-ra-play.com","origin":"https://book-of-ra-play.com","robots":{"status":"ok","http":200,"url":"https://book-of-ra-play.com/robots.txt","bytes":542,"groups":1},"bots":{"GPTBot":["partial","*",19,0],"ChatGPT-User":["partial","*",19,0],"OAI-SearchBot":["partial","*",19,0],"ClaudeBot":["partial","*",19,0],"Claude-User":["partial","*",19,0],"Claude-SearchBot":["partial","*",19,0],"anthropic-ai":["partial","*",19,0],"PerplexityBot":["partial","*",19,0],"Perplexity-User":["partial","*",19,0],"Google-Extended":["partial","*",19,0],"Applebot-Extended":["partial","*",19,0],"CCBot":["partial","*",19,0],"Bytespider":["partial","*",19,0],"Amazonbot":["partial","*",19,0],"meta-externalagent":["partial","*",19,0],"cohere-ai":["partial","*",19,0],"Diffbot":["partial","*",19,0],"ContextFetchBot":["partial","*",19,0]},"llms":{"status":"missing","http":404,"bytes":25407},"home":{"status":"ok","http":200,"url":"https://book-of-ra-play.com/","html_tokens":30591,"markdown_tokens":2402,"pruned_tokens":602}},
{"rank":8139,"domain":"surveygizmo.com","origin":"https://surveygizmo.com","robots":{"status":"missing","http":404,"url":"https://www.surveygizmo.com/robots.txt","bytes":365,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":343},"home":{"status":"ok","http":200,"url":"https://www.surveygizmo.com/","html_tokens":44,"markdown_tokens":2,"pruned_tokens":2}},
{"rank":8140,"domain":"medshop24h.top","origin":"http://medshop24h.top","robots":{"status":"server_error","http":503,"url":"http://medshop24h.top/robots.txt","bytes":55,"groups":0,"error":"HTTP 503"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":8141,"domain":"grohe.com","origin":"https://grohe.com","robots":{"status":"ok","http":200,"url":"https://www.grohe.com/robots.txt","bytes":246,"groups":1},"bots":{"GPTBot":["partial","*",8,1],"ChatGPT-User":["partial","*",8,1],"OAI-SearchBot":["partial","*",8,1],"ClaudeBot":["partial","*",8,1],"Claude-User":["partial","*",8,1],"Claude-SearchBot":["partial","*",8,1],"anthropic-ai":["partial","*",8,1],"PerplexityBot":["partial","*",8,1],"Perplexity-User":["partial","*",8,1],"Google-Extended":["partial","*",8,1],"Applebot-Extended":["partial","*",8,1],"CCBot":["partial","*",8,1],"Bytespider":["partial","*",8,1],"Amazonbot":["partial","*",8,1],"meta-externalagent":["partial","*",8,1],"cohere-ai":["partial","*",8,1],"Diffbot":["partial","*",8,1],"ContextFetchBot":["partial","*",8,1]},"llms":{"status":"html","http":200,"bytes":625953},"home":{"status":"ok","http":200,"url":"https://www.grohe.com/en-GB","html_tokens":156455,"markdown_tokens":117,"pruned_tokens":117}},
{"rank":8142,"domain":"markedu.com.cn","origin":"http://markedu.com.cn","robots":{"status":"missing","http":404,"url":"http://markedu.com.cn/robots.txt","bytes":1163,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":1163},"home":{"status":"ok","http":200,"url":"http://markedu.com.cn/","html_tokens":3928,"markdown_tokens":45,"pruned_tokens":45}},
{"rank":8143,"domain":"scouting.org","origin":"https://scouting.org","robots":{"status":"refused","http":403,"url":"https://scouting.org/robots.txt","bytes":146,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":8144,"domain":"mi.it","origin":"https://mi.it","robots":{"status":"unreachable","http":null,"url":"https://mi.it/robots.txt","bytes":0,"groups":0,"error":"cannot resolve mi.it"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":8145,"domain":"colourlovers.com","origin":"https://colourlovers.com","robots":{"status":"ok","http":200,"url":"https://www.colourlovers.com/robots.txt","bytes":4811,"groups":113},"bots":{"GPTBot":["partial","*",9,1],"ChatGPT-User":["partial","*",9,1],"OAI-SearchBot":["partial","*",9,1],"ClaudeBot":["partial","*",9,1],"Claude-User":["partial","*",9,1],"Claude-SearchBot":["partial","*",9,1],"anthropic-ai":["partial","*",9,1],"PerplexityBot":["partial","*",9,1],"Perplexity-User":["partial","*",9,1],"Google-Extended":["partial","*",9,1],"Applebot-Extended":["partial","*",9,1],"CCBot":["partial","*",9,1],"Bytespider":["partial","*",9,1],"Amazonbot":["partial","*",9,1],"meta-externalagent":["partial","*",9,1],"cohere-ai":["partial","*",9,1],"Diffbot":["partial","*",9,1],"ContextFetchBot":["partial","*",9,1]},"llms":{"status":"http_error","http":403,"bytes":5531},"home":{"status":"http_error","http":403,"url":"https://www.colourlovers.com/"}},
{"rank":8146,"domain":"lxhdz.com","origin":"http://lxhdz.com","robots":{"status":"missing","http":404,"url":"http://lxhdz.com/robots.txt","bytes":1163,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":1163},"home":{"status":"ok","http":200,"url":"http://www.lxhdz.com/","html_tokens":5494,"markdown_tokens":399,"pruned_tokens":173}},
{"rank":8147,"domain":"rba.gov.au","origin":"https://rba.gov.au","robots":{"status":"ok","http":200,"url":"https://www.rba.gov.au/robots.txt","bytes":476,"groups":2},"bots":{"GPTBot":["partial","*",6,0],"ChatGPT-User":["partial","*",6,0],"OAI-SearchBot":["partial","*",6,0],"ClaudeBot":["partial","*",6,0],"Claude-User":["partial","*",6,0],"Claude-SearchBot":["partial","*",6,0],"anthropic-ai":["partial","*",6,0],"PerplexityBot":["partial","*",6,0],"Perplexity-User":["partial","*",6,0],"Google-Extended":["partial","*",6,0],"Applebot-Extended":["partial","*",6,0],"CCBot":["partial","*",6,0],"Bytespider":["partial","*",6,0],"Amazonbot":["partial","*",6,0],"meta-externalagent":["partial","*",6,0],"cohere-ai":["partial","*",6,0],"Diffbot":["partial","*",6,0],"ContextFetchBot":["partial","*",6,0]},"llms":{"status":"missing","http":404,"bytes":18107},"home":{"status":"ok","http":200,"url":"https://www.rba.gov.au/","html_tokens":15364,"markdown_tokens":1176,"pruned_tokens":162}},
{"rank":8148,"domain":"nycgovparks.org","origin":"https://nycgovparks.org","robots":{"status":"ok","http":200,"url":"https://www.nycgovparks.org/robots.txt","bytes":2785,"groups":3},"bots":{"GPTBot":["partial","*",56,0],"ChatGPT-User":["partial","*",56,0],"OAI-SearchBot":["partial","*",56,0],"ClaudeBot":["partial","*",56,0],"Claude-User":["partial","*",56,0],"Claude-SearchBot":["partial","*",56,0],"anthropic-ai":["partial","*",56,0],"PerplexityBot":["partial","*",56,0],"Perplexity-User":["partial","*",56,0],"Google-Extended":["partial","*",56,0],"Applebot-Extended":["partial","*",56,0],"CCBot":["partial","*",56,0],"Bytespider":["partial","*",56,0],"Amazonbot":["partial","*",56,0],"meta-externalagent":["partial","*",56,0],"cohere-ai":["partial","*",56,0],"Diffbot":["partial","*",56,0],"ContextFetchBot":["partial","*",56,0]},"llms":{"status":"missing","http":404,"bytes":13633},"home":{"status":"http_error","http":403,"url":"https://www.nycgovparks.org/"}},
{"rank":8149,"domain":"benjerry.com","origin":"https://benjerry.com","robots":{"status":"refused","http":403,"url":"https://www.benjerry.com/robots.txt","bytes":382,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":8150,"domain":"lacma.org","origin":"https://lacma.org","robots":{"status":"refused","http":403,"url":"https://lacma.org/robots.txt","bytes":560,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":8151,"domain":"wku.edu","origin":"https://wku.edu","robots":{"status":"ok","http":200,"url":"https://www.wku.edu/robots.txt","bytes":160,"groups":1},"bots":{"GPTBot":["partial","*",7,0],"ChatGPT-User":["partial","*",7,0],"OAI-SearchBot":["partial","*",7,0],"ClaudeBot":["partial","*",7,0],"Claude-User":["partial","*",7,0],"Claude-SearchBot":["partial","*",7,0],"anthropic-ai":["partial","*",7,0],"PerplexityBot":["partial","*",7,0],"Perplexity-User":["partial","*",7,0],"Google-Extended":["partial","*",7,0],"Applebot-Extended":["partial","*",7,0],"CCBot":["partial","*",7,0],"Bytespider":["partial","*",7,0],"Amazonbot":["partial","*",7,0],"meta-externalagent":["partial","*",7,0],"cohere-ai":["partial","*",7,0],"Diffbot":["partial","*",7,0],"ContextFetchBot":["partial","*",7,0]},"llms":{"status":"missing","http":404,"bytes":22726},"home":{"status":"ok","http":200,"url":"https://www.wku.edu/","html_tokens":11678,"markdown_tokens":688,"pruned_tokens":151}},
{"rank":8152,"domain":"edpharmahub.store","origin":"https://edpharmahub.store","robots":{"status":"ok","http":200,"url":"https://edpharmahub.store/robots.txt","bytes":80,"groups":1},"bots":{"GPTBot":["partial","*",1,0],"ChatGPT-User":["partial","*",1,0],"OAI-SearchBot":["partial","*",1,0],"ClaudeBot":["partial","*",1,0],"Claude-User":["partial","*",1,0],"Claude-SearchBot":["partial","*",1,0],"anthropic-ai":["partial","*",1,0],"PerplexityBot":["partial","*",1,0],"Perplexity-User":["partial","*",1,0],"Google-Extended":["partial","*",1,0],"Applebot-Extended":["partial","*",1,0],"CCBot":["partial","*",1,0],"Bytespider":["partial","*",1,0],"Amazonbot":["partial","*",1,0],"meta-externalagent":["partial","*",1,0],"cohere-ai":["partial","*",1,0],"Diffbot":["partial","*",1,0],"ContextFetchBot":["partial","*",1,0]},"llms":{"status":"missing","http":404,"bytes":646},"home":{"status":"ok","http":200,"url":"https://edpharmahub.store/","html_tokens":22028,"markdown_tokens":1087,"pruned_tokens":167}},
{"rank":8153,"domain":"sougo.net","origin":"http://sougo.net","robots":{"status":"missing","http":404,"url":"http://sougo.net/robots.txt","bytes":3422,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":3418},"home":{"status":"ok","http":200,"url":"http://sougo.net/","html_tokens":3510,"markdown_tokens":22,"pruned_tokens":22}},
{"rank":8154,"domain":"shjyun.com","origin":"https://shjyun.com","robots":{"status":"ok","http":200,"url":"https://shjyun.com/robots.txt","bytes":2050,"groups":20},"bots":{"GPTBot":["allowed","named",0,1],"ChatGPT-User":["allowed","named",0,1],"OAI-SearchBot":["allowed","named",0,1],"ClaudeBot":["allowed","named",0,1],"Claude-User":["allowed","named",0,1],"Claude-SearchBot":["partial","*",7,3],"anthropic-ai":["partial","*",7,3],"PerplexityBot":["allowed","named",0,1],"Perplexity-User":["partial","*",7,3],"Google-Extended":["allowed","named",0,1],"Applebot-Extended":["allowed","named",0,1],"CCBot":["allowed","named",0,1],"Bytespider":["allowed","named",0,1],"Amazonbot":["partial","*",7,3],"meta-externalagent":["partial","*",7,3],"cohere-ai":["partial","*",7,3],"Diffbot":["partial","*",7,3],"ContextFetchBot":["partial","*",7,3]},"llms":{"status":"present","http":200,"bytes":4466},"home":{"status":"ok","http":200,"url":"https://shjyun.com/","html_tokens":7905,"markdown_tokens":178,"pruned_tokens":172}},
{"rank":8155,"domain":"sandiego.gov","origin":"https://sandiego.gov","robots":{"status":"refused","http":403,"url":"https://sandiego.gov/robots.txt","bytes":560,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":8156,"domain":"mp3.com","origin":"http://mp3.com","robots":{"status":"html","http":200,"url":"https://www.markmonitor.com/mp3com/","bytes":413501,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":8157,"domain":"tiny.cloud","origin":"https://tiny.cloud","robots":{"status":"ok","http":200,"url":"https://www.tiny.cloud/robots.txt","bytes":151,"groups":1},"bots":{"GPTBot":["allowed","*",0,1],"ChatGPT-User":["allowed","*",0,1],"OAI-SearchBot":["allowed","*",0,1],"ClaudeBot":["allowed","*",0,1],"Claude-User":["allowed","*",0,1],"Claude-SearchBot":["allowed","*",0,1],"anthropic-ai":["allowed","*",0,1],"PerplexityBot":["allowed","*",0,1],"Perplexity-User":["allowed","*",0,1],"Google-Extended":["allowed","*",0,1],"Applebot-Extended":["allowed","*",0,1],"CCBot":["allowed","*",0,1],"Bytespider":["allowed","*",0,1],"Amazonbot":["allowed","*",0,1],"meta-externalagent":["allowed","*",0,1],"cohere-ai":["allowed","*",0,1],"Diffbot":["allowed","*",0,1],"ContextFetchBot":["allowed","*",0,1]},"llms":{"status":"present","http":200,"bytes":31183},"home":{"status":"ok","http":200,"url":"https://www.tiny.cloud/","html_tokens":93699,"markdown_tokens":572,"pruned_tokens":190}},
{"rank":8158,"domain":"cnguancai.com","origin":"http://cnguancai.com","robots":{"status":"missing","http":404,"url":"http://cnguancai.com/robots.txt","bytes":5074,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":5070},"home":{"status":"ok","http":200,"url":"http://cnguancai.com/","html_tokens":7008,"markdown_tokens":207,"pruned_tokens":144}},
{"rank":8159,"domain":"powerdekor-woodfloor.com","origin":"https://powerdekor-woodfloor.com","robots":{"status":"missing","http":404,"url":"https://powerdekor-woodfloor.com/robots.txt","bytes":3433,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":3429},"home":{"status":"ok","http":200,"url":"https://powerdekor-woodfloor.com/","html_tokens":8684,"markdown_tokens":271,"pruned_tokens":159}},
{"rank":8160,"domain":"teachina.net","origin":"https://www.teachina.net","robots":{"status":"ok","http":200,"url":"https://www.teachina.net/robots.txt","bytes":553,"groups":1},"bots":{"GPTBot":["partial","*",24,0],"ChatGPT-User":["partial","*",24,0],"OAI-SearchBot":["partial","*",24,0],"ClaudeBot":["partial","*",24,0],"Claude-User":["partial","*",24,0],"Claude-SearchBot":["partial","*",24,0],"anthropic-ai":["partial","*",24,0],"PerplexityBot":["partial","*",24,0],"Perplexity-User":["partial","*",24,0],"Google-Extended":["partial","*",24,0],"Applebot-Extended":["partial","*",24,0],"CCBot":["partial","*",24,0],"Bytespider":["partial","*",24,0],"Amazonbot":["partial","*",24,0],"meta-externalagent":["partial","*",24,0],"cohere-ai":["partial","*",24,0],"Diffbot":["partial","*",24,0],"ContextFetchBot":["partial","*",24,0]},"llms":{"status":"disallowed"},"home":{"status":"ok","http":200,"url":"https://www.teachina.net/","html_tokens":30630,"markdown_tokens":42,"pruned_tokens":42}},
{"rank":8161,"domain":"thehartford.com","origin":"https://www.thehartford.com","robots":{"status":"ok","http":200,"url":"https://www.thehartford.com/robots.txt","bytes":403,"groups":1},"bots":{"GPTBot":["partial","*",11,0],"ChatGPT-User":["partial","*",11,0],"OAI-SearchBot":["partial","*",11,0],"ClaudeBot":["partial","*",11,0],"Claude-User":["partial","*",11,0],"Claude-SearchBot":["partial","*",11,0],"anthropic-ai":["partial","*",11,0],"PerplexityBot":["partial","*",11,0],"Perplexity-User":["partial","*",11,0],"Google-Extended":["partial","*",11,0],"Applebot-Extended":["partial","*",11,0],"CCBot":["partial","*",11,0],"Bytespider":["partial","*",11,0],"Amazonbot":["partial","*",11,0],"meta-externalagent":["partial","*",11,0],"cohere-ai":["partial","*",11,0],"Diffbot":["partial","*",11,0],"ContextFetchBot":["partial","*",11,0]},"llms":{"status":"unreachable","error":"<StreamReset stream_id:3, error_code:2, remote_reset:True>"},"home":{"status":"unreachable","error":"<StreamReset stream_id:5, error_code:2, remote_reset:True>"}},
{"rank":8162,"domain":"newarab.com","origin":"https://newarab.com","robots":{"status":"ok","http":200,"url":"https://www.newarab.com/robots.txt","bytes":1947,"groups":1},"bots":{"GPTBot":["partial","*",31,18],"ChatGPT-User":["partial","*",31,18],"OAI-SearchBot":["partial","*",31,18],"ClaudeBot":["partial","*",31,18],"Claude-User":["partial","*",31,18],"Claude-SearchBot":["partial","*",31,18],"anthropic-ai":["partial","*",31,18],"PerplexityBot":["partial","*",31,18],"Perplexity-User":["partial","*",31,18],"Google-Extended":["partial","*",31,18],"Applebot-Extended":["partial","*",31,18],"CCBot":["partial","*",31,18],"Bytespider":["partial","*",31,18],"Amazonbot":["partial","*",31,18],"meta-externalagent":["partial","*",31,18],"cohere-ai":["partial","*",31,18],"Diffbot":["partial","*",31,18],"ContextFetchBot":["partial","*",31,18]},"llms":{"status":"missing","http":404,"bytes":291},"home":{"status":"ok","http":200,"url":"https://www.newarab.com/","html_tokens":99641,"markdown_tokens":243,"pruned_tokens":70}},
{"rank":8163,"domain":"studocu.com","origin":"https://studocu.com","robots":{"status":"ok","http":200,"url":"https://www.studocu.com/robots.txt","bytes":2698,"groups":2},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["partial","*",61,0],"OAI-SearchBot":["partial","*",61,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",61,0],"Claude-SearchBot":["partial","*",61,0],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["partial","*",61,0],"Perplexity-User":["partial","*",61,0],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["partial","*",61,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",61,0]},"llms":{"status":"http_error","http":403,"bytes":60921},"home":{"status":"http_error","http":403,"url":"https://www.studocu.com/"}},
{"rank":8164,"domain":"midilibre.fr","origin":"https://midilibre.fr","robots":{"status":"ok","http":200,"url":"https://www.midilibre.fr/robots.txt","bytes":8903,"groups":240},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["allowed","named",0,0],"OAI-SearchBot":["allowed","named",0,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",5,0],"Claude-SearchBot":["partial","*",5,0],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["partial","*",5,0],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["partial","*",5,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["partial","*",5,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",5,0]},"llms":{"status":"missing","http":404,"bytes":109276},"home":{"status":"ok","http":200,"url":"https://www.midilibre.fr/","html_tokens":117148,"markdown_tokens":372,"pruned_tokens":149}},
{"rank":8165,"domain":"aoa.org","origin":"https://aoa.org","robots":{"status":"ok","http":200,"url":"https://www.aoa.org/robots.txt","bytes":931,"groups":1},"bots":{"GPTBot":["partial","*",24,0],"ChatGPT-User":["partial","*",24,0],"OAI-SearchBot":["partial","*",24,0],"ClaudeBot":["partial","*",24,0],"Claude-User":["partial","*",24,0],"Claude-SearchBot":["partial","*",24,0],"anthropic-ai":["partial","*",24,0],"PerplexityBot":["partial","*",24,0],"Perplexity-User":["partial","*",24,0],"Google-Extended":["partial","*",24,0],"Applebot-Extended":["partial","*",24,0],"CCBot":["partial","*",24,0],"Bytespider":["partial","*",24,0],"Amazonbot":["partial","*",24,0],"meta-externalagent":["partial","*",24,0],"cohere-ai":["partial","*",24,0],"Diffbot":["partial","*",24,0],"ContextFetchBot":["partial","*",24,0]},"llms":{"status":"missing","http":404,"bytes":101675},"home":{"status":"ok","http":200,"url":"https://www.aoa.org/","html_tokens":27541,"markdown_tokens":123,"pruned_tokens":123}},
{"rank":8166,"domain":"360910.com","origin":"http://360910.com","robots":{"status":"missing","http":404,"url":"http://360910.com/robots.txt","bytes":1308,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":1308},"home":{"status":"ok","http":200,"url":"http://360910.com/","html_tokens":27329,"markdown_tokens":186,"pruned_tokens":129}},
{"rank":8167,"domain":"magicly.bio","origin":"https://magicly.bio","robots":{"status":"ok","http":200,"url":"https://magicly.bio/robots.txt","bytes":559,"groups":6},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["blocked","named",1,0],"OAI-SearchBot":["partial","*",15,1],"ClaudeBot":["partial","*",15,1],"Claude-User":["partial","*",15,1],"Claude-SearchBot":["partial","*",15,1],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["partial","*",15,1],"Perplexity-User":["partial","*",15,1],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["partial","*",15,1],"CCBot":["blocked","named",1,0],"Bytespider":["partial","*",15,1],"Amazonbot":["partial","*",15,1],"meta-externalagent":["partial","*",15,1],"cohere-ai":["partial","*",15,1],"Diffbot":["partial","*",15,1],"ContextFetchBot":["partial","*",15,1]},"llms":{"status":"present","http":200,"bytes":2696},"home":{"status":"ok","http":200,"url":"https://magicly.bio/","html_tokens":28431,"markdown_tokens":95,"pruned_tokens":95}},
{"rank":8168,"domain":"vamtam.com","origin":"https://vamtam.com","robots":{"status":"ok","http":200,"url":"https://vamtam.com/robots.txt","bytes":737,"groups":1},"bots":{"GPTBot":["partial","*",8,1],"ChatGPT-User":["partial","*",8,1],"OAI-SearchBot":["partial","*",8,1],"ClaudeBot":["partial","*",8,1],"Claude-User":["partial","*",8,1],"Claude-SearchBot":["partial","*",8,1],"anthropic-ai":["partial","*",8,1],"PerplexityBot":["partial","*",8,1],"Perplexity-User":["partial","*",8,1],"Google-Extended":["partial","*",8,1],"Applebot-Extended":["partial","*",8,1],"CCBot":["partial","*",8,1],"Bytespider":["partial","*",8,1],"Amazonbot":["partial","*",8,1],"meta-externalagent":["partial","*",8,1],"cohere-ai":["partial","*",8,1],"Diffbot":["partial","*",8,1],"ContextFetchBot":["partial","*",8,1]},"llms":{"status":"other_text","http":200,"bytes":5459},"home":{"status":"ok","http":200,"url":"https://vamtam.com/","html_tokens":83681,"markdown_tokens":951,"pruned_tokens":90}},
{"rank":8169,"domain":"cindmin.com","origin":"http://cindmin.com","robots":{"status":"missing","http":404,"url":"http://cindmin.com/robots.txt","bytes":138,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":138},"home":{"status":"ok","http":200,"url":"http://cindmin.com/","html_tokens":30756,"markdown_tokens":14,"pruned_tokens":14}},
{"rank":8170,"domain":"gifyu.com","origin":"https://gifyu.com","robots":{"status":"ok","http":200,"url":"https://gifyu.com/robots.txt","bytes":3043,"groups":33},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["partial","*",17,2],"OAI-SearchBot":["partial","*",17,2],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",17,2],"Claude-SearchBot":["partial","*",17,2],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["partial","*",17,2],"Perplexity-User":["partial","*",17,2],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",17,2]},"llms":{"status":"missing","http":404,"bytes":51785},"home":{"status":"ok","http":200,"url":"https://gifyu.com/","html_tokens":12735,"markdown_tokens":66,"pruned_tokens":64}},
{"rank":8171,"domain":"cyberlink.com","origin":"https://cyberlink.com","robots":{"status":"ok","http":200,"url":"https://www.cyberlink.com/robots.txt","bytes":10030,"groups":12},"bots":{"GPTBot":["partial","named",30,0],"ChatGPT-User":["partial","named",30,0],"OAI-SearchBot":["partial","*",30,0],"ClaudeBot":["partial","named",30,0],"Claude-User":["partial","*",30,0],"Claude-SearchBot":["partial","*",30,0],"anthropic-ai":["partial","named",30,0],"PerplexityBot":["partial","named",30,0],"Perplexity-User":["partial","*",30,0],"Google-Extended":["partial","named",30,0],"Applebot-Extended":["partial","*",30,0],"CCBot":["partial","named",30,0],"Bytespider":["partial","*",30,0],"Amazonbot":["partial","*",30,0],"meta-externalagent":["partial","*",30,0],"cohere-ai":["partial","*",30,0],"Diffbot":["partial","*",30,0],"ContextFetchBot":["partial","*",30,0]},"llms":{"status":"disallowed"},"home":{"status":"ok","http":200,"url":"https://www.cyberlink.com/","html_tokens":20896,"markdown_tokens":86,"pruned_tokens":85}},
{"rank":8172,"domain":"nsidc.org","origin":"https://nsidc.org","robots":{"status":"ok","http":200,"url":"https://nsidc.org/robots.txt","bytes":2372,"groups":1},"bots":{"GPTBot":["partial","*",40,22],"ChatGPT-User":["partial","*",40,22],"OAI-SearchBot":["partial","*",40,22],"ClaudeBot":["partial","*",40,22],"Claude-User":["partial","*",40,22],"Claude-SearchBot":["partial","*",40,22],"anthropic-ai":["partial","*",40,22],"PerplexityBot":["partial","*",40,22],"Perplexity-User":["partial","*",40,22],"Google-Extended":["partial","*",40,22],"Applebot-Extended":["partial","*",40,22],"CCBot":["partial","*",40,22],"Bytespider":["partial","*",40,22],"Amazonbot":["partial","*",40,22],"meta-externalagent":["partial","*",40,22],"cohere-ai":["partial","*",40,22],"Diffbot":["partial","*",40,22],"ContextFetchBot":["partial","*",40,22]},"llms":{"status":"missing","http":404,"bytes":183},"home":{"status":"ok","http":200,"url":"https://nsidc.org/","html_tokens":48284,"markdown_tokens":874,"pruned_tokens":171}},
{"rank":8173,"domain":"m24.ru","origin":"https://m24.ru","robots":{"status":"ok","http":200,"url":"https://www.m24.ru/robots.txt","bytes":2825,"groups":1},"bots":{"GPTBot":["partial","*",78,0],"ChatGPT-User":["partial","*",78,0],"OAI-SearchBot":["partial","*",78,0],"ClaudeBot":["partial","*",78,0],"Claude-User":["partial","*",78,0],"Claude-SearchBot":["partial","*",78,0],"anthropic-ai":["partial","*",78,0],"PerplexityBot":["partial","*",78,0],"Perplexity-User":["partial","*",78,0],"Google-Extended":["partial","*",78,0],"Applebot-Extended":["partial","*",78,0],"CCBot":["partial","*",78,0],"Bytespider":["partial","*",78,0],"Amazonbot":["partial","*",78,0],"meta-externalagent":["partial","*",78,0],"cohere-ai":["partial","*",78,0],"Diffbot":["partial","*",78,0],"ContextFetchBot":["partial","*",78,0]},"llms":{"status":"missing","http":404,"bytes":74191},"home":{"status":"ok","http":200,"url":"https://www.m24.ru/","html_tokens":45961,"markdown_tokens":1302,"pruned_tokens":129}},
{"rank":8174,"domain":"ruvr.ru","origin":"https://ruvr.ru","robots":{"status":"unreachable","http":null,"url":"https://ruvr.ru/robots.txt","bytes":0,"groups":0,"error":"cannot resolve ruvr.ru"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":8175,"domain":"fox5atlanta.com","origin":"https://fox5atlanta.com","robots":{"status":"ok","http":200,"url":"https://www.fox5atlanta.com/robots.txt","bytes":818,"groups":7},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["partial","*",11,0],"OAI-SearchBot":["partial","*",11,0],"ClaudeBot":["partial","*",11,0],"Claude-User":["partial","*",11,0],"Claude-SearchBot":["partial","*",11,0],"anthropic-ai":["partial","*",11,0],"PerplexityBot":["partial","*",11,0],"Perplexity-User":["partial","*",11,0],"Google-Extended":["partial","*",11,0],"Applebot-Extended":["partial","*",11,0],"CCBot":["partial","*",11,0],"Bytespider":["partial","*",11,0],"Amazonbot":["partial","*",11,0],"meta-externalagent":["partial","*",11,0],"cohere-ai":["partial","*",11,0],"Diffbot":["partial","*",11,0],"ContextFetchBot":["partial","*",11,0]},"llms":{"status":"missing","http":404,"bytes":0},"home":{"status":"ok","http":200,"url":"https://www.fox5atlanta.com/","html_tokens":186628,"markdown_tokens":357,"pruned_tokens":138}},
{"rank":8176,"domain":"newsbtc.com","origin":"https://newsbtc.com","robots":{"status":"ok","http":200,"url":"https://www.newsbtc.com/robots.txt","bytes":335,"groups":1},"bots":{"GPTBot":["partial","*",6,0],"ChatGPT-User":["partial","*",6,0],"OAI-SearchBot":["partial","*",6,0],"ClaudeBot":["partial","*",6,0],"Claude-User":["partial","*",6,0],"Claude-SearchBot":["partial","*",6,0],"anthropic-ai":["partial","*",6,0],"PerplexityBot":["partial","*",6,0],"Perplexity-User":["partial","*",6,0],"Google-Extended":["partial","*",6,0],"Applebot-Extended":["partial","*",6,0],"CCBot":["partial","*",6,0],"Bytespider":["partial","*",6,0],"Amazonbot":["partial","*",6,0],"meta-externalagent":["partial","*",6,0],"cohere-ai":["partial","*",6,0],"Diffbot":["partial","*",6,0],"ContextFetchBot":["partial","*",6,0]},"llms":{"status":"missing","http":404,"bytes":231643},"home":{"status":"ok","http":200,"url":"https://www.newsbtc.com/","html_tokens":78865,"markdown_tokens":2234,"pruned_tokens":590}},
{"rank":8177,"domain":"ivanti.com","origin":"https://ivanti.com","robots":{"status":"ok","http":200,"url":"https://www.ivanti.com/robots.txt","bytes":628,"groups":1},"bots":{"GPTBot":["partial","*",8,0],"ChatGPT-User":["partial","*",8,0],"OAI-SearchBot":["partial","*",8,0],"ClaudeBot":["partial","*",8,0],"Claude-User":["partial","*",8,0],"Claude-SearchBot":["partial","*",8,0],"anthropic-ai":["partial","*",8,0],"PerplexityBot":["partial","*",8,0],"Perplexity-User":["partial","*",8,0],"Google-Extended":["partial","*",8,0],"Applebot-Extended":["partial","*",8,0],"CCBot":["partial","*",8,0],"Bytespider":["partial","*",8,0],"Amazonbot":["partial","*",8,0],"meta-externalagent":["partial","*",8,0],"cohere-ai":["partial","*",8,0],"Diffbot":["partial","*",8,0],"ContextFetchBot":["partial","*",8,0]},"llms":{"status":"missing","http":404,"bytes":1606},"home":{"status":"ok","http":200,"url":"https://www.ivanti.com/","html_tokens":21922,"markdown_tokens":775,"pruned_tokens":139}},
{"rank":8178,"domain":"annefrank.org","origin":"https://annefrank.org","robots":{"status":"server_error","http":500,"url":"https://annefrank.org/robots.txt","bytes":39115,"groups":0,"error":"HTTP 500"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":8179,"domain":"bailii.org","origin":"https://www.bailii.org","robots":{"status":"ok","http":200,"url":"https://www.bailii.org/robots.txt","bytes":360,"groups":2},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["partial","*",10,0],"OAI-SearchBot":["partial","*",10,0],"ClaudeBot":["partial","*",10,0],"Claude-User":["partial","*",10,0],"Claude-SearchBot":["partial","*",10,0],"anthropic-ai":["partial","*",10,0],"PerplexityBot":["partial","*",10,0],"Perplexity-User":["partial","*",10,0],"Google-Extended":["partial","*",10,0],"Applebot-Extended":["partial","*",10,0],"CCBot":["partial","*",10,0],"Bytespider":["partial","*",10,0],"Amazonbot":["partial","*",10,0],"meta-externalagent":["partial","*",10,0],"cohere-ai":["partial","*",10,0],"Diffbot":["partial","*",10,0],"ContextFetchBot":["partial","*",10,0]},"llms":{"status":"missing","http":404,"bytes":158},"home":{"status":"ok","http":200,"url":"https://www.bailii.org/","html_tokens":2784,"markdown_tokens":671,"pruned_tokens":308}},
{"rank":8180,"domain":"1c.ru","origin":"https://1c.ru","robots":{"status":"ok","http":200,"url":"https://1c.ru/robots.txt","bytes":1114,"groups":9},"bots":{"GPTBot":["partial","*",17,0],"ChatGPT-User":["partial","*",17,0],"OAI-SearchBot":["partial","*",17,0],"ClaudeBot":["partial","*",17,0],"Claude-User":["partial","*",17,0],"Claude-SearchBot":["partial","*",17,0],"anthropic-ai":["partial","*",17,0],"PerplexityBot":["partial","*",17,0],"Perplexity-User":["partial","*",17,0],"Google-Extended":["partial","*",17,0],"Applebot-Extended":["partial","*",17,0],"CCBot":["partial","*",17,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["partial","*",17,0],"meta-externalagent":["partial","*",17,0],"cohere-ai":["partial","*",17,0],"Diffbot":["partial","*",17,0],"ContextFetchBot":["partial","*",17,0]},"llms":{"status":"missing","http":404,"bytes":206},"home":{"status":"ok","http":200,"url":"https://1c.ru/","html_tokens":15771,"markdown_tokens":164,"pruned_tokens":164}},
{"rank":8181,"domain":"aizenway.cn","origin":"http://aizenway.cn","robots":{"status":"missing","http":404,"url":"http://aizenway.cn/robots.txt","bytes":1231,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":1231},"home":{"status":"timeout","error":"timed out after 15 s"}},
{"rank":8182,"domain":"hrp.org.uk","origin":"https://hrp.org.uk","robots":{"status":"ok","http":200,"url":"https://www.hrp.org.uk/robots.txt","bytes":297,"groups":1},"bots":{"GPTBot":["partial","*",9,1],"ChatGPT-User":["partial","*",9,1],"OAI-SearchBot":["partial","*",9,1],"ClaudeBot":["partial","*",9,1],"Claude-User":["partial","*",9,1],"Claude-SearchBot":["partial","*",9,1],"anthropic-ai":["partial","*",9,1],"PerplexityBot":["partial","*",9,1],"Perplexity-User":["partial","*",9,1],"Google-Extended":["partial","*",9,1],"Applebot-Extended":["partial","*",9,1],"CCBot":["partial","*",9,1],"Bytespider":["partial","*",9,1],"Amazonbot":["partial","*",9,1],"meta-externalagent":["partial","*",9,1],"cohere-ai":["partial","*",9,1],"Diffbot":["partial","*",9,1],"ContextFetchBot":["partial","*",9,1]},"llms":{"status":"missing","http":404,"bytes":517810},"home":{"status":"ok","http":200,"url":"https://www.hrp.org.uk/","html_tokens":136665,"markdown_tokens":568,"pruned_tokens":124}},
{"rank":8183,"domain":"google.com.cy","origin":"https://google.com.cy","robots":{"status":"ok","http":200,"url":"https://www.google.com.cy/robots.txt","bytes":6624,"groups":4},"bots":{"GPTBot":["partial","*",169,64],"ChatGPT-User":["partial","*",169,64],"OAI-SearchBot":["partial","*",169,64],"ClaudeBot":["partial","*",169,64],"Claude-User":["partial","*",169,64],"Claude-SearchBot":["partial","*",169,64],"anthropic-ai":["partial","*",169,64],"PerplexityBot":["partial","*",169,64],"Perplexity-User":["partial","*",169,64],"Google-Extended":["partial","*",169,64],"Applebot-Extended":["partial","*",169,64],"CCBot":["partial","*",169,64],"Bytespider":["partial","*",169,64],"Amazonbot":["partial","*",169,64],"meta-externalagent":["partial","*",169,64],"cohere-ai":["partial","*",169,64],"Diffbot":["partial","*",169,64],"ContextFetchBot":["partial","*",169,64]},"llms":{"status":"missing","http":404,"bytes":1569},"home":{"status":"ok","http":200,"url":"https://www.google.com/","html_tokens":21265,"markdown_tokens":30,"pruned_tokens":30}},
{"rank":8184,"domain":"austria.info","origin":"https://austria.info","robots":{"status":"ok","http":200,"url":"https://www.austria.info/robots.txt","bytes":158,"groups":1},"bots":{"GPTBot":["partial","*",5,0],"ChatGPT-User":["partial","*",5,0],"OAI-SearchBot":["partial","*",5,0],"ClaudeBot":["partial","*",5,0],"Claude-User":["partial","*",5,0],"Claude-SearchBot":["partial","*",5,0],"anthropic-ai":["partial","*",5,0],"PerplexityBot":["partial","*",5,0],"Perplexity-User":["partial","*",5,0],"Google-Extended":["partial","*",5,0],"Applebot-Extended":["partial","*",5,0],"CCBot":["partial","*",5,0],"Bytespider":["partial","*",5,0],"Amazonbot":["partial","*",5,0],"meta-externalagent":["partial","*",5,0],"cohere-ai":["partial","*",5,0],"Diffbot":["partial","*",5,0],"ContextFetchBot":["partial","*",5,0]},"llms":{"status":"http_error","http":403,"bytes":0},"home":{"status":"ok","http":200,"url":"https://www.austria.info/en-us/","html_tokens":807906,"markdown_tokens":114,"pruned_tokens":114}},
{"rank":8185,"domain":"erols.com","origin":"https://erols.com","robots":{"status":"unreachable","http":null,"url":"https://erols.com/robots.txt","bytes":0,"groups":0,"error":"cannot resolve erols.com"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":8186,"domain":"muji.com","origin":"https://muji.com","robots":{"status":"timeout","http":null,"url":"https://muji.com/robots.txt","bytes":0,"groups":0,"error":"ConnectTimeout"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":8187,"domain":"aapanel.com","origin":"https://aapanel.com","robots":{"status":"ok","http":200,"url":"https://www.aapanel.com/robots.txt","bytes":69,"groups":1},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"present","http":200,"bytes":2347},"home":{"status":"ok","http":200,"url":"https://www.aapanel.com/","html_tokens":64652,"markdown_tokens":1324,"pruned_tokens":393}},
{"rank":8188,"domain":"canon.jp","origin":"https://canon.jp","robots":{"status":"ok","http":200,"url":"https://canon.jp/robots.txt","bytes":201,"groups":1},"bots":{"GPTBot":["allowed","*",0,1],"ChatGPT-User":["allowed","*",0,1],"OAI-SearchBot":["allowed","*",0,1],"ClaudeBot":["allowed","*",0,1],"Claude-User":["allowed","*",0,1],"Claude-SearchBot":["allowed","*",0,1],"anthropic-ai":["allowed","*",0,1],"PerplexityBot":["allowed","*",0,1],"Perplexity-User":["allowed","*",0,1],"Google-Extended":["allowed","*",0,1],"Applebot-Extended":["allowed","*",0,1],"CCBot":["allowed","*",0,1],"Bytespider":["allowed","*",0,1],"Amazonbot":["allowed","*",0,1],"meta-externalagent":["allowed","*",0,1],"cohere-ai":["allowed","*",0,1],"Diffbot":["allowed","*",0,1],"ContextFetchBot":["allowed","*",0,1]},"llms":{"status":"missing","http":404,"bytes":103},"home":{"status":"ok","http":200,"url":"https://canon.jp/","html_tokens":14286,"markdown_tokens":416,"pruned_tokens":88}},
{"rank":8189,"domain":"ashpublications.org","origin":"https://ashpublications.org","robots":{"status":"ok","http":200,"url":"https://ashpublications.org/robots.txt","bytes":2066,"groups":2},"bots":{"GPTBot":["partial","*",41,3],"ChatGPT-User":["partial","*",41,3],"OAI-SearchBot":["partial","*",41,3],"ClaudeBot":["partial","*",41,3],"Claude-User":["partial","*",41,3],"Claude-SearchBot":["partial","*",41,3],"anthropic-ai":["partial","*",41,3],"PerplexityBot":["partial","*",41,3],"Perplexity-User":["partial","*",41,3],"Google-Extended":["partial","*",41,3],"Applebot-Extended":["partial","*",41,3],"CCBot":["partial","*",41,3],"Bytespider":["partial","*",41,3],"Amazonbot":["partial","*",41,3],"meta-externalagent":["partial","*",41,3],"cohere-ai":["partial","*",41,3],"Diffbot":["partial","*",41,3],"ContextFetchBot":["partial","*",41,3]},"llms":{"status":"http_error","http":403,"bytes":5530},"home":{"status":"http_error","http":403,"url":"https://ashpublications.org/"}},
{"rank":8190,"domain":"doingbusiness.org","origin":"https://doingbusiness.org","robots":{"status":"missing","http":404,"url":"https://www.worldbank.org/en/businessready/robots.txt","bytes":5128,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":5128},"home":{"status":"ok","http":200,"url":"https://www.worldbank.org/en/businessready","html_tokens":11849,"markdown_tokens":53,"pruned_tokens":53}},
{"rank":8191,"domain":"peopledaily.com.cn","origin":"https://peopledaily.com.cn","robots":{"status":"unreachable","http":null,"url":"https://peopledaily.com.cn/robots.txt","bytes":0,"groups":0,"error":"cannot resolve peopledaily.com.cn"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":8192,"domain":"google.com.af","origin":"https://google.com.af","robots":{"status":"ok","http":200,"url":"https://www.google.com.af/robots.txt","bytes":6624,"groups":4},"bots":{"GPTBot":["partial","*",169,64],"ChatGPT-User":["partial","*",169,64],"OAI-SearchBot":["partial","*",169,64],"ClaudeBot":["partial","*",169,64],"Claude-User":["partial","*",169,64],"Claude-SearchBot":["partial","*",169,64],"anthropic-ai":["partial","*",169,64],"PerplexityBot":["partial","*",169,64],"Perplexity-User":["partial","*",169,64],"Google-Extended":["partial","*",169,64],"Applebot-Extended":["partial","*",169,64],"CCBot":["partial","*",169,64],"Bytespider":["partial","*",169,64],"Amazonbot":["partial","*",169,64],"meta-externalagent":["partial","*",169,64],"cohere-ai":["partial","*",169,64],"Diffbot":["partial","*",169,64],"ContextFetchBot":["partial","*",169,64]},"llms":{"status":"missing","http":404,"bytes":1569},"home":{"status":"ok","http":200,"url":"https://www.google.com/","html_tokens":21291,"markdown_tokens":30,"pruned_tokens":30}},
{"rank":8193,"domain":"caeb.cn","origin":"https://caeb.cn","robots":{"status":"timeout","http":null,"url":"https://caeb.cn/robots.txt","bytes":0,"groups":0,"error":"ConnectTimeout"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":8194,"domain":"famitsu.com","origin":"https://famitsu.com","robots":{"status":"ok","http":200,"url":"https://www.famitsu.com/robots.txt","bytes":570,"groups":1},"bots":{"GPTBot":["partial","*",23,0],"ChatGPT-User":["partial","*",23,0],"OAI-SearchBot":["partial","*",23,0],"ClaudeBot":["partial","*",23,0],"Claude-User":["partial","*",23,0],"Claude-SearchBot":["partial","*",23,0],"anthropic-ai":["partial","*",23,0],"PerplexityBot":["partial","*",23,0],"Perplexity-User":["partial","*",23,0],"Google-Extended":["partial","*",23,0],"Applebot-Extended":["partial","*",23,0],"CCBot":["partial","*",23,0],"Bytespider":["partial","*",23,0],"Amazonbot":["partial","*",23,0],"meta-externalagent":["partial","*",23,0],"cohere-ai":["partial","*",23,0],"Diffbot":["partial","*",23,0],"ContextFetchBot":["partial","*",23,0]},"llms":{"status":"missing","http":404,"bytes":27739},"home":{"status":"ok","http":200,"url":"https://www.famitsu.com/","html_tokens":171778,"markdown_tokens":23,"pruned_tokens":23}},
{"rank":8195,"domain":"insta360.com","origin":"https://insta360.com","robots":{"status":"ok","http":200,"url":"https://www.insta360.com/robots.txt","bytes":714,"groups":1},"bots":{"GPTBot":["partial","*",5,1],"ChatGPT-User":["partial","*",5,1],"OAI-SearchBot":["partial","*",5,1],"ClaudeBot":["partial","*",5,1],"Claude-User":["partial","*",5,1],"Claude-SearchBot":["partial","*",5,1],"anthropic-ai":["partial","*",5,1],"PerplexityBot":["partial","*",5,1],"Perplexity-User":["partial","*",5,1],"Google-Extended":["partial","*",5,1],"Applebot-Extended":["partial","*",5,1],"CCBot":["partial","*",5,1],"Bytespider":["partial","*",5,1],"Amazonbot":["partial","*",5,1],"meta-externalagent":["partial","*",5,1],"cohere-ai":["partial","*",5,1],"Diffbot":["partial","*",5,1],"ContextFetchBot":["partial","*",5,1]},"llms":{"status":"missing","http":404,"bytes":9106},"home":{"status":"http_error","http":403,"url":"https://www.insta360.com/"}},
{"rank":8196,"domain":"thoughtworks.com","origin":"https://thoughtworks.com","robots":{"status":"ok","http":200,"url":"https://thoughtworks.com/robots.txt","bytes":4342,"groups":1},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"other_text","http":200,"bytes":17180},"home":{"status":"not_html","error":"unsupported content type text/markdown"}},
{"rank":8197,"domain":"berkshirehathaway.com","origin":"https://berkshirehathaway.com","robots":{"status":"missing","http":404,"url":"https://berkshirehathaway.com/robots.txt","bytes":355,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":355},"home":{"status":"ok","http":200,"url":"https://berkshirehathaway.com/","html_tokens":1384,"markdown_tokens":119,"pruned_tokens":83}},
{"rank":8198,"domain":"hust.edu.cn","origin":"https://hust.edu.cn","robots":{"status":"missing","http":404,"url":"https://hust.edu.cn/robots.txt","bytes":1693,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":1693},"home":{"status":"ok","http":200,"url":"https://hust.edu.cn/","html_tokens":17655,"markdown_tokens":13,"pruned_tokens":13}},
{"rank":8199,"domain":"nicemedia.com.cn","origin":"https://nicemedia.com.cn","robots":{"status":"unreachable","http":null,"url":"https://nicemedia.com.cn/robots.txt","bytes":0,"groups":0,"error":"ConnectError"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":8200,"domain":"yingdai.online","origin":"http://yingdai.online","robots":{"status":"missing","http":404,"url":"http://yingdai.online/robots.txt","bytes":6835,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":6835},"home":{"status":"ok","http":200,"url":"http://yingdai.online/","html_tokens":12652,"markdown_tokens":258,"pruned_tokens":83}},
{"rank":8201,"domain":"med.or.jp","origin":"https://med.or.jp","robots":{"status":"ok","http":200,"url":"https://med.or.jp/robots.txt","bytes":0,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":196},"home":{"status":"ok","http":200,"url":"https://med.or.jp/","html_tokens":7962,"markdown_tokens":9,"pruned_tokens":9}},
{"rank":8202,"domain":"wordwall.net","origin":"https://wordwall.net","robots":{"status":"ok","http":200,"url":"https://wordwall.net/robots.txt","bytes":185,"groups":1},"bots":{"GPTBot":["partial","*",6,0],"ChatGPT-User":["partial","*",6,0],"OAI-SearchBot":["partial","*",6,0],"ClaudeBot":["partial","*",6,0],"Claude-User":["partial","*",6,0],"Claude-SearchBot":["partial","*",6,0],"anthropic-ai":["partial","*",6,0],"PerplexityBot":["partial","*",6,0],"Perplexity-User":["partial","*",6,0],"Google-Extended":["partial","*",6,0],"Applebot-Extended":["partial","*",6,0],"CCBot":["partial","*",6,0],"Bytespider":["partial","*",6,0],"Amazonbot":["partial","*",6,0],"meta-externalagent":["partial","*",6,0],"cohere-ai":["partial","*",6,0],"Diffbot":["partial","*",6,0],"ContextFetchBot":["partial","*",6,0]},"llms":{"status":"missing","http":404,"bytes":26385},"home":{"status":"ok","http":200,"url":"https://wordwall.net/","html_tokens":13311,"markdown_tokens":408,"pruned_tokens":136}},
{"rank":8203,"domain":"glitch.me","origin":"https://www.glitch.me","robots":{"status":"missing","http":410,"url":"https://www.glitch.me/robots.txt","bytes":228794,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":410,"bytes":228794},"home":{"status":"http_error","http":410,"url":"https://www.glitch.me/"}},
{"rank":8204,"domain":"365danzhao.cn","origin":"http://365danzhao.cn","robots":{"status":"missing","http":404,"url":"http://365danzhao.cn/robots.txt","bytes":1308,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":1308},"home":{"status":"ok","http":200,"url":"http://365danzhao.cn/","html_tokens":10745,"markdown_tokens":18,"pruned_tokens":18}},
{"rank":8205,"domain":"ck.page","origin":"https://ck.page","robots":{"status":"server_error","http":525,"url":"https://ck.page/robots.txt","bytes":7695,"groups":0,"error":"HTTP 525"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":8206,"domain":"motogp.com","origin":"https://motogp.com","robots":{"status":"ok","http":200,"url":"https://www.motogp.com/robots.txt","bytes":1477,"groups":2},"bots":{"GPTBot":["partial","*",44,0],"ChatGPT-User":["partial","*",44,0],"OAI-SearchBot":["partial","*",44,0],"ClaudeBot":["partial","*",44,0],"Claude-User":["partial","*",44,0],"Claude-SearchBot":["partial","*",44,0],"anthropic-ai":["partial","*",44,0],"PerplexityBot":["partial","*",44,0],"Perplexity-User":["partial","*",44,0],"Google-Extended":["partial","*",44,0],"Applebot-Extended":["partial","*",44,0],"CCBot":["partial","*",44,0],"Bytespider":["partial","*",44,0],"Amazonbot":["partial","*",44,0],"meta-externalagent":["partial","*",44,0],"cohere-ai":["partial","*",44,0],"Diffbot":["partial","*",44,0],"ContextFetchBot":["partial","*",44,0]},"llms":{"status":"missing","http":404,"bytes":107735},"home":{"status":"ok","http":200,"url":"https://www.motogp.com/","html_tokens":104457,"markdown_tokens":2615,"pruned_tokens":957}},
{"rank":8207,"domain":"uml.edu","origin":"https://uml.edu","robots":{"status":"ok","http":200,"url":"https://www.uml.edu/robots.txt","bytes":16013,"groups":8},"bots":{"GPTBot":["partial","*",496,3],"ChatGPT-User":["partial","*",496,3],"OAI-SearchBot":["partial","*",496,3],"ClaudeBot":["partial","*",496,3],"Claude-User":["partial","*",496,3],"Claude-SearchBot":["partial","*",496,3],"anthropic-ai":["partial","*",496,3],"PerplexityBot":["partial","*",496,3],"Perplexity-User":["partial","*",496,3],"Google-Extended":["partial","*",496,3],"Applebot-Extended":["partial","*",496,3],"CCBot":["partial","*",496,3],"Bytespider":["partial","*",496,3],"Amazonbot":["partial","*",496,3],"meta-externalagent":["partial","*",496,3],"cohere-ai":["partial","*",496,3],"Diffbot":["partial","*",496,3],"ContextFetchBot":["partial","*",496,3]},"llms":{"status":"missing","http":404,"bytes":6176},"home":{"status":"ok","http":200,"url":"https://www.uml.edu/","html_tokens":7820,"markdown_tokens":210,"pruned_tokens":170}},
{"rank":8208,"domain":"xdfsh.com","origin":"http://xdfsh.com","robots":{"status":"html","http":200,"url":"https://open.weixin.qq.com/connect/oauth2/authorize?appid=wxfa499b2c3ab05c56&redirect_uri=http%3A%2F%2Fxdfsh.com%2Folv%2FwxLoginBack&response_type=code&scope=snsapi_userinfo&state=25b710d3-e9fa-4aff-8109-7720095a439f&connect_redirect=1#wechat_redirect","bytes":749,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":8209,"domain":"ascii.jp","origin":"https://ascii.jp","robots":{"status":"ok","http":200,"url":"https://ascii.jp/robots.txt","bytes":159,"groups":2},"bots":{"GPTBot":["partial","*",1,0],"ChatGPT-User":["partial","*",1,0],"OAI-SearchBot":["partial","*",1,0],"ClaudeBot":["partial","*",1,0],"Claude-User":["partial","*",1,0],"Claude-SearchBot":["partial","*",1,0],"anthropic-ai":["partial","*",1,0],"PerplexityBot":["partial","*",1,0],"Perplexity-User":["partial","*",1,0],"Google-Extended":["partial","*",1,0],"Applebot-Extended":["partial","*",1,0],"CCBot":["partial","*",1,0],"Bytespider":["partial","*",1,0],"Amazonbot":["partial","*",1,0],"meta-externalagent":["partial","*",1,0],"cohere-ai":["partial","*",1,0],"Diffbot":["partial","*",1,0],"ContextFetchBot":["partial","*",1,0]},"llms":{"status":"missing","http":404,"bytes":61982},"home":{"status":"ok","http":200,"url":"https://ascii.jp/","html_tokens":28479,"markdown_tokens":384,"pruned_tokens":163}},
{"rank":8210,"domain":"hiroshima.jp","origin":"https://hiroshima.jp","robots":{"status":"unreachable","http":null,"url":"https://hiroshima.jp/robots.txt","bytes":0,"groups":0,"error":"cannot resolve hiroshima.jp"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":8211,"domain":"acyclovir-400-mg.store","origin":"https://acyclovir-400-mg.store","robots":{"status":"ok","http":200,"url":"https://acyclovir-400-mg.store/robots.txt","bytes":83,"groups":1},"bots":{"GPTBot":["partial","*",1,0],"ChatGPT-User":["partial","*",1,0],"OAI-SearchBot":["partial","*",1,0],"ClaudeBot":["partial","*",1,0],"Claude-User":["partial","*",1,0],"Claude-SearchBot":["partial","*",1,0],"anthropic-ai":["partial","*",1,0],"PerplexityBot":["partial","*",1,0],"Perplexity-User":["partial","*",1,0],"Google-Extended":["partial","*",1,0],"Applebot-Extended":["partial","*",1,0],"CCBot":["partial","*",1,0],"Bytespider":["partial","*",1,0],"Amazonbot":["partial","*",1,0],"meta-externalagent":["partial","*",1,0],"cohere-ai":["partial","*",1,0],"Diffbot":["partial","*",1,0],"ContextFetchBot":["partial","*",1,0]},"llms":{"status":"missing","http":404,"bytes":651},"home":{"status":"ok","http":200,"url":"https://acyclovir-400-mg.store/","html_tokens":16683,"markdown_tokens":4468,"pruned_tokens":816}},
{"rank":8212,"domain":"hitnspinslots.com","origin":"https://hitnspinslots.com","robots":{"status":"unreachable","http":null,"url":"https://hitnspinslots.com/robots.txt","bytes":0,"groups":0,"error":"cannot resolve hitnspinslots.com"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":8213,"domain":"prnasia.com","origin":"https://prnasia.com","robots":{"status":"ok","http":200,"url":"https://www.prnasia.com/robots.txt","bytes":2563,"groups":1},"bots":{"GPTBot":["partial","*",101,0],"ChatGPT-User":["partial","*",101,0],"OAI-SearchBot":["partial","*",101,0],"ClaudeBot":["partial","*",101,0],"Claude-User":["partial","*",101,0],"Claude-SearchBot":["partial","*",101,0],"anthropic-ai":["partial","*",101,0],"PerplexityBot":["partial","*",101,0],"Perplexity-User":["partial","*",101,0],"Google-Extended":["partial","*",101,0],"Applebot-Extended":["partial","*",101,0],"CCBot":["partial","*",101,0],"Bytespider":["partial","*",101,0],"Amazonbot":["partial","*",101,0],"meta-externalagent":["partial","*",101,0],"cohere-ai":["partial","*",101,0],"Diffbot":["partial","*",101,0],"ContextFetchBot":["partial","*",101,0]},"llms":{"status":"missing","http":404,"bytes":944},"home":{"status":"ok","http":200,"url":"https://www.prnasia.com/","html_tokens":27933,"markdown_tokens":10,"pruned_tokens":10}},
{"rank":8214,"domain":"tiobe.com","origin":"https://tiobe.com","robots":{"status":"ok","http":200,"url":"https://tiobe.com/robots.txt","bytes":115,"groups":1},"bots":{"GPTBot":["partial","*",1,1],"ChatGPT-User":["partial","*",1,1],"OAI-SearchBot":["partial","*",1,1],"ClaudeBot":["partial","*",1,1],"Claude-User":["partial","*",1,1],"Claude-SearchBot":["partial","*",1,1],"anthropic-ai":["partial","*",1,1],"PerplexityBot":["partial","*",1,1],"Perplexity-User":["partial","*",1,1],"Google-Extended":["partial","*",1,1],"Applebot-Extended":["partial","*",1,1],"CCBot":["partial","*",1,1],"Bytespider":["partial","*",1,1],"Amazonbot":["partial","*",1,1],"meta-externalagent":["partial","*",1,1],"cohere-ai":["partial","*",1,1],"Diffbot":["partial","*",1,1],"ContextFetchBot":["partial","*",1,1]},"llms":{"status":"missing","http":404,"bytes":23121},"home":{"status":"ok","http":200,"url":"https://www.tiobe.com/","html_tokens":13488,"markdown_tokens":1026,"pruned_tokens":352}},
{"rank":8215,"domain":"qdedu.cn","origin":"https://qdedu.cn","robots":{"status":"unreachable","http":null,"url":"https://qdedu.cn/robots.txt","bytes":0,"groups":0,"error":"cannot resolve qdedu.cn"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":8216,"domain":"unioncrediting.com","origin":"http://unioncrediting.com","robots":{"status":"missing","http":404,"url":"http://unioncrediting.com/robots.txt","bytes":1163,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":1163},"home":{"status":"ok","http":200,"url":"http://unioncrediting.com/","html_tokens":7707,"markdown_tokens":126,"pruned_tokens":111}},
{"rank":8217,"domain":"rodeointl.com.cn","origin":"http://rodeointl.com.cn","robots":{"status":"missing","http":404,"url":"http://rodeointl.com.cn/robots.txt","bytes":315,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":315},"home":{"status":"http_error","http":404,"url":"http://rodeointl.com.cn/"}},
{"rank":8218,"domain":"perfil.com","origin":"https://perfil.com","robots":{"status":"ok","http":200,"url":"https://www.perfil.com/robots.txt","bytes":271,"groups":1},"bots":{"GPTBot":["allowed","*",0,1],"ChatGPT-User":["allowed","*",0,1],"OAI-SearchBot":["allowed","*",0,1],"ClaudeBot":["allowed","*",0,1],"Claude-User":["allowed","*",0,1],"Claude-SearchBot":["allowed","*",0,1],"anthropic-ai":["allowed","*",0,1],"PerplexityBot":["allowed","*",0,1],"Perplexity-User":["allowed","*",0,1],"Google-Extended":["allowed","*",0,1],"Applebot-Extended":["allowed","*",0,1],"CCBot":["allowed","*",0,1],"Bytespider":["allowed","*",0,1],"Amazonbot":["allowed","*",0,1],"meta-externalagent":["allowed","*",0,1],"cohere-ai":["allowed","*",0,1],"Diffbot":["allowed","*",0,1],"ContextFetchBot":["allowed","*",0,1]},"llms":{"status":"missing","http":404,"bytes":57228},"home":{"status":"ok","http":200,"url":"https://www.perfil.com/","html_tokens":128233,"markdown_tokens":94,"pruned_tokens":94}},
{"rank":8219,"domain":"pepperdine.edu","origin":"https://pepperdine.edu","robots":{"status":"ok","http":200,"url":"https://www.pepperdine.edu/robots.txt","bytes":6698,"groups":2},"bots":{"GPTBot":["partial","*",174,0],"ChatGPT-User":["partial","*",174,0],"OAI-SearchBot":["partial","*",174,0],"ClaudeBot":["partial","*",174,0],"Claude-User":["partial","*",174,0],"Claude-SearchBot":["partial","*",174,0],"anthropic-ai":["partial","*",174,0],"PerplexityBot":["partial","*",174,0],"Perplexity-User":["partial","*",174,0],"Google-Extended":["partial","*",174,0],"Applebot-Extended":["partial","*",174,0],"CCBot":["partial","*",174,0],"Bytespider":["partial","*",174,0],"Amazonbot":["partial","*",174,0],"meta-externalagent":["partial","*",174,0],"cohere-ai":["partial","*",174,0],"Diffbot":["partial","*",174,0],"ContextFetchBot":["partial","*",174,0]},"llms":{"status":"missing","http":404,"bytes":27222},"home":{"status":"ok","http":200,"url":"https://www.pepperdine.edu/","html_tokens":12691,"markdown_tokens":339,"pruned_tokens":162}},
{"rank":8220,"domain":"iqilu.com","origin":"https://iqilu.com","robots":{"status":"html","http":200,"url":"http://www.iqilu.com/html/404/index404.shtml","bytes":1528,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":8221,"domain":"storyblok.com","origin":"https://storyblok.com","robots":{"status":"ok","http":200,"url":"https://www.storyblok.com/robots.txt","bytes":78,"groups":1},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"present","http":200,"bytes":4669},"home":{"status":"ok","http":200,"url":"https://www.storyblok.com/","html_tokens":56596,"markdown_tokens":969,"pruned_tokens":183}},
{"rank":8222,"domain":"answersingenesis.org","origin":"https://answersingenesis.org","robots":{"status":"ok","http":200,"url":"https://answersingenesis.org/robots.txt","bytes":384,"groups":1},"bots":{"GPTBot":["partial","*",4,0],"ChatGPT-User":["partial","*",4,0],"OAI-SearchBot":["partial","*",4,0],"ClaudeBot":["partial","*",4,0],"Claude-User":["partial","*",4,0],"Claude-SearchBot":["partial","*",4,0],"anthropic-ai":["partial","*",4,0],"PerplexityBot":["partial","*",4,0],"Perplexity-User":["partial","*",4,0],"Google-Extended":["partial","*",4,0],"Applebot-Extended":["partial","*",4,0],"CCBot":["partial","*",4,0],"Bytespider":["partial","*",4,0],"Amazonbot":["partial","*",4,0],"meta-externalagent":["partial","*",4,0],"cohere-ai":["partial","*",4,0],"Diffbot":["partial","*",4,0],"ContextFetchBot":["partial","*",4,0]},"llms":{"status":"missing","http":404,"bytes":39382},"home":{"status":"ok","http":200,"url":"https://answersingenesis.org/","html_tokens":20359,"markdown_tokens":436,"pruned_tokens":142}},
{"rank":8223,"domain":"tulsaworld.com","origin":"https://tulsaworld.com","robots":{"status":"ok","http":200,"url":"https://tulsaworld.com/robots.txt","bytes":11087,"groups":1},"bots":{"GPTBot":["partial","*",134,0],"ChatGPT-User":["partial","*",134,0],"OAI-SearchBot":["partial","*",134,0],"ClaudeBot":["partial","*",134,0],"Claude-User":["partial","*",134,0],"Claude-SearchBot":["partial","*",134,0],"anthropic-ai":["partial","*",134,0],"PerplexityBot":["partial","*",134,0],"Perplexity-User":["partial","*",134,0],"Google-Extended":["partial","*",134,0],"Applebot-Extended":["partial","*",134,0],"CCBot":["partial","*",134,0],"Bytespider":["partial","*",134,0],"Amazonbot":["partial","*",134,0],"meta-externalagent":["partial","*",134,0],"cohere-ai":["partial","*",134,0],"Diffbot":["partial","*",134,0],"ContextFetchBot":["partial","*",134,0]},"llms":{"status":"missing","http":404,"bytes":2363},"home":{"status":"ok","http":200,"url":"https://tulsaworld.com/","html_tokens":129683,"markdown_tokens":1067,"pruned_tokens":48}},
{"rank":8224,"domain":"deutsche-rentenversicherung.de","origin":"https://deutsche-rentenversicherung.de","robots":{"status":"ok","http":200,"url":"https://www.deutsche-rentenversicherung.de/robots.txt","bytes":481,"groups":4},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["partial","*",5,5],"OAI-SearchBot":["partial","*",5,5],"ClaudeBot":["partial","*",5,5],"Claude-User":["partial","*",5,5],"Claude-SearchBot":["partial","*",5,5],"anthropic-ai":["partial","*",5,5],"PerplexityBot":["partial","*",5,5],"Perplexity-User":["partial","*",5,5],"Google-Extended":["partial","*",5,5],"Applebot-Extended":["partial","*",5,5],"CCBot":["partial","*",5,5],"Bytespider":["partial","*",5,5],"Amazonbot":["partial","*",5,5],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["partial","*",5,5],"Diffbot":["partial","*",5,5],"ContextFetchBot":["partial","*",5,5]},"llms":{"status":"missing","http":404,"bytes":140439},"home":{"status":"ok","http":200,"url":"https://www.deutsche-rentenversicherung.de/DRV/DE/Home","html_tokens":49668,"markdown_tokens":345,"pruned_tokens":107}},
{"rank":8225,"domain":"ausfreeslots.com","origin":"https://ausfreeslots.com","robots":{"status":"timeout","http":null,"url":"https://ausfreeslots.com/robots.txt","bytes":0,"groups":0,"error":"timed out after 10 s"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":8226,"domain":"50webs.com","origin":"https://50webs.com","robots":{"status":"ok","http":200,"url":"https://www.50webs.com/robots.txt","bytes":198,"groups":1},"bots":{"GPTBot":["partial","*",7,0],"ChatGPT-User":["partial","*",7,0],"OAI-SearchBot":["partial","*",7,0],"ClaudeBot":["partial","*",7,0],"Claude-User":["partial","*",7,0],"Claude-SearchBot":["partial","*",7,0],"anthropic-ai":["partial","*",7,0],"PerplexityBot":["partial","*",7,0],"Perplexity-User":["partial","*",7,0],"Google-Extended":["partial","*",7,0],"Applebot-Extended":["partial","*",7,0],"CCBot":["partial","*",7,0],"Bytespider":["partial","*",7,0],"Amazonbot":["partial","*",7,0],"meta-externalagent":["partial","*",7,0],"cohere-ai":["partial","*",7,0],"Diffbot":["partial","*",7,0],"ContextFetchBot":["partial","*",7,0]},"llms":{"status":"missing","http":404,"bytes":284303},"home":{"status":"ok","http":200,"url":"https://www.50webs.com/","html_tokens":202173,"markdown_tokens":280,"pruned_tokens":76}},
{"rank":8227,"domain":"mantoo.com.cn","origin":"http://mantoo.com.cn","robots":{"status":"html","http":200,"url":"https://wanwang.aliyun.com/hosting/ipvisit_stop","bytes":44540,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":8228,"domain":"scrapbox.io","origin":"https://scrapbox.io","robots":{"status":"ok","http":200,"url":"https://scrapbox.io/robots.txt","bytes":23,"groups":1},"bots":{"GPTBot":["allowed","*",0,1],"ChatGPT-User":["allowed","*",0,1],"OAI-SearchBot":["allowed","*",0,1],"ClaudeBot":["allowed","*",0,1],"Claude-User":["allowed","*",0,1],"Claude-SearchBot":["allowed","*",0,1],"anthropic-ai":["allowed","*",0,1],"PerplexityBot":["allowed","*",0,1],"Perplexity-User":["allowed","*",0,1],"Google-Extended":["allowed","*",0,1],"Applebot-Extended":["allowed","*",0,1],"CCBot":["allowed","*",0,1],"Bytespider":["allowed","*",0,1],"Amazonbot":["allowed","*",0,1],"meta-externalagent":["allowed","*",0,1],"cohere-ai":["allowed","*",0,1],"Diffbot":["allowed","*",0,1],"ContextFetchBot":["allowed","*",0,1]},"llms":{"status":"missing","http":404,"bytes":1473},"home":{"status":"ok","http":200,"url":"https://scrapbox.io/","html_tokens":1859,"markdown_tokens":75,"pruned_tokens":75}},
{"rank":8229,"domain":"tastingtable.com","origin":"https://tastingtable.com","robots":{"status":"ok","http":200,"url":"https://tastingtable.com/robots.txt","bytes":263,"groups":2},"bots":{"GPTBot":["partial","*",4,0],"ChatGPT-User":["partial","*",4,0],"OAI-SearchBot":["partial","*",4,0],"ClaudeBot":["partial","*",4,0],"Claude-User":["partial","*",4,0],"Claude-SearchBot":["partial","*",4,0],"anthropic-ai":["partial","*",4,0],"PerplexityBot":["partial","*",4,0],"Perplexity-User":["partial","*",4,0],"Google-Extended":["partial","*",4,0],"Applebot-Extended":["partial","*",4,0],"CCBot":["partial","*",4,0],"Bytespider":["partial","*",4,0],"Amazonbot":["partial","*",4,0],"meta-externalagent":["partial","*",4,0],"cohere-ai":["partial","*",4,0],"Diffbot":["partial","*",4,0],"ContextFetchBot":["partial","*",4,0]},"llms":{"status":"html","http":200,"bytes":102759},"home":{"status":"ok","http":200,"url":"https://www.tastingtable.com/","html_tokens":25689,"markdown_tokens":1174,"pruned_tokens":164}},
{"rank":8230,"domain":"kpbs.org","origin":"https://kpbs.org","robots":{"status":"ok","http":200,"url":"https://www.kpbs.org/robots.txt","bytes":198,"groups":1},"bots":{"GPTBot":["partial","*",2,0],"ChatGPT-User":["partial","*",2,0],"OAI-SearchBot":["partial","*",2,0],"ClaudeBot":["partial","*",2,0],"Claude-User":["partial","*",2,0],"Claude-SearchBot":["partial","*",2,0],"anthropic-ai":["partial","*",2,0],"PerplexityBot":["partial","*",2,0],"Perplexity-User":["partial","*",2,0],"Google-Extended":["partial","*",2,0],"Applebot-Extended":["partial","*",2,0],"CCBot":["partial","*",2,0],"Bytespider":["partial","*",2,0],"Amazonbot":["partial","*",2,0],"meta-externalagent":["partial","*",2,0],"cohere-ai":["partial","*",2,0],"Diffbot":["partial","*",2,0],"ContextFetchBot":["partial","*",2,0]},"llms":{"status":"missing","http":404,"bytes":275649},"home":{"status":"ok","http":200,"url":"https://www.kpbs.org/","html_tokens":98565,"markdown_tokens":765,"pruned_tokens":169}},
{"rank":8231,"domain":"hermanmiller.com","origin":"https://hermanmiller.com","robots":{"status":"ok","http":200,"url":"https://www.hermanmiller.com/robots.txt","bytes":551,"groups":1},"bots":{"GPTBot":["partial","*",18,0],"ChatGPT-User":["partial","*",18,0],"OAI-SearchBot":["partial","*",18,0],"ClaudeBot":["partial","*",18,0],"Claude-User":["partial","*",18,0],"Claude-SearchBot":["partial","*",18,0],"anthropic-ai":["partial","*",18,0],"PerplexityBot":["partial","*",18,0],"Perplexity-User":["partial","*",18,0],"Google-Extended":["partial","*",18,0],"Applebot-Extended":["partial","*",18,0],"CCBot":["partial","*",18,0],"Bytespider":["partial","*",18,0],"Amazonbot":["partial","*",18,0],"meta-externalagent":["partial","*",18,0],"cohere-ai":["partial","*",18,0],"Diffbot":["partial","*",18,0],"ContextFetchBot":["partial","*",18,0]},"llms":{"status":"missing","http":404,"bytes":115369},"home":{"status":"ok","http":200,"url":"https://www.hermanmiller.com/welcome/","html_tokens":3724,"markdown_tokens":57,"pruned_tokens":6}},
{"rank":8232,"domain":"xzgjp.com","origin":"https://xzgjp.com","robots":{"status":"missing","http":404,"url":"https://xzgjp.com/robots.txt","bytes":0,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":0},"home":{"status":"ok","http":200,"url":"https://xzgjp.com/","html_tokens":20884,"markdown_tokens":19,"pruned_tokens":19}},
{"rank":8233,"domain":"wtzm.cn","origin":"http://wtzm.cn","robots":{"status":"missing","http":404,"url":"http://wtzm.cn/robots.txt","bytes":1163,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":1163},"home":{"status":"ok","http":200,"url":"http://wtzm.cn/","html_tokens":2521,"markdown_tokens":104,"pruned_tokens":104}},
{"rank":8234,"domain":"xhamster.com","origin":"https://xhamster.com","robots":{"status":"ok","http":200,"url":"https://xhamster.com/robots.txt","bytes":13130,"groups":2},"bots":{"GPTBot":["partial","*",446,31],"ChatGPT-User":["partial","*",446,31],"OAI-SearchBot":["partial","*",446,31],"ClaudeBot":["partial","*",446,31],"Claude-User":["partial","*",446,31],"Claude-SearchBot":["partial","*",446,31],"anthropic-ai":["partial","*",446,31],"PerplexityBot":["partial","*",446,31],"Perplexity-User":["partial","*",446,31],"Google-Extended":["partial","*",446,31],"Applebot-Extended":["partial","*",446,31],"CCBot":["partial","*",446,31],"Bytespider":["partial","*",446,31],"Amazonbot":["partial","*",446,31],"meta-externalagent":["partial","*",446,31],"cohere-ai":["partial","*",446,31],"Diffbot":["partial","*",446,31],"ContextFetchBot":["partial","*",446,31]},"llms":{"status":"missing","http":404,"bytes":146},"home":{"status":"ok","http":200,"url":"https://xhamster.com/","html_tokens":117053,"markdown_tokens":22,"pruned_tokens":22}},
{"rank":8235,"domain":"iranintl.com","origin":"https://iranintl.com","robots":{"status":"ok","http":200,"url":"https://www.iranintl.com/robots.txt","bytes":140,"groups":3},"bots":{"GPTBot":["allowed","*",0,1],"ChatGPT-User":["allowed","*",0,1],"OAI-SearchBot":["allowed","*",0,1],"ClaudeBot":["allowed","*",0,1],"Claude-User":["allowed","*",0,1],"Claude-SearchBot":["allowed","*",0,1],"anthropic-ai":["allowed","*",0,1],"PerplexityBot":["allowed","*",0,1],"Perplexity-User":["allowed","*",0,1],"Google-Extended":["allowed","*",0,1],"Applebot-Extended":["allowed","*",0,1],"CCBot":["allowed","*",0,1],"Bytespider":["allowed","*",0,1],"Amazonbot":["allowed","*",0,1],"meta-externalagent":["allowed","*",0,1],"cohere-ai":["allowed","*",0,1],"Diffbot":["allowed","*",0,1],"ContextFetchBot":["allowed","*",0,1]},"llms":{"status":"present","http":200,"bytes":65883},"home":{"status":"ok","http":200,"url":"https://www.iranintl.com/","html_tokens":104390,"markdown_tokens":1038,"pruned_tokens":110}},
{"rank":8236,"domain":"codesandbox.io","origin":"https://codesandbox.io","robots":{"status":"refused","http":403,"url":"https://codesandbox.io/robots.txt","bytes":5560,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":8237,"domain":"howard.edu","origin":"https://howard.edu","robots":{"status":"ok","http":200,"url":"https://howard.edu/robots.txt","bytes":2141,"groups":1},"bots":{"GPTBot":["partial","*",35,22],"ChatGPT-User":["partial","*",35,22],"OAI-SearchBot":["partial","*",35,22],"ClaudeBot":["partial","*",35,22],"Claude-User":["partial","*",35,22],"Claude-SearchBot":["partial","*",35,22],"anthropic-ai":["partial","*",35,22],"PerplexityBot":["partial","*",35,22],"Perplexity-User":["partial","*",35,22],"Google-Extended":["partial","*",35,22],"Applebot-Extended":["partial","*",35,22],"CCBot":["partial","*",35,22],"Bytespider":["partial","*",35,22],"Amazonbot":["partial","*",35,22],"meta-externalagent":["partial","*",35,22],"cohere-ai":["partial","*",35,22],"Diffbot":["partial","*",35,22],"ContextFetchBot":["partial","*",35,22]},"llms":{"status":"missing","http":404,"bytes":184},"home":{"status":"ok","http":200,"url":"https://howard.edu/","html_tokens":44049,"markdown_tokens":338,"pruned_tokens":292}},
{"rank":8238,"domain":"vh1.com","origin":"https://vh1.com","robots":{"status":"ok","http":200,"url":"https://www.vh1.com/robots.txt","bytes":66,"groups":2},"bots":{"GPTBot":["partial","*",1,0],"ChatGPT-User":["partial","*",1,0],"OAI-SearchBot":["partial","*",1,0],"ClaudeBot":["partial","*",1,0],"Claude-User":["partial","*",1,0],"Claude-SearchBot":["partial","*",1,0],"anthropic-ai":["partial","*",1,0],"PerplexityBot":["partial","*",1,0],"Perplexity-User":["partial","*",1,0],"Google-Extended":["partial","*",1,0],"Applebot-Extended":["partial","*",1,0],"CCBot":["partial","*",1,0],"Bytespider":["partial","*",1,0],"Amazonbot":["partial","*",1,0],"meta-externalagent":["partial","*",1,0],"cohere-ai":["partial","*",1,0],"Diffbot":["partial","*",1,0],"ContextFetchBot":["partial","*",1,0]},"llms":{"status":"missing","http":404,"bytes":791745},"home":{"status":"ok","http":200,"url":"https://www.vh1.com","html_tokens":312030,"markdown_tokens":79,"pruned_tokens":79}},
{"rank":8239,"domain":"heinonline.org","origin":"https://heinonline.org","robots":{"status":"ok","http":200,"url":"https://access.heinonline.com/robots.txt","bytes":432,"groups":11},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":283},"home":{"status":"ok","http":200,"url":"https://access.heinonline.com/","html_tokens":52,"markdown_tokens":0,"pruned_tokens":0}},
{"rank":8240,"domain":"cbechero.net","origin":"http://cbechero.net","robots":{"status":"missing","http":404,"url":"http://cbechero.net/robots.txt","bytes":1831,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":1829},"home":{"status":"ok","http":200,"url":"http://cbechero.net/","html_tokens":2547,"markdown_tokens":12,"pruned_tokens":12}},
{"rank":8241,"domain":"google.rw","origin":"https://google.rw","robots":{"status":"ok","http":200,"url":"https://www.google.rw/robots.txt","bytes":6624,"groups":4},"bots":{"GPTBot":["partial","*",169,64],"ChatGPT-User":["partial","*",169,64],"OAI-SearchBot":["partial","*",169,64],"ClaudeBot":["partial","*",169,64],"Claude-User":["partial","*",169,64],"Claude-SearchBot":["partial","*",169,64],"anthropic-ai":["partial","*",169,64],"PerplexityBot":["partial","*",169,64],"Perplexity-User":["partial","*",169,64],"Google-Extended":["partial","*",169,64],"Applebot-Extended":["partial","*",169,64],"CCBot":["partial","*",169,64],"Bytespider":["partial","*",169,64],"Amazonbot":["partial","*",169,64],"meta-externalagent":["partial","*",169,64],"cohere-ai":["partial","*",169,64],"Diffbot":["partial","*",169,64],"ContextFetchBot":["partial","*",169,64]},"llms":{"status":"missing","http":404,"bytes":1569},"home":{"status":"ok","http":200,"url":"https://www.google.com/","html_tokens":21275,"markdown_tokens":30,"pruned_tokens":30}},
{"rank":8242,"domain":"periscope.tv","origin":"https://periscope.tv","robots":{"status":"ok","http":200,"url":"https://www.periscope.tv/robots.txt","bytes":151,"groups":2},"bots":{"GPTBot":["partial","*",4,0],"ChatGPT-User":["partial","*",4,0],"OAI-SearchBot":["partial","*",4,0],"ClaudeBot":["partial","*",4,0],"Claude-User":["partial","*",4,0],"Claude-SearchBot":["partial","*",4,0],"anthropic-ai":["partial","*",4,0],"PerplexityBot":["partial","*",4,0],"Perplexity-User":["partial","*",4,0],"Google-Extended":["partial","*",4,0],"Applebot-Extended":["partial","*",4,0],"CCBot":["partial","*",4,0],"Bytespider":["partial","*",4,0],"Amazonbot":["partial","*",4,0],"meta-externalagent":["partial","*",4,0],"cohere-ai":["partial","*",4,0],"Diffbot":["partial","*",4,0],"ContextFetchBot":["partial","*",4,0]},"llms":{"status":"http_error","http":400,"bytes":15378},"home":{"status":"ok","http":200,"url":"https://www.periscope.tv/","html_tokens":5353,"markdown_tokens":42,"pruned_tokens":42}},
{"rank":8243,"domain":"hebei.gov.cn","origin":"https://hebei.gov.cn","robots":{"status":"timeout","http":null,"url":"https://hebei.gov.cn/robots.txt","bytes":0,"groups":0,"error":"timed out after 10 s"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":8244,"domain":"maa.org","origin":"https://maa.org","robots":{"status":"refused","http":403,"url":"https://maa.org/robots.txt","bytes":103,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":8245,"domain":"zjzdlo.com","origin":"http://zjzdlo.com","robots":{"status":"missing","http":404,"url":"http://zjzdlo.com/robots.txt","bytes":146,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":146},"home":{"status":"ok","http":200,"url":"http://zjzdlo.com/","html_tokens":2411,"markdown_tokens":46,"pruned_tokens":46}},
{"rank":8246,"domain":"badische-zeitung.de","origin":"https://badische-zeitung.de","robots":{"status":"ok","http":200,"url":"https://www.badische-zeitung.de/robots.txt","bytes":702,"groups":17},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["blocked","named",1,0],"OAI-SearchBot":["partial","*",4,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["blocked","named",1,0],"Claude-SearchBot":["partial","*",4,0],"anthropic-ai":["partial","*",4,0],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["blocked","named",1,0],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["partial","*",4,0],"ContextFetchBot":["partial","*",4,0]},"llms":{"status":"missing","http":404,"bytes":66638},"home":{"status":"ok","http":200,"url":"https://www.badische-zeitung.de/","html_tokens":97897,"markdown_tokens":608,"pruned_tokens":149}},
{"rank":8247,"domain":"vip.com","origin":"https://vip.com","robots":{"status":"html","http":200,"url":"https://www.vip.com/","bytes":13353,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":8248,"domain":"rodeointl.cn","origin":"http://rodeointl.cn","robots":{"status":"missing","http":404,"url":"http://rodeointl.cn/robots.txt","bytes":315,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":315},"home":{"status":"http_error","http":404,"url":"http://rodeointl.cn/"}},
{"rank":8249,"domain":"shgjp.net","origin":"https://shgjp.net","robots":{"status":"missing","http":404,"url":"https://shgjp.net/robots.txt","bytes":0,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":0},"home":{"status":"ok","http":200,"url":"https://shgjp.net/","html_tokens":20880,"markdown_tokens":19,"pruned_tokens":19}},
{"rank":8250,"domain":"securitymagazine.com","origin":"https://securitymagazine.com","robots":{"status":"ok","http":200,"url":"https://www.securitymagazine.com/robots.txt","bytes":656,"groups":1},"bots":{"GPTBot":["partial","*",13,2],"ChatGPT-User":["partial","*",13,2],"OAI-SearchBot":["partial","*",13,2],"ClaudeBot":["partial","*",13,2],"Claude-User":["partial","*",13,2],"Claude-SearchBot":["partial","*",13,2],"anthropic-ai":["partial","*",13,2],"PerplexityBot":["partial","*",13,2],"Perplexity-User":["partial","*",13,2],"Google-Extended":["partial","*",13,2],"Applebot-Extended":["partial","*",13,2],"CCBot":["partial","*",13,2],"Bytespider":["partial","*",13,2],"Amazonbot":["partial","*",13,2],"meta-externalagent":["partial","*",13,2],"cohere-ai":["partial","*",13,2],"Diffbot":["partial","*",13,2],"ContextFetchBot":["partial","*",13,2]},"llms":{"status":"http_error","http":503,"bytes":0},"home":{"status":"ok","http":200,"url":"https://www.securitymagazine.com/","html_tokens":34481,"markdown_tokens":121,"pruned_tokens":121}},
{"rank":8251,"domain":"dgt.es","origin":"https://dgt.es","robots":{"status":"ok","http":200,"url":"https://www.dgt.es/robots.txt","bytes":8643,"groups":2},"bots":{"GPTBot":["partial","*",10,0],"ChatGPT-User":["partial","*",10,0],"OAI-SearchBot":["partial","*",10,0],"ClaudeBot":["partial","*",10,0],"Claude-User":["partial","*",10,0],"Claude-SearchBot":["partial","*",10,0],"anthropic-ai":["partial","*",10,0],"PerplexityBot":["partial","*",10,0],"Perplexity-User":["partial","*",10,0],"Google-Extended":["partial","*",10,0],"Applebot-Extended":["partial","*",10,0],"CCBot":["partial","*",10,0],"Bytespider":["partial","*",10,0],"Amazonbot":["partial","*",10,0],"meta-externalagent":["partial","*",10,0],"cohere-ai":["partial","*",10,0],"Diffbot":["partial","*",10,0],"ContextFetchBot":["partial","*",10,0]},"llms":{"status":"missing","http":404,"bytes":2161},"home":{"status":"ok","http":200,"url":"https://www.dgt.es/inicio/","html_tokens":66854,"markdown_tokens":151,"pruned_tokens":151}},
{"rank":8252,"domain":"ufpr.br","origin":"https://ufpr.br","robots":{"status":"ok","http":200,"url":"https://ufpr.br/robots.txt","bytes":108,"groups":1},"bots":{"GPTBot":["partial","*",1,1],"ChatGPT-User":["partial","*",1,1],"OAI-SearchBot":["partial","*",1,1],"ClaudeBot":["partial","*",1,1],"Claude-User":["partial","*",1,1],"Claude-SearchBot":["partial","*",1,1],"anthropic-ai":["partial","*",1,1],"PerplexityBot":["partial","*",1,1],"Perplexity-User":["partial","*",1,1],"Google-Extended":["partial","*",1,1],"Applebot-Extended":["partial","*",1,1],"CCBot":["partial","*",1,1],"Bytespider":["partial","*",1,1],"Amazonbot":["partial","*",1,1],"meta-externalagent":["partial","*",1,1],"cohere-ai":["partial","*",1,1],"Diffbot":["partial","*",1,1],"ContextFetchBot":["partial","*",1,1]},"llms":{"status":"missing","http":404,"bytes":184123},"home":{"status":"ok","http":200,"url":"https://ufpr.br/","html_tokens":46387,"markdown_tokens":31,"pruned_tokens":31}},
{"rank":8253,"domain":"syxtvip.cn","origin":"https://syxtvip.cn","robots":{"status":"unreachable","http":null,"url":"https://syxtvip.cn/robots.txt","bytes":0,"groups":0,"error":"cannot resolve syxtvip.cn"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":8254,"domain":"form-mailer.jp","origin":"https://form-mailer.jp","robots":{"status":"ok","http":200,"url":"https://www.form-mailer.jp/robots.txt","bytes":447,"groups":2},"bots":{"GPTBot":["partial","named",3,0],"ChatGPT-User":["partial","named",3,0],"OAI-SearchBot":["partial","named",3,0],"ClaudeBot":["partial","named",3,0],"Claude-User":["partial","named",3,0],"Claude-SearchBot":["partial","named",3,0],"anthropic-ai":["partial","*",3,0],"PerplexityBot":["partial","named",3,0],"Perplexity-User":["partial","named",3,0],"Google-Extended":["partial","named",3,0],"Applebot-Extended":["partial","named",3,0],"CCBot":["partial","*",3,0],"Bytespider":["partial","*",3,0],"Amazonbot":["partial","*",3,0],"meta-externalagent":["partial","*",3,0],"cohere-ai":["partial","*",3,0],"Diffbot":["partial","*",3,0],"ContextFetchBot":["partial","*",3,0]},"llms":{"status":"missing","http":404,"bytes":32218},"home":{"status":"ok","http":200,"url":"https://www.form-mailer.jp/","html_tokens":83946,"markdown_tokens":509,"pruned_tokens":173}},
{"rank":8255,"domain":"vicp.fun","origin":"http://vicp.fun","robots":{"status":"missing","http":404,"url":"http://vicp.fun/robots.txt","bytes":146,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":146},"home":{"status":"ok","http":200,"url":"http://vicp.fun/","html_tokens":757,"markdown_tokens":25,"pruned_tokens":25}},
{"rank":8256,"domain":"waltongroup.cn","origin":"http://waltongroup.cn","robots":{"status":"missing","http":404,"url":"http://waltongroup.cn/robots.txt","bytes":315,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":315},"home":{"status":"http_error","http":404,"url":"http://waltongroup.cn/"}},
{"rank":8257,"domain":"startersites.io","origin":"https://startersites.io","robots":{"status":"ok","http":200,"url":"https://startersites.io/robots.txt","bytes":455,"groups":13},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["blocked","named",1,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":507},"home":{"status":"http_error","http":403,"url":"https://startersites.io/"}},
{"rank":8258,"domain":"bostondynamics.com","origin":"https://bostondynamics.com","robots":{"status":"ok","http":200,"url":"https://bostondynamics.com/robots.txt","bytes":85,"groups":1},"bots":{"GPTBot":["partial","*",1,0],"ChatGPT-User":["partial","*",1,0],"OAI-SearchBot":["partial","*",1,0],"ClaudeBot":["partial","*",1,0],"Claude-User":["partial","*",1,0],"Claude-SearchBot":["partial","*",1,0],"anthropic-ai":["partial","*",1,0],"PerplexityBot":["partial","*",1,0],"Perplexity-User":["partial","*",1,0],"Google-Extended":["partial","*",1,0],"Applebot-Extended":["partial","*",1,0],"CCBot":["partial","*",1,0],"Bytespider":["partial","*",1,0],"Amazonbot":["partial","*",1,0],"meta-externalagent":["partial","*",1,0],"cohere-ai":["partial","*",1,0],"Diffbot":["partial","*",1,0],"ContextFetchBot":["partial","*",1,0]},"llms":{"status":"missing","http":404,"bytes":85842},"home":{"status":"ok","http":200,"url":"https://bostondynamics.com/","html_tokens":36542,"markdown_tokens":265,"pruned_tokens":164}},
{"rank":8259,"domain":"blackbaud.com","origin":"https://blackbaud.com","robots":{"status":"ok","http":200,"url":"https://www.blackbaud.com/robots.txt","bytes":101,"groups":2},"bots":{"GPTBot":["partial","*",1,0],"ChatGPT-User":["partial","*",1,0],"OAI-SearchBot":["partial","*",1,0],"ClaudeBot":["partial","*",1,0],"Claude-User":["partial","*",1,0],"Claude-SearchBot":["partial","*",1,0],"anthropic-ai":["partial","*",1,0],"PerplexityBot":["partial","*",1,0],"Perplexity-User":["partial","*",1,0],"Google-Extended":["partial","*",1,0],"Applebot-Extended":["partial","*",1,0],"CCBot":["partial","*",1,0],"Bytespider":["partial","*",1,0],"Amazonbot":["partial","*",1,0],"meta-externalagent":["partial","*",1,0],"cohere-ai":["partial","*",1,0],"Diffbot":["partial","*",1,0],"ContextFetchBot":["partial","*",1,0]},"llms":{"status":"missing","http":404,"bytes":166777},"home":{"status":"ok","http":200,"url":"https://www.blackbaud.com/","html_tokens":68222,"markdown_tokens":183,"pruned_tokens":58}},
{"rank":8260,"domain":"xn--xhq8a773e95j9l5a.com","origin":"http://xn--xhq8a773e95j9l5a.com","robots":{"status":"html","http":200,"url":"http://xn--xhq8a773e95j9l5a.com/robots.txt","bytes":762,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":8261,"domain":"floridatoday.com","origin":"https://floridatoday.com","robots":{"status":"ok","http":200,"url":"https://www.floridatoday.com/robots.txt","bytes":13527,"groups":288},"bots":{"GPTBot":["partial","named",1,5],"ChatGPT-User":["partial","named",1,5],"OAI-SearchBot":["partial","named",1,5],"ClaudeBot":["blocked","named",1,0],"Claude-User":["blocked","named",1,0],"Claude-SearchBot":["blocked","named",1,0],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["partial","named",1,5],"Perplexity-User":["blocked","named",1,0],"Google-Extended":["partial","named",1,5],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",2,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",32,0]},"llms":{"status":"http_error","http":402,"bytes":254},"home":{"status":"http_error","http":402,"url":"https://www.floridatoday.com/"}},
{"rank":8262,"domain":"amctheatres.com","origin":"https://amctheatres.com","robots":{"status":"ok","http":200,"url":"https://www.amctheatres.com/robots.txt","bytes":250,"groups":3},"bots":{"GPTBot":["partial","*",3,0],"ChatGPT-User":["partial","*",3,0],"OAI-SearchBot":["partial","*",3,0],"ClaudeBot":["partial","*",3,0],"Claude-User":["partial","*",3,0],"Claude-SearchBot":["partial","*",3,0],"anthropic-ai":["partial","*",3,0],"PerplexityBot":["partial","*",3,0],"Perplexity-User":["partial","*",3,0],"Google-Extended":["partial","*",3,0],"Applebot-Extended":["partial","*",3,0],"CCBot":["partial","*",3,0],"Bytespider":["partial","*",3,0],"Amazonbot":["partial","*",3,0],"meta-externalagent":["partial","named",1,0],"cohere-ai":["partial","*",3,0],"Diffbot":["partial","*",3,0],"ContextFetchBot":["partial","*",3,0]},"llms":{"status":"http_error","http":403,"bytes":4548},"home":{"status":"http_error","http":403,"url":"https://amctheatres.com/"}},
{"rank":8263,"domain":"flexera.com","origin":"https://flexera.com","robots":{"status":"refused","http":403,"url":"https://flexera.com/robots.txt","bytes":560,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":8264,"domain":"mybaccaratguide.com","origin":"https://mybaccaratguide.com","robots":{"status":"ok","http":200,"url":"https://mybaccaratguide.com/robots.txt","bytes":469,"groups":1},"bots":{"GPTBot":["partial","*",17,0],"ChatGPT-User":["partial","*",17,0],"OAI-SearchBot":["partial","*",17,0],"ClaudeBot":["partial","*",17,0],"Claude-User":["partial","*",17,0],"Claude-SearchBot":["partial","*",17,0],"anthropic-ai":["partial","*",17,0],"PerplexityBot":["partial","*",17,0],"Perplexity-User":["partial","*",17,0],"Google-Extended":["partial","*",17,0],"Applebot-Extended":["partial","*",17,0],"CCBot":["partial","*",17,0],"Bytespider":["partial","*",17,0],"Amazonbot":["partial","*",17,0],"meta-externalagent":["partial","*",17,0],"cohere-ai":["partial","*",17,0],"Diffbot":["partial","*",17,0],"ContextFetchBot":["partial","*",17,0]},"llms":{"status":"missing","http":404,"bytes":22314},"home":{"status":"ok","http":200,"url":"https://mybaccaratguide.com/","html_tokens":18407,"markdown_tokens":2255,"pruned_tokens":161}},
{"rank":8265,"domain":"getgrav.org","origin":"https://getgrav.org","robots":{"status":"ok","http":200,"url":"https://getgrav.org/robots.txt","bytes":1375,"groups":1},"bots":{"GPTBot":["partial","*",2,1],"ChatGPT-User":["partial","*",2,1],"OAI-SearchBot":["partial","*",2,1],"ClaudeBot":["partial","*",2,1],"Claude-User":["partial","*",2,1],"Claude-SearchBot":["partial","*",2,1],"anthropic-ai":["partial","*",2,1],"PerplexityBot":["partial","*",2,1],"Perplexity-User":["partial","*",2,1],"Google-Extended":["partial","*",2,1],"Applebot-Extended":["partial","*",2,1],"CCBot":["partial","*",2,1],"Bytespider":["partial","*",2,1],"Amazonbot":["partial","*",2,1],"meta-externalagent":["partial","*",2,1],"cohere-ai":["partial","*",2,1],"Diffbot":["partial","*",2,1],"ContextFetchBot":["partial","*",2,1]},"llms":{"status":"present","http":200,"bytes":2186},"home":{"status":"ok","http":200,"url":"https://getgrav.org/","html_tokens":24998,"markdown_tokens":1398,"pruned_tokens":152}},
{"rank":8266,"domain":"conrad.de","origin":"https://conrad.de","robots":{"status":"refused","http":403,"url":"https://www.conrad.de/robots.txt","bytes":5508,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":8267,"domain":"bostonreview.net","origin":"https://bostonreview.net","robots":{"status":"refused","http":403,"url":"https://www.bostonreview.net/robots.txt","bytes":4910,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":8268,"domain":"cnfol.com","origin":"https://cnfol.com","robots":{"status":"missing","http":404,"url":"https://cnfol.com/robots.txt","bytes":13076,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":13076},"home":{"status":"ok","http":200,"url":"https://cnfol.com/","html_tokens":38976,"markdown_tokens":162,"pruned_tokens":162}},
{"rank":8269,"domain":"cf.ac.uk","origin":"https://cf.ac.uk","robots":{"status":"refused","http":403,"url":"https://www.cardiff.ac.uk/robots.txt","bytes":5555,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":8270,"domain":"tokyo-np.co.jp","origin":"https://tokyo-np.co.jp","robots":{"status":"ok","http":200,"url":"https://www.tokyo-np.co.jp/robots.txt","bytes":2763,"groups":67},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["blocked","named",1,0],"OAI-SearchBot":["blocked","named",1,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["blocked","named",1,0],"Claude-SearchBot":["blocked","named",1,0],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["blocked","named",1,0],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",2,1]},"llms":{"status":"missing","http":404,"bytes":20589},"home":{"status":"ok","http":200,"url":"https://www.tokyo-np.co.jp/","html_tokens":38648,"markdown_tokens":25,"pruned_tokens":25}},
{"rank":8271,"domain":"mclaren.com","origin":"https://mclaren.com","robots":{"status":"ok","http":200,"url":"https://www.mclaren.com/robots.txt","bytes":102,"groups":1},"bots":{"GPTBot":["partial","*",2,1],"ChatGPT-User":["partial","*",2,1],"OAI-SearchBot":["partial","*",2,1],"ClaudeBot":["partial","*",2,1],"Claude-User":["partial","*",2,1],"Claude-SearchBot":["partial","*",2,1],"anthropic-ai":["partial","*",2,1],"PerplexityBot":["partial","*",2,1],"Perplexity-User":["partial","*",2,1],"Google-Extended":["partial","*",2,1],"Applebot-Extended":["partial","*",2,1],"CCBot":["partial","*",2,1],"Bytespider":["partial","*",2,1],"Amazonbot":["partial","*",2,1],"meta-externalagent":["partial","*",2,1],"cohere-ai":["partial","*",2,1],"Diffbot":["partial","*",2,1],"ContextFetchBot":["partial","*",2,1]},"llms":{"status":"missing","http":404,"bytes":21527},"home":{"status":"ok","http":200,"url":"https://www.mclaren.com/","html_tokens":6156,"markdown_tokens":9,"pruned_tokens":9}},
{"rank":8272,"domain":"korrespondent.net","origin":"https://korrespondent.net","robots":{"status":"ok","http":200,"url":"https://korrespondent.net/robots.txt","bytes":812,"groups":1},"bots":{"GPTBot":["partial","*",29,0],"ChatGPT-User":["partial","*",29,0],"OAI-SearchBot":["partial","*",29,0],"ClaudeBot":["partial","*",29,0],"Claude-User":["partial","*",29,0],"Claude-SearchBot":["partial","*",29,0],"anthropic-ai":["partial","*",29,0],"PerplexityBot":["partial","*",29,0],"Perplexity-User":["partial","*",29,0],"Google-Extended":["partial","*",29,0],"Applebot-Extended":["partial","*",29,0],"CCBot":["partial","*",29,0],"Bytespider":["partial","*",29,0],"Amazonbot":["partial","*",29,0],"meta-externalagent":["partial","*",29,0],"cohere-ai":["partial","*",29,0],"Diffbot":["partial","*",29,0],"ContextFetchBot":["partial","*",29,0]},"llms":{"status":"missing","http":404,"bytes":47717},"home":{"status":"ok","http":200,"url":"https://ua.korrespondent.net","html_tokens":39413,"markdown_tokens":34,"pruned_tokens":34}},
{"rank":8273,"domain":"doxycare.shop","origin":"https://doxycare.shop","robots":{"status":"html","http":200,"url":"https://doxyatlas.com/","bytes":12052,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":8274,"domain":"indexmundi.com","origin":"https://indexmundi.com","robots":{"status":"ok","http":200,"url":"https://www.indexmundi.com/robots.txt","bytes":589,"groups":13},"bots":{"GPTBot":["partial","*",1,0],"ChatGPT-User":["partial","*",1,0],"OAI-SearchBot":["partial","*",1,0],"ClaudeBot":["partial","*",1,0],"Claude-User":["partial","*",1,0],"Claude-SearchBot":["partial","*",1,0],"anthropic-ai":["partial","*",1,0],"PerplexityBot":["partial","*",1,0],"Perplexity-User":["partial","*",1,0],"Google-Extended":["partial","*",1,0],"Applebot-Extended":["partial","*",1,0],"CCBot":["partial","*",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["partial","*",1,0],"cohere-ai":["partial","*",1,0],"Diffbot":["partial","*",1,0],"ContextFetchBot":["partial","*",1,0]},"llms":{"status":"html","http":200,"bytes":1217},"home":{"status":"timeout","error":"timed out after 15 s"}},
{"rank":8275,"domain":"interfax-russia.ru","origin":"https://interfax-russia.ru","robots":{"status":"html","http":200,"url":"https://interfax-russia.ru/robots.txt","bytes":2160,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":8276,"domain":"sciendo.com","origin":"https://sciendo.com","robots":{"status":"ok","http":200,"url":"https://reference-global.com/robots.txt","bytes":387,"groups":6},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["partial","*",9,0],"OAI-SearchBot":["partial","*",9,0],"ClaudeBot":["partial","*",9,0],"Claude-User":["partial","*",9,0],"Claude-SearchBot":["partial","*",9,0],"anthropic-ai":["partial","*",9,0],"PerplexityBot":["partial","*",9,0],"Perplexity-User":["partial","*",9,0],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["partial","*",9,0],"CCBot":["blocked","named",1,0],"Bytespider":["partial","*",9,0],"Amazonbot":["partial","*",9,0],"meta-externalagent":["partial","*",9,0],"cohere-ai":["partial","*",9,0],"Diffbot":["partial","*",9,0],"ContextFetchBot":["partial","*",9,0]},"llms":{"status":"missing","http":404,"bytes":205315},"home":{"status":"ok","http":200,"url":"https://reference-global.com/","html_tokens":127100,"markdown_tokens":56,"pruned_tokens":56}},
{"rank":8277,"domain":"tianyancha.com","origin":"https://tianyancha.com","robots":{"status":"ok","http":200,"url":"https://www.tianyancha.com/robots.txt","bytes":906,"groups":8},"bots":{"GPTBot":["partial","*",30,0],"ChatGPT-User":["partial","*",30,0],"OAI-SearchBot":["partial","*",30,0],"ClaudeBot":["partial","*",30,0],"Claude-User":["partial","*",30,0],"Claude-SearchBot":["partial","*",30,0],"anthropic-ai":["partial","*",30,0],"PerplexityBot":["partial","*",30,0],"Perplexity-User":["partial","*",30,0],"Google-Extended":["partial","*",30,0],"Applebot-Extended":["partial","*",30,0],"CCBot":["partial","*",30,0],"Bytespider":["partial","*",30,0],"Amazonbot":["partial","*",30,0],"meta-externalagent":["partial","*",30,0],"cohere-ai":["partial","*",30,0],"Diffbot":["partial","*",30,0],"ContextFetchBot":["partial","*",30,0]},"llms":{"status":"missing","http":404,"bytes":25094},"home":{"status":"http_error","http":419,"url":"https://www.tianyancha.com/"}},
{"rank":8278,"domain":"mdjdkfyp.com","origin":"http://mdjdkfyp.com","robots":{"status":"missing","http":404,"url":"http://mdjdkfyp.com/robots.txt","bytes":0,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":0},"home":{"status":"ok","http":200,"url":"http://mdjdkfyp.com/","html_tokens":5739,"markdown_tokens":88,"pruned_tokens":88}},
{"rank":8279,"domain":"ytshop.cn","origin":"https://www.ytshop.cn","robots":{"status":"ok","http":200,"url":"https://www.ytshop.cn/robots.txt","bytes":553,"groups":1},"bots":{"GPTBot":["partial","*",24,0],"ChatGPT-User":["partial","*",24,0],"OAI-SearchBot":["partial","*",24,0],"ClaudeBot":["partial","*",24,0],"Claude-User":["partial","*",24,0],"Claude-SearchBot":["partial","*",24,0],"anthropic-ai":["partial","*",24,0],"PerplexityBot":["partial","*",24,0],"Perplexity-User":["partial","*",24,0],"Google-Extended":["partial","*",24,0],"Applebot-Extended":["partial","*",24,0],"CCBot":["partial","*",24,0],"Bytespider":["partial","*",24,0],"Amazonbot":["partial","*",24,0],"meta-externalagent":["partial","*",24,0],"cohere-ai":["partial","*",24,0],"Diffbot":["partial","*",24,0],"ContextFetchBot":["partial","*",24,0]},"llms":{"status":"disallowed"},"home":{"status":"ok","http":200,"url":"https://www.ytshop.cn/","html_tokens":32035,"markdown_tokens":25,"pruned_tokens":15}},
{"rank":8280,"domain":"valvetc.com","origin":"http://valvetc.com","robots":{"status":"ok","http":200,"url":"http://valvetc.com/robots.txt","bytes":2123,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"html","http":200,"bytes":2123},"home":{"status":"ok","http":200,"url":"http://valvetc.com/","html_tokens":18344,"markdown_tokens":66,"pruned_tokens":66}},
{"rank":8281,"domain":"cryptobriefing.com","origin":"https://cryptobriefing.com","robots":{"status":"refused","http":403,"url":"https://cryptobriefing.com/robots.txt","bytes":4912,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":8282,"domain":"devin.ai","origin":"https://devin.ai","robots":{"status":"ok","http":200,"url":"https://devin.ai/robots.txt","bytes":370,"groups":1},"bots":{"GPTBot":["partial","named",2,1],"ChatGPT-User":["partial","named",2,1],"OAI-SearchBot":["partial","named",2,1],"ClaudeBot":["partial","named",2,1],"Claude-User":["partial","*",2,1],"Claude-SearchBot":["partial","named",2,1],"anthropic-ai":["partial","*",2,1],"PerplexityBot":["partial","named",2,1],"Perplexity-User":["partial","*",2,1],"Google-Extended":["partial","named",2,1],"Applebot-Extended":["partial","named",2,1],"CCBot":["partial","*",2,1],"Bytespider":["partial","*",2,1],"Amazonbot":["partial","*",2,1],"meta-externalagent":["partial","*",2,1],"cohere-ai":["partial","*",2,1],"Diffbot":["partial","*",2,1],"ContextFetchBot":["partial","*",2,1]},"llms":{"status":"present","http":200,"bytes":2514},"home":{"status":"ok","http":200,"url":"https://devin.ai/","html_tokens":378191,"markdown_tokens":195,"pruned_tokens":150}},
{"rank":8283,"domain":"abajournal.com","origin":"https://abajournal.com","robots":{"status":"ok","http":200,"url":"https://www.abajournal.com/robots.txt","bytes":4048,"groups":6},"bots":{"GPTBot":["partial","named",12,0],"ChatGPT-User":["partial","named",14,0],"OAI-SearchBot":["partial","named",14,0],"ClaudeBot":["partial","named",12,0],"Claude-User":["partial","named",14,0],"Claude-SearchBot":["partial","named",14,0],"anthropic-ai":["partial","*",33,0],"PerplexityBot":["partial","named",14,0],"Perplexity-User":["partial","named",14,0],"Google-Extended":["partial","named",12,0],"Applebot-Extended":["partial","named",12,0],"CCBot":["partial","named",12,0],"Bytespider":["partial","named",12,0],"Amazonbot":["partial","*",33,0],"meta-externalagent":["partial","named",12,0],"cohere-ai":["partial","*",33,0],"Diffbot":["partial","*",33,0],"ContextFetchBot":["partial","*",33,0]},"llms":{"status":"present","http":200,"bytes":2678},"home":{"status":"ok","http":200,"url":"https://www.abajournal.com/","html_tokens":10419,"markdown_tokens":82,"pruned_tokens":82}},
{"rank":8284,"domain":"google.mk","origin":"https://google.mk","robots":{"status":"ok","http":200,"url":"https://www.google.mk/robots.txt","bytes":6624,"groups":4},"bots":{"GPTBot":["partial","*",169,64],"ChatGPT-User":["partial","*",169,64],"OAI-SearchBot":["partial","*",169,64],"ClaudeBot":["partial","*",169,64],"Claude-User":["partial","*",169,64],"Claude-SearchBot":["partial","*",169,64],"anthropic-ai":["partial","*",169,64],"PerplexityBot":["partial","*",169,64],"Perplexity-User":["partial","*",169,64],"Google-Extended":["partial","*",169,64],"Applebot-Extended":["partial","*",169,64],"CCBot":["partial","*",169,64],"Bytespider":["partial","*",169,64],"Amazonbot":["partial","*",169,64],"meta-externalagent":["partial","*",169,64],"cohere-ai":["partial","*",169,64],"Diffbot":["partial","*",169,64],"ContextFetchBot":["partial","*",169,64]},"llms":{"status":"missing","http":404,"bytes":1569},"home":{"status":"ok","http":200,"url":"https://www.google.com/","html_tokens":21293,"markdown_tokens":30,"pruned_tokens":30}},
{"rank":8285,"domain":"wma.net","origin":"https://wma.net","robots":{"status":"ok","http":200,"url":"https://www.wma.net/robots.txt","bytes":0,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":50982},"home":{"status":"ok","http":200,"url":"https://www.wma.net/","html_tokens":16898,"markdown_tokens":104,"pruned_tokens":62}},
{"rank":8286,"domain":"smzdm.com","origin":"https://smzdm.com","robots":{"status":"ok","http":200,"url":"https://www.smzdm.com/robots.txt","bytes":692,"groups":1},"bots":{"GPTBot":["partial","*",10,0],"ChatGPT-User":["partial","*",10,0],"OAI-SearchBot":["partial","*",10,0],"ClaudeBot":["partial","*",10,0],"Claude-User":["partial","*",10,0],"Claude-SearchBot":["partial","*",10,0],"anthropic-ai":["partial","*",10,0],"PerplexityBot":["partial","*",10,0],"Perplexity-User":["partial","*",10,0],"Google-Extended":["partial","*",10,0],"Applebot-Extended":["partial","*",10,0],"CCBot":["partial","*",10,0],"Bytespider":["partial","*",10,0],"Amazonbot":["partial","*",10,0],"meta-externalagent":["partial","*",10,0],"cohere-ai":["partial","*",10,0],"Diffbot":["partial","*",10,0],"ContextFetchBot":["partial","*",10,0]},"llms":{"status":"present","http":200,"bytes":6814},"home":{"status":"ok","http":200,"url":"https://www.smzdm.com/","html_tokens":274077,"markdown_tokens":34,"pruned_tokens":27}},
{"rank":8287,"domain":"perm.ru","origin":"https://perm.ru","robots":{"status":"timeout","http":null,"url":"https://perm.ru/robots.txt","bytes":0,"groups":0,"error":"ConnectTimeout"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":8288,"domain":"fdbaudio.com","origin":"http://fdbaudio.com","robots":{"status":"missing","http":404,"url":"http://fdbaudio.com/robots.txt","bytes":208,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":206},"home":{"status":"ok","http":200,"url":"http://fdbaudio.com/","html_tokens":10600,"markdown_tokens":209,"pruned_tokens":173}},
{"rank":8289,"domain":"guangaishangcheng.com","origin":"https://guangaishangcheng.com","robots":{"status":"unreachable","http":null,"url":"https://guangaishangcheng.com/robots.txt","bytes":0,"groups":0,"error":"cannot resolve guangaishangcheng.com"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":8290,"domain":"lxypt.net","origin":"https://www.lxypt.net","robots":{"status":"missing","http":404,"url":"https://www.lxypt.net/robots.txt","bytes":63,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":63},"home":{"status":"ok","http":200,"url":"https://www.lxypt.net/","html_tokens":12777,"markdown_tokens":130,"pruned_tokens":78}},
{"rank":8291,"domain":"providencejournal.com","origin":"https://providencejournal.com","robots":{"status":"ok","http":200,"url":"https://www.providencejournal.com/robots.txt","bytes":14093,"groups":288},"bots":{"GPTBot":["partial","named",1,4],"ChatGPT-User":["partial","named",1,4],"OAI-SearchBot":["partial","named",1,4],"ClaudeBot":["blocked","named",1,0],"Claude-User":["blocked","named",1,0],"Claude-SearchBot":["blocked","named",1,0],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["partial","named",1,4],"Perplexity-User":["blocked","named",1,0],"Google-Extended":["partial","named",1,4],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",2,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",88,0]},"llms":{"status":"http_error","http":402,"bytes":254},"home":{"status":"http_error","http":402,"url":"https://www.providencejournal.com/"}},
{"rank":8292,"domain":"deeplearning.ai","origin":"https://deeplearning.ai","robots":{"status":"ok","http":200,"url":"https://www.deeplearning.ai/robots.txt","bytes":480,"groups":3},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["allowed","named",0,1],"OAI-SearchBot":["allowed","named",0,1],"ClaudeBot":["blocked","named",1,0],"Claude-User":["allowed","named",0,1],"Claude-SearchBot":["allowed","named",0,1],"anthropic-ai":["allowed","*",0,1],"PerplexityBot":["allowed","named",0,1],"Perplexity-User":["allowed","named",0,1],"Google-Extended":["allowed","named",0,1],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["allowed","*",0,1],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["allowed","*",0,1],"Diffbot":["allowed","*",0,1],"ContextFetchBot":["allowed","*",0,1]},"llms":{"status":"present","http":200,"bytes":37904},"home":{"status":"ok","http":200,"url":"https://www.deeplearning.ai/","html_tokens":111474,"markdown_tokens":131,"pruned_tokens":4}},
{"rank":8293,"domain":"google.cd","origin":"https://google.cd","robots":{"status":"ok","http":200,"url":"https://www.google.cd/robots.txt","bytes":6624,"groups":4},"bots":{"GPTBot":["partial","*",169,64],"ChatGPT-User":["partial","*",169,64],"OAI-SearchBot":["partial","*",169,64],"ClaudeBot":["partial","*",169,64],"Claude-User":["partial","*",169,64],"Claude-SearchBot":["partial","*",169,64],"anthropic-ai":["partial","*",169,64],"PerplexityBot":["partial","*",169,64],"Perplexity-User":["partial","*",169,64],"Google-Extended":["partial","*",169,64],"Applebot-Extended":["partial","*",169,64],"CCBot":["partial","*",169,64],"Bytespider":["partial","*",169,64],"Amazonbot":["partial","*",169,64],"meta-externalagent":["partial","*",169,64],"cohere-ai":["partial","*",169,64],"Diffbot":["partial","*",169,64],"ContextFetchBot":["partial","*",169,64]},"llms":{"status":"missing","http":404,"bytes":1569},"home":{"status":"ok","http":200,"url":"https://www.google.com/","html_tokens":21299,"markdown_tokens":30,"pruned_tokens":30}},
{"rank":8294,"domain":"byf8855.com","origin":"http://byf8855.com","robots":{"status":"missing","http":404,"url":"http://byf8855.com/robots.txt","bytes":1175,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":1175},"home":{"status":"ok","http":200,"url":"http://byf8855.com/","html_tokens":11207,"markdown_tokens":56,"pruned_tokens":56}},
{"rank":8295,"domain":"zooios.com","origin":"https://zooios.com","robots":{"status":"unreachable","http":null,"url":"https://zooios.com/robots.txt","bytes":0,"groups":0,"error":"cannot resolve zooios.com"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":8296,"domain":"mid-day.com","origin":"https://mid-day.com","robots":{"status":"ok","http":200,"url":"https://www.mid-day.com/robots.txt","bytes":3363,"groups":1},"bots":{"GPTBot":["partial","*",38,1],"ChatGPT-User":["partial","*",38,1],"OAI-SearchBot":["partial","*",38,1],"ClaudeBot":["partial","*",38,1],"Claude-User":["partial","*",38,1],"Claude-SearchBot":["partial","*",38,1],"anthropic-ai":["partial","*",38,1],"PerplexityBot":["partial","*",38,1],"Perplexity-User":["partial","*",38,1],"Google-Extended":["partial","*",38,1],"Applebot-Extended":["partial","*",38,1],"CCBot":["partial","*",38,1],"Bytespider":["partial","*",38,1],"Amazonbot":["partial","*",38,1],"meta-externalagent":["partial","*",38,1],"cohere-ai":["partial","*",38,1],"Diffbot":["partial","*",38,1],"ContextFetchBot":["partial","*",38,1]},"llms":{"status":"missing","http":404,"bytes":22675},"home":{"status":"ok","http":200,"url":"https://www.mid-day.com/","html_tokens":10316,"markdown_tokens":47,"pruned_tokens":47}},
{"rank":8297,"domain":"inps.it","origin":"https://inps.it","robots":{"status":"ok","http":200,"url":"https://www.inps.it/robots.txt","bytes":118,"groups":1},"bots":{"GPTBot":["partial","*",1,1],"ChatGPT-User":["partial","*",1,1],"OAI-SearchBot":["partial","*",1,1],"ClaudeBot":["partial","*",1,1],"Claude-User":["partial","*",1,1],"Claude-SearchBot":["partial","*",1,1],"anthropic-ai":["partial","*",1,1],"PerplexityBot":["partial","*",1,1],"Perplexity-User":["partial","*",1,1],"Google-Extended":["partial","*",1,1],"Applebot-Extended":["partial","*",1,1],"CCBot":["partial","*",1,1],"Bytespider":["partial","*",1,1],"Amazonbot":["partial","*",1,1],"meta-externalagent":["partial","*",1,1],"cohere-ai":["partial","*",1,1],"Diffbot":["partial","*",1,1],"ContextFetchBot":["partial","*",1,1]},"llms":{"status":"missing","http":404,"bytes":653103},"home":{"status":"ok","http":200,"url":"https://www.inps.it/","html_tokens":220293,"markdown_tokens":1140,"pruned_tokens":98}},
{"rank":8298,"domain":"usm.edu","origin":"https://usm.edu","robots":{"status":"ok","http":200,"url":"https://www.usm.edu/robots.txt","bytes":683,"groups":1},"bots":{"GPTBot":["partial","*",23,0],"ChatGPT-User":["partial","*",23,0],"OAI-SearchBot":["partial","*",23,0],"ClaudeBot":["partial","*",23,0],"Claude-User":["partial","*",23,0],"Claude-SearchBot":["partial","*",23,0],"anthropic-ai":["partial","*",23,0],"PerplexityBot":["partial","*",23,0],"Perplexity-User":["partial","*",23,0],"Google-Extended":["partial","*",23,0],"Applebot-Extended":["partial","*",23,0],"CCBot":["partial","*",23,0],"Bytespider":["partial","*",23,0],"Amazonbot":["partial","*",23,0],"meta-externalagent":["partial","*",23,0],"cohere-ai":["partial","*",23,0],"Diffbot":["partial","*",23,0],"ContextFetchBot":["partial","*",23,0]},"llms":{"status":"missing","http":404,"bytes":45985},"home":{"status":"ok","http":200,"url":"https://www.usm.edu/","html_tokens":15696,"markdown_tokens":42,"pruned_tokens":42}},
{"rank":8299,"domain":"google.mv","origin":"https://google.mv","robots":{"status":"ok","http":200,"url":"https://www.google.mv/robots.txt","bytes":6624,"groups":4},"bots":{"GPTBot":["partial","*",169,64],"ChatGPT-User":["partial","*",169,64],"OAI-SearchBot":["partial","*",169,64],"ClaudeBot":["partial","*",169,64],"Claude-User":["partial","*",169,64],"Claude-SearchBot":["partial","*",169,64],"anthropic-ai":["partial","*",169,64],"PerplexityBot":["partial","*",169,64],"Perplexity-User":["partial","*",169,64],"Google-Extended":["partial","*",169,64],"Applebot-Extended":["partial","*",169,64],"CCBot":["partial","*",169,64],"Bytespider":["partial","*",169,64],"Amazonbot":["partial","*",169,64],"meta-externalagent":["partial","*",169,64],"cohere-ai":["partial","*",169,64],"Diffbot":["partial","*",169,64],"ContextFetchBot":["partial","*",169,64]},"llms":{"status":"missing","http":404,"bytes":1569},"home":{"status":"ok","http":200,"url":"https://www.google.com/","html_tokens":21361,"markdown_tokens":30,"pruned_tokens":30}},
{"rank":8300,"domain":"kb.se","origin":"https://kb.se","robots":{"status":"ok","http":200,"url":"https://www.kb.se/robots.txt","bytes":508,"groups":1},"bots":{"GPTBot":["partial","*",15,0],"ChatGPT-User":["partial","*",15,0],"OAI-SearchBot":["partial","*",15,0],"ClaudeBot":["partial","*",15,0],"Claude-User":["partial","*",15,0],"Claude-SearchBot":["partial","*",15,0],"anthropic-ai":["partial","*",15,0],"PerplexityBot":["partial","*",15,0],"Perplexity-User":["partial","*",15,0],"Google-Extended":["partial","*",15,0],"Applebot-Extended":["partial","*",15,0],"CCBot":["partial","*",15,0],"Bytespider":["partial","*",15,0],"Amazonbot":["partial","*",15,0],"meta-externalagent":["partial","*",15,0],"cohere-ai":["partial","*",15,0],"Diffbot":["partial","*",15,0],"ContextFetchBot":["partial","*",15,0]},"llms":{"status":"unreachable","error":"Server disconnected"},"home":{"status":"ok","http":200,"url":"https://www.kb.se/","html_tokens":32093,"markdown_tokens":313,"pruned_tokens":8}},
{"rank":8301,"domain":"profimontaze.sk","origin":"https://profimontaze.sk","robots":{"status":"ok","http":200,"url":"https://profimontaze.sk/robots.txt","bytes":69,"groups":1},"bots":{"GPTBot":["allowed","*",0,1],"ChatGPT-User":["allowed","*",0,1],"OAI-SearchBot":["allowed","*",0,1],"ClaudeBot":["allowed","*",0,1],"Claude-User":["allowed","*",0,1],"Claude-SearchBot":["allowed","*",0,1],"anthropic-ai":["allowed","*",0,1],"PerplexityBot":["allowed","*",0,1],"Perplexity-User":["allowed","*",0,1],"Google-Extended":["allowed","*",0,1],"Applebot-Extended":["allowed","*",0,1],"CCBot":["allowed","*",0,1],"Bytespider":["allowed","*",0,1],"Amazonbot":["allowed","*",0,1],"meta-externalagent":["allowed","*",0,1],"cohere-ai":["allowed","*",0,1],"Diffbot":["allowed","*",0,1],"ContextFetchBot":["allowed","*",0,1]},"llms":{"status":"missing","http":404,"bytes":644},"home":{"status":"ok","http":200,"url":"https://profimontaze.sk/","html_tokens":45787,"markdown_tokens":26466,"pruned_tokens":62}},
{"rank":8302,"domain":"yhuochong.cn","origin":"https://www.yhuochong.cn","robots":{"status":"missing","http":404,"url":"https://www.yhuochong.cn/robots.txt","bytes":153,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":153},"home":{"status":"ok","http":200,"url":"https://www.yhuochong.cn/","html_tokens":5384,"markdown_tokens":138,"pruned_tokens":138}},
{"rank":8303,"domain":"wttw.com","origin":"https://wttw.com","robots":{"status":"ok","http":200,"url":"https://www.wttw.com/robots.txt","bytes":2068,"groups":1},"bots":{"GPTBot":["partial","*",36,18],"ChatGPT-User":["partial","*",36,18],"OAI-SearchBot":["partial","*",36,18],"ClaudeBot":["partial","*",36,18],"Claude-User":["partial","*",36,18],"Claude-SearchBot":["partial","*",36,18],"anthropic-ai":["partial","*",36,18],"PerplexityBot":["partial","*",36,18],"Perplexity-User":["partial","*",36,18],"Google-Extended":["partial","*",36,18],"Applebot-Extended":["partial","*",36,18],"CCBot":["partial","*",36,18],"Bytespider":["partial","*",36,18],"Amazonbot":["partial","*",36,18],"meta-externalagent":["partial","*",36,18],"cohere-ai":["partial","*",36,18],"Diffbot":["partial","*",36,18],"ContextFetchBot":["partial","*",36,18]},"llms":{"status":"missing","http":404,"bytes":2818},"home":{"status":"http_error","http":403,"url":"https://wttw.com/"}},
{"rank":8304,"domain":"uniandes.edu.co","origin":"https://www.uniandes.edu.co","robots":{"status":"ok","http":200,"url":"https://www.uniandes.edu.co/robots.txt","bytes":495,"groups":1},"bots":{"GPTBot":["partial","*",13,1],"ChatGPT-User":["partial","*",13,1],"OAI-SearchBot":["partial","*",13,1],"ClaudeBot":["partial","*",13,1],"Claude-User":["partial","*",13,1],"Claude-SearchBot":["partial","*",13,1],"anthropic-ai":["partial","*",13,1],"PerplexityBot":["partial","*",13,1],"Perplexity-User":["partial","*",13,1],"Google-Extended":["partial","*",13,1],"Applebot-Extended":["partial","*",13,1],"CCBot":["partial","*",13,1],"Bytespider":["partial","*",13,1],"Amazonbot":["partial","*",13,1],"meta-externalagent":["partial","*",13,1],"cohere-ai":["partial","*",13,1],"Diffbot":["partial","*",13,1],"ContextFetchBot":["partial","*",13,1]},"llms":{"status":"present","http":200,"bytes":2385},"home":{"status":"ok","http":200,"url":"https://www.uniandes.edu.co/","html_tokens":39355,"markdown_tokens":596,"pruned_tokens":35}},
{"rank":8305,"domain":"ljhls.cn","origin":"http://ljhls.cn","robots":{"status":"ok","http":200,"url":"http://ljhls.cn/robots.txt","bytes":25,"groups":1},"bots":{"GPTBot":["allowed","*",0,1],"ChatGPT-User":["allowed","*",0,1],"OAI-SearchBot":["allowed","*",0,1],"ClaudeBot":["allowed","*",0,1],"Claude-User":["allowed","*",0,1],"Claude-SearchBot":["allowed","*",0,1],"anthropic-ai":["allowed","*",0,1],"PerplexityBot":["allowed","*",0,1],"Perplexity-User":["allowed","*",0,1],"Google-Extended":["allowed","*",0,1],"Applebot-Extended":["allowed","*",0,1],"CCBot":["allowed","*",0,1],"Bytespider":["allowed","*",0,1],"Amazonbot":["allowed","*",0,1],"meta-externalagent":["allowed","*",0,1],"cohere-ai":["allowed","*",0,1],"Diffbot":["allowed","*",0,1],"ContextFetchBot":["allowed","*",0,1]},"llms":{"status":"missing","http":404,"bytes":1163},"home":{"status":"http_error","http":403,"url":"http://ljhls.cn/"}},
{"rank":8306,"domain":"wan-peng.com","origin":"http://wan-peng.com","robots":{"status":"missing","http":404,"url":"http://wan-peng.com/robots.txt","bytes":1231,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":1231},"home":{"status":"ok","http":200,"url":"http://wan-peng.com/","html_tokens":3469,"markdown_tokens":77,"pruned_tokens":77}},
{"rank":8307,"domain":"airbnb.fr","origin":"https://airbnb.fr","robots":{"status":"ok","http":200,"url":"https://www.airbnb.fr/robots.txt","bytes":20929,"groups":18},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["partial","named",61,2],"OAI-SearchBot":["partial","named",61,2],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",61,2],"Claude-SearchBot":["partial","*",61,2],"anthropic-ai":["partial","named",63,2],"PerplexityBot":["partial","named",63,2],"Perplexity-User":["partial","*",61,2],"Google-Extended":["partial","*",61,2],"Applebot-Extended":["blocked","named",1,0],"CCBot":["partial","*",61,2],"Bytespider":["partial","*",61,2],"Amazonbot":["partial","*",61,2],"meta-externalagent":["partial","named",61,2],"cohere-ai":["partial","named",66,2],"Diffbot":["partial","*",61,2],"ContextFetchBot":["partial","*",61,2]},"llms":{"status":"missing","http":404,"bytes":2947},"home":{"status":"ok","http":200,"url":"https://www.airbnb.fr/","html_tokens":159536,"markdown_tokens":1002,"pruned_tokens":111}},
{"rank":8308,"domain":"fanpage.it","origin":"https://fanpage.it","robots":{"status":"refused","http":403,"url":"https://www.fanpage.it/robots.txt","bytes":382,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":8309,"domain":"adopenstatic.com","origin":"https://adopenstatic.com","robots":{"status":"timeout","http":null,"url":"https://adopenstatic.com/robots.txt","bytes":0,"groups":0,"error":"ConnectTimeout"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":8310,"domain":"google.gm","origin":"https://google.gm","robots":{"status":"ok","http":200,"url":"https://www.google.gm/robots.txt","bytes":6624,"groups":4},"bots":{"GPTBot":["partial","*",169,64],"ChatGPT-User":["partial","*",169,64],"OAI-SearchBot":["partial","*",169,64],"ClaudeBot":["partial","*",169,64],"Claude-User":["partial","*",169,64],"Claude-SearchBot":["partial","*",169,64],"anthropic-ai":["partial","*",169,64],"PerplexityBot":["partial","*",169,64],"Perplexity-User":["partial","*",169,64],"Google-Extended":["partial","*",169,64],"Applebot-Extended":["partial","*",169,64],"CCBot":["partial","*",169,64],"Bytespider":["partial","*",169,64],"Amazonbot":["partial","*",169,64],"meta-externalagent":["partial","*",169,64],"cohere-ai":["partial","*",169,64],"Diffbot":["partial","*",169,64],"ContextFetchBot":["partial","*",169,64]},"llms":{"status":"missing","http":404,"bytes":1569},"home":{"status":"ok","http":200,"url":"https://www.google.com/","html_tokens":21270,"markdown_tokens":30,"pruned_tokens":30}},
{"rank":8311,"domain":"desmos.com","origin":"https://desmos.com","robots":{"status":"ok","http":200,"url":"https://www.desmos.com/robots.txt","bytes":54,"groups":1},"bots":{"GPTBot":["partial","*",2,0],"ChatGPT-User":["partial","*",2,0],"OAI-SearchBot":["partial","*",2,0],"ClaudeBot":["partial","*",2,0],"Claude-User":["partial","*",2,0],"Claude-SearchBot":["partial","*",2,0],"anthropic-ai":["partial","*",2,0],"PerplexityBot":["partial","*",2,0],"Perplexity-User":["partial","*",2,0],"Google-Extended":["partial","*",2,0],"Applebot-Extended":["partial","*",2,0],"CCBot":["partial","*",2,0],"Bytespider":["partial","*",2,0],"Amazonbot":["partial","*",2,0],"meta-externalagent":["partial","*",2,0],"cohere-ai":["partial","*",2,0],"Diffbot":["partial","*",2,0],"ContextFetchBot":["partial","*",2,0]},"llms":{"status":"missing","http":404,"bytes":9696},"home":{"status":"ok","http":200,"url":"https://www.desmos.com/","html_tokens":2424,"markdown_tokens":3,"pruned_tokens":3}},
{"rank":8312,"domain":"documentfoundation.org","origin":"https://documentfoundation.org","robots":{"status":"missing","http":404,"url":"https://www.documentfoundation.org/robots.txt","bytes":146,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":146},"home":{"status":"ok","http":200,"url":"https://www.documentfoundation.org/","html_tokens":2958,"markdown_tokens":744,"pruned_tokens":166}},
{"rank":8313,"domain":"cipd.org","origin":"https://cipd.org","robots":{"status":"ok","http":200,"url":"https://www.cipd.org/robots.txt","bytes":431,"groups":12},"bots":{"GPTBot":["allowed","named",0,1],"ChatGPT-User":["allowed","named",0,1],"OAI-SearchBot":["allowed","named",0,1],"ClaudeBot":["allowed","named",0,1],"Claude-User":["allowed","*",0,1],"Claude-SearchBot":["allowed","*",0,1],"anthropic-ai":["allowed","named",0,1],"PerplexityBot":["allowed","named",0,1],"Perplexity-User":["allowed","*",0,1],"Google-Extended":["allowed","named",0,1],"Applebot-Extended":["allowed","*",0,1],"CCBot":["allowed","named",0,1],"Bytespider":["allowed","*",0,1],"Amazonbot":["allowed","*",0,1],"meta-externalagent":["allowed","*",0,1],"cohere-ai":["allowed","*",0,1],"Diffbot":["allowed","*",0,1],"ContextFetchBot":["allowed","*",0,1]},"llms":{"status":"missing","http":404,"bytes":212001},"home":{"status":"ok","http":200,"url":"https://www.cipd.org/en/","html_tokens":62244,"markdown_tokens":287,"pruned_tokens":80}},
{"rank":8314,"domain":"google.ws","origin":"https://google.ws","robots":{"status":"ok","http":200,"url":"https://www.google.ws/robots.txt","bytes":6624,"groups":4},"bots":{"GPTBot":["partial","*",169,64],"ChatGPT-User":["partial","*",169,64],"OAI-SearchBot":["partial","*",169,64],"ClaudeBot":["partial","*",169,64],"Claude-User":["partial","*",169,64],"Claude-SearchBot":["partial","*",169,64],"anthropic-ai":["partial","*",169,64],"PerplexityBot":["partial","*",169,64],"Perplexity-User":["partial","*",169,64],"Google-Extended":["partial","*",169,64],"Applebot-Extended":["partial","*",169,64],"CCBot":["partial","*",169,64],"Bytespider":["partial","*",169,64],"Amazonbot":["partial","*",169,64],"meta-externalagent":["partial","*",169,64],"cohere-ai":["partial","*",169,64],"Diffbot":["partial","*",169,64],"ContextFetchBot":["partial","*",169,64]},"llms":{"status":"missing","http":404,"bytes":1569},"home":{"status":"ok","http":200,"url":"https://www.google.com/","html_tokens":21366,"markdown_tokens":30,"pruned_tokens":30}},
{"rank":8315,"domain":"avis.com","origin":"https://avis.com","robots":{"status":"ok","http":200,"url":"https://www.avis.com/robots.txt","bytes":324,"groups":1},"bots":{"GPTBot":["partial","*",9,1],"ChatGPT-User":["partial","*",9,1],"OAI-SearchBot":["partial","*",9,1],"ClaudeBot":["partial","*",9,1],"Claude-User":["partial","*",9,1],"Claude-SearchBot":["partial","*",9,1],"anthropic-ai":["partial","*",9,1],"PerplexityBot":["partial","*",9,1],"Perplexity-User":["partial","*",9,1],"Google-Extended":["partial","*",9,1],"Applebot-Extended":["partial","*",9,1],"CCBot":["partial","*",9,1],"Bytespider":["partial","*",9,1],"Amazonbot":["partial","*",9,1],"meta-externalagent":["partial","*",9,1],"cohere-ai":["partial","*",9,1],"Diffbot":["partial","*",9,1],"ContextFetchBot":["partial","*",9,1]},"llms":{"status":"present","http":200,"bytes":3632},"home":{"status":"ok","http":200,"url":"https://www.avis.com/en/home","html_tokens":203491,"markdown_tokens":133,"pruned_tokens":31}},
{"rank":8316,"domain":"lewrockwell.com","origin":"https://lewrockwell.com","robots":{"status":"ok","http":200,"url":"https://lewrockwell.com/robots.txt","bytes":70,"groups":1},"bots":{"GPTBot":["partial","*",1,0],"ChatGPT-User":["partial","*",1,0],"OAI-SearchBot":["partial","*",1,0],"ClaudeBot":["partial","*",1,0],"Claude-User":["partial","*",1,0],"Claude-SearchBot":["partial","*",1,0],"anthropic-ai":["partial","*",1,0],"PerplexityBot":["partial","*",1,0],"Perplexity-User":["partial","*",1,0],"Google-Extended":["partial","*",1,0],"Applebot-Extended":["partial","*",1,0],"CCBot":["partial","*",1,0],"Bytespider":["partial","*",1,0],"Amazonbot":["partial","*",1,0],"meta-externalagent":["partial","*",1,0],"cohere-ai":["partial","*",1,0],"Diffbot":["partial","*",1,0],"ContextFetchBot":["partial","*",1,0]},"llms":{"status":"crawl_delay","error":"Crawl-delay 100 s"},"home":{"status":"crawl_delay","error":"Crawl-delay 100 s"}},
{"rank":8317,"domain":"marksmile.com","origin":"https://marksmile.com","robots":{"status":"unreachable","http":null,"url":"https://marksmile.com/robots.txt","bytes":0,"groups":0,"error":"[SSL: CERTIFICATE_VERIFY_FAILED] certificate verify failed: unable to get local issuer certificate (_ssl.c:1028)"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":8318,"domain":"sinolight.cn","origin":"https://sinolight.cn","robots":{"status":"server_error","http":500,"url":"https://sinolight.cn/robots.txt","bytes":5102,"groups":0,"error":"HTTP 500"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":8319,"domain":"fs.blog","origin":"https://fs.blog","robots":{"status":"ok","http":200,"url":"https://fs.blog/robots.txt","bytes":3741,"groups":51},"bots":{"GPTBot":["blocked","named",2,0],"ChatGPT-User":["allowed","named",0,1],"OAI-SearchBot":["allowed","named",0,1],"ClaudeBot":["blocked","named",2,0],"Claude-User":["allowed","named",0,1],"Claude-SearchBot":["allowed","named",0,1],"anthropic-ai":["blocked","named",2,0],"PerplexityBot":["allowed","named",0,1],"Perplexity-User":["allowed","named",0,1],"Google-Extended":["blocked","named",2,0],"Applebot-Extended":["blocked","named",2,0],"CCBot":["blocked","named",2,0],"Bytespider":["blocked","named",2,0],"Amazonbot":["blocked","named",2,0],"meta-externalagent":["blocked","named",2,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["allowed","*",0,1]},"llms":{"status":"other_text","http":200,"bytes":2098},"home":{"status":"ok","http":200,"url":"https://fs.blog/","html_tokens":36760,"markdown_tokens":309,"pruned_tokens":4}},
{"rank":8320,"domain":"harrods.com","origin":"https://harrods.com","robots":{"status":"refused","http":403,"url":"https://www.harrods.com/robots.txt","bytes":807,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":8321,"domain":"vg247.com","origin":"https://vg247.com","robots":{"status":"ok","http":200,"url":"https://www.vg247.com/robots.txt","bytes":5606,"groups":2},"bots":{"GPTBot":["partial","named",1,1],"ChatGPT-User":["partial","named",1,1],"OAI-SearchBot":["partial","named",1,1],"ClaudeBot":["partial","named",1,1],"Claude-User":["partial","named",1,1],"Claude-SearchBot":["partial","named",1,1],"anthropic-ai":["partial","named",1,1],"PerplexityBot":["partial","named",1,1],"Perplexity-User":["partial","named",1,1],"Google-Extended":["partial","*",9,0],"Applebot-Extended":["partial","named",1,1],"CCBot":["partial","named",1,1],"Bytespider":["partial","named",1,1],"Amazonbot":["partial","named",1,1],"meta-externalagent":["partial","named",1,1],"cohere-ai":["partial","named",1,1],"Diffbot":["partial","named",1,1],"ContextFetchBot":["partial","*",9,0]},"llms":{"status":"missing","http":404,"bytes":2271},"home":{"status":"ok","http":200,"url":"https://www.vg247.com/","html_tokens":69819,"markdown_tokens":2054,"pruned_tokens":142}},
{"rank":8322,"domain":"ibs.it","origin":"https://ibs.it","robots":{"status":"ok","http":200,"url":"https://www.ibs.it/robots.txt","bytes":5868,"groups":2},"bots":{"GPTBot":["partial","*",26,11],"ChatGPT-User":["partial","*",26,11],"OAI-SearchBot":["partial","*",26,11],"ClaudeBot":["partial","*",26,11],"Claude-User":["partial","*",26,11],"Claude-SearchBot":["partial","*",26,11],"anthropic-ai":["partial","*",26,11],"PerplexityBot":["partial","*",26,11],"Perplexity-User":["partial","*",26,11],"Google-Extended":["partial","*",26,11],"Applebot-Extended":["partial","*",26,11],"CCBot":["partial","*",26,11],"Bytespider":["partial","*",26,11],"Amazonbot":["partial","*",26,11],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["partial","*",26,11],"Diffbot":["partial","*",26,11],"ContextFetchBot":["partial","*",26,11]},"llms":{"status":"missing","http":404,"bytes":473299},"home":{"status":"ok","http":200,"url":"https://www.ibs.it/","html_tokens":204232,"markdown_tokens":138,"pruned_tokens":138}},
{"rank":8323,"domain":"lshtm.ac.uk","origin":"https://lshtm.ac.uk","robots":{"status":"ok","http":200,"url":"https://www.lshtm.ac.uk/robots.txt","bytes":2005,"groups":1},"bots":{"GPTBot":["partial","*",33,18],"ChatGPT-User":["partial","*",33,18],"OAI-SearchBot":["partial","*",33,18],"ClaudeBot":["partial","*",33,18],"Claude-User":["partial","*",33,18],"Claude-SearchBot":["partial","*",33,18],"anthropic-ai":["partial","*",33,18],"PerplexityBot":["partial","*",33,18],"Perplexity-User":["partial","*",33,18],"Google-Extended":["partial","*",33,18],"Applebot-Extended":["partial","*",33,18],"CCBot":["partial","*",33,18],"Bytespider":["partial","*",33,18],"Amazonbot":["partial","*",33,18],"meta-externalagent":["partial","*",33,18],"cohere-ai":["partial","*",33,18],"Diffbot":["partial","*",33,18],"ContextFetchBot":["partial","*",33,18]},"llms":{"status":"missing","http":404,"bytes":550},"home":{"status":"http_error","http":403,"url":"https://www.lshtm.ac.uk/"}},
{"rank":8324,"domain":"google.nu","origin":"https://google.nu","robots":{"status":"ok","http":200,"url":"https://www.google.nu/robots.txt","bytes":6624,"groups":4},"bots":{"GPTBot":["partial","*",169,64],"ChatGPT-User":["partial","*",169,64],"OAI-SearchBot":["partial","*",169,64],"ClaudeBot":["partial","*",169,64],"Claude-User":["partial","*",169,64],"Claude-SearchBot":["partial","*",169,64],"anthropic-ai":["partial","*",169,64],"PerplexityBot":["partial","*",169,64],"Perplexity-User":["partial","*",169,64],"Google-Extended":["partial","*",169,64],"Applebot-Extended":["partial","*",169,64],"CCBot":["partial","*",169,64],"Bytespider":["partial","*",169,64],"Amazonbot":["partial","*",169,64],"meta-externalagent":["partial","*",169,64],"cohere-ai":["partial","*",169,64],"Diffbot":["partial","*",169,64],"ContextFetchBot":["partial","*",169,64]},"llms":{"status":"missing","http":404,"bytes":1569},"home":{"status":"ok","http":200,"url":"https://www.google.com/","html_tokens":21320,"markdown_tokens":30,"pruned_tokens":30}},
{"rank":8325,"domain":"utep.edu","origin":"https://utep.edu","robots":{"status":"missing","http":404,"url":"https://www.utep.edu/robots.txt","bytes":13141,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":13141},"home":{"status":"ok","http":200,"url":"https://www.utep.edu/","html_tokens":28597,"markdown_tokens":158,"pruned_tokens":158}},
{"rank":8326,"domain":"weblio.jp","origin":"https://weblio.jp","robots":{"status":"missing","http":404,"url":"https://weblio.jp/robots.txt","bytes":236,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":236},"home":{"status":"http_error","http":404,"url":"https://weblio.jp/"}},
{"rank":8327,"domain":"novinky.cz","origin":"https://novinky.cz","robots":{"status":"ok","http":200,"url":"https://www.novinky.cz/robots.txt","bytes":1280,"groups":3},"bots":{"GPTBot":["partial","*",23,0],"ChatGPT-User":["partial","*",23,0],"OAI-SearchBot":["partial","*",23,0],"ClaudeBot":["partial","*",23,0],"Claude-User":["partial","*",23,0],"Claude-SearchBot":["partial","*",23,0],"anthropic-ai":["partial","*",23,0],"PerplexityBot":["partial","*",23,0],"Perplexity-User":["partial","*",23,0],"Google-Extended":["partial","*",23,0],"Applebot-Extended":["partial","*",23,0],"CCBot":["partial","*",23,0],"Bytespider":["partial","*",23,0],"Amazonbot":["partial","*",23,0],"meta-externalagent":["partial","*",23,0],"cohere-ai":["partial","*",23,0],"Diffbot":["partial","*",23,0],"ContextFetchBot":["partial","*",23,0]},"llms":{"status":"missing","http":404,"bytes":71156},"home":{"status":"disallowed","error":"robots.txt of login.seznam.cz disallows /api/v1/autologin?service=novinky&return_url=https%3A%2F%2Fwww.novinky.cz%2F%3Fnoredirect%3D1"}},
{"rank":8328,"domain":"or.cr","origin":"https://or.cr","robots":{"status":"unreachable","http":null,"url":"https://or.cr/robots.txt","bytes":0,"groups":0,"error":"cannot resolve or.cr"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":8329,"domain":"mlr.press","origin":"https://mlr.press","robots":{"status":"missing","http":404,"url":"https://mlr.press/robots.txt","bytes":9115,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":9115},"home":{"status":"http_error","http":404,"url":"https://mlr.press/"}},
{"rank":8330,"domain":"phoenixcontact.com","origin":"https://phoenixcontact.com","robots":{"status":"refused","http":403,"url":"https://www.phoenixcontact.com/robots.txt","bytes":390,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":8331,"domain":"nagoya.jp","origin":"https://nagoya.jp","robots":{"status":"unreachable","http":null,"url":"https://nagoya.jp/robots.txt","bytes":0,"groups":0,"error":"cannot resolve nagoya.jp"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":8332,"domain":"beatstars.com","origin":"https://beatstars.com","robots":{"status":"ok","http":200,"url":"https://www.beatstars.com/robots.txt","bytes":1079,"groups":19},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["partial","*",12,1],"OAI-SearchBot":["partial","*",12,1],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",12,1],"Claude-SearchBot":["partial","*",12,1],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["partial","*",12,1],"Perplexity-User":["partial","*",12,1],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",12,1]},"llms":{"status":"present","http":200,"bytes":1780},"home":{"status":"ok","http":200,"url":"https://www.beatstars.com/","html_tokens":13715,"markdown_tokens":0,"pruned_tokens":0}},
{"rank":8333,"domain":"mysterythemes.com","origin":"https://mysterythemes.com","robots":{"status":"ok","http":200,"url":"https://mysterythemes.com/robots.txt","bytes":233,"groups":1},"bots":{"GPTBot":["partial","*",2,0],"ChatGPT-User":["partial","*",2,0],"OAI-SearchBot":["partial","*",2,0],"ClaudeBot":["partial","*",2,0],"Claude-User":["partial","*",2,0],"Claude-SearchBot":["partial","*",2,0],"anthropic-ai":["partial","*",2,0],"PerplexityBot":["partial","*",2,0],"Perplexity-User":["partial","*",2,0],"Google-Extended":["partial","*",2,0],"Applebot-Extended":["partial","*",2,0],"CCBot":["partial","*",2,0],"Bytespider":["partial","*",2,0],"Amazonbot":["partial","*",2,0],"meta-externalagent":["partial","*",2,0],"cohere-ai":["partial","*",2,0],"Diffbot":["partial","*",2,0],"ContextFetchBot":["partial","*",2,0]},"llms":{"status":"other_text","http":200,"bytes":2522},"home":{"status":"ok","http":200,"url":"https://mysterythemes.com/","html_tokens":33369,"markdown_tokens":473,"pruned_tokens":150}},
{"rank":8334,"domain":"fiercepharma.com","origin":"https://fiercepharma.com","robots":{"status":"ok","http":200,"url":"https://www.fiercepharma.com/robots.txt","bytes":1620,"groups":1},"bots":{"GPTBot":["partial","*",23,18],"ChatGPT-User":["partial","*",23,18],"OAI-SearchBot":["partial","*",23,18],"ClaudeBot":["partial","*",23,18],"Claude-User":["partial","*",23,18],"Claude-SearchBot":["partial","*",23,18],"anthropic-ai":["partial","*",23,18],"PerplexityBot":["partial","*",23,18],"Perplexity-User":["partial","*",23,18],"Google-Extended":["partial","*",23,18],"Applebot-Extended":["partial","*",23,18],"CCBot":["partial","*",23,18],"Bytespider":["partial","*",23,18],"Amazonbot":["partial","*",23,18],"meta-externalagent":["partial","*",23,18],"cohere-ai":["partial","*",23,18],"Diffbot":["partial","*",23,18],"ContextFetchBot":["partial","*",23,18]},"llms":{"status":"http_error","http":403,"bytes":507},"home":{"status":"http_error","http":403,"url":"https://fiercepharma.com/"}},
{"rank":8335,"domain":"scrum.org","origin":"https://scrum.org","robots":{"status":"refused","http":403,"url":"https://scrum.org/robots.txt","bytes":919,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":8336,"domain":"divadloustolu.cz","origin":"https://divadloustolu.cz","robots":{"status":"ok","http":200,"url":"https://www.divadloustolu.cz/robots.txt","bytes":145,"groups":1},"bots":{"GPTBot":["partial","*",2,1],"ChatGPT-User":["partial","*",2,1],"OAI-SearchBot":["partial","*",2,1],"ClaudeBot":["partial","*",2,1],"Claude-User":["partial","*",2,1],"Claude-SearchBot":["partial","*",2,1],"anthropic-ai":["partial","*",2,1],"PerplexityBot":["partial","*",2,1],"Perplexity-User":["partial","*",2,1],"Google-Extended":["partial","*",2,1],"Applebot-Extended":["partial","*",2,1],"CCBot":["partial","*",2,1],"Bytespider":["partial","*",2,1],"Amazonbot":["partial","*",2,1],"meta-externalagent":["partial","*",2,1],"cohere-ai":["partial","*",2,1],"Diffbot":["partial","*",2,1],"ContextFetchBot":["partial","*",2,1]},"llms":{"status":"missing","http":404,"bytes":2017},"home":{"status":"ok","http":200,"url":"https://www.divadloustolu.cz/","html_tokens":14182,"markdown_tokens":6302,"pruned_tokens":86}},
{"rank":8337,"domain":"km.ru","origin":"https://km.ru","robots":{"status":"ok","http":200,"url":"https://www.km.ru/robots.txt","bytes":2361,"groups":19},"bots":{"GPTBot":["partial","named",1,0],"ChatGPT-User":["partial","named",1,0],"OAI-SearchBot":["partial","named",1,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",7,0],"Claude-SearchBot":["partial","*",7,0],"anthropic-ai":["partial","*",7,0],"PerplexityBot":["partial","named",1,0],"Perplexity-User":["partial","*",7,0],"Google-Extended":["partial","named",1,0],"Applebot-Extended":["partial","named",1,0],"CCBot":["partial","named",1,0],"Bytespider":["partial","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["partial","named",1,0],"cohere-ai":["partial","*",7,0],"Diffbot":["partial","*",7,0],"ContextFetchBot":["partial","*",7,0]},"llms":{"status":"missing","http":404,"bytes":2153},"home":{"status":"ok","http":200,"url":"https://www.km.ru/","html_tokens":24569,"markdown_tokens":192,"pruned_tokens":145}},
{"rank":8338,"domain":"rtings.com","origin":"https://rtings.com","robots":{"status":"ok","http":200,"url":"https://www.rtings.com/robots.txt","bytes":159,"groups":1},"bots":{"GPTBot":["partial","*",2,0],"ChatGPT-User":["partial","*",2,0],"OAI-SearchBot":["partial","*",2,0],"ClaudeBot":["partial","*",2,0],"Claude-User":["partial","*",2,0],"Claude-SearchBot":["partial","*",2,0],"anthropic-ai":["partial","*",2,0],"PerplexityBot":["partial","*",2,0],"Perplexity-User":["partial","*",2,0],"Google-Extended":["partial","*",2,0],"Applebot-Extended":["partial","*",2,0],"CCBot":["partial","*",2,0],"Bytespider":["partial","*",2,0],"Amazonbot":["partial","*",2,0],"meta-externalagent":["partial","*",2,0],"cohere-ai":["partial","*",2,0],"Diffbot":["partial","*",2,0],"ContextFetchBot":["partial","*",2,0]},"llms":{"status":"missing","http":404,"bytes":171391},"home":{"status":"ok","http":200,"url":"https://www.rtings.com/","html_tokens":67532,"markdown_tokens":131,"pruned_tokens":131}},
{"rank":8339,"domain":"jcpenney.com","origin":"https://jcpenney.com","robots":{"status":"timeout","http":null,"url":"https://jcpenney.com/robots.txt","bytes":0,"groups":0,"error":"timed out after 10 s"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":8340,"domain":"justpaste.me","origin":"https://justpaste.me","robots":{"status":"html","http":200,"url":"https://justpaste.me/","bytes":14584,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":8341,"domain":"atsjournals.org","origin":"https://atsjournals.org","robots":{"status":"ok","http":200,"url":"https://atsjournals.org/robots.txt","bytes":4269,"groups":2},"bots":{"GPTBot":["partial","*",61,3],"ChatGPT-User":["partial","*",61,3],"OAI-SearchBot":["partial","*",61,3],"ClaudeBot":["partial","*",61,3],"Claude-User":["partial","*",61,3],"Claude-SearchBot":["partial","*",61,3],"anthropic-ai":["partial","*",61,3],"PerplexityBot":["partial","*",61,3],"Perplexity-User":["partial","*",61,3],"Google-Extended":["partial","*",61,3],"Applebot-Extended":["partial","*",61,3],"CCBot":["partial","*",61,3],"Bytespider":["partial","*",61,3],"Amazonbot":["partial","*",61,3],"meta-externalagent":["partial","*",61,3],"cohere-ai":["partial","*",61,3],"Diffbot":["partial","*",61,3],"ContextFetchBot":["partial","*",61,3]},"llms":{"status":"http_error","http":403,"bytes":5536},"home":{"status":"http_error","http":403,"url":"https://academic.oup.com/atsjournals"}},
{"rank":8342,"domain":"licdn.com","origin":"https://licdn.com","robots":{"status":"unreachable","http":null,"url":"https://licdn.com/robots.txt","bytes":0,"groups":0,"error":"cannot resolve licdn.com"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":8343,"domain":"unwater.org","origin":"https://unwater.org","robots":{"status":"refused","http":403,"url":"https://unwater.org/robots.txt","bytes":5527,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":8344,"domain":"kmu.gov.ua","origin":"https://www.kmu.gov.ua","robots":{"status":"ok","http":200,"url":"https://www.kmu.gov.ua/robots.txt","bytes":410,"groups":2},"bots":{"GPTBot":["partial","*",11,0],"ChatGPT-User":["partial","*",11,0],"OAI-SearchBot":["partial","*",11,0],"ClaudeBot":["partial","*",11,0],"Claude-User":["partial","*",11,0],"Claude-SearchBot":["partial","*",11,0],"anthropic-ai":["partial","*",11,0],"PerplexityBot":["partial","*",11,0],"Perplexity-User":["partial","*",11,0],"Google-Extended":["partial","*",11,0],"Applebot-Extended":["partial","*",11,0],"CCBot":["partial","*",11,0],"Bytespider":["partial","*",11,0],"Amazonbot":["partial","*",11,0],"meta-externalagent":["partial","*",11,0],"cohere-ai":["partial","*",11,0],"Diffbot":["partial","*",11,0],"ContextFetchBot":["partial","*",11,0]},"llms":{"status":"html","http":200,"bytes":2516},"home":{"status":"ok","http":200,"url":"https://www.kmu.gov.ua/","html_tokens":108780,"markdown_tokens":295,"pruned_tokens":138}},
{"rank":8345,"domain":"marsh.com","origin":"https://marsh.com","robots":{"status":"ok","http":200,"url":"https://www.marsh.com/robots.txt","bytes":654,"groups":1},"bots":{"GPTBot":["partial","*",19,0],"ChatGPT-User":["partial","*",19,0],"OAI-SearchBot":["partial","*",19,0],"ClaudeBot":["partial","*",19,0],"Claude-User":["partial","*",19,0],"Claude-SearchBot":["partial","*",19,0],"anthropic-ai":["partial","*",19,0],"PerplexityBot":["partial","*",19,0],"Perplexity-User":["partial","*",19,0],"Google-Extended":["partial","*",19,0],"Applebot-Extended":["partial","*",19,0],"CCBot":["partial","*",19,0],"Bytespider":["partial","*",19,0],"Amazonbot":["partial","*",19,0],"meta-externalagent":["partial","*",19,0],"cohere-ai":["partial","*",19,0],"Diffbot":["partial","*",19,0],"ContextFetchBot":["partial","*",19,0]},"llms":{"status":"missing","http":404,"bytes":238612},"home":{"status":"ok","http":200,"url":"https://www.marsh.com/en/home.html","html_tokens":17600,"markdown_tokens":376,"pruned_tokens":188}},
{"rank":8346,"domain":"intoday.in","origin":"https://intoday.in","robots":{"status":"html","http":200,"url":"https://www.indiatodaygroup.com/","bytes":190751,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":8347,"domain":"cancer.ca","origin":"https://cancer.ca","robots":{"status":"ok","http":200,"url":"https://cancer.ca/robots.txt","bytes":109,"groups":1},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"present","http":200,"bytes":22104},"home":{"status":"ok","http":200,"url":"https://cancer.ca/en/","html_tokens":37256,"markdown_tokens":214,"pruned_tokens":103}},
{"rank":8348,"domain":"xtliyi.cn","origin":"http://xtliyi.cn","robots":{"status":"missing","http":404,"url":"http://xtliyi.cn/robots.txt","bytes":1308,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":1308},"home":{"status":"ok","http":200,"url":"http://xtliyi.cn/","html_tokens":28518,"markdown_tokens":193,"pruned_tokens":144}},
{"rank":8349,"domain":"odin.com","origin":"https://odin.com","robots":{"status":"unreachable","http":null,"url":"https://odin.com/robots.txt","bytes":0,"groups":0,"error":"cannot resolve odin.com"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":8350,"domain":"malwarebytes.org","origin":"https://malwarebytes.org","robots":{"status":"ok","http":200,"url":"https://www.malwarebytes.com/robots.txt","bytes":1816,"groups":1},"bots":{"GPTBot":["partial","named",42,1],"ChatGPT-User":["partial","*",42,1],"OAI-SearchBot":["partial","named",42,1],"ClaudeBot":["partial","named",42,1],"Claude-User":["partial","*",42,1],"Claude-SearchBot":["partial","*",42,1],"anthropic-ai":["partial","*",42,1],"PerplexityBot":["partial","named",42,1],"Perplexity-User":["partial","*",42,1],"Google-Extended":["partial","*",42,1],"Applebot-Extended":["partial","*",42,1],"CCBot":["partial","*",42,1],"Bytespider":["partial","*",42,1],"Amazonbot":["partial","*",42,1],"meta-externalagent":["partial","*",42,1],"cohere-ai":["partial","*",42,1],"Diffbot":["partial","*",42,1],"ContextFetchBot":["partial","*",42,1]},"llms":{"status":"present","http":200,"bytes":7001},"home":{"status":"ok","http":200,"url":"https://www.malwarebytes.com/","html_tokens":256868,"markdown_tokens":1111,"pruned_tokens":268}},
{"rank":8351,"domain":"meyerweb.com","origin":"https://meyerweb.com","robots":{"status":"ok","http":200,"url":"https://meyerweb.com/robots.txt","bytes":60,"groups":2},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"missing","http":404,"bytes":6470},"home":{"status":"ok","http":200,"url":"https://meyerweb.com/","html_tokens":23040,"markdown_tokens":1714,"pruned_tokens":160}},
{"rank":8352,"domain":"base44.app","origin":"https://base44.app","robots":{"status":"ok","http":200,"url":"https://base44.app/robots.txt","bytes":67,"groups":1},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"missing","http":404,"bytes":0},"home":{"status":"ok","http":200,"url":"https://base44.com","html_tokens":68784,"markdown_tokens":521,"pruned_tokens":341}},
{"rank":8353,"domain":"href.li","origin":"https://href.li","robots":{"status":"ok","http":200,"url":"https://href.li/robots.txt","bytes":26,"groups":1},"bots":{"GPTBot":["blocked","*",1,0],"ChatGPT-User":["blocked","*",1,0],"OAI-SearchBot":["blocked","*",1,0],"ClaudeBot":["blocked","*",1,0],"Claude-User":["blocked","*",1,0],"Claude-SearchBot":["blocked","*",1,0],"anthropic-ai":["blocked","*",1,0],"PerplexityBot":["blocked","*",1,0],"Perplexity-User":["blocked","*",1,0],"Google-Extended":["blocked","*",1,0],"Applebot-Extended":["blocked","*",1,0],"CCBot":["blocked","*",1,0],"Bytespider":["blocked","*",1,0],"Amazonbot":["blocked","*",1,0],"meta-externalagent":["blocked","*",1,0],"cohere-ai":["blocked","*",1,0],"Diffbot":["blocked","*",1,0],"ContextFetchBot":["blocked","*",1,0]},"llms":{"status":"disallowed"},"home":{"status":"disallowed"}},
{"rank":8354,"domain":"vivo.com","origin":"https://vivo.com","robots":{"status":"ok","http":200,"url":"https://www.vivo.com/robots.txt","bytes":559,"groups":1},"bots":{"GPTBot":["partial","*",15,0],"ChatGPT-User":["partial","*",15,0],"OAI-SearchBot":["partial","*",15,0],"ClaudeBot":["partial","*",15,0],"Claude-User":["partial","*",15,0],"Claude-SearchBot":["partial","*",15,0],"anthropic-ai":["partial","*",15,0],"PerplexityBot":["partial","*",15,0],"Perplexity-User":["partial","*",15,0],"Google-Extended":["partial","*",15,0],"Applebot-Extended":["partial","*",15,0],"CCBot":["partial","*",15,0],"Bytespider":["partial","*",15,0],"Amazonbot":["partial","*",15,0],"meta-externalagent":["partial","*",15,0],"cohere-ai":["partial","*",15,0],"Diffbot":["partial","*",15,0],"ContextFetchBot":["partial","*",15,0]},"llms":{"status":"missing","http":404,"bytes":146},"home":{"status":"ok","http":200,"url":"https://www.vivo.com/en","html_tokens":20698,"markdown_tokens":44,"pruned_tokens":44}},
{"rank":8355,"domain":"silive.com","origin":"https://silive.com","robots":{"status":"ok","http":200,"url":"https://www.silive.com/robots.txt","bytes":2818,"groups":4},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["partial","named",1,11],"OAI-SearchBot":["partial","named",1,11],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","named",1,11],"Claude-SearchBot":["partial","named",1,11],"anthropic-ai":["partial","*",30,3],"PerplexityBot":["partial","named",1,11],"Perplexity-User":["partial","named",1,11],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["partial","*",30,3],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",30,3]},"llms":{"status":"missing","http":404,"bytes":249785},"home":{"status":"http_error","http":403,"url":"https://www.silive.com/"}},
{"rank":8356,"domain":"digitalmarketinginstitute.com","origin":"https://digitalmarketinginstitute.com","robots":{"status":"missing","http":404,"url":"https://digitalmarketinginstitute.com/robots.txt","bytes":81695,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":81695},"home":{"status":"ok","http":200,"url":"https://digitalmarketinginstitute.com/","html_tokens":373543,"markdown_tokens":4429,"pruned_tokens":106}},
{"rank":8357,"domain":"blog.br","origin":"https://blog.br","robots":{"status":"unreachable","http":null,"url":"https://blog.br/robots.txt","bytes":0,"groups":0,"error":"cannot resolve blog.br"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":8358,"domain":"nest.com","origin":"https://nest.com","robots":{"status":"ok","http":200,"url":"https://nest.com/robots.txt","bytes":352,"groups":2},"bots":{"GPTBot":["partial","*",10,1],"ChatGPT-User":["partial","*",10,1],"OAI-SearchBot":["partial","*",10,1],"ClaudeBot":["partial","*",10,1],"Claude-User":["partial","*",10,1],"Claude-SearchBot":["partial","*",10,1],"anthropic-ai":["partial","*",10,1],"PerplexityBot":["partial","*",10,1],"Perplexity-User":["partial","*",10,1],"Google-Extended":["partial","*",10,1],"Applebot-Extended":["partial","*",10,1],"CCBot":["partial","*",10,1],"Bytespider":["partial","*",10,1],"Amazonbot":["partial","*",10,1],"meta-externalagent":["partial","*",10,1],"cohere-ai":["partial","*",10,1],"Diffbot":["partial","*",10,1],"ContextFetchBot":["partial","*",10,1]},"llms":{"status":"html","http":200,"bytes":30890},"home":{"status":"ok","http":200,"url":"https://store.google.com/us/category/connected_home?hl=en-US&GoogleNest&utm_source=nest_redirect&utm_medium=google_oo&utm_campaign=homepage","html_tokens":682718,"markdown_tokens":100,"pruned_tokens":69}},
{"rank":8359,"domain":"mongodb.org","origin":"https://mongodb.org","robots":{"status":"ok","http":200,"url":"https://www.mongodb.com/robots.txt","bytes":2859,"groups":7},"bots":{"GPTBot":["partial","*",63,0],"ChatGPT-User":["partial","*",63,0],"OAI-SearchBot":["partial","*",63,0],"ClaudeBot":["partial","*",63,0],"Claude-User":["partial","*",63,0],"Claude-SearchBot":["partial","*",63,0],"anthropic-ai":["partial","*",63,0],"PerplexityBot":["partial","*",63,0],"Perplexity-User":["partial","*",63,0],"Google-Extended":["partial","*",63,0],"Applebot-Extended":["partial","*",63,0],"CCBot":["partial","*",63,0],"Bytespider":["partial","*",63,0],"Amazonbot":["partial","*",63,0],"meta-externalagent":["partial","*",63,0],"cohere-ai":["partial","*",63,0],"Diffbot":["partial","*",63,0],"ContextFetchBot":["partial","*",63,0]},"llms":{"status":"present","http":200,"bytes":10385},"home":{"status":"ok","http":200,"url":"https://www.mongodb.com/","html_tokens":250186,"markdown_tokens":726,"pruned_tokens":325}},
{"rank":8360,"domain":"wikimatrix.org","origin":"https://wikimatrix.org","robots":{"status":"html","http":200,"url":"https://www.wikimatrix.org/","bytes":24303,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":8361,"domain":"gdchemistry.cn","origin":"https://gdchemistry.cn","robots":{"status":"ok","http":200,"url":"https://gdchemistry.cn/robots.txt","bytes":115,"groups":3},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":1231},"home":{"status":"ok","http":200,"url":"https://gdchemistry.cn/","html_tokens":6710,"markdown_tokens":31,"pruned_tokens":31}},
{"rank":8362,"domain":"mintel.com","origin":"https://mintel.com","robots":{"status":"html","http":200,"url":"https://www.mintel.com","bytes":331046,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":8363,"domain":"createspace.com","origin":"https://createspace.com","robots":{"status":"html","http":200,"url":"https://kdp.amazon.com/createspace-transfer","bytes":153117,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":8364,"domain":"pe.kr","origin":"https://pe.kr","robots":{"status":"unreachable","http":null,"url":"https://pe.kr/robots.txt","bytes":0,"groups":0,"error":"cannot resolve pe.kr"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":8365,"domain":"canon-europe.com","origin":"https://canon-europe.com","robots":{"status":"refused","http":403,"url":"https://www.canon-europe.com/robots.txt","bytes":392,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":8366,"domain":"jyhuanyu.cn","origin":"http://jyhuanyu.cn","robots":{"status":"refused","http":444,"url":"http://jyhuanyu.cn/robots.txt","bytes":0,"groups":0,"error":"HTTP 444"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":8367,"domain":"poshmark.com","origin":"https://poshmark.com","robots":{"status":"ok","http":200,"url":"https://poshmark.com/robots.txt","bytes":1027,"groups":2},"bots":{"GPTBot":["partial","*",42,0],"ChatGPT-User":["partial","*",42,0],"OAI-SearchBot":["partial","*",42,0],"ClaudeBot":["partial","*",42,0],"Claude-User":["partial","*",42,0],"Claude-SearchBot":["partial","*",42,0],"anthropic-ai":["partial","*",42,0],"PerplexityBot":["partial","*",42,0],"Perplexity-User":["partial","*",42,0],"Google-Extended":["partial","*",42,0],"Applebot-Extended":["partial","*",42,0],"CCBot":["partial","*",42,0],"Bytespider":["partial","*",42,0],"Amazonbot":["partial","*",42,0],"meta-externalagent":["partial","*",42,0],"cohere-ai":["partial","*",42,0],"Diffbot":["partial","*",42,0],"ContextFetchBot":["partial","*",42,0]},"llms":{"status":"missing","http":404,"bytes":199704},"home":{"status":"ok","http":200,"url":"https://poshmark.com/","html_tokens":226788,"markdown_tokens":183,"pruned_tokens":150}},
{"rank":8368,"domain":"tradetracker.net","origin":"https://tradetracker.net","robots":{"status":"ok","http":200,"url":"https://tradetracker.net/robots.txt","bytes":23,"groups":1},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"html","http":200,"bytes":39316},"home":{"status":"ok","http":200,"url":"https://tradetracker.com","html_tokens":9824,"markdown_tokens":120,"pruned_tokens":120}},
{"rank":8369,"domain":"vpro.nl","origin":"https://vpro.nl","robots":{"status":"ok","http":200,"url":"https://www.vpro.nl/robots.txt","bytes":645,"groups":14},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["blocked","named",1,0],"OAI-SearchBot":["partial","*",6,1],"ClaudeBot":["partial","*",6,1],"Claude-User":["partial","*",6,1],"Claude-SearchBot":["partial","*",6,1],"anthropic-ai":["partial","*",6,1],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["partial","*",6,1],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["partial","*",6,1],"CCBot":["blocked","named",1,0],"Bytespider":["partial","*",6,1],"Amazonbot":["partial","*",6,1],"meta-externalagent":["partial","*",6,1],"cohere-ai":["partial","*",6,1],"Diffbot":["partial","*",6,1],"ContextFetchBot":["partial","*",6,1]},"llms":{"status":"missing","http":404,"bytes":73539},"home":{"status":"ok","http":200,"url":"https://www.vpro.nl/","html_tokens":41990,"markdown_tokens":500,"pruned_tokens":202}},
{"rank":8370,"domain":"metopera.org","origin":"https://metopera.org","robots":{"status":"ok","http":200,"url":"https://metopera.org/robots.txt","bytes":2676,"groups":32},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["allowed","*",0,1],"OAI-SearchBot":["allowed","*",0,1],"ClaudeBot":["blocked","named",1,0],"Claude-User":["allowed","*",0,1],"Claude-SearchBot":["allowed","*",0,1],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["allowed","*",0,1],"Perplexity-User":["allowed","*",0,1],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["allowed","*",0,1]},"llms":{"status":"disallowed","error":"robots.txt of metopera.queue-it.net disallows /?c=metopera&e=metoperawaitingroom&ver=javascript-4.4.4&cver=187&man=24%2F7%20Site%20Protection&enqu"},"home":{"status":"disallowed","error":"robots.txt of metopera.queue-it.net disallows /?c=metopera&e=metoperawaitingroom&ver=javascript-4.4.4&cver=187&man=24%2F7%20Site%20Protection&enqu"}},
{"rank":8371,"domain":"broadinstitute.org","origin":"https://broadinstitute.org","robots":{"status":"ok","http":200,"url":"https://www.broadinstitute.org/robots.txt","bytes":2122,"groups":11},"bots":{"GPTBot":["partial","*",28,18],"ChatGPT-User":["partial","*",28,18],"OAI-SearchBot":["partial","*",28,18],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",28,18],"Claude-SearchBot":["partial","*",28,18],"anthropic-ai":["partial","*",28,18],"PerplexityBot":["partial","*",28,18],"Perplexity-User":["partial","*",28,18],"Google-Extended":["partial","*",28,18],"Applebot-Extended":["partial","*",28,18],"CCBot":["partial","*",28,18],"Bytespider":["partial","*",28,18],"Amazonbot":["partial","*",28,18],"meta-externalagent":["partial","*",28,18],"cohere-ai":["partial","*",28,18],"Diffbot":["partial","*",28,18],"ContextFetchBot":["partial","*",28,18]},"llms":{"status":"missing","http":404,"bytes":196},"home":{"status":"ok","http":200,"url":"https://www.broadinstitute.org/","html_tokens":54623,"markdown_tokens":792,"pruned_tokens":165}},
{"rank":8372,"domain":"tuwien.at","origin":"https://tuwien.at","robots":{"status":"ok","http":200,"url":"https://www.tuwien.at/robots.txt","bytes":285,"groups":2},"bots":{"GPTBot":["partial","*",4,1],"ChatGPT-User":["partial","*",4,1],"OAI-SearchBot":["partial","*",4,1],"ClaudeBot":["partial","*",4,1],"Claude-User":["partial","*",4,1],"Claude-SearchBot":["partial","*",4,1],"anthropic-ai":["partial","*",4,1],"PerplexityBot":["partial","*",4,1],"Perplexity-User":["partial","*",4,1],"Google-Extended":["partial","*",4,1],"Applebot-Extended":["partial","*",4,1],"CCBot":["partial","*",4,1],"Bytespider":["partial","*",4,1],"Amazonbot":["partial","*",4,1],"meta-externalagent":["partial","*",4,1],"cohere-ai":["partial","*",4,1],"Diffbot":["partial","*",4,1],"ContextFetchBot":["partial","*",4,1]},"llms":{"status":"missing","http":404,"bytes":256994},"home":{"status":"ok","http":200,"url":"https://www.tuwien.at/","html_tokens":98225,"markdown_tokens":237,"pruned_tokens":115}},
{"rank":8373,"domain":"sysinternals.com","origin":"https://sysinternals.com","robots":{"status":"missing","http":404,"url":"https://learn.microsoft.com/en-us/sysinternals/robots.txt","bytes":17593,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":17593},"home":{"status":"ok","http":200,"url":"https://learn.microsoft.com/en-us/sysinternals/","html_tokens":11086,"markdown_tokens":1283,"pruned_tokens":260}},
{"rank":8374,"domain":"zhedinglaw.com","origin":"http://zhedinglaw.com","robots":{"status":"missing","http":404,"url":"http://zhedinglaw.com/robots.txt","bytes":146,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":146},"home":{"status":"ok","http":200,"url":"http://zhedinglaw.com/","html_tokens":2411,"markdown_tokens":46,"pruned_tokens":46}},
{"rank":8375,"domain":"okdiario.com","origin":"https://okdiario.com","robots":{"status":"ok","http":200,"url":"https://okdiario.com/robots.txt","bytes":2678,"groups":28},"bots":{"GPTBot":["partial","*",29,0],"ChatGPT-User":["partial","*",29,0],"OAI-SearchBot":["partial","*",29,0],"ClaudeBot":["partial","*",29,0],"Claude-User":["partial","*",29,0],"Claude-SearchBot":["partial","*",29,0],"anthropic-ai":["partial","*",29,0],"PerplexityBot":["partial","*",29,0],"Perplexity-User":["partial","*",29,0],"Google-Extended":["partial","*",29,0],"Applebot-Extended":["partial","*",29,0],"CCBot":["partial","*",29,0],"Bytespider":["partial","*",29,0],"Amazonbot":["partial","*",29,0],"meta-externalagent":["partial","*",29,0],"cohere-ai":["partial","*",29,0],"Diffbot":["partial","*",29,0],"ContextFetchBot":["partial","*",29,0]},"llms":{"status":"missing","http":404,"bytes":672078},"home":{"status":"ok","http":200,"url":"https://okdiario.com/","html_tokens":479944,"markdown_tokens":1586,"pruned_tokens":102}},
{"rank":8376,"domain":"nursingworld.org","origin":"https://nursingworld.org","robots":{"status":"ok","http":200,"url":"https://www.nursingworld.org/robots.txt","bytes":455,"groups":1},"bots":{"GPTBot":["partial","*",14,0],"ChatGPT-User":["partial","*",14,0],"OAI-SearchBot":["partial","*",14,0],"ClaudeBot":["partial","*",14,0],"Claude-User":["partial","*",14,0],"Claude-SearchBot":["partial","*",14,0],"anthropic-ai":["partial","*",14,0],"PerplexityBot":["partial","*",14,0],"Perplexity-User":["partial","*",14,0],"Google-Extended":["partial","*",14,0],"Applebot-Extended":["partial","*",14,0],"CCBot":["partial","*",14,0],"Bytespider":["partial","*",14,0],"Amazonbot":["partial","*",14,0],"meta-externalagent":["partial","*",14,0],"cohere-ai":["partial","*",14,0],"Diffbot":["partial","*",14,0],"ContextFetchBot":["partial","*",14,0]},"llms":{"status":"missing","http":404,"bytes":1333},"home":{"status":"ok","http":200,"url":"https://www.nursingworld.org/","html_tokens":38267,"markdown_tokens":104,"pruned_tokens":104}},
{"rank":8377,"domain":"ren.tv","origin":"https://ren.tv","robots":{"status":"refused","http":403,"url":"https://ren.tv/robots.txt","bytes":902,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":8378,"domain":"vdrsystems.net","origin":"https://vdrsystems.net","robots":{"status":"ok","http":200,"url":"https://vdrsystems.net/robots.txt","bytes":339,"groups":1},"bots":{"GPTBot":["partial","*",11,6],"ChatGPT-User":["partial","*",11,6],"OAI-SearchBot":["partial","*",11,6],"ClaudeBot":["partial","*",11,6],"Claude-User":["partial","*",11,6],"Claude-SearchBot":["partial","*",11,6],"anthropic-ai":["partial","*",11,6],"PerplexityBot":["partial","*",11,6],"Perplexity-User":["partial","*",11,6],"Google-Extended":["partial","*",11,6],"Applebot-Extended":["partial","*",11,6],"CCBot":["partial","*",11,6],"Bytespider":["partial","*",11,6],"Amazonbot":["partial","*",11,6],"meta-externalagent":["partial","*",11,6],"cohere-ai":["partial","*",11,6],"Diffbot":["partial","*",11,6],"ContextFetchBot":["partial","*",11,6]},"llms":{"status":"missing","http":404,"bytes":0},"home":{"status":"ok","http":200,"url":"https://vdrsystems.net/","html_tokens":4256,"markdown_tokens":0,"pruned_tokens":0}},
{"rank":8379,"domain":"antiwar.com","origin":"https://antiwar.com","robots":{"status":"ok","http":200,"url":"https://antiwar.com/robots.txt","bytes":683,"groups":2},"bots":{"GPTBot":["partial","*",27,0],"ChatGPT-User":["partial","*",27,0],"OAI-SearchBot":["partial","*",27,0],"ClaudeBot":["partial","*",27,0],"Claude-User":["partial","*",27,0],"Claude-SearchBot":["partial","*",27,0],"anthropic-ai":["partial","*",27,0],"PerplexityBot":["partial","*",27,0],"Perplexity-User":["partial","*",27,0],"Google-Extended":["partial","*",27,0],"Applebot-Extended":["partial","*",27,0],"CCBot":["partial","*",27,0],"Bytespider":["partial","*",27,0],"Amazonbot":["partial","*",27,0],"meta-externalagent":["partial","*",27,0],"cohere-ai":["partial","*",27,0],"Diffbot":["partial","*",27,0],"ContextFetchBot":["partial","*",27,0]},"llms":{"status":"missing","http":404,"bytes":196},"home":{"status":"ok","http":200,"url":"https://antiwar.com/","html_tokens":26645,"markdown_tokens":25,"pruned_tokens":25}},
{"rank":8380,"domain":"gcjszfl.com","origin":"http://gcjszfl.com","robots":{"status":"missing","http":404,"url":"http://gcjszfl.com/robots.txt","bytes":5062,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":5058},"home":{"status":"ok","http":200,"url":"http://gcjszfl.com/","html_tokens":6450,"markdown_tokens":68,"pruned_tokens":68}},
{"rank":8381,"domain":"phoenixnewtimes.com","origin":"https://phoenixnewtimes.com","robots":{"status":"ok","http":200,"url":"https://www.phoenixnewtimes.com/robots.txt","bytes":1420,"groups":2},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["blocked","named",1,0],"OAI-SearchBot":["blocked","named",1,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["blocked","named",1,0],"Claude-SearchBot":["blocked","named",1,0],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["blocked","named",1,0],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",1,1]},"llms":{"status":"missing","http":404,"bytes":117045},"home":{"status":"ok","http":200,"url":"https://www.phoenixnewtimes.com/","html_tokens":71649,"markdown_tokens":43,"pruned_tokens":43}},
{"rank":8382,"domain":"native-instruments.com","origin":"https://native-instruments.com","robots":{"status":"ok","http":200,"url":"https://www.native-instruments.com/robots.txt","bytes":6412,"groups":7},"bots":{"GPTBot":["partial","*",46,0],"ChatGPT-User":["partial","*",46,0],"OAI-SearchBot":["partial","*",46,0],"ClaudeBot":["partial","*",46,0],"Claude-User":["partial","*",46,0],"Claude-SearchBot":["partial","*",46,0],"anthropic-ai":["partial","*",46,0],"PerplexityBot":["partial","*",46,0],"Perplexity-User":["partial","*",46,0],"Google-Extended":["partial","*",46,0],"Applebot-Extended":["partial","*",46,0],"CCBot":["partial","*",46,0],"Bytespider":["partial","*",46,0],"Amazonbot":["partial","*",46,0],"meta-externalagent":["partial","*",46,0],"cohere-ai":["partial","*",46,0],"Diffbot":["partial","*",46,0],"ContextFetchBot":["partial","*",46,0]},"llms":{"status":"present","http":200,"bytes":5800},"home":{"status":"ok","http":200,"url":"https://www.native-instruments.com/","html_tokens":245571,"markdown_tokens":8,"pruned_tokens":8}},
{"rank":8383,"domain":"bmel.de","origin":"https://bmel.de","robots":{"status":"ok","http":200,"url":"https://www.bmleh.de/robots.txt","bytes":241,"groups":1},"bots":{"GPTBot":["partial","*",3,3],"ChatGPT-User":["partial","*",3,3],"OAI-SearchBot":["partial","*",3,3],"ClaudeBot":["partial","*",3,3],"Claude-User":["partial","*",3,3],"Claude-SearchBot":["partial","*",3,3],"anthropic-ai":["partial","*",3,3],"PerplexityBot":["partial","*",3,3],"Perplexity-User":["partial","*",3,3],"Google-Extended":["partial","*",3,3],"Applebot-Extended":["partial","*",3,3],"CCBot":["partial","*",3,3],"Bytespider":["partial","*",3,3],"Amazonbot":["partial","*",3,3],"meta-externalagent":["partial","*",3,3],"cohere-ai":["partial","*",3,3],"Diffbot":["partial","*",3,3],"ContextFetchBot":["partial","*",3,3]},"llms":{"status":"missing","http":404,"bytes":749042},"home":{"status":"timeout","error":"timed out after 15 s"}},
{"rank":8384,"domain":"sysu.edu.cn","origin":"https://sysu.edu.cn","robots":{"status":"ok","http":200,"url":"https://www.sysu.edu.cn","bytes":107937,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"html","http":200,"bytes":107937},"home":{"status":"ok","http":200,"url":"https://www.sysu.edu.cn","html_tokens":25328,"markdown_tokens":54,"pruned_tokens":54}},
{"rank":8385,"domain":"sebi.gov.in","origin":"https://sebi.gov.in","robots":{"status":"html","http":200,"url":"https://www.sebi.gov.in/","bytes":54906,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":8386,"domain":"americamagazine.org","origin":"https://americamagazine.org","robots":{"status":"ok","http":200,"url":"https://www.americamagazine.org/robots.txt","bytes":292,"groups":1},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"missing","http":404,"bytes":261196},"home":{"status":"ok","http":200,"url":"https://www.americamagazine.org/","html_tokens":113459,"markdown_tokens":203,"pruned_tokens":113}},
{"rank":8387,"domain":"nate.com","origin":"https://nate.com","robots":{"status":"ok","http":200,"url":"https://nate.com/robots.txt","bytes":103,"groups":1},"bots":{"GPTBot":["partial","*",1,4],"ChatGPT-User":["partial","*",1,4],"OAI-SearchBot":["partial","*",1,4],"ClaudeBot":["partial","*",1,4],"Claude-User":["partial","*",1,4],"Claude-SearchBot":["partial","*",1,4],"anthropic-ai":["partial","*",1,4],"PerplexityBot":["partial","*",1,4],"Perplexity-User":["partial","*",1,4],"Google-Extended":["partial","*",1,4],"Applebot-Extended":["partial","*",1,4],"CCBot":["partial","*",1,4],"Bytespider":["partial","*",1,4],"Amazonbot":["partial","*",1,4],"meta-externalagent":["partial","*",1,4],"cohere-ai":["partial","*",1,4],"Diffbot":["partial","*",1,4],"ContextFetchBot":["partial","*",1,4]},"llms":{"status":"disallowed"},"home":{"status":"error","http":200,"url":"https://nate.com/","error":"unparseable HTML"}},
{"rank":8388,"domain":"snapcraft.io","origin":"https://snapcraft.io","robots":{"status":"ok","http":200,"url":"https://snapcraft.io/robots.txt","bytes":2355,"groups":1},"bots":{"GPTBot":["partial","*",37,3],"ChatGPT-User":["partial","*",37,3],"OAI-SearchBot":["partial","*",37,3],"ClaudeBot":["partial","*",37,3],"Claude-User":["partial","*",37,3],"Claude-SearchBot":["partial","*",37,3],"anthropic-ai":["partial","*",37,3],"PerplexityBot":["partial","*",37,3],"Perplexity-User":["partial","*",37,3],"Google-Extended":["partial","*",37,3],"Applebot-Extended":["partial","*",37,3],"CCBot":["partial","*",37,3],"Bytespider":["partial","*",37,3],"Amazonbot":["partial","*",37,3],"meta-externalagent":["partial","*",37,3],"cohere-ai":["partial","*",37,3],"Diffbot":["partial","*",37,3],"ContextFetchBot":["partial","*",37,3]},"llms":{"status":"present","http":200,"bytes":3144},"home":{"status":"ok","http":200,"url":"https://snapcraft.io/","html_tokens":21690,"markdown_tokens":413,"pruned_tokens":120}},
{"rank":8389,"domain":"plus.com","origin":"https://plus.com","robots":{"status":"server_error","http":500,"url":"https://plus.com/robots.txt","bytes":37759,"groups":0,"error":"HTTP 500"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":8390,"domain":"rsa.com","origin":"https://rsa.com","robots":{"status":"refused","http":403,"url":"https://rsa.com/robots.txt","bytes":560,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":8391,"domain":"nodak.edu","origin":"https://nodak.edu","robots":{"status":"unreachable","http":null,"url":"https://nodak.edu/robots.txt","bytes":0,"groups":0,"error":"cannot resolve nodak.edu"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":8392,"domain":"hbgjp.cn","origin":"https://hbgjp.cn","robots":{"status":"missing","http":404,"url":"https://hbgjp.cn/robots.txt","bytes":0,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":0},"home":{"status":"ok","http":200,"url":"https://hbgjp.cn/","html_tokens":18529,"markdown_tokens":19,"pruned_tokens":19}},
{"rank":8393,"domain":"kinokuniya.co.jp","origin":"https://www.kinokuniya.co.jp","robots":{"status":"ok","http":200,"url":"https://www.kinokuniya.co.jp/robots.txt","bytes":1033,"groups":6},"bots":{"GPTBot":["partial","*",9,0],"ChatGPT-User":["partial","*",9,0],"OAI-SearchBot":["partial","*",9,0],"ClaudeBot":["partial","*",9,0],"Claude-User":["partial","*",9,0],"Claude-SearchBot":["partial","*",9,0],"anthropic-ai":["partial","*",9,0],"PerplexityBot":["partial","*",9,0],"Perplexity-User":["partial","*",9,0],"Google-Extended":["partial","*",9,0],"Applebot-Extended":["partial","*",9,0],"CCBot":["partial","*",9,0],"Bytespider":["partial","*",9,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["partial","*",9,0],"cohere-ai":["partial","*",9,0],"Diffbot":["partial","*",9,0],"ContextFetchBot":["partial","*",9,0]},"llms":{"status":"missing","http":404,"bytes":6123},"home":{"status":"ok","http":200,"url":"https://www.kinokuniya.co.jp/","html_tokens":23881,"markdown_tokens":506,"pruned_tokens":141}},
{"rank":8394,"domain":"pretix.eu","origin":"https://pretix.eu","robots":{"status":"ok","http":200,"url":"https://pretix.eu/robots.txt","bytes":178,"groups":1},"bots":{"GPTBot":["partial","*",8,0],"ChatGPT-User":["partial","*",8,0],"OAI-SearchBot":["partial","*",8,0],"ClaudeBot":["partial","*",8,0],"Claude-User":["partial","*",8,0],"Claude-SearchBot":["partial","*",8,0],"anthropic-ai":["partial","*",8,0],"PerplexityBot":["partial","*",8,0],"Perplexity-User":["partial","*",8,0],"Google-Extended":["partial","*",8,0],"Applebot-Extended":["partial","*",8,0],"CCBot":["partial","*",8,0],"Bytespider":["partial","*",8,0],"Amazonbot":["partial","*",8,0],"meta-externalagent":["partial","*",8,0],"cohere-ai":["partial","*",8,0],"Diffbot":["partial","*",8,0],"ContextFetchBot":["partial","*",8,0]},"llms":{"status":"missing","http":404,"bytes":955},"home":{"status":"ok","http":200,"url":"https://pretix.eu/about/en/","html_tokens":9470,"markdown_tokens":16,"pruned_tokens":16}},
{"rank":8395,"domain":"dslreports.com","origin":"https://dslreports.com","robots":{"status":"server_error","http":503,"url":"https://dslreports.com/robots.txt","bytes":862,"groups":0,"error":"HTTP 503"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":8396,"domain":"uriweb.kr","origin":"https://uriweb.kr","robots":{"status":"ok","http":200,"url":"https://uriweb.kr/robots.txt","bytes":205,"groups":1},"bots":{"GPTBot":["partial","*",7,1],"ChatGPT-User":["partial","*",7,1],"OAI-SearchBot":["partial","*",7,1],"ClaudeBot":["partial","*",7,1],"Claude-User":["partial","*",7,1],"Claude-SearchBot":["partial","*",7,1],"anthropic-ai":["partial","*",7,1],"PerplexityBot":["partial","*",7,1],"Perplexity-User":["partial","*",7,1],"Google-Extended":["partial","*",7,1],"Applebot-Extended":["partial","*",7,1],"CCBot":["partial","*",7,1],"Bytespider":["partial","*",7,1],"Amazonbot":["partial","*",7,1],"meta-externalagent":["partial","*",7,1],"cohere-ai":["partial","*",7,1],"Diffbot":["partial","*",7,1],"ContextFetchBot":["partial","*",7,1]},"llms":{"status":"other_text","http":200,"bytes":74},"home":{"status":"ok","http":200,"url":"https://uriweb.kr/","html_tokens":457728,"markdown_tokens":117,"pruned_tokens":117}},
{"rank":8397,"domain":"elsevier.es","origin":"https://elsevier.es","robots":{"status":"ok","http":200,"url":"https://www.elsevier.es/robots.txt","bytes":369,"groups":9},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["blocked","named",1,0],"OAI-SearchBot":["blocked","named",1,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",2,0],"Claude-SearchBot":["partial","*",2,0],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["partial","*",2,0],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["partial","*",2,0],"CCBot":["partial","*",2,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["partial","*",2,0],"meta-externalagent":["partial","*",2,0],"cohere-ai":["partial","*",2,0],"Diffbot":["partial","*",2,0],"ContextFetchBot":["partial","*",2,0]},"llms":{"status":"missing","http":404,"bytes":22357},"home":{"status":"ok","http":200,"url":"https://www.elsevier.com/es-es","html_tokens":94718,"markdown_tokens":1287,"pruned_tokens":120}},
{"rank":8398,"domain":"google.mg","origin":"https://google.mg","robots":{"status":"ok","http":200,"url":"https://www.google.mg/robots.txt","bytes":6624,"groups":4},"bots":{"GPTBot":["partial","*",169,64],"ChatGPT-User":["partial","*",169,64],"OAI-SearchBot":["partial","*",169,64],"ClaudeBot":["partial","*",169,64],"Claude-User":["partial","*",169,64],"Claude-SearchBot":["partial","*",169,64],"anthropic-ai":["partial","*",169,64],"PerplexityBot":["partial","*",169,64],"Perplexity-User":["partial","*",169,64],"Google-Extended":["partial","*",169,64],"Applebot-Extended":["partial","*",169,64],"CCBot":["partial","*",169,64],"Bytespider":["partial","*",169,64],"Amazonbot":["partial","*",169,64],"meta-externalagent":["partial","*",169,64],"cohere-ai":["partial","*",169,64],"Diffbot":["partial","*",169,64],"ContextFetchBot":["partial","*",169,64]},"llms":{"status":"missing","http":404,"bytes":1569},"home":{"status":"ok","http":200,"url":"https://www.google.com/","html_tokens":21339,"markdown_tokens":30,"pruned_tokens":30}},
{"rank":8399,"domain":"google.com.sl","origin":"https://google.com.sl","robots":{"status":"ok","http":200,"url":"https://www.google.com.sl/robots.txt","bytes":6624,"groups":4},"bots":{"GPTBot":["partial","*",169,64],"ChatGPT-User":["partial","*",169,64],"OAI-SearchBot":["partial","*",169,64],"ClaudeBot":["partial","*",169,64],"Claude-User":["partial","*",169,64],"Claude-SearchBot":["partial","*",169,64],"anthropic-ai":["partial","*",169,64],"PerplexityBot":["partial","*",169,64],"Perplexity-User":["partial","*",169,64],"Google-Extended":["partial","*",169,64],"Applebot-Extended":["partial","*",169,64],"CCBot":["partial","*",169,64],"Bytespider":["partial","*",169,64],"Amazonbot":["partial","*",169,64],"meta-externalagent":["partial","*",169,64],"cohere-ai":["partial","*",169,64],"Diffbot":["partial","*",169,64],"ContextFetchBot":["partial","*",169,64]},"llms":{"status":"missing","http":404,"bytes":1569},"home":{"status":"ok","http":200,"url":"https://www.google.com/","html_tokens":21377,"markdown_tokens":30,"pruned_tokens":30}},
{"rank":8400,"domain":"babelio.com","origin":"https://babelio.com","robots":{"status":"timeout","http":null,"url":"https://babelio.com/robots.txt","bytes":0,"groups":0,"error":"timed out after 10 s"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":8401,"domain":"rainforest-alliance.org","origin":"https://rainforest-alliance.org","robots":{"status":"ok","http":200,"url":"https://www.rainforest-alliance.org/robots.txt","bytes":1370,"groups":1},"bots":{"GPTBot":["partial","*",6,2],"ChatGPT-User":["partial","*",6,2],"OAI-SearchBot":["partial","*",6,2],"ClaudeBot":["partial","*",6,2],"Claude-User":["partial","*",6,2],"Claude-SearchBot":["partial","*",6,2],"anthropic-ai":["partial","*",6,2],"PerplexityBot":["partial","*",6,2],"Perplexity-User":["partial","*",6,2],"Google-Extended":["partial","*",6,2],"Applebot-Extended":["partial","*",6,2],"CCBot":["partial","*",6,2],"Bytespider":["partial","*",6,2],"Amazonbot":["partial","*",6,2],"meta-externalagent":["partial","*",6,2],"cohere-ai":["partial","*",6,2],"Diffbot":["partial","*",6,2],"ContextFetchBot":["partial","*",6,2]},"llms":{"status":"other_text","http":200,"bytes":15998},"home":{"status":"ok","http":200,"url":"https://www.rainforest-alliance.org/","html_tokens":162347,"markdown_tokens":401,"pruned_tokens":33}},
{"rank":8402,"domain":"perforce.com","origin":"https://perforce.com","robots":{"status":"ok","http":200,"url":"https://www.perforce.com/robots.txt","bytes":6273,"groups":2},"bots":{"GPTBot":["partial","*",138,22],"ChatGPT-User":["partial","*",138,22],"OAI-SearchBot":["partial","*",138,22],"ClaudeBot":["partial","*",138,22],"Claude-User":["partial","*",138,22],"Claude-SearchBot":["partial","*",138,22],"anthropic-ai":["partial","*",138,22],"PerplexityBot":["partial","*",138,22],"Perplexity-User":["partial","*",138,22],"Google-Extended":["partial","*",138,22],"Applebot-Extended":["partial","*",138,22],"CCBot":["partial","*",138,22],"Bytespider":["partial","*",138,22],"Amazonbot":["partial","*",138,22],"meta-externalagent":["partial","*",138,22],"cohere-ai":["partial","*",138,22],"Diffbot":["partial","*",138,22],"ContextFetchBot":["partial","*",138,22]},"llms":{"status":"missing","http":404,"bytes":551},"home":{"status":"ok","http":200,"url":"https://www.perforce.com/","html_tokens":59107,"markdown_tokens":791,"pruned_tokens":663}},
{"rank":8403,"domain":"blogher.com","origin":"https://blogher.com","robots":{"status":"ok","http":200,"url":"https://www.shemedia.com/robots.txt","bytes":1507,"groups":1},"bots":{"GPTBot":["partial","named",27,1],"ChatGPT-User":["partial","*",27,1],"OAI-SearchBot":["partial","*",27,1],"ClaudeBot":["partial","named",27,1],"Claude-User":["partial","*",27,1],"Claude-SearchBot":["partial","*",27,1],"anthropic-ai":["partial","named",27,1],"PerplexityBot":["partial","*",27,1],"Perplexity-User":["partial","*",27,1],"Google-Extended":["partial","named",27,1],"Applebot-Extended":["partial","named",27,1],"CCBot":["partial","named",27,1],"Bytespider":["partial","named",27,1],"Amazonbot":["partial","named",27,1],"meta-externalagent":["partial","named",27,1],"cohere-ai":["partial","named",27,1],"Diffbot":["partial","*",27,1],"ContextFetchBot":["partial","*",27,1]},"llms":{"status":"missing","http":404,"bytes":0},"home":{"status":"ok","http":200,"url":"https://www.shemedia.com/","html_tokens":444977,"markdown_tokens":501,"pruned_tokens":140}},
{"rank":8404,"domain":"jskejing.com","origin":"http://jskejing.com","robots":{"status":"ok","http":200,"url":"http://jskejing.com/robots.txt","bytes":485,"groups":1},"bots":{"GPTBot":["partial","*",21,0],"ChatGPT-User":["partial","*",21,0],"OAI-SearchBot":["partial","*",21,0],"ClaudeBot":["partial","*",21,0],"Claude-User":["partial","*",21,0],"Claude-SearchBot":["partial","*",21,0],"anthropic-ai":["partial","*",21,0],"PerplexityBot":["partial","*",21,0],"Perplexity-User":["partial","*",21,0],"Google-Extended":["partial","*",21,0],"Applebot-Extended":["partial","*",21,0],"CCBot":["partial","*",21,0],"Bytespider":["partial","*",21,0],"Amazonbot":["partial","*",21,0],"meta-externalagent":["partial","*",21,0],"cohere-ai":["partial","*",21,0],"Diffbot":["partial","*",21,0],"ContextFetchBot":["partial","*",21,0]},"llms":{"status":"missing","http":404,"bytes":4728},"home":{"status":"ok","http":200,"url":"http://jskejing.com/","html_tokens":19747,"markdown_tokens":100,"pruned_tokens":27}},
{"rank":8405,"domain":"nh.us","origin":"https://nh.us","robots":{"status":"unreachable","http":null,"url":"https://nh.us/robots.txt","bytes":0,"groups":0,"error":"cannot resolve nh.us"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":8406,"domain":"scripps.edu","origin":"https://scripps.edu","robots":{"status":"ok","http":200,"url":"https://www.scripps.edu/robots.txt","bytes":189,"groups":1},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"http_error","http":403,"bytes":5542},"home":{"status":"http_error","http":403,"url":"https://scripps.edu/"}},
{"rank":8407,"domain":"onlinehome.us","origin":"http://onlinehome.us","robots":{"status":"missing","http":404,"url":"http://onlinehome.us/robots.txt","bytes":601,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":601},"home":{"status":"http_error","http":403,"url":"http://onlinehome.us/"}},
{"rank":8408,"domain":"lawinsider.com","origin":"https://lawinsider.com","robots":{"status":"ok","http":200,"url":"https://www.lawinsider.com/robots.txt","bytes":23164,"groups":1},"bots":{"GPTBot":["partial","*",767,0],"ChatGPT-User":["partial","*",767,0],"OAI-SearchBot":["partial","*",767,0],"ClaudeBot":["partial","*",767,0],"Claude-User":["partial","*",767,0],"Claude-SearchBot":["partial","*",767,0],"anthropic-ai":["partial","*",767,0],"PerplexityBot":["partial","*",767,0],"Perplexity-User":["partial","*",767,0],"Google-Extended":["partial","*",767,0],"Applebot-Extended":["partial","*",767,0],"CCBot":["partial","*",767,0],"Bytespider":["partial","*",767,0],"Amazonbot":["partial","*",767,0],"meta-externalagent":["partial","*",767,0],"cohere-ai":["partial","*",767,0],"Diffbot":["partial","*",767,0],"ContextFetchBot":["partial","*",767,0]},"llms":{"status":"missing","http":404,"bytes":12763},"home":{"status":"ok","http":200,"url":"https://www.lawinsider.com/","html_tokens":11062,"markdown_tokens":1279,"pruned_tokens":115}},
{"rank":8409,"domain":"hollywoodlife.com","origin":"https://hollywoodlife.com","robots":{"status":"ok","http":200,"url":"https://hollywoodlife.com/robots.txt","bytes":3436,"groups":3},"bots":{"GPTBot":["partial","*",61,2],"ChatGPT-User":["partial","*",61,2],"OAI-SearchBot":["partial","*",61,2],"ClaudeBot":["partial","*",61,2],"Claude-User":["partial","*",61,2],"Claude-SearchBot":["partial","*",61,2],"anthropic-ai":["partial","*",61,2],"PerplexityBot":["partial","*",61,2],"Perplexity-User":["partial","*",61,2],"Google-Extended":["partial","*",61,2],"Applebot-Extended":["partial","*",61,2],"CCBot":["partial","*",61,2],"Bytespider":["partial","*",61,2],"Amazonbot":["partial","*",61,2],"meta-externalagent":["partial","*",61,2],"cohere-ai":["partial","*",61,2],"Diffbot":["partial","*",61,2],"ContextFetchBot":["partial","*",61,2]},"llms":{"status":"missing","http":404,"bytes":171960},"home":{"status":"http_error","http":403,"url":"https://hollywoodlife.com/"}},
{"rank":8410,"domain":"bostonmagazine.com","origin":"https://bostonmagazine.com","robots":{"status":"ok","http":200,"url":"https://www.bostonmagazine.com/robots.txt","bytes":1245,"groups":26},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["partial","*",2,1],"OAI-SearchBot":["partial","*",2,1],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",2,1],"Claude-SearchBot":["partial","*",2,1],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["partial","*",2,1],"Perplexity-User":["partial","*",2,1],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",2,1]},"llms":{"status":"missing","http":404,"bytes":150254},"home":{"status":"ok","http":200,"url":"https://www.bostonmagazine.com/","html_tokens":54451,"markdown_tokens":174,"pruned_tokens":174}},
{"rank":8411,"domain":"byboai.com","origin":"http://byboai.com","robots":{"status":"missing","http":404,"url":"http://byboai.com/robots.txt","bytes":1163,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":1163},"home":{"status":"ok","http":200,"url":"http://byboai.com/","html_tokens":9281,"markdown_tokens":56,"pruned_tokens":56}},
{"rank":8412,"domain":"cnif.cn","origin":"https://cnif.cn","robots":{"status":"unreachable","http":null,"url":"https://cnif.cn/robots.txt","bytes":0,"groups":0,"error":"[SSL: CERTIFICATE_VERIFY_FAILED] certificate verify failed: Hostname mismatch, certificate is not valid for 'cnif.cn'. (_ssl.c:1028)"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":8413,"domain":"abbvie.com","origin":"https://abbvie.com","robots":{"status":"ok","http":200,"url":"https://www.abbvie.com/robots.txt","bytes":1541,"groups":2},"bots":{"GPTBot":["partial","*",9,1],"ChatGPT-User":["partial","*",9,1],"OAI-SearchBot":["partial","*",9,1],"ClaudeBot":["partial","*",9,1],"Claude-User":["partial","*",9,1],"Claude-SearchBot":["partial","*",9,1],"anthropic-ai":["partial","*",9,1],"PerplexityBot":["partial","*",9,1],"Perplexity-User":["partial","*",9,1],"Google-Extended":["partial","*",9,1],"Applebot-Extended":["partial","*",9,1],"CCBot":["partial","*",9,1],"Bytespider":["partial","*",9,1],"Amazonbot":["partial","*",9,1],"meta-externalagent":["partial","*",9,1],"cohere-ai":["partial","*",9,1],"Diffbot":["partial","*",9,1],"ContextFetchBot":["partial","*",9,1]},"llms":{"status":"http_error","http":403,"bytes":4904},"home":{"status":"http_error","http":403,"url":"https://www.abbvie.com/"}},
{"rank":8414,"domain":"bde.es","origin":"https://bde.es","robots":{"status":"ok","http":200,"url":"https://www.bde.es/robots.txt","bytes":514,"groups":1},"bots":{"GPTBot":["partial","*",10,0],"ChatGPT-User":["partial","*",10,0],"OAI-SearchBot":["partial","*",10,0],"ClaudeBot":["partial","*",10,0],"Claude-User":["partial","*",10,0],"Claude-SearchBot":["partial","*",10,0],"anthropic-ai":["partial","*",10,0],"PerplexityBot":["partial","*",10,0],"Perplexity-User":["partial","*",10,0],"Google-Extended":["partial","*",10,0],"Applebot-Extended":["partial","*",10,0],"CCBot":["partial","*",10,0],"Bytespider":["partial","*",10,0],"Amazonbot":["partial","*",10,0],"meta-externalagent":["partial","*",10,0],"cohere-ai":["partial","*",10,0],"Diffbot":["partial","*",10,0],"ContextFetchBot":["partial","*",10,0]},"llms":{"status":"disallowed","error":"robots.txt of app.bde.es not readable (unreachable)"},"home":{"status":"ok","http":200,"url":"https://www.bde.es/wbe/es/","html_tokens":178892,"markdown_tokens":125,"pruned_tokens":125}},
{"rank":8415,"domain":"herts.ac.uk","origin":"https://herts.ac.uk","robots":{"status":"refused","http":403,"url":"https://herts.ac.uk/robots.txt","bytes":5506,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":8416,"domain":"usensejn.com","origin":"https://usensejn.com","robots":{"status":"unreachable","http":null,"url":"https://usensejn.com/robots.txt","bytes":0,"groups":0,"error":"cannot resolve usensejn.com"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":8417,"domain":"datatilsynet.dk","origin":"https://datatilsynet.dk","robots":{"status":"ok","http":200,"url":"https://datatilsynet.dk/robots.txt","bytes":69,"groups":1},"bots":{"GPTBot":["allowed","*",0,1],"ChatGPT-User":["allowed","*",0,1],"OAI-SearchBot":["allowed","*",0,1],"ClaudeBot":["allowed","*",0,1],"Claude-User":["allowed","*",0,1],"Claude-SearchBot":["allowed","*",0,1],"anthropic-ai":["allowed","*",0,1],"PerplexityBot":["allowed","*",0,1],"Perplexity-User":["allowed","*",0,1],"Google-Extended":["allowed","*",0,1],"Applebot-Extended":["allowed","*",0,1],"CCBot":["allowed","*",0,1],"Bytespider":["allowed","*",0,1],"Amazonbot":["allowed","*",0,1],"meta-externalagent":["allowed","*",0,1],"cohere-ai":["allowed","*",0,1],"Diffbot":["allowed","*",0,1],"ContextFetchBot":["allowed","*",0,1]},"llms":{"status":"missing","http":404,"bytes":27230},"home":{"status":"ok","http":200,"url":"https://www.datatilsynet.dk/","html_tokens":21463,"markdown_tokens":62,"pruned_tokens":62}},
{"rank":8418,"domain":"canalplus.com","origin":"https://canalplus.com","robots":{"status":"refused","http":403,"url":"https://www.canalplus.com/robots.txt","bytes":113,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":8419,"domain":"aacap.org","origin":"https://aacap.org","robots":{"status":"ok","http":200,"url":"https://aacap.org/robots.txt","bytes":775,"groups":12},"bots":{"GPTBot":["partial","*",14,0],"ChatGPT-User":["partial","*",14,0],"OAI-SearchBot":["partial","*",14,0],"ClaudeBot":["partial","*",14,0],"Claude-User":["partial","*",14,0],"Claude-SearchBot":["partial","*",14,0],"anthropic-ai":["partial","*",14,0],"PerplexityBot":["partial","*",14,0],"Perplexity-User":["partial","*",14,0],"Google-Extended":["partial","*",14,0],"Applebot-Extended":["partial","*",14,0],"CCBot":["partial","*",14,0],"Bytespider":["partial","*",14,0],"Amazonbot":["partial","*",14,0],"meta-externalagent":["partial","*",14,0],"cohere-ai":["partial","*",14,0],"Diffbot":["partial","*",14,0],"ContextFetchBot":["partial","*",14,0]},"llms":{"status":"missing","http":404,"bytes":49566},"home":{"status":"ok","http":200,"url":"https://aacap.org/","html_tokens":23138,"markdown_tokens":110,"pruned_tokens":110}},
{"rank":8420,"domain":"liveabout.com","origin":"https://liveabout.com","robots":{"status":"ok","http":200,"url":"https://www.liveabout.com/robots.txt","bytes":3471,"groups":6},"bots":{"GPTBot":["partial","named",1,0],"ChatGPT-User":["partial","named",1,0],"OAI-SearchBot":["partial","named",1,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["blocked","named",1,0],"Claude-SearchBot":["partial","named",1,2],"anthropic-ai":["partial","named",1,2],"PerplexityBot":["partial","named",1,2],"Perplexity-User":["blocked","named",1,0],"Google-Extended":["partial","named",1,2],"Applebot-Extended":["blocked","named",1,0],"CCBot":["partial","named",1,2],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["partial","named",1,2],"cohere-ai":["partial","named",1,2],"Diffbot":["partial","*",2,0],"ContextFetchBot":["partial","*",2,0]},"llms":{"status":"http_error","http":402,"bytes":612},"home":{"status":"http_error","http":402,"url":"https://www.liveabout.com/"}},
{"rank":8421,"domain":"google.kg","origin":"https://google.kg","robots":{"status":"ok","http":200,"url":"https://www.google.kg/robots.txt","bytes":6624,"groups":4},"bots":{"GPTBot":["partial","*",169,64],"ChatGPT-User":["partial","*",169,64],"OAI-SearchBot":["partial","*",169,64],"ClaudeBot":["partial","*",169,64],"Claude-User":["partial","*",169,64],"Claude-SearchBot":["partial","*",169,64],"anthropic-ai":["partial","*",169,64],"PerplexityBot":["partial","*",169,64],"Perplexity-User":["partial","*",169,64],"Google-Extended":["partial","*",169,64],"Applebot-Extended":["partial","*",169,64],"CCBot":["partial","*",169,64],"Bytespider":["partial","*",169,64],"Amazonbot":["partial","*",169,64],"meta-externalagent":["partial","*",169,64],"cohere-ai":["partial","*",169,64],"Diffbot":["partial","*",169,64],"ContextFetchBot":["partial","*",169,64]},"llms":{"status":"missing","http":404,"bytes":1569},"home":{"status":"ok","http":200,"url":"https://www.google.com/","html_tokens":21446,"markdown_tokens":30,"pruned_tokens":30}},
{"rank":8422,"domain":"notebooklm.google","origin":"https://notebooklm.google","robots":{"status":"ok","http":200,"url":"https://notebooklm.google/robots.txt","bytes":71,"groups":1},"bots":{"GPTBot":["allowed","*",0,1],"ChatGPT-User":["allowed","*",0,1],"OAI-SearchBot":["allowed","*",0,1],"ClaudeBot":["allowed","*",0,1],"Claude-User":["allowed","*",0,1],"Claude-SearchBot":["allowed","*",0,1],"anthropic-ai":["allowed","*",0,1],"PerplexityBot":["allowed","*",0,1],"Perplexity-User":["allowed","*",0,1],"Google-Extended":["allowed","*",0,1],"Applebot-Extended":["allowed","*",0,1],"CCBot":["allowed","*",0,1],"Bytespider":["allowed","*",0,1],"Amazonbot":["allowed","*",0,1],"meta-externalagent":["allowed","*",0,1],"cohere-ai":["allowed","*",0,1],"Diffbot":["allowed","*",0,1],"ContextFetchBot":["allowed","*",0,1]},"llms":{"status":"html","http":200,"bytes":117991},"home":{"status":"ok","http":200,"url":"https://notebook.google/","html_tokens":29498,"markdown_tokens":0,"pruned_tokens":0}},
{"rank":8423,"domain":"volkswagen.de","origin":"https://volkswagen.de","robots":{"status":"ok","http":200,"url":"https://www.volkswagen.de/robots.txt","bytes":1650,"groups":8},"bots":{"GPTBot":["partial","*",20,1],"ChatGPT-User":["partial","*",20,1],"OAI-SearchBot":["partial","*",20,1],"ClaudeBot":["partial","*",20,1],"Claude-User":["partial","*",20,1],"Claude-SearchBot":["partial","*",20,1],"anthropic-ai":["partial","*",20,1],"PerplexityBot":["partial","*",20,1],"Perplexity-User":["partial","*",20,1],"Google-Extended":["partial","*",20,1],"Applebot-Extended":["partial","*",20,1],"CCBot":["partial","*",20,1],"Bytespider":["partial","*",20,1],"Amazonbot":["partial","*",20,1],"meta-externalagent":["partial","*",20,1],"cohere-ai":["partial","*",20,1],"Diffbot":["partial","*",20,1],"ContextFetchBot":["partial","*",20,1]},"llms":{"status":"missing","http":404,"bytes":30801},"home":{"status":"ok","http":200,"url":"https://www.volkswagen.de/de.html","html_tokens":149730,"markdown_tokens":301,"pruned_tokens":80}},
{"rank":8424,"domain":"transifex.com","origin":"https://transifex.com","robots":{"status":"ok","http":200,"url":"https://www.transifex.com/robots.txt","bytes":217,"groups":1},"bots":{"GPTBot":["partial","*",5,1],"ChatGPT-User":["partial","*",5,1],"OAI-SearchBot":["partial","*",5,1],"ClaudeBot":["partial","*",5,1],"Claude-User":["partial","*",5,1],"Claude-SearchBot":["partial","*",5,1],"anthropic-ai":["partial","*",5,1],"PerplexityBot":["partial","*",5,1],"Perplexity-User":["partial","*",5,1],"Google-Extended":["partial","*",5,1],"Applebot-Extended":["partial","*",5,1],"CCBot":["partial","*",5,1],"Bytespider":["partial","*",5,1],"Amazonbot":["partial","*",5,1],"meta-externalagent":["partial","*",5,1],"cohere-ai":["partial","*",5,1],"Diffbot":["partial","*",5,1],"ContextFetchBot":["partial","*",5,1]},"llms":{"status":"present","http":200,"bytes":25560},"home":{"status":"ok","http":200,"url":"https://www.transifex.com/","html_tokens":86476,"markdown_tokens":420,"pruned_tokens":166}},
{"rank":8425,"domain":"icmje.org","origin":"https://icmje.org","robots":{"status":"ok","http":200,"url":"https://icmje.org/robots.txt","bytes":153,"groups":1},"bots":{"GPTBot":["partial","*",1,0],"ChatGPT-User":["partial","*",1,0],"OAI-SearchBot":["partial","*",1,0],"ClaudeBot":["partial","*",1,0],"Claude-User":["partial","*",1,0],"Claude-SearchBot":["partial","*",1,0],"anthropic-ai":["partial","*",1,0],"PerplexityBot":["partial","*",1,0],"Perplexity-User":["partial","*",1,0],"Google-Extended":["partial","*",1,0],"Applebot-Extended":["partial","*",1,0],"CCBot":["partial","*",1,0],"Bytespider":["partial","*",1,0],"Amazonbot":["partial","*",1,0],"meta-externalagent":["partial","*",1,0],"cohere-ai":["partial","*",1,0],"Diffbot":["partial","*",1,0],"ContextFetchBot":["partial","*",1,0]},"llms":{"status":"missing","http":404,"bytes":563},"home":{"status":"ok","http":200,"url":"https://icmje.org/","html_tokens":6676,"markdown_tokens":274,"pruned_tokens":113}},
{"rank":8426,"domain":"valuvdo.com","origin":"https://valuvdo.com","robots":{"status":"ok","http":200,"url":"https://valuvdo.com/robots.txt","bytes":485,"groups":1},"bots":{"GPTBot":["partial","*",21,0],"ChatGPT-User":["partial","*",21,0],"OAI-SearchBot":["partial","*",21,0],"ClaudeBot":["partial","*",21,0],"Claude-User":["partial","*",21,0],"Claude-SearchBot":["partial","*",21,0],"anthropic-ai":["partial","*",21,0],"PerplexityBot":["partial","*",21,0],"Perplexity-User":["partial","*",21,0],"Google-Extended":["partial","*",21,0],"Applebot-Extended":["partial","*",21,0],"CCBot":["partial","*",21,0],"Bytespider":["partial","*",21,0],"Amazonbot":["partial","*",21,0],"meta-externalagent":["partial","*",21,0],"cohere-ai":["partial","*",21,0],"Diffbot":["partial","*",21,0],"ContextFetchBot":["partial","*",21,0]},"llms":{"status":"missing","http":404,"bytes":4726},"home":{"status":"ok","http":200,"url":"https://valuvdo.com/","html_tokens":5594,"markdown_tokens":47,"pruned_tokens":47}},
{"rank":8427,"domain":"psg.fr","origin":"https://psg.fr","robots":{"status":"ok","http":200,"url":"https://www.psg.fr/robots.txt","bytes":545,"groups":2},"bots":{"GPTBot":["partial","*",16,1],"ChatGPT-User":["partial","*",16,1],"OAI-SearchBot":["partial","*",16,1],"ClaudeBot":["partial","*",16,1],"Claude-User":["partial","*",16,1],"Claude-SearchBot":["partial","*",16,1],"anthropic-ai":["partial","*",16,1],"PerplexityBot":["partial","*",16,1],"Perplexity-User":["partial","*",16,1],"Google-Extended":["partial","*",16,1],"Applebot-Extended":["partial","*",16,1],"CCBot":["partial","*",16,1],"Bytespider":["partial","*",16,1],"Amazonbot":["partial","*",16,1],"meta-externalagent":["partial","*",16,1],"cohere-ai":["partial","*",16,1],"Diffbot":["partial","*",16,1],"ContextFetchBot":["partial","*",16,1]},"llms":{"status":"missing","http":404,"bytes":51590},"home":{"status":"ok","http":200,"url":"https://www.psg.fr/en","html_tokens":125092,"markdown_tokens":89,"pruned_tokens":89}},
{"rank":8428,"domain":"lyon.fr","origin":"https://lyon.fr","robots":{"status":"error","http":null,"url":"https://lyon.fr/robots.txt","bytes":0,"groups":0,"error":"For absolute URLs, path must be empty or begin with '/'"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":8429,"domain":"ucr.ac.cr","origin":"https://ucr.ac.cr","robots":{"status":"ok","http":200,"url":"https://www.ucr.ac.cr/robots.txt","bytes":200,"groups":1},"bots":{"GPTBot":["partial","*",6,0],"ChatGPT-User":["partial","*",6,0],"OAI-SearchBot":["partial","*",6,0],"ClaudeBot":["partial","*",6,0],"Claude-User":["partial","*",6,0],"Claude-SearchBot":["partial","*",6,0],"anthropic-ai":["partial","*",6,0],"PerplexityBot":["partial","*",6,0],"Perplexity-User":["partial","*",6,0],"Google-Extended":["partial","*",6,0],"Applebot-Extended":["partial","*",6,0],"CCBot":["partial","*",6,0],"Bytespider":["partial","*",6,0],"Amazonbot":["partial","*",6,0],"meta-externalagent":["partial","*",6,0],"cohere-ai":["partial","*",6,0],"Diffbot":["partial","*",6,0],"ContextFetchBot":["partial","*",6,0]},"llms":{"status":"missing","http":404,"bytes":51472},"home":{"status":"ok","http":200,"url":"https://www.ucr.ac.cr/","html_tokens":58887,"markdown_tokens":1193,"pruned_tokens":149}},
{"rank":8430,"domain":"google.cm","origin":"https://google.cm","robots":{"status":"ok","http":200,"url":"https://www.google.cm/robots.txt","bytes":6624,"groups":4},"bots":{"GPTBot":["partial","*",169,64],"ChatGPT-User":["partial","*",169,64],"OAI-SearchBot":["partial","*",169,64],"ClaudeBot":["partial","*",169,64],"Claude-User":["partial","*",169,64],"Claude-SearchBot":["partial","*",169,64],"anthropic-ai":["partial","*",169,64],"PerplexityBot":["partial","*",169,64],"Perplexity-User":["partial","*",169,64],"Google-Extended":["partial","*",169,64],"Applebot-Extended":["partial","*",169,64],"CCBot":["partial","*",169,64],"Bytespider":["partial","*",169,64],"Amazonbot":["partial","*",169,64],"meta-externalagent":["partial","*",169,64],"cohere-ai":["partial","*",169,64],"Diffbot":["partial","*",169,64],"ContextFetchBot":["partial","*",169,64]},"llms":{"status":"missing","http":404,"bytes":1569},"home":{"status":"ok","http":200,"url":"https://www.google.com/","html_tokens":21321,"markdown_tokens":30,"pruned_tokens":30}},
{"rank":8431,"domain":"shtehe.com","origin":"https://shtehe.com","robots":{"status":"unreachable","http":null,"url":"https://shtehe.com/robots.txt","bytes":0,"groups":0,"error":"cannot resolve shtehe.com"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":8432,"domain":"dmagazine.com","origin":"https://dmagazine.com","robots":{"status":"refused","http":403,"url":"https://dmagazine.com/robots.txt","bytes":5530,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":8433,"domain":"huanghuai.edu.cn","origin":"https://huanghuai.edu.cn","robots":{"status":"unreachable","http":null,"url":"https://huanghuai.edu.cn/robots.txt","bytes":0,"groups":0,"error":"[SSL: CERTIFICATE_VERIFY_FAILED] certificate verify failed: self-signed certificate (_ssl.c:1028)"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":8434,"domain":"hitachi.co.jp","origin":"https://www.hitachi.co.jp","robots":{"status":"ok","http":200,"url":"https://www.hitachi.co.jp/robots.txt","bytes":31,"groups":1},"bots":{"GPTBot":["partial","*",1,0],"ChatGPT-User":["partial","*",1,0],"OAI-SearchBot":["partial","*",1,0],"ClaudeBot":["partial","*",1,0],"Claude-User":["partial","*",1,0],"Claude-SearchBot":["partial","*",1,0],"anthropic-ai":["partial","*",1,0],"PerplexityBot":["partial","*",1,0],"Perplexity-User":["partial","*",1,0],"Google-Extended":["partial","*",1,0],"Applebot-Extended":["partial","*",1,0],"CCBot":["partial","*",1,0],"Bytespider":["partial","*",1,0],"Amazonbot":["partial","*",1,0],"meta-externalagent":["partial","*",1,0],"cohere-ai":["partial","*",1,0],"Diffbot":["partial","*",1,0],"ContextFetchBot":["partial","*",1,0]},"llms":{"status":"missing","http":404,"bytes":8902},"home":{"status":"ok","http":200,"url":"https://www.hitachi.com/ja-jp/","html_tokens":48843,"markdown_tokens":69,"pruned_tokens":69}},
{"rank":8435,"domain":"inrs.fr","origin":"https://inrs.fr","robots":{"status":"ok","http":200,"url":"https://www.inrs.fr/robots.txt","bytes":325,"groups":1},"bots":{"GPTBot":["partial","*",7,0],"ChatGPT-User":["partial","*",7,0],"OAI-SearchBot":["partial","*",7,0],"ClaudeBot":["partial","*",7,0],"Claude-User":["partial","*",7,0],"Claude-SearchBot":["partial","*",7,0],"anthropic-ai":["partial","*",7,0],"PerplexityBot":["partial","*",7,0],"Perplexity-User":["partial","*",7,0],"Google-Extended":["partial","*",7,0],"Applebot-Extended":["partial","*",7,0],"CCBot":["partial","*",7,0],"Bytespider":["partial","*",7,0],"Amazonbot":["partial","*",7,0],"meta-externalagent":["partial","*",7,0],"cohere-ai":["partial","*",7,0],"Diffbot":["partial","*",7,0],"ContextFetchBot":["partial","*",7,0]},"llms":{"status":"missing","http":404,"bytes":52693},"home":{"status":"ok","http":200,"url":"https://www.inrs.fr/","html_tokens":23638,"markdown_tokens":209,"pruned_tokens":66}},
{"rank":8436,"domain":"scholasticahq.com","origin":"https://scholasticahq.com","robots":{"status":"ok","http":200,"url":"https://scholasticahq.com/robots.txt","bytes":85,"groups":1},"bots":{"GPTBot":["partial","*",1,1],"ChatGPT-User":["partial","*",1,1],"OAI-SearchBot":["partial","*",1,1],"ClaudeBot":["partial","*",1,1],"Claude-User":["partial","*",1,1],"Claude-SearchBot":["partial","*",1,1],"anthropic-ai":["partial","*",1,1],"PerplexityBot":["partial","*",1,1],"Perplexity-User":["partial","*",1,1],"Google-Extended":["partial","*",1,1],"Applebot-Extended":["partial","*",1,1],"CCBot":["partial","*",1,1],"Bytespider":["partial","*",1,1],"Amazonbot":["partial","*",1,1],"meta-externalagent":["partial","*",1,1],"cohere-ai":["partial","*",1,1],"Diffbot":["partial","*",1,1],"ContextFetchBot":["partial","*",1,1]},"llms":{"status":"missing","http":404,"bytes":2334},"home":{"status":"ok","http":200,"url":"https://scholasticahq.com/","html_tokens":7076,"markdown_tokens":233,"pruned_tokens":63}},
{"rank":8437,"domain":"kodi.tv","origin":"https://kodi.tv","robots":{"status":"missing","http":404,"url":"https://kodi.tv/robots.txt","bytes":26089,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":26089},"home":{"status":"http_error","http":403,"url":"https://kodi.tv/"}},
{"rank":8438,"domain":"i.tianqi.com","origin":"https://i.tianqi.com","robots":{"status":"missing","http":404,"url":"https://i.tianqi.com/robots.txt","bytes":150,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":150},"home":{"status":"error","http":200,"url":"https://i.tianqi.com/","error":"unparseable HTML"}},
{"rank":8439,"domain":"wantengyuanlin.com","origin":"https://www.wantengyuanlin.com","robots":{"status":"missing","http":404,"url":"https://www.wantengyuanlin.com/robots.txt","bytes":3049,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":3045},"home":{"status":"error","http":200,"url":"https://www.wantengyuanlin.com/","error":"unparseable HTML"}},
{"rank":8440,"domain":"xn--l9qya375kzx4a.com","origin":"http://xn--l9qya375kzx4a.com","robots":{"status":"missing","http":404,"url":"http://xn--l9qya375kzx4a.com/robots.txt","bytes":4734,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":4730},"home":{"status":"ok","http":200,"url":"http://xn--l9qya375kzx4a.com/","html_tokens":13309,"markdown_tokens":246,"pruned_tokens":93}},
{"rank":8441,"domain":"bungie.net","origin":"https://bungie.net","robots":{"status":"ok","http":200,"url":"https://www.bungie.net/robots.txt","bytes":382,"groups":1},"bots":{"GPTBot":["partial","*",16,0],"ChatGPT-User":["partial","*",16,0],"OAI-SearchBot":["partial","*",16,0],"ClaudeBot":["partial","*",16,0],"Claude-User":["partial","*",16,0],"Claude-SearchBot":["partial","*",16,0],"anthropic-ai":["partial","*",16,0],"PerplexityBot":["partial","*",16,0],"Perplexity-User":["partial","*",16,0],"Google-Extended":["partial","*",16,0],"Applebot-Extended":["partial","*",16,0],"CCBot":["partial","*",16,0],"Bytespider":["partial","*",16,0],"Amazonbot":["partial","*",16,0],"meta-externalagent":["partial","*",16,0],"cohere-ai":["partial","*",16,0],"Diffbot":["partial","*",16,0],"ContextFetchBot":["partial","*",16,0]},"llms":{"status":"html","http":200,"bytes":883444},"home":{"status":"ok","http":200,"url":"https://www.bungie.net/7","html_tokens":220849,"markdown_tokens":12,"pruned_tokens":12}},
{"rank":8442,"domain":"diabetes.org.uk","origin":"https://diabetes.org.uk","robots":{"status":"ok","http":200,"url":"https://www.diabetes.org.uk/robots.txt","bytes":2268,"groups":1},"bots":{"GPTBot":["partial","*",36,18],"ChatGPT-User":["partial","*",36,18],"OAI-SearchBot":["partial","*",36,18],"ClaudeBot":["partial","*",36,18],"Claude-User":["partial","*",36,18],"Claude-SearchBot":["partial","*",36,18],"anthropic-ai":["partial","*",36,18],"PerplexityBot":["partial","*",36,18],"Perplexity-User":["partial","*",36,18],"Google-Extended":["partial","*",36,18],"Applebot-Extended":["partial","*",36,18],"CCBot":["partial","*",36,18],"Bytespider":["partial","*",36,18],"Amazonbot":["partial","*",36,18],"meta-externalagent":["partial","*",36,18],"cohere-ai":["partial","*",36,18],"Diffbot":["partial","*",36,18],"ContextFetchBot":["partial","*",36,18]},"llms":{"status":"missing","http":404,"bytes":193},"home":{"status":"ok","http":200,"url":"https://www.diabetes.org.uk/","html_tokens":51391,"markdown_tokens":108,"pruned_tokens":58}},
{"rank":8443,"domain":"you-fuli.com","origin":"https://www.you-fuli.com","robots":{"status":"ok","http":200,"url":"https://www.you-fuli.com/robots.txt","bytes":313,"groups":1},"bots":{"GPTBot":["partial","*",12,1],"ChatGPT-User":["partial","*",12,1],"OAI-SearchBot":["partial","*",12,1],"ClaudeBot":["partial","*",12,1],"Claude-User":["partial","*",12,1],"Claude-SearchBot":["partial","*",12,1],"anthropic-ai":["partial","*",12,1],"PerplexityBot":["partial","*",12,1],"Perplexity-User":["partial","*",12,1],"Google-Extended":["partial","*",12,1],"Applebot-Extended":["partial","*",12,1],"CCBot":["partial","*",12,1],"Bytespider":["partial","*",12,1],"Amazonbot":["partial","*",12,1],"meta-externalagent":["partial","*",12,1],"cohere-ai":["partial","*",12,1],"Diffbot":["partial","*",12,1],"ContextFetchBot":["partial","*",12,1]},"llms":{"status":"missing","http":404,"bytes":4734},"home":{"status":"ok","http":200,"url":"https://www.you-fuli.com/","html_tokens":6219,"markdown_tokens":23,"pruned_tokens":23}},
{"rank":8444,"domain":"palantir.com","origin":"https://palantir.com","robots":{"status":"ok","http":200,"url":"https://www.palantir.com/robots.txt","bytes":175,"groups":1},"bots":{"GPTBot":["allowed","*",0,1],"ChatGPT-User":["allowed","*",0,1],"OAI-SearchBot":["allowed","*",0,1],"ClaudeBot":["allowed","*",0,1],"Claude-User":["allowed","*",0,1],"Claude-SearchBot":["allowed","*",0,1],"anthropic-ai":["allowed","*",0,1],"PerplexityBot":["allowed","*",0,1],"Perplexity-User":["allowed","*",0,1],"Google-Extended":["allowed","*",0,1],"Applebot-Extended":["allowed","*",0,1],"CCBot":["allowed","*",0,1],"Bytespider":["allowed","*",0,1],"Amazonbot":["allowed","*",0,1],"meta-externalagent":["allowed","*",0,1],"cohere-ai":["allowed","*",0,1],"Diffbot":["allowed","*",0,1],"ContextFetchBot":["allowed","*",0,1]},"llms":{"status":"html","http":200,"bytes":422773},"home":{"status":"ok","http":200,"url":"https://www.palantir.com/","html_tokens":115925,"markdown_tokens":3,"pruned_tokens":3}},
{"rank":8445,"domain":"yourhosting.nl","origin":"https://yourhosting.nl","robots":{"status":"ok","http":200,"url":"https://yourhosting.nl/robots.txt","bytes":661,"groups":2},"bots":{"GPTBot":["partial","*",9,1],"ChatGPT-User":["partial","*",9,1],"OAI-SearchBot":["partial","*",9,1],"ClaudeBot":["partial","*",9,1],"Claude-User":["partial","*",9,1],"Claude-SearchBot":["partial","*",9,1],"anthropic-ai":["partial","*",9,1],"PerplexityBot":["partial","*",9,1],"Perplexity-User":["partial","*",9,1],"Google-Extended":["partial","*",9,1],"Applebot-Extended":["partial","*",9,1],"CCBot":["partial","*",9,1],"Bytespider":["partial","*",9,1],"Amazonbot":["partial","*",9,1],"meta-externalagent":["partial","*",9,1],"cohere-ai":["partial","*",9,1],"Diffbot":["partial","*",9,1],"ContextFetchBot":["partial","*",9,1]},"llms":{"status":"other_text","http":200,"bytes":15516},"home":{"status":"ok","http":200,"url":"https://www.yourhosting.nl/","html_tokens":50169,"markdown_tokens":1184,"pruned_tokens":111}},
{"rank":8446,"domain":"leica-camera.com","origin":"https://leica-camera.com","robots":{"status":"ok","http":200,"url":"https://leica-camera.com/robots.txt","bytes":2343,"groups":1},"bots":{"GPTBot":["partial","*",51,18],"ChatGPT-User":["partial","*",51,18],"OAI-SearchBot":["partial","*",51,18],"ClaudeBot":["partial","*",51,18],"Claude-User":["partial","*",51,18],"Claude-SearchBot":["partial","*",51,18],"anthropic-ai":["partial","*",51,18],"PerplexityBot":["partial","*",51,18],"Perplexity-User":["partial","*",51,18],"Google-Extended":["partial","*",51,18],"Applebot-Extended":["partial","*",51,18],"CCBot":["partial","*",51,18],"Bytespider":["partial","*",51,18],"Amazonbot":["partial","*",51,18],"meta-externalagent":["partial","*",51,18],"cohere-ai":["partial","*",51,18],"Diffbot":["partial","*",51,18],"ContextFetchBot":["partial","*",51,18]},"llms":{"status":"http_error","http":403,"bytes":153},"home":{"status":"ok","http":200,"url":"https://leica-camera.com/en-US","html_tokens":346718,"markdown_tokens":531,"pruned_tokens":430}},
{"rank":8447,"domain":"google.la","origin":"https://google.la","robots":{"status":"ok","http":200,"url":"https://www.google.la/robots.txt","bytes":6624,"groups":4},"bots":{"GPTBot":["partial","*",169,64],"ChatGPT-User":["partial","*",169,64],"OAI-SearchBot":["partial","*",169,64],"ClaudeBot":["partial","*",169,64],"Claude-User":["partial","*",169,64],"Claude-SearchBot":["partial","*",169,64],"anthropic-ai":["partial","*",169,64],"PerplexityBot":["partial","*",169,64],"Perplexity-User":["partial","*",169,64],"Google-Extended":["partial","*",169,64],"Applebot-Extended":["partial","*",169,64],"CCBot":["partial","*",169,64],"Bytespider":["partial","*",169,64],"Amazonbot":["partial","*",169,64],"meta-externalagent":["partial","*",169,64],"cohere-ai":["partial","*",169,64],"Diffbot":["partial","*",169,64],"ContextFetchBot":["partial","*",169,64]},"llms":{"status":"missing","http":404,"bytes":1569},"home":{"status":"ok","http":200,"url":"https://www.google.com/","html_tokens":21453,"markdown_tokens":30,"pruned_tokens":30}},
{"rank":8448,"domain":"aiga.org","origin":"https://aiga.org","robots":{"status":"ok","http":200,"url":"https://www.aiga.org/robots.txt","bytes":2005,"groups":1},"bots":{"GPTBot":["partial","*",33,18],"ChatGPT-User":["partial","*",33,18],"OAI-SearchBot":["partial","*",33,18],"ClaudeBot":["partial","*",33,18],"Claude-User":["partial","*",33,18],"Claude-SearchBot":["partial","*",33,18],"anthropic-ai":["partial","*",33,18],"PerplexityBot":["partial","*",33,18],"Perplexity-User":["partial","*",33,18],"Google-Extended":["partial","*",33,18],"Applebot-Extended":["partial","*",33,18],"CCBot":["partial","*",33,18],"Bytespider":["partial","*",33,18],"Amazonbot":["partial","*",33,18],"meta-externalagent":["partial","*",33,18],"cohere-ai":["partial","*",33,18],"Diffbot":["partial","*",33,18],"ContextFetchBot":["partial","*",33,18]},"llms":{"status":"missing","http":404,"bytes":547},"home":{"status":"http_error","http":403,"url":"https://aiga.org/"}},
{"rank":8449,"domain":"dpma.de","origin":"https://dpma.de","robots":{"status":"ok","http":200,"url":"https://dpma.de/robots.txt","bytes":35,"groups":1},"bots":{"GPTBot":["partial","*",1,0],"ChatGPT-User":["partial","*",1,0],"OAI-SearchBot":["partial","*",1,0],"ClaudeBot":["partial","*",1,0],"Claude-User":["partial","*",1,0],"Claude-SearchBot":["partial","*",1,0],"anthropic-ai":["partial","*",1,0],"PerplexityBot":["partial","*",1,0],"Perplexity-User":["partial","*",1,0],"Google-Extended":["partial","*",1,0],"Applebot-Extended":["partial","*",1,0],"CCBot":["partial","*",1,0],"Bytespider":["partial","*",1,0],"Amazonbot":["partial","*",1,0],"meta-externalagent":["partial","*",1,0],"cohere-ai":["partial","*",1,0],"Diffbot":["partial","*",1,0],"ContextFetchBot":["partial","*",1,0]},"llms":{"status":"missing","http":404,"bytes":23823},"home":{"status":"ok","http":200,"url":"https://dpma.de/","html_tokens":12992,"markdown_tokens":209,"pruned_tokens":74}},
{"rank":8450,"domain":"lexmark.com","origin":"https://lexmark.com","robots":{"status":"ok","http":200,"url":"https://www.lexmark.com/robots.txt","bytes":4512,"groups":1},"bots":{"GPTBot":["partial","*",7,0],"ChatGPT-User":["partial","*",7,0],"OAI-SearchBot":["partial","*",7,0],"ClaudeBot":["partial","*",7,0],"Claude-User":["partial","*",7,0],"Claude-SearchBot":["partial","*",7,0],"anthropic-ai":["partial","*",7,0],"PerplexityBot":["partial","*",7,0],"Perplexity-User":["partial","*",7,0],"Google-Extended":["partial","*",7,0],"Applebot-Extended":["partial","*",7,0],"CCBot":["partial","*",7,0],"Bytespider":["partial","*",7,0],"Amazonbot":["partial","*",7,0],"meta-externalagent":["partial","*",7,0],"cohere-ai":["partial","*",7,0],"Diffbot":["partial","*",7,0],"ContextFetchBot":["partial","*",7,0]},"llms":{"status":"missing","http":404,"bytes":47293},"home":{"status":"ok","http":200,"url":"https://www.lexmark.com/en_us.html","html_tokens":17750,"markdown_tokens":35,"pruned_tokens":35}},
{"rank":8451,"domain":"footprintnetwork.org","origin":"https://footprintnetwork.org","robots":{"status":"missing","http":404,"url":"https://footprintnetwork.org/robots.txt","bytes":236,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":236},"home":{"status":"ok","http":200,"url":"https://footprintnetwork.org/","html_tokens":46481,"markdown_tokens":160,"pruned_tokens":88}},
{"rank":8452,"domain":"de-ele.com","origin":"https://de-ele.com","robots":{"status":"ok","http":200,"url":"https://www.de-ele.com/robots.txt","bytes":311,"groups":1},"bots":{"GPTBot":["partial","*",12,1],"ChatGPT-User":["partial","*",12,1],"OAI-SearchBot":["partial","*",12,1],"ClaudeBot":["partial","*",12,1],"Claude-User":["partial","*",12,1],"Claude-SearchBot":["partial","*",12,1],"anthropic-ai":["partial","*",12,1],"PerplexityBot":["partial","*",12,1],"Perplexity-User":["partial","*",12,1],"Google-Extended":["partial","*",12,1],"Applebot-Extended":["partial","*",12,1],"CCBot":["partial","*",12,1],"Bytespider":["partial","*",12,1],"Amazonbot":["partial","*",12,1],"meta-externalagent":["partial","*",12,1],"cohere-ai":["partial","*",12,1],"Diffbot":["partial","*",12,1],"ContextFetchBot":["partial","*",12,1]},"llms":{"status":"html","http":200,"bytes":5456},"home":{"status":"ok","http":200,"url":"https://www.de-ele.com/","html_tokens":106566,"markdown_tokens":207,"pruned_tokens":122}},
{"rank":8453,"domain":"mixi.jp","origin":"https://mixi.jp","robots":{"status":"ok","http":200,"url":"https://mixi.jp/robots.txt","bytes":965,"groups":3},"bots":{"GPTBot":["partial","*",33,0],"ChatGPT-User":["partial","*",33,0],"OAI-SearchBot":["partial","*",33,0],"ClaudeBot":["partial","*",33,0],"Claude-User":["partial","*",33,0],"Claude-SearchBot":["partial","*",33,0],"anthropic-ai":["partial","*",33,0],"PerplexityBot":["partial","*",33,0],"Perplexity-User":["partial","*",33,0],"Google-Extended":["partial","*",33,0],"Applebot-Extended":["partial","*",33,0],"CCBot":["partial","*",33,0],"Bytespider":["partial","*",33,0],"Amazonbot":["partial","*",33,0],"meta-externalagent":["partial","*",33,0],"cohere-ai":["partial","*",33,0],"Diffbot":["partial","*",33,0],"ContextFetchBot":["partial","*",33,0]},"llms":{"status":"missing","http":404,"bytes":8424},"home":{"status":"ok","http":200,"url":"https://mixi.jp/","html_tokens":9127,"markdown_tokens":52,"pruned_tokens":52}},
{"rank":8454,"domain":"abqjournal.com","origin":"https://abqjournal.com","robots":{"status":"ok","http":200,"url":"https://www.abqjournal.com/robots.txt","bytes":4755,"groups":4},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["allowed","named",0,7],"OAI-SearchBot":["allowed","named",0,7],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",29,28],"Claude-SearchBot":["allowed","named",0,7],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["allowed","named",0,7],"Perplexity-User":["allowed","named",0,7],"Google-Extended":["allowed","named",0,7],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["partial","*",29,28],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",29,28]},"llms":{"status":"present","http":200,"bytes":814},"home":{"status":"ok","http":200,"url":"https://www.abqjournal.com/","html_tokens":122653,"markdown_tokens":420,"pruned_tokens":159}},
{"rank":8455,"domain":"iamsterdam.com","origin":"https://iamsterdam.com","robots":{"status":"ok","http":200,"url":"https://www.iamsterdam.com/robots.txt","bytes":1371,"groups":9},"bots":{"GPTBot":["partial","*",3,2],"ChatGPT-User":["partial","*",3,2],"OAI-SearchBot":["partial","*",3,2],"ClaudeBot":["partial","*",3,2],"Claude-User":["partial","*",3,2],"Claude-SearchBot":["partial","*",3,2],"anthropic-ai":["partial","*",3,2],"PerplexityBot":["partial","*",3,2],"Perplexity-User":["partial","*",3,2],"Google-Extended":["partial","*",3,2],"Applebot-Extended":["partial","*",3,2],"CCBot":["partial","*",3,2],"Bytespider":["blocked","named",1,0],"Amazonbot":["partial","*",3,2],"meta-externalagent":["partial","*",3,2],"cohere-ai":["partial","*",3,2],"Diffbot":["partial","*",3,2],"ContextFetchBot":["partial","*",3,2]},"llms":{"status":"missing","http":404,"bytes":396527},"home":{"status":"ok","http":200,"url":"https://www.iamsterdam.com/","html_tokens":141766,"markdown_tokens":179,"pruned_tokens":128}},
{"rank":8456,"domain":"wish.com","origin":"https://wish.com","robots":{"status":"ok","http":200,"url":"https://www.wish.com/robots.txt","bytes":565,"groups":3},"bots":{"GPTBot":["partial","*",21,0],"ChatGPT-User":["partial","*",21,0],"OAI-SearchBot":["partial","*",21,0],"ClaudeBot":["partial","*",21,0],"Claude-User":["partial","*",21,0],"Claude-SearchBot":["partial","*",21,0],"anthropic-ai":["partial","*",21,0],"PerplexityBot":["partial","*",21,0],"Perplexity-User":["partial","*",21,0],"Google-Extended":["partial","*",21,0],"Applebot-Extended":["partial","*",21,0],"CCBot":["partial","*",21,0],"Bytespider":["partial","*",21,0],"Amazonbot":["partial","*",21,0],"meta-externalagent":["partial","*",21,0],"cohere-ai":["partial","*",21,0],"Diffbot":["partial","*",21,0],"ContextFetchBot":["partial","*",21,0]},"llms":{"status":"missing","http":404,"bytes":12052},"home":{"status":"ok","http":200,"url":"https://www.wish.com/","html_tokens":19316,"markdown_tokens":0,"pruned_tokens":0}},
{"rank":8457,"domain":"iiss.org","origin":"https://iiss.org","robots":{"status":"ok","http":200,"url":"https://www.iiss.org/robots.txt","bytes":54,"groups":1},"bots":{"GPTBot":["partial","*",2,0],"ChatGPT-User":["partial","*",2,0],"OAI-SearchBot":["partial","*",2,0],"ClaudeBot":["partial","*",2,0],"Claude-User":["partial","*",2,0],"Claude-SearchBot":["partial","*",2,0],"anthropic-ai":["partial","*",2,0],"PerplexityBot":["partial","*",2,0],"Perplexity-User":["partial","*",2,0],"Google-Extended":["partial","*",2,0],"Applebot-Extended":["partial","*",2,0],"CCBot":["partial","*",2,0],"Bytespider":["partial","*",2,0],"Amazonbot":["partial","*",2,0],"meta-externalagent":["partial","*",2,0],"cohere-ai":["partial","*",2,0],"Diffbot":["partial","*",2,0],"ContextFetchBot":["partial","*",2,0]},"llms":{"status":"missing","http":404,"bytes":22651},"home":{"status":"ok","http":200,"url":"https://www.iiss.org/","html_tokens":27271,"markdown_tokens":0,"pruned_tokens":0}},
{"rank":8458,"domain":"montclair.edu","origin":"https://montclair.edu","robots":{"status":"refused","http":403,"url":"https://montclair.edu/robots.txt","bytes":560,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":8459,"domain":"haisum.com","origin":"https://haisum.com","robots":{"status":"unreachable","http":null,"url":"https://haisum.com/robots.txt","bytes":0,"groups":0,"error":"cannot resolve haisum.com"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":8460,"domain":"thebusinessresearchcompany.com","origin":"https://thebusinessresearchcompany.com","robots":{"status":"ok","http":200,"url":"https://www.thebusinessresearchcompany.com/robots.txt","bytes":1364,"groups":13},"bots":{"GPTBot":["allowed","named",0,1],"ChatGPT-User":["allowed","named",0,1],"OAI-SearchBot":["partial","*",24,0],"ClaudeBot":["allowed","named",0,1],"Claude-User":["partial","*",24,0],"Claude-SearchBot":["partial","*",24,0],"anthropic-ai":["partial","*",24,0],"PerplexityBot":["allowed","named",0,1],"Perplexity-User":["partial","*",24,0],"Google-Extended":["allowed","named",0,1],"Applebot-Extended":["allowed","named",0,1],"CCBot":["allowed","named",0,1],"Bytespider":["partial","*",24,0],"Amazonbot":["allowed","named",0,1],"meta-externalagent":["partial","*",24,0],"cohere-ai":["allowed","named",0,1],"Diffbot":["partial","*",24,0],"ContextFetchBot":["partial","*",24,0]},"llms":{"status":"present","http":200,"bytes":768},"home":{"status":"ok","http":200,"url":"https://www.thebusinessresearchcompany.com/","html_tokens":57236,"markdown_tokens":1375,"pruned_tokens":155}},
{"rank":8461,"domain":"digitalcameraworld.com","origin":"https://digitalcameraworld.com","robots":{"status":"ok","http":200,"url":"https://www.digitalcameraworld.com/robots.txt","bytes":1744,"groups":4},"bots":{"GPTBot":["partial","*",18,0],"ChatGPT-User":["partial","*",18,0],"OAI-SearchBot":["partial","*",18,0],"ClaudeBot":["partial","*",18,0],"Claude-User":["partial","*",18,0],"Claude-SearchBot":["partial","*",18,0],"anthropic-ai":["partial","*",18,0],"PerplexityBot":["partial","*",18,0],"Perplexity-User":["partial","*",18,0],"Google-Extended":["partial","*",18,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["partial","*",18,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["partial","*",18,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",18,0]},"llms":{"status":"missing","http":404,"bytes":1103752},"home":{"status":"ok","http":200,"url":"https://www.digitalcameraworld.com/","html_tokens":405876,"markdown_tokens":58,"pruned_tokens":58}},
{"rank":8462,"domain":"prnewswire.co.uk","origin":"https://prnewswire.co.uk","robots":{"status":"ok","http":200,"url":"https://www.prnewswire.co.uk/robots.txt","bytes":159,"groups":1},"bots":{"GPTBot":["partial","*",1,0],"ChatGPT-User":["partial","*",1,0],"OAI-SearchBot":["partial","*",1,0],"ClaudeBot":["partial","*",1,0],"Claude-User":["partial","*",1,0],"Claude-SearchBot":["partial","*",1,0],"anthropic-ai":["partial","*",1,0],"PerplexityBot":["partial","*",1,0],"Perplexity-User":["partial","*",1,0],"Google-Extended":["partial","*",1,0],"Applebot-Extended":["partial","*",1,0],"CCBot":["partial","*",1,0],"Bytespider":["partial","*",1,0],"Amazonbot":["partial","*",1,0],"meta-externalagent":["partial","*",1,0],"cohere-ai":["partial","*",1,0],"Diffbot":["partial","*",1,0],"ContextFetchBot":["partial","*",1,0]},"llms":{"status":"missing","http":404,"bytes":0},"home":{"status":"ok","http":200,"url":"https://www.prnewswire.co.uk/","html_tokens":44647,"markdown_tokens":126,"pruned_tokens":126}},
{"rank":8463,"domain":"parl.ca","origin":"https://parl.ca","robots":{"status":"ok","http":200,"url":"https://www.parl.ca/robots.txt","bytes":126,"groups":1},"bots":{"GPTBot":["partial","*",4,0],"ChatGPT-User":["partial","*",4,0],"OAI-SearchBot":["partial","*",4,0],"ClaudeBot":["partial","*",4,0],"Claude-User":["partial","*",4,0],"Claude-SearchBot":["partial","*",4,0],"anthropic-ai":["partial","*",4,0],"PerplexityBot":["partial","*",4,0],"Perplexity-User":["partial","*",4,0],"Google-Extended":["partial","*",4,0],"Applebot-Extended":["partial","*",4,0],"CCBot":["partial","*",4,0],"Bytespider":["partial","*",4,0],"Amazonbot":["partial","*",4,0],"meta-externalagent":["partial","*",4,0],"cohere-ai":["partial","*",4,0],"Diffbot":["partial","*",4,0],"ContextFetchBot":["partial","*",4,0]},"llms":{"status":"disallowed","error":"robots.txt of www.parl.ca disallows /ErrorPage/Default.aspx?Url=https%3a%2f%2fwww.parl.ca%2fllms.txt&StatusCode=404"},"home":{"status":"ok","http":200,"url":"https://www.parl.ca/","html_tokens":597,"markdown_tokens":44,"pruned_tokens":34}},
{"rank":8464,"domain":"musescore.com","origin":"https://musescore.com","robots":{"status":"ok","http":200,"url":"https://musescore.com/robots.txt","bytes":3374,"groups":2},"bots":{"GPTBot":["partial","*",28,9],"ChatGPT-User":["partial","*",28,9],"OAI-SearchBot":["partial","*",28,9],"ClaudeBot":["partial","*",28,9],"Claude-User":["partial","*",28,9],"Claude-SearchBot":["partial","*",28,9],"anthropic-ai":["partial","*",28,9],"PerplexityBot":["partial","*",28,9],"Perplexity-User":["partial","*",28,9],"Google-Extended":["partial","*",28,9],"Applebot-Extended":["partial","*",28,9],"CCBot":["partial","*",28,9],"Bytespider":["partial","*",28,9],"Amazonbot":["partial","*",28,9],"meta-externalagent":["partial","*",28,9],"cohere-ai":["partial","*",28,9],"Diffbot":["partial","*",28,9],"ContextFetchBot":["partial","*",28,9]},"llms":{"status":"http_error","http":403,"bytes":15467},"home":{"status":"http_error","http":403,"url":"https://musescore.com/"}},
{"rank":8465,"domain":"mtholyoke.edu","origin":"https://mtholyoke.edu","robots":{"status":"ok","http":200,"url":"https://www.mtholyoke.edu/robots.txt","bytes":2151,"groups":1},"bots":{"GPTBot":["partial","*",36,18],"ChatGPT-User":["partial","*",36,18],"OAI-SearchBot":["partial","*",36,18],"ClaudeBot":["partial","*",36,18],"Claude-User":["partial","*",36,18],"Claude-SearchBot":["partial","*",36,18],"anthropic-ai":["partial","*",36,18],"PerplexityBot":["partial","*",36,18],"Perplexity-User":["partial","*",36,18],"Google-Extended":["partial","*",36,18],"Applebot-Extended":["partial","*",36,18],"CCBot":["partial","*",36,18],"Bytespider":["partial","*",36,18],"Amazonbot":["partial","*",36,18],"meta-externalagent":["partial","*",36,18],"cohere-ai":["partial","*",36,18],"Diffbot":["partial","*",36,18],"ContextFetchBot":["partial","*",36,18]},"llms":{"status":"missing","http":404,"bytes":574},"home":{"status":"ok","http":200,"url":"https://www.mtholyoke.edu/","html_tokens":29222,"markdown_tokens":621,"pruned_tokens":116}},
{"rank":8466,"domain":"imagevenue.com","origin":"https://imagevenue.com","robots":{"status":"ok","http":200,"url":"https://imagevenue.com/robots.txt","bytes":24,"groups":1},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"missing","http":404,"bytes":1552},"home":{"status":"ok","http":200,"url":"https://imagevenue.com/","html_tokens":4718,"markdown_tokens":327,"pruned_tokens":151}},
{"rank":8467,"domain":"bleedingcool.com","origin":"https://bleedingcool.com","robots":{"status":"ok","http":200,"url":"https://bleedingcool.com/robots.txt","bytes":167,"groups":3},"bots":{"GPTBot":["allowed","*",0,1],"ChatGPT-User":["allowed","*",0,1],"OAI-SearchBot":["allowed","*",0,1],"ClaudeBot":["allowed","*",0,1],"Claude-User":["allowed","*",0,1],"Claude-SearchBot":["allowed","*",0,1],"anthropic-ai":["allowed","*",0,1],"PerplexityBot":["allowed","*",0,1],"Perplexity-User":["allowed","*",0,1],"Google-Extended":["allowed","*",0,1],"Applebot-Extended":["allowed","*",0,1],"CCBot":["allowed","*",0,1],"Bytespider":["allowed","*",0,1],"Amazonbot":["allowed","*",0,1],"meta-externalagent":["allowed","*",0,1],"cohere-ai":["allowed","*",0,1],"Diffbot":["allowed","*",0,1],"ContextFetchBot":["allowed","*",0,1]},"llms":{"status":"missing","http":404,"bytes":17305},"home":{"status":"ok","http":200,"url":"https://bleedingcool.com/","html_tokens":35629,"markdown_tokens":929,"pruned_tokens":148}},
{"rank":8468,"domain":"uri.sh","origin":"https://uri.sh","robots":{"status":"unreachable","http":null,"url":"https://uri.sh/robots.txt","bytes":0,"groups":0,"error":"cannot resolve uri.sh"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":8469,"domain":"jornada.com.mx","origin":"https://jornada.com.mx","robots":{"status":"ok","http":200,"url":"https://www.jornada.com.mx/robots.txt","bytes":1303,"groups":11},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["partial","*",15,1],"OAI-SearchBot":["partial","*",15,1],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",15,1],"Claude-SearchBot":["partial","*",15,1],"anthropic-ai":["partial","*",15,1],"PerplexityBot":["partial","*",15,1],"Perplexity-User":["partial","*",15,1],"Google-Extended":["partial","*",15,1],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["partial","*",15,1],"Diffbot":["partial","*",15,1],"ContextFetchBot":["partial","*",15,1]},"llms":{"status":"missing","http":404,"bytes":106},"home":{"status":"http_error","http":403,"url":"https://www.jornada.com.mx/"}},
{"rank":8470,"domain":"gelocal.it","origin":"https://gelocal.it","robots":{"status":"unreachable","http":null,"url":"https://gelocal.it/robots.txt","bytes":0,"groups":0,"error":"cannot resolve gelocal.it"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":8471,"domain":"telesurtv.net","origin":"https://telesurtv.net","robots":{"status":"ok","http":200,"url":"https://www.telesurtv.net/robots.txt","bytes":766,"groups":7},"bots":{"GPTBot":["partial","*",12,3],"ChatGPT-User":["partial","*",12,3],"OAI-SearchBot":["partial","*",12,3],"ClaudeBot":["partial","*",12,3],"Claude-User":["partial","*",12,3],"Claude-SearchBot":["partial","*",12,3],"anthropic-ai":["partial","*",12,3],"PerplexityBot":["partial","*",12,3],"Perplexity-User":["partial","*",12,3],"Google-Extended":["partial","*",12,3],"Applebot-Extended":["partial","*",12,3],"CCBot":["partial","*",12,3],"Bytespider":["partial","*",12,3],"Amazonbot":["partial","*",12,3],"meta-externalagent":["partial","*",12,3],"cohere-ai":["partial","*",12,3],"Diffbot":["partial","*",12,3],"ContextFetchBot":["partial","*",12,3]},"llms":{"status":"missing","http":404,"bytes":144528},"home":{"status":"ok","http":200,"url":"https://www.telesurtv.net/","html_tokens":369032,"markdown_tokens":2818,"pruned_tokens":137}},
{"rank":8472,"domain":"invespcro.com","origin":"https://invespcro.com","robots":{"status":"ok","http":200,"url":"https://www.invespcro.com/robots.txt","bytes":1451,"groups":1},"bots":{"GPTBot":["partial","*",13,1],"ChatGPT-User":["partial","*",13,1],"OAI-SearchBot":["partial","*",13,1],"ClaudeBot":["partial","*",13,1],"Claude-User":["partial","*",13,1],"Claude-SearchBot":["partial","*",13,1],"anthropic-ai":["partial","*",13,1],"PerplexityBot":["partial","*",13,1],"Perplexity-User":["partial","*",13,1],"Google-Extended":["partial","*",13,1],"Applebot-Extended":["partial","*",13,1],"CCBot":["partial","*",13,1],"Bytespider":["partial","*",13,1],"Amazonbot":["partial","*",13,1],"meta-externalagent":["partial","*",13,1],"cohere-ai":["partial","*",13,1],"Diffbot":["partial","*",13,1],"ContextFetchBot":["partial","*",13,1]},"llms":{"status":"missing","http":404,"bytes":76320},"home":{"status":"http_error","http":403,"url":"https://invespcro.com/"}},
{"rank":8473,"domain":"visionofhumanity.org","origin":"https://visionofhumanity.org","robots":{"status":"ok","http":200,"url":"https://visionofhumanity.org/robots.txt","bytes":200,"groups":1},"bots":{"GPTBot":["partial","*",4,1],"ChatGPT-User":["partial","*",4,1],"OAI-SearchBot":["partial","*",4,1],"ClaudeBot":["partial","*",4,1],"Claude-User":["partial","*",4,1],"Claude-SearchBot":["partial","*",4,1],"anthropic-ai":["partial","*",4,1],"PerplexityBot":["partial","*",4,1],"Perplexity-User":["partial","*",4,1],"Google-Extended":["partial","*",4,1],"Applebot-Extended":["partial","*",4,1],"CCBot":["partial","*",4,1],"Bytespider":["partial","*",4,1],"Amazonbot":["partial","*",4,1],"meta-externalagent":["partial","*",4,1],"cohere-ai":["partial","*",4,1],"Diffbot":["partial","*",4,1],"ContextFetchBot":["partial","*",4,1]},"llms":{"status":"http_error","http":403,"bytes":5595},"home":{"status":"http_error","http":403,"url":"https://visionofhumanity.org/"}},
{"rank":8474,"domain":"earth.com","origin":"https://earth.com","robots":{"status":"ok","http":200,"url":"https://www.earth.com/robots.txt","bytes":3727,"groups":98},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["partial","*",4,1],"OAI-SearchBot":["partial","*",4,1],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",4,1],"Claude-SearchBot":["partial","*",4,1],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["partial","*",4,1],"Perplexity-User":["partial","*",4,1],"Google-Extended":["partial","*",4,1],"Applebot-Extended":["partial","*",4,1],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["partial","*",4,1],"meta-externalagent":["partial","*",4,1],"cohere-ai":["partial","*",4,1],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",4,1]},"llms":{"status":"missing","http":404,"bytes":20342},"home":{"status":"ok","http":200,"url":"https://www.earth.com/","html_tokens":152135,"markdown_tokens":149,"pruned_tokens":149}},
{"rank":8475,"domain":"huffingtonpost.es","origin":"https://huffingtonpost.es","robots":{"status":"ok","http":200,"url":"https://www.huffingtonpost.es/robots.txt","bytes":733,"groups":2},"bots":{"GPTBot":["partial","*",16,0],"ChatGPT-User":["partial","*",16,0],"OAI-SearchBot":["partial","*",16,0],"ClaudeBot":["partial","*",16,0],"Claude-User":["partial","*",16,0],"Claude-SearchBot":["partial","*",16,0],"anthropic-ai":["partial","*",16,0],"PerplexityBot":["partial","*",16,0],"Perplexity-User":["partial","*",16,0],"Google-Extended":["partial","*",16,0],"Applebot-Extended":["partial","*",16,0],"CCBot":["partial","*",16,0],"Bytespider":["partial","*",16,0],"Amazonbot":["partial","*",16,0],"meta-externalagent":["partial","*",16,0],"cohere-ai":["partial","*",16,0],"Diffbot":["partial","*",16,0],"ContextFetchBot":["partial","*",16,0]},"llms":{"status":"missing","http":404,"bytes":176997},"home":{"status":"http_error","http":403,"url":"https://huffingtonpost.es/"}},
{"rank":8476,"domain":"deregen.com","origin":"https://deregen.com","robots":{"status":"ok","http":200,"url":"https://deregen.com/robots.txt","bytes":313,"groups":1},"bots":{"GPTBot":["partial","*",12,1],"ChatGPT-User":["partial","*",12,1],"OAI-SearchBot":["partial","*",12,1],"ClaudeBot":["partial","*",12,1],"Claude-User":["partial","*",12,1],"Claude-SearchBot":["partial","*",12,1],"anthropic-ai":["partial","*",12,1],"PerplexityBot":["partial","*",12,1],"Perplexity-User":["partial","*",12,1],"Google-Extended":["partial","*",12,1],"Applebot-Extended":["partial","*",12,1],"CCBot":["partial","*",12,1],"Bytespider":["partial","*",12,1],"Amazonbot":["partial","*",12,1],"meta-externalagent":["partial","*",12,1],"cohere-ai":["partial","*",12,1],"Diffbot":["partial","*",12,1],"ContextFetchBot":["partial","*",12,1]},"llms":{"status":"missing","http":404,"bytes":1163},"home":{"status":"ok","http":200,"url":"https://deregen.com/","html_tokens":39201,"markdown_tokens":74,"pruned_tokens":74}},
{"rank":8477,"domain":"resmigazete.gov.tr","origin":"https://resmigazete.gov.tr","robots":{"status":"timeout","http":null,"url":"https://resmigazete.gov.tr/robots.txt","bytes":0,"groups":0,"error":"timed out after 10 s"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":8478,"domain":"rikunabi.com","origin":"https://www.rikunabi.com","robots":{"status":"html","http":200,"url":"https://job.rikunabi.com/?mode=intern","bytes":322218,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":8479,"domain":"eveonline.com","origin":"https://eveonline.com","robots":{"status":"ok","http":200,"url":"https://www.eveonline.com/robots.txt","bytes":70,"groups":1},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"missing","http":404,"bytes":10487},"home":{"status":"ok","http":200,"url":"https://www.eveonline.com/","html_tokens":25855,"markdown_tokens":13,"pruned_tokens":13}},
{"rank":8480,"domain":"bne.es","origin":"https://bne.es","robots":{"status":"ok","http":200,"url":"https://www.bne.es/robots.txt","bytes":2027,"groups":1},"bots":{"GPTBot":["partial","*",34,18],"ChatGPT-User":["partial","*",34,18],"OAI-SearchBot":["partial","*",34,18],"ClaudeBot":["partial","*",34,18],"Claude-User":["partial","*",34,18],"Claude-SearchBot":["partial","*",34,18],"anthropic-ai":["partial","*",34,18],"PerplexityBot":["partial","*",34,18],"Perplexity-User":["partial","*",34,18],"Google-Extended":["partial","*",34,18],"Applebot-Extended":["partial","*",34,18],"CCBot":["partial","*",34,18],"Bytespider":["partial","*",34,18],"Amazonbot":["partial","*",34,18],"meta-externalagent":["partial","*",34,18],"cohere-ai":["partial","*",34,18],"Diffbot":["partial","*",34,18],"ContextFetchBot":["partial","*",34,18]},"llms":{"status":"missing","http":404,"bytes":79702},"home":{"status":"http_error","http":403,"url":"https://www.bne.es/"}},
{"rank":8481,"domain":"clicrbs.com.br","origin":"https://clicrbs.com.br","robots":{"status":"ok","http":200,"url":"https://clicrbs.com.br/robots.txt","bytes":942,"groups":26},"bots":{"GPTBot":["allowed","*",0,1],"ChatGPT-User":["allowed","*",0,1],"OAI-SearchBot":["allowed","*",0,1],"ClaudeBot":["blocked","named",1,0],"Claude-User":["blocked","named",1,0],"Claude-SearchBot":["blocked","named",1,0],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["blocked","named",1,0],"Google-Extended":["allowed","named",0,1],"Applebot-Extended":["allowed","*",0,1],"CCBot":["allowed","*",0,1],"Bytespider":["allowed","*",0,1],"Amazonbot":["allowed","*",0,1],"meta-externalagent":["allowed","*",0,1],"cohere-ai":["blocked","named",1,0],"Diffbot":["allowed","*",0,1],"ContextFetchBot":["allowed","*",0,1]},"llms":{"status":"timeout","error":"timed out after 10 s"},"home":{"status":"ok","http":200,"url":"https://clicrbs.com.br/","html_tokens":105919,"markdown_tokens":25,"pruned_tokens":25}},
{"rank":8482,"domain":"webador.com","origin":"https://webador.com","robots":{"status":"ok","http":200,"url":"https://www.webador.com/robots.txt","bytes":88,"groups":1},"bots":{"GPTBot":["partial","*",1,0],"ChatGPT-User":["partial","*",1,0],"OAI-SearchBot":["partial","*",1,0],"ClaudeBot":["partial","*",1,0],"Claude-User":["partial","*",1,0],"Claude-SearchBot":["partial","*",1,0],"anthropic-ai":["partial","*",1,0],"PerplexityBot":["partial","*",1,0],"Perplexity-User":["partial","*",1,0],"Google-Extended":["partial","*",1,0],"Applebot-Extended":["partial","*",1,0],"CCBot":["partial","*",1,0],"Bytespider":["partial","*",1,0],"Amazonbot":["partial","*",1,0],"meta-externalagent":["partial","*",1,0],"cohere-ai":["partial","*",1,0],"Diffbot":["partial","*",1,0],"ContextFetchBot":["partial","*",1,0]},"llms":{"status":"present","http":200,"bytes":1620},"home":{"status":"ok","http":200,"url":"https://www.webador.com/","html_tokens":33653,"markdown_tokens":545,"pruned_tokens":169}},
{"rank":8483,"domain":"eurovision.tv","origin":"https://eurovision.tv","robots":{"status":"refused","http":403,"url":"https://eurovision.tv/robots.txt","bytes":4907,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":8484,"domain":"cenforce.online","origin":"https://cenforce.online","robots":{"status":"ok","http":200,"url":"https://cenforce.online/robots.txt","bytes":110,"groups":1},"bots":{"GPTBot":["partial","*",3,0],"ChatGPT-User":["partial","*",3,0],"OAI-SearchBot":["partial","*",3,0],"ClaudeBot":["partial","*",3,0],"Claude-User":["partial","*",3,0],"Claude-SearchBot":["partial","*",3,0],"anthropic-ai":["partial","*",3,0],"PerplexityBot":["partial","*",3,0],"Perplexity-User":["partial","*",3,0],"Google-Extended":["partial","*",3,0],"Applebot-Extended":["partial","*",3,0],"CCBot":["partial","*",3,0],"Bytespider":["partial","*",3,0],"Amazonbot":["partial","*",3,0],"meta-externalagent":["partial","*",3,0],"cohere-ai":["partial","*",3,0],"Diffbot":["partial","*",3,0],"ContextFetchBot":["partial","*",3,0]},"llms":{"status":"missing","http":404,"bytes":644},"home":{"status":"ok","http":200,"url":"https://cenforce.online/","html_tokens":9335,"markdown_tokens":2353,"pruned_tokens":519}},
{"rank":8485,"domain":"xnfhtech.com","origin":"http://xnfhtech.com","robots":{"status":"missing","http":404,"url":"http://xnfhtech.com/robots.txt","bytes":1163,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":1163},"home":{"status":"ok","http":200,"url":"http://xnfhtech.com/","html_tokens":4545,"markdown_tokens":44,"pruned_tokens":44}},
{"rank":8486,"domain":"matadornetwork.com","origin":"https://matadornetwork.com","robots":{"status":"ok","http":200,"url":"https://matadornetwork.com/robots.txt","bytes":430,"groups":1},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"missing","http":404,"bytes":75010},"home":{"status":"ok","http":200,"url":"https://matadornetwork.com/","html_tokens":105014,"markdown_tokens":295,"pruned_tokens":118}},
{"rank":8487,"domain":"ots.at","origin":"https://ots.at","robots":{"status":"ok","http":200,"url":"https://www.ots.at/robots.txt","bytes":2118,"groups":1},"bots":{"GPTBot":["partial","*",5,0],"ChatGPT-User":["partial","*",5,0],"OAI-SearchBot":["partial","*",5,0],"ClaudeBot":["partial","*",5,0],"Claude-User":["partial","*",5,0],"Claude-SearchBot":["partial","*",5,0],"anthropic-ai":["partial","*",5,0],"PerplexityBot":["partial","*",5,0],"Perplexity-User":["partial","*",5,0],"Google-Extended":["partial","*",5,0],"Applebot-Extended":["partial","*",5,0],"CCBot":["partial","*",5,0],"Bytespider":["partial","*",5,0],"Amazonbot":["partial","*",5,0],"meta-externalagent":["partial","*",5,0],"cohere-ai":["partial","*",5,0],"Diffbot":["partial","*",5,0],"ContextFetchBot":["partial","*",5,0]},"llms":{"status":"missing","http":404,"bytes":170296},"home":{"status":"http_error","http":403,"url":"https://ots.at/"}},
{"rank":8488,"domain":"rcsb.org","origin":"https://rcsb.org","robots":{"status":"ok","http":200,"url":"https://rcsb.org/robots.txt","bytes":381,"groups":1},"bots":{"GPTBot":["partial","*",6,8],"ChatGPT-User":["partial","*",6,8],"OAI-SearchBot":["partial","*",6,8],"ClaudeBot":["partial","*",6,8],"Claude-User":["partial","*",6,8],"Claude-SearchBot":["partial","*",6,8],"anthropic-ai":["partial","*",6,8],"PerplexityBot":["partial","*",6,8],"Perplexity-User":["partial","*",6,8],"Google-Extended":["partial","*",6,8],"Applebot-Extended":["partial","*",6,8],"CCBot":["partial","*",6,8],"Bytespider":["partial","*",6,8],"Amazonbot":["partial","*",6,8],"meta-externalagent":["partial","*",6,8],"cohere-ai":["partial","*",6,8],"Diffbot":["partial","*",6,8],"ContextFetchBot":["partial","*",6,8]},"llms":{"status":"missing","http":404,"bytes":40706},"home":{"status":"ok","http":200,"url":"https://rcsb.org/","html_tokens":21006,"markdown_tokens":82,"pruned_tokens":82}},
{"rank":8489,"domain":"orbitz.com","origin":"https://orbitz.com","robots":{"status":"ok","http":200,"url":"https://www.orbitz.com/robots.txt","bytes":2046,"groups":4},"bots":{"GPTBot":["partial","*",51,0],"ChatGPT-User":["partial","named",5,1],"OAI-SearchBot":["partial","named",5,1],"ClaudeBot":["partial","*",51,0],"Claude-User":["partial","named",5,1],"Claude-SearchBot":["partial","named",5,1],"anthropic-ai":["partial","*",51,0],"PerplexityBot":["partial","named",5,1],"Perplexity-User":["partial","named",5,1],"Google-Extended":["partial","*",51,0],"Applebot-Extended":["partial","*",51,0],"CCBot":["partial","*",51,0],"Bytespider":["partial","*",51,0],"Amazonbot":["partial","*",51,0],"meta-externalagent":["partial","*",51,0],"cohere-ai":["partial","*",51,0],"Diffbot":["partial","*",51,0],"ContextFetchBot":["partial","*",51,0]},"llms":{"status":"http_error","http":403,"bytes":243},"home":{"status":"http_error","http":429,"url":"https://www.orbitz.com/"}},
{"rank":8490,"domain":"gengxinhuandai.com","origin":"https://gengxinhuandai.com","robots":{"status":"unreachable","http":null,"url":"https://gengxinhuandai.com/robots.txt","bytes":0,"groups":0,"error":"cannot resolve gengxinhuandai.com"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":8491,"domain":"saitama.jp","origin":"https://saitama.jp","robots":{"status":"unreachable","http":null,"url":"https://saitama.jp/robots.txt","bytes":0,"groups":0,"error":"cannot resolve saitama.jp"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":8492,"domain":"spectrumnews1.com","origin":"https://spectrumnews1.com","robots":{"status":"ok","http":200,"url":"https://spectrumnews1.com/robots.txt","bytes":1785,"groups":24},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["partial","*",24,12],"OAI-SearchBot":["partial","*",24,12],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",24,12],"Claude-SearchBot":["partial","*",24,12],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["partial","*",24,12],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",24,12]},"llms":{"status":"missing","http":404,"bytes":34815},"home":{"status":"ok","http":200,"url":"https://spectrumnews1.com/","html_tokens":2988,"markdown_tokens":0,"pruned_tokens":0}},
{"rank":8493,"domain":"autobild.de","origin":"https://autobild.de","robots":{"status":"ok","http":200,"url":"https://www.autobild.de/robots.txt","bytes":2857,"groups":2},"bots":{"GPTBot":["partial","*",26,0],"ChatGPT-User":["partial","*",26,0],"OAI-SearchBot":["partial","*",26,0],"ClaudeBot":["partial","*",26,0],"Claude-User":["partial","*",26,0],"Claude-SearchBot":["partial","*",26,0],"anthropic-ai":["partial","*",26,0],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["blocked","named",1,0],"Google-Extended":["partial","*",26,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["partial","*",26,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",26,0]},"llms":{"status":"http_error","http":403,"bytes":381},"home":{"status":"ok","http":200,"url":"https://www.autobild.de/","html_tokens":147510,"markdown_tokens":77,"pruned_tokens":77}},
{"rank":8494,"domain":"apsense.com","origin":"https://apsense.com","robots":{"status":"ok","http":200,"url":"https://www.apsense.com/robots.txt","bytes":363,"groups":4},"bots":{"GPTBot":["partial","*",3,0],"ChatGPT-User":["partial","*",3,0],"OAI-SearchBot":["partial","*",3,0],"ClaudeBot":["partial","*",3,0],"Claude-User":["partial","*",3,0],"Claude-SearchBot":["partial","*",3,0],"anthropic-ai":["partial","*",3,0],"PerplexityBot":["partial","*",3,0],"Perplexity-User":["partial","*",3,0],"Google-Extended":["partial","*",3,0],"Applebot-Extended":["partial","*",3,0],"CCBot":["partial","*",3,0],"Bytespider":["partial","*",3,0],"Amazonbot":["partial","*",3,0],"meta-externalagent":["partial","*",3,0],"cohere-ai":["partial","*",3,0],"Diffbot":["partial","*",3,0],"ContextFetchBot":["partial","*",3,0]},"llms":{"status":"missing","http":404,"bytes":1823},"home":{"status":"http_error","http":403,"url":"https://apsense.com/"}},
{"rank":8495,"domain":"ftdichip.com","origin":"https://ftdichip.com","robots":{"status":"refused","http":403,"url":"https://ftdichip.com/robots.txt","bytes":5400,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":8496,"domain":"chinadxscycp.org","origin":"http://chinadxscycp.org","robots":{"status":"missing","http":404,"url":"http://chinadxscycp.org/robots.txt","bytes":1163,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":1163},"home":{"status":"ok","http":200,"url":"http://chinadxscycp.org/","html_tokens":80643,"markdown_tokens":508,"pruned_tokens":145}},
{"rank":8497,"domain":"newamerica.org","origin":"https://newamerica.org","robots":{"status":"ok","http":200,"url":"https://www.newamerica.org/robots.txt","bytes":176,"groups":1},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"other_text","http":200,"bytes":6661},"home":{"status":"ok","http":200,"url":"https://www.newamerica.org/","html_tokens":27648,"markdown_tokens":155,"pruned_tokens":155}},
{"rank":8498,"domain":"vans.com","origin":"https://vans.com","robots":{"status":"refused","http":403,"url":"https://vans.com/robots.txt","bytes":370,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":8499,"domain":"metronidazoleinfo.com","origin":"https://metronidazoleinfo.com","robots":{"status":"ok","http":200,"url":"https://metronidazoleinfo.com/robots.txt","bytes":84,"groups":1},"bots":{"GPTBot":["partial","*",1,0],"ChatGPT-User":["partial","*",1,0],"OAI-SearchBot":["partial","*",1,0],"ClaudeBot":["partial","*",1,0],"Claude-User":["partial","*",1,0],"Claude-SearchBot":["partial","*",1,0],"anthropic-ai":["partial","*",1,0],"PerplexityBot":["partial","*",1,0],"Perplexity-User":["partial","*",1,0],"Google-Extended":["partial","*",1,0],"Applebot-Extended":["partial","*",1,0],"CCBot":["partial","*",1,0],"Bytespider":["partial","*",1,0],"Amazonbot":["partial","*",1,0],"meta-externalagent":["partial","*",1,0],"cohere-ai":["partial","*",1,0],"Diffbot":["partial","*",1,0],"ContextFetchBot":["partial","*",1,0]},"llms":{"status":"missing","http":404,"bytes":650},"home":{"status":"ok","http":200,"url":"https://metronidazoleinfo.com/","html_tokens":3230,"markdown_tokens":469,"pruned_tokens":284}},
{"rank":8500,"domain":"barackobama.com","origin":"https://barackobama.com","robots":{"status":"ok","http":200,"url":"https://barackobama.com/robots.txt","bytes":78,"groups":1},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"missing","http":404,"bytes":10004},"home":{"status":"ok","http":200,"url":"https://barackobama.com/","html_tokens":2642,"markdown_tokens":18,"pruned_tokens":18}},
{"rank":8501,"domain":"vjing.com.cn","origin":"http://vjing.com.cn","robots":{"status":"missing","http":404,"url":"http://vjing.com.cn/robots.txt","bytes":315,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":315},"home":{"status":"http_error","http":404,"url":"http://vjing.com.cn/"}},
{"rank":8502,"domain":"privateinternetaccess.com","origin":"https://privateinternetaccess.com","robots":{"status":"missing","http":404,"url":"https://privateinternetaccess.com/robots.txt","bytes":1,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":56276},"home":{"status":"ok","http":200,"url":"https://www.privateinternetaccess.com/","html_tokens":69605,"markdown_tokens":3268,"pruned_tokens":79}},
{"rank":8503,"domain":"ajmc.com","origin":"https://ajmc.com","robots":{"status":"ok","http":200,"url":"https://www.ajmc.com/robots.txt","bytes":208,"groups":1},"bots":{"GPTBot":["partial","*",3,0],"ChatGPT-User":["partial","*",3,0],"OAI-SearchBot":["partial","*",3,0],"ClaudeBot":["partial","*",3,0],"Claude-User":["partial","*",3,0],"Claude-SearchBot":["partial","*",3,0],"anthropic-ai":["partial","*",3,0],"PerplexityBot":["partial","*",3,0],"Perplexity-User":["partial","*",3,0],"Google-Extended":["partial","*",3,0],"Applebot-Extended":["partial","*",3,0],"CCBot":["partial","*",3,0],"Bytespider":["partial","*",3,0],"Amazonbot":["partial","*",3,0],"meta-externalagent":["partial","*",3,0],"cohere-ai":["partial","*",3,0],"Diffbot":["partial","*",3,0],"ContextFetchBot":["partial","*",3,0]},"llms":{"status":"present","http":200,"bytes":7248},"home":{"status":"ok","http":200,"url":"https://www.ajmc.com/","html_tokens":183080,"markdown_tokens":260,"pruned_tokens":108}},
{"rank":8504,"domain":"epochtimes.com","origin":"https://epochtimes.com","robots":{"status":"ok","http":200,"url":"https://www.epochtimes.com/robots.txt","bytes":8230,"groups":10},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["allowed","named",0,1],"OAI-SearchBot":["partial","named",1,173],"ClaudeBot":["blocked","named",1,0],"Claude-User":["allowed","named",0,1],"Claude-SearchBot":["partial","named",1,173],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["partial","named",1,173],"Perplexity-User":["allowed","named",0,1],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["allowed","named",0,1],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",31,0]},"llms":{"status":"missing","http":404,"bytes":162},"home":{"status":"ok","http":200,"url":"https://www.epochtimes.com/","html_tokens":33873,"markdown_tokens":69,"pruned_tokens":69}},
{"rank":8505,"domain":"dgtcjn.com","origin":"https://dgtcjn.com","robots":{"status":"unreachable","http":null,"url":"https://dgtcjn.com/robots.txt","bytes":0,"groups":0,"error":"[SSL: TLSV1_ALERT_INTERNAL_ERROR] tlsv1 alert internal error (_ssl.c:1028)"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":8506,"domain":"feb.ltd","origin":"http://feb.ltd","robots":{"status":"missing","http":404,"url":"http://feb.ltd/robots.txt","bytes":1245,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":1245},"home":{"status":"ok","http":200,"url":"http://feb.ltd/","html_tokens":3775,"markdown_tokens":283,"pruned_tokens":72}},
{"rank":8507,"domain":"knowbe4.com","origin":"https://knowbe4.com","robots":{"status":"ok","http":200,"url":"https://www.knowbe4.com/robots.txt","bytes":523,"groups":1},"bots":{"GPTBot":["partial","*",16,0],"ChatGPT-User":["partial","*",16,0],"OAI-SearchBot":["partial","*",16,0],"ClaudeBot":["partial","*",16,0],"Claude-User":["partial","*",16,0],"Claude-SearchBot":["partial","*",16,0],"anthropic-ai":["partial","*",16,0],"PerplexityBot":["partial","*",16,0],"Perplexity-User":["partial","*",16,0],"Google-Extended":["partial","*",16,0],"Applebot-Extended":["partial","*",16,0],"CCBot":["partial","*",16,0],"Bytespider":["partial","*",16,0],"Amazonbot":["partial","*",16,0],"meta-externalagent":["partial","*",16,0],"cohere-ai":["partial","*",16,0],"Diffbot":["partial","*",16,0],"ContextFetchBot":["partial","*",16,0]},"llms":{"status":"missing","http":404,"bytes":157124},"home":{"status":"ok","http":200,"url":"https://www.knowbe4.com/","html_tokens":49495,"markdown_tokens":965,"pruned_tokens":170}},
{"rank":8508,"domain":"nongrenzhaipin.com","origin":"http://nongrenzhaipin.com","robots":{"status":"refused","http":403,"url":"http://nongrenzhaipin.com/robots.txt","bytes":0,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":8509,"domain":"gyxrmyy.cn","origin":"https://gyxrmyy.cn","robots":{"status":"timeout","http":null,"url":"https://gyxrmyy.cn/robots.txt","bytes":0,"groups":0,"error":"ConnectTimeout"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":8510,"domain":"metr.org","origin":"https://metr.org","robots":{"status":"ok","http":200,"url":"https://metr.org/robots.txt","bytes":207,"groups":1},"bots":{"GPTBot":["partial","*",5,0],"ChatGPT-User":["partial","*",5,0],"OAI-SearchBot":["partial","*",5,0],"ClaudeBot":["partial","*",5,0],"Claude-User":["partial","*",5,0],"Claude-SearchBot":["partial","*",5,0],"anthropic-ai":["partial","*",5,0],"PerplexityBot":["partial","*",5,0],"Perplexity-User":["partial","*",5,0],"Google-Extended":["partial","*",5,0],"Applebot-Extended":["partial","*",5,0],"CCBot":["partial","*",5,0],"Bytespider":["partial","*",5,0],"Amazonbot":["partial","*",5,0],"meta-externalagent":["partial","*",5,0],"cohere-ai":["partial","*",5,0],"Diffbot":["partial","*",5,0],"ContextFetchBot":["partial","*",5,0]},"llms":{"status":"missing","http":404,"bytes":38546},"home":{"status":"ok","http":200,"url":"https://metr.org/","html_tokens":34280,"markdown_tokens":735,"pruned_tokens":44}},
{"rank":8511,"domain":"uillinois.edu","origin":"https://uillinois.edu","robots":{"status":"refused","http":403,"url":"https://uillinois.edu/robots.txt","bytes":5551,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":8512,"domain":"jumeirah.com","origin":"https://jumeirah.com","robots":{"status":"ok","http":200,"url":"https://www.jumeirah.com/robots.txt","bytes":2114,"groups":1},"bots":{"GPTBot":["partial","*",47,0],"ChatGPT-User":["partial","*",47,0],"OAI-SearchBot":["partial","*",47,0],"ClaudeBot":["partial","*",47,0],"Claude-User":["partial","*",47,0],"Claude-SearchBot":["partial","*",47,0],"anthropic-ai":["partial","*",47,0],"PerplexityBot":["partial","*",47,0],"Perplexity-User":["partial","*",47,0],"Google-Extended":["partial","*",47,0],"Applebot-Extended":["partial","*",47,0],"CCBot":["partial","*",47,0],"Bytespider":["partial","*",47,0],"Amazonbot":["partial","*",47,0],"meta-externalagent":["partial","*",47,0],"cohere-ai":["partial","*",47,0],"Diffbot":["partial","*",47,0],"ContextFetchBot":["partial","*",47,0]},"llms":{"status":"missing","http":404,"bytes":602903},"home":{"status":"ok","http":200,"url":"https://www.jumeirah.com/en","html_tokens":167769,"markdown_tokens":0,"pruned_tokens":0}},
{"rank":8513,"domain":"ia.us","origin":"https://ia.us","robots":{"status":"unreachable","http":null,"url":"https://ia.us/robots.txt","bytes":0,"groups":0,"error":"cannot resolve ia.us"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":8514,"domain":"wal.ltd","origin":"http://wal.ltd","robots":{"status":"missing","http":404,"url":"http://wal.ltd/robots.txt","bytes":1245,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":1245},"home":{"status":"ok","http":200,"url":"http://wal.ltd/","html_tokens":3775,"markdown_tokens":283,"pruned_tokens":82}},
{"rank":8515,"domain":"gamejolt.com","origin":"https://gamejolt.com","robots":{"status":"ok","http":200,"url":"https://gamejolt.com/robots.txt","bytes":328,"groups":1},"bots":{"GPTBot":["partial","*",13,0],"ChatGPT-User":["partial","*",13,0],"OAI-SearchBot":["partial","*",13,0],"ClaudeBot":["partial","*",13,0],"Claude-User":["partial","*",13,0],"Claude-SearchBot":["partial","*",13,0],"anthropic-ai":["partial","*",13,0],"PerplexityBot":["partial","*",13,0],"Perplexity-User":["partial","*",13,0],"Google-Extended":["partial","*",13,0],"Applebot-Extended":["partial","*",13,0],"CCBot":["partial","*",13,0],"Bytespider":["partial","*",13,0],"Amazonbot":["partial","*",13,0],"meta-externalagent":["partial","*",13,0],"cohere-ai":["partial","*",13,0],"Diffbot":["partial","*",13,0],"ContextFetchBot":["partial","*",13,0]},"llms":{"status":"missing","http":404,"bytes":507},"home":{"status":"ok","http":200,"url":"https://gamejolt.com/","html_tokens":3907,"markdown_tokens":0,"pruned_tokens":0}},
{"rank":8516,"domain":"openlearning.com","origin":"https://openlearning.com","robots":{"status":"missing","http":404,"url":"https://www.openlearning.com/robots.txt/","bytes":128132,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":128120},"home":{"status":"ok","http":200,"url":"https://www.openlearning.com/","html_tokens":9983,"markdown_tokens":0,"pruned_tokens":0}},
{"rank":8517,"domain":"cnstock.com","origin":"https://www.cnstock.com","robots":{"status":"missing","http":404,"url":"https://www.cnstock.com/robots.txt","bytes":150,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":150},"home":{"status":"ok","http":200,"url":"https://www.cnstock.com/","html_tokens":19237,"markdown_tokens":74,"pruned_tokens":74}},
{"rank":8518,"domain":"jchldryq.com","origin":"http://jchldryq.com","robots":{"status":"missing","http":404,"url":"http://jchldryq.com/robots.txt","bytes":4737,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":4733},"home":{"status":"ok","http":200,"url":"http://jchldryq.com/","html_tokens":5607,"markdown_tokens":112,"pruned_tokens":112}},
{"rank":8519,"domain":"royal-elementor-addons.com","origin":"https://royal-elementor-addons.com","robots":{"status":"refused","http":403,"url":"https://royal-elementor-addons.com/robots.txt","bytes":103,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":8520,"domain":"google.sh","origin":"https://google.sh","robots":{"status":"ok","http":200,"url":"https://www.google.sh/robots.txt","bytes":6624,"groups":4},"bots":{"GPTBot":["partial","*",169,64],"ChatGPT-User":["partial","*",169,64],"OAI-SearchBot":["partial","*",169,64],"ClaudeBot":["partial","*",169,64],"Claude-User":["partial","*",169,64],"Claude-SearchBot":["partial","*",169,64],"anthropic-ai":["partial","*",169,64],"PerplexityBot":["partial","*",169,64],"Perplexity-User":["partial","*",169,64],"Google-Extended":["partial","*",169,64],"Applebot-Extended":["partial","*",169,64],"CCBot":["partial","*",169,64],"Bytespider":["partial","*",169,64],"Amazonbot":["partial","*",169,64],"meta-externalagent":["partial","*",169,64],"cohere-ai":["partial","*",169,64],"Diffbot":["partial","*",169,64],"ContextFetchBot":["partial","*",169,64]},"llms":{"status":"missing","http":404,"bytes":1569},"home":{"status":"ok","http":200,"url":"https://www.google.com/","html_tokens":21446,"markdown_tokens":30,"pruned_tokens":30}},
{"rank":8521,"domain":"hut.fi","origin":"https://hut.fi","robots":{"status":"unreachable","http":null,"url":"https://hut.fi/robots.txt","bytes":0,"groups":0,"error":"cannot resolve hut.fi"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":8522,"domain":"hiwufhi.com","origin":"http://hiwufhi.com","robots":{"status":"missing","http":404,"url":"http://hiwufhi.com/robots.txt","bytes":1163,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":1163},"home":{"status":"http_error","http":403,"url":"http://hiwufhi.com/"}},
{"rank":8523,"domain":"1stdibs.com","origin":"https://1stdibs.com","robots":{"status":"timeout","http":null,"url":"https://1stdibs.com/robots.txt","bytes":0,"groups":0,"error":"timed out after 10 s"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":8524,"domain":"simdif.com","origin":"https://simdif.com","robots":{"status":"ok","http":200,"url":"https://simdif.com/robots.txt","bytes":144,"groups":1},"bots":{"GPTBot":["partial","*",1,1],"ChatGPT-User":["partial","*",1,1],"OAI-SearchBot":["partial","*",1,1],"ClaudeBot":["partial","*",1,1],"Claude-User":["partial","*",1,1],"Claude-SearchBot":["partial","*",1,1],"anthropic-ai":["partial","*",1,1],"PerplexityBot":["partial","*",1,1],"Perplexity-User":["partial","*",1,1],"Google-Extended":["partial","*",1,1],"Applebot-Extended":["partial","*",1,1],"CCBot":["partial","*",1,1],"Bytespider":["partial","*",1,1],"Amazonbot":["partial","*",1,1],"meta-externalagent":["partial","*",1,1],"cohere-ai":["partial","*",1,1],"Diffbot":["partial","*",1,1],"ContextFetchBot":["partial","*",1,1]},"llms":{"status":"present","http":200,"bytes":5463},"home":{"status":"ok","http":200,"url":"https://www.simdif.com/en/","html_tokens":16790,"markdown_tokens":1293,"pruned_tokens":265}},
{"rank":8525,"domain":"newworldencyclopedia.org","origin":"https://newworldencyclopedia.org","robots":{"status":"ok","http":200,"url":"https://www.newworldencyclopedia.org/robots.txt","bytes":588,"groups":2},"bots":{"GPTBot":["partial","*",12,2],"ChatGPT-User":["partial","*",12,2],"OAI-SearchBot":["partial","*",12,2],"ClaudeBot":["partial","*",12,2],"Claude-User":["partial","*",12,2],"Claude-SearchBot":["partial","*",12,2],"anthropic-ai":["partial","*",12,2],"PerplexityBot":["partial","*",12,2],"Perplexity-User":["partial","*",12,2],"Google-Extended":["partial","*",12,2],"Applebot-Extended":["partial","*",12,2],"CCBot":["partial","*",12,2],"Bytespider":["partial","*",12,2],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["partial","*",12,2],"Diffbot":["partial","*",12,2],"ContextFetchBot":["partial","*",12,2]},"llms":{"status":"html","http":200,"bytes":23345},"home":{"status":"ok","http":200,"url":"https://www.newworldencyclopedia.org/entry/Info:Main_Page","html_tokens":11296,"markdown_tokens":274,"pruned_tokens":155}},
{"rank":8526,"domain":"okcupid.com","origin":"https://okcupid.com","robots":{"status":"refused","http":403,"url":"https://okcupid.com/robots.txt","bytes":4544,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":8527,"domain":"seiren-sh.com","origin":"https://seiren-sh.com","robots":{"status":"unreachable","http":null,"url":"https://seiren-sh.com/robots.txt","bytes":0,"groups":0,"error":"cannot resolve seiren-sh.com"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":8528,"domain":"sitejabber.com","origin":"https://sitejabber.com","robots":{"status":"ok","http":200,"url":"https://www.smartcustomer.com/robots.txt","bytes":8136,"groups":11},"bots":{"GPTBot":["partial","*",75,95],"ChatGPT-User":["partial","*",75,95],"OAI-SearchBot":["partial","*",75,95],"ClaudeBot":["partial","*",75,95],"Claude-User":["partial","*",75,95],"Claude-SearchBot":["partial","*",75,95],"anthropic-ai":["partial","*",75,95],"PerplexityBot":["partial","*",75,95],"Perplexity-User":["partial","*",75,95],"Google-Extended":["partial","*",75,95],"Applebot-Extended":["partial","*",75,95],"CCBot":["partial","*",75,95],"Bytespider":["partial","*",75,95],"Amazonbot":["partial","*",75,95],"meta-externalagent":["partial","*",75,95],"cohere-ai":["partial","*",75,95],"Diffbot":["partial","*",75,95],"ContextFetchBot":["partial","*",75,95]},"llms":{"status":"present","http":200,"bytes":6525},"home":{"status":"ok","http":200,"url":"https://www.smartcustomer.com/","html_tokens":84523,"markdown_tokens":5743,"pruned_tokens":161}},
{"rank":8529,"domain":"renault.com","origin":"https://renault.com","robots":{"status":"ok","http":200,"url":"https://www.renaultgroup.com/robots.txt","bytes":3143,"groups":16},"bots":{"GPTBot":["partial","named",7,1],"ChatGPT-User":["partial","named",7,1],"OAI-SearchBot":["partial","named",7,1],"ClaudeBot":["partial","named",7,1],"Claude-User":["partial","*",12,1],"Claude-SearchBot":["partial","*",12,1],"anthropic-ai":["partial","named",7,1],"PerplexityBot":["partial","named",7,1],"Perplexity-User":["partial","*",12,1],"Google-Extended":["partial","named",7,1],"Applebot-Extended":["partial","named",7,1],"CCBot":["partial","named",7,1],"Bytespider":["partial","named",7,1],"Amazonbot":["partial","named",7,1],"meta-externalagent":["partial","*",12,1],"cohere-ai":["partial","*",12,1],"Diffbot":["partial","*",12,1],"ContextFetchBot":["partial","*",12,1]},"llms":{"status":"missing","http":404,"bytes":229454},"home":{"status":"ok","http":200,"url":"https://www.renaultgroup.com/","html_tokens":87051,"markdown_tokens":15,"pruned_tokens":15}},
{"rank":8530,"domain":"haberler.com","origin":"https://haberler.com","robots":{"status":"ok","http":200,"url":"https://www.haberler.com/robots.txt","bytes":1511,"groups":2},"bots":{"GPTBot":["partial","*",19,2],"ChatGPT-User":["partial","*",19,2],"OAI-SearchBot":["partial","*",19,2],"ClaudeBot":["partial","*",19,2],"Claude-User":["partial","*",19,2],"Claude-SearchBot":["partial","*",19,2],"anthropic-ai":["partial","*",19,2],"PerplexityBot":["partial","*",19,2],"Perplexity-User":["partial","*",19,2],"Google-Extended":["partial","named",19,2],"Applebot-Extended":["partial","*",19,2],"CCBot":["partial","*",19,2],"Bytespider":["partial","*",19,2],"Amazonbot":["partial","*",19,2],"meta-externalagent":["partial","*",19,2],"cohere-ai":["partial","*",19,2],"Diffbot":["partial","*",19,2],"ContextFetchBot":["partial","*",19,2]},"llms":{"status":"present","http":200,"bytes":7252},"home":{"status":"ok","http":200,"url":"https://www.haberler.com/","html_tokens":104206,"markdown_tokens":52,"pruned_tokens":52}},
{"rank":8531,"domain":"rabobank.nl","origin":"https://rabobank.nl","robots":{"status":"refused","http":403,"url":"https://rabobank.nl/robots.txt","bytes":18728,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":8532,"domain":"google.co.zw","origin":"https://google.co.zw","robots":{"status":"ok","http":200,"url":"https://www.google.co.zw/robots.txt","bytes":6624,"groups":4},"bots":{"GPTBot":["partial","*",169,64],"ChatGPT-User":["partial","*",169,64],"OAI-SearchBot":["partial","*",169,64],"ClaudeBot":["partial","*",169,64],"Claude-User":["partial","*",169,64],"Claude-SearchBot":["partial","*",169,64],"anthropic-ai":["partial","*",169,64],"PerplexityBot":["partial","*",169,64],"Perplexity-User":["partial","*",169,64],"Google-Extended":["partial","*",169,64],"Applebot-Extended":["partial","*",169,64],"CCBot":["partial","*",169,64],"Bytespider":["partial","*",169,64],"Amazonbot":["partial","*",169,64],"meta-externalagent":["partial","*",169,64],"cohere-ai":["partial","*",169,64],"Diffbot":["partial","*",169,64],"ContextFetchBot":["partial","*",169,64]},"llms":{"status":"missing","http":404,"bytes":1569},"home":{"status":"ok","http":200,"url":"https://www.google.com/","html_tokens":21282,"markdown_tokens":30,"pruned_tokens":30}},
{"rank":8533,"domain":"bundesgerichtshof.de","origin":"https://bundesgerichtshof.de","robots":{"status":"ok","http":200,"url":"https://www.bundesgerichtshof.de/robots.txt","bytes":281,"groups":1},"bots":{"GPTBot":["partial","*",3,4],"ChatGPT-User":["partial","*",3,4],"OAI-SearchBot":["partial","*",3,4],"ClaudeBot":["partial","*",3,4],"Claude-User":["partial","*",3,4],"Claude-SearchBot":["partial","*",3,4],"anthropic-ai":["partial","*",3,4],"PerplexityBot":["partial","*",3,4],"Perplexity-User":["partial","*",3,4],"Google-Extended":["partial","*",3,4],"Applebot-Extended":["partial","*",3,4],"CCBot":["partial","*",3,4],"Bytespider":["partial","*",3,4],"Amazonbot":["partial","*",3,4],"meta-externalagent":["partial","*",3,4],"cohere-ai":["partial","*",3,4],"Diffbot":["partial","*",3,4],"ContextFetchBot":["partial","*",3,4]},"llms":{"status":"missing","http":404,"bytes":29959},"home":{"status":"timeout","error":"timed out after 15 s"}},
{"rank":8534,"domain":"wellcomecollection.org","origin":"https://wellcomecollection.org","robots":{"status":"ok","http":200,"url":"https://wellcomecollection.org/robots.txt","bytes":695,"groups":17},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["partial","*",1,1],"OAI-SearchBot":["partial","*",1,1],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",1,1],"Claude-SearchBot":["partial","*",1,1],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["partial","*",1,1],"Perplexity-User":["partial","*",1,1],"Google-Extended":["partial","*",1,1],"Applebot-Extended":["blocked","named",1,0],"CCBot":["partial","*",1,1],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["partial","*",1,1],"cohere-ai":["blocked","named",1,0],"Diffbot":["partial","*",1,1],"ContextFetchBot":["partial","*",1,1]},"llms":{"status":"missing","http":404,"bytes":76789},"home":{"status":"ok","http":200,"url":"https://wellcomecollection.org/","html_tokens":54196,"markdown_tokens":51,"pruned_tokens":51}},
{"rank":8535,"domain":"sabah.com.tr","origin":"https://sabah.com.tr","robots":{"status":"ok","http":200,"url":"https://www.sabah.com.tr/robots.txt","bytes":962,"groups":1},"bots":{"GPTBot":["partial","*",28,4],"ChatGPT-User":["partial","*",28,4],"OAI-SearchBot":["partial","*",28,4],"ClaudeBot":["partial","*",28,4],"Claude-User":["partial","*",28,4],"Claude-SearchBot":["partial","*",28,4],"anthropic-ai":["partial","*",28,4],"PerplexityBot":["partial","*",28,4],"Perplexity-User":["partial","*",28,4],"Google-Extended":["partial","*",28,4],"Applebot-Extended":["partial","*",28,4],"CCBot":["partial","*",28,4],"Bytespider":["partial","*",28,4],"Amazonbot":["partial","*",28,4],"meta-externalagent":["partial","*",28,4],"cohere-ai":["partial","*",28,4],"Diffbot":["partial","*",28,4],"ContextFetchBot":["partial","*",28,4]},"llms":{"status":"other_text","http":200,"bytes":3806},"home":{"status":"ok","http":200,"url":"https://www.sabah.com.tr/","html_tokens":128760,"markdown_tokens":29,"pruned_tokens":29}},
{"rank":8536,"domain":"topsitenet.com","origin":"https://topsitenet.com","robots":{"status":"missing","http":404,"url":"https://topsitenet.com/robots.txt","bytes":5674,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":5672},"home":{"status":"ok","http":200,"url":"https://topsitenet.com/","html_tokens":3281,"markdown_tokens":642,"pruned_tokens":127}},
{"rank":8537,"domain":"gradle.org","origin":"https://gradle.org","robots":{"status":"ok","http":200,"url":"https://gradle.org/robots.txt","bytes":40,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":43276},"home":{"status":"ok","http":200,"url":"https://gradle.org/","html_tokens":16809,"markdown_tokens":68,"pruned_tokens":68}},
{"rank":8538,"domain":"yakjk.com","origin":"https://yakjk.com","robots":{"status":"missing","http":404,"url":"https://yakjk.com/robots.txt","bytes":4725,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":4721},"home":{"status":"ok","http":200,"url":"https://yakjk.com/","html_tokens":16433,"markdown_tokens":31,"pruned_tokens":28}},
{"rank":8539,"domain":"obozrevatel.com","origin":"https://obozrevatel.com","robots":{"status":"ok","http":200,"url":"https://www.obozrevatel.com/robots.txt","bytes":1052,"groups":3},"bots":{"GPTBot":["partial","*",6,12],"ChatGPT-User":["partial","*",6,12],"OAI-SearchBot":["partial","*",6,12],"ClaudeBot":["partial","*",6,12],"Claude-User":["partial","*",6,12],"Claude-SearchBot":["partial","*",6,12],"anthropic-ai":["partial","*",6,12],"PerplexityBot":["partial","*",6,12],"Perplexity-User":["partial","*",6,12],"Google-Extended":["partial","*",6,12],"Applebot-Extended":["partial","*",6,12],"CCBot":["partial","*",6,12],"Bytespider":["partial","*",6,12],"Amazonbot":["partial","*",6,12],"meta-externalagent":["partial","*",6,12],"cohere-ai":["partial","*",6,12],"Diffbot":["partial","*",6,12],"ContextFetchBot":["partial","*",6,12]},"llms":{"status":"missing","http":404,"bytes":91327},"home":{"status":"ok","http":200,"url":"https://www.obozrevatel.com/","html_tokens":55157,"markdown_tokens":25,"pruned_tokens":25}},
{"rank":8540,"domain":"sld.cu","origin":"https://sld.cu","robots":{"status":"unreachable","http":null,"url":"https://sld.cu/robots.txt","bytes":0,"groups":0,"error":"cannot resolve sld.cu"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":8541,"domain":"infosec.exchange","origin":"https://infosec.exchange","robots":{"status":"ok","http":200,"url":"https://infosec.exchange/robots.txt","bytes":231,"groups":2},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["partial","*",3,0],"OAI-SearchBot":["partial","*",3,0],"ClaudeBot":["partial","*",3,0],"Claude-User":["partial","*",3,0],"Claude-SearchBot":["partial","*",3,0],"anthropic-ai":["partial","*",3,0],"PerplexityBot":["partial","*",3,0],"Perplexity-User":["partial","*",3,0],"Google-Extended":["partial","*",3,0],"Applebot-Extended":["partial","*",3,0],"CCBot":["partial","*",3,0],"Bytespider":["partial","*",3,0],"Amazonbot":["partial","*",3,0],"meta-externalagent":["partial","*",3,0],"cohere-ai":["partial","*",3,0],"Diffbot":["partial","*",3,0],"ContextFetchBot":["partial","*",3,0]},"llms":{"status":"missing","http":404,"bytes":1828},"home":{"status":"ok","http":200,"url":"https://infosec.exchange/","html_tokens":14712,"markdown_tokens":34,"pruned_tokens":34}},
{"rank":8542,"domain":"avcweigh.com","origin":"https://avcweigh.com","robots":{"status":"unreachable","http":null,"url":"https://avcweigh.com/robots.txt","bytes":0,"groups":0,"error":"[SSL: CERTIFICATE_VERIFY_FAILED] certificate verify failed: self-signed certificate (_ssl.c:1028)"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":8543,"domain":"wfpms.com","origin":"http://wfpms.com","robots":{"status":"missing","http":404,"url":"http://wfpms.com/robots.txt","bytes":1831,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":1829},"home":{"status":"ok","http":200,"url":"http://wfpms.com/","html_tokens":11520,"markdown_tokens":165,"pruned_tokens":165}},
{"rank":8544,"domain":"nccu.edu.tw","origin":"https://nccu.edu.tw","robots":{"status":"ok","http":200,"url":"https://nccu.edu.tw/robots.txt","bytes":26,"groups":1},"bots":{"GPTBot":["blocked","*",1,0],"ChatGPT-User":["blocked","*",1,0],"OAI-SearchBot":["blocked","*",1,0],"ClaudeBot":["blocked","*",1,0],"Claude-User":["blocked","*",1,0],"Claude-SearchBot":["blocked","*",1,0],"anthropic-ai":["blocked","*",1,0],"PerplexityBot":["blocked","*",1,0],"Perplexity-User":["blocked","*",1,0],"Google-Extended":["blocked","*",1,0],"Applebot-Extended":["blocked","*",1,0],"CCBot":["blocked","*",1,0],"Bytespider":["blocked","*",1,0],"Amazonbot":["blocked","*",1,0],"meta-externalagent":["blocked","*",1,0],"cohere-ai":["blocked","*",1,0],"Diffbot":["blocked","*",1,0],"ContextFetchBot":["blocked","*",1,0]},"llms":{"status":"disallowed"},"home":{"status":"disallowed"}},
{"rank":8545,"domain":"trivago.com","origin":"https://trivago.com","robots":{"status":"ok","http":200,"url":"https://www.trivago.com/robots.txt","bytes":4794,"groups":2},"bots":{"GPTBot":["partial","*",104,1],"ChatGPT-User":["partial","*",104,1],"OAI-SearchBot":["partial","*",104,1],"ClaudeBot":["partial","named",1,13],"Claude-User":["partial","*",104,1],"Claude-SearchBot":["partial","*",104,1],"anthropic-ai":["partial","*",104,1],"PerplexityBot":["partial","*",104,1],"Perplexity-User":["partial","*",104,1],"Google-Extended":["partial","*",104,1],"Applebot-Extended":["partial","*",104,1],"CCBot":["partial","*",104,1],"Bytespider":["partial","*",104,1],"Amazonbot":["partial","*",104,1],"meta-externalagent":["partial","*",104,1],"cohere-ai":["partial","*",104,1],"Diffbot":["partial","*",104,1],"ContextFetchBot":["partial","*",104,1]},"llms":{"status":"http_error","http":403,"bytes":3825},"home":{"status":"http_error","http":403,"url":"https://trivago.com/"}},
{"rank":8546,"domain":"quicksprout.com","origin":"https://quicksprout.com","robots":{"status":"ok","http":200,"url":"https://www.quicksprout.com/robots.txt","bytes":218,"groups":1},"bots":{"GPTBot":["partial","*",9,0],"ChatGPT-User":["partial","*",9,0],"OAI-SearchBot":["partial","*",9,0],"ClaudeBot":["partial","*",9,0],"Claude-User":["partial","*",9,0],"Claude-SearchBot":["partial","*",9,0],"anthropic-ai":["partial","*",9,0],"PerplexityBot":["partial","*",9,0],"Perplexity-User":["partial","*",9,0],"Google-Extended":["partial","*",9,0],"Applebot-Extended":["partial","*",9,0],"CCBot":["partial","*",9,0],"Bytespider":["partial","*",9,0],"Amazonbot":["partial","*",9,0],"meta-externalagent":["partial","*",9,0],"cohere-ai":["partial","*",9,0],"Diffbot":["partial","*",9,0],"ContextFetchBot":["partial","*",9,0]},"llms":{"status":"http_error","http":403,"bytes":5568},"home":{"status":"http_error","http":403,"url":"https://quicksprout.com/"}},
{"rank":8547,"domain":"amherst.edu","origin":"https://amherst.edu","robots":{"status":"refused","http":405,"url":"https://www.amherst.edu/robots.txt","bytes":2535,"groups":0,"error":"HTTP 405"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":8548,"domain":"pokiesmoky.com","origin":"https://pokiesmoky.com","robots":{"status":"ok","http":200,"url":"https://pokiesmoky.com/robots.txt","bytes":533,"groups":1},"bots":{"GPTBot":["partial","*",19,0],"ChatGPT-User":["partial","*",19,0],"OAI-SearchBot":["partial","*",19,0],"ClaudeBot":["partial","*",19,0],"Claude-User":["partial","*",19,0],"Claude-SearchBot":["partial","*",19,0],"anthropic-ai":["partial","*",19,0],"PerplexityBot":["partial","*",19,0],"Perplexity-User":["partial","*",19,0],"Google-Extended":["partial","*",19,0],"Applebot-Extended":["partial","*",19,0],"CCBot":["partial","*",19,0],"Bytespider":["partial","*",19,0],"Amazonbot":["partial","*",19,0],"meta-externalagent":["partial","*",19,0],"cohere-ai":["partial","*",19,0],"Diffbot":["partial","*",19,0],"ContextFetchBot":["partial","*",19,0]},"llms":{"status":"missing","http":404,"bytes":48843},"home":{"status":"ok","http":200,"url":"https://pokiesmoky.com/","html_tokens":28785,"markdown_tokens":1930,"pruned_tokens":132}},
{"rank":8549,"domain":"360cities.net","origin":"https://360cities.net","robots":{"status":"refused","http":403,"url":"https://360cities.net/robots.txt","bytes":5551,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":8550,"domain":"cncqe.cn","origin":"https://cncqe.cn","robots":{"status":"unreachable","http":null,"url":"https://cncqe.cn/robots.txt","bytes":0,"groups":0,"error":"<StreamReset stream_id:1, error_code:1, remote_reset:True>"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":8551,"domain":"rusi.org","origin":"https://rusi.org","robots":{"status":"ok","http":200,"url":"https://www.rusi.org/robots.txt","bytes":145,"groups":1},"bots":{"GPTBot":["partial","*",3,0],"ChatGPT-User":["partial","*",3,0],"OAI-SearchBot":["partial","*",3,0],"ClaudeBot":["partial","*",3,0],"Claude-User":["partial","*",3,0],"Claude-SearchBot":["partial","*",3,0],"anthropic-ai":["partial","*",3,0],"PerplexityBot":["partial","*",3,0],"Perplexity-User":["partial","*",3,0],"Google-Extended":["partial","*",3,0],"Applebot-Extended":["partial","*",3,0],"CCBot":["partial","*",3,0],"Bytespider":["partial","*",3,0],"Amazonbot":["partial","*",3,0],"meta-externalagent":["partial","*",3,0],"cohere-ai":["partial","*",3,0],"Diffbot":["partial","*",3,0],"ContextFetchBot":["partial","*",3,0]},"llms":{"status":"missing","http":404,"bytes":601678},"home":{"status":"ok","http":200,"url":"https://www.rusi.org/","html_tokens":200929,"markdown_tokens":179,"pruned_tokens":157}},
{"rank":8552,"domain":"hirri.cn","origin":"https://hirri.cn","robots":{"status":"missing","http":404,"url":"https://hirri.cn/robots.txt","bytes":138,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":138},"home":{"status":"error","http":200,"url":"https://hirri.cn/","error":"unparseable HTML"}},
{"rank":8553,"domain":"yljxf.com","origin":"https://yljxf.com","robots":{"status":"missing","http":404,"url":"https://yljxf.com/robots.txt","bytes":315,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":315},"home":{"status":"http_error","http":404,"url":"https://yljxf.com/"}},
{"rank":8554,"domain":"fanduel.com","origin":"https://fanduel.com","robots":{"status":"ok","http":200,"url":"https://fanduel.com/robots.txt","bytes":510,"groups":1},"bots":{"GPTBot":["partial","*",13,2],"ChatGPT-User":["partial","*",13,2],"OAI-SearchBot":["partial","*",13,2],"ClaudeBot":["partial","*",13,2],"Claude-User":["partial","*",13,2],"Claude-SearchBot":["partial","*",13,2],"anthropic-ai":["partial","*",13,2],"PerplexityBot":["partial","*",13,2],"Perplexity-User":["partial","*",13,2],"Google-Extended":["partial","*",13,2],"Applebot-Extended":["partial","*",13,2],"CCBot":["partial","*",13,2],"Bytespider":["partial","*",13,2],"Amazonbot":["partial","*",13,2],"meta-externalagent":["partial","*",13,2],"cohere-ai":["partial","*",13,2],"Diffbot":["partial","*",13,2],"ContextFetchBot":["partial","*",13,2]},"llms":{"status":"present","http":200,"bytes":1305},"home":{"status":"ok","http":200,"url":"https://www.fanduel.com/","html_tokens":83020,"markdown_tokens":619,"pruned_tokens":585}},
{"rank":8555,"domain":"tk.de","origin":"https://tk.de","robots":{"status":"ok","http":200,"url":"https://www.tk.de/robots.txt","bytes":1305,"groups":1},"bots":{"GPTBot":["partial","*",4,0],"ChatGPT-User":["partial","*",4,0],"OAI-SearchBot":["partial","*",4,0],"ClaudeBot":["partial","*",4,0],"Claude-User":["partial","*",4,0],"Claude-SearchBot":["partial","*",4,0],"anthropic-ai":["partial","*",4,0],"PerplexityBot":["partial","*",4,0],"Perplexity-User":["partial","*",4,0],"Google-Extended":["partial","*",4,0],"Applebot-Extended":["partial","*",4,0],"CCBot":["partial","*",4,0],"Bytespider":["partial","*",4,0],"Amazonbot":["partial","*",4,0],"meta-externalagent":["partial","*",4,0],"cohere-ai":["partial","*",4,0],"Diffbot":["partial","*",4,0],"ContextFetchBot":["partial","*",4,0]},"llms":{"status":"missing","http":404,"bytes":2062},"home":{"status":"ok","http":200,"url":"https://www.tk.de/techniker","html_tokens":20160,"markdown_tokens":30,"pruned_tokens":30}},
{"rank":8556,"domain":"jcu.edu.au","origin":"https://jcu.edu.au","robots":{"status":"refused","http":403,"url":"https://jcu.edu.au/robots.txt","bytes":5611,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":8557,"domain":"homesandgardens.com","origin":"https://homesandgardens.com","robots":{"status":"ok","http":200,"url":"https://www.homesandgardens.com/robots.txt","bytes":1762,"groups":4},"bots":{"GPTBot":["partial","*",18,0],"ChatGPT-User":["partial","*",18,0],"OAI-SearchBot":["partial","*",18,0],"ClaudeBot":["partial","*",18,0],"Claude-User":["partial","*",18,0],"Claude-SearchBot":["partial","*",18,0],"anthropic-ai":["partial","*",18,0],"PerplexityBot":["partial","*",18,0],"Perplexity-User":["partial","*",18,0],"Google-Extended":["partial","*",18,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["partial","*",18,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["partial","*",18,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",18,0]},"llms":{"status":"missing","http":404,"bytes":1322981},"home":{"status":"ok","http":200,"url":"https://www.homesandgardens.com/","html_tokens":469072,"markdown_tokens":1400,"pruned_tokens":103}},
{"rank":8558,"domain":"namesilo.com","origin":"https://namesilo.com","robots":{"status":"refused","http":403,"url":"https://namesilo.com/robots.txt","bytes":5507,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":8559,"domain":"jp.sharp","origin":"https://jp.sharp","robots":{"status":"ok","http":200,"url":"https://jp.sharp/robots.txt","bytes":321,"groups":1},"bots":{"GPTBot":["partial","*",8,0],"ChatGPT-User":["partial","*",8,0],"OAI-SearchBot":["partial","*",8,0],"ClaudeBot":["partial","*",8,0],"Claude-User":["partial","*",8,0],"Claude-SearchBot":["partial","*",8,0],"anthropic-ai":["partial","*",8,0],"PerplexityBot":["partial","*",8,0],"Perplexity-User":["partial","*",8,0],"Google-Extended":["partial","*",8,0],"Applebot-Extended":["partial","*",8,0],"CCBot":["partial","*",8,0],"Bytespider":["partial","*",8,0],"Amazonbot":["partial","*",8,0],"meta-externalagent":["partial","*",8,0],"cohere-ai":["partial","*",8,0],"Diffbot":["partial","*",8,0],"ContextFetchBot":["partial","*",8,0]},"llms":{"status":"missing","http":404,"bytes":7009},"home":{"status":"ok","http":200,"url":"https://jp.sharp/","html_tokens":15204,"markdown_tokens":167,"pruned_tokens":167}},
{"rank":8560,"domain":"singularityhub.com","origin":"https://singularityhub.com","robots":{"status":"ok","http":200,"url":"https://singularityhub.com/robots.txt","bytes":368,"groups":2},"bots":{"GPTBot":["partial","*",9,1],"ChatGPT-User":["partial","*",9,1],"OAI-SearchBot":["partial","*",9,1],"ClaudeBot":["partial","*",9,1],"Claude-User":["partial","*",9,1],"Claude-SearchBot":["partial","*",9,1],"anthropic-ai":["partial","*",9,1],"PerplexityBot":["partial","*",9,1],"Perplexity-User":["partial","*",9,1],"Google-Extended":["partial","*",9,1],"Applebot-Extended":["partial","*",9,1],"CCBot":["partial","*",9,1],"Bytespider":["partial","*",9,1],"Amazonbot":["partial","*",9,1],"meta-externalagent":["partial","*",9,1],"cohere-ai":["partial","*",9,1],"Diffbot":["partial","*",9,1],"ContextFetchBot":["partial","*",9,1]},"llms":{"status":"missing","http":404,"bytes":259721},"home":{"status":"ok","http":200,"url":"https://singularityhub.com/","html_tokens":120769,"markdown_tokens":219,"pruned_tokens":59}},
{"rank":8561,"domain":"ul.ie","origin":"https://ul.ie","robots":{"status":"ok","http":200,"url":"https://www.ul.ie/robots.txt","bytes":2167,"groups":1},"bots":{"GPTBot":["partial","*",36,22],"ChatGPT-User":["partial","*",36,22],"OAI-SearchBot":["partial","*",36,22],"ClaudeBot":["partial","*",36,22],"Claude-User":["partial","*",36,22],"Claude-SearchBot":["partial","*",36,22],"anthropic-ai":["partial","*",36,22],"PerplexityBot":["partial","*",36,22],"Perplexity-User":["partial","*",36,22],"Google-Extended":["partial","*",36,22],"Applebot-Extended":["partial","*",36,22],"CCBot":["partial","*",36,22],"Bytespider":["partial","*",36,22],"Amazonbot":["partial","*",36,22],"meta-externalagent":["partial","*",36,22],"cohere-ai":["partial","*",36,22],"Diffbot":["partial","*",36,22],"ContextFetchBot":["partial","*",36,22]},"llms":{"status":"missing","http":404,"bytes":674},"home":{"status":"ok","http":200,"url":"https://www.ul.ie/","html_tokens":63070,"markdown_tokens":700,"pruned_tokens":73}},
{"rank":8562,"domain":"jianweifz.com","origin":"https://www.jianweifz.com","robots":{"status":"missing","http":404,"url":"https://www.jianweifz.com/robots.txt","bytes":2431,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":2431},"home":{"status":"ok","http":200,"url":"https://www.jianweifz.com/","html_tokens":15782,"markdown_tokens":265,"pruned_tokens":160}},
{"rank":8563,"domain":"financial-ombudsman.org.uk","origin":"https://financial-ombudsman.org.uk","robots":{"status":"ok","http":200,"url":"https://www.financial-ombudsman.org.uk/robots.txt","bytes":342,"groups":1},"bots":{"GPTBot":["partial","*",4,0],"ChatGPT-User":["partial","*",4,0],"OAI-SearchBot":["partial","*",4,0],"ClaudeBot":["partial","*",4,0],"Claude-User":["partial","*",4,0],"Claude-SearchBot":["partial","*",4,0],"anthropic-ai":["partial","*",4,0],"PerplexityBot":["partial","*",4,0],"Perplexity-User":["partial","*",4,0],"Google-Extended":["partial","*",4,0],"Applebot-Extended":["partial","*",4,0],"CCBot":["partial","*",4,0],"Bytespider":["partial","*",4,0],"Amazonbot":["partial","*",4,0],"meta-externalagent":["partial","*",4,0],"cohere-ai":["partial","*",4,0],"Diffbot":["partial","*",4,0],"ContextFetchBot":["partial","*",4,0]},"llms":{"status":"missing","http":404,"bytes":70072},"home":{"status":"ok","http":200,"url":"https://www.financial-ombudsman.org.uk/","html_tokens":21971,"markdown_tokens":267,"pruned_tokens":154}},
{"rank":8564,"domain":"timeweb.com","origin":"https://timeweb.com","robots":{"status":"ok","http":200,"url":"https://timeweb.com/robots.txt","bytes":932,"groups":1},"bots":{"GPTBot":["partial","*",8,10],"ChatGPT-User":["partial","*",8,10],"OAI-SearchBot":["partial","*",8,10],"ClaudeBot":["partial","*",8,10],"Claude-User":["partial","*",8,10],"Claude-SearchBot":["partial","*",8,10],"anthropic-ai":["partial","*",8,10],"PerplexityBot":["partial","*",8,10],"Perplexity-User":["partial","*",8,10],"Google-Extended":["partial","*",8,10],"Applebot-Extended":["partial","*",8,10],"CCBot":["partial","*",8,10],"Bytespider":["partial","*",8,10],"Amazonbot":["partial","*",8,10],"meta-externalagent":["partial","*",8,10],"cohere-ai":["partial","*",8,10],"Diffbot":["partial","*",8,10],"ContextFetchBot":["partial","*",8,10]},"llms":{"status":"missing","http":404,"bytes":146},"home":{"status":"ok","http":200,"url":"https://timeweb.com/","html_tokens":118408,"markdown_tokens":1567,"pruned_tokens":50}},
{"rank":8565,"domain":"coach.com","origin":"https://coach.com","robots":{"status":"ok","http":200,"url":"https://www.coach.com/robots.txt","bytes":6148,"groups":10},"bots":{"GPTBot":["partial","named",20,1],"ChatGPT-User":["partial","named",20,1],"OAI-SearchBot":["partial","named",19,1],"ClaudeBot":["partial","named",20,1],"Claude-User":["partial","named",20,1],"Claude-SearchBot":["partial","*",22,0],"anthropic-ai":["partial","*",22,0],"PerplexityBot":["partial","named",20,1],"Perplexity-User":["partial","*",22,0],"Google-Extended":["partial","named",20,1],"Applebot-Extended":["partial","*",22,0],"CCBot":["partial","*",22,0],"Bytespider":["partial","*",22,0],"Amazonbot":["partial","*",22,0],"meta-externalagent":["partial","*",22,0],"cohere-ai":["partial","*",22,0],"Diffbot":["partial","*",22,0],"ContextFetchBot":["partial","*",22,0]},"llms":{"status":"http_error","http":403,"bytes":371},"home":{"status":"http_error","http":403,"url":"https://coach.com/"}},
{"rank":8566,"domain":"pan.pl","origin":"https://pan.pl","robots":{"status":"server_error","http":500,"url":"https://pan.pl/robots.txt","bytes":39108,"groups":0,"error":"HTTP 500"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":8567,"domain":"undway.com","origin":"http://undway.com","robots":{"status":"missing","http":404,"url":"http://undway.com/robots.txt","bytes":1867,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":1867},"home":{"status":"ok","http":200,"url":"http://undway.com/","html_tokens":5577,"markdown_tokens":247,"pruned_tokens":124}},
{"rank":8568,"domain":"google.com.fj","origin":"https://google.com.fj","robots":{"status":"ok","http":200,"url":"https://www.google.com.fj/robots.txt","bytes":6624,"groups":4},"bots":{"GPTBot":["partial","*",169,64],"ChatGPT-User":["partial","*",169,64],"OAI-SearchBot":["partial","*",169,64],"ClaudeBot":["partial","*",169,64],"Claude-User":["partial","*",169,64],"Claude-SearchBot":["partial","*",169,64],"anthropic-ai":["partial","*",169,64],"PerplexityBot":["partial","*",169,64],"Perplexity-User":["partial","*",169,64],"Google-Extended":["partial","*",169,64],"Applebot-Extended":["partial","*",169,64],"CCBot":["partial","*",169,64],"Bytespider":["partial","*",169,64],"Amazonbot":["partial","*",169,64],"meta-externalagent":["partial","*",169,64],"cohere-ai":["partial","*",169,64],"Diffbot":["partial","*",169,64],"ContextFetchBot":["partial","*",169,64]},"llms":{"status":"missing","http":404,"bytes":1569},"home":{"status":"ok","http":200,"url":"https://www.google.com/","html_tokens":21278,"markdown_tokens":30,"pruned_tokens":30}},
{"rank":8569,"domain":"google.com.bh","origin":"https://google.com.bh","robots":{"status":"ok","http":200,"url":"https://www.google.com.bh/robots.txt","bytes":6624,"groups":4},"bots":{"GPTBot":["partial","*",169,64],"ChatGPT-User":["partial","*",169,64],"OAI-SearchBot":["partial","*",169,64],"ClaudeBot":["partial","*",169,64],"Claude-User":["partial","*",169,64],"Claude-SearchBot":["partial","*",169,64],"anthropic-ai":["partial","*",169,64],"PerplexityBot":["partial","*",169,64],"Perplexity-User":["partial","*",169,64],"Google-Extended":["partial","*",169,64],"Applebot-Extended":["partial","*",169,64],"CCBot":["partial","*",169,64],"Bytespider":["partial","*",169,64],"Amazonbot":["partial","*",169,64],"meta-externalagent":["partial","*",169,64],"cohere-ai":["partial","*",169,64],"Diffbot":["partial","*",169,64],"ContextFetchBot":["partial","*",169,64]},"llms":{"status":"missing","http":404,"bytes":1569},"home":{"status":"ok","http":200,"url":"https://www.google.com/","html_tokens":21257,"markdown_tokens":30,"pruned_tokens":30}},
{"rank":8570,"domain":"thoughtlanes.net","origin":"https://thoughtlanes.net","robots":{"status":"ok","http":200,"url":"https://thoughtlanes.net/robots.txt","bytes":128,"groups":1},"bots":{"GPTBot":["partial","*",3,0],"ChatGPT-User":["partial","*",3,0],"OAI-SearchBot":["partial","*",3,0],"ClaudeBot":["partial","*",3,0],"Claude-User":["partial","*",3,0],"Claude-SearchBot":["partial","*",3,0],"anthropic-ai":["partial","*",3,0],"PerplexityBot":["partial","*",3,0],"Perplexity-User":["partial","*",3,0],"Google-Extended":["partial","*",3,0],"Applebot-Extended":["partial","*",3,0],"CCBot":["partial","*",3,0],"Bytespider":["partial","*",3,0],"Amazonbot":["partial","*",3,0],"meta-externalagent":["partial","*",3,0],"cohere-ai":["partial","*",3,0],"Diffbot":["partial","*",3,0],"ContextFetchBot":["partial","*",3,0]},"llms":{"status":"missing","http":404,"bytes":38904},"home":{"status":"http_error","http":500,"url":"https://thoughtlanes.net/"}},
{"rank":8571,"domain":"rosbalt.ru","origin":"https://rosbalt.ru","robots":{"status":"html","http":200,"url":"https://www.rosbalt.ru/robots.txt","bytes":13602,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":8572,"domain":"zhibo8.com","origin":"https://zhibo8.com","robots":{"status":"missing","http":404,"url":"https://zhibo8.com/robots.txt","bytes":168,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":168},"home":{"status":"ok","http":200,"url":"https://zhibo8.com/","html_tokens":169671,"markdown_tokens":526,"pruned_tokens":168}},
{"rank":8573,"domain":"freedomscientific.com","origin":"https://freedomscientific.com","robots":{"status":"ok","http":200,"url":"https://www.freedomscientific.com/robots.txt","bytes":199,"groups":1},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"http_error","http":403,"bytes":5809},"home":{"status":"http_error","http":403,"url":"https://freedomscientific.com/"}},
{"rank":8574,"domain":"cas.go.jp","origin":"https://www.cas.go.jp","robots":{"status":"missing","http":404,"url":"https://www.cas.go.jp/robots.txt","bytes":6304,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":6304},"home":{"status":"ok","http":200,"url":"https://www.cas.go.jp/","html_tokens":3017,"markdown_tokens":216,"pruned_tokens":161}},
{"rank":8575,"domain":"picsart.com","origin":"https://picsart.com","robots":{"status":"ok","http":200,"url":"https://picsart.com/robots.txt","bytes":1963,"groups":2},"bots":{"GPTBot":["partial","*",55,1],"ChatGPT-User":["partial","*",55,1],"OAI-SearchBot":["partial","*",55,1],"ClaudeBot":["partial","*",55,1],"Claude-User":["partial","*",55,1],"Claude-SearchBot":["partial","*",55,1],"anthropic-ai":["partial","*",55,1],"PerplexityBot":["partial","*",55,1],"Perplexity-User":["partial","*",55,1],"Google-Extended":["partial","*",55,1],"Applebot-Extended":["partial","*",55,1],"CCBot":["partial","*",55,1],"Bytespider":["partial","*",55,1],"Amazonbot":["partial","*",55,1],"meta-externalagent":["partial","*",55,1],"cohere-ai":["partial","*",55,1],"Diffbot":["partial","*",55,1],"ContextFetchBot":["partial","*",55,1]},"llms":{"status":"present","http":200,"bytes":1670},"home":{"status":"ok","http":200,"url":"https://picsart.com/","html_tokens":752160,"markdown_tokens":288,"pruned_tokens":153}},
{"rank":8576,"domain":"dashlane.com","origin":"https://dashlane.com","robots":{"status":"ok","http":200,"url":"https://www.dashlane.com/robots.txt","bytes":2149,"groups":6},"bots":{"GPTBot":["partial","named",1,1],"ChatGPT-User":["partial","named",1,1],"OAI-SearchBot":["partial","named",1,1],"ClaudeBot":["partial","named",1,1],"Claude-User":["partial","*",1,0],"Claude-SearchBot":["partial","*",1,0],"anthropic-ai":["partial","named",1,1],"PerplexityBot":["partial","named",1,1],"Perplexity-User":["partial","*",1,0],"Google-Extended":["partial","named",1,1],"Applebot-Extended":["partial","*",1,0],"CCBot":["partial","named",1,1],"Bytespider":["partial","*",1,0],"Amazonbot":["partial","named",1,1],"meta-externalagent":["partial","named",1,1],"cohere-ai":["partial","named",1,1],"Diffbot":["partial","*",1,0],"ContextFetchBot":["partial","*",1,0]},"llms":{"status":"present","http":200,"bytes":7594},"home":{"status":"ok","http":200,"url":"https://www.dashlane.com/","html_tokens":122390,"markdown_tokens":181,"pruned_tokens":53}},
{"rank":8577,"domain":"estranky.cz","origin":"https://estranky.cz","robots":{"status":"ok","http":200,"url":"https://estranky.cz/robots.txt","bytes":98,"groups":3},"bots":{"GPTBot":["allowed","*",0,1],"ChatGPT-User":["allowed","*",0,1],"OAI-SearchBot":["allowed","*",0,1],"ClaudeBot":["allowed","*",0,1],"Claude-User":["allowed","*",0,1],"Claude-SearchBot":["allowed","*",0,1],"anthropic-ai":["allowed","*",0,1],"PerplexityBot":["allowed","*",0,1],"Perplexity-User":["allowed","*",0,1],"Google-Extended":["allowed","*",0,1],"Applebot-Extended":["allowed","*",0,1],"CCBot":["allowed","*",0,1],"Bytespider":["allowed","*",0,1],"Amazonbot":["allowed","*",0,1],"meta-externalagent":["allowed","*",0,1],"cohere-ai":["allowed","*",0,1],"Diffbot":["allowed","*",0,1],"ContextFetchBot":["allowed","*",0,1]},"llms":{"status":"missing","http":404,"bytes":17856},"home":{"status":"ok","http":200,"url":"https://www.estranky.cz/","html_tokens":8389,"markdown_tokens":247,"pruned_tokens":106}},
{"rank":8578,"domain":"anobii.com","origin":"https://anobii.com","robots":{"status":"ok","http":200,"url":"https://www.anobii.com/robots.txt","bytes":3791,"groups":6},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["partial","*",4,0],"OAI-SearchBot":["partial","*",4,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",4,0],"Claude-SearchBot":["partial","*",4,0],"anthropic-ai":["partial","*",4,0],"PerplexityBot":["partial","*",4,0],"Perplexity-User":["partial","*",4,0],"Google-Extended":["partial","*",4,0],"Applebot-Extended":["partial","*",4,0],"CCBot":["partial","*",4,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["partial","*",4,0],"Diffbot":["partial","*",4,0],"ContextFetchBot":["partial","*",4,0]},"llms":{"status":"missing","http":404,"bytes":146},"home":{"status":"ok","http":200,"url":"https://www.anobii.com/en","html_tokens":1239,"markdown_tokens":15,"pruned_tokens":15}},
{"rank":8579,"domain":"siggraph.org","origin":"https://siggraph.org","robots":{"status":"ok","http":200,"url":"https://www.siggraph.org/robots.txt","bytes":209,"groups":1},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"missing","http":404,"bytes":136547},"home":{"status":"ok","http":307,"url":"https://siggraph.org/","html_tokens":360,"markdown_tokens":23,"pruned_tokens":23}},
{"rank":8580,"domain":"survio.com","origin":"https://survio.com","robots":{"status":"ok","http":200,"url":"https://www.survio.com/robots.txt","bytes":6646,"groups":2},"bots":{"GPTBot":["partial","*",86,0],"ChatGPT-User":["partial","*",86,0],"OAI-SearchBot":["partial","named",85,0],"ClaudeBot":["partial","*",86,0],"Claude-User":["partial","*",86,0],"Claude-SearchBot":["partial","*",86,0],"anthropic-ai":["partial","*",86,0],"PerplexityBot":["partial","*",86,0],"Perplexity-User":["partial","*",86,0],"Google-Extended":["partial","*",86,0],"Applebot-Extended":["partial","*",86,0],"CCBot":["partial","*",86,0],"Bytespider":["partial","*",86,0],"Amazonbot":["partial","*",86,0],"meta-externalagent":["partial","*",86,0],"cohere-ai":["partial","*",86,0],"Diffbot":["partial","*",86,0],"ContextFetchBot":["partial","*",86,0]},"llms":{"status":"present","http":200,"bytes":41568},"home":{"status":"ok","http":200,"url":"https://www.survio.com/en/","html_tokens":21397,"markdown_tokens":1172,"pruned_tokens":640}},
{"rank":8581,"domain":"additudemag.com","origin":"https://additudemag.com","robots":{"status":"ok","http":200,"url":"https://additudemag.com/robots.txt","bytes":1713,"groups":25},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["allowed","named",0,1],"OAI-SearchBot":["allowed","named",0,1],"ClaudeBot":["blocked","named",1,0],"Claude-User":["allowed","named",0,1],"Claude-SearchBot":["allowed","named",0,1],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["allowed","named",0,1],"Perplexity-User":["allowed","named",0,1],"Google-Extended":["allowed","named",0,1],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",12,2]},"llms":{"status":"present","http":200,"bytes":6914},"home":{"status":"ok","http":200,"url":"https://additudemag.com/","html_tokens":77810,"markdown_tokens":525,"pruned_tokens":155}},
{"rank":8582,"domain":"atlascopco.com","origin":"https://atlascopco.com","robots":{"status":"ok","http":200,"url":"https://www.atlascopco.com/robots.txt","bytes":816,"groups":1},"bots":{"GPTBot":["partial","*",17,2],"ChatGPT-User":["partial","*",17,2],"OAI-SearchBot":["partial","*",17,2],"ClaudeBot":["partial","*",17,2],"Claude-User":["partial","*",17,2],"Claude-SearchBot":["partial","*",17,2],"anthropic-ai":["partial","*",17,2],"PerplexityBot":["partial","*",17,2],"Perplexity-User":["partial","*",17,2],"Google-Extended":["partial","*",17,2],"Applebot-Extended":["partial","*",17,2],"CCBot":["partial","*",17,2],"Bytespider":["partial","*",17,2],"Amazonbot":["partial","*",17,2],"meta-externalagent":["partial","*",17,2],"cohere-ai":["partial","*",17,2],"Diffbot":["partial","*",17,2],"ContextFetchBot":["partial","*",17,2]},"llms":{"status":"missing","http":404,"bytes":40748},"home":{"status":"ok","http":200,"url":"https://www.atlascopco.com/","html_tokens":36635,"markdown_tokens":63,"pruned_tokens":63}},
{"rank":8583,"domain":"bmwallpaper.com","origin":"https://bmwallpaper.com","robots":{"status":"ok","http":200,"url":"https://bmwallpaper.com/robots.txt","bytes":485,"groups":1},"bots":{"GPTBot":["partial","*",21,0],"ChatGPT-User":["partial","*",21,0],"OAI-SearchBot":["partial","*",21,0],"ClaudeBot":["partial","*",21,0],"Claude-User":["partial","*",21,0],"Claude-SearchBot":["partial","*",21,0],"anthropic-ai":["partial","*",21,0],"PerplexityBot":["partial","*",21,0],"Perplexity-User":["partial","*",21,0],"Google-Extended":["partial","*",21,0],"Applebot-Extended":["partial","*",21,0],"CCBot":["partial","*",21,0],"Bytespider":["partial","*",21,0],"Amazonbot":["partial","*",21,0],"meta-externalagent":["partial","*",21,0],"cohere-ai":["partial","*",21,0],"Diffbot":["partial","*",21,0],"ContextFetchBot":["partial","*",21,0]},"llms":{"status":"missing","http":404,"bytes":1163},"home":{"status":"ok","http":200,"url":"https://bmwallpaper.com/","html_tokens":24879,"markdown_tokens":95,"pruned_tokens":95}},
{"rank":8584,"domain":"chinazxscp.org","origin":"https://chinazxscp.org","robots":{"status":"timeout","http":null,"url":"https://chinazxscp.org/robots.txt","bytes":0,"groups":0,"error":"ConnectTimeout"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":8585,"domain":"ycharts.com","origin":"https://ycharts.com","robots":{"status":"refused","http":405,"url":"https://ycharts.com/robots.txt","bytes":2138,"groups":0,"error":"HTTP 405"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":8586,"domain":"stcn.com","origin":"https://stcn.com","robots":{"status":"missing","http":404,"url":"https://stcn.com/robots.txt","bytes":28394,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":28394},"home":{"status":"ok","http":200,"url":"https://stcn.com/","html_tokens":38768,"markdown_tokens":19,"pruned_tokens":19}},
{"rank":8587,"domain":"mplayerhq.hu","origin":"https://mplayerhq.hu","robots":{"status":"ok","http":200,"url":"https://mplayerhq.hu/robots.txt","bytes":24,"groups":1},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"missing","http":404,"bytes":196},"home":{"status":"ok","http":200,"url":"https://mplayerhq.hu/","html_tokens":150,"markdown_tokens":19,"pruned_tokens":19}},
{"rank":8588,"domain":"sz-cooleasy.com","origin":"http://sz-cooleasy.com","robots":{"status":"missing","http":404,"url":"http://sz-cooleasy.com/robots.txt","bytes":1231,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":1231},"home":{"status":"ok","http":200,"url":"http://sz-cooleasy.com/","html_tokens":4028,"markdown_tokens":3,"pruned_tokens":3}},
{"rank":8589,"domain":"nazwa.pl","origin":"https://nazwa.pl","robots":{"status":"ok","http":200,"url":"https://www.nazwa.pl/robots.txt","bytes":217,"groups":1},"bots":{"GPTBot":["partial","*",6,0],"ChatGPT-User":["partial","*",6,0],"OAI-SearchBot":["partial","*",6,0],"ClaudeBot":["partial","*",6,0],"Claude-User":["partial","*",6,0],"Claude-SearchBot":["partial","*",6,0],"anthropic-ai":["partial","*",6,0],"PerplexityBot":["partial","*",6,0],"Perplexity-User":["partial","*",6,0],"Google-Extended":["partial","*",6,0],"Applebot-Extended":["partial","*",6,0],"CCBot":["partial","*",6,0],"Bytespider":["partial","*",6,0],"Amazonbot":["partial","*",6,0],"meta-externalagent":["partial","*",6,0],"cohere-ai":["partial","*",6,0],"Diffbot":["partial","*",6,0],"ContextFetchBot":["partial","*",6,0]},"llms":{"status":"present","http":200,"bytes":3600},"home":{"status":"ok","http":200,"url":"https://www.nazwa.pl/","html_tokens":38773,"markdown_tokens":650,"pruned_tokens":243}},
{"rank":8590,"domain":"pix11.com","origin":"https://pix11.com","robots":{"status":"ok","http":200,"url":"https://pix11.com/robots.txt","bytes":2297,"groups":40},"bots":{"GPTBot":["blocked","named",2,0],"ChatGPT-User":["blocked","named",1,0],"OAI-SearchBot":["blocked","named",1,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",12,1],"Claude-SearchBot":["partial","*",12,1],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["blocked","named",2,0],"Perplexity-User":["partial","*",12,1],"Google-Extended":["blocked","named",2,0],"Applebot-Extended":["blocked","named",2,0],"CCBot":["blocked","named",2,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["partial","*",12,1],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",12,1]},"llms":{"status":"http_error","http":403,"bytes":6514},"home":{"status":"http_error","http":403,"url":"https://pix11.com/"}},
{"rank":8591,"domain":"unomaha.edu","origin":"https://unomaha.edu","robots":{"status":"ok","http":200,"url":"https://www.unomaha.edu/robots.txt","bytes":3246,"groups":1},"bots":{"GPTBot":["partial","*",54,4],"ChatGPT-User":["partial","*",54,4],"OAI-SearchBot":["partial","*",54,4],"ClaudeBot":["partial","*",54,4],"Claude-User":["partial","*",54,4],"Claude-SearchBot":["partial","*",54,4],"anthropic-ai":["partial","*",54,4],"PerplexityBot":["partial","*",54,4],"Perplexity-User":["partial","*",54,4],"Google-Extended":["partial","*",54,4],"Applebot-Extended":["partial","*",54,4],"CCBot":["partial","*",54,4],"Bytespider":["partial","*",54,4],"Amazonbot":["partial","*",54,4],"meta-externalagent":["partial","*",54,4],"cohere-ai":["partial","*",54,4],"Diffbot":["partial","*",54,4],"ContextFetchBot":["partial","*",54,4]},"llms":{"status":"missing","http":404,"bytes":69766},"home":{"status":"ok","http":200,"url":"https://www.unomaha.edu/","html_tokens":21133,"markdown_tokens":327,"pruned_tokens":72}},
{"rank":8592,"domain":"cas.org","origin":"https://cas.org","robots":{"status":"ok","http":200,"url":"https://www.cas.org/robots.txt","bytes":493,"groups":1},"bots":{"GPTBot":["partial","*",13,1],"ChatGPT-User":["partial","*",13,1],"OAI-SearchBot":["partial","*",13,1],"ClaudeBot":["partial","*",13,1],"Claude-User":["partial","*",13,1],"Claude-SearchBot":["partial","*",13,1],"anthropic-ai":["partial","*",13,1],"PerplexityBot":["partial","*",13,1],"Perplexity-User":["partial","*",13,1],"Google-Extended":["partial","*",13,1],"Applebot-Extended":["partial","*",13,1],"CCBot":["partial","*",13,1],"Bytespider":["partial","*",13,1],"Amazonbot":["partial","*",13,1],"meta-externalagent":["partial","*",13,1],"cohere-ai":["partial","*",13,1],"Diffbot":["partial","*",13,1],"ContextFetchBot":["partial","*",13,1]},"llms":{"status":"missing","http":404,"bytes":96941},"home":{"status":"ok","http":200,"url":"https://www.cas.org/","html_tokens":39036,"markdown_tokens":255,"pruned_tokens":152}},
{"rank":8593,"domain":"xiaoxingauto.com","origin":"https://xiaoxingauto.com","robots":{"status":"missing","http":404,"url":"https://xiaoxingauto.com/robots.txt","bytes":315,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":315},"home":{"status":"http_error","http":404,"url":"https://xiaoxingauto.com/"}},
{"rank":8594,"domain":"enterprise.com","origin":"https://enterprise.com","robots":{"status":"ok","http":200,"url":"https://www.enterprise.com/robots.txt","bytes":28939,"groups":25},"bots":{"GPTBot":["partial","named",26,0],"ChatGPT-User":["partial","named",26,0],"OAI-SearchBot":["partial","*",26,0],"ClaudeBot":["partial","named",26,0],"Claude-User":["partial","named",26,0],"Claude-SearchBot":["partial","named",26,0],"anthropic-ai":["partial","*",26,0],"PerplexityBot":["partial","named",26,0],"Perplexity-User":["partial","named",26,0],"Google-Extended":["partial","named",26,0],"Applebot-Extended":["partial","named",26,0],"CCBot":["partial","named",26,0],"Bytespider":["partial","named",26,0],"Amazonbot":["partial","*",26,0],"meta-externalagent":["partial","named",26,0],"cohere-ai":["partial","*",26,0],"Diffbot":["partial","*",26,0],"ContextFetchBot":["partial","*",26,0]},"llms":{"status":"present","http":200,"bytes":4100},"home":{"status":"ok","http":200,"url":"https://www.enterprise.com/en/home.html","html_tokens":177111,"markdown_tokens":32,"pruned_tokens":32}},
{"rank":8595,"domain":"accountingweb.co.uk","origin":"https://www.accountingweb.co.uk","robots":{"status":"ok","http":200,"url":"https://www.accountingweb.co.uk/robots.txt","bytes":2624,"groups":1},"bots":{"GPTBot":["partial","*",54,32],"ChatGPT-User":["partial","*",54,32],"OAI-SearchBot":["partial","*",54,32],"ClaudeBot":["partial","*",54,32],"Claude-User":["partial","*",54,32],"Claude-SearchBot":["partial","*",54,32],"anthropic-ai":["partial","*",54,32],"PerplexityBot":["partial","*",54,32],"Perplexity-User":["partial","*",54,32],"Google-Extended":["partial","*",54,32],"Applebot-Extended":["partial","*",54,32],"CCBot":["partial","*",54,32],"Bytespider":["partial","*",54,32],"Amazonbot":["partial","*",54,32],"meta-externalagent":["partial","*",54,32],"cohere-ai":["partial","*",54,32],"Diffbot":["partial","*",54,32],"ContextFetchBot":["partial","*",54,32]},"llms":{"status":"missing","http":404,"bytes":291},"home":{"status":"http_error","http":403,"url":"https://www.accountingweb.co.uk/"}},
{"rank":8596,"domain":"upatras.gr","origin":"https://upatras.gr","robots":{"status":"ok","http":200,"url":"https://upatras.gr/robots.txt","bytes":119,"groups":1},"bots":{"GPTBot":["partial","*",1,1],"ChatGPT-User":["partial","*",1,1],"OAI-SearchBot":["partial","*",1,1],"ClaudeBot":["partial","*",1,1],"Claude-User":["partial","*",1,1],"Claude-SearchBot":["partial","*",1,1],"anthropic-ai":["partial","*",1,1],"PerplexityBot":["partial","*",1,1],"Perplexity-User":["partial","*",1,1],"Google-Extended":["partial","*",1,1],"Applebot-Extended":["partial","*",1,1],"CCBot":["partial","*",1,1],"Bytespider":["partial","*",1,1],"Amazonbot":["partial","*",1,1],"meta-externalagent":["partial","*",1,1],"cohere-ai":["partial","*",1,1],"Diffbot":["partial","*",1,1],"ContextFetchBot":["partial","*",1,1]},"llms":{"status":"missing","http":404,"bytes":152387},"home":{"status":"ok","http":200,"url":"https://www.upatras.gr/","html_tokens":43812,"markdown_tokens":47,"pruned_tokens":47}},
{"rank":8597,"domain":"cnppri.com","origin":"https://cnppri.com","robots":{"status":"missing","http":404,"url":"https://cnppri.com/robots.txt","bytes":315,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":315},"home":{"status":"http_error","http":404,"url":"https://cnppri.com/"}},
{"rank":8598,"domain":"americanpregnancy.org","origin":"https://americanpregnancy.org","robots":{"status":"ok","http":200,"url":"https://americanpregnancy.org/robots.txt","bytes":577,"groups":1},"bots":{"GPTBot":["partial","*",20,1],"ChatGPT-User":["partial","*",20,1],"OAI-SearchBot":["partial","*",20,1],"ClaudeBot":["partial","*",20,1],"Claude-User":["partial","*",20,1],"Claude-SearchBot":["partial","*",20,1],"anthropic-ai":["partial","*",20,1],"PerplexityBot":["partial","*",20,1],"Perplexity-User":["partial","*",20,1],"Google-Extended":["partial","*",20,1],"Applebot-Extended":["partial","*",20,1],"CCBot":["partial","*",20,1],"Bytespider":["partial","*",20,1],"Amazonbot":["partial","*",20,1],"meta-externalagent":["partial","*",20,1],"cohere-ai":["partial","*",20,1],"Diffbot":["partial","*",20,1],"ContextFetchBot":["partial","*",20,1]},"llms":{"status":"other_text","http":200,"bytes":2586},"home":{"status":"ok","http":200,"url":"https://americanpregnancy.org/","html_tokens":293844,"markdown_tokens":702,"pruned_tokens":75}},
{"rank":8599,"domain":"bundeswehr.de","origin":"https://bundeswehr.de","robots":{"status":"ok","http":200,"url":"https://www.bundeswehr.de/robots.txt","bytes":146,"groups":1},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"missing","http":404,"bytes":1795},"home":{"status":"ok","http":200,"url":"https://www.bundeswehr.de/de/","html_tokens":41714,"markdown_tokens":260,"pruned_tokens":109}},
{"rank":8600,"domain":"localinfo.jp","origin":"https://localinfo.jp","robots":{"status":"unreachable","http":null,"url":"https://localinfo.jp/robots.txt","bytes":0,"groups":0,"error":"cannot resolve localinfo.jp"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":8601,"domain":"dfb.de","origin":"https://dfb.de","robots":{"status":"ok","http":200,"url":"https://www.dfb.de/robots.txt","bytes":24,"groups":1},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"missing","http":404,"bytes":10232},"home":{"status":"ok","http":200,"url":"https://www.dfb.de/","html_tokens":75246,"markdown_tokens":4909,"pruned_tokens":136}},
{"rank":8602,"domain":"mdwrite.net","origin":"https://mdwrite.net","robots":{"status":"ok","http":200,"url":"https://mdwrite.net/robots.txt","bytes":118,"groups":1},"bots":{"GPTBot":["partial","*",3,0],"ChatGPT-User":["partial","*",3,0],"OAI-SearchBot":["partial","*",3,0],"ClaudeBot":["partial","*",3,0],"Claude-User":["partial","*",3,0],"Claude-SearchBot":["partial","*",3,0],"anthropic-ai":["partial","*",3,0],"PerplexityBot":["partial","*",3,0],"Perplexity-User":["partial","*",3,0],"Google-Extended":["partial","*",3,0],"Applebot-Extended":["partial","*",3,0],"CCBot":["partial","*",3,0],"Bytespider":["partial","*",3,0],"Amazonbot":["partial","*",3,0],"meta-externalagent":["partial","*",3,0],"cohere-ai":["partial","*",3,0],"Diffbot":["partial","*",3,0],"ContextFetchBot":["partial","*",3,0]},"llms":{"status":"missing","http":404,"bytes":38297},"home":{"status":"http_error","http":500,"url":"https://mdwrite.net/"}},
{"rank":8603,"domain":"pucp.edu.pe","origin":"https://www.pucp.edu.pe","robots":{"status":"ok","http":200,"url":"https://www.pucp.edu.pe/robots.txt","bytes":173,"groups":1},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"empty","http":200,"bytes":0},"home":{"status":"ok","http":200,"url":"https://www.pucp.edu.pe/","html_tokens":29226,"markdown_tokens":66,"pruned_tokens":66}},
{"rank":8604,"domain":"allbusiness.com","origin":"https://allbusiness.com","robots":{"status":"ok","http":200,"url":"https://www.allbusiness.com/robots.txt","bytes":739,"groups":16},"bots":{"GPTBot":["partial","*",4,0],"ChatGPT-User":["partial","*",4,0],"OAI-SearchBot":["partial","*",4,0],"ClaudeBot":["partial","*",4,0],"Claude-User":["partial","*",4,0],"Claude-SearchBot":["partial","*",4,0],"anthropic-ai":["partial","*",4,0],"PerplexityBot":["partial","*",4,0],"Perplexity-User":["partial","*",4,0],"Google-Extended":["partial","*",4,0],"Applebot-Extended":["partial","*",4,0],"CCBot":["blocked","named",1,0],"Bytespider":["partial","*",4,0],"Amazonbot":["partial","*",4,0],"meta-externalagent":["partial","*",4,0],"cohere-ai":["partial","*",4,0],"Diffbot":["partial","*",4,0],"ContextFetchBot":["partial","*",4,0]},"llms":{"status":"missing","http":404,"bytes":150104},"home":{"status":"ok","http":200,"url":"https://www.allbusiness.com/","html_tokens":39861,"markdown_tokens":158,"pruned_tokens":113}},
{"rank":8605,"domain":"afip.gob.ar","origin":"https://www.afip.gob.ar","robots":{"status":"missing","http":404,"url":"https://www.afip.gob.ar/robots.txt","bytes":42424,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":42424},"home":{"status":"ok","http":200,"url":"https://www.afip.gob.ar/","html_tokens":69,"markdown_tokens":0,"pruned_tokens":0}},
{"rank":8606,"domain":"foodsafety.gov","origin":"https://foodsafety.gov","robots":{"status":"refused","http":403,"url":"https://www.foodsafety.gov/robots.txt","bytes":386,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":8607,"domain":"china-embassy.gov.cn","origin":"https://china-embassy.gov.cn","robots":{"status":"unreachable","http":null,"url":"https://china-embassy.gov.cn/robots.txt","bytes":0,"groups":0,"error":"cannot resolve china-embassy.gov.cn"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":8608,"domain":"familydoctor.org","origin":"https://familydoctor.org","robots":{"status":"ok","http":200,"url":"https://familydoctor.org/robots.txt","bytes":426,"groups":1},"bots":{"GPTBot":["partial","*",9,5],"ChatGPT-User":["partial","*",9,5],"OAI-SearchBot":["partial","*",9,5],"ClaudeBot":["partial","*",9,5],"Claude-User":["partial","*",9,5],"Claude-SearchBot":["partial","*",9,5],"anthropic-ai":["partial","*",9,5],"PerplexityBot":["partial","*",9,5],"Perplexity-User":["partial","*",9,5],"Google-Extended":["partial","*",9,5],"Applebot-Extended":["partial","*",9,5],"CCBot":["partial","*",9,5],"Bytespider":["partial","*",9,5],"Amazonbot":["partial","*",9,5],"meta-externalagent":["partial","*",9,5],"cohere-ai":["partial","*",9,5],"Diffbot":["partial","*",9,5],"ContextFetchBot":["partial","*",9,5]},"llms":{"status":"http_error","http":403,"bytes":5569},"home":{"status":"http_error","http":403,"url":"https://familydoctor.org/"}},
{"rank":8609,"domain":"base-search.net","origin":"https://base-search.net","robots":{"status":"html","http":200,"url":"https://www.base-search.net","bytes":7783,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":8610,"domain":"theworld.org","origin":"https://theworld.org","robots":{"status":"ok","http":200,"url":"https://theworld.org/robots.txt","bytes":323,"groups":4},"bots":{"GPTBot":["partial","*",5,0],"ChatGPT-User":["partial","*",5,0],"OAI-SearchBot":["partial","*",5,0],"ClaudeBot":["partial","*",5,0],"Claude-User":["partial","*",5,0],"Claude-SearchBot":["partial","*",5,0],"anthropic-ai":["partial","*",5,0],"PerplexityBot":["partial","*",5,0],"Perplexity-User":["partial","*",5,0],"Google-Extended":["partial","*",5,0],"Applebot-Extended":["partial","*",5,0],"CCBot":["partial","*",5,0],"Bytespider":["partial","*",5,0],"Amazonbot":["partial","*",5,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["partial","*",5,0],"Diffbot":["partial","*",5,0],"ContextFetchBot":["partial","*",5,0]},"llms":{"status":"missing","http":404,"bytes":31181},"home":{"status":"ok","http":200,"url":"https://theworld.org/","html_tokens":1066756,"markdown_tokens":94,"pruned_tokens":94}},
{"rank":8611,"domain":"biologists.com","origin":"https://biologists.com","robots":{"status":"ok","http":200,"url":"https://www.biologists.com/robots.txt","bytes":176,"groups":1},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"missing","http":404,"bytes":139162},"home":{"status":"ok","http":200,"url":"https://www.biologists.com/","html_tokens":48833,"markdown_tokens":543,"pruned_tokens":91}},
{"rank":8612,"domain":"amazoncognito.com","origin":"https://amazoncognito.com","robots":{"status":"unreachable","http":null,"url":"https://amazoncognito.com/robots.txt","bytes":0,"groups":0,"error":"cannot resolve amazoncognito.com"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":8613,"domain":"motorolasolutions.com","origin":"https://motorolasolutions.com","robots":{"status":"ok","http":200,"url":"https://www.motorolasolutions.com/robots.txt","bytes":7404,"groups":1},"bots":{"GPTBot":["partial","*",209,1],"ChatGPT-User":["partial","*",209,1],"OAI-SearchBot":["partial","*",209,1],"ClaudeBot":["partial","*",209,1],"Claude-User":["partial","*",209,1],"Claude-SearchBot":["partial","*",209,1],"anthropic-ai":["partial","*",209,1],"PerplexityBot":["partial","*",209,1],"Perplexity-User":["partial","*",209,1],"Google-Extended":["partial","*",209,1],"Applebot-Extended":["partial","*",209,1],"CCBot":["partial","*",209,1],"Bytespider":["partial","*",209,1],"Amazonbot":["partial","*",209,1],"meta-externalagent":["partial","*",209,1],"cohere-ai":["partial","*",209,1],"Diffbot":["partial","*",209,1],"ContextFetchBot":["partial","*",209,1]},"llms":{"status":"missing","http":404,"bytes":369},"home":{"status":"ok","http":200,"url":"https://www.motorolasolutions.com/en_us.html?geo=redirect","html_tokens":369515,"markdown_tokens":2127,"pruned_tokens":103}},
{"rank":8614,"domain":"cnam.fr","origin":"https://cnam.fr","robots":{"status":"ok","http":200,"url":"https://www.cnam.fr/robots.txt","bytes":331,"groups":1},"bots":{"GPTBot":["partial","*",10,0],"ChatGPT-User":["partial","*",10,0],"OAI-SearchBot":["partial","*",10,0],"ClaudeBot":["partial","*",10,0],"Claude-User":["partial","*",10,0],"Claude-SearchBot":["partial","*",10,0],"anthropic-ai":["partial","*",10,0],"PerplexityBot":["partial","*",10,0],"Perplexity-User":["partial","*",10,0],"Google-Extended":["partial","*",10,0],"Applebot-Extended":["partial","*",10,0],"CCBot":["partial","*",10,0],"Bytespider":["partial","*",10,0],"Amazonbot":["partial","*",10,0],"meta-externalagent":["partial","*",10,0],"cohere-ai":["partial","*",10,0],"Diffbot":["partial","*",10,0],"ContextFetchBot":["partial","*",10,0]},"llms":{"status":"missing","http":404,"bytes":16248},"home":{"status":"ok","http":200,"url":"https://www.cnam.fr/","html_tokens":26488,"markdown_tokens":475,"pruned_tokens":175}},
{"rank":8615,"domain":"flockingchina.com","origin":"https://flockingchina.com","robots":{"status":"unreachable","http":null,"url":"https://flockingchina.com/robots.txt","bytes":0,"groups":0,"error":"[SSL: CERTIFICATE_VERIFY_FAILED] certificate verify failed: self-signed certificate (_ssl.c:1028)"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":8616,"domain":"sz-cxzn.com","origin":"http://sz-cxzn.com","robots":{"status":"ok","http":200,"url":"http://sz-cxzn.com/robots.txt","bytes":252,"groups":1},"bots":{"GPTBot":["partial","*",12,0],"ChatGPT-User":["partial","*",12,0],"OAI-SearchBot":["partial","*",12,0],"ClaudeBot":["partial","*",12,0],"Claude-User":["partial","*",12,0],"Claude-SearchBot":["partial","*",12,0],"anthropic-ai":["partial","*",12,0],"PerplexityBot":["partial","*",12,0],"Perplexity-User":["partial","*",12,0],"Google-Extended":["partial","*",12,0],"Applebot-Extended":["partial","*",12,0],"CCBot":["partial","*",12,0],"Bytespider":["partial","*",12,0],"Amazonbot":["partial","*",12,0],"meta-externalagent":["partial","*",12,0],"cohere-ai":["partial","*",12,0],"Diffbot":["partial","*",12,0],"ContextFetchBot":["partial","*",12,0]},"llms":{"status":"missing","http":404,"bytes":4838},"home":{"status":"ok","http":200,"url":"http://sz-cxzn.com/","html_tokens":8351,"markdown_tokens":176,"pruned_tokens":163}},
{"rank":8617,"domain":"websiteplanet.com","origin":"https://websiteplanet.com","robots":{"status":"ok","http":200,"url":"https://www.websiteplanet.com/robots.txt","bytes":586,"groups":1},"bots":{"GPTBot":["partial","*",8,13],"ChatGPT-User":["partial","*",8,13],"OAI-SearchBot":["partial","*",8,13],"ClaudeBot":["partial","*",8,13],"Claude-User":["partial","*",8,13],"Claude-SearchBot":["partial","*",8,13],"anthropic-ai":["partial","*",8,13],"PerplexityBot":["partial","*",8,13],"Perplexity-User":["partial","*",8,13],"Google-Extended":["partial","*",8,13],"Applebot-Extended":["partial","*",8,13],"CCBot":["partial","*",8,13],"Bytespider":["partial","*",8,13],"Amazonbot":["partial","*",8,13],"meta-externalagent":["partial","*",8,13],"cohere-ai":["partial","*",8,13],"Diffbot":["partial","*",8,13],"ContextFetchBot":["partial","*",8,13]},"llms":{"status":"http_error","http":403,"bytes":159},"home":{"status":"ok","http":200,"url":"https://www.websiteplanet.com/","html_tokens":56845,"markdown_tokens":619,"pruned_tokens":121}},
{"rank":8618,"domain":"fapesp.br","origin":"https://fapesp.br","robots":{"status":"ok","http":200,"url":"https://fapesp.br/robots.txt","bytes":24,"groups":1},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"html","http":200,"bytes":49818},"home":{"status":"ok","http":200,"url":"https://fapesp.br/","html_tokens":16749,"markdown_tokens":588,"pruned_tokens":7}},
{"rank":8619,"domain":"canlii.org","origin":"https://canlii.org","robots":{"status":"ok","http":200,"url":"https://www.canlii.org/robots.txt","bytes":50527,"groups":8},"bots":{"GPTBot":["partial","named",495,0],"ChatGPT-User":["blocked","*",1,0],"OAI-SearchBot":["blocked","*",1,0],"ClaudeBot":["blocked","*",1,0],"Claude-User":["blocked","*",1,0],"Claude-SearchBot":["blocked","*",1,0],"anthropic-ai":["blocked","*",1,0],"PerplexityBot":["blocked","*",1,0],"Perplexity-User":["blocked","*",1,0],"Google-Extended":["blocked","*",1,0],"Applebot-Extended":["blocked","*",1,0],"CCBot":["blocked","*",1,0],"Bytespider":["blocked","*",1,0],"Amazonbot":["blocked","*",1,0],"meta-externalagent":["blocked","*",1,0],"cohere-ai":["blocked","*",1,0],"Diffbot":["blocked","*",1,0],"ContextFetchBot":["blocked","*",1,0]},"llms":{"status":"disallowed"},"home":{"status":"disallowed"}},
{"rank":8620,"domain":"gametrailers.com","origin":"https://gametrailers.com","robots":{"status":"unreachable","http":null,"url":"https://gametrailers.com/robots.txt","bytes":0,"groups":0,"error":"cannot resolve gametrailers.com"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":8621,"domain":"gz.gov.cn","origin":"https://www.gz.gov.cn","robots":{"status":"missing","http":404,"url":"https://www.gz.gov.cn/robots.txt","bytes":146,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":146},"home":{"status":"ok","http":200,"url":"https://www.gz.gov.cn/","html_tokens":28934,"markdown_tokens":422,"pruned_tokens":175}},
{"rank":8622,"domain":"gsi.go.jp","origin":"https://gsi.go.jp","robots":{"status":"unreachable","http":null,"url":"https://gsi.go.jp/robots.txt","bytes":0,"groups":0,"error":"cannot resolve gsi.go.jp"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":8623,"domain":"fxstreet.com","origin":"https://fxstreet.com","robots":{"status":"ok","http":200,"url":"https://www.fxstreet.com/robots.txt","bytes":1966,"groups":2},"bots":{"GPTBot":["partial","*",33,0],"ChatGPT-User":["partial","*",33,0],"OAI-SearchBot":["partial","*",33,0],"ClaudeBot":["partial","*",33,0],"Claude-User":["partial","*",33,0],"Claude-SearchBot":["partial","*",33,0],"anthropic-ai":["partial","*",33,0],"PerplexityBot":["partial","*",33,0],"Perplexity-User":["partial","*",33,0],"Google-Extended":["partial","*",33,0],"Applebot-Extended":["partial","*",33,0],"CCBot":["partial","*",33,0],"Bytespider":["partial","*",33,0],"Amazonbot":["partial","*",33,0],"meta-externalagent":["partial","*",33,0],"cohere-ai":["partial","*",33,0],"Diffbot":["partial","*",33,0],"ContextFetchBot":["partial","*",33,0]},"llms":{"status":"missing","http":404,"bytes":67708},"home":{"status":"ok","http":200,"url":"https://www.fxstreet.com/","html_tokens":163535,"markdown_tokens":306,"pruned_tokens":163}},
{"rank":8624,"domain":"mea.gov.in","origin":"https://mea.gov.in","robots":{"status":"ok","http":200,"url":"https://www.mea.gov.in/robots.txt","bytes":25,"groups":1},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"html","http":200,"bytes":16981},"home":{"status":"ok","http":200,"url":"https://www.mea.gov.in/","html_tokens":20605,"markdown_tokens":276,"pruned_tokens":119}},
{"rank":8625,"domain":"hkexnews.hk","origin":"https://hkexnews.hk","robots":{"status":"refused","http":400,"url":"https://hkexnews.hk/robots.txt","bytes":357,"groups":0,"error":"HTTP 400"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":8626,"domain":"daimler.com","origin":"http://daimler.com","robots":{"status":"refused","http":403,"url":"http://daimler.com/robots.txt","bytes":9,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":8627,"domain":"pchome.net","origin":"https://pchome.net","robots":{"status":"unreachable","http":null,"url":"https://pchome.net/robots.txt","bytes":0,"groups":0,"error":"All connection attempts failed"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":8628,"domain":"purewow.com","origin":"https://purewow.com","robots":{"status":"ok","http":200,"url":"https://www.purewow.com/robots.txt","bytes":456,"groups":3},"bots":{"GPTBot":["partial","*",6,1],"ChatGPT-User":["partial","*",6,1],"OAI-SearchBot":["partial","*",6,1],"ClaudeBot":["partial","*",6,1],"Claude-User":["partial","*",6,1],"Claude-SearchBot":["partial","*",6,1],"anthropic-ai":["partial","*",6,1],"PerplexityBot":["partial","*",6,1],"Perplexity-User":["partial","*",6,1],"Google-Extended":["partial","*",6,1],"Applebot-Extended":["partial","*",6,1],"CCBot":["partial","*",6,1],"Bytespider":["partial","*",6,1],"Amazonbot":["partial","*",6,1],"meta-externalagent":["partial","*",6,1],"cohere-ai":["partial","*",6,1],"Diffbot":["partial","*",6,1],"ContextFetchBot":["partial","*",6,1]},"llms":{"status":"missing","http":404,"bytes":49523},"home":{"status":"ok","http":200,"url":"https://www.purewow.com/","html_tokens":54699,"markdown_tokens":18,"pruned_tokens":18}},
{"rank":8629,"domain":"granicus.com","origin":"https://granicus.com","robots":{"status":"refused","http":403,"url":"https://granicus.com/robots.txt","bytes":560,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":8630,"domain":"google.tm","origin":"https://google.tm","robots":{"status":"ok","http":200,"url":"https://www.google.tm/robots.txt","bytes":6624,"groups":4},"bots":{"GPTBot":["partial","*",169,64],"ChatGPT-User":["partial","*",169,64],"OAI-SearchBot":["partial","*",169,64],"ClaudeBot":["partial","*",169,64],"Claude-User":["partial","*",169,64],"Claude-SearchBot":["partial","*",169,64],"anthropic-ai":["partial","*",169,64],"PerplexityBot":["partial","*",169,64],"Perplexity-User":["partial","*",169,64],"Google-Extended":["partial","*",169,64],"Applebot-Extended":["partial","*",169,64],"CCBot":["partial","*",169,64],"Bytespider":["partial","*",169,64],"Amazonbot":["partial","*",169,64],"meta-externalagent":["partial","*",169,64],"cohere-ai":["partial","*",169,64],"Diffbot":["partial","*",169,64],"ContextFetchBot":["partial","*",169,64]},"llms":{"status":"missing","http":404,"bytes":1569},"home":{"status":"ok","http":200,"url":"https://www.google.com/","html_tokens":21454,"markdown_tokens":30,"pruned_tokens":30}},
{"rank":8631,"domain":"arborday.org","origin":"https://arborday.org","robots":{"status":"ok","http":200,"url":"https://www.arborday.org/robots.txt","bytes":2098,"groups":1},"bots":{"GPTBot":["partial","*",34,18],"ChatGPT-User":["partial","*",34,18],"OAI-SearchBot":["partial","*",34,18],"ClaudeBot":["partial","*",34,18],"Claude-User":["partial","*",34,18],"Claude-SearchBot":["partial","*",34,18],"anthropic-ai":["partial","*",34,18],"PerplexityBot":["partial","*",34,18],"Perplexity-User":["partial","*",34,18],"Google-Extended":["partial","*",34,18],"Applebot-Extended":["partial","*",34,18],"CCBot":["partial","*",34,18],"Bytespider":["partial","*",34,18],"Amazonbot":["partial","*",34,18],"meta-externalagent":["partial","*",34,18],"cohere-ai":["partial","*",34,18],"Diffbot":["partial","*",34,18],"ContextFetchBot":["partial","*",34,18]},"llms":{"status":"missing","http":404,"bytes":551},"home":{"status":"ok","http":200,"url":"https://www.arborday.org/","html_tokens":23788,"markdown_tokens":207,"pruned_tokens":128}},
{"rank":8632,"domain":"hearthis.at","origin":"https://hearthis.at","robots":{"status":"ok","http":200,"url":"https://hearthis.at/robots.txt","bytes":3226,"groups":2},"bots":{"GPTBot":["partial","*",4,0],"ChatGPT-User":["partial","*",4,0],"OAI-SearchBot":["partial","*",4,0],"ClaudeBot":["partial","*",4,0],"Claude-User":["partial","*",4,0],"Claude-SearchBot":["partial","*",4,0],"anthropic-ai":["partial","*",4,0],"PerplexityBot":["partial","*",4,0],"Perplexity-User":["partial","*",4,0],"Google-Extended":["partial","*",4,0],"Applebot-Extended":["partial","*",4,0],"CCBot":["partial","*",4,0],"Bytespider":["partial","*",4,0],"Amazonbot":["partial","*",4,0],"meta-externalagent":["partial","*",4,0],"cohere-ai":["partial","*",4,0],"Diffbot":["partial","*",4,0],"ContextFetchBot":["partial","*",4,0]},"llms":{"status":"missing","http":404,"bytes":61318},"home":{"status":"ok","http":200,"url":"https://hearthis.at/","html_tokens":73576,"markdown_tokens":828,"pruned_tokens":67}},
{"rank":8633,"domain":"kare11.com","origin":"https://kare11.com","robots":{"status":"ok","http":200,"url":"https://www.kare11.com/robots.txt","bytes":450,"groups":2},"bots":{"GPTBot":["partial","*",10,0],"ChatGPT-User":["partial","*",10,0],"OAI-SearchBot":["partial","*",10,0],"ClaudeBot":["partial","*",10,0],"Claude-User":["partial","*",10,0],"Claude-SearchBot":["partial","*",10,0],"anthropic-ai":["partial","*",10,0],"PerplexityBot":["partial","*",10,0],"Perplexity-User":["partial","*",10,0],"Google-Extended":["partial","*",10,0],"Applebot-Extended":["partial","*",10,0],"CCBot":["partial","*",10,0],"Bytespider":["partial","*",10,0],"Amazonbot":["partial","*",10,0],"meta-externalagent":["partial","*",10,0],"cohere-ai":["partial","*",10,0],"Diffbot":["partial","*",10,0],"ContextFetchBot":["partial","*",10,0]},"llms":{"status":"missing","http":404,"bytes":6845},"home":{"status":"ok","http":200,"url":"https://www.kare11.com/","html_tokens":99219,"markdown_tokens":1642,"pruned_tokens":169}},
{"rank":8634,"domain":"nsfc.gov.cn","origin":"https://nsfc.gov.cn","robots":{"status":"missing","http":404,"url":"https://nsfc.gov.cn/robots.txt","bytes":146,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":146},"home":{"status":"http_error","http":404,"url":"https://nsfc.gov.cn/"}},
{"rank":8635,"domain":"posteezy.com","origin":"https://posteezy.com","robots":{"status":"ok","http":200,"url":"https://posteezy.com/robots.txt","bytes":1706,"groups":1},"bots":{"GPTBot":["partial","*",26,18],"ChatGPT-User":["partial","*",26,18],"OAI-SearchBot":["partial","*",26,18],"ClaudeBot":["partial","*",26,18],"Claude-User":["partial","*",26,18],"Claude-SearchBot":["partial","*",26,18],"anthropic-ai":["partial","*",26,18],"PerplexityBot":["partial","*",26,18],"Perplexity-User":["partial","*",26,18],"Google-Extended":["partial","*",26,18],"Applebot-Extended":["partial","*",26,18],"CCBot":["partial","*",26,18],"Bytespider":["partial","*",26,18],"Amazonbot":["partial","*",26,18],"meta-externalagent":["partial","*",26,18],"cohere-ai":["partial","*",26,18],"Diffbot":["partial","*",26,18],"ContextFetchBot":["partial","*",26,18]},"llms":{"status":"missing","http":404,"bytes":186},"home":{"status":"ok","http":200,"url":"https://posteezy.com/","html_tokens":5251,"markdown_tokens":188,"pruned_tokens":188}},
{"rank":8636,"domain":"fastweb.com","origin":"https://fastweb.com","robots":{"status":"ok","http":200,"url":"https://www.fastweb.com/robots.txt","bytes":401,"groups":3},"bots":{"GPTBot":["partial","named",3,1],"ChatGPT-User":["partial","*",3,0],"OAI-SearchBot":["partial","*",3,0],"ClaudeBot":["partial","*",3,0],"Claude-User":["partial","*",3,0],"Claude-SearchBot":["partial","*",3,0],"anthropic-ai":["partial","*",3,0],"PerplexityBot":["partial","*",3,0],"Perplexity-User":["partial","*",3,0],"Google-Extended":["partial","*",3,0],"Applebot-Extended":["partial","*",3,0],"CCBot":["partial","*",3,0],"Bytespider":["partial","*",3,0],"Amazonbot":["partial","*",3,0],"meta-externalagent":["partial","*",3,0],"cohere-ai":["partial","*",3,0],"Diffbot":["partial","*",3,0],"ContextFetchBot":["partial","*",3,0]},"llms":{"status":"present","http":200,"bytes":2999},"home":{"status":"ok","http":200,"url":"https://www.fastweb.com/","html_tokens":18692,"markdown_tokens":765,"pruned_tokens":138}},
{"rank":8637,"domain":"jalan.net","origin":"https://jalan.net","robots":{"status":"ok","http":200,"url":"https://www.jalan.net/robots.txt","bytes":1645,"groups":1},"bots":{"GPTBot":["partial","*",21,2],"ChatGPT-User":["partial","*",21,2],"OAI-SearchBot":["partial","*",21,2],"ClaudeBot":["partial","*",21,2],"Claude-User":["partial","*",21,2],"Claude-SearchBot":["partial","*",21,2],"anthropic-ai":["partial","*",21,2],"PerplexityBot":["partial","*",21,2],"Perplexity-User":["partial","*",21,2],"Google-Extended":["partial","*",21,2],"Applebot-Extended":["partial","*",21,2],"CCBot":["partial","*",21,2],"Bytespider":["partial","*",21,2],"Amazonbot":["partial","*",21,2],"meta-externalagent":["partial","*",21,2],"cohere-ai":["partial","*",21,2],"Diffbot":["partial","*",21,2],"ContextFetchBot":["partial","*",21,2]},"llms":{"status":"html","http":200,"bytes":6741},"home":{"status":"ok","http":200,"url":"https://www.jalan.net/","html_tokens":47812,"markdown_tokens":20,"pruned_tokens":17}},
{"rank":8638,"domain":"haberturk.com","origin":"https://haberturk.com","robots":{"status":"ok","http":200,"url":"https://www.haberturk.com/robots.txt","bytes":2949,"groups":1},"bots":{"GPTBot":["partial","*",85,2],"ChatGPT-User":["partial","*",85,2],"OAI-SearchBot":["partial","*",85,2],"ClaudeBot":["partial","*",85,2],"Claude-User":["partial","*",85,2],"Claude-SearchBot":["partial","*",85,2],"anthropic-ai":["partial","*",85,2],"PerplexityBot":["partial","*",85,2],"Perplexity-User":["partial","*",85,2],"Google-Extended":["partial","*",85,2],"Applebot-Extended":["partial","*",85,2],"CCBot":["partial","*",85,2],"Bytespider":["partial","*",85,2],"Amazonbot":["partial","*",85,2],"meta-externalagent":["partial","*",85,2],"cohere-ai":["partial","*",85,2],"Diffbot":["partial","*",85,2],"ContextFetchBot":["partial","*",85,2]},"llms":{"status":"missing","http":404,"bytes":6845},"home":{"status":"ok","http":200,"url":"https://www.haberturk.com/","html_tokens":90980,"markdown_tokens":1670,"pruned_tokens":163}},
{"rank":8639,"domain":"brit.co","origin":"https://brit.co","robots":{"status":"ok","http":200,"url":"https://www.brit.co/robots.txt","bytes":255,"groups":3},"bots":{"GPTBot":["allowed","named",0,0],"ChatGPT-User":["partial","*",3,0],"OAI-SearchBot":["partial","*",3,0],"ClaudeBot":["partial","*",3,0],"Claude-User":["partial","*",3,0],"Claude-SearchBot":["partial","*",3,0],"anthropic-ai":["partial","*",3,0],"PerplexityBot":["partial","*",3,0],"Perplexity-User":["partial","*",3,0],"Google-Extended":["partial","*",3,0],"Applebot-Extended":["partial","*",3,0],"CCBot":["partial","*",3,0],"Bytespider":["partial","*",3,0],"Amazonbot":["partial","*",3,0],"meta-externalagent":["partial","*",3,0],"cohere-ai":["partial","*",3,0],"Diffbot":["partial","*",3,0],"ContextFetchBot":["partial","*",3,0]},"llms":{"status":"present","http":200,"bytes":40019},"home":{"status":"ok","http":200,"url":"https://www.brit.co/","html_tokens":82614,"markdown_tokens":20,"pruned_tokens":20}},
{"rank":8640,"domain":"lmstudio.ai","origin":"https://lmstudio.ai","robots":{"status":"ok","http":200,"url":"https://lmstudio.ai/robots.txt","bytes":64,"groups":1},"bots":{"GPTBot":["allowed","*",0,1],"ChatGPT-User":["allowed","*",0,1],"OAI-SearchBot":["allowed","*",0,1],"ClaudeBot":["allowed","*",0,1],"Claude-User":["allowed","*",0,1],"Claude-SearchBot":["allowed","*",0,1],"anthropic-ai":["allowed","*",0,1],"PerplexityBot":["allowed","*",0,1],"Perplexity-User":["allowed","*",0,1],"Google-Extended":["allowed","*",0,1],"Applebot-Extended":["allowed","*",0,1],"CCBot":["allowed","*",0,1],"Bytespider":["allowed","*",0,1],"Amazonbot":["allowed","*",0,1],"meta-externalagent":["allowed","*",0,1],"cohere-ai":["allowed","*",0,1],"Diffbot":["allowed","*",0,1],"ContextFetchBot":["allowed","*",0,1]},"llms":{"status":"present","http":200,"bytes":240265},"home":{"status":"ok","http":200,"url":"https://lmstudio.ai/","html_tokens":21979,"markdown_tokens":69,"pruned_tokens":69}},
{"rank":8641,"domain":"playclub-es.com","origin":"https://playclub-es.com","robots":{"status":"ok","http":200,"url":"https://playclub-es.com/robots.txt","bytes":535,"groups":1},"bots":{"GPTBot":["partial","*",19,0],"ChatGPT-User":["partial","*",19,0],"OAI-SearchBot":["partial","*",19,0],"ClaudeBot":["partial","*",19,0],"Claude-User":["partial","*",19,0],"Claude-SearchBot":["partial","*",19,0],"anthropic-ai":["partial","*",19,0],"PerplexityBot":["partial","*",19,0],"Perplexity-User":["partial","*",19,0],"Google-Extended":["partial","*",19,0],"Applebot-Extended":["partial","*",19,0],"CCBot":["partial","*",19,0],"Bytespider":["partial","*",19,0],"Amazonbot":["partial","*",19,0],"meta-externalagent":["partial","*",19,0],"cohere-ai":["partial","*",19,0],"Diffbot":["partial","*",19,0],"ContextFetchBot":["partial","*",19,0]},"llms":{"status":"missing","http":404,"bytes":12945},"home":{"status":"ok","http":200,"url":"https://playclub-es.com/","html_tokens":35881,"markdown_tokens":697,"pruned_tokens":146}},
{"rank":8642,"domain":"gilead.com","origin":"https://gilead.com","robots":{"status":"ok","http":200,"url":"https://www.gilead.com/robots.txt","bytes":2688,"groups":1},"bots":{"GPTBot":["partial","*",23,1],"ChatGPT-User":["partial","*",23,1],"OAI-SearchBot":["partial","*",23,1],"ClaudeBot":["partial","*",23,1],"Claude-User":["partial","*",23,1],"Claude-SearchBot":["partial","*",23,1],"anthropic-ai":["partial","*",23,1],"PerplexityBot":["partial","*",23,1],"Perplexity-User":["partial","*",23,1],"Google-Extended":["partial","*",23,1],"Applebot-Extended":["partial","*",23,1],"CCBot":["partial","*",23,1],"Bytespider":["partial","*",23,1],"Amazonbot":["partial","*",23,1],"meta-externalagent":["partial","*",23,1],"cohere-ai":["partial","*",23,1],"Diffbot":["partial","*",23,1],"ContextFetchBot":["partial","*",23,1]},"llms":{"status":"missing","http":404,"bytes":79981},"home":{"status":"ok","http":200,"url":"https://www.gilead.com/","html_tokens":29178,"markdown_tokens":165,"pruned_tokens":93}},
{"rank":8643,"domain":"zoll.de","origin":"https://zoll.de","robots":{"status":"ok","http":200,"url":"https://www.zoll.de/robots.txt","bytes":337,"groups":1},"bots":{"GPTBot":["partial","*",4,4],"ChatGPT-User":["partial","*",4,4],"OAI-SearchBot":["partial","*",4,4],"ClaudeBot":["partial","*",4,4],"Claude-User":["partial","*",4,4],"Claude-SearchBot":["partial","*",4,4],"anthropic-ai":["partial","*",4,4],"PerplexityBot":["partial","*",4,4],"Perplexity-User":["partial","*",4,4],"Google-Extended":["partial","*",4,4],"Applebot-Extended":["partial","*",4,4],"CCBot":["partial","*",4,4],"Bytespider":["partial","*",4,4],"Amazonbot":["partial","*",4,4],"meta-externalagent":["partial","*",4,4],"cohere-ai":["partial","*",4,4],"Diffbot":["partial","*",4,4],"ContextFetchBot":["partial","*",4,4]},"llms":{"status":"crawl_delay","error":"Crawl-delay 180 s"},"home":{"status":"crawl_delay","error":"Crawl-delay 180 s"}},
{"rank":8644,"domain":"hubstack.net","origin":"https://hubstack.net","robots":{"status":"ok","http":200,"url":"https://hubstack.net/robots.txt","bytes":124,"groups":1},"bots":{"GPTBot":["partial","*",3,0],"ChatGPT-User":["partial","*",3,0],"OAI-SearchBot":["partial","*",3,0],"ClaudeBot":["partial","*",3,0],"Claude-User":["partial","*",3,0],"Claude-SearchBot":["partial","*",3,0],"anthropic-ai":["partial","*",3,0],"PerplexityBot":["partial","*",3,0],"Perplexity-User":["partial","*",3,0],"Google-Extended":["partial","*",3,0],"Applebot-Extended":["partial","*",3,0],"CCBot":["partial","*",3,0],"Bytespider":["partial","*",3,0],"Amazonbot":["partial","*",3,0],"meta-externalagent":["partial","*",3,0],"cohere-ai":["partial","*",3,0],"Diffbot":["partial","*",3,0],"ContextFetchBot":["partial","*",3,0]},"llms":{"status":"missing","http":404,"bytes":36529},"home":{"status":"http_error","http":500,"url":"https://hubstack.net/"}},
{"rank":8645,"domain":"din.de","origin":"https://din.de","robots":{"status":"ok","http":200,"url":"https://www.din.de/robots.txt","bytes":1690,"groups":42},"bots":{"GPTBot":["partial","*",4,0],"ChatGPT-User":["partial","*",4,0],"OAI-SearchBot":["partial","*",4,0],"ClaudeBot":["partial","*",4,0],"Claude-User":["partial","*",4,0],"Claude-SearchBot":["partial","*",4,0],"anthropic-ai":["partial","*",4,0],"PerplexityBot":["partial","*",4,0],"Perplexity-User":["partial","*",4,0],"Google-Extended":["partial","*",4,0],"Applebot-Extended":["partial","*",4,0],"CCBot":["partial","*",4,0],"Bytespider":["partial","*",4,0],"Amazonbot":["partial","*",4,0],"meta-externalagent":["partial","*",4,0],"cohere-ai":["partial","*",4,0],"Diffbot":["partial","*",4,0],"ContextFetchBot":["partial","*",4,0]},"llms":{"status":"missing","http":404,"bytes":209291},"home":{"status":"ok","http":200,"url":"https://www.din.de/de","html_tokens":65134,"markdown_tokens":389,"pruned_tokens":60}},
{"rank":8646,"domain":"dzszyjsxx.com","origin":"http://dzszyjsxx.com","robots":{"status":"ok","http":200,"url":"http://dzszyjsxx.com/robots.txt","bytes":25,"groups":1},"bots":{"GPTBot":["blocked","*",1,0],"ChatGPT-User":["blocked","*",1,0],"OAI-SearchBot":["blocked","*",1,0],"ClaudeBot":["blocked","*",1,0],"Claude-User":["blocked","*",1,0],"Claude-SearchBot":["blocked","*",1,0],"anthropic-ai":["blocked","*",1,0],"PerplexityBot":["blocked","*",1,0],"Perplexity-User":["blocked","*",1,0],"Google-Extended":["blocked","*",1,0],"Applebot-Extended":["blocked","*",1,0],"CCBot":["blocked","*",1,0],"Bytespider":["blocked","*",1,0],"Amazonbot":["blocked","*",1,0],"meta-externalagent":["blocked","*",1,0],"cohere-ai":["blocked","*",1,0],"Diffbot":["blocked","*",1,0],"ContextFetchBot":["blocked","*",1,0]},"llms":{"status":"disallowed"},"home":{"status":"disallowed"}},
{"rank":8647,"domain":"mitsui-shopping-park.com","origin":"https://mitsui-shopping-park.com","robots":{"status":"ok","http":200,"url":"https://mitsui-shopping-park.com/robots.txt","bytes":923,"groups":1},"bots":{"GPTBot":["partial","*",14,0],"ChatGPT-User":["partial","*",14,0],"OAI-SearchBot":["partial","*",14,0],"ClaudeBot":["partial","*",14,0],"Claude-User":["partial","*",14,0],"Claude-SearchBot":["partial","*",14,0],"anthropic-ai":["partial","*",14,0],"PerplexityBot":["partial","*",14,0],"Perplexity-User":["partial","*",14,0],"Google-Extended":["partial","*",14,0],"Applebot-Extended":["partial","*",14,0],"CCBot":["partial","*",14,0],"Bytespider":["partial","*",14,0],"Amazonbot":["partial","*",14,0],"meta-externalagent":["partial","*",14,0],"cohere-ai":["partial","*",14,0],"Diffbot":["partial","*",14,0],"ContextFetchBot":["partial","*",14,0]},"llms":{"status":"missing","http":404,"bytes":2944},"home":{"status":"ok","http":200,"url":"https://mitsui-shopping-park.com/","html_tokens":28643,"markdown_tokens":36,"pruned_tokens":36}},
{"rank":8648,"domain":"live-website.com","origin":"https://www.live-website.com","robots":{"status":"missing","http":404,"url":"https://www.live-website.com/robots.txt","bytes":355,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":355},"home":{"status":"http_error","http":403,"url":"https://www.live-website.com/"}},
{"rank":8649,"domain":"tepco.co.jp","origin":"https://www.tepco.co.jp","robots":{"status":"refused","http":403,"url":"https://www.tepco.co.jp/robots.txt","bytes":387,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":8650,"domain":"sparktoro.com","origin":"https://sparktoro.com","robots":{"status":"ok","http":200,"url":"https://sparktoro.com/robots.txt","bytes":267,"groups":2},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"present","http":200,"bytes":894},"home":{"status":"ok","http":200,"url":"https://sparktoro.com/","html_tokens":16231,"markdown_tokens":277,"pruned_tokens":57}},
{"rank":8651,"domain":"biologicaldiversity.org","origin":"https://biologicaldiversity.org","robots":{"status":"ok","http":200,"url":"https://biologicaldiversity.org/robots.txt","bytes":729,"groups":2},"bots":{"GPTBot":["partial","*",11,3],"ChatGPT-User":["partial","*",11,3],"OAI-SearchBot":["partial","*",11,3],"ClaudeBot":["partial","*",11,3],"Claude-User":["partial","*",11,3],"Claude-SearchBot":["partial","*",11,3],"anthropic-ai":["partial","*",11,3],"PerplexityBot":["partial","*",11,3],"Perplexity-User":["partial","*",11,3],"Google-Extended":["partial","*",11,3],"Applebot-Extended":["partial","*",11,3],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["partial","*",11,3],"meta-externalagent":["partial","*",11,3],"cohere-ai":["partial","*",11,3],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",11,3]},"llms":{"status":"missing","http":404,"bytes":31979},"home":{"status":"ok","http":200,"url":"https://biologicaldiversity.org/","html_tokens":12550,"markdown_tokens":7,"pruned_tokens":7}},
{"rank":8652,"domain":"my1.ru","origin":"https://my1.ru","robots":{"status":"ok","http":200,"url":"https://www.ucoz.ru/robots.txt","bytes":121,"groups":1},"bots":{"GPTBot":["partial","*",5,0],"ChatGPT-User":["partial","*",5,0],"OAI-SearchBot":["partial","*",5,0],"ClaudeBot":["partial","*",5,0],"Claude-User":["partial","*",5,0],"Claude-SearchBot":["partial","*",5,0],"anthropic-ai":["partial","*",5,0],"PerplexityBot":["partial","*",5,0],"Perplexity-User":["partial","*",5,0],"Google-Extended":["partial","*",5,0],"Applebot-Extended":["partial","*",5,0],"CCBot":["partial","*",5,0],"Bytespider":["partial","*",5,0],"Amazonbot":["partial","*",5,0],"meta-externalagent":["partial","*",5,0],"cohere-ai":["partial","*",5,0],"Diffbot":["partial","*",5,0],"ContextFetchBot":["partial","*",5,0]},"llms":{"status":"present","http":200,"bytes":33987},"home":{"status":"ok","http":200,"url":"https://www.ucoz.ru/","html_tokens":13583,"markdown_tokens":183,"pruned_tokens":133}},
{"rank":8653,"domain":"news.com","origin":"https://news.com","robots":{"status":"html","http":200,"url":"https://www.cnet.com/news/","bytes":291454,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":8654,"domain":"diamond.jp","origin":"https://diamond.jp","robots":{"status":"ok","http":200,"url":"https://diamond.jp/robots.txt","bytes":26,"groups":1},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"missing","http":404,"bytes":6116},"home":{"status":"ok","http":200,"url":"https://diamond.jp/","html_tokens":43814,"markdown_tokens":21,"pruned_tokens":21}},
{"rank":8655,"domain":"epsilon.com","origin":"https://epsilon.com","robots":{"status":"ok","http":200,"url":"https://www.epsilon.com/robots.txt","bytes":197,"groups":2},"bots":{"GPTBot":["partial","*",2,1],"ChatGPT-User":["partial","*",2,1],"OAI-SearchBot":["partial","*",2,1],"ClaudeBot":["partial","*",2,1],"Claude-User":["partial","*",2,1],"Claude-SearchBot":["partial","*",2,1],"anthropic-ai":["partial","*",2,1],"PerplexityBot":["partial","*",2,1],"Perplexity-User":["partial","*",2,1],"Google-Extended":["partial","*",2,1],"Applebot-Extended":["partial","*",2,1],"CCBot":["partial","*",2,1],"Bytespider":["partial","*",2,1],"Amazonbot":["partial","*",2,1],"meta-externalagent":["partial","*",2,1],"cohere-ai":["partial","*",2,1],"Diffbot":["partial","*",2,1],"ContextFetchBot":["partial","*",2,1]},"llms":{"status":"too_large","error":"response larger than 5 MB"},"home":{"status":"ok","http":200,"url":"https://www.epsilon.com/us","html_tokens":83506,"markdown_tokens":37,"pruned_tokens":37}},
{"rank":8656,"domain":"themezee.com","origin":"https://themezee.com","robots":{"status":"ok","http":200,"url":"https://themezee.com/robots.txt","bytes":170,"groups":1},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"missing","http":404,"bytes":112689},"home":{"status":"ok","http":200,"url":"https://themezee.com/","html_tokens":39408,"markdown_tokens":438,"pruned_tokens":59}},
{"rank":8657,"domain":"google.vg","origin":"https://google.vg","robots":{"status":"ok","http":200,"url":"https://www.google.vg/robots.txt","bytes":6624,"groups":4},"bots":{"GPTBot":["partial","*",169,64],"ChatGPT-User":["partial","*",169,64],"OAI-SearchBot":["partial","*",169,64],"ClaudeBot":["partial","*",169,64],"Claude-User":["partial","*",169,64],"Claude-SearchBot":["partial","*",169,64],"anthropic-ai":["partial","*",169,64],"PerplexityBot":["partial","*",169,64],"Perplexity-User":["partial","*",169,64],"Google-Extended":["partial","*",169,64],"Applebot-Extended":["partial","*",169,64],"CCBot":["partial","*",169,64],"Bytespider":["partial","*",169,64],"Amazonbot":["partial","*",169,64],"meta-externalagent":["partial","*",169,64],"cohere-ai":["partial","*",169,64],"Diffbot":["partial","*",169,64],"ContextFetchBot":["partial","*",169,64]},"llms":{"status":"missing","http":404,"bytes":1569},"home":{"status":"ok","http":200,"url":"https://www.google.vg/","html_tokens":21322,"markdown_tokens":32,"pruned_tokens":32}},
{"rank":8658,"domain":"onlinedown.net","origin":"https://onlinedown.net","robots":{"status":"server_error","http":567,"url":"https://onlinedown.net/robots.txt","bytes":7351,"groups":0,"error":"HTTP 567"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":8659,"domain":"yingdailianmeng.com","origin":"https://yingdailianmeng.com","robots":{"status":"unreachable","http":null,"url":"https://yingdailianmeng.com/robots.txt","bytes":0,"groups":0,"error":"All connection attempts failed"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":8660,"domain":"housingwire.com","origin":"https://housingwire.com","robots":{"status":"ok","http":200,"url":"https://www.housingwire.com/robots.txt","bytes":856,"groups":2},"bots":{"GPTBot":["partial","*",17,0],"ChatGPT-User":["partial","*",17,0],"OAI-SearchBot":["partial","*",17,0],"ClaudeBot":["partial","*",17,0],"Claude-User":["partial","*",17,0],"Claude-SearchBot":["partial","*",17,0],"anthropic-ai":["partial","*",17,0],"PerplexityBot":["partial","*",17,0],"Perplexity-User":["partial","*",17,0],"Google-Extended":["partial","*",17,0],"Applebot-Extended":["partial","*",17,0],"CCBot":["partial","*",17,0],"Bytespider":["partial","*",17,0],"Amazonbot":["partial","*",17,0],"meta-externalagent":["partial","*",17,0],"cohere-ai":["partial","*",17,0],"Diffbot":["partial","*",17,0],"ContextFetchBot":["partial","*",17,0]},"llms":{"status":"missing","http":404,"bytes":222112},"home":{"status":"ok","http":200,"url":"https://www.housingwire.com/","html_tokens":109837,"markdown_tokens":71,"pruned_tokens":71}},
{"rank":8661,"domain":"tripadvisor.com.au","origin":"https://tripadvisor.com.au","robots":{"status":"ok","http":200,"url":"https://www.tripadvisor.com.au/robots.txt","bytes":22939,"groups":9},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["partial","named",623,8],"OAI-SearchBot":["partial","named",623,8],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",621,8],"Claude-SearchBot":["partial","*",621,8],"anthropic-ai":["partial","*",621,8],"PerplexityBot":["partial","named",625,8],"Perplexity-User":["partial","*",621,8],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["partial","*",621,8],"ContextFetchBot":["partial","*",621,8]},"llms":{"status":"http_error","http":403,"bytes":778},"home":{"status":"http_error","http":403,"url":"https://www.tripadvisor.com.au/"}},
{"rank":8662,"domain":"raindrop.io","origin":"https://raindrop.io","robots":{"status":"ok","http":200,"url":"https://raindrop.io/robots.txt","bytes":114,"groups":1},"bots":{"GPTBot":["allowed","*",0,1],"ChatGPT-User":["allowed","*",0,1],"OAI-SearchBot":["allowed","*",0,1],"ClaudeBot":["allowed","*",0,1],"Claude-User":["allowed","*",0,1],"Claude-SearchBot":["allowed","*",0,1],"anthropic-ai":["allowed","*",0,1],"PerplexityBot":["allowed","*",0,1],"Perplexity-User":["allowed","*",0,1],"Google-Extended":["allowed","*",0,1],"Applebot-Extended":["allowed","*",0,1],"CCBot":["allowed","*",0,1],"Bytespider":["allowed","*",0,1],"Amazonbot":["allowed","*",0,1],"meta-externalagent":["allowed","*",0,1],"cohere-ai":["allowed","*",0,1],"Diffbot":["allowed","*",0,1],"ContextFetchBot":["allowed","*",0,1]},"llms":{"status":"missing","http":404,"bytes":3594},"home":{"status":"ok","http":200,"url":"https://raindrop.io/","html_tokens":14781,"markdown_tokens":219,"pruned_tokens":44}},
{"rank":8663,"domain":"wttc.org","origin":"https://wttc.org","robots":{"status":"ok","http":200,"url":"https://wttc.org/robots.txt","bytes":115,"groups":1},"bots":{"GPTBot":["partial","*",3,0],"ChatGPT-User":["partial","*",3,0],"OAI-SearchBot":["partial","*",3,0],"ClaudeBot":["partial","*",3,0],"Claude-User":["partial","*",3,0],"Claude-SearchBot":["partial","*",3,0],"anthropic-ai":["partial","*",3,0],"PerplexityBot":["partial","*",3,0],"Perplexity-User":["partial","*",3,0],"Google-Extended":["partial","*",3,0],"Applebot-Extended":["partial","*",3,0],"CCBot":["partial","*",3,0],"Bytespider":["partial","*",3,0],"Amazonbot":["partial","*",3,0],"meta-externalagent":["partial","*",3,0],"cohere-ai":["partial","*",3,0],"Diffbot":["partial","*",3,0],"ContextFetchBot":["partial","*",3,0]},"llms":{"status":"missing","http":404,"bytes":69594},"home":{"status":"ok","http":200,"url":"https://wttc.org/","html_tokens":25806,"markdown_tokens":784,"pruned_tokens":443}},
{"rank":8664,"domain":"beepworld.de","origin":"https://beepworld.de","robots":{"status":"ok","http":200,"url":"https://beepworld.de/robots.txt","bytes":224,"groups":1},"bots":{"GPTBot":["partial","*",6,1],"ChatGPT-User":["partial","*",6,1],"OAI-SearchBot":["partial","*",6,1],"ClaudeBot":["partial","*",6,1],"Claude-User":["partial","*",6,1],"Claude-SearchBot":["partial","*",6,1],"anthropic-ai":["partial","*",6,1],"PerplexityBot":["partial","*",6,1],"Perplexity-User":["partial","*",6,1],"Google-Extended":["partial","*",6,1],"Applebot-Extended":["partial","*",6,1],"CCBot":["partial","*",6,1],"Bytespider":["partial","*",6,1],"Amazonbot":["partial","*",6,1],"meta-externalagent":["partial","*",6,1],"cohere-ai":["partial","*",6,1],"Diffbot":["partial","*",6,1],"ContextFetchBot":["partial","*",6,1]},"llms":{"status":"missing","http":404,"bytes":3026},"home":{"status":"ok","http":200,"url":"https://beepworld.de/","html_tokens":9502,"markdown_tokens":546,"pruned_tokens":323}},
{"rank":8665,"domain":"betterplace.org","origin":"https://betterplace.org","robots":{"status":"ok","http":200,"url":"https://www.betterplace.org/robots.txt","bytes":124,"groups":1},"bots":{"GPTBot":["partial","*",2,1],"ChatGPT-User":["partial","*",2,1],"OAI-SearchBot":["partial","*",2,1],"ClaudeBot":["partial","*",2,1],"Claude-User":["partial","*",2,1],"Claude-SearchBot":["partial","*",2,1],"anthropic-ai":["partial","*",2,1],"PerplexityBot":["partial","*",2,1],"Perplexity-User":["partial","*",2,1],"Google-Extended":["partial","*",2,1],"Applebot-Extended":["partial","*",2,1],"CCBot":["partial","*",2,1],"Bytespider":["partial","*",2,1],"Amazonbot":["partial","*",2,1],"meta-externalagent":["partial","*",2,1],"cohere-ai":["partial","*",2,1],"Diffbot":["partial","*",2,1],"ContextFetchBot":["partial","*",2,1]},"llms":{"status":"missing","http":404,"bytes":0},"home":{"status":"ok","http":200,"url":"https://www.betterplace.org/de","html_tokens":20203,"markdown_tokens":159,"pruned_tokens":128}},
{"rank":8666,"domain":"ubergizmo.com","origin":"https://ubergizmo.com","robots":{"status":"ok","http":200,"url":"https://www.ubergizmo.com/robots.txt","bytes":502,"groups":2},"bots":{"GPTBot":["partial","*",14,0],"ChatGPT-User":["partial","*",14,0],"OAI-SearchBot":["partial","*",14,0],"ClaudeBot":["partial","*",14,0],"Claude-User":["partial","*",14,0],"Claude-SearchBot":["partial","*",14,0],"anthropic-ai":["partial","*",14,0],"PerplexityBot":["partial","*",14,0],"Perplexity-User":["partial","*",14,0],"Google-Extended":["partial","*",14,0],"Applebot-Extended":["partial","*",14,0],"CCBot":["partial","*",14,0],"Bytespider":["partial","*",14,0],"Amazonbot":["partial","*",14,0],"meta-externalagent":["partial","*",14,0],"cohere-ai":["partial","*",14,0],"Diffbot":["partial","*",14,0],"ContextFetchBot":["partial","*",14,0]},"llms":{"status":"missing","http":404,"bytes":114654},"home":{"status":"ok","http":200,"url":"https://www.ubergizmo.com/","html_tokens":33564,"markdown_tokens":17,"pruned_tokens":17}},
{"rank":8667,"domain":"jylebao.com","origin":"http://jylebao.com","robots":{"status":"ok","http":200,"url":"http://jylebao.com/robots.txt","bytes":485,"groups":1},"bots":{"GPTBot":["partial","*",21,0],"ChatGPT-User":["partial","*",21,0],"OAI-SearchBot":["partial","*",21,0],"ClaudeBot":["partial","*",21,0],"Claude-User":["partial","*",21,0],"Claude-SearchBot":["partial","*",21,0],"anthropic-ai":["partial","*",21,0],"PerplexityBot":["partial","*",21,0],"Perplexity-User":["partial","*",21,0],"Google-Extended":["partial","*",21,0],"Applebot-Extended":["partial","*",21,0],"CCBot":["partial","*",21,0],"Bytespider":["partial","*",21,0],"Amazonbot":["partial","*",21,0],"meta-externalagent":["partial","*",21,0],"cohere-ai":["partial","*",21,0],"Diffbot":["partial","*",21,0],"ContextFetchBot":["partial","*",21,0]},"llms":{"status":"missing","http":404,"bytes":4726},"home":{"status":"ok","http":200,"url":"http://jylebao.com/","html_tokens":8332,"markdown_tokens":54,"pruned_tokens":54}},
{"rank":8668,"domain":"tourismthailand.org","origin":"https://tourismthailand.org","robots":{"status":"missing","http":404,"url":"https://www.tourismthailand.org/robots.txt","bytes":0,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":0},"home":{"status":"http_error","http":403,"url":"https://tourismthailand.org/"}},
{"rank":8669,"domain":"politiken.dk","origin":"https://politiken.dk","robots":{"status":"ok","http":200,"url":"https://politiken.dk/robots.txt","bytes":789,"groups":19},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["blocked","named",1,0],"OAI-SearchBot":["partial","*",4,0],"ClaudeBot":["partial","*",4,0],"Claude-User":["partial","*",4,0],"Claude-SearchBot":["partial","*",4,0],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["partial","*",4,0],"Perplexity-User":["partial","*",4,0],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["partial","*",4,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["partial","*",4,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["partial","*",4,0],"ContextFetchBot":["partial","*",4,0]},"llms":{"status":"missing","http":404,"bytes":1039286},"home":{"status":"ok","http":200,"url":"https://politiken.dk/","html_tokens":102160,"markdown_tokens":7,"pruned_tokens":7}},
{"rank":8670,"domain":"deutschebahn.com","origin":"https://deutschebahn.com","robots":{"status":"ok","http":200,"url":"https://www.deutschebahn.com/robots.txt","bytes":97,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":113729},"home":{"status":"ok","http":200,"url":"https://www.deutschebahn.com/de","html_tokens":67170,"markdown_tokens":103,"pruned_tokens":103}},
{"rank":8671,"domain":"snob.ru","origin":"https://snob.ru","robots":{"status":"ok","http":200,"url":"https://snob.ru/robots.txt","bytes":9552,"groups":6},"bots":{"GPTBot":["partial","*",90,1],"ChatGPT-User":["partial","*",90,1],"OAI-SearchBot":["partial","*",90,1],"ClaudeBot":["partial","*",90,1],"Claude-User":["partial","*",90,1],"Claude-SearchBot":["partial","*",90,1],"anthropic-ai":["partial","*",90,1],"PerplexityBot":["partial","*",90,1],"Perplexity-User":["partial","*",90,1],"Google-Extended":["partial","*",90,1],"Applebot-Extended":["partial","*",90,1],"CCBot":["partial","*",90,1],"Bytespider":["partial","*",90,1],"Amazonbot":["partial","*",90,1],"meta-externalagent":["partial","*",90,1],"cohere-ai":["partial","*",90,1],"Diffbot":["partial","*",90,1],"ContextFetchBot":["partial","*",90,1]},"llms":{"status":"missing","http":404,"bytes":0},"home":{"status":"ok","http":200,"url":"https://snob.ru/","html_tokens":74047,"markdown_tokens":28,"pruned_tokens":28}},
{"rank":8672,"domain":"cast.org","origin":"https://cast.org","robots":{"status":"refused","http":403,"url":"https://cast.org/robots.txt","bytes":560,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":8673,"domain":"zysycp.com","origin":"https://zysycp.com","robots":{"status":"timeout","http":null,"url":"https://zysycp.com/robots.txt","bytes":0,"groups":0,"error":"ConnectTimeout"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":8674,"domain":"conservation.org","origin":"https://conservation.org","robots":{"status":"ok","http":200,"url":"https://www.conservation.org/robots.txt","bytes":74,"groups":1},"bots":{"GPTBot":["allowed","*",0,1],"ChatGPT-User":["allowed","*",0,1],"OAI-SearchBot":["allowed","*",0,1],"ClaudeBot":["allowed","*",0,1],"Claude-User":["allowed","*",0,1],"Claude-SearchBot":["allowed","*",0,1],"anthropic-ai":["allowed","*",0,1],"PerplexityBot":["allowed","*",0,1],"Perplexity-User":["allowed","*",0,1],"Google-Extended":["allowed","*",0,1],"Applebot-Extended":["allowed","*",0,1],"CCBot":["allowed","*",0,1],"Bytespider":["allowed","*",0,1],"Amazonbot":["allowed","*",0,1],"meta-externalagent":["allowed","*",0,1],"cohere-ai":["allowed","*",0,1],"Diffbot":["allowed","*",0,1],"ContextFetchBot":["allowed","*",0,1]},"llms":{"status":"present","http":200,"bytes":2619},"home":{"status":"ok","http":200,"url":"https://www.conservation.org/","html_tokens":65037,"markdown_tokens":334,"pruned_tokens":16}},
{"rank":8675,"domain":"tn.it","origin":"https://tn.it","robots":{"status":"unreachable","http":null,"url":"https://tn.it/robots.txt","bytes":0,"groups":0,"error":"cannot resolve tn.it"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":8676,"domain":"who.is","origin":"https://who.is","robots":{"status":"ok","http":200,"url":"https://who.is/robots.txt","bytes":75,"groups":1},"bots":{"GPTBot":["partial","*",1,1],"ChatGPT-User":["partial","*",1,1],"OAI-SearchBot":["partial","*",1,1],"ClaudeBot":["partial","*",1,1],"Claude-User":["partial","*",1,1],"Claude-SearchBot":["partial","*",1,1],"anthropic-ai":["partial","*",1,1],"PerplexityBot":["partial","*",1,1],"Perplexity-User":["partial","*",1,1],"Google-Extended":["partial","*",1,1],"Applebot-Extended":["partial","*",1,1],"CCBot":["partial","*",1,1],"Bytespider":["partial","*",1,1],"Amazonbot":["partial","*",1,1],"meta-externalagent":["partial","*",1,1],"cohere-ai":["partial","*",1,1],"Diffbot":["partial","*",1,1],"ContextFetchBot":["partial","*",1,1]},"llms":{"status":"missing","http":404,"bytes":18318},"home":{"status":"ok","http":200,"url":"https://who.is/","html_tokens":9246,"markdown_tokens":415,"pruned_tokens":128}},
{"rank":8677,"domain":"foxhis.com","origin":"https://foxhis.com","robots":{"status":"missing","http":404,"url":"https://www.foxhis.com/robots.txt","bytes":1163,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":1163},"home":{"status":"ok","http":200,"url":"https://www.foxhis.com/","html_tokens":3993,"markdown_tokens":37,"pruned_tokens":37}},
{"rank":8678,"domain":"bdysgz.cn","origin":"https://bdysgz.cn","robots":{"status":"server_error","http":502,"url":"https://bdysgz.cn/robots.txt","bytes":122,"groups":0,"error":"HTTP 502"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":8679,"domain":"haendlerbund.de","origin":"https://haendlerbund.de","robots":{"status":"ok","http":200,"url":"https://www.haendlerbund.de/robots.txt","bytes":913,"groups":2},"bots":{"GPTBot":["partial","*",13,0],"ChatGPT-User":["partial","*",13,0],"OAI-SearchBot":["partial","*",13,0],"ClaudeBot":["partial","*",13,0],"Claude-User":["partial","*",13,0],"Claude-SearchBot":["partial","*",13,0],"anthropic-ai":["partial","*",13,0],"PerplexityBot":["partial","*",13,0],"Perplexity-User":["partial","*",13,0],"Google-Extended":["partial","*",13,0],"Applebot-Extended":["partial","*",13,0],"CCBot":["partial","*",13,0],"Bytespider":["partial","*",13,0],"Amazonbot":["partial","*",13,0],"meta-externalagent":["partial","*",13,0],"cohere-ai":["partial","*",13,0],"Diffbot":["partial","*",13,0],"ContextFetchBot":["partial","*",13,0]},"llms":{"status":"missing","http":404,"bytes":419267},"home":{"status":"ok","http":200,"url":"https://www.haendlerbund.de/de/","html_tokens":131011,"markdown_tokens":414,"pruned_tokens":105}},
{"rank":8680,"domain":"match.com","origin":"https://match.com","robots":{"status":"refused","http":403,"url":"https://www.match.com/robots.txt","bytes":4542,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":8681,"domain":"ie.edu","origin":"https://ie.edu","robots":{"status":"ok","http":200,"url":"https://www.ie.edu/robots.txt","bytes":4410,"groups":1},"bots":{"GPTBot":["partial","*",96,0],"ChatGPT-User":["partial","*",96,0],"OAI-SearchBot":["partial","*",96,0],"ClaudeBot":["partial","*",96,0],"Claude-User":["partial","*",96,0],"Claude-SearchBot":["partial","*",96,0],"anthropic-ai":["partial","*",96,0],"PerplexityBot":["partial","*",96,0],"Perplexity-User":["partial","*",96,0],"Google-Extended":["partial","*",96,0],"Applebot-Extended":["partial","*",96,0],"CCBot":["partial","*",96,0],"Bytespider":["partial","*",96,0],"Amazonbot":["partial","*",96,0],"meta-externalagent":["partial","*",96,0],"cohere-ai":["partial","*",96,0],"Diffbot":["partial","*",96,0],"ContextFetchBot":["partial","*",96,0]},"llms":{"status":"too_large","error":"response larger than 5 MB"},"home":{"status":"ok","http":200,"url":"https://www.ie.edu/","html_tokens":196877,"markdown_tokens":412,"pruned_tokens":374}},
{"rank":8682,"domain":"kijiji.ca","origin":"https://kijiji.ca","robots":{"status":"ok","http":200,"url":"https://www.kijiji.ca/robots.txt","bytes":7789,"groups":2},"bots":{"GPTBot":["partial","*",233,0],"ChatGPT-User":["partial","*",233,0],"OAI-SearchBot":["partial","*",233,0],"ClaudeBot":["partial","*",233,0],"Claude-User":["partial","*",233,0],"Claude-SearchBot":["partial","*",233,0],"anthropic-ai":["partial","*",233,0],"PerplexityBot":["partial","*",233,0],"Perplexity-User":["partial","*",233,0],"Google-Extended":["partial","*",233,0],"Applebot-Extended":["partial","*",233,0],"CCBot":["partial","*",233,0],"Bytespider":["partial","*",233,0],"Amazonbot":["partial","*",233,0],"meta-externalagent":["partial","*",233,0],"cohere-ai":["partial","*",233,0],"Diffbot":["partial","*",233,0],"ContextFetchBot":["partial","*",233,0]},"llms":{"status":"present","http":200,"bytes":2933},"home":{"status":"ok","http":200,"url":"https://www.kijiji.ca","html_tokens":58295,"markdown_tokens":46,"pruned_tokens":46}},
{"rank":8683,"domain":"vidalisted.shop","origin":"https://vidalisted.shop","robots":{"status":"html","http":200,"url":"https://edpharmahub.com/vidalista/","bytes":32142,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":8684,"domain":"sorbonne-universite.fr","origin":"https://sorbonne-universite.fr","robots":{"status":"ok","http":200,"url":"https://www.sorbonne-universite.fr/robots.txt","bytes":2320,"groups":3},"bots":{"GPTBot":["partial","*",35,22],"ChatGPT-User":["partial","*",35,22],"OAI-SearchBot":["partial","*",35,22],"ClaudeBot":["partial","*",35,22],"Claude-User":["partial","*",35,22],"Claude-SearchBot":["partial","*",35,22],"anthropic-ai":["partial","*",35,22],"PerplexityBot":["partial","*",35,22],"Perplexity-User":["partial","*",35,22],"Google-Extended":["partial","*",35,22],"Applebot-Extended":["partial","*",35,22],"CCBot":["partial","*",35,22],"Bytespider":["partial","*",35,22],"Amazonbot":["partial","*",35,22],"meta-externalagent":["partial","*",35,22],"cohere-ai":["partial","*",35,22],"Diffbot":["partial","*",35,22],"ContextFetchBot":["partial","*",35,22]},"llms":{"status":"missing","http":404,"bytes":583},"home":{"status":"ok","http":200,"url":"https://www.sorbonne-universite.fr/","html_tokens":61750,"markdown_tokens":164,"pruned_tokens":145}},
{"rank":8685,"domain":"emc.com","origin":"https://www.emc.com","robots":{"status":"ok","http":200,"url":"https://www.emc.com/robots.txt","bytes":58,"groups":1},"bots":{"GPTBot":["allowed","*",0,1],"ChatGPT-User":["allowed","*",0,1],"OAI-SearchBot":["allowed","*",0,1],"ClaudeBot":["allowed","*",0,1],"Claude-User":["allowed","*",0,1],"Claude-SearchBot":["allowed","*",0,1],"anthropic-ai":["allowed","*",0,1],"PerplexityBot":["allowed","*",0,1],"Perplexity-User":["allowed","*",0,1],"Google-Extended":["allowed","*",0,1],"Applebot-Extended":["allowed","*",0,1],"CCBot":["allowed","*",0,1],"Bytespider":["allowed","*",0,1],"Amazonbot":["allowed","*",0,1],"meta-externalagent":["allowed","*",0,1],"cohere-ai":["allowed","*",0,1],"Diffbot":["allowed","*",0,1],"ContextFetchBot":["allowed","*",0,1]},"llms":{"status":"missing","http":404,"bytes":271},"home":{"status":"disallowed","error":"robots.txt of www.delltechnologies.com not readable (refused)"}},
{"rank":8686,"domain":"slimframework.com","origin":"https://slimframework.com","robots":{"status":"ok","http":200,"url":"https://www.slimframework.com/robots.txt","bytes":51,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":5067},"home":{"status":"ok","http":200,"url":"https://www.slimframework.com/","html_tokens":3277,"markdown_tokens":612,"pruned_tokens":216}},
{"rank":8687,"domain":"pewsocialtrends.org","origin":"https://pewsocialtrends.org","robots":{"status":"missing","http":404,"url":"https://www.pewresearch.org/social-trends/robots.txt","bytes":287694,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":287688},"home":{"status":"ok","http":200,"url":"https://www.pewresearch.org/publications/?programs=social-trends#recent-publications","html_tokens":131832,"markdown_tokens":138,"pruned_tokens":138}},
{"rank":8688,"domain":"jy-xm.com","origin":"http://jy-xm.com","robots":{"status":"ok","http":200,"url":"http://jy-xm.com/robots.txt","bytes":485,"groups":1},"bots":{"GPTBot":["partial","*",21,0],"ChatGPT-User":["partial","*",21,0],"OAI-SearchBot":["partial","*",21,0],"ClaudeBot":["partial","*",21,0],"Claude-User":["partial","*",21,0],"Claude-SearchBot":["partial","*",21,0],"anthropic-ai":["partial","*",21,0],"PerplexityBot":["partial","*",21,0],"Perplexity-User":["partial","*",21,0],"Google-Extended":["partial","*",21,0],"Applebot-Extended":["partial","*",21,0],"CCBot":["partial","*",21,0],"Bytespider":["partial","*",21,0],"Amazonbot":["partial","*",21,0],"meta-externalagent":["partial","*",21,0],"cohere-ai":["partial","*",21,0],"Diffbot":["partial","*",21,0],"ContextFetchBot":["partial","*",21,0]},"llms":{"status":"missing","http":404,"bytes":4721},"home":{"status":"ok","http":200,"url":"http://jy-xm.com/","html_tokens":6213,"markdown_tokens":73,"pruned_tokens":73}},
{"rank":8689,"domain":"italki.com","origin":"https://italki.com","robots":{"status":"ok","http":200,"url":"https://www.italki.com/robots.txt","bytes":7651,"groups":8},"bots":{"GPTBot":["partial","*",27,0],"ChatGPT-User":["partial","named",27,0],"OAI-SearchBot":["partial","named",27,0],"ClaudeBot":["partial","*",27,0],"Claude-User":["partial","named",27,0],"Claude-SearchBot":["partial","named",27,0],"anthropic-ai":["partial","*",27,0],"PerplexityBot":["partial","named",27,0],"Perplexity-User":["partial","named",27,0],"Google-Extended":["partial","*",27,0],"Applebot-Extended":["partial","*",27,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["partial","named",27,0],"meta-externalagent":["partial","*",27,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",27,0]},"llms":{"status":"present","http":200,"bytes":9398},"home":{"status":"ok","http":200,"url":"https://www.italki.com/","html_tokens":109478,"markdown_tokens":160,"pruned_tokens":160}},
{"rank":8690,"domain":"google.vu","origin":"https://google.vu","robots":{"status":"ok","http":200,"url":"https://www.google.vu/robots.txt","bytes":6624,"groups":4},"bots":{"GPTBot":["partial","*",169,64],"ChatGPT-User":["partial","*",169,64],"OAI-SearchBot":["partial","*",169,64],"ClaudeBot":["partial","*",169,64],"Claude-User":["partial","*",169,64],"Claude-SearchBot":["partial","*",169,64],"anthropic-ai":["partial","*",169,64],"PerplexityBot":["partial","*",169,64],"Perplexity-User":["partial","*",169,64],"Google-Extended":["partial","*",169,64],"Applebot-Extended":["partial","*",169,64],"CCBot":["partial","*",169,64],"Bytespider":["partial","*",169,64],"Amazonbot":["partial","*",169,64],"meta-externalagent":["partial","*",169,64],"cohere-ai":["partial","*",169,64],"Diffbot":["partial","*",169,64],"ContextFetchBot":["partial","*",169,64]},"llms":{"status":"missing","http":404,"bytes":1569},"home":{"status":"ok","http":200,"url":"https://www.google.com/","html_tokens":21434,"markdown_tokens":30,"pruned_tokens":30}},
{"rank":8691,"domain":"ita-airways.com","origin":"https://ita-airways.com","robots":{"status":"refused","http":403,"url":"https://www.ita-airways.com","bytes":6187,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":8692,"domain":"www.judiciary.uk","origin":"https://www.judiciary.uk","robots":{"status":"ok","http":200,"url":"https://www.judiciary.uk/robots.txt","bytes":174,"groups":1},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"missing","http":404,"bytes":63084},"home":{"status":"ok","http":200,"url":"https://www.judiciary.uk/","html_tokens":24164,"markdown_tokens":124,"pruned_tokens":124}},
{"rank":8693,"domain":"tehrantimes.com","origin":"https://tehrantimes.com","robots":{"status":"ok","http":200,"url":"https://www.tehrantimes.com/robots.txt","bytes":1832,"groups":1},"bots":{"GPTBot":["partial","*",1,0],"ChatGPT-User":["partial","*",1,0],"OAI-SearchBot":["partial","*",1,0],"ClaudeBot":["partial","*",1,0],"Claude-User":["partial","*",1,0],"Claude-SearchBot":["partial","*",1,0],"anthropic-ai":["partial","*",1,0],"PerplexityBot":["partial","*",1,0],"Perplexity-User":["partial","*",1,0],"Google-Extended":["partial","*",1,0],"Applebot-Extended":["partial","*",1,0],"CCBot":["partial","*",1,0],"Bytespider":["partial","*",1,0],"Amazonbot":["partial","*",1,0],"meta-externalagent":["partial","*",1,0],"cohere-ai":["partial","*",1,0],"Diffbot":["partial","*",1,0],"ContextFetchBot":["partial","*",1,0]},"llms":{"status":"missing","http":404,"bytes":1695},"home":{"status":"ok","http":200,"url":"https://www.tehrantimes.com/","html_tokens":22863,"markdown_tokens":1917,"pruned_tokens":167}},
{"rank":8694,"domain":"soundclick.com","origin":"https://soundclick.com","robots":{"status":"ok","http":200,"url":"https://www.soundclick.com/robots.txt","bytes":2200,"groups":19},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["partial","*",7,0],"OAI-SearchBot":["blocked","named",1,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",7,0],"Claude-SearchBot":["partial","named",7,0],"anthropic-ai":["partial","*",7,0],"PerplexityBot":["partial","*",7,0],"Perplexity-User":["partial","*",7,0],"Google-Extended":["partial","*",7,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["partial","*",7,0],"Bytespider":["partial","*",7,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["partial","*",7,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",7,0]},"llms":{"status":"html","http":200,"bytes":1330},"home":{"status":"ok","http":200,"url":"https://www.soundclick.com/","html_tokens":332,"markdown_tokens":13,"pruned_tokens":13}},
{"rank":8695,"domain":"cnjingying88.com","origin":"http://cnjingying88.com","robots":{"status":"refused","http":403,"url":"http://cnjingying88.com/robots.txt","bytes":237,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":8696,"domain":"xyzs.cc","origin":"https://xyzs.cc","robots":{"status":"missing","http":404,"url":"https://xyzs.cc/robots.txt","bytes":5070,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":5066},"home":{"status":"disallowed","error":"robots.txt of www.xyzs.cc not readable (unreachable)"}},
{"rank":8697,"domain":"yangguangdd.com","origin":"https://yangguangdd.com","robots":{"status":"unreachable","http":null,"url":"https://yangguangdd.com/robots.txt","bytes":0,"groups":0,"error":"cannot resolve yangguangdd.com"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":8698,"domain":"formzu.net","origin":"https://www.formzu.net","robots":{"status":"refused","http":403,"url":"https://www.formzu.net/robots.txt","bytes":276,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":8699,"domain":"allabout.co.jp","origin":"https://allabout.co.jp","robots":{"status":"ok","http":200,"url":"https://allabout.co.jp/robots.txt","bytes":732,"groups":2},"bots":{"GPTBot":["partial","*",24,0],"ChatGPT-User":["partial","*",24,0],"OAI-SearchBot":["partial","*",24,0],"ClaudeBot":["partial","*",24,0],"Claude-User":["partial","*",24,0],"Claude-SearchBot":["partial","*",24,0],"anthropic-ai":["partial","*",24,0],"PerplexityBot":["partial","*",24,0],"Perplexity-User":["partial","*",24,0],"Google-Extended":["partial","*",24,0],"Applebot-Extended":["partial","*",24,0],"CCBot":["partial","*",24,0],"Bytespider":["partial","*",24,0],"Amazonbot":["partial","*",24,0],"meta-externalagent":["partial","*",24,0],"cohere-ai":["partial","*",24,0],"Diffbot":["partial","*",24,0],"ContextFetchBot":["partial","*",24,0]},"llms":{"status":"missing","http":404,"bytes":3127},"home":{"status":"ok","http":200,"url":"https://allabout.co.jp/","html_tokens":61421,"markdown_tokens":14,"pruned_tokens":14}},
{"rank":8700,"domain":"sf.gov","origin":"https://sf.gov","robots":{"status":"ok","http":200,"url":"https://www.sf.gov/robots.txt","bytes":78,"groups":1},"bots":{"GPTBot":["partial","*",3,0],"ChatGPT-User":["partial","*",3,0],"OAI-SearchBot":["partial","*",3,0],"ClaudeBot":["partial","*",3,0],"Claude-User":["partial","*",3,0],"Claude-SearchBot":["partial","*",3,0],"anthropic-ai":["partial","*",3,0],"PerplexityBot":["partial","*",3,0],"Perplexity-User":["partial","*",3,0],"Google-Extended":["partial","*",3,0],"Applebot-Extended":["partial","*",3,0],"CCBot":["partial","*",3,0],"Bytespider":["partial","*",3,0],"Amazonbot":["partial","*",3,0],"meta-externalagent":["partial","*",3,0],"cohere-ai":["partial","*",3,0],"Diffbot":["partial","*",3,0],"ContextFetchBot":["partial","*",3,0]},"llms":{"status":"missing","http":404,"bytes":91950},"home":{"status":"ok","http":200,"url":"https://www.sf.gov","html_tokens":67734,"markdown_tokens":26,"pruned_tokens":26}},
{"rank":8701,"domain":"confease.cn","origin":"https://confease.cn","robots":{"status":"unreachable","http":null,"url":"https://confease.cn/robots.txt","bytes":0,"groups":0,"error":"cannot resolve confease.cn"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":8702,"domain":"woot.com","origin":"https://woot.com","robots":{"status":"ok","http":200,"url":"https://www.woot.com/robots.txt","bytes":306,"groups":1},"bots":{"GPTBot":["partial","*",12,0],"ChatGPT-User":["partial","*",12,0],"OAI-SearchBot":["partial","*",12,0],"ClaudeBot":["partial","*",12,0],"Claude-User":["partial","*",12,0],"Claude-SearchBot":["partial","*",12,0],"anthropic-ai":["partial","*",12,0],"PerplexityBot":["partial","*",12,0],"Perplexity-User":["partial","*",12,0],"Google-Extended":["partial","*",12,0],"Applebot-Extended":["partial","*",12,0],"CCBot":["partial","*",12,0],"Bytespider":["partial","*",12,0],"Amazonbot":["partial","*",12,0],"meta-externalagent":["partial","*",12,0],"cohere-ai":["partial","*",12,0],"Diffbot":["partial","*",12,0],"ContextFetchBot":["partial","*",12,0]},"llms":{"status":"missing","http":404,"bytes":4815},"home":{"status":"ok","http":200,"url":"https://www.woot.com/","html_tokens":7925,"markdown_tokens":63,"pruned_tokens":63}},
{"rank":8703,"domain":"xinyunda-gov.cn","origin":"https://xinyunda-gov.cn","robots":{"status":"unreachable","http":null,"url":"https://xinyunda-gov.cn/robots.txt","bytes":0,"groups":0,"error":"cannot resolve xinyunda-gov.cn"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":8704,"domain":"spicethemes.com","origin":"https://spicethemes.com","robots":{"status":"ok","http":200,"url":"https://spicethemes.com/robots.txt","bytes":180,"groups":1},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"other_text","http":200,"bytes":77069},"home":{"status":"ok","http":200,"url":"https://spicethemes.com/","html_tokens":20723,"markdown_tokens":643,"pruned_tokens":165}},
{"rank":8705,"domain":"open-meteo.com","origin":"https://open-meteo.com","robots":{"status":"ok","http":200,"url":"https://open-meteo.com/robots.txt","bytes":22,"groups":1},"bots":{"GPTBot":["allowed","*",0,1],"ChatGPT-User":["allowed","*",0,1],"OAI-SearchBot":["allowed","*",0,1],"ClaudeBot":["allowed","*",0,1],"Claude-User":["allowed","*",0,1],"Claude-SearchBot":["allowed","*",0,1],"anthropic-ai":["allowed","*",0,1],"PerplexityBot":["allowed","*",0,1],"Perplexity-User":["allowed","*",0,1],"Google-Extended":["allowed","*",0,1],"Applebot-Extended":["allowed","*",0,1],"CCBot":["allowed","*",0,1],"Bytespider":["allowed","*",0,1],"Amazonbot":["allowed","*",0,1],"meta-externalagent":["allowed","*",0,1],"cohere-ai":["allowed","*",0,1],"Diffbot":["allowed","*",0,1],"ContextFetchBot":["allowed","*",0,1]},"llms":{"status":"missing","http":404,"bytes":25655},"home":{"status":"ok","http":200,"url":"https://open-meteo.com/","html_tokens":12902,"markdown_tokens":871,"pruned_tokens":167}},
{"rank":8706,"domain":"womansday.com","origin":"https://womansday.com","robots":{"status":"ok","http":200,"url":"https://womansday.com/robots.txt","bytes":963,"groups":3},"bots":{"GPTBot":["partial","named",32,0],"ChatGPT-User":["partial","named",32,0],"OAI-SearchBot":["partial","named",32,0],"ClaudeBot":["partial","*",19,0],"Claude-User":["partial","*",19,0],"Claude-SearchBot":["partial","*",19,0],"anthropic-ai":["partial","*",19,0],"PerplexityBot":["partial","*",19,0],"Perplexity-User":["partial","*",19,0],"Google-Extended":["partial","*",19,0],"Applebot-Extended":["partial","*",19,0],"CCBot":["partial","*",19,0],"Bytespider":["partial","*",19,0],"Amazonbot":["partial","*",19,0],"meta-externalagent":["partial","*",19,0],"cohere-ai":["partial","*",19,0],"Diffbot":["partial","*",19,0],"ContextFetchBot":["partial","*",19,0]},"llms":{"status":"missing","http":404,"bytes":84},"home":{"status":"ok","http":200,"url":"https://www.womansday.com/","html_tokens":152022,"markdown_tokens":24,"pruned_tokens":24}},
{"rank":8707,"domain":"soundonsound.com","origin":"https://soundonsound.com","robots":{"status":"refused","http":403,"url":"https://soundonsound.com/robots.txt","bytes":560,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":8708,"domain":"bfm.ru","origin":"https://bfm.ru","robots":{"status":"ok","http":200,"url":"https://www.bfm.ru/robots.txt","bytes":543,"groups":5},"bots":{"GPTBot":["partial","*",5,0],"ChatGPT-User":["partial","*",5,0],"OAI-SearchBot":["partial","*",5,0],"ClaudeBot":["partial","*",5,0],"Claude-User":["partial","*",5,0],"Claude-SearchBot":["partial","*",5,0],"anthropic-ai":["partial","*",5,0],"PerplexityBot":["partial","*",5,0],"Perplexity-User":["partial","*",5,0],"Google-Extended":["partial","*",5,0],"Applebot-Extended":["partial","*",5,0],"CCBot":["partial","*",5,0],"Bytespider":["partial","*",5,0],"Amazonbot":["partial","*",5,0],"meta-externalagent":["partial","*",5,0],"cohere-ai":["partial","*",5,0],"Diffbot":["partial","*",5,0],"ContextFetchBot":["partial","*",5,0]},"llms":{"status":"missing","http":404,"bytes":2505},"home":{"status":"ok","http":200,"url":"https://www.bfm.ru/","html_tokens":26805,"markdown_tokens":57,"pruned_tokens":57}},
{"rank":8709,"domain":"larepublica.pe","origin":"https://larepublica.pe","robots":{"status":"ok","http":200,"url":"https://larepublica.pe/robots.txt","bytes":3864,"groups":1},"bots":{"GPTBot":["partial","named",22,5],"ChatGPT-User":["partial","named",22,5],"OAI-SearchBot":["partial","named",22,5],"ClaudeBot":["partial","named",22,5],"Claude-User":["partial","*",22,5],"Claude-SearchBot":["partial","*",22,5],"anthropic-ai":["partial","named",22,5],"PerplexityBot":["partial","named",22,5],"Perplexity-User":["partial","*",22,5],"Google-Extended":["partial","named",22,5],"Applebot-Extended":["partial","*",22,5],"CCBot":["partial","named",22,5],"Bytespider":["partial","named",22,5],"Amazonbot":["partial","named",22,5],"meta-externalagent":["partial","*",22,5],"cohere-ai":["partial","named",22,5],"Diffbot":["partial","named",22,5],"ContextFetchBot":["partial","*",22,5]},"llms":{"status":"missing","http":404,"bytes":141301},"home":{"status":"ok","http":200,"url":"https://larepublica.pe/","html_tokens":416654,"markdown_tokens":472,"pruned_tokens":151}},
{"rank":8710,"domain":"ridewithgps.com","origin":"https://ridewithgps.com","robots":{"status":"ok","http":200,"url":"https://ridewithgps.com/robots.txt","bytes":1232,"groups":3},"bots":{"GPTBot":["partial","*",14,0],"ChatGPT-User":["partial","*",14,0],"OAI-SearchBot":["partial","*",14,0],"ClaudeBot":["partial","*",14,0],"Claude-User":["partial","*",14,0],"Claude-SearchBot":["partial","*",14,0],"anthropic-ai":["partial","*",14,0],"PerplexityBot":["partial","*",14,0],"Perplexity-User":["partial","*",14,0],"Google-Extended":["partial","*",14,0],"Applebot-Extended":["partial","*",14,0],"CCBot":["partial","*",14,0],"Bytespider":["partial","*",14,0],"Amazonbot":["partial","*",14,0],"meta-externalagent":["partial","*",14,0],"cohere-ai":["partial","*",14,0],"Diffbot":["partial","*",14,0],"ContextFetchBot":["partial","*",14,0]},"llms":{"status":"present","http":200,"bytes":7705},"home":{"status":"ok","http":200,"url":"https://ridewithgps.com/","html_tokens":42879,"markdown_tokens":335,"pruned_tokens":175}},
{"rank":8711,"domain":"airfrance.com","origin":"https://airfrance.com","robots":{"status":"timeout","http":null,"url":"https://airfrance.com/robots.txt","bytes":0,"groups":0,"error":"timed out after 10 s"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":8712,"domain":"jysenxin.com","origin":"http://jysenxin.com","robots":{"status":"ok","http":200,"url":"http://jysenxin.com/robots.txt","bytes":485,"groups":1},"bots":{"GPTBot":["partial","*",21,0],"ChatGPT-User":["partial","*",21,0],"OAI-SearchBot":["partial","*",21,0],"ClaudeBot":["partial","*",21,0],"Claude-User":["partial","*",21,0],"Claude-SearchBot":["partial","*",21,0],"anthropic-ai":["partial","*",21,0],"PerplexityBot":["partial","*",21,0],"Perplexity-User":["partial","*",21,0],"Google-Extended":["partial","*",21,0],"Applebot-Extended":["partial","*",21,0],"CCBot":["partial","*",21,0],"Bytespider":["partial","*",21,0],"Amazonbot":["partial","*",21,0],"meta-externalagent":["partial","*",21,0],"cohere-ai":["partial","*",21,0],"Diffbot":["partial","*",21,0],"ContextFetchBot":["partial","*",21,0]},"llms":{"status":"missing","http":404,"bytes":4728},"home":{"status":"ok","http":200,"url":"http://jysenxin.com/","html_tokens":7814,"markdown_tokens":105,"pruned_tokens":105}},
{"rank":8713,"domain":"zds8158.com","origin":"http://zds8158.com","robots":{"status":"ok","http":200,"url":"http://zds8158.com/robots.txt","bytes":126,"groups":1},"bots":{"GPTBot":["partial","*",1,0],"ChatGPT-User":["partial","*",1,0],"OAI-SearchBot":["partial","*",1,0],"ClaudeBot":["partial","*",1,0],"Claude-User":["partial","*",1,0],"Claude-SearchBot":["partial","*",1,0],"anthropic-ai":["partial","*",1,0],"PerplexityBot":["partial","*",1,0],"Perplexity-User":["partial","*",1,0],"Google-Extended":["partial","*",1,0],"Applebot-Extended":["partial","*",1,0],"CCBot":["partial","*",1,0],"Bytespider":["partial","*",1,0],"Amazonbot":["partial","*",1,0],"meta-externalagent":["partial","*",1,0],"cohere-ai":["partial","*",1,0],"Diffbot":["partial","*",1,0],"ContextFetchBot":["partial","*",1,0]},"llms":{"status":"missing","http":404,"bytes":4719},"home":{"status":"ok","http":200,"url":"http://zds8158.com/","html_tokens":4760,"markdown_tokens":50,"pruned_tokens":50}},
{"rank":8714,"domain":"jylt888.cn","origin":"http://jylt888.cn","robots":{"status":"ok","http":200,"url":"http://jylt888.cn/robots.txt","bytes":485,"groups":1},"bots":{"GPTBot":["partial","*",21,0],"ChatGPT-User":["partial","*",21,0],"OAI-SearchBot":["partial","*",21,0],"ClaudeBot":["partial","*",21,0],"Claude-User":["partial","*",21,0],"Claude-SearchBot":["partial","*",21,0],"anthropic-ai":["partial","*",21,0],"PerplexityBot":["partial","*",21,0],"Perplexity-User":["partial","*",21,0],"Google-Extended":["partial","*",21,0],"Applebot-Extended":["partial","*",21,0],"CCBot":["partial","*",21,0],"Bytespider":["partial","*",21,0],"Amazonbot":["partial","*",21,0],"meta-externalagent":["partial","*",21,0],"cohere-ai":["partial","*",21,0],"Diffbot":["partial","*",21,0],"ContextFetchBot":["partial","*",21,0]},"llms":{"status":"missing","http":404,"bytes":4725},"home":{"status":"ok","http":200,"url":"http://jylt888.cn/","html_tokens":22092,"markdown_tokens":240,"pruned_tokens":171}},
{"rank":8715,"domain":"pc-xd.com","origin":"https://pc-xd.com","robots":{"status":"ok","http":200,"url":"https://pc-xd.com/robots.txt","bytes":485,"groups":1},"bots":{"GPTBot":["partial","*",21,0],"ChatGPT-User":["partial","*",21,0],"OAI-SearchBot":["partial","*",21,0],"ClaudeBot":["partial","*",21,0],"Claude-User":["partial","*",21,0],"Claude-SearchBot":["partial","*",21,0],"anthropic-ai":["partial","*",21,0],"PerplexityBot":["partial","*",21,0],"Perplexity-User":["partial","*",21,0],"Google-Extended":["partial","*",21,0],"Applebot-Extended":["partial","*",21,0],"CCBot":["partial","*",21,0],"Bytespider":["partial","*",21,0],"Amazonbot":["partial","*",21,0],"meta-externalagent":["partial","*",21,0],"cohere-ai":["partial","*",21,0],"Diffbot":["partial","*",21,0],"ContextFetchBot":["partial","*",21,0]},"llms":{"status":"missing","http":404,"bytes":4721},"home":{"status":"ok","http":200,"url":"https://pc-xd.com/","html_tokens":13059,"markdown_tokens":82,"pruned_tokens":82}},
{"rank":8716,"domain":"spreadshirt.com","origin":"https://spreadshirt.com","robots":{"status":"refused","http":403,"url":"https://spreadshirt.com/robots.txt","bytes":34,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":8717,"domain":"google.co.tz","origin":"https://google.co.tz","robots":{"status":"ok","http":200,"url":"https://www.google.co.tz/robots.txt","bytes":6624,"groups":4},"bots":{"GPTBot":["partial","*",169,64],"ChatGPT-User":["partial","*",169,64],"OAI-SearchBot":["partial","*",169,64],"ClaudeBot":["partial","*",169,64],"Claude-User":["partial","*",169,64],"Claude-SearchBot":["partial","*",169,64],"anthropic-ai":["partial","*",169,64],"PerplexityBot":["partial","*",169,64],"Perplexity-User":["partial","*",169,64],"Google-Extended":["partial","*",169,64],"Applebot-Extended":["partial","*",169,64],"CCBot":["partial","*",169,64],"Bytespider":["partial","*",169,64],"Amazonbot":["partial","*",169,64],"meta-externalagent":["partial","*",169,64],"cohere-ai":["partial","*",169,64],"Diffbot":["partial","*",169,64],"ContextFetchBot":["partial","*",169,64]},"llms":{"status":"missing","http":404,"bytes":1569},"home":{"status":"ok","http":200,"url":"https://www.google.com/","html_tokens":21267,"markdown_tokens":30,"pruned_tokens":30}},
{"rank":8718,"domain":"google.me","origin":"https://google.me","robots":{"status":"ok","http":200,"url":"https://www.google.me/robots.txt","bytes":6624,"groups":4},"bots":{"GPTBot":["partial","*",169,64],"ChatGPT-User":["partial","*",169,64],"OAI-SearchBot":["partial","*",169,64],"ClaudeBot":["partial","*",169,64],"Claude-User":["partial","*",169,64],"Claude-SearchBot":["partial","*",169,64],"anthropic-ai":["partial","*",169,64],"PerplexityBot":["partial","*",169,64],"Perplexity-User":["partial","*",169,64],"Google-Extended":["partial","*",169,64],"Applebot-Extended":["partial","*",169,64],"CCBot":["partial","*",169,64],"Bytespider":["partial","*",169,64],"Amazonbot":["partial","*",169,64],"meta-externalagent":["partial","*",169,64],"cohere-ai":["partial","*",169,64],"Diffbot":["partial","*",169,64],"ContextFetchBot":["partial","*",169,64]},"llms":{"status":"missing","http":404,"bytes":1569},"home":{"status":"ok","http":200,"url":"https://www.google.com/","html_tokens":21469,"markdown_tokens":30,"pruned_tokens":30}},
{"rank":8719,"domain":"logmein.com","origin":"https://logmein.com","robots":{"status":"ok","http":200,"url":"https://www.logmein.com/robots.txt","bytes":413,"groups":1},"bots":{"GPTBot":["partial","*",5,0],"ChatGPT-User":["partial","*",5,0],"OAI-SearchBot":["partial","*",5,0],"ClaudeBot":["partial","*",5,0],"Claude-User":["partial","*",5,0],"Claude-SearchBot":["partial","*",5,0],"anthropic-ai":["partial","*",5,0],"PerplexityBot":["partial","*",5,0],"Perplexity-User":["partial","*",5,0],"Google-Extended":["partial","*",5,0],"Applebot-Extended":["partial","*",5,0],"CCBot":["partial","*",5,0],"Bytespider":["partial","*",5,0],"Amazonbot":["partial","*",5,0],"meta-externalagent":["partial","*",5,0],"cohere-ai":["partial","*",5,0],"Diffbot":["partial","*",5,0],"ContextFetchBot":["partial","*",5,0]},"llms":{"status":"missing","http":404,"bytes":251189},"home":{"status":"ok","http":200,"url":"https://www.logmein.com/","html_tokens":103842,"markdown_tokens":133,"pruned_tokens":76}},
{"rank":8720,"domain":"gfk.com","origin":"https://gfk.com","robots":{"status":"ok","http":200,"url":"https://www.gfk.com/robots.txt","bytes":1113,"groups":1},"bots":{"GPTBot":["partial","*",42,0],"ChatGPT-User":["partial","*",42,0],"OAI-SearchBot":["partial","*",42,0],"ClaudeBot":["partial","*",42,0],"Claude-User":["partial","*",42,0],"Claude-SearchBot":["partial","*",42,0],"anthropic-ai":["partial","*",42,0],"PerplexityBot":["partial","*",42,0],"Perplexity-User":["partial","*",42,0],"Google-Extended":["partial","*",42,0],"Applebot-Extended":["partial","*",42,0],"CCBot":["partial","*",42,0],"Bytespider":["partial","*",42,0],"Amazonbot":["partial","*",42,0],"meta-externalagent":["partial","*",42,0],"cohere-ai":["partial","*",42,0],"Diffbot":["partial","*",42,0],"ContextFetchBot":["partial","*",42,0]},"llms":{"status":"missing","http":404,"bytes":40045},"home":{"status":"ok","http":200,"url":"https://nielseniq.com/global/en/","html_tokens":90951,"markdown_tokens":351,"pruned_tokens":105}},
{"rank":8721,"domain":"jymds.com","origin":"http://jymds.com","robots":{"status":"ok","http":200,"url":"http://jymds.com/robots.txt","bytes":485,"groups":1},"bots":{"GPTBot":["partial","*",21,0],"ChatGPT-User":["partial","*",21,0],"OAI-SearchBot":["partial","*",21,0],"ClaudeBot":["partial","*",21,0],"Claude-User":["partial","*",21,0],"Claude-SearchBot":["partial","*",21,0],"anthropic-ai":["partial","*",21,0],"PerplexityBot":["partial","*",21,0],"Perplexity-User":["partial","*",21,0],"Google-Extended":["partial","*",21,0],"Applebot-Extended":["partial","*",21,0],"CCBot":["partial","*",21,0],"Bytespider":["partial","*",21,0],"Amazonbot":["partial","*",21,0],"meta-externalagent":["partial","*",21,0],"cohere-ai":["partial","*",21,0],"Diffbot":["partial","*",21,0],"ContextFetchBot":["partial","*",21,0]},"llms":{"status":"missing","http":404,"bytes":5066},"home":{"status":"ok","http":200,"url":"http://jymds.com/","html_tokens":10240,"markdown_tokens":54,"pruned_tokens":54}},
{"rank":8722,"domain":"nikkansports.com","origin":"https://nikkansports.com","robots":{"status":"ok","http":200,"url":"https://www.nikkansports.com/robots.txt","bytes":429,"groups":11},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["blocked","named",1,0],"OAI-SearchBot":["partial","*",1,0],"ClaudeBot":["partial","*",1,0],"Claude-User":["partial","*",1,0],"Claude-SearchBot":["partial","*",1,0],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["partial","*",1,0],"Perplexity-User":["partial","*",1,0],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["partial","*",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["partial","*",1,0],"Amazonbot":["partial","*",1,0],"meta-externalagent":["partial","*",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["partial","*",1,0],"ContextFetchBot":["partial","*",1,0]},"llms":{"status":"missing","http":404,"bytes":65116},"home":{"status":"ok","http":200,"url":"https://www.nikkansports.com/","html_tokens":47057,"markdown_tokens":31,"pruned_tokens":31}},
{"rank":8723,"domain":"thriftbooks.com","origin":"https://thriftbooks.com","robots":{"status":"ok","http":200,"url":"https://www.thriftbooks.com/robots.txt","bytes":904,"groups":5},"bots":{"GPTBot":["partial","*",25,0],"ChatGPT-User":["partial","*",25,0],"OAI-SearchBot":["partial","*",25,0],"ClaudeBot":["partial","*",25,0],"Claude-User":["partial","*",25,0],"Claude-SearchBot":["partial","*",25,0],"anthropic-ai":["partial","*",25,0],"PerplexityBot":["partial","*",25,0],"Perplexity-User":["partial","*",25,0],"Google-Extended":["partial","*",25,0],"Applebot-Extended":["partial","*",25,0],"CCBot":["partial","*",25,0],"Bytespider":["partial","*",25,0],"Amazonbot":["partial","*",25,0],"meta-externalagent":["partial","*",25,0],"cohere-ai":["partial","*",25,0],"Diffbot":["partial","*",25,0],"ContextFetchBot":["partial","*",25,0]},"llms":{"status":"http_error","http":406,"bytes":0},"home":{"status":"http_error","http":406,"url":"https://www.thriftbooks.com/"}},
{"rank":8724,"domain":"norwegian.com","origin":"https://norwegian.com","robots":{"status":"refused","http":403,"url":"https://www.norwegian.com/robots.txt","bytes":5180,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":8725,"domain":"wien.info","origin":"https://wien.info","robots":{"status":"ok","http":200,"url":"https://www.wien.info/robots.txt","bytes":1418,"groups":7},"bots":{"GPTBot":["partial","*",3,0],"ChatGPT-User":["partial","*",3,0],"OAI-SearchBot":["partial","*",3,0],"ClaudeBot":["partial","*",3,0],"Claude-User":["partial","*",3,0],"Claude-SearchBot":["partial","*",3,0],"anthropic-ai":["partial","*",3,0],"PerplexityBot":["partial","*",3,0],"Perplexity-User":["partial","*",3,0],"Google-Extended":["partial","*",3,0],"Applebot-Extended":["partial","*",3,0],"CCBot":["partial","*",3,0],"Bytespider":["partial","*",3,0],"Amazonbot":["partial","*",3,0],"meta-externalagent":["partial","*",3,0],"cohere-ai":["partial","*",3,0],"Diffbot":["partial","*",3,0],"ContextFetchBot":["partial","*",3,0]},"llms":{"status":"missing","http":404,"bytes":95347},"home":{"status":"ok","http":200,"url":"https://www.wien.info/en","html_tokens":36806,"markdown_tokens":420,"pruned_tokens":84}},
{"rank":8726,"domain":"torino.it","origin":"https://torino.it","robots":{"status":"unreachable","http":null,"url":"https://torino.it/robots.txt","bytes":0,"groups":0,"error":"cannot resolve torino.it"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":8727,"domain":"seld.be","origin":"https://seld.be","robots":{"status":"missing","http":404,"url":"https://seld.be/robots.txt","bytes":7404,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":7404},"home":{"status":"ok","http":200,"url":"https://seld.be/","html_tokens":4253,"markdown_tokens":537,"pruned_tokens":140}},
{"rank":8728,"domain":"goodtherapy.org","origin":"https://goodtherapy.org","robots":{"status":"ok","http":200,"url":"https://www.goodtherapy.org/robots.txt","bytes":693,"groups":1},"bots":{"GPTBot":["partial","*",24,2],"ChatGPT-User":["partial","*",24,2],"OAI-SearchBot":["partial","*",24,2],"ClaudeBot":["partial","*",24,2],"Claude-User":["partial","*",24,2],"Claude-SearchBot":["partial","*",24,2],"anthropic-ai":["partial","*",24,2],"PerplexityBot":["partial","*",24,2],"Perplexity-User":["partial","*",24,2],"Google-Extended":["partial","*",24,2],"Applebot-Extended":["partial","*",24,2],"CCBot":["partial","*",24,2],"Bytespider":["partial","*",24,2],"Amazonbot":["partial","*",24,2],"meta-externalagent":["partial","*",24,2],"cohere-ai":["partial","*",24,2],"Diffbot":["partial","*",24,2],"ContextFetchBot":["partial","*",24,2]},"llms":{"status":"present","http":200,"bytes":1440},"home":{"status":"http_error","http":403,"url":"https://www.goodtherapy.org/"}},
{"rank":8729,"domain":"goodmenproject.com","origin":"https://goodmenproject.com","robots":{"status":"ok","http":200,"url":"https://goodmenproject.com/robots.txt","bytes":376,"groups":6},"bots":{"GPTBot":["partial","*",6,0],"ChatGPT-User":["partial","*",6,0],"OAI-SearchBot":["partial","*",6,0],"ClaudeBot":["partial","*",6,0],"Claude-User":["partial","*",6,0],"Claude-SearchBot":["partial","*",6,0],"anthropic-ai":["partial","*",6,0],"PerplexityBot":["partial","*",6,0],"Perplexity-User":["partial","*",6,0],"Google-Extended":["partial","*",6,0],"Applebot-Extended":["partial","*",6,0],"CCBot":["partial","*",6,0],"Bytespider":["partial","*",6,0],"Amazonbot":["partial","*",6,0],"meta-externalagent":["partial","*",6,0],"cohere-ai":["partial","*",6,0],"Diffbot":["partial","*",6,0],"ContextFetchBot":["partial","*",6,0]},"llms":{"status":"missing","http":404,"bytes":253939},"home":{"status":"http_error","http":403,"url":"https://goodmenproject.com/"}},
{"rank":8730,"domain":"visitportugal.com","origin":"https://visitportugal.com","robots":{"status":"ok","http":200,"url":"https://www.visitportugal.com/robots.txt","bytes":2413,"groups":1},"bots":{"GPTBot":["partial","*",46,32],"ChatGPT-User":["partial","*",46,32],"OAI-SearchBot":["partial","*",46,32],"ClaudeBot":["partial","*",46,32],"Claude-User":["partial","*",46,32],"Claude-SearchBot":["partial","*",46,32],"anthropic-ai":["partial","*",46,32],"PerplexityBot":["partial","*",46,32],"Perplexity-User":["partial","*",46,32],"Google-Extended":["partial","*",46,32],"Applebot-Extended":["partial","*",46,32],"CCBot":["partial","*",46,32],"Bytespider":["partial","*",46,32],"Amazonbot":["partial","*",46,32],"meta-externalagent":["partial","*",46,32],"cohere-ai":["partial","*",46,32],"Diffbot":["partial","*",46,32],"ContextFetchBot":["partial","*",46,32]},"llms":{"status":"html","http":200,"bytes":7472},"home":{"status":"ok","http":200,"url":"https://www.visitportugal.com/","html_tokens":1866,"markdown_tokens":269,"pruned_tokens":153}},
{"rank":8731,"domain":"ebu.ch","origin":"https://ebu.ch","robots":{"status":"ok","http":200,"url":"https://www.ebu.ch/robots.txt","bytes":10329,"groups":4},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["partial","named",44,15],"OAI-SearchBot":["partial","named",44,15],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","named",44,15],"Claude-SearchBot":["partial","named",44,15],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["partial","named",44,15],"Perplexity-User":["partial","named",44,15],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",42,15]},"llms":{"status":"present","http":200,"bytes":10016},"home":{"status":"http_error","http":403,"url":"https://www.ebu.ch/"}},
{"rank":8732,"domain":"xjtu.edu.cn","origin":"https://www.xjtu.edu.cn","robots":{"status":"missing","http":404,"url":"https://www.xjtu.edu.cn/robots.txt","bytes":1693,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":1693},"home":{"status":"ok","http":200,"url":"https://www.xjtu.edu.cn/","html_tokens":13528,"markdown_tokens":22,"pruned_tokens":22}},
{"rank":8733,"domain":"katu.com","origin":"https://katu.com","robots":{"status":"ok","http":200,"url":"https://katu.com/robots.txt","bytes":565,"groups":12},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["blocked","named",1,0],"OAI-SearchBot":["partial","*",1,1],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",1,1],"Claude-SearchBot":["partial","*",1,1],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["partial","*",1,1],"Perplexity-User":["partial","*",1,1],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["partial","*",1,1],"Amazonbot":["partial","*",1,1],"meta-externalagent":["partial","*",1,1],"cohere-ai":["blocked","named",1,0],"Diffbot":["partial","*",1,1],"ContextFetchBot":["partial","*",1,1]},"llms":{"status":"http_error","http":400,"bytes":45},"home":{"status":"ok","http":200,"url":"https://katu.com/","html_tokens":131606,"markdown_tokens":62,"pruned_tokens":62}},
{"rank":8734,"domain":"ello.co","origin":"https://ello.co","robots":{"status":"html","http":200,"url":"https://notgamstopcasinos.co/","bytes":197255,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":8735,"domain":"hbszyysy.cn","origin":"https://www.hbszyysy.cn","robots":{"status":"missing","http":404,"url":"https://www.hbszyysy.cn/robots.txt","bytes":261,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":261},"home":{"status":"ok","http":200,"url":"https://www.hbszyysy.cn/","html_tokens":14107,"markdown_tokens":6,"pruned_tokens":6}},
{"rank":8736,"domain":"jswanjiasupply.com","origin":"http://jswanjiasupply.com","robots":{"status":"ok","http":200,"url":"http://jswanjiasupply.com/robots.txt","bytes":485,"groups":1},"bots":{"GPTBot":["partial","*",21,0],"ChatGPT-User":["partial","*",21,0],"OAI-SearchBot":["partial","*",21,0],"ClaudeBot":["partial","*",21,0],"Claude-User":["partial","*",21,0],"Claude-SearchBot":["partial","*",21,0],"anthropic-ai":["partial","*",21,0],"PerplexityBot":["partial","*",21,0],"Perplexity-User":["partial","*",21,0],"Google-Extended":["partial","*",21,0],"Applebot-Extended":["partial","*",21,0],"CCBot":["partial","*",21,0],"Bytespider":["partial","*",21,0],"Amazonbot":["partial","*",21,0],"meta-externalagent":["partial","*",21,0],"cohere-ai":["partial","*",21,0],"Diffbot":["partial","*",21,0],"ContextFetchBot":["partial","*",21,0]},"llms":{"status":"missing","http":404,"bytes":4740},"home":{"status":"ok","http":200,"url":"http://jswanjiasupply.com/","html_tokens":9132,"markdown_tokens":96,"pruned_tokens":96}},
{"rank":8737,"domain":"creativelive.com","origin":"https://creativelive.com","robots":{"status":"ok","http":200,"url":"https://www.creativelive.com/robots.txt","bytes":403,"groups":1},"bots":{"GPTBot":["partial","*",14,0],"ChatGPT-User":["partial","*",14,0],"OAI-SearchBot":["partial","*",14,0],"ClaudeBot":["partial","*",14,0],"Claude-User":["partial","*",14,0],"Claude-SearchBot":["partial","*",14,0],"anthropic-ai":["partial","*",14,0],"PerplexityBot":["partial","*",14,0],"Perplexity-User":["partial","*",14,0],"Google-Extended":["partial","*",14,0],"Applebot-Extended":["partial","*",14,0],"CCBot":["partial","*",14,0],"Bytespider":["partial","*",14,0],"Amazonbot":["partial","*",14,0],"meta-externalagent":["partial","*",14,0],"cohere-ai":["partial","*",14,0],"Diffbot":["partial","*",14,0],"ContextFetchBot":["partial","*",14,0]},"llms":{"status":"present","http":200,"bytes":422294},"home":{"status":"ok","http":200,"url":"https://www.creativelive.com/","html_tokens":87822,"markdown_tokens":215,"pruned_tokens":28}},
{"rank":8738,"domain":"blogspot.gr","origin":"https://blogspot.gr","robots":{"status":"ok","http":200,"url":"https://blogspot.gr/robots.txt","bytes":6624,"groups":4},"bots":{"GPTBot":["partial","*",169,64],"ChatGPT-User":["partial","*",169,64],"OAI-SearchBot":["partial","*",169,64],"ClaudeBot":["partial","*",169,64],"Claude-User":["partial","*",169,64],"Claude-SearchBot":["partial","*",169,64],"anthropic-ai":["partial","*",169,64],"PerplexityBot":["partial","*",169,64],"Perplexity-User":["partial","*",169,64],"Google-Extended":["partial","*",169,64],"Applebot-Extended":["partial","*",169,64],"CCBot":["partial","*",169,64],"Bytespider":["partial","*",169,64],"Amazonbot":["partial","*",169,64],"meta-externalagent":["partial","*",169,64],"cohere-ai":["partial","*",169,64],"Diffbot":["partial","*",169,64],"ContextFetchBot":["partial","*",169,64]},"llms":{"status":"missing","http":404,"bytes":1569},"home":{"status":"ok","http":200,"url":"https://www.google.com/","html_tokens":21335,"markdown_tokens":30,"pruned_tokens":30}},
{"rank":8739,"domain":"sdyzskqyy.com","origin":"http://sdyzskqyy.com","robots":{"status":"html","http":200,"url":"http://sdyzskqyy.com/robots.txt","bytes":3368,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":8740,"domain":"uodo.gov.pl","origin":"https://uodo.gov.pl","robots":{"status":"ok","http":200,"url":"https://uodo.gov.pl/robots.txt","bytes":117,"groups":1},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"missing","http":404,"bytes":807},"home":{"status":"ok","http":200,"url":"https://uodo.gov.pl/","html_tokens":43967,"markdown_tokens":374,"pruned_tokens":91}},
{"rank":8741,"domain":"allaboutdnt.com","origin":"https://allaboutdnt.com","robots":{"status":"refused","http":403,"url":"https://allaboutdnt.com/robots.txt","bytes":560,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":8742,"domain":"blog.hu","origin":"https://blog.hu","robots":{"status":"ok","http":200,"url":"https://blog.hu/robots.txt","bytes":520,"groups":1},"bots":{"GPTBot":["partial","*",7,0],"ChatGPT-User":["partial","*",7,0],"OAI-SearchBot":["partial","*",7,0],"ClaudeBot":["partial","*",7,0],"Claude-User":["partial","*",7,0],"Claude-SearchBot":["partial","*",7,0],"anthropic-ai":["partial","*",7,0],"PerplexityBot":["partial","*",7,0],"Perplexity-User":["partial","*",7,0],"Google-Extended":["partial","*",7,0],"Applebot-Extended":["partial","*",7,0],"CCBot":["partial","*",7,0],"Bytespider":["partial","*",7,0],"Amazonbot":["partial","*",7,0],"meta-externalagent":["partial","*",7,0],"cohere-ai":["partial","*",7,0],"Diffbot":["partial","*",7,0],"ContextFetchBot":["partial","*",7,0]},"llms":{"status":"http_error","http":501,"bytes":0},"home":{"status":"ok","http":200,"url":"https://blog.hu/","html_tokens":21156,"markdown_tokens":59,"pruned_tokens":59}},
{"rank":8743,"domain":"wowhead.com","origin":"https://wowhead.com","robots":{"status":"ok","http":200,"url":"https://www.wowhead.com/robots.txt","bytes":68848,"groups":3},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["partial","*",1599,0],"OAI-SearchBot":["partial","*",1599,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",1599,0],"Claude-SearchBot":["partial","*",1599,0],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["partial","*",1599,0],"Perplexity-User":["partial","*",1599,0],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["partial","*",1599,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",1599,0]},"llms":{"status":"missing","http":404,"bytes":24763},"home":{"status":"ok","http":200,"url":"https://www.wowhead.com/","html_tokens":43058,"markdown_tokens":23,"pruned_tokens":23}},
{"rank":8744,"domain":"ipfs.io","origin":"https://ipfs.io","robots":{"status":"ok","http":200,"url":"https://ipfs.tech/robots.txt","bytes":68,"groups":1},"bots":{"GPTBot":["allowed","*",0,1],"ChatGPT-User":["allowed","*",0,1],"OAI-SearchBot":["allowed","*",0,1],"ClaudeBot":["allowed","*",0,1],"Claude-User":["allowed","*",0,1],"Claude-SearchBot":["allowed","*",0,1],"anthropic-ai":["allowed","*",0,1],"PerplexityBot":["allowed","*",0,1],"Perplexity-User":["allowed","*",0,1],"Google-Extended":["allowed","*",0,1],"Applebot-Extended":["allowed","*",0,1],"CCBot":["allowed","*",0,1],"Bytespider":["allowed","*",0,1],"Amazonbot":["allowed","*",0,1],"meta-externalagent":["allowed","*",0,1],"cohere-ai":["allowed","*",0,1],"Diffbot":["allowed","*",0,1],"ContextFetchBot":["allowed","*",0,1]},"llms":{"status":"missing","http":404,"bytes":9379},"home":{"status":"ok","http":200,"url":"https://about.ipfs.io/","html_tokens":3603,"markdown_tokens":1056,"pruned_tokens":905}},
{"rank":8745,"domain":"shop.com","origin":"https://shop.com","robots":{"status":"ok","http":200,"url":"https://shop.com/robots.txt","bytes":2823,"groups":1},"bots":{"GPTBot":["partial","*",37,2],"ChatGPT-User":["partial","*",37,2],"OAI-SearchBot":["partial","*",37,2],"ClaudeBot":["partial","*",37,2],"Claude-User":["partial","*",37,2],"Claude-SearchBot":["partial","*",37,2],"anthropic-ai":["partial","*",37,2],"PerplexityBot":["partial","*",37,2],"Perplexity-User":["partial","*",37,2],"Google-Extended":["partial","*",37,2],"Applebot-Extended":["partial","*",37,2],"CCBot":["partial","*",37,2],"Bytespider":["partial","*",37,2],"Amazonbot":["partial","*",37,2],"meta-externalagent":["partial","*",37,2],"cohere-ai":["partial","*",37,2],"Diffbot":["partial","*",37,2],"ContextFetchBot":["partial","*",37,2]},"llms":{"status":"present","http":200,"bytes":3534},"home":{"status":"http_error","http":429,"url":"https://shop.com/"}},
{"rank":8746,"domain":"dotdashmeredith.com","origin":"https://dotdashmeredith.com","robots":{"status":"ok","http":200,"url":"https://www.people.inc/robots.txt","bytes":1994,"groups":26},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["blocked","named",1,0],"OAI-SearchBot":["blocked","named",1,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",2,0],"Claude-SearchBot":["partial","*",2,0],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["partial","*",2,0],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",2,0]},"llms":{"status":"missing","http":404,"bytes":7990},"home":{"status":"ok","http":200,"url":"https://www.people.inc/","html_tokens":74935,"markdown_tokens":15,"pruned_tokens":15}},
{"rank":8747,"domain":"scielo.org","origin":"https://scielo.org","robots":{"status":"ok","http":200,"url":"https://www.scielo.org/robots.txt","bytes":609,"groups":15},"bots":{"GPTBot":["blocked","*",1,0],"ChatGPT-User":["blocked","*",1,0],"OAI-SearchBot":["blocked","*",1,0],"ClaudeBot":["blocked","*",1,0],"Claude-User":["blocked","*",1,0],"Claude-SearchBot":["blocked","*",1,0],"anthropic-ai":["blocked","*",1,0],"PerplexityBot":["blocked","*",1,0],"Perplexity-User":["blocked","*",1,0],"Google-Extended":["blocked","*",1,0],"Applebot-Extended":["blocked","*",1,0],"CCBot":["blocked","*",1,0],"Bytespider":["blocked","*",1,0],"Amazonbot":["blocked","*",1,0],"meta-externalagent":["blocked","*",1,0],"cohere-ai":["blocked","*",1,0],"Diffbot":["blocked","*",1,0],"ContextFetchBot":["blocked","*",1,0]},"llms":{"status":"disallowed"},"home":{"status":"disallowed"}},
{"rank":8748,"domain":"dtyuanyang.com","origin":"http://dtyuanyang.com","robots":{"status":"ok","http":200,"url":"http://dtyuanyang.com/robots.txt","bytes":485,"groups":1},"bots":{"GPTBot":["partial","*",21,0],"ChatGPT-User":["partial","*",21,0],"OAI-SearchBot":["partial","*",21,0],"ClaudeBot":["partial","*",21,0],"Claude-User":["partial","*",21,0],"Claude-SearchBot":["partial","*",21,0],"anthropic-ai":["partial","*",21,0],"PerplexityBot":["partial","*",21,0],"Perplexity-User":["partial","*",21,0],"Google-Extended":["partial","*",21,0],"Applebot-Extended":["partial","*",21,0],"CCBot":["partial","*",21,0],"Bytespider":["partial","*",21,0],"Amazonbot":["partial","*",21,0],"meta-externalagent":["partial","*",21,0],"cohere-ai":["partial","*",21,0],"Diffbot":["partial","*",21,0],"ContextFetchBot":["partial","*",21,0]},"llms":{"status":"missing","http":404,"bytes":5071},"home":{"status":"ok","http":200,"url":"http://dtyuanyang.com/","html_tokens":19800,"markdown_tokens":90,"pruned_tokens":90}},
{"rank":8749,"domain":"churchcenter.com","origin":"https://churchcenter.com","robots":{"status":"ok","http":200,"url":"https://churchcenter.com/robots.txt","bytes":328,"groups":2},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["partial","*",3,0],"OAI-SearchBot":["partial","*",3,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",3,0],"Claude-SearchBot":["partial","*",3,0],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["partial","*",3,0],"Perplexity-User":["partial","*",3,0],"Google-Extended":["partial","*",3,0],"Applebot-Extended":["partial","*",3,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["partial","*",3,0],"Diffbot":["partial","*",3,0],"ContextFetchBot":["partial","*",3,0]},"llms":{"status":"missing","http":404,"bytes":9},"home":{"status":"ok","http":200,"url":"https://churchcenter.com/setup","html_tokens":2493,"markdown_tokens":22,"pruned_tokens":22}},
{"rank":8750,"domain":"loudersound.com","origin":"https://loudersound.com","robots":{"status":"ok","http":200,"url":"https://www.loudersound.com/robots.txt","bytes":1484,"groups":4},"bots":{"GPTBot":["partial","*",18,0],"ChatGPT-User":["partial","*",18,0],"OAI-SearchBot":["partial","*",18,0],"ClaudeBot":["partial","*",18,0],"Claude-User":["partial","*",18,0],"Claude-SearchBot":["partial","*",18,0],"anthropic-ai":["partial","*",18,0],"PerplexityBot":["partial","*",18,0],"Perplexity-User":["partial","*",18,0],"Google-Extended":["partial","*",18,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["partial","*",18,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["partial","*",18,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",18,0]},"llms":{"status":"missing","http":404,"bytes":481935},"home":{"status":"ok","http":200,"url":"https://www.loudersound.com/","html_tokens":234441,"markdown_tokens":60,"pruned_tokens":60}},
{"rank":8751,"domain":"politicopro.com","origin":"https://politicopro.com","robots":{"status":"ok","http":200,"url":"https://www.politicopro.com/robots.txt","bytes":547,"groups":1},"bots":{"GPTBot":["partial","*",16,1],"ChatGPT-User":["partial","*",16,1],"OAI-SearchBot":["partial","*",16,1],"ClaudeBot":["partial","*",16,1],"Claude-User":["partial","*",16,1],"Claude-SearchBot":["partial","*",16,1],"anthropic-ai":["partial","*",16,1],"PerplexityBot":["partial","*",16,1],"Perplexity-User":["partial","*",16,1],"Google-Extended":["partial","*",16,1],"Applebot-Extended":["partial","*",16,1],"CCBot":["partial","*",16,1],"Bytespider":["partial","*",16,1],"Amazonbot":["partial","*",16,1],"meta-externalagent":["partial","*",16,1],"cohere-ai":["partial","*",16,1],"Diffbot":["partial","*",16,1],"ContextFetchBot":["partial","*",16,1]},"llms":{"status":"http_error","http":403,"bytes":5568},"home":{"status":"http_error","http":403,"url":"https://politicopro.com/"}},
{"rank":8752,"domain":"artbasel.com","origin":"https://artbasel.com","robots":{"status":"ok","http":200,"url":"https://www.artbasel.com/robots.txt","bytes":203,"groups":4},"bots":{"GPTBot":["partial","*",3,1],"ChatGPT-User":["partial","*",3,1],"OAI-SearchBot":["partial","*",3,1],"ClaudeBot":["partial","*",3,1],"Claude-User":["partial","*",3,1],"Claude-SearchBot":["partial","*",3,1],"anthropic-ai":["partial","*",3,1],"PerplexityBot":["partial","*",3,1],"Perplexity-User":["partial","*",3,1],"Google-Extended":["partial","*",3,1],"Applebot-Extended":["partial","*",3,1],"CCBot":["partial","*",3,1],"Bytespider":["partial","*",3,1],"Amazonbot":["partial","*",3,1],"meta-externalagent":["partial","*",3,1],"cohere-ai":["partial","*",3,1],"Diffbot":["partial","*",3,1],"ContextFetchBot":["partial","*",3,1]},"llms":{"status":"missing","http":404,"bytes":236308},"home":{"status":"ok","http":200,"url":"https://www.artbasel.com/","html_tokens":73439,"markdown_tokens":102,"pruned_tokens":102}},
{"rank":8753,"domain":"ilpost.it","origin":"https://ilpost.it","robots":{"status":"ok","http":200,"url":"https://www.ilpost.it/robots.txt","bytes":576,"groups":4},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["partial","*",7,0],"OAI-SearchBot":["partial","*",7,0],"ClaudeBot":["partial","*",7,0],"Claude-User":["partial","*",7,0],"Claude-SearchBot":["partial","*",7,0],"anthropic-ai":["partial","*",7,0],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["partial","*",7,0],"Google-Extended":["partial","*",7,0],"Applebot-Extended":["partial","*",7,0],"CCBot":["partial","*",7,0],"Bytespider":["partial","*",7,0],"Amazonbot":["partial","*",7,0],"meta-externalagent":["partial","*",7,0],"cohere-ai":["partial","*",7,0],"Diffbot":["partial","*",7,0],"ContextFetchBot":["partial","*",7,0]},"llms":{"status":"missing","http":404,"bytes":1210},"home":{"status":"ok","http":200,"url":"https://www.ilpost.it/","html_tokens":68187,"markdown_tokens":182,"pruned_tokens":116}},
{"rank":8754,"domain":"good.is","origin":"https://good.is","robots":{"status":"ok","http":200,"url":"https://www.good.is/robots.txt","bytes":347,"groups":2},"bots":{"GPTBot":["partial","*",1,0],"ChatGPT-User":["partial","*",1,0],"OAI-SearchBot":["partial","*",1,0],"ClaudeBot":["partial","*",1,0],"Claude-User":["partial","*",1,0],"Claude-SearchBot":["partial","*",1,0],"anthropic-ai":["partial","*",1,0],"PerplexityBot":["partial","*",1,0],"Perplexity-User":["partial","*",1,0],"Google-Extended":["partial","*",1,0],"Applebot-Extended":["partial","*",1,0],"CCBot":["partial","*",1,0],"Bytespider":["partial","*",1,0],"Amazonbot":["partial","*",1,0],"meta-externalagent":["partial","*",1,0],"cohere-ai":["partial","*",1,0],"Diffbot":["partial","*",1,0],"ContextFetchBot":["partial","*",1,0]},"llms":{"status":"missing","http":404,"bytes":197607},"home":{"status":"ok","http":200,"url":"https://www.good.is/","html_tokens":59391,"markdown_tokens":36,"pruned_tokens":34}},
{"rank":8755,"domain":"people-press.org","origin":"https://people-press.org","robots":{"status":"missing","http":404,"url":"https://www.pewresearch.org/politics/robots.txt","bytes":287679,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":287673},"home":{"status":"error","error":"more than 5 redirects"}},
{"rank":8756,"domain":"zibll.com","origin":"https://zibll.com","robots":{"status":"ok","http":200,"url":"https://www.zibll.com/robots.txt","bytes":114,"groups":1},"bots":{"GPTBot":["partial","*",1,1],"ChatGPT-User":["partial","*",1,1],"OAI-SearchBot":["partial","*",1,1],"ClaudeBot":["partial","*",1,1],"Claude-User":["partial","*",1,1],"Claude-SearchBot":["partial","*",1,1],"anthropic-ai":["partial","*",1,1],"PerplexityBot":["partial","*",1,1],"Perplexity-User":["partial","*",1,1],"Google-Extended":["partial","*",1,1],"Applebot-Extended":["partial","*",1,1],"CCBot":["partial","*",1,1],"Bytespider":["partial","*",1,1],"Amazonbot":["partial","*",1,1],"meta-externalagent":["partial","*",1,1],"cohere-ai":["partial","*",1,1],"Diffbot":["partial","*",1,1],"ContextFetchBot":["partial","*",1,1]},"llms":{"status":"missing","http":404,"bytes":84654},"home":{"status":"ok","http":200,"url":"https://www.zibll.com/","html_tokens":31792,"markdown_tokens":30,"pruned_tokens":30}},
{"rank":8757,"domain":"proza.ru","origin":"https://proza.ru","robots":{"status":"ok","http":200,"url":"https://proza.ru/robots.txt","bytes":238,"groups":1},"bots":{"GPTBot":["partial","*",9,0],"ChatGPT-User":["partial","*",9,0],"OAI-SearchBot":["partial","*",9,0],"ClaudeBot":["partial","*",9,0],"Claude-User":["partial","*",9,0],"Claude-SearchBot":["partial","*",9,0],"anthropic-ai":["partial","*",9,0],"PerplexityBot":["partial","*",9,0],"Perplexity-User":["partial","*",9,0],"Google-Extended":["partial","*",9,0],"Applebot-Extended":["partial","*",9,0],"CCBot":["partial","*",9,0],"Bytespider":["partial","*",9,0],"Amazonbot":["partial","*",9,0],"meta-externalagent":["partial","*",9,0],"cohere-ai":["partial","*",9,0],"Diffbot":["partial","*",9,0],"ContextFetchBot":["partial","*",9,0]},"llms":{"status":"missing","http":404,"bytes":7212},"home":{"status":"ok","http":200,"url":"https://proza.ru/","html_tokens":6944,"markdown_tokens":131,"pruned_tokens":131}},
{"rank":8758,"domain":"guoxiaodonggroup.com","origin":"http://guoxiaodonggroup.com","robots":{"status":"missing","http":404,"url":"http://guoxiaodonggroup.com/robots.txt","bytes":4743,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":4739},"home":{"status":"ok","http":200,"url":"http://guoxiaodonggroup.com/","html_tokens":4551,"markdown_tokens":144,"pruned_tokens":144}},
{"rank":8759,"domain":"enc.net.cn","origin":"http://enc.net.cn","robots":{"status":"missing","http":404,"url":"http://enc.net.cn/robots.txt","bytes":315,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":315},"home":{"status":"http_error","http":404,"url":"http://enc.net.cn/"}},
{"rank":8760,"domain":"boardhost.com","origin":"https://boardhost.com","robots":{"status":"ok","http":200,"url":"https://boardhost.com/robots.txt","bytes":36,"groups":1},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"html","http":200,"bytes":1151},"home":{"status":"ok","http":200,"url":"https://boardhost.com/","html_tokens":1619,"markdown_tokens":182,"pruned_tokens":73}},
{"rank":8761,"domain":"uni-karlsruhe.de","origin":"https://www.uni-karlsruhe.de","robots":{"status":"ok","http":200,"url":"https://www.kit.edu/robots.txt","bytes":300,"groups":2},"bots":{"GPTBot":["partial","*",6,0],"ChatGPT-User":["partial","*",6,0],"OAI-SearchBot":["partial","*",6,0],"ClaudeBot":["partial","*",6,0],"Claude-User":["partial","*",6,0],"Claude-SearchBot":["partial","*",6,0],"anthropic-ai":["partial","*",6,0],"PerplexityBot":["partial","*",6,0],"Perplexity-User":["partial","*",6,0],"Google-Extended":["partial","*",6,0],"Applebot-Extended":["partial","*",6,0],"CCBot":["partial","*",6,0],"Bytespider":["partial","*",6,0],"Amazonbot":["partial","*",6,0],"meta-externalagent":["partial","*",6,0],"cohere-ai":["partial","*",6,0],"Diffbot":["partial","*",6,0],"ContextFetchBot":["partial","*",6,0]},"llms":{"status":"missing","http":404,"bytes":98946},"home":{"status":"ok","http":200,"url":"https://www.kit.edu/","html_tokens":18711,"markdown_tokens":209,"pruned_tokens":136}},
{"rank":8762,"domain":"ipe.ac.cn","origin":"https://www.ipe.ac.cn","robots":{"status":"missing","http":404,"url":"https://www.ipe.ac.cn/robots.txt","bytes":315,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":315},"home":{"status":"http_error","http":404,"url":"https://www.ipe.ac.cn/"}},
{"rank":8763,"domain":"dakuan.net","origin":"http://dakuan.net","robots":{"status":"missing","http":404,"url":"http://dakuan.net/robots.txt","bytes":4728,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":4724},"home":{"status":"ok","http":200,"url":"http://dakuan.net/","html_tokens":10412,"markdown_tokens":158,"pruned_tokens":158}},
{"rank":8764,"domain":"afternic.com","origin":"https://afternic.com","robots":{"status":"ok","http":200,"url":"https://www.afternic.com/robots.txt","bytes":85,"groups":1},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"missing","http":404,"bytes":328565},"home":{"status":"ok","http":200,"url":"https://www.afternic.com/","html_tokens":33726,"markdown_tokens":116,"pruned_tokens":35}},
{"rank":8765,"domain":"manageengine.com","origin":"https://manageengine.com","robots":{"status":"ok","http":200,"url":"https://www.manageengine.com/robots.txt","bytes":19833,"groups":1},"bots":{"GPTBot":["partial","named",336,1],"ChatGPT-User":["partial","named",336,1],"OAI-SearchBot":["partial","named",336,1],"ClaudeBot":["partial","named",336,1],"Claude-User":["partial","named",336,1],"Claude-SearchBot":["partial","named",336,1],"anthropic-ai":["partial","*",336,1],"PerplexityBot":["partial","named",336,1],"Perplexity-User":["partial","named",336,1],"Google-Extended":["partial","named",336,1],"Applebot-Extended":["partial","named",336,1],"CCBot":["partial","named",336,1],"Bytespider":["partial","named",336,1],"Amazonbot":["partial","named",336,1],"meta-externalagent":["partial","named",336,1],"cohere-ai":["partial","named",336,1],"Diffbot":["partial","named",336,1],"ContextFetchBot":["partial","*",336,1]},"llms":{"status":"too_large","error":"response larger than 5 MB"},"home":{"status":"ok","http":200,"url":"https://www.manageengine.com/","html_tokens":13285,"markdown_tokens":461,"pruned_tokens":35}},
{"rank":8766,"domain":"nyctourism.com","origin":"https://nyctourism.com","robots":{"status":"ok","http":200,"url":"https://www.nyctourism.com/robots.txt","bytes":347,"groups":1},"bots":{"GPTBot":["partial","*",6,0],"ChatGPT-User":["partial","*",6,0],"OAI-SearchBot":["partial","*",6,0],"ClaudeBot":["partial","*",6,0],"Claude-User":["partial","*",6,0],"Claude-SearchBot":["partial","*",6,0],"anthropic-ai":["partial","*",6,0],"PerplexityBot":["partial","*",6,0],"Perplexity-User":["partial","*",6,0],"Google-Extended":["partial","*",6,0],"Applebot-Extended":["partial","*",6,0],"CCBot":["partial","*",6,0],"Bytespider":["partial","*",6,0],"Amazonbot":["partial","*",6,0],"meta-externalagent":["partial","*",6,0],"cohere-ai":["partial","*",6,0],"Diffbot":["partial","*",6,0],"ContextFetchBot":["partial","*",6,0]},"llms":{"status":"http_error","http":500,"bytes":2105},"home":{"status":"ok","http":200,"url":"https://www.nyctourism.com/","html_tokens":195501,"markdown_tokens":243,"pruned_tokens":169}},
{"rank":8767,"domain":"mediaoptions.com","origin":"https://mediaoptions.com","robots":{"status":"ok","http":200,"url":"https://mediaoptions.com/robots.txt","bytes":528,"groups":1},"bots":{"GPTBot":["partial","*",15,2],"ChatGPT-User":["partial","*",15,2],"OAI-SearchBot":["partial","*",15,2],"ClaudeBot":["partial","*",15,2],"Claude-User":["partial","*",15,2],"Claude-SearchBot":["partial","*",15,2],"anthropic-ai":["partial","*",15,2],"PerplexityBot":["partial","*",15,2],"Perplexity-User":["partial","*",15,2],"Google-Extended":["partial","*",15,2],"Applebot-Extended":["partial","*",15,2],"CCBot":["partial","*",15,2],"Bytespider":["partial","*",15,2],"Amazonbot":["partial","*",15,2],"meta-externalagent":["partial","*",15,2],"cohere-ai":["partial","*",15,2],"Diffbot":["partial","*",15,2],"ContextFetchBot":["partial","*",15,2]},"llms":{"status":"http_error","http":403,"bytes":5569},"home":{"status":"http_error","http":403,"url":"https://mediaoptions.com/"}},
{"rank":8768,"domain":"odessa.ua","origin":"https://odessa.ua","robots":{"status":"timeout","http":null,"url":"https://odessa.ua/robots.txt","bytes":0,"groups":0,"error":"ConnectTimeout"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":8769,"domain":"pcauto.com.cn","origin":"https://pcauto.com.cn","robots":{"status":"html","http":200,"url":"https://www.pcauto.com.cn/","bytes":783074,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":8770,"domain":"koosoo.cn","origin":"https://koosoo.cn","robots":{"status":"unreachable","http":null,"url":"https://koosoo.cn/robots.txt","bytes":0,"groups":0,"error":"cannot resolve koosoo.cn"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":8771,"domain":"hnfxjt.com","origin":"http://hnfxjt.com","robots":{"status":"missing","http":404,"url":"http://hnfxjt.com/robots.txt","bytes":15,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":910},"home":{"status":"ok","http":200,"url":"http://hnfxjt.com/","html_tokens":42325,"markdown_tokens":16,"pruned_tokens":16}},
{"rank":8772,"domain":"accessnewswire.com","origin":"https://accessnewswire.com","robots":{"status":"ok","http":200,"url":"https://www.accessnewswire.com/robots.txt","bytes":286,"groups":1},"bots":{"GPTBot":["partial","*",11,0],"ChatGPT-User":["partial","*",11,0],"OAI-SearchBot":["partial","*",11,0],"ClaudeBot":["partial","*",11,0],"Claude-User":["partial","*",11,0],"Claude-SearchBot":["partial","*",11,0],"anthropic-ai":["partial","*",11,0],"PerplexityBot":["partial","*",11,0],"Perplexity-User":["partial","*",11,0],"Google-Extended":["partial","*",11,0],"Applebot-Extended":["partial","*",11,0],"CCBot":["partial","*",11,0],"Bytespider":["partial","*",11,0],"Amazonbot":["partial","*",11,0],"meta-externalagent":["partial","*",11,0],"cohere-ai":["partial","*",11,0],"Diffbot":["partial","*",11,0],"ContextFetchBot":["partial","*",11,0]},"llms":{"status":"present","http":200,"bytes":5871},"home":{"status":"http_error","http":403,"url":"https://accessnewswire.com/"}},
{"rank":8773,"domain":"google.sn","origin":"https://google.sn","robots":{"status":"ok","http":200,"url":"https://www.google.sn/robots.txt","bytes":6624,"groups":4},"bots":{"GPTBot":["partial","*",169,64],"ChatGPT-User":["partial","*",169,64],"OAI-SearchBot":["partial","*",169,64],"ClaudeBot":["partial","*",169,64],"Claude-User":["partial","*",169,64],"Claude-SearchBot":["partial","*",169,64],"anthropic-ai":["partial","*",169,64],"PerplexityBot":["partial","*",169,64],"Perplexity-User":["partial","*",169,64],"Google-Extended":["partial","*",169,64],"Applebot-Extended":["partial","*",169,64],"CCBot":["partial","*",169,64],"Bytespider":["partial","*",169,64],"Amazonbot":["partial","*",169,64],"meta-externalagent":["partial","*",169,64],"cohere-ai":["partial","*",169,64],"Diffbot":["partial","*",169,64],"ContextFetchBot":["partial","*",169,64]},"llms":{"status":"missing","http":404,"bytes":1569},"home":{"status":"ok","http":200,"url":"https://www.google.com/","html_tokens":21273,"markdown_tokens":30,"pruned_tokens":30}},
{"rank":8774,"domain":"degitec-jiangyin.com","origin":"https://degitec-jiangyin.com","robots":{"status":"missing","http":404,"url":"https://degitec-jiangyin.com/robots.txt","bytes":4741,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":4737},"home":{"status":"ok","http":200,"url":"https://degitec-jiangyin.com/","html_tokens":6605,"markdown_tokens":120,"pruned_tokens":120}},
{"rank":8775,"domain":"hqmachinery.net","origin":"http://hqmachinery.net","robots":{"status":"ok","http":200,"url":"http://hqmachinery.net/robots.txt","bytes":485,"groups":1},"bots":{"GPTBot":["partial","*",21,0],"ChatGPT-User":["partial","*",21,0],"OAI-SearchBot":["partial","*",21,0],"ClaudeBot":["partial","*",21,0],"Claude-User":["partial","*",21,0],"Claude-SearchBot":["partial","*",21,0],"anthropic-ai":["partial","*",21,0],"PerplexityBot":["partial","*",21,0],"Perplexity-User":["partial","*",21,0],"Google-Extended":["partial","*",21,0],"Applebot-Extended":["partial","*",21,0],"CCBot":["partial","*",21,0],"Bytespider":["partial","*",21,0],"Amazonbot":["partial","*",21,0],"meta-externalagent":["partial","*",21,0],"cohere-ai":["partial","*",21,0],"Diffbot":["partial","*",21,0],"ContextFetchBot":["partial","*",21,0]},"llms":{"status":"missing","http":404,"bytes":4734},"home":{"status":"ok","http":200,"url":"http://hqmachinery.net/","html_tokens":4850,"markdown_tokens":160,"pruned_tokens":160}},
{"rank":8776,"domain":"jiangpingjx.cn","origin":"http://jiangpingjx.cn","robots":{"status":"ok","http":200,"url":"http://jiangpingjx.cn/robots.txt","bytes":485,"groups":1},"bots":{"GPTBot":["partial","*",21,0],"ChatGPT-User":["partial","*",21,0],"OAI-SearchBot":["partial","*",21,0],"ClaudeBot":["partial","*",21,0],"Claude-User":["partial","*",21,0],"Claude-SearchBot":["partial","*",21,0],"anthropic-ai":["partial","*",21,0],"PerplexityBot":["partial","*",21,0],"Perplexity-User":["partial","*",21,0],"Google-Extended":["partial","*",21,0],"Applebot-Extended":["partial","*",21,0],"CCBot":["partial","*",21,0],"Bytespider":["partial","*",21,0],"Amazonbot":["partial","*",21,0],"meta-externalagent":["partial","*",21,0],"cohere-ai":["partial","*",21,0],"Diffbot":["partial","*",21,0],"ContextFetchBot":["partial","*",21,0]},"llms":{"status":"missing","http":404,"bytes":4733},"home":{"status":"ok","http":200,"url":"http://jiangpingjx.cn/","html_tokens":5180,"markdown_tokens":206,"pruned_tokens":171}},
{"rank":8777,"domain":"fcbayern.com","origin":"https://fcbayern.com","robots":{"status":"refused","http":403,"url":"https://fcbayern.com/robots.txt","bytes":3823,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":8778,"domain":"legal500.com","origin":"https://legal500.com","robots":{"status":"ok","http":200,"url":"https://www.legal500.com/robots.txt","bytes":79,"groups":1},"bots":{"GPTBot":["partial","*",1,0],"ChatGPT-User":["partial","*",1,0],"OAI-SearchBot":["partial","*",1,0],"ClaudeBot":["partial","*",1,0],"Claude-User":["partial","*",1,0],"Claude-SearchBot":["partial","*",1,0],"anthropic-ai":["partial","*",1,0],"PerplexityBot":["partial","*",1,0],"Perplexity-User":["partial","*",1,0],"Google-Extended":["partial","*",1,0],"Applebot-Extended":["partial","*",1,0],"CCBot":["partial","*",1,0],"Bytespider":["partial","*",1,0],"Amazonbot":["partial","*",1,0],"meta-externalagent":["partial","*",1,0],"cohere-ai":["partial","*",1,0],"Diffbot":["partial","*",1,0],"ContextFetchBot":["partial","*",1,0]},"llms":{"status":"missing","http":404,"bytes":104761},"home":{"status":"ok","http":200,"url":"https://www.legal500.com/","html_tokens":45361,"markdown_tokens":32,"pruned_tokens":32}},
{"rank":8779,"domain":"ewtn.com","origin":"https://ewtn.com","robots":{"status":"ok","http":200,"url":"https://www.ewtn.com/robots.txt","bytes":99,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":110192},"home":{"status":"ok","http":200,"url":"https://www.ewtn.com/","html_tokens":373846,"markdown_tokens":40,"pruned_tokens":10}},
{"rank":8780,"domain":"talentlms.com","origin":"https://talentlms.com","robots":{"status":"ok","http":200,"url":"https://www.talentlms.com/robots.txt","bytes":311,"groups":3},"bots":{"GPTBot":["allowed","named",0,1],"ChatGPT-User":["allowed","named",0,1],"OAI-SearchBot":["allowed","named",0,1],"ClaudeBot":["allowed","named",0,1],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","named",0,1],"Perplexity-User":["allowed","named",0,1],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"other_text","http":200,"bytes":5923},"home":{"status":"ok","http":200,"url":"https://www.talentlms.com/","html_tokens":91683,"markdown_tokens":608,"pruned_tokens":328}},
{"rank":8781,"domain":"group.nec","origin":"https://group.nec","robots":{"status":"refused","http":403,"url":"https://group.nec/robots.txt","bytes":373,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":8782,"domain":"fz-juelich.de","origin":"https://fz-juelich.de","robots":{"status":"ok","http":200,"url":"https://www.fz-juelich.de/robots.txt","bytes":38,"groups":1},"bots":{"GPTBot":["partial","*",1,0],"ChatGPT-User":["partial","*",1,0],"OAI-SearchBot":["partial","*",1,0],"ClaudeBot":["partial","*",1,0],"Claude-User":["partial","*",1,0],"Claude-SearchBot":["partial","*",1,0],"anthropic-ai":["partial","*",1,0],"PerplexityBot":["partial","*",1,0],"Perplexity-User":["partial","*",1,0],"Google-Extended":["partial","*",1,0],"Applebot-Extended":["partial","*",1,0],"CCBot":["partial","*",1,0],"Bytespider":["partial","*",1,0],"Amazonbot":["partial","*",1,0],"meta-externalagent":["partial","*",1,0],"cohere-ai":["partial","*",1,0],"Diffbot":["partial","*",1,0],"ContextFetchBot":["partial","*",1,0]},"llms":{"status":"missing","http":404,"bytes":164526},"home":{"status":"ok","http":200,"url":"https://www.fz-juelich.de/de","html_tokens":172324,"markdown_tokens":91,"pruned_tokens":91}},
{"rank":8783,"domain":"circleci.com","origin":"https://circleci.com","robots":{"status":"ok","http":200,"url":"https://circleci.com/robots.txt","bytes":995,"groups":2},"bots":{"GPTBot":["partial","*",7,0],"ChatGPT-User":["partial","named",6,0],"OAI-SearchBot":["partial","*",7,0],"ClaudeBot":["partial","*",7,0],"Claude-User":["partial","named",6,0],"Claude-SearchBot":["partial","*",7,0],"anthropic-ai":["partial","*",7,0],"PerplexityBot":["partial","*",7,0],"Perplexity-User":["partial","named",6,0],"Google-Extended":["partial","*",7,0],"Applebot-Extended":["partial","*",7,0],"CCBot":["partial","*",7,0],"Bytespider":["partial","*",7,0],"Amazonbot":["partial","*",7,0],"meta-externalagent":["partial","*",7,0],"cohere-ai":["partial","*",7,0],"Diffbot":["partial","*",7,0],"ContextFetchBot":["partial","*",7,0]},"llms":{"status":"present","http":200,"bytes":9806},"home":{"status":"ok","http":200,"url":"https://circleci.com/","html_tokens":149692,"markdown_tokens":2361,"pruned_tokens":420}},
{"rank":8784,"domain":"voguebusiness.com","origin":"https://voguebusiness.com","robots":{"status":"missing","http":404,"url":"https://www.vogue.com/business/robots.txt","bytes":792378,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":792208},"home":{"status":"ok","http":200,"url":"https://www.vogue.com/business","html_tokens":706171,"markdown_tokens":38,"pruned_tokens":38}},
{"rank":8785,"domain":"betify-casino-poker-fr.fr","origin":"https://betify-casino-poker-fr.fr","robots":{"status":"ok","http":200,"url":"https://betify-casino-poker-fr.fr/robots.txt","bytes":123,"groups":1},"bots":{"GPTBot":["partial","*",1,1],"ChatGPT-User":["partial","*",1,1],"OAI-SearchBot":["partial","*",1,1],"ClaudeBot":["partial","*",1,1],"Claude-User":["partial","*",1,1],"Claude-SearchBot":["partial","*",1,1],"anthropic-ai":["partial","*",1,1],"PerplexityBot":["partial","*",1,1],"Perplexity-User":["partial","*",1,1],"Google-Extended":["partial","*",1,1],"Applebot-Extended":["partial","*",1,1],"CCBot":["partial","*",1,1],"Bytespider":["partial","*",1,1],"Amazonbot":["partial","*",1,1],"meta-externalagent":["partial","*",1,1],"cohere-ai":["partial","*",1,1],"Diffbot":["partial","*",1,1],"ContextFetchBot":["partial","*",1,1]},"llms":{"status":"missing","http":404,"bytes":20472},"home":{"status":"ok","http":200,"url":"https://betify-casino-poker-fr.fr/","html_tokens":9918,"markdown_tokens":3658,"pruned_tokens":130}},
{"rank":8786,"domain":"sj.qq.com","origin":"https://sj.qq.com","robots":{"status":"ok","http":200,"url":"https://sj.qq.com/robots.txt","bytes":461,"groups":5},"bots":{"GPTBot":["partial","*",11,0],"ChatGPT-User":["partial","*",11,0],"OAI-SearchBot":["partial","*",11,0],"ClaudeBot":["partial","*",11,0],"Claude-User":["partial","*",11,0],"Claude-SearchBot":["partial","*",11,0],"anthropic-ai":["partial","*",11,0],"PerplexityBot":["partial","*",11,0],"Perplexity-User":["partial","*",11,0],"Google-Extended":["partial","*",11,0],"Applebot-Extended":["partial","*",11,0],"CCBot":["partial","*",11,0],"Bytespider":["partial","*",11,0],"Amazonbot":["partial","*",11,0],"meta-externalagent":["partial","*",11,0],"cohere-ai":["partial","*",11,0],"Diffbot":["partial","*",11,0],"ContextFetchBot":["partial","*",11,0]},"llms":{"status":"missing","http":404,"bytes":98971},"home":{"status":"ok","http":200,"url":"https://sj.qq.com/","html_tokens":186060,"markdown_tokens":4,"pruned_tokens":4}},
{"rank":8787,"domain":"markdownguide.org","origin":"https://markdownguide.org","robots":{"status":"ok","http":200,"url":"https://www.markdownguide.org/robots.txt","bytes":151,"groups":1},"bots":{"GPTBot":["partial","*",2,0],"ChatGPT-User":["partial","*",2,0],"OAI-SearchBot":["partial","*",2,0],"ClaudeBot":["partial","*",2,0],"Claude-User":["partial","*",2,0],"Claude-SearchBot":["partial","*",2,0],"anthropic-ai":["partial","*",2,0],"PerplexityBot":["partial","*",2,0],"Perplexity-User":["partial","*",2,0],"Google-Extended":["partial","*",2,0],"Applebot-Extended":["partial","*",2,0],"CCBot":["partial","*",2,0],"Bytespider":["partial","*",2,0],"Amazonbot":["partial","*",2,0],"meta-externalagent":["partial","*",2,0],"cohere-ai":["partial","*",2,0],"Diffbot":["partial","*",2,0],"ContextFetchBot":["partial","*",2,0]},"llms":{"status":"missing","http":404,"bytes":5888},"home":{"status":"ok","http":200,"url":"https://www.markdownguide.org/","html_tokens":1960,"markdown_tokens":169,"pruned_tokens":139}},
{"rank":8788,"domain":"bnu.edu.cn","origin":"https://www.bnu.edu.cn","robots":{"status":"missing","http":404,"url":"https://www.bnu.edu.cn/robots.txt","bytes":1094,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":1094},"home":{"status":"ok","http":200,"url":"https://www.bnu.edu.cn/","html_tokens":9175,"markdown_tokens":273,"pruned_tokens":103}},
{"rank":8789,"domain":"jyjcms.cn","origin":"http://jyjcms.cn","robots":{"status":"ok","http":200,"url":"http://jyjcms.cn/robots.txt","bytes":485,"groups":1},"bots":{"GPTBot":["partial","*",21,0],"ChatGPT-User":["partial","*",21,0],"OAI-SearchBot":["partial","*",21,0],"ClaudeBot":["partial","*",21,0],"Claude-User":["partial","*",21,0],"Claude-SearchBot":["partial","*",21,0],"anthropic-ai":["partial","*",21,0],"PerplexityBot":["partial","*",21,0],"Perplexity-User":["partial","*",21,0],"Google-Extended":["partial","*",21,0],"Applebot-Extended":["partial","*",21,0],"CCBot":["partial","*",21,0],"Bytespider":["partial","*",21,0],"Amazonbot":["partial","*",21,0],"meta-externalagent":["partial","*",21,0],"cohere-ai":["partial","*",21,0],"Diffbot":["partial","*",21,0],"ContextFetchBot":["partial","*",21,0]},"llms":{"status":"missing","http":404,"bytes":4723},"home":{"status":"ok","http":200,"url":"http://jyjcms.cn/","html_tokens":6295,"markdown_tokens":119,"pruned_tokens":120}},
{"rank":8790,"domain":"jsyraf.cn","origin":"http://jsyraf.cn","robots":{"status":"ok","http":200,"url":"http://jsyraf.cn/robots.txt","bytes":485,"groups":1},"bots":{"GPTBot":["partial","*",21,0],"ChatGPT-User":["partial","*",21,0],"OAI-SearchBot":["partial","*",21,0],"ClaudeBot":["partial","*",21,0],"Claude-User":["partial","*",21,0],"Claude-SearchBot":["partial","*",21,0],"anthropic-ai":["partial","*",21,0],"PerplexityBot":["partial","*",21,0],"Perplexity-User":["partial","*",21,0],"Google-Extended":["partial","*",21,0],"Applebot-Extended":["partial","*",21,0],"CCBot":["partial","*",21,0],"Bytespider":["partial","*",21,0],"Amazonbot":["partial","*",21,0],"meta-externalagent":["partial","*",21,0],"cohere-ai":["partial","*",21,0],"Diffbot":["partial","*",21,0],"ContextFetchBot":["partial","*",21,0]},"llms":{"status":"missing","http":404,"bytes":4723},"home":{"status":"ok","http":200,"url":"http://jsyraf.cn/","html_tokens":13041,"markdown_tokens":197,"pruned_tokens":160}},
{"rank":8791,"domain":"restaurant.org","origin":"https://restaurant.org","robots":{"status":"ok","http":200,"url":"https://restaurant.org/robots.txt","bytes":75,"groups":1},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"missing","http":404,"bytes":75479},"home":{"status":"ok","http":200,"url":"https://restaurant.org/","html_tokens":25419,"markdown_tokens":105,"pruned_tokens":105}},
{"rank":8792,"domain":"startpagina.nl","origin":"https://startpagina.nl","robots":{"status":"ok","http":200,"url":"https://www.startpagina.nl/robots.txt","bytes":364,"groups":2},"bots":{"GPTBot":["partial","*",1,0],"ChatGPT-User":["partial","*",1,0],"OAI-SearchBot":["partial","*",1,0],"ClaudeBot":["partial","*",1,0],"Claude-User":["partial","*",1,0],"Claude-SearchBot":["partial","*",1,0],"anthropic-ai":["partial","*",1,0],"PerplexityBot":["partial","*",1,0],"Perplexity-User":["partial","*",1,0],"Google-Extended":["partial","*",1,0],"Applebot-Extended":["partial","*",1,0],"CCBot":["partial","*",1,0],"Bytespider":["partial","*",1,0],"Amazonbot":["partial","*",1,0],"meta-externalagent":["partial","*",1,0],"cohere-ai":["partial","*",1,0],"Diffbot":["partial","*",1,0],"ContextFetchBot":["partial","*",1,0]},"llms":{"status":"missing","http":404,"bytes":2189063},"home":{"status":"ok","http":200,"url":"https://www.startpagina.nl/","html_tokens":547146,"markdown_tokens":82,"pruned_tokens":13}},
{"rank":8793,"domain":"fotki.com","origin":"https://fotki.com","robots":{"status":"ok","http":200,"url":"https://www.fotki.com/robots.txt","bytes":861,"groups":25},"bots":{"GPTBot":["blocked","*",1,0],"ChatGPT-User":["blocked","*",1,0],"OAI-SearchBot":["blocked","*",1,0],"ClaudeBot":["blocked","*",1,0],"Claude-User":["blocked","*",1,0],"Claude-SearchBot":["blocked","*",1,0],"anthropic-ai":["blocked","*",1,0],"PerplexityBot":["blocked","*",1,0],"Perplexity-User":["blocked","*",1,0],"Google-Extended":["blocked","*",1,0],"Applebot-Extended":["blocked","*",1,0],"CCBot":["blocked","*",1,0],"Bytespider":["allowed","named",0,1],"Amazonbot":["blocked","*",1,0],"meta-externalagent":["blocked","*",1,0],"cohere-ai":["blocked","*",1,0],"Diffbot":["blocked","*",1,0],"ContextFetchBot":["blocked","*",1,0]},"llms":{"status":"disallowed"},"home":{"status":"disallowed"}},
{"rank":8794,"domain":"shangjin88.com","origin":"http://shangjin88.com","robots":{"status":"ok","http":200,"url":"http://shangjin88.com/robots.txt","bytes":485,"groups":1},"bots":{"GPTBot":["partial","*",21,0],"ChatGPT-User":["partial","*",21,0],"OAI-SearchBot":["partial","*",21,0],"ClaudeBot":["partial","*",21,0],"Claude-User":["partial","*",21,0],"Claude-SearchBot":["partial","*",21,0],"anthropic-ai":["partial","*",21,0],"PerplexityBot":["partial","*",21,0],"Perplexity-User":["partial","*",21,0],"Google-Extended":["partial","*",21,0],"Applebot-Extended":["partial","*",21,0],"CCBot":["partial","*",21,0],"Bytespider":["partial","*",21,0],"Amazonbot":["partial","*",21,0],"meta-externalagent":["partial","*",21,0],"cohere-ai":["partial","*",21,0],"Diffbot":["partial","*",21,0],"ContextFetchBot":["partial","*",21,0]},"llms":{"status":"missing","http":404,"bytes":4732},"home":{"status":"ok","http":200,"url":"http://shangjin88.com/","html_tokens":5572,"markdown_tokens":139,"pruned_tokens":138}},
{"rank":8795,"domain":"techjury.net","origin":"https://techjury.net","robots":{"status":"ok","http":200,"url":"https://techjury.net/robots.txt","bytes":1035,"groups":1},"bots":{"GPTBot":["partial","*",21,4],"ChatGPT-User":["partial","*",21,4],"OAI-SearchBot":["partial","*",21,4],"ClaudeBot":["partial","*",21,4],"Claude-User":["partial","*",21,4],"Claude-SearchBot":["partial","*",21,4],"anthropic-ai":["partial","*",21,4],"PerplexityBot":["partial","*",21,4],"Perplexity-User":["partial","*",21,4],"Google-Extended":["partial","*",21,4],"Applebot-Extended":["partial","*",21,4],"CCBot":["partial","*",21,4],"Bytespider":["partial","*",21,4],"Amazonbot":["partial","*",21,4],"meta-externalagent":["partial","*",21,4],"cohere-ai":["partial","*",21,4],"Diffbot":["partial","*",21,4],"ContextFetchBot":["partial","*",21,4]},"llms":{"status":"missing","http":404,"bytes":42282},"home":{"status":"ok","http":200,"url":"https://techjury.net/","html_tokens":26027,"markdown_tokens":1254,"pruned_tokens":164}},
{"rank":8796,"domain":"google.sm","origin":"https://google.sm","robots":{"status":"ok","http":200,"url":"https://www.google.sm/robots.txt","bytes":6624,"groups":4},"bots":{"GPTBot":["partial","*",169,64],"ChatGPT-User":["partial","*",169,64],"OAI-SearchBot":["partial","*",169,64],"ClaudeBot":["partial","*",169,64],"Claude-User":["partial","*",169,64],"Claude-SearchBot":["partial","*",169,64],"anthropic-ai":["partial","*",169,64],"PerplexityBot":["partial","*",169,64],"Perplexity-User":["partial","*",169,64],"Google-Extended":["partial","*",169,64],"Applebot-Extended":["partial","*",169,64],"CCBot":["partial","*",169,64],"Bytespider":["partial","*",169,64],"Amazonbot":["partial","*",169,64],"meta-externalagent":["partial","*",169,64],"cohere-ai":["partial","*",169,64],"Diffbot":["partial","*",169,64],"ContextFetchBot":["partial","*",169,64]},"llms":{"status":"missing","http":404,"bytes":1569},"home":{"status":"ok","http":200,"url":"https://www.google.com/","html_tokens":21294,"markdown_tokens":30,"pruned_tokens":30}},
{"rank":8797,"domain":"armorgames.com","origin":"https://armorgames.com","robots":{"status":"ok","http":200,"url":"https://armorgames.com/robots.txt","bytes":773,"groups":2},"bots":{"GPTBot":["partial","*",14,2],"ChatGPT-User":["partial","*",14,2],"OAI-SearchBot":["partial","*",14,2],"ClaudeBot":["partial","*",14,2],"Claude-User":["partial","*",14,2],"Claude-SearchBot":["partial","*",14,2],"anthropic-ai":["partial","*",14,2],"PerplexityBot":["partial","*",14,2],"Perplexity-User":["partial","*",14,2],"Google-Extended":["partial","*",14,2],"Applebot-Extended":["partial","*",14,2],"CCBot":["partial","*",14,2],"Bytespider":["partial","*",14,2],"Amazonbot":["partial","*",14,2],"meta-externalagent":["partial","*",14,2],"cohere-ai":["partial","*",14,2],"Diffbot":["partial","*",14,2],"ContextFetchBot":["partial","*",14,2]},"llms":{"status":"missing","http":404,"bytes":29494},"home":{"status":"ok","http":200,"url":"https://armorgames.com/","html_tokens":59847,"markdown_tokens":174,"pruned_tokens":174}},
{"rank":8798,"domain":"jyjbjx.cn","origin":"http://jyjbjx.cn","robots":{"status":"ok","http":200,"url":"http://jyjbjx.cn/robots.txt","bytes":485,"groups":1},"bots":{"GPTBot":["partial","*",21,0],"ChatGPT-User":["partial","*",21,0],"OAI-SearchBot":["partial","*",21,0],"ClaudeBot":["partial","*",21,0],"Claude-User":["partial","*",21,0],"Claude-SearchBot":["partial","*",21,0],"anthropic-ai":["partial","*",21,0],"PerplexityBot":["partial","*",21,0],"Perplexity-User":["partial","*",21,0],"Google-Extended":["partial","*",21,0],"Applebot-Extended":["partial","*",21,0],"CCBot":["partial","*",21,0],"Bytespider":["partial","*",21,0],"Amazonbot":["partial","*",21,0],"meta-externalagent":["partial","*",21,0],"cohere-ai":["partial","*",21,0],"Diffbot":["partial","*",21,0],"ContextFetchBot":["partial","*",21,0]},"llms":{"status":"missing","http":404,"bytes":4723},"home":{"status":"ok","http":200,"url":"http://jyjbjx.cn/","html_tokens":10354,"markdown_tokens":59,"pruned_tokens":59}},
{"rank":8799,"domain":"carfax.eu","origin":"https://carfax.eu","robots":{"status":"ok","http":200,"url":"https://www.carfax.eu/robots.txt","bytes":372,"groups":2},"bots":{"GPTBot":["partial","*",12,0],"ChatGPT-User":["partial","*",12,0],"OAI-SearchBot":["partial","*",12,0],"ClaudeBot":["partial","*",12,0],"Claude-User":["partial","*",12,0],"Claude-SearchBot":["partial","*",12,0],"anthropic-ai":["partial","*",12,0],"PerplexityBot":["partial","*",12,0],"Perplexity-User":["partial","*",12,0],"Google-Extended":["partial","*",12,0],"Applebot-Extended":["partial","*",12,0],"CCBot":["partial","*",12,0],"Bytespider":["partial","*",12,0],"Amazonbot":["partial","*",12,0],"meta-externalagent":["partial","*",12,0],"cohere-ai":["partial","*",12,0],"Diffbot":["partial","*",12,0],"ContextFetchBot":["partial","*",12,0]},"llms":{"status":"missing","http":404,"bytes":100545},"home":{"status":"ok","http":200,"url":"https://www.carfax.eu/","html_tokens":58824,"markdown_tokens":2537,"pruned_tokens":451}},
{"rank":8800,"domain":"onlyoffice.com","origin":"https://onlyoffice.com","robots":{"status":"ok","http":200,"url":"https://www.onlyoffice.com/robots.txt","bytes":6689,"groups":1},"bots":{"GPTBot":["partial","*",188,0],"ChatGPT-User":["partial","*",188,0],"OAI-SearchBot":["partial","*",188,0],"ClaudeBot":["partial","*",188,0],"Claude-User":["partial","*",188,0],"Claude-SearchBot":["partial","*",188,0],"anthropic-ai":["partial","*",188,0],"PerplexityBot":["partial","*",188,0],"Perplexity-User":["partial","*",188,0],"Google-Extended":["partial","*",188,0],"Applebot-Extended":["partial","*",188,0],"CCBot":["partial","*",188,0],"Bytespider":["partial","*",188,0],"Amazonbot":["partial","*",188,0],"meta-externalagent":["partial","*",188,0],"cohere-ai":["partial","*",188,0],"Diffbot":["partial","*",188,0],"ContextFetchBot":["partial","*",188,0]},"llms":{"status":"missing","http":404,"bytes":27848},"home":{"status":"ok","http":200,"url":"https://www.onlyoffice.com/","html_tokens":36173,"markdown_tokens":109,"pruned_tokens":80}},
{"rank":8801,"domain":"w3c.org","origin":"http://w3c.org","robots":{"status":"ok","http":200,"url":"https://www.w3.org/robots.txt","bytes":3844,"groups":2},"bots":{"GPTBot":["partial","*",112,0],"ChatGPT-User":["partial","*",112,0],"OAI-SearchBot":["partial","*",112,0],"ClaudeBot":["partial","*",112,0],"Claude-User":["partial","*",112,0],"Claude-SearchBot":["partial","*",112,0],"anthropic-ai":["partial","*",112,0],"PerplexityBot":["partial","*",112,0],"Perplexity-User":["partial","*",112,0],"Google-Extended":["partial","*",112,0],"Applebot-Extended":["partial","*",112,0],"CCBot":["partial","*",112,0],"Bytespider":["partial","*",112,0],"Amazonbot":["partial","*",112,0],"meta-externalagent":["partial","*",112,0],"cohere-ai":["partial","*",112,0],"Diffbot":["partial","*",112,0],"ContextFetchBot":["partial","*",112,0]},"llms":{"status":"missing","http":404,"bytes":20893},"home":{"status":"ok","http":200,"url":"https://www.w3.org","html_tokens":12603,"markdown_tokens":378,"pruned_tokens":160}},
{"rank":8802,"domain":"verizon.net","origin":"https://www.verizon.net","robots":{"status":"html","http":200,"url":"https://www.verizon.com/","bytes":125207,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":8803,"domain":"xinmin.cn","origin":"https://xinmin.cn","robots":{"status":"ok","http":200,"url":"https://xinmin.cn/robots.txt","bytes":127,"groups":1},"bots":{"GPTBot":["allowed","*",0,1],"ChatGPT-User":["allowed","*",0,1],"OAI-SearchBot":["allowed","*",0,1],"ClaudeBot":["allowed","*",0,1],"Claude-User":["allowed","*",0,1],"Claude-SearchBot":["allowed","*",0,1],"anthropic-ai":["allowed","*",0,1],"PerplexityBot":["allowed","*",0,1],"Perplexity-User":["allowed","*",0,1],"Google-Extended":["allowed","*",0,1],"Applebot-Extended":["allowed","*",0,1],"CCBot":["allowed","*",0,1],"Bytespider":["allowed","*",0,1],"Amazonbot":["allowed","*",0,1],"meta-externalagent":["allowed","*",0,1],"cohere-ai":["allowed","*",0,1],"Diffbot":["allowed","*",0,1],"ContextFetchBot":["allowed","*",0,1]},"llms":{"status":"html","http":200,"bytes":32481},"home":{"status":"ok","http":200,"url":"https://xinmin.cn/","html_tokens":5039,"markdown_tokens":103,"pruned_tokens":17}},
{"rank":8804,"domain":"jinkailijx.cn","origin":"https://jinkailijx.cn","robots":{"status":"unreachable","http":null,"url":"https://jinkailijx.cn/robots.txt","bytes":0,"groups":0,"error":"cannot resolve jinkailijx.cn"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":8805,"domain":"clipchamp.com","origin":"https://clipchamp.com","robots":{"status":"ok","http":200,"url":"https://clipchamp.com/robots.txt","bytes":167,"groups":1},"bots":{"GPTBot":["partial","*",5,0],"ChatGPT-User":["partial","*",5,0],"OAI-SearchBot":["partial","*",5,0],"ClaudeBot":["partial","*",5,0],"Claude-User":["partial","*",5,0],"Claude-SearchBot":["partial","*",5,0],"anthropic-ai":["partial","*",5,0],"PerplexityBot":["partial","*",5,0],"Perplexity-User":["partial","*",5,0],"Google-Extended":["partial","*",5,0],"Applebot-Extended":["partial","*",5,0],"CCBot":["partial","*",5,0],"Bytespider":["partial","*",5,0],"Amazonbot":["partial","*",5,0],"meta-externalagent":["partial","*",5,0],"cohere-ai":["partial","*",5,0],"Diffbot":["partial","*",5,0],"ContextFetchBot":["partial","*",5,0]},"llms":{"status":"html","http":200,"bytes":320670},"home":{"status":"ok","http":200,"url":"https://clipchamp.com/en/","html_tokens":80142,"markdown_tokens":518,"pruned_tokens":211}},
{"rank":8806,"domain":"moj.gov.tw","origin":"https://www.moj.gov.tw","robots":{"status":"ok","http":200,"url":"https://www.moj.gov.tw/robots","bytes":52,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":6306},"home":{"status":"ok","http":200,"url":"https://www.moj.gov.tw/","html_tokens":63283,"markdown_tokens":8,"pruned_tokens":5}},
{"rank":8807,"domain":"dxc.com","origin":"https://dxc.com","robots":{"status":"ok","http":200,"url":"https://dxc.com/robots.txt","bytes":2237,"groups":5},"bots":{"GPTBot":["partial","*",67,9],"ChatGPT-User":["allowed","named",0,1],"OAI-SearchBot":["allowed","named",0,1],"ClaudeBot":["partial","*",67,9],"Claude-User":["partial","*",67,9],"Claude-SearchBot":["partial","*",67,9],"anthropic-ai":["partial","*",67,9],"PerplexityBot":["allowed","named",0,1],"Perplexity-User":["partial","*",67,9],"Google-Extended":["partial","*",67,9],"Applebot-Extended":["partial","*",67,9],"CCBot":["partial","*",67,9],"Bytespider":["partial","*",67,9],"Amazonbot":["partial","*",67,9],"meta-externalagent":["partial","*",67,9],"cohere-ai":["partial","*",67,9],"Diffbot":["partial","*",67,9],"ContextFetchBot":["partial","*",67,9]},"llms":{"status":"present","http":200,"bytes":1341},"home":{"status":"ok","http":200,"url":"https://dxc.com/","html_tokens":49767,"markdown_tokens":224,"pruned_tokens":166}},
{"rank":8808,"domain":"universalmusic.com","origin":"https://universalmusic.com","robots":{"status":"ok","http":200,"url":"https://www.universalmusic.com/robots.txt","bytes":110088,"groups":2},"bots":{"GPTBot":["blocked","named",1171,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"missing","http":404,"bytes":88811},"home":{"status":"ok","http":200,"url":"https://www.universalmusic.com/","html_tokens":25160,"markdown_tokens":159,"pruned_tokens":106}},
{"rank":8809,"domain":"terrapinn.com","origin":"https://www.terrapinn.com","robots":{"status":"ok","http":200,"url":"https://www.terrapinn.com/robots.txt","bytes":810,"groups":2},"bots":{"GPTBot":["partial","*",20,8],"ChatGPT-User":["partial","*",20,8],"OAI-SearchBot":["partial","*",20,8],"ClaudeBot":["partial","*",20,8],"Claude-User":["partial","*",20,8],"Claude-SearchBot":["partial","*",20,8],"anthropic-ai":["partial","*",20,8],"PerplexityBot":["partial","*",20,8],"Perplexity-User":["partial","*",20,8],"Google-Extended":["partial","*",20,8],"Applebot-Extended":["partial","*",20,8],"CCBot":["partial","*",20,8],"Bytespider":["partial","*",20,8],"Amazonbot":["partial","*",20,8],"meta-externalagent":["partial","*",20,8],"cohere-ai":["partial","*",20,8],"Diffbot":["partial","*",20,8],"ContextFetchBot":["partial","*",20,8]},"llms":{"status":"missing","http":404,"bytes":0},"home":{"status":"ok","http":200,"url":"https://www.terrapinn.com/","html_tokens":10059,"markdown_tokens":242,"pruned_tokens":7}},
{"rank":8810,"domain":"radiopublic.com","origin":"https://radiopublic.com","robots":{"status":"unreachable","http":null,"url":"https://radiopublic.com/robots.txt","bytes":0,"groups":0,"error":"cannot resolve radiopublic.com"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":8811,"domain":"challenge-hct.com","origin":"https://challenge-hct.com","robots":{"status":"timeout","http":null,"url":"https://challenge-hct.com/robots.txt","bytes":0,"groups":0,"error":"ConnectTimeout"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":8812,"domain":"government.bg","origin":"https://government.bg","robots":{"status":"ok","http":200,"url":"https://government.bg/robots.txt","bytes":24,"groups":1},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"html","http":200,"bytes":1992},"home":{"status":"ok","http":200,"url":"https://government.bg/","html_tokens":5426,"markdown_tokens":11,"pruned_tokens":11}},
{"rank":8813,"domain":"globalgiving.org","origin":"https://www.globalgiving.org","robots":{"status":"ok","http":200,"url":"https://www.globalgiving.org/robots.txt","bytes":283,"groups":1},"bots":{"GPTBot":["partial","*",8,0],"ChatGPT-User":["partial","*",8,0],"OAI-SearchBot":["partial","*",8,0],"ClaudeBot":["partial","*",8,0],"Claude-User":["partial","*",8,0],"Claude-SearchBot":["partial","*",8,0],"anthropic-ai":["partial","*",8,0],"PerplexityBot":["partial","*",8,0],"Perplexity-User":["partial","*",8,0],"Google-Extended":["partial","*",8,0],"Applebot-Extended":["partial","*",8,0],"CCBot":["partial","*",8,0],"Bytespider":["partial","*",8,0],"Amazonbot":["partial","*",8,0],"meta-externalagent":["partial","*",8,0],"cohere-ai":["partial","*",8,0],"Diffbot":["partial","*",8,0],"ContextFetchBot":["partial","*",8,0]},"llms":{"status":"http_error","http":403,"bytes":5381},"home":{"status":"http_error","http":403,"url":"https://www.globalgiving.org/"}},
{"rank":8814,"domain":"bsu.edu","origin":"https://bsu.edu","robots":{"status":"refused","http":403,"url":"https://bsu.edu/robots.txt","bytes":371,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":8815,"domain":"calvin.edu","origin":"https://calvin.edu","robots":{"status":"refused","http":403,"url":"https://calvin.edu/robots.txt","bytes":560,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":8816,"domain":"andreabai.com","origin":"https://andreabai.com","robots":{"status":"ok","http":200,"url":"https://andreabai.com/robots.txt","bytes":67,"groups":1},"bots":{"GPTBot":["allowed","*",0,1],"ChatGPT-User":["allowed","*",0,1],"OAI-SearchBot":["allowed","*",0,1],"ClaudeBot":["allowed","*",0,1],"Claude-User":["allowed","*",0,1],"Claude-SearchBot":["allowed","*",0,1],"anthropic-ai":["allowed","*",0,1],"PerplexityBot":["allowed","*",0,1],"Perplexity-User":["allowed","*",0,1],"Google-Extended":["allowed","*",0,1],"Applebot-Extended":["allowed","*",0,1],"CCBot":["allowed","*",0,1],"Bytespider":["allowed","*",0,1],"Amazonbot":["allowed","*",0,1],"meta-externalagent":["allowed","*",0,1],"cohere-ai":["allowed","*",0,1],"Diffbot":["allowed","*",0,1],"ContextFetchBot":["allowed","*",0,1]},"llms":{"status":"missing","http":404,"bytes":642},"home":{"status":"ok","http":200,"url":"https://andreabai.com/","html_tokens":40285,"markdown_tokens":22917,"pruned_tokens":114}},
{"rank":8817,"domain":"sinopet-xm.org","origin":"http://sinopet-xm.org","robots":{"status":"missing","http":404,"url":"http://sinopet-xm.org/robots.txt","bytes":4740,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":4736},"home":{"status":"ok","http":200,"url":"http://sinopet-xm.org/","html_tokens":13530,"markdown_tokens":153,"pruned_tokens":148}},
{"rank":8818,"domain":"bdkgroup.cn","origin":"http://bdkgroup.cn","robots":{"status":"ok","http":200,"url":"http://bdkgroup.cn/robots.txt","bytes":485,"groups":1},"bots":{"GPTBot":["partial","*",21,0],"ChatGPT-User":["partial","*",21,0],"OAI-SearchBot":["partial","*",21,0],"ClaudeBot":["partial","*",21,0],"Claude-User":["partial","*",21,0],"Claude-SearchBot":["partial","*",21,0],"anthropic-ai":["partial","*",21,0],"PerplexityBot":["partial","*",21,0],"Perplexity-User":["partial","*",21,0],"Google-Extended":["partial","*",21,0],"Applebot-Extended":["partial","*",21,0],"CCBot":["partial","*",21,0],"Bytespider":["partial","*",21,0],"Amazonbot":["partial","*",21,0],"meta-externalagent":["partial","*",21,0],"cohere-ai":["partial","*",21,0],"Diffbot":["partial","*",21,0],"ContextFetchBot":["partial","*",21,0]},"llms":{"status":"missing","http":404,"bytes":4727},"home":{"status":"ok","http":200,"url":"http://bdkgroup.cn/","html_tokens":4436,"markdown_tokens":8,"pruned_tokens":8}},
{"rank":8819,"domain":"chaint.net","origin":"https://chaint.net","robots":{"status":"unreachable","http":null,"url":"https://chaint.net/robots.txt","bytes":0,"groups":0,"error":"[SSL: CERTIFICATE_VERIFY_FAILED] certificate verify failed: Hostname mismatch, certificate is not valid for 'chaint.net'. (_ssl.c:1028)"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":8820,"domain":"cj.com","origin":"https://www.cj.com","robots":{"status":"ok","http":200,"url":"https://www.cj.com/robots.txt","bytes":420,"groups":1},"bots":{"GPTBot":["partial","*",14,0],"ChatGPT-User":["partial","*",14,0],"OAI-SearchBot":["partial","*",14,0],"ClaudeBot":["partial","*",14,0],"Claude-User":["partial","*",14,0],"Claude-SearchBot":["partial","*",14,0],"anthropic-ai":["partial","*",14,0],"PerplexityBot":["partial","*",14,0],"Perplexity-User":["partial","*",14,0],"Google-Extended":["partial","*",14,0],"Applebot-Extended":["partial","*",14,0],"CCBot":["partial","*",14,0],"Bytespider":["partial","*",14,0],"Amazonbot":["partial","*",14,0],"meta-externalagent":["partial","*",14,0],"cohere-ai":["partial","*",14,0],"Diffbot":["partial","*",14,0],"ContextFetchBot":["partial","*",14,0]},"llms":{"status":"missing","http":404,"bytes":39267},"home":{"status":"ok","http":200,"url":"https://www.cj.com/","html_tokens":46295,"markdown_tokens":208,"pruned_tokens":36}},
{"rank":8821,"domain":"xhslink.com","origin":"https://xhslink.com","robots":{"status":"missing","http":404,"url":"https://xhslink.com/robots.txt","bytes":18,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":18},"home":{"status":"disallowed","error":"robots.txt of www.xiaohongshu.com disallows /"}},
{"rank":8822,"domain":"najlepsieonlinekasino.casino","origin":"https://najlepsieonlinekasino.casino","robots":{"status":"ok","http":200,"url":"https://najlepsieonlinekasino.casino/robots.txt","bytes":82,"groups":1},"bots":{"GPTBot":["allowed","*",0,1],"ChatGPT-User":["allowed","*",0,1],"OAI-SearchBot":["allowed","*",0,1],"ClaudeBot":["allowed","*",0,1],"Claude-User":["allowed","*",0,1],"Claude-SearchBot":["allowed","*",0,1],"anthropic-ai":["allowed","*",0,1],"PerplexityBot":["allowed","*",0,1],"Perplexity-User":["allowed","*",0,1],"Google-Extended":["allowed","*",0,1],"Applebot-Extended":["allowed","*",0,1],"CCBot":["allowed","*",0,1],"Bytespider":["allowed","*",0,1],"Amazonbot":["allowed","*",0,1],"meta-externalagent":["allowed","*",0,1],"cohere-ai":["allowed","*",0,1],"Diffbot":["allowed","*",0,1],"ContextFetchBot":["allowed","*",0,1]},"llms":{"status":"missing","http":404,"bytes":657},"home":{"status":"ok","http":200,"url":"https://najlepsieonlinekasino.casino/","html_tokens":46452,"markdown_tokens":27688,"pruned_tokens":429}},
{"rank":8823,"domain":"dyhyjxzz.cn","origin":"http://dyhyjxzz.cn","robots":{"status":"ok","http":200,"url":"http://dyhyjxzz.cn/robots.txt","bytes":485,"groups":1},"bots":{"GPTBot":["partial","*",21,0],"ChatGPT-User":["partial","*",21,0],"OAI-SearchBot":["partial","*",21,0],"ClaudeBot":["partial","*",21,0],"Claude-User":["partial","*",21,0],"Claude-SearchBot":["partial","*",21,0],"anthropic-ai":["partial","*",21,0],"PerplexityBot":["partial","*",21,0],"Perplexity-User":["partial","*",21,0],"Google-Extended":["partial","*",21,0],"Applebot-Extended":["partial","*",21,0],"CCBot":["partial","*",21,0],"Bytespider":["partial","*",21,0],"Amazonbot":["partial","*",21,0],"meta-externalagent":["partial","*",21,0],"cohere-ai":["partial","*",21,0],"Diffbot":["partial","*",21,0],"ContextFetchBot":["partial","*",21,0]},"llms":{"status":"missing","http":404,"bytes":4727},"home":{"status":"ok","http":200,"url":"http://dyhyjxzz.cn/","html_tokens":5564,"markdown_tokens":144,"pruned_tokens":144}},
{"rank":8824,"domain":"bonfire.com","origin":"https://bonfire.com","robots":{"status":"ok","http":200,"url":"https://www.bonfire.com/robots.txt","bytes":2515,"groups":2},"bots":{"GPTBot":["partial","named",35,3],"ChatGPT-User":["partial","named",35,3],"OAI-SearchBot":["partial","named",35,3],"ClaudeBot":["partial","named",35,3],"Claude-User":["partial","*",14,3],"Claude-SearchBot":["partial","*",14,3],"anthropic-ai":["partial","*",14,3],"PerplexityBot":["partial","named",35,3],"Perplexity-User":["partial","*",14,3],"Google-Extended":["partial","named",35,3],"Applebot-Extended":["partial","*",14,3],"CCBot":["partial","*",14,3],"Bytespider":["partial","named",35,3],"Amazonbot":["partial","named",35,3],"meta-externalagent":["partial","named",35,3],"cohere-ai":["partial","*",14,3],"Diffbot":["partial","*",14,3],"ContextFetchBot":["partial","*",14,3]},"llms":{"status":"present","http":200,"bytes":5182},"home":{"status":"ok","http":200,"url":"https://www.bonfire.com/","html_tokens":160088,"markdown_tokens":74,"pruned_tokens":74}},
{"rank":8825,"domain":"ice.com","origin":"https://ice.com","robots":{"status":"ok","http":200,"url":"https://www.ice.com/robots.txt","bytes":308,"groups":1},"bots":{"GPTBot":["partial","*",9,0],"ChatGPT-User":["partial","*",9,0],"OAI-SearchBot":["partial","*",9,0],"ClaudeBot":["partial","*",9,0],"Claude-User":["partial","*",9,0],"Claude-SearchBot":["partial","*",9,0],"anthropic-ai":["partial","*",9,0],"PerplexityBot":["partial","*",9,0],"Perplexity-User":["partial","*",9,0],"Google-Extended":["partial","*",9,0],"Applebot-Extended":["partial","*",9,0],"CCBot":["partial","*",9,0],"Bytespider":["partial","*",9,0],"Amazonbot":["partial","*",9,0],"meta-externalagent":["partial","*",9,0],"cohere-ai":["partial","*",9,0],"Diffbot":["partial","*",9,0],"ContextFetchBot":["partial","*",9,0]},"llms":{"status":"missing","http":404,"bytes":66134},"home":{"status":"ok","http":200,"url":"https://www.ice.com/index","html_tokens":27983,"markdown_tokens":142,"pruned_tokens":96}},
{"rank":8826,"domain":"bdc.ca","origin":"https://bdc.ca","robots":{"status":"ok","http":200,"url":"https://www.bdc.ca/robots.txt","bytes":24616,"groups":2},"bots":{"GPTBot":["partial","*",83,262],"ChatGPT-User":["partial","*",83,262],"OAI-SearchBot":["partial","*",83,262],"ClaudeBot":["partial","*",83,262],"Claude-User":["partial","*",83,262],"Claude-SearchBot":["partial","*",83,262],"anthropic-ai":["partial","*",83,262],"PerplexityBot":["partial","*",83,262],"Perplexity-User":["partial","*",83,262],"Google-Extended":["partial","*",83,262],"Applebot-Extended":["partial","*",83,262],"CCBot":["partial","*",83,262],"Bytespider":["partial","*",83,262],"Amazonbot":["partial","*",83,262],"meta-externalagent":["partial","*",83,262],"cohere-ai":["partial","*",83,262],"Diffbot":["partial","*",83,262],"ContextFetchBot":["partial","*",83,262]},"llms":{"status":"missing","http":404,"bytes":296143},"home":{"status":"ok","http":200,"url":"https://www.bdc.ca/en","html_tokens":82445,"markdown_tokens":202,"pruned_tokens":94}},
{"rank":8827,"domain":"jjzlcc.com","origin":"http://jjzlcc.com","robots":{"status":"ok","http":200,"url":"http://jjzlcc.com/robots.txt","bytes":247,"groups":1},"bots":{"GPTBot":["partial","*",10,0],"ChatGPT-User":["partial","*",10,0],"OAI-SearchBot":["partial","*",10,0],"ClaudeBot":["partial","*",10,0],"Claude-User":["partial","*",10,0],"Claude-SearchBot":["partial","*",10,0],"anthropic-ai":["partial","*",10,0],"PerplexityBot":["partial","*",10,0],"Perplexity-User":["partial","*",10,0],"Google-Extended":["partial","*",10,0],"Applebot-Extended":["partial","*",10,0],"CCBot":["partial","*",10,0],"Bytespider":["partial","*",10,0],"Amazonbot":["partial","*",10,0],"meta-externalagent":["partial","*",10,0],"cohere-ai":["partial","*",10,0],"Diffbot":["partial","*",10,0],"ContextFetchBot":["partial","*",10,0]},"llms":{"status":"missing","http":404,"bytes":3693},"home":{"status":"ok","http":200,"url":"http://jjzlcc.com/","html_tokens":3754,"markdown_tokens":69,"pruned_tokens":69}},
{"rank":8828,"domain":"zpkeji.com","origin":"https://zpkeji.com","robots":{"status":"ok","http":200,"url":"https://zpkeji.com/robots.txt","bytes":116,"groups":2},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":4724},"home":{"status":"ok","http":200,"url":"https://zpkeji.com/","html_tokens":12295,"markdown_tokens":30,"pruned_tokens":25}},
{"rank":8829,"domain":"graphql.org","origin":"https://graphql.org","robots":{"status":"missing","http":404,"url":"https://graphql.org/robots.txt","bytes":45955,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"present","http":200,"bytes":9053},"home":{"status":"ok","http":200,"url":"https://graphql.org/","html_tokens":60706,"markdown_tokens":1283,"pruned_tokens":1199}},
{"rank":8830,"domain":"nanowrimo.org","origin":"https://nanowrimo.org","robots":{"status":"ok","http":200,"url":"https://nanowrimo.org/robots.txt","bytes":66,"groups":1},"bots":{"GPTBot":["allowed","*",0,1],"ChatGPT-User":["allowed","*",0,1],"OAI-SearchBot":["allowed","*",0,1],"ClaudeBot":["allowed","*",0,1],"Claude-User":["allowed","*",0,1],"Claude-SearchBot":["allowed","*",0,1],"anthropic-ai":["allowed","*",0,1],"PerplexityBot":["allowed","*",0,1],"Perplexity-User":["allowed","*",0,1],"Google-Extended":["allowed","*",0,1],"Applebot-Extended":["allowed","*",0,1],"CCBot":["allowed","*",0,1],"Bytespider":["allowed","*",0,1],"Amazonbot":["allowed","*",0,1],"meta-externalagent":["allowed","*",0,1],"cohere-ai":["allowed","*",0,1],"Diffbot":["allowed","*",0,1],"ContextFetchBot":["allowed","*",0,1]},"llms":{"status":"other_text","http":200,"bytes":65},"home":{"status":"ok","http":200,"url":"https://nanowrimo.org/","html_tokens":29,"markdown_tokens":0,"pruned_tokens":0}},
{"rank":8831,"domain":"zohopublic.eu","origin":"https://zohopublic.eu","robots":{"status":"missing","http":404,"url":"https://zohopublic.eu/robots.txt","bytes":105,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":105},"home":{"status":"not_html","error":"unsupported content type application/json"}},
{"rank":8832,"domain":"calstate.edu","origin":"https://calstate.edu","robots":{"status":"refused","http":403,"url":"https://www.calstate.edu/robots.txt","bytes":815484,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":8833,"domain":"weheartit.com","origin":"https://weheartit.com","robots":{"status":"refused","http":403,"url":"https://weheartit.com/robots.txt","bytes":4913,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":8834,"domain":"91tcw.cc","origin":"https://91tcw.cc","robots":{"status":"missing","http":404,"url":"https://91tcw.cc/robots.txt","bytes":162,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":162},"home":{"status":"ok","http":200,"url":"http://www.91tcw.cc","html_tokens":7579,"markdown_tokens":225,"pruned_tokens":56}},
{"rank":8835,"domain":"cecchina.com","origin":"https://cecchina.com","robots":{"status":"unreachable","http":null,"url":"https://cecchina.com/robots.txt","bytes":0,"groups":0,"error":"cannot resolve cecchina.com"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":8836,"domain":"yjtag.jp","origin":"https://yjtag.jp","robots":{"status":"missing","http":404,"url":"https://yjtag.jp/robots.txt","bytes":6802,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":6802},"home":{"status":"http_error","http":403,"url":"https://yjtag.jp/"}},
{"rank":8837,"domain":"politie.nl","origin":"https://politie.nl","robots":{"status":"ok","http":200,"url":"https://www.politie.nl/robots.txt","bytes":409,"groups":3},"bots":{"GPTBot":["partial","*",10,0],"ChatGPT-User":["partial","*",10,0],"OAI-SearchBot":["partial","*",10,0],"ClaudeBot":["partial","*",10,0],"Claude-User":["partial","*",10,0],"Claude-SearchBot":["partial","*",10,0],"anthropic-ai":["partial","*",10,0],"PerplexityBot":["partial","*",10,0],"Perplexity-User":["partial","*",10,0],"Google-Extended":["partial","*",10,0],"Applebot-Extended":["partial","*",10,0],"CCBot":["partial","*",10,0],"Bytespider":["partial","*",10,0],"Amazonbot":["partial","*",10,0],"meta-externalagent":["partial","*",10,0],"cohere-ai":["partial","*",10,0],"Diffbot":["partial","*",10,0],"ContextFetchBot":["partial","*",10,0]},"llms":{"status":"missing","http":404,"bytes":17393},"home":{"status":"ok","http":200,"url":"https://www.politie.nl/","html_tokens":21842,"markdown_tokens":11,"pruned_tokens":11}},
{"rank":8838,"domain":"facs.org","origin":"https://facs.org","robots":{"status":"ok","http":200,"url":"https://www.facs.org/robots.txt","bytes":76,"groups":1},"bots":{"GPTBot":["partial","*",1,0],"ChatGPT-User":["partial","*",1,0],"OAI-SearchBot":["partial","*",1,0],"ClaudeBot":["partial","*",1,0],"Claude-User":["partial","*",1,0],"Claude-SearchBot":["partial","*",1,0],"anthropic-ai":["partial","*",1,0],"PerplexityBot":["partial","*",1,0],"Perplexity-User":["partial","*",1,0],"Google-Extended":["partial","*",1,0],"Applebot-Extended":["partial","*",1,0],"CCBot":["partial","*",1,0],"Bytespider":["partial","*",1,0],"Amazonbot":["partial","*",1,0],"meta-externalagent":["partial","*",1,0],"cohere-ai":["partial","*",1,0],"Diffbot":["partial","*",1,0],"ContextFetchBot":["partial","*",1,0]},"llms":{"status":"html","http":200,"bytes":4986},"home":{"status":"ok","http":200,"url":"https://www.facs.org/","html_tokens":28094,"markdown_tokens":200,"pruned_tokens":137}},
{"rank":8839,"domain":"yudu.com","origin":"https://yudu.com","robots":{"status":"ok","http":200,"url":"https://www.yudu.com/robots.txt","bytes":165,"groups":1},"bots":{"GPTBot":["partial","*",5,0],"ChatGPT-User":["partial","*",5,0],"OAI-SearchBot":["partial","*",5,0],"ClaudeBot":["partial","*",5,0],"Claude-User":["partial","*",5,0],"Claude-SearchBot":["partial","*",5,0],"anthropic-ai":["partial","*",5,0],"PerplexityBot":["partial","*",5,0],"Perplexity-User":["partial","*",5,0],"Google-Extended":["partial","*",5,0],"Applebot-Extended":["partial","*",5,0],"CCBot":["partial","*",5,0],"Bytespider":["partial","*",5,0],"Amazonbot":["partial","*",5,0],"meta-externalagent":["partial","*",5,0],"cohere-ai":["partial","*",5,0],"Diffbot":["partial","*",5,0],"ContextFetchBot":["partial","*",5,0]},"llms":{"status":"missing","http":404,"bytes":44384},"home":{"status":"ok","http":200,"url":"https://www.yudu.com/","html_tokens":33334,"markdown_tokens":526,"pruned_tokens":153}},
{"rank":8840,"domain":"geekbench.com","origin":"https://geekbench.com","robots":{"status":"ok","http":200,"url":"https://www.geekbench.com/robots.txt","bytes":47,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":162},"home":{"status":"http_error","http":403,"url":"https://geekbench.com/"}},
{"rank":8841,"domain":"thenewstribune.com","origin":"https://thenewstribune.com","robots":{"status":"unreachable","http":null,"url":"https://thenewstribune.com/robots.txt","bytes":0,"groups":0,"error":"<StreamReset stream_id:1, error_code:2, remote_reset:True>"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":8842,"domain":"downforeveryoneorjustme.com","origin":"https://downforeveryoneorjustme.com","robots":{"status":"ok","http":200,"url":"https://downforeveryoneorjustme.com/robots.txt","bytes":77,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"html","http":200,"bytes":23524},"home":{"status":"http_error","http":403,"url":"https://downforeveryoneorjustme.com/"}},
{"rank":8843,"domain":"metooo.io","origin":"https://metooo.io","robots":{"status":"ok","http":200,"url":"https://www.metooo.io/robots.txt","bytes":156,"groups":1},"bots":{"GPTBot":["partial","*",2,0],"ChatGPT-User":["partial","*",2,0],"OAI-SearchBot":["partial","*",2,0],"ClaudeBot":["partial","*",2,0],"Claude-User":["partial","*",2,0],"Claude-SearchBot":["partial","*",2,0],"anthropic-ai":["partial","*",2,0],"PerplexityBot":["partial","*",2,0],"Perplexity-User":["partial","*",2,0],"Google-Extended":["partial","*",2,0],"Applebot-Extended":["partial","*",2,0],"CCBot":["partial","*",2,0],"Bytespider":["partial","*",2,0],"Amazonbot":["partial","*",2,0],"meta-externalagent":["partial","*",2,0],"cohere-ai":["partial","*",2,0],"Diffbot":["partial","*",2,0],"ContextFetchBot":["partial","*",2,0]},"llms":{"status":"missing","http":404,"bytes":2388},"home":{"status":"ok","http":200,"url":"https://www.metooo.io/","html_tokens":23314,"markdown_tokens":674,"pruned_tokens":329}},
{"rank":8844,"domain":"visitbritain.com","origin":"https://visitbritain.com","robots":{"status":"ok","http":200,"url":"https://www.visitbritain.com/robots.txt","bytes":1888,"groups":1},"bots":{"GPTBot":["partial","*",40,18],"ChatGPT-User":["partial","*",40,18],"OAI-SearchBot":["partial","*",40,18],"ClaudeBot":["partial","*",40,18],"Claude-User":["partial","*",40,18],"Claude-SearchBot":["partial","*",40,18],"anthropic-ai":["partial","*",40,18],"PerplexityBot":["partial","*",40,18],"Perplexity-User":["partial","*",40,18],"Google-Extended":["partial","*",40,18],"Applebot-Extended":["partial","*",40,18],"CCBot":["partial","*",40,18],"Bytespider":["partial","*",40,18],"Amazonbot":["partial","*",40,18],"meta-externalagent":["partial","*",40,18],"cohere-ai":["partial","*",40,18],"Diffbot":["partial","*",40,18],"ContextFetchBot":["partial","*",40,18]},"llms":{"status":"missing","http":404,"bytes":555},"home":{"status":"ok","http":200,"url":"https://www.visitbritain.com/en","html_tokens":147294,"markdown_tokens":961,"pruned_tokens":21}},
{"rank":8845,"domain":"aston.ac.uk","origin":"https://aston.ac.uk","robots":{"status":"ok","http":200,"url":"https://www.aston.ac.uk/robots.txt","bytes":2092,"groups":1},"bots":{"GPTBot":["partial","*",35,18],"ChatGPT-User":["partial","*",35,18],"OAI-SearchBot":["partial","*",35,18],"ClaudeBot":["partial","*",35,18],"Claude-User":["partial","*",35,18],"Claude-SearchBot":["partial","*",35,18],"anthropic-ai":["partial","*",35,18],"PerplexityBot":["partial","*",35,18],"Perplexity-User":["partial","*",35,18],"Google-Extended":["partial","*",35,18],"Applebot-Extended":["partial","*",35,18],"CCBot":["partial","*",35,18],"Bytespider":["partial","*",35,18],"Amazonbot":["partial","*",35,18],"meta-externalagent":["partial","*",35,18],"cohere-ai":["partial","*",35,18],"Diffbot":["partial","*",35,18],"ContextFetchBot":["partial","*",35,18]},"llms":{"status":"missing","http":404,"bytes":146},"home":{"status":"ok","http":200,"url":"https://www.aston.ac.uk/","html_tokens":33108,"markdown_tokens":405,"pruned_tokens":136}},
{"rank":8846,"domain":"section508.gov","origin":"https://section508.gov","robots":{"status":"ok","http":200,"url":"https://www.section508.gov/robots.txt","bytes":48,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":20841},"home":{"status":"ok","http":200,"url":"https://www.section508.gov/","html_tokens":8576,"markdown_tokens":224,"pruned_tokens":151}},
{"rank":8847,"domain":"dyhj66.cn","origin":"http://dyhj66.cn","robots":{"status":"ok","http":200,"url":"http://dyhj66.cn/robots.txt","bytes":485,"groups":1},"bots":{"GPTBot":["partial","*",21,0],"ChatGPT-User":["partial","*",21,0],"OAI-SearchBot":["partial","*",21,0],"ClaudeBot":["partial","*",21,0],"Claude-User":["partial","*",21,0],"Claude-SearchBot":["partial","*",21,0],"anthropic-ai":["partial","*",21,0],"PerplexityBot":["partial","*",21,0],"Perplexity-User":["partial","*",21,0],"Google-Extended":["partial","*",21,0],"Applebot-Extended":["partial","*",21,0],"CCBot":["partial","*",21,0],"Bytespider":["partial","*",21,0],"Amazonbot":["partial","*",21,0],"meta-externalagent":["partial","*",21,0],"cohere-ai":["partial","*",21,0],"Diffbot":["partial","*",21,0],"ContextFetchBot":["partial","*",21,0]},"llms":{"status":"missing","http":404,"bytes":4722},"home":{"status":"ok","http":200,"url":"http://dyhj66.cn/","html_tokens":13198,"markdown_tokens":53,"pruned_tokens":53}},
{"rank":8848,"domain":"kokusen.go.jp","origin":"https://www.kokusen.go.jp","robots":{"status":"missing","http":404,"url":"https://www.kokusen.go.jp/robots.txt","bytes":7212,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":7212},"home":{"status":"ok","http":200,"url":"https://www.kokusen.go.jp/","html_tokens":3512,"markdown_tokens":25,"pruned_tokens":19}},
{"rank":8849,"domain":"tether.to","origin":"https://tether.to","robots":{"status":"ok","http":200,"url":"https://tether.to/robots.txt","bytes":62,"groups":1},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"html","http":200,"bytes":4761},"home":{"status":"ok","http":200,"url":"https://tether.to/","html_tokens":1425,"markdown_tokens":0,"pruned_tokens":0}},
{"rank":8850,"domain":"onetonline.org","origin":"https://onetonline.org","robots":{"status":"ok","http":200,"url":"https://www.onetonline.org/robots.txt","bytes":678,"groups":3},"bots":{"GPTBot":["partial","*",23,0],"ChatGPT-User":["partial","*",23,0],"OAI-SearchBot":["partial","*",23,0],"ClaudeBot":["partial","*",23,0],"Claude-User":["partial","*",23,0],"Claude-SearchBot":["partial","*",23,0],"anthropic-ai":["partial","*",23,0],"PerplexityBot":["partial","*",23,0],"Perplexity-User":["partial","*",23,0],"Google-Extended":["partial","*",23,0],"Applebot-Extended":["partial","*",23,0],"CCBot":["partial","*",23,0],"Bytespider":["partial","*",23,0],"Amazonbot":["partial","*",23,0],"meta-externalagent":["partial","*",23,0],"cohere-ai":["partial","*",23,0],"Diffbot":["partial","*",23,0],"ContextFetchBot":["partial","*",23,0]},"llms":{"status":"missing","http":404,"bytes":32818},"home":{"status":"ok","http":200,"url":"https://www.onetonline.org/","html_tokens":20968,"markdown_tokens":775,"pruned_tokens":173}},
{"rank":8851,"domain":"tennessee.edu","origin":"https://tennessee.edu","robots":{"status":"ok","http":200,"url":"https://tennessee.edu/robots.txt","bytes":187,"groups":1},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"http_error","http":403,"bytes":5544},"home":{"status":"http_error","http":403,"url":"https://tennessee.edu/"}},
{"rank":8852,"domain":"renderosity.com","origin":"https://renderosity.com","robots":{"status":"ok","http":200,"url":"https://www.renderosity.com/robots.txt","bytes":523,"groups":1},"bots":{"GPTBot":["partial","*",19,0],"ChatGPT-User":["partial","*",19,0],"OAI-SearchBot":["partial","*",19,0],"ClaudeBot":["partial","*",19,0],"Claude-User":["partial","*",19,0],"Claude-SearchBot":["partial","*",19,0],"anthropic-ai":["partial","*",19,0],"PerplexityBot":["partial","*",19,0],"Perplexity-User":["partial","*",19,0],"Google-Extended":["partial","*",19,0],"Applebot-Extended":["partial","*",19,0],"CCBot":["partial","*",19,0],"Bytespider":["partial","*",19,0],"Amazonbot":["partial","*",19,0],"meta-externalagent":["partial","*",19,0],"cohere-ai":["partial","*",19,0],"Diffbot":["partial","*",19,0],"ContextFetchBot":["partial","*",19,0]},"llms":{"status":"missing","http":404,"bytes":47405},"home":{"status":"ok","http":200,"url":"https://www.renderosity.com/","html_tokens":69026,"markdown_tokens":168,"pruned_tokens":154}},
{"rank":8853,"domain":"valuewalk.com","origin":"https://valuewalk.com","robots":{"status":"ok","http":200,"url":"https://www.valuewalk.com/robots.txt","bytes":729,"groups":2},"bots":{"GPTBot":["partial","*",13,3],"ChatGPT-User":["partial","*",13,3],"OAI-SearchBot":["partial","*",13,3],"ClaudeBot":["partial","*",13,3],"Claude-User":["partial","*",13,3],"Claude-SearchBot":["partial","*",13,3],"anthropic-ai":["partial","*",13,3],"PerplexityBot":["partial","*",13,3],"Perplexity-User":["partial","*",13,3],"Google-Extended":["partial","*",13,3],"Applebot-Extended":["partial","*",13,3],"CCBot":["partial","*",13,3],"Bytespider":["partial","*",13,3],"Amazonbot":["partial","*",13,3],"meta-externalagent":["partial","*",13,3],"cohere-ai":["partial","*",13,3],"Diffbot":["partial","*",13,3],"ContextFetchBot":["partial","*",13,3]},"llms":{"status":"missing","http":404,"bytes":60506},"home":{"status":"http_error","http":403,"url":"https://valuewalk.com/"}},
{"rank":8854,"domain":"autoscout24.de","origin":"https://autoscout24.de","robots":{"status":"ok","http":200,"url":"https://www.autoscout24.de/robots.txt","bytes":2912,"groups":9},"bots":{"GPTBot":["partial","named",1,10],"ChatGPT-User":["partial","*",66,0],"OAI-SearchBot":["partial","*",66,0],"ClaudeBot":["partial","named",1,10],"Claude-User":["partial","*",66,0],"Claude-SearchBot":["partial","*",66,0],"anthropic-ai":["partial","*",66,0],"PerplexityBot":["partial","*",66,0],"Perplexity-User":["partial","*",66,0],"Google-Extended":["partial","named",1,10],"Applebot-Extended":["partial","named",1,10],"CCBot":["partial","named",1,10],"Bytespider":["partial","*",66,0],"Amazonbot":["partial","*",66,0],"meta-externalagent":["partial","*",66,0],"cohere-ai":["partial","*",66,0],"Diffbot":["partial","*",66,0],"ContextFetchBot":["partial","*",66,0]},"llms":{"status":"missing","http":404,"bytes":74646},"home":{"status":"ok","http":200,"url":"https://www.autoscout24.de/","html_tokens":255643,"markdown_tokens":781,"pruned_tokens":165}},
{"rank":8855,"domain":"erau.edu","origin":"https://erau.edu","robots":{"status":"ok","http":200,"url":"https://erau.edu/robots.txt","bytes":27,"groups":1},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"missing","http":404,"bytes":142991},"home":{"status":"ok","http":200,"url":"https://erau.edu/","html_tokens":48334,"markdown_tokens":264,"pruned_tokens":154}},
{"rank":8856,"domain":"emailmeform.com","origin":"https://emailmeform.com","robots":{"status":"ok","http":200,"url":"https://www.emailmeform.com/robots.txt","bytes":14,"groups":1},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"missing","http":404,"bytes":261008},"home":{"status":"ok","http":200,"url":"https://www.emailmeform.com","html_tokens":111942,"markdown_tokens":241,"pruned_tokens":28}},
{"rank":8857,"domain":"maxforceracing.com","origin":"https://maxforceracing.com","robots":{"status":"ok","http":200,"url":"https://maxforceracing.com/robots.txt","bytes":469,"groups":1},"bots":{"GPTBot":["partial","*",17,0],"ChatGPT-User":["partial","*",17,0],"OAI-SearchBot":["partial","*",17,0],"ClaudeBot":["partial","*",17,0],"Claude-User":["partial","*",17,0],"Claude-SearchBot":["partial","*",17,0],"anthropic-ai":["partial","*",17,0],"PerplexityBot":["partial","*",17,0],"Perplexity-User":["partial","*",17,0],"Google-Extended":["partial","*",17,0],"Applebot-Extended":["partial","*",17,0],"CCBot":["partial","*",17,0],"Bytespider":["partial","*",17,0],"Amazonbot":["partial","*",17,0],"meta-externalagent":["partial","*",17,0],"cohere-ai":["partial","*",17,0],"Diffbot":["partial","*",17,0],"ContextFetchBot":["partial","*",17,0]},"llms":{"status":"missing","http":404,"bytes":27615},"home":{"status":"ok","http":200,"url":"https://maxforceracing.com/","html_tokens":24419,"markdown_tokens":1927,"pruned_tokens":155}},
{"rank":8858,"domain":"sf-express.com","origin":"https://www.sf-express.com","robots":{"status":"missing","http":404,"url":"https://www.sf-express.com/robots.txt","bytes":4090,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":4090},"home":{"status":"ok","http":200,"url":"https://www.sf-express.com/","html_tokens":29,"markdown_tokens":0,"pruned_tokens":0}},
{"rank":8859,"domain":"tut.fi","origin":"https://tut.fi","robots":{"status":"html","http":200,"url":"https://www.tuni.fi/fi","bytes":148152,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":8860,"domain":"vxuebao.com","origin":"https://vxuebao.com","robots":{"status":"timeout","http":null,"url":"https://vxuebao.com/robots.txt","bytes":0,"groups":0,"error":"ConnectTimeout"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":8861,"domain":"jsyongsheng.cn","origin":"http://jsyongsheng.cn","robots":{"status":"ok","http":200,"url":"http://jsyongsheng.cn/robots.txt","bytes":249,"groups":1},"bots":{"GPTBot":["partial","*",10,0],"ChatGPT-User":["partial","*",10,0],"OAI-SearchBot":["partial","*",10,0],"ClaudeBot":["partial","*",10,0],"Claude-User":["partial","*",10,0],"Claude-SearchBot":["partial","*",10,0],"anthropic-ai":["partial","*",10,0],"PerplexityBot":["partial","*",10,0],"Perplexity-User":["partial","*",10,0],"Google-Extended":["partial","*",10,0],"Applebot-Extended":["partial","*",10,0],"CCBot":["partial","*",10,0],"Bytespider":["partial","*",10,0],"Amazonbot":["partial","*",10,0],"meta-externalagent":["partial","*",10,0],"cohere-ai":["partial","*",10,0],"Diffbot":["partial","*",10,0],"ContextFetchBot":["partial","*",10,0]},"llms":{"status":"missing","http":404,"bytes":3749},"home":{"status":"ok","http":200,"url":"http://jsyongsheng.cn/","html_tokens":7724,"markdown_tokens":99,"pruned_tokens":99}},
{"rank":8862,"domain":"jsvlsjx.com","origin":"http://jsvlsjx.com","robots":{"status":"missing","http":404,"url":"http://jsvlsjx.com/robots.txt","bytes":4713,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":4709},"home":{"status":"disallowed","error":"robots.txt of stopnote.vhostgo.com not readable (unreachable)"}},
{"rank":8863,"domain":"js-hongdi.com","origin":"http://js-hongdi.com","robots":{"status":"ok","http":200,"url":"http://js-hongdi.com/robots.txt","bytes":247,"groups":1},"bots":{"GPTBot":["partial","*",10,0],"ChatGPT-User":["partial","*",10,0],"OAI-SearchBot":["partial","*",10,0],"ClaudeBot":["partial","*",10,0],"Claude-User":["partial","*",10,0],"Claude-SearchBot":["partial","*",10,0],"anthropic-ai":["partial","*",10,0],"PerplexityBot":["partial","*",10,0],"Perplexity-User":["partial","*",10,0],"Google-Extended":["partial","*",10,0],"Applebot-Extended":["partial","*",10,0],"CCBot":["partial","*",10,0],"Bytespider":["partial","*",10,0],"Amazonbot":["partial","*",10,0],"meta-externalagent":["partial","*",10,0],"cohere-ai":["partial","*",10,0],"Diffbot":["partial","*",10,0],"ContextFetchBot":["partial","*",10,0]},"llms":{"status":"missing","http":404,"bytes":3693},"home":{"status":"ok","http":200,"url":"http://js-hongdi.com/","html_tokens":10814,"markdown_tokens":111,"pruned_tokens":111}},
{"rank":8864,"domain":"videogameschronicle.com","origin":"https://videogameschronicle.com","robots":{"status":"ok","http":200,"url":"https://www.videogameschronicle.com/robots.txt","bytes":267,"groups":1},"bots":{"GPTBot":["partial","*",1,1],"ChatGPT-User":["partial","*",1,1],"OAI-SearchBot":["partial","*",1,1],"ClaudeBot":["partial","*",1,1],"Claude-User":["partial","*",1,1],"Claude-SearchBot":["partial","*",1,1],"anthropic-ai":["partial","*",1,1],"PerplexityBot":["partial","*",1,1],"Perplexity-User":["partial","*",1,1],"Google-Extended":["partial","*",1,1],"Applebot-Extended":["partial","*",1,1],"CCBot":["partial","*",1,1],"Bytespider":["partial","*",1,1],"Amazonbot":["partial","*",1,1],"meta-externalagent":["partial","*",1,1],"cohere-ai":["partial","*",1,1],"Diffbot":["partial","*",1,1],"ContextFetchBot":["partial","*",1,1]},"llms":{"status":"missing","http":404,"bytes":38957},"home":{"status":"ok","http":200,"url":"https://www.videogameschronicle.com/","html_tokens":32349,"markdown_tokens":26,"pruned_tokens":26}},
{"rank":8865,"domain":"golf.com","origin":"https://golf.com","robots":{"status":"refused","http":403,"url":"https://golf.com/robots.txt","bytes":560,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":8866,"domain":"goop.com","origin":"https://goop.com","robots":{"status":"refused","http":403,"url":"https://goop.com/robots.txt","bytes":5524,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":8867,"domain":"washingtoncitypaper.com","origin":"https://washingtoncitypaper.com","robots":{"status":"ok","http":200,"url":"https://washingtoncitypaper.com/robots.txt","bytes":979,"groups":24},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["blocked","named",1,0],"OAI-SearchBot":["blocked","named",1,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["blocked","named",1,0],"Claude-SearchBot":["blocked","named",1,0],"anthropic-ai":["partial","*",1,2],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["partial","*",1,2],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["partial","*",1,2],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["partial","*",1,2],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["partial","*",1,2],"Diffbot":["partial","*",1,2],"ContextFetchBot":["partial","*",1,2]},"llms":{"status":"other_text","http":200,"bytes":4952},"home":{"status":"ok","http":200,"url":"https://washingtoncitypaper.com/","html_tokens":85629,"markdown_tokens":86,"pruned_tokens":20}},
{"rank":8868,"domain":"fiercebiotech.com","origin":"https://fiercebiotech.com","robots":{"status":"ok","http":200,"url":"https://www.fiercebiotech.com/robots.txt","bytes":1620,"groups":1},"bots":{"GPTBot":["partial","*",23,18],"ChatGPT-User":["partial","*",23,18],"OAI-SearchBot":["partial","*",23,18],"ClaudeBot":["partial","*",23,18],"Claude-User":["partial","*",23,18],"Claude-SearchBot":["partial","*",23,18],"anthropic-ai":["partial","*",23,18],"PerplexityBot":["partial","*",23,18],"Perplexity-User":["partial","*",23,18],"Google-Extended":["partial","*",23,18],"Applebot-Extended":["partial","*",23,18],"CCBot":["partial","*",23,18],"Bytespider":["partial","*",23,18],"Amazonbot":["partial","*",23,18],"meta-externalagent":["partial","*",23,18],"cohere-ai":["partial","*",23,18],"Diffbot":["partial","*",23,18],"ContextFetchBot":["partial","*",23,18]},"llms":{"status":"http_error","http":403,"bytes":507},"home":{"status":"http_error","http":403,"url":"https://fiercebiotech.com/"}},
{"rank":8869,"domain":"pcloud.com","origin":"https://pcloud.com","robots":{"status":"ok","http":200,"url":"https://www.pcloud.com/robots.txt","bytes":512,"groups":1},"bots":{"GPTBot":["partial","*",14,1],"ChatGPT-User":["partial","*",14,1],"OAI-SearchBot":["partial","*",14,1],"ClaudeBot":["partial","*",14,1],"Claude-User":["partial","*",14,1],"Claude-SearchBot":["partial","*",14,1],"anthropic-ai":["partial","*",14,1],"PerplexityBot":["partial","*",14,1],"Perplexity-User":["partial","*",14,1],"Google-Extended":["partial","*",14,1],"Applebot-Extended":["partial","*",14,1],"CCBot":["partial","*",14,1],"Bytespider":["partial","*",14,1],"Amazonbot":["partial","*",14,1],"meta-externalagent":["partial","*",14,1],"cohere-ai":["partial","*",14,1],"Diffbot":["partial","*",14,1],"ContextFetchBot":["partial","*",14,1]},"llms":{"status":"html","http":200,"bytes":555930},"home":{"status":"ok","http":200,"url":"https://www.pcloud.com/","html_tokens":138785,"markdown_tokens":74,"pruned_tokens":74}},
{"rank":8870,"domain":"dyhrsj.com","origin":"http://dyhrsj.com","robots":{"status":"ok","http":200,"url":"http://dyhrsj.com/robots.txt","bytes":485,"groups":1},"bots":{"GPTBot":["partial","*",21,0],"ChatGPT-User":["partial","*",21,0],"OAI-SearchBot":["partial","*",21,0],"ClaudeBot":["partial","*",21,0],"Claude-User":["partial","*",21,0],"Claude-SearchBot":["partial","*",21,0],"anthropic-ai":["partial","*",21,0],"PerplexityBot":["partial","*",21,0],"Perplexity-User":["partial","*",21,0],"Google-Extended":["partial","*",21,0],"Applebot-Extended":["partial","*",21,0],"CCBot":["partial","*",21,0],"Bytespider":["partial","*",21,0],"Amazonbot":["partial","*",21,0],"meta-externalagent":["partial","*",21,0],"cohere-ai":["partial","*",21,0],"Diffbot":["partial","*",21,0],"ContextFetchBot":["partial","*",21,0]},"llms":{"status":"missing","http":404,"bytes":4724},"home":{"status":"ok","http":200,"url":"http://dyhrsj.com/","html_tokens":18194,"markdown_tokens":52,"pruned_tokens":52}},
{"rank":8871,"domain":"yamanashi.jp","origin":"https://yamanashi.jp","robots":{"status":"unreachable","http":null,"url":"https://yamanashi.jp/robots.txt","bytes":0,"groups":0,"error":"cannot resolve yamanashi.jp"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":8872,"domain":"globalwitness.org","origin":"https://globalwitness.org","robots":{"status":"ok","http":200,"url":"https://globalwitness.org/robots.txt","bytes":79,"groups":1},"bots":{"GPTBot":["partial","*",1,0],"ChatGPT-User":["partial","*",1,0],"OAI-SearchBot":["partial","*",1,0],"ClaudeBot":["partial","*",1,0],"Claude-User":["partial","*",1,0],"Claude-SearchBot":["partial","*",1,0],"anthropic-ai":["partial","*",1,0],"PerplexityBot":["partial","*",1,0],"Perplexity-User":["partial","*",1,0],"Google-Extended":["partial","*",1,0],"Applebot-Extended":["partial","*",1,0],"CCBot":["partial","*",1,0],"Bytespider":["partial","*",1,0],"Amazonbot":["partial","*",1,0],"meta-externalagent":["partial","*",1,0],"cohere-ai":["partial","*",1,0],"Diffbot":["partial","*",1,0],"ContextFetchBot":["partial","*",1,0]},"llms":{"status":"missing","http":404,"bytes":59838},"home":{"status":"ok","http":200,"url":"https://globalwitness.org/en/","html_tokens":30286,"markdown_tokens":990,"pruned_tokens":96}},
{"rank":8873,"domain":"liputan6.com","origin":"https://liputan6.com","robots":{"status":"ok","http":200,"url":"https://www.liputan6.com/robots.txt","bytes":443,"groups":1},"bots":{"GPTBot":["partial","*",17,0],"ChatGPT-User":["partial","*",17,0],"OAI-SearchBot":["partial","*",17,0],"ClaudeBot":["partial","*",17,0],"Claude-User":["partial","*",17,0],"Claude-SearchBot":["partial","*",17,0],"anthropic-ai":["partial","*",17,0],"PerplexityBot":["partial","*",17,0],"Perplexity-User":["partial","*",17,0],"Google-Extended":["partial","*",17,0],"Applebot-Extended":["partial","*",17,0],"CCBot":["partial","*",17,0],"Bytespider":["partial","*",17,0],"Amazonbot":["partial","*",17,0],"meta-externalagent":["partial","*",17,0],"cohere-ai":["partial","*",17,0],"Diffbot":["partial","*",17,0],"ContextFetchBot":["partial","*",17,0]},"llms":{"status":"missing","http":404,"bytes":101593},"home":{"status":"ok","http":200,"url":"https://www.liputan6.com/","html_tokens":75383,"markdown_tokens":15,"pruned_tokens":15}},
{"rank":8874,"domain":"tubitak.gov.tr","origin":"https://tubitak.gov.tr","robots":{"status":"ok","http":200,"url":"https://tubitak.gov.tr/robots.txt","bytes":2027,"groups":1},"bots":{"GPTBot":["partial","*",34,18],"ChatGPT-User":["partial","*",34,18],"OAI-SearchBot":["partial","*",34,18],"ClaudeBot":["partial","*",34,18],"Claude-User":["partial","*",34,18],"Claude-SearchBot":["partial","*",34,18],"anthropic-ai":["partial","*",34,18],"PerplexityBot":["partial","*",34,18],"Perplexity-User":["partial","*",34,18],"Google-Extended":["partial","*",34,18],"Applebot-Extended":["partial","*",34,18],"CCBot":["partial","*",34,18],"Bytespider":["partial","*",34,18],"Amazonbot":["partial","*",34,18],"meta-externalagent":["partial","*",34,18],"cohere-ai":["partial","*",34,18],"Diffbot":["partial","*",34,18],"ContextFetchBot":["partial","*",34,18]},"llms":{"status":"html","http":200,"bytes":10358},"home":{"status":"ok","http":200,"url":"https://tubitak.gov.tr/tr","html_tokens":25126,"markdown_tokens":221,"pruned_tokens":125}},
{"rank":8875,"domain":"murata.com","origin":"https://murata.com","robots":{"status":"html","http":200,"url":"https://www.murata.com/","bytes":88819,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":8876,"domain":"kazanmarathon.org","origin":"https://kazanmarathon.org","robots":{"status":"refused","http":403,"url":"https://kazanmarathon.org/robots.txt","bytes":4917,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":8877,"domain":"piwik.org","origin":"https://piwik.org","robots":{"status":"ok","http":200,"url":"https://matomo.org/robots.txt","bytes":345,"groups":1},"bots":{"GPTBot":["partial","*",9,1],"ChatGPT-User":["partial","*",9,1],"OAI-SearchBot":["partial","*",9,1],"ClaudeBot":["partial","*",9,1],"Claude-User":["partial","*",9,1],"Claude-SearchBot":["partial","*",9,1],"anthropic-ai":["partial","*",9,1],"PerplexityBot":["partial","*",9,1],"Perplexity-User":["partial","*",9,1],"Google-Extended":["partial","*",9,1],"Applebot-Extended":["partial","*",9,1],"CCBot":["partial","*",9,1],"Bytespider":["partial","*",9,1],"Amazonbot":["partial","*",9,1],"meta-externalagent":["partial","*",9,1],"cohere-ai":["partial","*",9,1],"Diffbot":["partial","*",9,1],"ContextFetchBot":["partial","*",9,1]},"llms":{"status":"timeout","error":"timed out after 10 s"},"home":{"status":"timeout","error":"timed out after 15 s"}},
{"rank":8878,"domain":"rednet.cn","origin":"https://www.rednet.cn","robots":{"status":"ok","http":200,"url":"https://www.rednet.cn/robots.txt","bytes":48,"groups":1},"bots":{"GPTBot":["partial","*",1,0],"ChatGPT-User":["partial","*",1,0],"OAI-SearchBot":["partial","*",1,0],"ClaudeBot":["partial","*",1,0],"Claude-User":["partial","*",1,0],"Claude-SearchBot":["partial","*",1,0],"anthropic-ai":["partial","*",1,0],"PerplexityBot":["partial","*",1,0],"Perplexity-User":["partial","*",1,0],"Google-Extended":["partial","*",1,0],"Applebot-Extended":["partial","*",1,0],"CCBot":["partial","*",1,0],"Bytespider":["partial","*",1,0],"Amazonbot":["partial","*",1,0],"meta-externalagent":["partial","*",1,0],"cohere-ai":["partial","*",1,0],"Diffbot":["partial","*",1,0],"ContextFetchBot":["partial","*",1,0]},"llms":{"status":"missing","http":404,"bytes":1572},"home":{"status":"ok","http":200,"url":"https://www.rednet.cn/","html_tokens":41819,"markdown_tokens":1,"pruned_tokens":1}},
{"rank":8879,"domain":"china-heating.org.cn","origin":"https://china-heating.org.cn","robots":{"status":"ok","http":200,"url":"https://china-heating.org.cn/robots.txt","bytes":26,"groups":1},"bots":{"GPTBot":["blocked","*",1,0],"ChatGPT-User":["blocked","*",1,0],"OAI-SearchBot":["blocked","*",1,0],"ClaudeBot":["blocked","*",1,0],"Claude-User":["blocked","*",1,0],"Claude-SearchBot":["blocked","*",1,0],"anthropic-ai":["blocked","*",1,0],"PerplexityBot":["blocked","*",1,0],"Perplexity-User":["blocked","*",1,0],"Google-Extended":["blocked","*",1,0],"Applebot-Extended":["blocked","*",1,0],"CCBot":["blocked","*",1,0],"Bytespider":["blocked","*",1,0],"Amazonbot":["blocked","*",1,0],"meta-externalagent":["blocked","*",1,0],"cohere-ai":["blocked","*",1,0],"Diffbot":["blocked","*",1,0],"ContextFetchBot":["blocked","*",1,0]},"llms":{"status":"disallowed"},"home":{"status":"disallowed"}},
{"rank":8880,"domain":"interviewmagazine.com","origin":"https://interviewmagazine.com","robots":{"status":"ok","http":200,"url":"https://www.interviewmagazine.com/robots.txt","bytes":178,"groups":1},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"missing","http":404,"bytes":60},"home":{"status":"ok","http":200,"url":"https://www.interviewmagazine.com/","html_tokens":18210,"markdown_tokens":1151,"pruned_tokens":111}},
{"rank":8881,"domain":"welcometothejungle.com","origin":"https://welcometothejungle.com","robots":{"status":"ok","http":200,"url":"https://www.welcometothejungle.com/robots.txt","bytes":216,"groups":1},"bots":{"GPTBot":["partial","*",5,2],"ChatGPT-User":["partial","*",5,2],"OAI-SearchBot":["partial","*",5,2],"ClaudeBot":["partial","*",5,2],"Claude-User":["partial","*",5,2],"Claude-SearchBot":["partial","*",5,2],"anthropic-ai":["partial","*",5,2],"PerplexityBot":["partial","*",5,2],"Perplexity-User":["partial","*",5,2],"Google-Extended":["partial","*",5,2],"Applebot-Extended":["partial","*",5,2],"CCBot":["partial","*",5,2],"Bytespider":["partial","*",5,2],"Amazonbot":["partial","*",5,2],"meta-externalagent":["partial","*",5,2],"cohere-ai":["partial","*",5,2],"Diffbot":["partial","*",5,2],"ContextFetchBot":["partial","*",5,2]},"llms":{"status":"html","http":200,"bytes":31965},"home":{"status":"ok","http":200,"url":"https://www.welcometothejungle.com/en","html_tokens":133110,"markdown_tokens":502,"pruned_tokens":72}},
{"rank":8882,"domain":"spaceflightnow.com","origin":"https://spaceflightnow.com","robots":{"status":"ok","http":200,"url":"https://spaceflightnow.com/robots.txt","bytes":119,"groups":1},"bots":{"GPTBot":["partial","*",1,1],"ChatGPT-User":["partial","*",1,1],"OAI-SearchBot":["partial","*",1,1],"ClaudeBot":["partial","*",1,1],"Claude-User":["partial","*",1,1],"Claude-SearchBot":["partial","*",1,1],"anthropic-ai":["partial","*",1,1],"PerplexityBot":["partial","*",1,1],"Perplexity-User":["partial","*",1,1],"Google-Extended":["partial","*",1,1],"Applebot-Extended":["partial","*",1,1],"CCBot":["partial","*",1,1],"Bytespider":["partial","*",1,1],"Amazonbot":["partial","*",1,1],"meta-externalagent":["partial","*",1,1],"cohere-ai":["partial","*",1,1],"Diffbot":["partial","*",1,1],"ContextFetchBot":["partial","*",1,1]},"llms":{"status":"missing","http":404,"bytes":107258},"home":{"status":"ok","http":200,"url":"https://spaceflightnow.com/","html_tokens":28902,"markdown_tokens":649,"pruned_tokens":154}},
{"rank":8883,"domain":"heraldnet.com","origin":"https://heraldnet.com","robots":{"status":"ok","http":200,"url":"https://www.heraldnet.com/robots.txt","bytes":312,"groups":1},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"missing","http":404,"bytes":124441},"home":{"status":"ok","http":200,"url":"https://www.heraldnet.com/","html_tokens":69201,"markdown_tokens":37,"pruned_tokens":37}},
{"rank":8884,"domain":"calacademy.org","origin":"https://calacademy.org","robots":{"status":"ok","http":200,"url":"https://calacademy.org/robots.txt","bytes":2130,"groups":2},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["blocked","named",1,0],"OAI-SearchBot":["partial","*",42,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",42,0],"Claude-SearchBot":["partial","*",42,0],"anthropic-ai":["partial","*",42,0],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["partial","*",42,0],"Google-Extended":["partial","*",42,0],"Applebot-Extended":["partial","*",42,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["partial","*",42,0],"meta-externalagent":["partial","*",42,0],"cohere-ai":["partial","*",42,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",42,0]},"llms":{"status":"missing","http":404,"bytes":294},"home":{"status":"ok","http":200,"url":"https://www.calacademy.org/","html_tokens":24573,"markdown_tokens":188,"pruned_tokens":148}},
{"rank":8885,"domain":"fairphone.com","origin":"https://fairphone.com","robots":{"status":"ok","http":200,"url":"https://www.fairphone.com/robots.txt","bytes":1482,"groups":1},"bots":{"GPTBot":["partial","*",41,0],"ChatGPT-User":["partial","*",41,0],"OAI-SearchBot":["partial","*",41,0],"ClaudeBot":["partial","*",41,0],"Claude-User":["partial","*",41,0],"Claude-SearchBot":["partial","*",41,0],"anthropic-ai":["partial","*",41,0],"PerplexityBot":["partial","*",41,0],"Perplexity-User":["partial","*",41,0],"Google-Extended":["partial","*",41,0],"Applebot-Extended":["partial","*",41,0],"CCBot":["partial","*",41,0],"Bytespider":["partial","*",41,0],"Amazonbot":["partial","*",41,0],"meta-externalagent":["partial","*",41,0],"cohere-ai":["partial","*",41,0],"Diffbot":["partial","*",41,0],"ContextFetchBot":["partial","*",41,0]},"llms":{"status":"missing","http":404,"bytes":90297},"home":{"status":"ok","http":200,"url":"https://www.fairphone.com/","html_tokens":40968,"markdown_tokens":772,"pruned_tokens":134}},
{"rank":8886,"domain":"ujaen.es","origin":"https://ujaen.es","robots":{"status":"html","http":200,"url":"https://www.ujaen.es/","bytes":128441,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":8887,"domain":"acronymfinder.com","origin":"https://acronymfinder.com","robots":{"status":"ok","http":200,"url":"https://acronymfinder.com/robots.txt","bytes":987,"groups":4},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["blocked","named",1,0],"OAI-SearchBot":["blocked","named",1,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["blocked","named",1,0],"Claude-SearchBot":["blocked","named",1,0],"anthropic-ai":["partial","*",4,0],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["partial","*",4,0],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["partial","*",4,0],"Diffbot":["partial","*",4,0],"ContextFetchBot":["partial","*",4,0]},"llms":{"status":"missing","http":404,"bytes":423},"home":{"status":"http_error","http":403,"url":"https://acronymfinder.com/"}},
{"rank":8888,"domain":"thegiftsports.com","origin":"https://thegiftsports.com","robots":{"status":"ok","http":200,"url":"https://thegiftsports.com/robots.txt","bytes":2747,"groups":33},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["allowed","*",0,2],"OAI-SearchBot":["allowed","*",0,2],"ClaudeBot":["blocked","named",1,0],"Claude-User":["allowed","*",0,2],"Claude-SearchBot":["allowed","*",0,2],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["allowed","*",0,2],"Perplexity-User":["allowed","*",0,2],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["allowed","*",0,2]},"llms":{"status":"missing","http":404,"bytes":646},"home":{"status":"ok","http":200,"url":"https://thegiftsports.com/","html_tokens":42890,"markdown_tokens":24518,"pruned_tokens":142}},
{"rank":8889,"domain":"aetherherbs.com","origin":"https://aetherherbs.com","robots":{"status":"timeout","http":null,"url":"https://aetherherbs.com/robots.txt","bytes":0,"groups":0,"error":"ConnectTimeout"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":8890,"domain":"mp.br","origin":"https://mp.br","robots":{"status":"unreachable","http":null,"url":"https://mp.br/robots.txt","bytes":0,"groups":0,"error":"cannot resolve mp.br"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":8891,"domain":"deepai.org","origin":"https://deepai.org","robots":{"status":"ok","http":200,"url":"https://deepai.org/robots.txt","bytes":71,"groups":1},"bots":{"GPTBot":["partial","*",1,0],"ChatGPT-User":["partial","*",1,0],"OAI-SearchBot":["partial","*",1,0],"ClaudeBot":["partial","*",1,0],"Claude-User":["partial","*",1,0],"Claude-SearchBot":["partial","*",1,0],"anthropic-ai":["partial","*",1,0],"PerplexityBot":["partial","*",1,0],"Perplexity-User":["partial","*",1,0],"Google-Extended":["partial","*",1,0],"Applebot-Extended":["partial","*",1,0],"CCBot":["partial","*",1,0],"Bytespider":["partial","*",1,0],"Amazonbot":["partial","*",1,0],"meta-externalagent":["partial","*",1,0],"cohere-ai":["partial","*",1,0],"Diffbot":["partial","*",1,0],"ContextFetchBot":["partial","*",1,0]},"llms":{"status":"present","http":200,"bytes":405},"home":{"status":"ok","http":200,"url":"https://deepai.org/","html_tokens":114914,"markdown_tokens":1645,"pruned_tokens":503}},
{"rank":8892,"domain":"lycos.de","origin":"https://lycos.de","robots":{"status":"unreachable","http":null,"url":"https://lycos.de/robots.txt","bytes":0,"groups":0,"error":"cannot resolve lycos.de"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":8893,"domain":"jyclbj.cn","origin":"http://jyclbj.cn","robots":{"status":"ok","http":200,"url":"http://jyclbj.cn/robots.txt","bytes":485,"groups":1},"bots":{"GPTBot":["partial","*",21,0],"ChatGPT-User":["partial","*",21,0],"OAI-SearchBot":["partial","*",21,0],"ClaudeBot":["partial","*",21,0],"Claude-User":["partial","*",21,0],"Claude-SearchBot":["partial","*",21,0],"anthropic-ai":["partial","*",21,0],"PerplexityBot":["partial","*",21,0],"Perplexity-User":["partial","*",21,0],"Google-Extended":["partial","*",21,0],"Applebot-Extended":["partial","*",21,0],"CCBot":["partial","*",21,0],"Bytespider":["partial","*",21,0],"Amazonbot":["partial","*",21,0],"meta-externalagent":["partial","*",21,0],"cohere-ai":["partial","*",21,0],"Diffbot":["partial","*",21,0],"ContextFetchBot":["partial","*",21,0]},"llms":{"status":"missing","http":404,"bytes":4723},"home":{"status":"ok","http":200,"url":"http://jyclbj.cn/","html_tokens":11219,"markdown_tokens":45,"pruned_tokens":45}},
{"rank":8894,"domain":"jykalige.cn","origin":"http://jykalige.cn","robots":{"status":"ok","http":200,"url":"http://jykalige.cn/robots.txt","bytes":485,"groups":1},"bots":{"GPTBot":["partial","*",21,0],"ChatGPT-User":["partial","*",21,0],"OAI-SearchBot":["partial","*",21,0],"ClaudeBot":["partial","*",21,0],"Claude-User":["partial","*",21,0],"Claude-SearchBot":["partial","*",21,0],"anthropic-ai":["partial","*",21,0],"PerplexityBot":["partial","*",21,0],"Perplexity-User":["partial","*",21,0],"Google-Extended":["partial","*",21,0],"Applebot-Extended":["partial","*",21,0],"CCBot":["partial","*",21,0],"Bytespider":["partial","*",21,0],"Amazonbot":["partial","*",21,0],"meta-externalagent":["partial","*",21,0],"cohere-ai":["partial","*",21,0],"Diffbot":["partial","*",21,0],"ContextFetchBot":["partial","*",21,0]},"llms":{"status":"missing","http":404,"bytes":4727},"home":{"status":"ok","http":200,"url":"http://jykalige.cn/","html_tokens":18521,"markdown_tokens":98,"pruned_tokens":98}},
{"rank":8895,"domain":"fox2now.com","origin":"https://fox2now.com","robots":{"status":"ok","http":200,"url":"https://fox2now.com/robots.txt","bytes":2303,"groups":40},"bots":{"GPTBot":["blocked","named",2,0],"ChatGPT-User":["blocked","named",1,0],"OAI-SearchBot":["blocked","named",1,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",12,1],"Claude-SearchBot":["partial","*",12,1],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["blocked","named",2,0],"Perplexity-User":["partial","*",12,1],"Google-Extended":["blocked","named",2,0],"Applebot-Extended":["blocked","named",2,0],"CCBot":["blocked","named",2,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["partial","*",12,1],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",12,1]},"llms":{"status":"http_error","http":403,"bytes":6514},"home":{"status":"http_error","http":403,"url":"https://fox2now.com/"}},
{"rank":8896,"domain":"healthit.gov","origin":"https://healthit.gov","robots":{"status":"ok","http":200,"url":"https://healthit.gov/robots.txt","bytes":335,"groups":1},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"present","http":200,"bytes":2409},"home":{"status":"ok","http":200,"url":"https://healthit.gov/","html_tokens":114006,"markdown_tokens":281,"pruned_tokens":105}},
{"rank":8897,"domain":"greenpeace.de","origin":"https://greenpeace.de","robots":{"status":"ok","http":200,"url":"https://www.greenpeace.de/robots.txt","bytes":1998,"groups":1},"bots":{"GPTBot":["partial","*",21,0],"ChatGPT-User":["partial","*",21,0],"OAI-SearchBot":["partial","*",21,0],"ClaudeBot":["partial","*",21,0],"Claude-User":["partial","*",21,0],"Claude-SearchBot":["partial","*",21,0],"anthropic-ai":["partial","*",21,0],"PerplexityBot":["partial","*",21,0],"Perplexity-User":["partial","*",21,0],"Google-Extended":["partial","*",21,0],"Applebot-Extended":["partial","*",21,0],"CCBot":["partial","*",21,0],"Bytespider":["partial","*",21,0],"Amazonbot":["partial","*",21,0],"meta-externalagent":["partial","*",21,0],"cohere-ai":["partial","*",21,0],"Diffbot":["partial","*",21,0],"ContextFetchBot":["partial","*",21,0]},"llms":{"status":"http_error","http":403,"bytes":150},"home":{"status":"ok","http":200,"url":"https://www.greenpeace.de/","html_tokens":70159,"markdown_tokens":616,"pruned_tokens":250}},
{"rank":8898,"domain":"cgdev.org","origin":"https://cgdev.org","robots":{"status":"ok","http":200,"url":"https://www.cgdev.org/robots.txt","bytes":2460,"groups":1},"bots":{"GPTBot":["partial","*",39,22],"ChatGPT-User":["partial","*",39,22],"OAI-SearchBot":["partial","*",39,22],"ClaudeBot":["partial","*",39,22],"Claude-User":["partial","*",39,22],"Claude-SearchBot":["partial","*",39,22],"anthropic-ai":["partial","*",39,22],"PerplexityBot":["partial","*",39,22],"Perplexity-User":["partial","*",39,22],"Google-Extended":["partial","*",39,22],"Applebot-Extended":["partial","*",39,22],"CCBot":["partial","*",39,22],"Bytespider":["partial","*",39,22],"Amazonbot":["partial","*",39,22],"meta-externalagent":["partial","*",39,22],"cohere-ai":["partial","*",39,22],"Diffbot":["partial","*",39,22],"ContextFetchBot":["partial","*",39,22]},"llms":{"status":"missing","http":404,"bytes":548},"home":{"status":"http_error","http":403,"url":"https://cgdev.org/"}},
{"rank":8899,"domain":"nthu.edu.tw","origin":"https://www.nthu.edu.tw","robots":{"status":"ok","http":200,"url":"https://www.nthu.edu.tw/robots.txt","bytes":114,"groups":1},"bots":{"GPTBot":["partial","*",2,1],"ChatGPT-User":["partial","*",2,1],"OAI-SearchBot":["partial","*",2,1],"ClaudeBot":["partial","*",2,1],"Claude-User":["partial","*",2,1],"Claude-SearchBot":["partial","*",2,1],"anthropic-ai":["partial","*",2,1],"PerplexityBot":["partial","*",2,1],"Perplexity-User":["partial","*",2,1],"Google-Extended":["partial","*",2,1],"Applebot-Extended":["partial","*",2,1],"CCBot":["partial","*",2,1],"Bytespider":["partial","*",2,1],"Amazonbot":["partial","*",2,1],"meta-externalagent":["partial","*",2,1],"cohere-ai":["partial","*",2,1],"Diffbot":["partial","*",2,1],"ContextFetchBot":["partial","*",2,1]},"llms":{"status":"missing","http":404,"bytes":1250},"home":{"status":"ok","http":200,"url":"https://www.nthu.edu.tw/","html_tokens":7915,"markdown_tokens":24,"pruned_tokens":24}},
{"rank":8900,"domain":"lejia114.com","origin":"https://lejia114.com","robots":{"status":"unreachable","http":null,"url":"https://lejia114.com/robots.txt","bytes":0,"groups":0,"error":"cannot resolve lejia114.com"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":8901,"domain":"zbfcxx.cn","origin":"https://www.zbfcxx.cn","robots":{"status":"ok","http":200,"url":"https://www.zbfcxx.cn/robots.txt","bytes":57,"groups":1},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"present","http":200,"bytes":1524},"home":{"status":"ok","http":200,"url":"https://www.zbfcxx.cn/","html_tokens":13165,"markdown_tokens":132,"pruned_tokens":132}},
{"rank":8902,"domain":"aihw.gov.au","origin":"https://aihw.gov.au","robots":{"status":"refused","http":403,"url":"https://aihw.gov.au/robots.txt","bytes":5506,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":8903,"domain":"market.us","origin":"https://market.us","robots":{"status":"ok","http":200,"url":"https://market.us/robots.txt","bytes":734,"groups":4},"bots":{"GPTBot":["partial","*",9,1],"ChatGPT-User":["partial","*",9,1],"OAI-SearchBot":["partial","*",9,1],"ClaudeBot":["partial","*",9,1],"Claude-User":["partial","*",9,1],"Claude-SearchBot":["partial","*",9,1],"anthropic-ai":["partial","*",9,1],"PerplexityBot":["partial","*",9,1],"Perplexity-User":["partial","*",9,1],"Google-Extended":["partial","*",9,1],"Applebot-Extended":["partial","*",9,1],"CCBot":["partial","*",9,1],"Bytespider":["partial","*",9,1],"Amazonbot":["partial","*",9,1],"meta-externalagent":["partial","*",9,1],"cohere-ai":["partial","*",9,1],"Diffbot":["partial","*",9,1],"ContextFetchBot":["partial","*",9,1]},"llms":{"status":"other_text","http":200,"bytes":3934},"home":{"status":"ok","http":200,"url":"https://market.us/","html_tokens":58139,"markdown_tokens":635,"pruned_tokens":134}},
{"rank":8904,"domain":"website3.me","origin":"https://website3.me","robots":{"status":"ok","http":200,"url":"https://www.website3.me/robots.txt","bytes":320,"groups":1},"bots":{"GPTBot":["partial","*",4,9],"ChatGPT-User":["partial","*",4,9],"OAI-SearchBot":["partial","*",4,9],"ClaudeBot":["partial","*",4,9],"Claude-User":["partial","*",4,9],"Claude-SearchBot":["partial","*",4,9],"anthropic-ai":["partial","*",4,9],"PerplexityBot":["partial","*",4,9],"Perplexity-User":["partial","*",4,9],"Google-Extended":["partial","*",4,9],"Applebot-Extended":["partial","*",4,9],"CCBot":["partial","*",4,9],"Bytespider":["partial","*",4,9],"Amazonbot":["partial","*",4,9],"meta-externalagent":["partial","*",4,9],"cohere-ai":["partial","*",4,9],"Diffbot":["partial","*",4,9],"ContextFetchBot":["partial","*",4,9]},"llms":{"status":"missing","http":404,"bytes":2022},"home":{"status":"ok","http":200,"url":"https://www.website3.me/","html_tokens":53475,"markdown_tokens":331,"pruned_tokens":112}},
{"rank":8905,"domain":"kinja.com","origin":"https://kinja.com","robots":{"status":"refused","http":429,"url":"https://kinja.com/robots.txt","bytes":34224,"groups":0,"error":"HTTP 429"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":8906,"domain":"news.livedoor.com","origin":"https://news.livedoor.com","robots":{"status":"ok","http":200,"url":"https://news.livedoor.com/robots.txt","bytes":340,"groups":3},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["allowed","*",0,1],"OAI-SearchBot":["allowed","*",0,1],"ClaudeBot":["blocked","named",1,0],"Claude-User":["allowed","*",0,1],"Claude-SearchBot":["allowed","*",0,1],"anthropic-ai":["allowed","*",0,1],"PerplexityBot":["allowed","*",0,1],"Perplexity-User":["allowed","*",0,1],"Google-Extended":["allowed","*",0,1],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["allowed","*",0,1],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["allowed","*",0,1],"Diffbot":["allowed","*",0,1],"ContextFetchBot":["allowed","*",0,1]},"llms":{"status":"missing","http":404,"bytes":2771},"home":{"status":"ok","http":200,"url":"https://news.livedoor.com/","html_tokens":14131,"markdown_tokens":15,"pruned_tokens":15}},
{"rank":8907,"domain":"verbraucher-schlichter.de","origin":"https://verbraucher-schlichter.de","robots":{"status":"missing","http":404,"url":"https://www.universalschlichtungsstelle.de/robots.txt","bytes":153,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"html","http":200,"bytes":24593},"home":{"status":"ok","http":200,"url":"https://www.universalschlichtungsstelle.de/","html_tokens":6110,"markdown_tokens":126,"pruned_tokens":126}},
{"rank":8908,"domain":"campaign.gov.uk","origin":"https://campaign.gov.uk","robots":{"status":"ok","http":200,"url":"https://campaign.gov.uk/robots.txt","bytes":26,"groups":1},"bots":{"GPTBot":["blocked","*",1,0],"ChatGPT-User":["blocked","*",1,0],"OAI-SearchBot":["blocked","*",1,0],"ClaudeBot":["blocked","*",1,0],"Claude-User":["blocked","*",1,0],"Claude-SearchBot":["blocked","*",1,0],"anthropic-ai":["blocked","*",1,0],"PerplexityBot":["blocked","*",1,0],"Perplexity-User":["blocked","*",1,0],"Google-Extended":["blocked","*",1,0],"Applebot-Extended":["blocked","*",1,0],"CCBot":["blocked","*",1,0],"Bytespider":["blocked","*",1,0],"Amazonbot":["blocked","*",1,0],"meta-externalagent":["blocked","*",1,0],"cohere-ai":["blocked","*",1,0],"Diffbot":["blocked","*",1,0],"ContextFetchBot":["blocked","*",1,0]},"llms":{"status":"disallowed"},"home":{"status":"disallowed"}},
{"rank":8909,"domain":"firstthings.com","origin":"https://firstthings.com","robots":{"status":"ok","http":200,"url":"https://firstthings.com/robots.txt","bytes":530,"groups":2},"bots":{"GPTBot":["partial","*",10,1],"ChatGPT-User":["partial","*",10,1],"OAI-SearchBot":["partial","*",10,1],"ClaudeBot":["partial","*",10,1],"Claude-User":["partial","*",10,1],"Claude-SearchBot":["partial","*",10,1],"anthropic-ai":["partial","*",10,1],"PerplexityBot":["partial","*",10,1],"Perplexity-User":["partial","*",10,1],"Google-Extended":["partial","*",10,1],"Applebot-Extended":["partial","*",10,1],"CCBot":["partial","*",10,1],"Bytespider":["partial","*",10,1],"Amazonbot":["partial","*",10,1],"meta-externalagent":["partial","*",10,1],"cohere-ai":["partial","*",10,1],"Diffbot":["partial","*",10,1],"ContextFetchBot":["partial","*",10,1]},"llms":{"status":"missing","http":404,"bytes":136180},"home":{"status":"ok","http":200,"url":"https://firstthings.com/","html_tokens":132187,"markdown_tokens":103,"pruned_tokens":101}},
{"rank":8910,"domain":"moe.edu.cn","origin":"https://moe.edu.cn","robots":{"status":"unreachable","http":null,"url":"https://moe.edu.cn/robots.txt","bytes":0,"groups":0,"error":"cannot resolve moe.edu.cn"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":8911,"domain":"healthcareitnews.com","origin":"https://healthcareitnews.com","robots":{"status":"ok","http":200,"url":"https://www.healthcareitnews.com/robots.txt","bytes":2141,"groups":1},"bots":{"GPTBot":["partial","*",35,22],"ChatGPT-User":["partial","*",35,22],"OAI-SearchBot":["partial","*",35,22],"ClaudeBot":["partial","*",35,22],"Claude-User":["partial","*",35,22],"Claude-SearchBot":["partial","*",35,22],"anthropic-ai":["partial","*",35,22],"PerplexityBot":["partial","*",35,22],"Perplexity-User":["partial","*",35,22],"Google-Extended":["partial","*",35,22],"Applebot-Extended":["partial","*",35,22],"CCBot":["partial","*",35,22],"Bytespider":["partial","*",35,22],"Amazonbot":["partial","*",35,22],"meta-externalagent":["partial","*",35,22],"cohere-ai":["partial","*",35,22],"Diffbot":["partial","*",35,22],"ContextFetchBot":["partial","*",35,22]},"llms":{"status":"missing","http":404,"bytes":198},"home":{"status":"http_error","http":403,"url":"https://www.healthcareitnews.com/"}},
{"rank":8912,"domain":"81.cn","origin":"https://81.cn","robots":{"status":"missing","http":404,"url":"http://www.81.cn/robots.txt","bytes":94,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":94},"home":{"status":"ok","http":200,"url":"http://www.81.cn/","html_tokens":34896,"markdown_tokens":64,"pruned_tokens":64}},
{"rank":8913,"domain":"elushretail.com","origin":"https://elushretail.com","robots":{"status":"ok","http":200,"url":"https://elushretail.com/robots.txt","bytes":67,"groups":1},"bots":{"GPTBot":["partial","*",1,1],"ChatGPT-User":["partial","*",1,1],"OAI-SearchBot":["partial","*",1,1],"ClaudeBot":["partial","*",1,1],"Claude-User":["partial","*",1,1],"Claude-SearchBot":["partial","*",1,1],"anthropic-ai":["partial","*",1,1],"PerplexityBot":["partial","*",1,1],"Perplexity-User":["partial","*",1,1],"Google-Extended":["partial","*",1,1],"Applebot-Extended":["partial","*",1,1],"CCBot":["partial","*",1,1],"Bytespider":["partial","*",1,1],"Amazonbot":["partial","*",1,1],"meta-externalagent":["partial","*",1,1],"cohere-ai":["partial","*",1,1],"Diffbot":["partial","*",1,1],"ContextFetchBot":["partial","*",1,1]},"llms":{"status":"missing","http":404,"bytes":23659},"home":{"status":"ok","http":200,"url":"https://www.elushretail.com/","html_tokens":6961,"markdown_tokens":333,"pruned_tokens":91}},
{"rank":8914,"domain":"itftennis.com","origin":"https://itftennis.com","robots":{"status":"ok","http":200,"url":"https://www.itftennis.com/robots.txt","bytes":128,"groups":1},"bots":{"GPTBot":["partial","*",1,1],"ChatGPT-User":["partial","*",1,1],"OAI-SearchBot":["partial","*",1,1],"ClaudeBot":["partial","*",1,1],"Claude-User":["partial","*",1,1],"Claude-SearchBot":["partial","*",1,1],"anthropic-ai":["partial","*",1,1],"PerplexityBot":["partial","*",1,1],"Perplexity-User":["partial","*",1,1],"Google-Extended":["partial","*",1,1],"Applebot-Extended":["partial","*",1,1],"CCBot":["partial","*",1,1],"Bytespider":["partial","*",1,1],"Amazonbot":["partial","*",1,1],"meta-externalagent":["partial","*",1,1],"cohere-ai":["partial","*",1,1],"Diffbot":["partial","*",1,1],"ContextFetchBot":["partial","*",1,1]},"llms":{"status":"html","http":200,"bytes":212},"home":{"status":"ok","http":200,"url":"https://www.itftennis.com/","html_tokens":53,"markdown_tokens":0,"pruned_tokens":0}},
{"rank":8915,"domain":"hollywood.com","origin":"https://hollywood.com","robots":{"status":"ok","http":200,"url":"https://www.hollywood.com/robots.txt","bytes":175,"groups":1},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"missing","http":404,"bytes":168508},"home":{"status":"ok","http":200,"url":"https://www.hollywood.com/","html_tokens":63067,"markdown_tokens":6114,"pruned_tokens":696}},
{"rank":8916,"domain":"rehairhub.store","origin":"https://rehairhub.store","robots":{"status":"ok","http":200,"url":"https://rehairhub.store/robots.txt","bytes":76,"groups":1},"bots":{"GPTBot":["partial","*",1,0],"ChatGPT-User":["partial","*",1,0],"OAI-SearchBot":["partial","*",1,0],"ClaudeBot":["partial","*",1,0],"Claude-User":["partial","*",1,0],"Claude-SearchBot":["partial","*",1,0],"anthropic-ai":["partial","*",1,0],"PerplexityBot":["partial","*",1,0],"Perplexity-User":["partial","*",1,0],"Google-Extended":["partial","*",1,0],"Applebot-Extended":["partial","*",1,0],"CCBot":["partial","*",1,0],"Bytespider":["partial","*",1,0],"Amazonbot":["partial","*",1,0],"meta-externalagent":["partial","*",1,0],"cohere-ai":["partial","*",1,0],"Diffbot":["partial","*",1,0],"ContextFetchBot":["partial","*",1,0]},"llms":{"status":"missing","http":404,"bytes":644},"home":{"status":"ok","http":200,"url":"https://rehairhub.store/","html_tokens":24127,"markdown_tokens":1663,"pruned_tokens":163}},
{"rank":8917,"domain":"atos.net","origin":"https://atos.net","robots":{"status":"ok","http":200,"url":"https://atos.net/robots.txt","bytes":76,"groups":1},"bots":{"GPTBot":["partial","*",1,0],"ChatGPT-User":["partial","*",1,0],"OAI-SearchBot":["partial","*",1,0],"ClaudeBot":["partial","*",1,0],"Claude-User":["partial","*",1,0],"Claude-SearchBot":["partial","*",1,0],"anthropic-ai":["partial","*",1,0],"PerplexityBot":["partial","*",1,0],"Perplexity-User":["partial","*",1,0],"Google-Extended":["partial","*",1,0],"Applebot-Extended":["partial","*",1,0],"CCBot":["partial","*",1,0],"Bytespider":["partial","*",1,0],"Amazonbot":["partial","*",1,0],"meta-externalagent":["partial","*",1,0],"cohere-ai":["partial","*",1,0],"Diffbot":["partial","*",1,0],"ContextFetchBot":["partial","*",1,0]},"llms":{"status":"missing","http":404,"bytes":3368},"home":{"status":"ok","http":200,"url":"https://atos.net/en-us/usa","html_tokens":24669,"markdown_tokens":162,"pruned_tokens":162}},
{"rank":8918,"domain":"jyshengqifz.cn","origin":"http://jyshengqifz.cn","robots":{"status":"ok","http":200,"url":"http://jyshengqifz.cn/robots.txt","bytes":485,"groups":1},"bots":{"GPTBot":["partial","*",21,0],"ChatGPT-User":["partial","*",21,0],"OAI-SearchBot":["partial","*",21,0],"ClaudeBot":["partial","*",21,0],"Claude-User":["partial","*",21,0],"Claude-SearchBot":["partial","*",21,0],"anthropic-ai":["partial","*",21,0],"PerplexityBot":["partial","*",21,0],"Perplexity-User":["partial","*",21,0],"Google-Extended":["partial","*",21,0],"Applebot-Extended":["partial","*",21,0],"CCBot":["partial","*",21,0],"Bytespider":["partial","*",21,0],"Amazonbot":["partial","*",21,0],"meta-externalagent":["partial","*",21,0],"cohere-ai":["partial","*",21,0],"Diffbot":["partial","*",21,0],"ContextFetchBot":["partial","*",21,0]},"llms":{"status":"missing","http":404,"bytes":4733},"home":{"status":"ok","http":200,"url":"http://jyshengqifz.cn/","html_tokens":12065,"markdown_tokens":9,"pruned_tokens":9}},
{"rank":8919,"domain":"traveloka.com","origin":"https://traveloka.com","robots":{"status":"ok","http":200,"url":"https://www.traveloka.com/robots.txt","bytes":21243,"groups":23},"bots":{"GPTBot":["allowed","named",0,1],"ChatGPT-User":["allowed","named",0,1],"OAI-SearchBot":["allowed","named",0,1],"ClaudeBot":["allowed","named",0,1],"Claude-User":["allowed","named",0,1],"Claude-SearchBot":["allowed","named",0,1],"anthropic-ai":["partial","*",86,23],"PerplexityBot":["allowed","named",0,1],"Perplexity-User":["allowed","named",0,1],"Google-Extended":["allowed","named",0,1],"Applebot-Extended":["allowed","named",0,1],"CCBot":["partial","*",86,23],"Bytespider":["allowed","named",0,1],"Amazonbot":["allowed","named",0,1],"meta-externalagent":["allowed","named",0,1],"cohere-ai":["partial","*",86,23],"Diffbot":["partial","*",86,23],"ContextFetchBot":["partial","*",86,23]},"llms":{"status":"present","http":200,"bytes":9738},"home":{"status":"ok","http":200,"url":"https://www.traveloka.com/en-en","html_tokens":266002,"markdown_tokens":26,"pruned_tokens":26}},
{"rank":8920,"domain":"skatteverket.se","origin":"https://skatteverket.se","robots":{"status":"unreachable","http":null,"url":"https://skatteverket.se/robots.txt","bytes":0,"groups":0,"error":"ReadError"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":8921,"domain":"gottman.com","origin":"https://gottman.com","robots":{"status":"ok","http":200,"url":"https://gottman.com/robots.txt","bytes":257,"groups":1},"bots":{"GPTBot":["partial","*",4,3],"ChatGPT-User":["partial","*",4,3],"OAI-SearchBot":["partial","*",4,3],"ClaudeBot":["partial","*",4,3],"Claude-User":["partial","*",4,3],"Claude-SearchBot":["partial","*",4,3],"anthropic-ai":["partial","*",4,3],"PerplexityBot":["partial","*",4,3],"Perplexity-User":["partial","*",4,3],"Google-Extended":["partial","*",4,3],"Applebot-Extended":["partial","*",4,3],"CCBot":["partial","*",4,3],"Bytespider":["partial","*",4,3],"Amazonbot":["partial","*",4,3],"meta-externalagent":["partial","*",4,3],"cohere-ai":["partial","*",4,3],"Diffbot":["partial","*",4,3],"ContextFetchBot":["partial","*",4,3]},"llms":{"status":"missing","http":404,"bytes":312420},"home":{"status":"ok","http":200,"url":"https://www.gottman.com/","html_tokens":106339,"markdown_tokens":820,"pruned_tokens":395}},
{"rank":8922,"domain":"whitepages.com","origin":"https://whitepages.com","robots":{"status":"ok","http":200,"url":"https://www.whitepages.com/robots.txt","bytes":2975,"groups":2},"bots":{"GPTBot":["partial","*",14,80],"ChatGPT-User":["partial","*",14,80],"OAI-SearchBot":["partial","*",14,80],"ClaudeBot":["partial","*",14,80],"Claude-User":["partial","*",14,80],"Claude-SearchBot":["partial","*",14,80],"anthropic-ai":["partial","*",14,80],"PerplexityBot":["partial","*",14,80],"Perplexity-User":["partial","*",14,80],"Google-Extended":["partial","*",14,80],"Applebot-Extended":["partial","*",14,80],"CCBot":["partial","*",14,80],"Bytespider":["partial","*",14,80],"Amazonbot":["partial","*",14,80],"meta-externalagent":["partial","*",14,80],"cohere-ai":["partial","*",14,80],"Diffbot":["partial","*",14,80],"ContextFetchBot":["partial","*",14,80]},"llms":{"status":"http_error","http":403,"bytes":5550},"home":{"status":"ok","http":200,"url":"https://www.whitepages.com/","html_tokens":95228,"markdown_tokens":997,"pruned_tokens":974}},
{"rank":8923,"domain":"zdyyxnk.com","origin":"https://zdyyxnk.com","robots":{"status":"missing","http":404,"url":"https://zdyyxnk.com/robots.txt","bytes":1163,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":1163},"home":{"status":"ok","http":200,"url":"https://zdyyxnk.com/","html_tokens":8992,"markdown_tokens":109,"pruned_tokens":109}},
{"rank":8924,"domain":"fox13news.com","origin":"https://fox13news.com","robots":{"status":"ok","http":200,"url":"https://www.fox13news.com/robots.txt","bytes":810,"groups":7},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["partial","*",11,0],"OAI-SearchBot":["partial","*",11,0],"ClaudeBot":["partial","*",11,0],"Claude-User":["partial","*",11,0],"Claude-SearchBot":["partial","*",11,0],"anthropic-ai":["partial","*",11,0],"PerplexityBot":["partial","*",11,0],"Perplexity-User":["partial","*",11,0],"Google-Extended":["partial","*",11,0],"Applebot-Extended":["partial","*",11,0],"CCBot":["partial","*",11,0],"Bytespider":["partial","*",11,0],"Amazonbot":["partial","*",11,0],"meta-externalagent":["partial","*",11,0],"cohere-ai":["partial","*",11,0],"Diffbot":["partial","*",11,0],"ContextFetchBot":["partial","*",11,0]},"llms":{"status":"missing","http":404,"bytes":0},"home":{"status":"ok","http":200,"url":"https://www.fox13news.com/","html_tokens":123464,"markdown_tokens":194,"pruned_tokens":143}},
{"rank":8925,"domain":"indy100.com","origin":"https://indy100.com","robots":{"status":"ok","http":200,"url":"https://www.indy100.com/robots.txt","bytes":346,"groups":1},"bots":{"GPTBot":["partial","*",7,1],"ChatGPT-User":["partial","*",7,1],"OAI-SearchBot":["partial","*",7,1],"ClaudeBot":["partial","*",7,1],"Claude-User":["partial","*",7,1],"Claude-SearchBot":["partial","*",7,1],"anthropic-ai":["partial","*",7,1],"PerplexityBot":["partial","*",7,1],"Perplexity-User":["partial","*",7,1],"Google-Extended":["partial","*",7,1],"Applebot-Extended":["partial","*",7,1],"CCBot":["partial","*",7,1],"Bytespider":["partial","*",7,1],"Amazonbot":["partial","*",7,1],"meta-externalagent":["partial","*",7,1],"cohere-ai":["partial","*",7,1],"Diffbot":["partial","*",7,1],"ContextFetchBot":["partial","*",7,1]},"llms":{"status":"missing","http":404,"bytes":524907},"home":{"status":"ok","http":200,"url":"https://www.indy100.com/","html_tokens":151136,"markdown_tokens":492,"pruned_tokens":173}},
{"rank":8926,"domain":"clf.cn","origin":"https://clf.cn","robots":{"status":"unreachable","http":null,"url":"https://clf.cn/robots.txt","bytes":0,"groups":0,"error":"[SSL: CERTIFICATE_VERIFY_FAILED] certificate verify failed: Hostname mismatch, certificate is not valid for 'clf.cn'. (_ssl.c:1028)"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":8927,"domain":"google.ac","origin":"https://google.ac","robots":{"status":"ok","http":200,"url":"https://www.google.ac/robots.txt","bytes":6624,"groups":4},"bots":{"GPTBot":["partial","*",169,64],"ChatGPT-User":["partial","*",169,64],"OAI-SearchBot":["partial","*",169,64],"ClaudeBot":["partial","*",169,64],"Claude-User":["partial","*",169,64],"Claude-SearchBot":["partial","*",169,64],"anthropic-ai":["partial","*",169,64],"PerplexityBot":["partial","*",169,64],"Perplexity-User":["partial","*",169,64],"Google-Extended":["partial","*",169,64],"Applebot-Extended":["partial","*",169,64],"CCBot":["partial","*",169,64],"Bytespider":["partial","*",169,64],"Amazonbot":["partial","*",169,64],"meta-externalagent":["partial","*",169,64],"cohere-ai":["partial","*",169,64],"Diffbot":["partial","*",169,64],"ContextFetchBot":["partial","*",169,64]},"llms":{"status":"missing","http":404,"bytes":1569},"home":{"status":"ok","http":200,"url":"https://www.google.ac/","html_tokens":21534,"markdown_tokens":32,"pruned_tokens":32}},
{"rank":8928,"domain":"posthog.com","origin":"https://posthog.com","robots":{"status":"ok","http":200,"url":"https://posthog.com/robots.txt","bytes":209,"groups":4},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"present","http":200,"bytes":448488},"home":{"status":"ok","http":200,"url":"https://posthog.com/","html_tokens":430031,"markdown_tokens":806,"pruned_tokens":192}},
{"rank":8929,"domain":"rf.gd","origin":"https://rf.gd","robots":{"status":"missing","http":404,"url":"https://rf.gd/robots.txt","bytes":411,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":411},"home":{"status":"ok","http":200,"url":"https://rf.gd/","html_tokens":1789,"markdown_tokens":228,"pruned_tokens":134}},
{"rank":8930,"domain":"itp-forum.ru","origin":"https://itp-forum.ru","robots":{"status":"refused","http":403,"url":"https://itp-forum.ru/robots.txt","bytes":4912,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":8931,"domain":"cshchina.com","origin":"https://cshchina.com","robots":{"status":"unreachable","http":null,"url":"https://cshchina.com/robots.txt","bytes":0,"groups":0,"error":"cannot resolve cshchina.com"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":8932,"domain":"zxxjscp.com","origin":"https://zxxjscp.com","robots":{"status":"timeout","http":null,"url":"https://zxxjscp.com/robots.txt","bytes":0,"groups":0,"error":"ConnectTimeout"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":8933,"domain":"stake.com","origin":"https://stake.com","robots":{"status":"refused","http":403,"url":"https://stake.com/robots.txt","bytes":5504,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":8934,"domain":"lvmh.com","origin":"https://lvmh.com","robots":{"status":"ok","http":200,"url":"https://www.lvmh.com/robots.txt","bytes":153,"groups":1},"bots":{"GPTBot":["partial","*",3,1],"ChatGPT-User":["partial","*",3,1],"OAI-SearchBot":["partial","*",3,1],"ClaudeBot":["partial","*",3,1],"Claude-User":["partial","*",3,1],"Claude-SearchBot":["partial","*",3,1],"anthropic-ai":["partial","*",3,1],"PerplexityBot":["partial","*",3,1],"Perplexity-User":["partial","*",3,1],"Google-Extended":["partial","*",3,1],"Applebot-Extended":["partial","*",3,1],"CCBot":["partial","*",3,1],"Bytespider":["partial","*",3,1],"Amazonbot":["partial","*",3,1],"meta-externalagent":["partial","*",3,1],"cohere-ai":["partial","*",3,1],"Diffbot":["partial","*",3,1],"ContextFetchBot":["partial","*",3,1]},"llms":{"status":"missing","http":404,"bytes":13148},"home":{"status":"ok","http":200,"url":"https://www.lvmh.com/en","html_tokens":511371,"markdown_tokens":425,"pruned_tokens":420}},
{"rank":8935,"domain":"jeuxvideo.com","origin":"https://jeuxvideo.com","robots":{"status":"ok","http":200,"url":"https://www.jeuxvideo.com/robots.txt","bytes":1927,"groups":12},"bots":{"GPTBot":["partial","*",39,0],"ChatGPT-User":["partial","*",39,0],"OAI-SearchBot":["partial","*",39,0],"ClaudeBot":["partial","*",39,0],"Claude-User":["partial","*",39,0],"Claude-SearchBot":["partial","*",39,0],"anthropic-ai":["partial","*",39,0],"PerplexityBot":["partial","*",39,0],"Perplexity-User":["partial","*",39,0],"Google-Extended":["partial","*",39,0],"Applebot-Extended":["partial","*",39,0],"CCBot":["partial","*",39,0],"Bytespider":["partial","*",39,0],"Amazonbot":["partial","*",39,0],"meta-externalagent":["partial","*",39,0],"cohere-ai":["partial","*",39,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",39,0]},"llms":{"status":"http_error","http":403,"bytes":5523},"home":{"status":"http_error","http":403,"url":"https://jeuxvideo.com/"}},
{"rank":8936,"domain":"mcclatchydc.com","origin":"https://mcclatchydc.com","robots":{"status":"unreachable","http":null,"url":"https://mcclatchydc.com/robots.txt","bytes":0,"groups":0,"error":"<StreamReset stream_id:1, error_code:2, remote_reset:True>"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":8937,"domain":"ember-energy.org","origin":"https://ember-energy.org","robots":{"status":"ok","http":200,"url":"https://ember-energy.org/robots.txt","bytes":282,"groups":2},"bots":{"GPTBot":["partial","*",3,1],"ChatGPT-User":["partial","*",3,1],"OAI-SearchBot":["partial","*",3,1],"ClaudeBot":["partial","*",3,1],"Claude-User":["partial","*",3,1],"Claude-SearchBot":["partial","*",3,1],"anthropic-ai":["partial","*",3,1],"PerplexityBot":["partial","*",3,1],"Perplexity-User":["partial","*",3,1],"Google-Extended":["partial","*",3,1],"Applebot-Extended":["partial","*",3,1],"CCBot":["partial","*",3,1],"Bytespider":["partial","*",3,1],"Amazonbot":["partial","*",3,1],"meta-externalagent":["partial","*",3,1],"cohere-ai":["partial","*",3,1],"Diffbot":["partial","*",3,1],"ContextFetchBot":["partial","*",3,1]},"llms":{"status":"html","http":200,"bytes":86815},"home":{"status":"ok","http":200,"url":"https://ember-energy.org/","html_tokens":29990,"markdown_tokens":546,"pruned_tokens":145}},
{"rank":8938,"domain":"mail.com","origin":"https://mail.com","robots":{"status":"ok","http":200,"url":"https://www.mail.com/robots.txt","bytes":3560,"groups":90},"bots":{"GPTBot":["partial","*",6,0],"ChatGPT-User":["partial","*",6,0],"OAI-SearchBot":["partial","*",6,0],"ClaudeBot":["partial","*",6,0],"Claude-User":["partial","*",6,0],"Claude-SearchBot":["partial","*",6,0],"anthropic-ai":["partial","*",6,0],"PerplexityBot":["partial","*",6,0],"Perplexity-User":["partial","*",6,0],"Google-Extended":["partial","*",6,0],"Applebot-Extended":["partial","*",6,0],"CCBot":["partial","*",6,0],"Bytespider":["partial","*",6,0],"Amazonbot":["partial","*",6,0],"meta-externalagent":["partial","*",6,0],"cohere-ai":["partial","*",6,0],"Diffbot":["partial","*",6,0],"ContextFetchBot":["partial","*",6,0]},"llms":{"status":"missing","http":404,"bytes":121997},"home":{"status":"ok","http":200,"url":"https://www.mail.com/","html_tokens":36446,"markdown_tokens":41,"pruned_tokens":41}},
{"rank":8939,"domain":"rolls-royce.com","origin":"https://rolls-royce.com","robots":{"status":"refused","http":403,"url":"https://www.rolls-royce.com/robots.txt","bytes":4909,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":8940,"domain":"ceu.edu","origin":"https://ceu.edu","robots":{"status":"ok","http":200,"url":"https://www.ceu.edu/robots.txt","bytes":25989,"groups":674},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["blocked","named",1,0],"OAI-SearchBot":["partial","*",36,19],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",36,19],"Claude-SearchBot":["partial","*",36,19],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["partial","*",36,19],"Google-Extended":["partial","*",36,19],"Applebot-Extended":["partial","*",36,19],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["partial","*",36,19],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",36,19]},"llms":{"status":"missing","http":404,"bytes":185},"home":{"status":"ok","http":200,"url":"https://www.ceu.edu/","html_tokens":29776,"markdown_tokens":1169,"pruned_tokens":635}},
{"rank":8941,"domain":"ninemsn.com.au","origin":"https://ninemsn.com.au","robots":{"status":"html","http":200,"url":"https://www.nine.com.au/","bytes":1396010,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":8942,"domain":"cnzhongxing.com","origin":"https://cnzhongxing.com","robots":{"status":"timeout","http":null,"url":"https://cnzhongxing.com/robots.txt","bytes":0,"groups":0,"error":"ConnectTimeout"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":8943,"domain":"airc.cn","origin":"https://airc.cn","robots":{"status":"ok","http":200,"url":"https://airc.cn/robots.txt","bytes":488,"groups":1},"bots":{"GPTBot":["partial","*",17,0],"ChatGPT-User":["partial","*",17,0],"OAI-SearchBot":["partial","*",17,0],"ClaudeBot":["partial","*",17,0],"Claude-User":["partial","*",17,0],"Claude-SearchBot":["partial","*",17,0],"anthropic-ai":["partial","*",17,0],"PerplexityBot":["partial","*",17,0],"Perplexity-User":["partial","*",17,0],"Google-Extended":["partial","*",17,0],"Applebot-Extended":["partial","*",17,0],"CCBot":["partial","*",17,0],"Bytespider":["partial","*",17,0],"Amazonbot":["partial","*",17,0],"meta-externalagent":["partial","*",17,0],"cohere-ai":["partial","*",17,0],"Diffbot":["partial","*",17,0],"ContextFetchBot":["partial","*",17,0]},"llms":{"status":"missing","http":404,"bytes":146},"home":{"status":"ok","http":200,"url":"https://airc.cn/","html_tokens":11833,"markdown_tokens":51,"pruned_tokens":51}},
{"rank":8944,"domain":"aicpa-cima.com","origin":"https://aicpa-cima.com","robots":{"status":"ok","http":200,"url":"https://www.aicpa-cima.com/robots.txt","bytes":299,"groups":5},"bots":{"GPTBot":["partial","*",4,0],"ChatGPT-User":["partial","*",4,0],"OAI-SearchBot":["partial","*",4,0],"ClaudeBot":["partial","*",4,0],"Claude-User":["partial","*",4,0],"Claude-SearchBot":["partial","*",4,0],"anthropic-ai":["partial","*",4,0],"PerplexityBot":["partial","*",4,0],"Perplexity-User":["partial","*",4,0],"Google-Extended":["partial","*",4,0],"Applebot-Extended":["partial","*",4,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["partial","*",4,0],"meta-externalagent":["partial","*",4,0],"cohere-ai":["partial","*",4,0],"Diffbot":["partial","*",4,0],"ContextFetchBot":["partial","*",4,0]},"llms":{"status":"missing","http":404,"bytes":233432},"home":{"status":"ok","http":200,"url":"https://www.aicpa-cima.com/","html_tokens":55348,"markdown_tokens":14,"pruned_tokens":14}},
{"rank":8945,"domain":"preprints.org","origin":"https://preprints.org","robots":{"status":"refused","http":403,"url":"https://preprints.org/robots.txt","bytes":377,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":8946,"domain":"rub.de","origin":"https://rub.de","robots":{"status":"ok","http":200,"url":"https://www.ruhr-uni-bochum.de/robots.txt","bytes":1618,"groups":1},"bots":{"GPTBot":["partial","*",23,18],"ChatGPT-User":["partial","*",23,18],"OAI-SearchBot":["partial","*",23,18],"ClaudeBot":["partial","*",23,18],"Claude-User":["partial","*",23,18],"Claude-SearchBot":["partial","*",23,18],"anthropic-ai":["partial","*",23,18],"PerplexityBot":["partial","*",23,18],"Perplexity-User":["partial","*",23,18],"Google-Extended":["partial","*",23,18],"Applebot-Extended":["partial","*",23,18],"CCBot":["partial","*",23,18],"Bytespider":["partial","*",23,18],"Amazonbot":["partial","*",23,18],"meta-externalagent":["partial","*",23,18],"cohere-ai":["partial","*",23,18],"Diffbot":["partial","*",23,18],"ContextFetchBot":["partial","*",23,18]},"llms":{"status":"missing","http":404,"bytes":6214},"home":{"status":"ok","http":200,"url":"https://www.ruhr-uni-bochum.de/de","html_tokens":30037,"markdown_tokens":206,"pruned_tokens":31}},
{"rank":8947,"domain":"alltheweb.com","origin":"https://alltheweb.com","robots":{"status":"ok","http":200,"url":"https://search.yahoo.com/robots.txt","bytes":2177,"groups":47},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["blocked","named",1,0],"OAI-SearchBot":["blocked","named",1,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",24,1],"Claude-SearchBot":["partial","*",24,1],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["partial","*",24,1],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["partial","*",24,1],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["partial","*",24,1],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",24,1]},"llms":{"status":"missing","http":404,"bytes":582},"home":{"status":"ok","http":200,"url":"https://search.yahoo.com/","html_tokens":19118,"markdown_tokens":24,"pruned_tokens":24}},
{"rank":8948,"domain":"sonatype.com","origin":"https://sonatype.com","robots":{"status":"ok","http":200,"url":"https://www.sonatype.com/robots.txt","bytes":774,"groups":1},"bots":{"GPTBot":["partial","*",21,0],"ChatGPT-User":["partial","*",21,0],"OAI-SearchBot":["partial","*",21,0],"ClaudeBot":["partial","*",21,0],"Claude-User":["partial","*",21,0],"Claude-SearchBot":["partial","*",21,0],"anthropic-ai":["partial","*",21,0],"PerplexityBot":["partial","*",21,0],"Perplexity-User":["partial","*",21,0],"Google-Extended":["partial","*",21,0],"Applebot-Extended":["partial","*",21,0],"CCBot":["partial","*",21,0],"Bytespider":["partial","*",21,0],"Amazonbot":["partial","*",21,0],"meta-externalagent":["partial","*",21,0],"cohere-ai":["partial","*",21,0],"Diffbot":["partial","*",21,0],"ContextFetchBot":["partial","*",21,0]},"llms":{"status":"present","http":200,"bytes":9141},"home":{"status":"ok","http":200,"url":"https://www.sonatype.com/","html_tokens":134396,"markdown_tokens":505,"pruned_tokens":176}},
{"rank":8949,"domain":"lzzhmp.com","origin":"https://lzzhmp.com","robots":{"status":"unreachable","http":null,"url":"https://lzzhmp.com/robots.txt","bytes":0,"groups":0,"error":"cannot resolve lzzhmp.com"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":8950,"domain":"telus.com","origin":"https://telus.com","robots":{"status":"refused","http":403,"url":"https://www.telus.com/robots.txt","bytes":4542,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":8951,"domain":"mayoclinicproceedings.org","origin":"https://mayoclinicproceedings.org","robots":{"status":"ok","http":200,"url":"https://www.mayoclinicproceedings.org/robots.txt","bytes":1858,"groups":22},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["blocked","named",1,0],"OAI-SearchBot":["partial","*",19,6],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",19,6],"Claude-SearchBot":["partial","*",19,6],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["partial","*",19,6],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["partial","*",19,6],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",19,6]},"llms":{"status":"http_error","http":403,"bytes":5578},"home":{"status":"http_error","http":403,"url":"https://mayoclinicproceedings.org/"}},
{"rank":8952,"domain":"autozone.com","origin":"https://autozone.com","robots":{"status":"ok","http":200,"url":"https://www.autozone.com/robots.txt","bytes":3952,"groups":17},"bots":{"GPTBot":["partial","*",47,1],"ChatGPT-User":["partial","*",47,1],"OAI-SearchBot":["partial","*",47,1],"ClaudeBot":["partial","*",47,1],"Claude-User":["partial","*",47,1],"Claude-SearchBot":["partial","*",47,1],"anthropic-ai":["partial","*",47,1],"PerplexityBot":["partial","*",47,1],"Perplexity-User":["partial","*",47,1],"Google-Extended":["partial","*",47,1],"Applebot-Extended":["partial","*",47,1],"CCBot":["partial","*",47,1],"Bytespider":["partial","*",47,1],"Amazonbot":["partial","*",47,1],"meta-externalagent":["partial","*",47,1],"cohere-ai":["partial","*",47,1],"Diffbot":["partial","*",47,1],"ContextFetchBot":["partial","*",47,1]},"llms":{"status":"present","http":200,"bytes":6230},"home":{"status":"http_error","http":403,"url":"https://www.autozone.com/"}},
{"rank":8953,"domain":"infor.com","origin":"https://infor.com","robots":{"status":"missing","http":404,"url":"https://infor.com/robots.txt","bytes":1245,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"present","http":200,"bytes":9422},"home":{"status":"ok","http":200,"url":"https://www.infor.com","html_tokens":82341,"markdown_tokens":231,"pruned_tokens":174}},
{"rank":8954,"domain":"sktthemes.org","origin":"https://sktthemes.org","robots":{"status":"ok","http":200,"url":"https://sktthemes.org/robots.txt","bytes":901,"groups":1},"bots":{"GPTBot":["partial","*",29,0],"ChatGPT-User":["partial","*",29,0],"OAI-SearchBot":["partial","*",29,0],"ClaudeBot":["partial","*",29,0],"Claude-User":["partial","*",29,0],"Claude-SearchBot":["partial","*",29,0],"anthropic-ai":["partial","*",29,0],"PerplexityBot":["partial","*",29,0],"Perplexity-User":["partial","*",29,0],"Google-Extended":["partial","*",29,0],"Applebot-Extended":["partial","*",29,0],"CCBot":["partial","*",29,0],"Bytespider":["partial","*",29,0],"Amazonbot":["partial","*",29,0],"meta-externalagent":["partial","*",29,0],"cohere-ai":["partial","*",29,0],"Diffbot":["partial","*",29,0],"ContextFetchBot":["partial","*",29,0]},"llms":{"status":"present","http":200,"bytes":234480},"home":{"status":"ok","http":200,"url":"https://www.sktthemes.org/","html_tokens":29811,"markdown_tokens":2779,"pruned_tokens":143}},
{"rank":8955,"domain":"zgcxy.cn","origin":"https://zgcxy.cn","robots":{"status":"unreachable","http":null,"url":"https://zgcxy.cn/robots.txt","bytes":0,"groups":0,"error":"cannot resolve zgcxy.cn"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":8956,"domain":"jymljs.cn","origin":"http://jymljs.cn","robots":{"status":"ok","http":200,"url":"http://jymljs.cn/robots.txt","bytes":485,"groups":1},"bots":{"GPTBot":["partial","*",21,0],"ChatGPT-User":["partial","*",21,0],"OAI-SearchBot":["partial","*",21,0],"ClaudeBot":["partial","*",21,0],"Claude-User":["partial","*",21,0],"Claude-SearchBot":["partial","*",21,0],"anthropic-ai":["partial","*",21,0],"PerplexityBot":["partial","*",21,0],"Perplexity-User":["partial","*",21,0],"Google-Extended":["partial","*",21,0],"Applebot-Extended":["partial","*",21,0],"CCBot":["partial","*",21,0],"Bytespider":["partial","*",21,0],"Amazonbot":["partial","*",21,0],"meta-externalagent":["partial","*",21,0],"cohere-ai":["partial","*",21,0],"Diffbot":["partial","*",21,0],"ContextFetchBot":["partial","*",21,0]},"llms":{"status":"missing","http":404,"bytes":4723},"home":{"status":"ok","http":200,"url":"http://jymljs.cn/","html_tokens":9080,"markdown_tokens":113,"pruned_tokens":76}},
{"rank":8957,"domain":"allaboutvision.com","origin":"https://allaboutvision.com","robots":{"status":"ok","http":200,"url":"https://www.allaboutvision.com/robots.txt","bytes":1068,"groups":2},"bots":{"GPTBot":["partial","*",2,0],"ChatGPT-User":["partial","*",2,0],"OAI-SearchBot":["partial","*",2,0],"ClaudeBot":["partial","*",2,0],"Claude-User":["partial","*",2,0],"Claude-SearchBot":["partial","*",2,0],"anthropic-ai":["partial","*",2,0],"PerplexityBot":["partial","*",2,0],"Perplexity-User":["partial","*",2,0],"Google-Extended":["partial","*",2,0],"Applebot-Extended":["partial","*",2,0],"CCBot":["partial","*",2,0],"Bytespider":["partial","*",2,0],"Amazonbot":["partial","*",2,0],"meta-externalagent":["partial","*",2,0],"cohere-ai":["partial","*",2,0],"Diffbot":["partial","*",2,0],"ContextFetchBot":["partial","*",2,0]},"llms":{"status":"missing","http":404,"bytes":37430},"home":{"status":"ok","http":200,"url":"https://www.allaboutvision.com/","html_tokens":42204,"markdown_tokens":139,"pruned_tokens":139}},
{"rank":8958,"domain":"auspost.com.au","origin":"https://auspost.com.au","robots":{"status":"ok","http":200,"url":"https://auspost.com.au/robots.txt","bytes":3795,"groups":2},"bots":{"GPTBot":["partial","*",63,0],"ChatGPT-User":["partial","*",63,0],"OAI-SearchBot":["partial","*",63,0],"ClaudeBot":["partial","*",63,0],"Claude-User":["partial","*",63,0],"Claude-SearchBot":["partial","*",63,0],"anthropic-ai":["partial","*",63,0],"PerplexityBot":["partial","*",63,0],"Perplexity-User":["partial","*",63,0],"Google-Extended":["partial","*",63,0],"Applebot-Extended":["partial","*",63,0],"CCBot":["partial","*",63,0],"Bytespider":["partial","*",63,0],"Amazonbot":["partial","*",63,0],"meta-externalagent":["partial","*",63,0],"cohere-ai":["partial","*",63,0],"Diffbot":["partial","*",63,0],"ContextFetchBot":["partial","*",63,0]},"llms":{"status":"missing","http":404,"bytes":813010},"home":{"status":"ok","http":200,"url":"https://auspost.com.au/","html_tokens":234230,"markdown_tokens":173,"pruned_tokens":173}},
{"rank":8959,"domain":"booth.pm","origin":"https://booth.pm","robots":{"status":"ok","http":200,"url":"https://booth.pm/robots.txt","bytes":63,"groups":1},"bots":{"GPTBot":["partial","*",3,0],"ChatGPT-User":["partial","*",3,0],"OAI-SearchBot":["partial","*",3,0],"ClaudeBot":["partial","*",3,0],"Claude-User":["partial","*",3,0],"Claude-SearchBot":["partial","*",3,0],"anthropic-ai":["partial","*",3,0],"PerplexityBot":["partial","*",3,0],"Perplexity-User":["partial","*",3,0],"Google-Extended":["partial","*",3,0],"Applebot-Extended":["partial","*",3,0],"CCBot":["partial","*",3,0],"Bytespider":["partial","*",3,0],"Amazonbot":["partial","*",3,0],"meta-externalagent":["partial","*",3,0],"cohere-ai":["partial","*",3,0],"Diffbot":["partial","*",3,0],"ContextFetchBot":["partial","*",3,0]},"llms":{"status":"missing","http":404,"bytes":1694},"home":{"status":"ok","http":200,"url":"https://booth.pm/en","html_tokens":46445,"markdown_tokens":34,"pruned_tokens":31}},
{"rank":8960,"domain":"missingkids.org","origin":"https://missingkids.org","robots":{"status":"ok","http":200,"url":"https://missingkids.org/robots.txt","bytes":60,"groups":2},"bots":{"GPTBot":["blocked","*",1,0],"ChatGPT-User":["blocked","*",1,0],"OAI-SearchBot":["blocked","*",1,0],"ClaudeBot":["blocked","*",1,0],"Claude-User":["blocked","*",1,0],"Claude-SearchBot":["blocked","*",1,0],"anthropic-ai":["blocked","*",1,0],"PerplexityBot":["blocked","*",1,0],"Perplexity-User":["blocked","*",1,0],"Google-Extended":["blocked","*",1,0],"Applebot-Extended":["blocked","*",1,0],"CCBot":["blocked","*",1,0],"Bytespider":["blocked","*",1,0],"Amazonbot":["blocked","*",1,0],"meta-externalagent":["blocked","*",1,0],"cohere-ai":["blocked","*",1,0],"Diffbot":["blocked","*",1,0],"ContextFetchBot":["blocked","*",1,0]},"llms":{"status":"disallowed"},"home":{"status":"disallowed"}},
{"rank":8961,"domain":"travelers.com","origin":"https://travelers.com","robots":{"status":"ok","http":200,"url":"https://www.travelers.com/robots.txt","bytes":512,"groups":1},"bots":{"GPTBot":["partial","*",8,0],"ChatGPT-User":["partial","*",8,0],"OAI-SearchBot":["partial","*",8,0],"ClaudeBot":["partial","*",8,0],"Claude-User":["partial","*",8,0],"Claude-SearchBot":["partial","*",8,0],"anthropic-ai":["partial","*",8,0],"PerplexityBot":["partial","*",8,0],"Perplexity-User":["partial","*",8,0],"Google-Extended":["partial","*",8,0],"Applebot-Extended":["partial","*",8,0],"CCBot":["partial","*",8,0],"Bytespider":["partial","*",8,0],"Amazonbot":["partial","*",8,0],"meta-externalagent":["partial","*",8,0],"cohere-ai":["partial","*",8,0],"Diffbot":["partial","*",8,0],"ContextFetchBot":["partial","*",8,0]},"llms":{"status":"missing","http":404,"bytes":167975},"home":{"status":"ok","http":200,"url":"https://www.travelers.com/","html_tokens":63778,"markdown_tokens":276,"pruned_tokens":109}},
{"rank":8962,"domain":"intertek.com","origin":"https://intertek.com","robots":{"status":"missing","http":404,"url":"https://www.intertek.com/robots.txt","bytes":795,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":795},"home":{"status":"ok","http":200,"url":"https://www.intertek.com/","html_tokens":27092,"markdown_tokens":50,"pruned_tokens":50}},
{"rank":8963,"domain":"eccic.com.cn","origin":"http://eccic.com.cn","robots":{"status":"missing","http":404,"url":"http://eccic.com.cn/robots.txt","bytes":1308,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":1308},"home":{"status":"ok","http":200,"url":"http://eccic.com.cn/","html_tokens":16665,"markdown_tokens":76,"pruned_tokens":74}},
{"rank":8964,"domain":"write.as","origin":"https://write.as","robots":{"status":"ok","http":200,"url":"https://write.as/robots.txt","bytes":445,"groups":6},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["partial","*",11,1],"OAI-SearchBot":["partial","*",11,1],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",11,1],"Claude-SearchBot":["partial","*",11,1],"anthropic-ai":["partial","*",11,1],"PerplexityBot":["partial","*",11,1],"Perplexity-User":["partial","*",11,1],"Google-Extended":["partial","*",11,1],"Applebot-Extended":["partial","*",11,1],"CCBot":["partial","*",11,1],"Bytespider":["partial","*",11,1],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["partial","*",11,1],"Diffbot":["partial","*",11,1],"ContextFetchBot":["partial","*",11,1]},"llms":{"status":"missing","http":404,"bytes":15},"home":{"status":"ok","http":200,"url":"https://write.as/","html_tokens":4686,"markdown_tokens":604,"pruned_tokens":244}},
{"rank":8965,"domain":"tongji.edu.cn","origin":"https://www.tongji.edu.cn","robots":{"status":"missing","http":404,"url":"https://www.tongji.edu.cn/robots.txt","bytes":1693,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":1693},"home":{"status":"ok","http":200,"url":"https://www.tongji.edu.cn/","html_tokens":30448,"markdown_tokens":14,"pruned_tokens":14}},
{"rank":8966,"domain":"thewest.com.au","origin":"https://thewest.com.au","robots":{"status":"ok","http":200,"url":"https://thewest.com.au/robots.txt","bytes":203,"groups":2},"bots":{"GPTBot":["partial","*",1,0],"ChatGPT-User":["partial","*",1,0],"OAI-SearchBot":["partial","*",1,0],"ClaudeBot":["partial","*",1,0],"Claude-User":["partial","*",1,0],"Claude-SearchBot":["partial","*",1,0],"anthropic-ai":["partial","*",1,0],"PerplexityBot":["partial","*",1,0],"Perplexity-User":["partial","*",1,0],"Google-Extended":["partial","*",1,0],"Applebot-Extended":["partial","*",1,0],"CCBot":["partial","*",1,0],"Bytespider":["partial","*",1,0],"Amazonbot":["partial","*",1,0],"meta-externalagent":["partial","*",1,0],"cohere-ai":["partial","*",1,0],"Diffbot":["partial","*",1,0],"ContextFetchBot":["partial","*",1,0]},"llms":{"status":"missing","http":404,"bytes":336383},"home":{"status":"ok","http":200,"url":"https://thewest.com.au/","html_tokens":599048,"markdown_tokens":81,"pruned_tokens":81}},
{"rank":8967,"domain":"zmxfybjy.com","origin":"https://zmxfybjy.com","robots":{"status":"unreachable","http":null,"url":"https://zmxfybjy.com/robots.txt","bytes":0,"groups":0,"error":"cannot resolve zmxfybjy.com"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":8968,"domain":"xn--lovu62d.cn","origin":"http://xn--lovu62d.cn","robots":{"status":"missing","http":404,"url":"http://xn--lovu62d.cn/robots.txt","bytes":5070,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":5066},"home":{"status":"error","http":200,"url":"http://xn--lovu62d.cn/","error":"unparseable HTML"}},
{"rank":8969,"domain":"1und1.de","origin":"https://1und1.de","robots":{"status":"ok","http":200,"url":"https://www.1und1.de/robots.txt","bytes":2596,"groups":3},"bots":{"GPTBot":["allowed","named",0,1],"ChatGPT-User":["allowed","named",0,1],"OAI-SearchBot":["allowed","named",0,1],"ClaudeBot":["allowed","named",0,1],"Claude-User":["allowed","named",0,1],"Claude-SearchBot":["allowed","named",0,1],"anthropic-ai":["partial","*",62,5],"PerplexityBot":["allowed","named",0,1],"Perplexity-User":["allowed","named",0,1],"Google-Extended":["allowed","named",0,1],"Applebot-Extended":["allowed","named",0,1],"CCBot":["partial","*",62,5],"Bytespider":["allowed","named",0,1],"Amazonbot":["partial","*",62,5],"meta-externalagent":["allowed","named",0,1],"cohere-ai":["partial","*",62,5],"Diffbot":["partial","*",62,5],"ContextFetchBot":["partial","*",62,5]},"llms":{"status":"present","http":200,"bytes":4091},"home":{"status":"ok","http":200,"url":"https://www.1und1.de/","html_tokens":91248,"markdown_tokens":432,"pruned_tokens":85}},
{"rank":8970,"domain":"schengenvisainfo.com","origin":"https://schengenvisainfo.com","robots":{"status":"refused","http":405,"url":"https://schengenvisainfo.com/robots.txt","bytes":627,"groups":0,"error":"HTTP 405"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":8971,"domain":"senato.it","origin":"https://senato.it","robots":{"status":"html","http":202,"url":"https://www.senato.it/robots.txt","bytes":2319,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":8972,"domain":"ajol.info","origin":"https://ajol.info","robots":{"status":"ok","http":200,"url":"https://www.ajol.info/robots.txt","bytes":1032,"groups":3},"bots":{"GPTBot":["partial","*",12,1],"ChatGPT-User":["partial","*",12,1],"OAI-SearchBot":["partial","*",12,1],"ClaudeBot":["partial","*",12,1],"Claude-User":["partial","*",12,1],"Claude-SearchBot":["partial","*",12,1],"anthropic-ai":["partial","*",12,1],"PerplexityBot":["partial","*",12,1],"Perplexity-User":["partial","*",12,1],"Google-Extended":["partial","*",12,1],"Applebot-Extended":["partial","*",12,1],"CCBot":["partial","*",12,1],"Bytespider":["partial","*",12,1],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["partial","*",12,1],"cohere-ai":["partial","*",12,1],"Diffbot":["partial","*",12,1],"ContextFetchBot":["partial","*",12,1]},"llms":{"status":"html","http":200,"bytes":85154},"home":{"status":"ok","http":200,"url":"https://www.ajol.info/index.php/ajol","html_tokens":21286,"markdown_tokens":71,"pruned_tokens":71}},
{"rank":8973,"domain":"pcpop.com","origin":"https://pcpop.com","robots":{"status":"html","http":200,"url":"https://pcpop.com/robots.txt","bytes":39,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":8974,"domain":"beisilechina.com","origin":"http://beisilechina.com","robots":{"status":"ok","http":200,"url":"http://beisilechina.com/robots.txt","bytes":485,"groups":1},"bots":{"GPTBot":["partial","*",21,0],"ChatGPT-User":["partial","*",21,0],"OAI-SearchBot":["partial","*",21,0],"ClaudeBot":["partial","*",21,0],"Claude-User":["partial","*",21,0],"Claude-SearchBot":["partial","*",21,0],"anthropic-ai":["partial","*",21,0],"PerplexityBot":["partial","*",21,0],"Perplexity-User":["partial","*",21,0],"Google-Extended":["partial","*",21,0],"Applebot-Extended":["partial","*",21,0],"CCBot":["partial","*",21,0],"Bytespider":["partial","*",21,0],"Amazonbot":["partial","*",21,0],"meta-externalagent":["partial","*",21,0],"cohere-ai":["partial","*",21,0],"Diffbot":["partial","*",21,0],"ContextFetchBot":["partial","*",21,0]},"llms":{"status":"missing","http":404,"bytes":4736},"home":{"status":"ok","http":200,"url":"http://beisilechina.com/","html_tokens":5806,"markdown_tokens":119,"pruned_tokens":119}},
{"rank":8975,"domain":"speed-sci.net","origin":"https://speed-sci.net","robots":{"status":"timeout","http":null,"url":"https://speed-sci.net/robots.txt","bytes":0,"groups":0,"error":"ConnectTimeout"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":8976,"domain":"eventim-light.com","origin":"https://eventim-light.com","robots":{"status":"refused","http":403,"url":"https://www.eventim-light.com/robots.txt","bytes":391,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":8977,"domain":"argus-po.sk","origin":"https://argus-po.sk","robots":{"status":"ok","http":200,"url":"https://argus-po.sk/robots.txt","bytes":65,"groups":1},"bots":{"GPTBot":["allowed","*",0,1],"ChatGPT-User":["allowed","*",0,1],"OAI-SearchBot":["allowed","*",0,1],"ClaudeBot":["allowed","*",0,1],"Claude-User":["allowed","*",0,1],"Claude-SearchBot":["allowed","*",0,1],"anthropic-ai":["allowed","*",0,1],"PerplexityBot":["allowed","*",0,1],"Perplexity-User":["allowed","*",0,1],"Google-Extended":["allowed","*",0,1],"Applebot-Extended":["allowed","*",0,1],"CCBot":["allowed","*",0,1],"Bytespider":["allowed","*",0,1],"Amazonbot":["allowed","*",0,1],"meta-externalagent":["allowed","*",0,1],"cohere-ai":["allowed","*",0,1],"Diffbot":["allowed","*",0,1],"ContextFetchBot":["allowed","*",0,1]},"llms":{"status":"missing","http":404,"bytes":640},"home":{"status":"ok","http":200,"url":"https://argus-po.sk/","html_tokens":39966,"markdown_tokens":23536,"pruned_tokens":117}},
{"rank":8978,"domain":"meihuahj.cn","origin":"https://meihuahj.cn","robots":{"status":"ok","http":200,"url":"https://www.meihuahj.cn/robots.txt","bytes":636,"groups":1},"bots":{"GPTBot":["partial","*",24,0],"ChatGPT-User":["partial","*",24,0],"OAI-SearchBot":["partial","*",24,0],"ClaudeBot":["partial","*",24,0],"Claude-User":["partial","*",24,0],"Claude-SearchBot":["partial","*",24,0],"anthropic-ai":["partial","*",24,0],"PerplexityBot":["partial","*",24,0],"Perplexity-User":["partial","*",24,0],"Google-Extended":["partial","*",24,0],"Applebot-Extended":["partial","*",24,0],"CCBot":["partial","*",24,0],"Bytespider":["partial","*",24,0],"Amazonbot":["partial","*",24,0],"meta-externalagent":["partial","*",24,0],"cohere-ai":["partial","*",24,0],"Diffbot":["partial","*",24,0],"ContextFetchBot":["partial","*",24,0]},"llms":{"status":"missing","http":404,"bytes":4730},"home":{"status":"ok","http":200,"url":"https://www.meihuahj.cn/","html_tokens":16089,"markdown_tokens":97,"pruned_tokens":74}},
{"rank":8979,"domain":"ntjlzz.cn","origin":"http://ntjlzz.cn","robots":{"status":"ok","http":200,"url":"http://ntjlzz.cn/robots.txt","bytes":485,"groups":1},"bots":{"GPTBot":["partial","*",21,0],"ChatGPT-User":["partial","*",21,0],"OAI-SearchBot":["partial","*",21,0],"ClaudeBot":["partial","*",21,0],"Claude-User":["partial","*",21,0],"Claude-SearchBot":["partial","*",21,0],"anthropic-ai":["partial","*",21,0],"PerplexityBot":["partial","*",21,0],"Perplexity-User":["partial","*",21,0],"Google-Extended":["partial","*",21,0],"Applebot-Extended":["partial","*",21,0],"CCBot":["partial","*",21,0],"Bytespider":["partial","*",21,0],"Amazonbot":["partial","*",21,0],"meta-externalagent":["partial","*",21,0],"cohere-ai":["partial","*",21,0],"Diffbot":["partial","*",21,0],"ContextFetchBot":["partial","*",21,0]},"llms":{"status":"missing","http":404,"bytes":4723},"home":{"status":"ok","http":200,"url":"http://ntjlzz.cn/","html_tokens":6316,"markdown_tokens":129,"pruned_tokens":129}},
{"rank":8980,"domain":"expressnews.com","origin":"https://expressnews.com","robots":{"status":"ok","http":200,"url":"https://www.expressnews.com/robots.txt","bytes":1111,"groups":3},"bots":{"GPTBot":["partial","*",10,0],"ChatGPT-User":["partial","*",10,0],"OAI-SearchBot":["partial","*",10,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["blocked","named",1,0],"Claude-SearchBot":["partial","*",10,0],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["blocked","named",1,0],"Google-Extended":["partial","*",10,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["partial","*",10,0],"meta-externalagent":["partial","*",10,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",10,0]},"llms":{"status":"missing","http":404,"bytes":70126},"home":{"status":"ok","http":200,"url":"https://www.expressnews.com/","html_tokens":275425,"markdown_tokens":29,"pruned_tokens":29}},
{"rank":8981,"domain":"google.gy","origin":"https://google.gy","robots":{"status":"ok","http":200,"url":"https://www.google.gy/robots.txt","bytes":6624,"groups":4},"bots":{"GPTBot":["partial","*",169,64],"ChatGPT-User":["partial","*",169,64],"OAI-SearchBot":["partial","*",169,64],"ClaudeBot":["partial","*",169,64],"Claude-User":["partial","*",169,64],"Claude-SearchBot":["partial","*",169,64],"anthropic-ai":["partial","*",169,64],"PerplexityBot":["partial","*",169,64],"Perplexity-User":["partial","*",169,64],"Google-Extended":["partial","*",169,64],"Applebot-Extended":["partial","*",169,64],"CCBot":["partial","*",169,64],"Bytespider":["partial","*",169,64],"Amazonbot":["partial","*",169,64],"meta-externalagent":["partial","*",169,64],"cohere-ai":["partial","*",169,64],"Diffbot":["partial","*",169,64],"ContextFetchBot":["partial","*",169,64]},"llms":{"status":"missing","http":404,"bytes":1569},"home":{"status":"ok","http":200,"url":"https://www.google.com/","html_tokens":21355,"markdown_tokens":30,"pruned_tokens":30}},
{"rank":8982,"domain":"volkswagen-group.com","origin":"https://volkswagen-group.com","robots":{"status":"ok","http":200,"url":"https://www.volkswagen-group.com/robots.txt","bytes":107,"groups":1},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"present","http":200,"bytes":6740},"home":{"status":"ok","http":200,"url":"https://www.volkswagen-group.com/en","html_tokens":12946,"markdown_tokens":237,"pruned_tokens":102}},
{"rank":8983,"domain":"antibact.sbs","origin":"https://antibact.sbs","robots":{"status":"unreachable","http":null,"url":"https://antibact.sbs/robots.txt","bytes":0,"groups":0,"error":"cannot resolve antibact.sbs"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":8984,"domain":"hypothes.is","origin":"https://hypothes.is","robots":{"status":"ok","http":200,"url":"https://hypothes.is/robots.txt","bytes":51,"groups":1},"bots":{"GPTBot":["partial","*",1,1],"ChatGPT-User":["partial","*",1,1],"OAI-SearchBot":["partial","*",1,1],"ClaudeBot":["partial","*",1,1],"Claude-User":["partial","*",1,1],"Claude-SearchBot":["partial","*",1,1],"anthropic-ai":["partial","*",1,1],"PerplexityBot":["partial","*",1,1],"Perplexity-User":["partial","*",1,1],"Google-Extended":["partial","*",1,1],"Applebot-Extended":["partial","*",1,1],"CCBot":["partial","*",1,1],"Bytespider":["partial","*",1,1],"Amazonbot":["partial","*",1,1],"meta-externalagent":["partial","*",1,1],"cohere-ai":["partial","*",1,1],"Diffbot":["partial","*",1,1],"ContextFetchBot":["partial","*",1,1]},"llms":{"status":"missing","http":404,"bytes":5412},"home":{"status":"disallowed","error":"robots.txt of web.hypothes.is not readable (refused)"}},
{"rank":8985,"domain":"pandoc.org","origin":"https://pandoc.org","robots":{"status":"ok","http":200,"url":"https://pandoc.org/robots.txt","bytes":1248,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":1522},"home":{"status":"ok","http":200,"url":"https://pandoc.org/","html_tokens":4738,"markdown_tokens":946,"pruned_tokens":252}},
{"rank":8986,"domain":"sunway-wine.com.cn","origin":"http://sunway-wine.com.cn","robots":{"status":"ok","http":200,"url":"http://sunway-wine.com.cn/robots.txt","bytes":485,"groups":1},"bots":{"GPTBot":["partial","*",21,0],"ChatGPT-User":["partial","*",21,0],"OAI-SearchBot":["partial","*",21,0],"ClaudeBot":["partial","*",21,0],"Claude-User":["partial","*",21,0],"Claude-SearchBot":["partial","*",21,0],"anthropic-ai":["partial","*",21,0],"PerplexityBot":["partial","*",21,0],"Perplexity-User":["partial","*",21,0],"Google-Extended":["partial","*",21,0],"Applebot-Extended":["partial","*",21,0],"CCBot":["partial","*",21,0],"Bytespider":["partial","*",21,0],"Amazonbot":["partial","*",21,0],"meta-externalagent":["partial","*",21,0],"cohere-ai":["partial","*",21,0],"Diffbot":["partial","*",21,0],"ContextFetchBot":["partial","*",21,0]},"llms":{"status":"missing","http":404,"bytes":4732},"home":{"status":"ok","http":200,"url":"http://sunway-wine.com.cn/","html_tokens":6151,"markdown_tokens":64,"pruned_tokens":64}},
{"rank":8987,"domain":"huzhou.org","origin":"https://huzhou.org","robots":{"status":"unreachable","http":null,"url":"https://huzhou.org/robots.txt","bytes":0,"groups":0,"error":"cannot resolve huzhou.org"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":8988,"domain":"lesnumeriques.com","origin":"https://lesnumeriques.com","robots":{"status":"ok","http":200,"url":"https://www.lesnumeriques.com/robots.txt","bytes":2027,"groups":2},"bots":{"GPTBot":["partial","*",62,0],"ChatGPT-User":["partial","*",62,0],"OAI-SearchBot":["partial","*",62,0],"ClaudeBot":["partial","*",62,0],"Claude-User":["partial","*",62,0],"Claude-SearchBot":["partial","*",62,0],"anthropic-ai":["partial","*",62,0],"PerplexityBot":["partial","*",62,0],"Perplexity-User":["partial","*",62,0],"Google-Extended":["partial","*",62,0],"Applebot-Extended":["partial","*",62,0],"CCBot":["partial","*",62,0],"Bytespider":["partial","*",62,0],"Amazonbot":["partial","*",62,0],"meta-externalagent":["partial","*",62,0],"cohere-ai":["partial","*",62,0],"Diffbot":["partial","*",62,0],"ContextFetchBot":["partial","*",62,0]},"llms":{"status":"missing","http":410,"bytes":132238},"home":{"status":"ok","http":200,"url":"https://www.lesnumeriques.com/","html_tokens":149450,"markdown_tokens":30,"pruned_tokens":30}},
{"rank":8989,"domain":"benq.com","origin":"https://benq.com","robots":{"status":"ok","http":200,"url":"https://benq.com/robots.txt","bytes":37,"groups":1},"bots":{"GPTBot":["partial","*",1,1],"ChatGPT-User":["partial","*",1,1],"OAI-SearchBot":["partial","*",1,1],"ClaudeBot":["partial","*",1,1],"Claude-User":["partial","*",1,1],"Claude-SearchBot":["partial","*",1,1],"anthropic-ai":["partial","*",1,1],"PerplexityBot":["partial","*",1,1],"Perplexity-User":["partial","*",1,1],"Google-Extended":["partial","*",1,1],"Applebot-Extended":["partial","*",1,1],"CCBot":["partial","*",1,1],"Bytespider":["partial","*",1,1],"Amazonbot":["partial","*",1,1],"meta-externalagent":["partial","*",1,1],"cohere-ai":["partial","*",1,1],"Diffbot":["partial","*",1,1],"ContextFetchBot":["partial","*",1,1]},"llms":{"status":"disallowed"},"home":{"status":"ok","http":200,"url":"https://www.benq.com/en-us/index.html","html_tokens":102823,"markdown_tokens":592,"pruned_tokens":232}},
{"rank":8990,"domain":"djljz.cn","origin":"https://djljz.cn","robots":{"status":"ok","http":200,"url":"https://www.djljz.cn/robots.txt","bytes":879,"groups":1},"bots":{"GPTBot":["partial","*",21,6],"ChatGPT-User":["partial","*",21,6],"OAI-SearchBot":["partial","*",21,6],"ClaudeBot":["partial","*",21,6],"Claude-User":["partial","*",21,6],"Claude-SearchBot":["partial","*",21,6],"anthropic-ai":["partial","*",21,6],"PerplexityBot":["partial","*",21,6],"Perplexity-User":["partial","*",21,6],"Google-Extended":["partial","*",21,6],"Applebot-Extended":["partial","*",21,6],"CCBot":["partial","*",21,6],"Bytespider":["partial","*",21,6],"Amazonbot":["partial","*",21,6],"meta-externalagent":["partial","*",21,6],"cohere-ai":["partial","*",21,6],"Diffbot":["partial","*",21,6],"ContextFetchBot":["partial","*",21,6]},"llms":{"status":"missing","http":404,"bytes":2098},"home":{"status":"ok","http":200,"url":"https://www.djljz.cn/","html_tokens":26232,"markdown_tokens":25,"pruned_tokens":25}},
{"rank":8991,"domain":"ntguchi.cn","origin":"https://ntguchi.cn","robots":{"status":"timeout","http":null,"url":"https://ntguchi.cn/robots.txt","bytes":0,"groups":0,"error":"ConnectTimeout"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":8992,"domain":"gktvip.cn","origin":"https://gktvip.cn","robots":{"status":"unreachable","http":null,"url":"https://gktvip.cn/robots.txt","bytes":0,"groups":0,"error":"cannot resolve gktvip.cn"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":8993,"domain":"tech.co","origin":"https://tech.co","robots":{"status":"missing","http":404,"url":"https://iris.mvfglobal.com/wp-signup.php?new=tech.co","bytes":16286,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"disallowed","error":"robots.txt of iris.mvfglobal.com disallows /wp-signup.php?new=tech.co"},"home":{"status":"ok","http":200,"url":"https://tech.co/","html_tokens":45643,"markdown_tokens":166,"pruned_tokens":166}},
{"rank":8994,"domain":"championat.com","origin":"https://championat.com","robots":{"status":"ok","http":200,"url":"https://www.championat.com/robots.txt","bytes":1893,"groups":3},"bots":{"GPTBot":["partial","*",22,0],"ChatGPT-User":["partial","*",22,0],"OAI-SearchBot":["partial","*",22,0],"ClaudeBot":["partial","*",22,0],"Claude-User":["partial","*",22,0],"Claude-SearchBot":["partial","*",22,0],"anthropic-ai":["partial","*",22,0],"PerplexityBot":["partial","*",22,0],"Perplexity-User":["partial","*",22,0],"Google-Extended":["partial","*",22,0],"Applebot-Extended":["partial","*",22,0],"CCBot":["partial","*",22,0],"Bytespider":["partial","*",22,0],"Amazonbot":["partial","*",22,0],"meta-externalagent":["partial","*",22,0],"cohere-ai":["partial","*",22,0],"Diffbot":["partial","*",22,0],"ContextFetchBot":["partial","*",22,0]},"llms":{"status":"html","http":200,"bytes":740},"home":{"status":"ok","http":200,"url":"https://www.championat.com/","html_tokens":170,"markdown_tokens":14,"pruned_tokens":14}},
{"rank":8995,"domain":"thueringen.de","origin":"https://thueringen.de","robots":{"status":"html","http":200,"url":"https://thueringen.de/robots.txt","bytes":45764,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":8996,"domain":"allaboutcircuits.com","origin":"https://allaboutcircuits.com","robots":{"status":"ok","http":200,"url":"https://www.allaboutcircuits.com/robots.txt","bytes":667,"groups":2},"bots":{"GPTBot":["partial","*",15,0],"ChatGPT-User":["partial","*",15,0],"OAI-SearchBot":["partial","*",15,0],"ClaudeBot":["partial","*",15,0],"Claude-User":["partial","*",15,0],"Claude-SearchBot":["partial","*",15,0],"anthropic-ai":["partial","*",15,0],"PerplexityBot":["partial","*",15,0],"Perplexity-User":["partial","*",15,0],"Google-Extended":["partial","*",15,0],"Applebot-Extended":["partial","*",15,0],"CCBot":["partial","*",15,0],"Bytespider":["partial","*",15,0],"Amazonbot":["partial","*",15,0],"meta-externalagent":["partial","*",15,0],"cohere-ai":["partial","*",15,0],"Diffbot":["partial","*",15,0],"ContextFetchBot":["partial","*",15,0]},"llms":{"status":"missing","http":404,"bytes":119634},"home":{"status":"http_error","http":403,"url":"https://allaboutcircuits.com/"}},
{"rank":8997,"domain":"rikang.com","origin":"http://rikang.com","robots":{"status":"ok","http":200,"url":"http://www.rikang.com/robots.txt","bytes":208,"groups":1},"bots":{"GPTBot":["partial","*",4,0],"ChatGPT-User":["partial","*",4,0],"OAI-SearchBot":["partial","*",4,0],"ClaudeBot":["partial","*",4,0],"Claude-User":["partial","*",4,0],"Claude-SearchBot":["partial","*",4,0],"anthropic-ai":["partial","*",4,0],"PerplexityBot":["partial","*",4,0],"Perplexity-User":["partial","*",4,0],"Google-Extended":["partial","*",4,0],"Applebot-Extended":["partial","*",4,0],"CCBot":["partial","*",4,0],"Bytespider":["partial","*",4,0],"Amazonbot":["partial","*",4,0],"meta-externalagent":["partial","*",4,0],"cohere-ai":["partial","*",4,0],"Diffbot":["partial","*",4,0],"ContextFetchBot":["partial","*",4,0]},"llms":{"status":"missing","http":404,"bytes":1231},"home":{"status":"ok","http":200,"url":"http://www.rikang.com/","html_tokens":8753,"markdown_tokens":260,"pruned_tokens":169}},
{"rank":8998,"domain":"montereybayaquarium.org","origin":"https://montereybayaquarium.org","robots":{"status":"ok","http":200,"url":"https://www.montereybayaquarium.org/robots.txt","bytes":258,"groups":1},"bots":{"GPTBot":["partial","*",2,0],"ChatGPT-User":["partial","*",2,0],"OAI-SearchBot":["partial","*",2,0],"ClaudeBot":["partial","*",2,0],"Claude-User":["partial","*",2,0],"Claude-SearchBot":["partial","*",2,0],"anthropic-ai":["partial","*",2,0],"PerplexityBot":["partial","*",2,0],"Perplexity-User":["partial","*",2,0],"Google-Extended":["partial","*",2,0],"Applebot-Extended":["partial","*",2,0],"CCBot":["partial","*",2,0],"Bytespider":["partial","*",2,0],"Amazonbot":["partial","*",2,0],"meta-externalagent":["partial","*",2,0],"cohere-ai":["partial","*",2,0],"Diffbot":["partial","*",2,0],"ContextFetchBot":["partial","*",2,0]},"llms":{"status":"http_error","http":405,"bytes":2547},"home":{"status":"http_error","http":405,"url":"https://montereybayaquarium.org/"}},
{"rank":8999,"domain":"convertio.co","origin":"https://convertio.co","robots":{"status":"ok","http":200,"url":"https://convertio.co/robots.txt","bytes":146,"groups":2},"bots":{"GPTBot":["partial","*",3,0],"ChatGPT-User":["partial","*",3,0],"OAI-SearchBot":["partial","*",3,0],"ClaudeBot":["partial","*",3,0],"Claude-User":["partial","*",3,0],"Claude-SearchBot":["partial","*",3,0],"anthropic-ai":["partial","*",3,0],"PerplexityBot":["partial","*",3,0],"Perplexity-User":["partial","*",3,0],"Google-Extended":["partial","*",3,0],"Applebot-Extended":["partial","*",3,0],"CCBot":["partial","*",3,0],"Bytespider":["partial","*",3,0],"Amazonbot":["partial","*",3,0],"meta-externalagent":["partial","*",3,0],"cohere-ai":["partial","*",3,0],"Diffbot":["partial","*",3,0],"ContextFetchBot":["partial","*",3,0]},"llms":{"status":"missing","http":404,"bytes":58071},"home":{"status":"ok","http":200,"url":"https://convertio.co/","html_tokens":33117,"markdown_tokens":276,"pruned_tokens":156}},
{"rank":9000,"domain":"homes.com","origin":"https://homes.com","robots":{"status":"refused","http":403,"url":"https://homes.com/robots.txt","bytes":373,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":9001,"domain":"google.com.vc","origin":"https://google.com.vc","robots":{"status":"ok","http":200,"url":"https://www.google.com.vc/robots.txt","bytes":6624,"groups":4},"bots":{"GPTBot":["partial","*",169,64],"ChatGPT-User":["partial","*",169,64],"OAI-SearchBot":["partial","*",169,64],"ClaudeBot":["partial","*",169,64],"Claude-User":["partial","*",169,64],"Claude-SearchBot":["partial","*",169,64],"anthropic-ai":["partial","*",169,64],"PerplexityBot":["partial","*",169,64],"Perplexity-User":["partial","*",169,64],"Google-Extended":["partial","*",169,64],"Applebot-Extended":["partial","*",169,64],"CCBot":["partial","*",169,64],"Bytespider":["partial","*",169,64],"Amazonbot":["partial","*",169,64],"meta-externalagent":["partial","*",169,64],"cohere-ai":["partial","*",169,64],"Diffbot":["partial","*",169,64],"ContextFetchBot":["partial","*",169,64]},"llms":{"status":"missing","http":404,"bytes":1569},"home":{"status":"ok","http":200,"url":"https://www.google.com/","html_tokens":21244,"markdown_tokens":30,"pruned_tokens":30}},
{"rank":9002,"domain":"controlc.com","origin":"https://controlc.com","robots":{"status":"ok","http":200,"url":"https://controlc.com/robots.txt","bytes":1248,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":3481},"home":{"status":"ok","http":200,"url":"https://controlc.com/","html_tokens":5073,"markdown_tokens":73,"pruned_tokens":73}},
{"rank":9003,"domain":"behance.com","origin":"https://behance.com","robots":{"status":"ok","http":200,"url":"https://www.behance.net/robots.txt","bytes":1317,"groups":2},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["partial","*",24,0],"OAI-SearchBot":["partial","*",24,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",24,0],"Claude-SearchBot":["partial","*",24,0],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["partial","*",24,0],"Perplexity-User":["partial","*",24,0],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["partial","*",24,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["partial","*",24,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",24,0]},"llms":{"status":"missing","http":404,"bytes":385084},"home":{"status":"http_error","http":403,"url":"https://www.behance.net/"}},
{"rank":9004,"domain":"slu.se","origin":"https://slu.se","robots":{"status":"ok","http":200,"url":"https://www.slu.se/robots.txt","bytes":120,"groups":1},"bots":{"GPTBot":["partial","*",3,0],"ChatGPT-User":["partial","*",3,0],"OAI-SearchBot":["partial","*",3,0],"ClaudeBot":["partial","*",3,0],"Claude-User":["partial","*",3,0],"Claude-SearchBot":["partial","*",3,0],"anthropic-ai":["partial","*",3,0],"PerplexityBot":["partial","*",3,0],"Perplexity-User":["partial","*",3,0],"Google-Extended":["partial","*",3,0],"Applebot-Extended":["partial","*",3,0],"CCBot":["partial","*",3,0],"Bytespider":["partial","*",3,0],"Amazonbot":["partial","*",3,0],"meta-externalagent":["partial","*",3,0],"cohere-ai":["partial","*",3,0],"Diffbot":["partial","*",3,0],"ContextFetchBot":["partial","*",3,0]},"llms":{"status":"missing","http":404,"bytes":62518},"home":{"status":"ok","http":200,"url":"https://www.slu.se/","html_tokens":26936,"markdown_tokens":304,"pruned_tokens":101}},
{"rank":9005,"domain":"siliconrepublic.com","origin":"https://siliconrepublic.com","robots":{"status":"ok","http":200,"url":"https://www.siliconrepublic.com/robots.txt","bytes":297,"groups":2},"bots":{"GPTBot":["partial","*",2,0],"ChatGPT-User":["partial","*",2,0],"OAI-SearchBot":["partial","*",2,0],"ClaudeBot":["partial","*",2,0],"Claude-User":["partial","*",2,0],"Claude-SearchBot":["partial","*",2,0],"anthropic-ai":["partial","*",2,0],"PerplexityBot":["partial","*",2,0],"Perplexity-User":["partial","*",2,0],"Google-Extended":["partial","*",2,0],"Applebot-Extended":["partial","*",2,0],"CCBot":["partial","*",2,0],"Bytespider":["partial","*",2,0],"Amazonbot":["partial","*",2,0],"meta-externalagent":["partial","*",2,0],"cohere-ai":["partial","*",2,0],"Diffbot":["partial","*",2,0],"ContextFetchBot":["partial","*",2,0]},"llms":{"status":"missing","http":404,"bytes":53879},"home":{"status":"ok","http":200,"url":"https://www.siliconrepublic.com/","html_tokens":47848,"markdown_tokens":46,"pruned_tokens":46}},
{"rank":9006,"domain":"phorum.pl","origin":"https://phorum.pl","robots":{"status":"unreachable","http":null,"url":"https://phorum.pl/robots.txt","bytes":0,"groups":0,"error":"[SSL: SSLV3_ALERT_HANDSHAKE_FAILURE] sslv3 alert handshake failure (_ssl.c:1028)"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":9007,"domain":"skyscrapercity.com","origin":"https://skyscrapercity.com","robots":{"status":"timeout","http":null,"url":"https://skyscrapercity.com/robots.txt","bytes":0,"groups":0,"error":"timed out after 10 s"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":9008,"domain":"netease.com","origin":"https://www.netease.com","robots":{"status":"missing","http":404,"url":"https://www.netease.com/robots.txt","bytes":2426,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":2426},"home":{"status":"ok","http":200,"url":"https://www.netease.com/","html_tokens":5206,"markdown_tokens":146,"pruned_tokens":146}},
{"rank":9009,"domain":"bolt.eu","origin":"https://bolt.eu","robots":{"status":"ok","http":200,"url":"https://bolt.eu/robots.txt","bytes":337,"groups":2},"bots":{"GPTBot":["allowed","*",0,1],"ChatGPT-User":["allowed","*",0,1],"OAI-SearchBot":["allowed","named",0,1],"ClaudeBot":["allowed","*",0,1],"Claude-User":["allowed","*",0,1],"Claude-SearchBot":["allowed","*",0,1],"anthropic-ai":["allowed","*",0,1],"PerplexityBot":["allowed","*",0,1],"Perplexity-User":["allowed","*",0,1],"Google-Extended":["allowed","*",0,1],"Applebot-Extended":["allowed","*",0,1],"CCBot":["allowed","*",0,1],"Bytespider":["allowed","*",0,1],"Amazonbot":["allowed","*",0,1],"meta-externalagent":["allowed","*",0,1],"cohere-ai":["allowed","*",0,1],"Diffbot":["allowed","*",0,1],"ContextFetchBot":["allowed","*",0,1]},"llms":{"status":"missing","http":404,"bytes":15218},"home":{"status":"ok","http":200,"url":"https://bolt.eu/en/","html_tokens":110419,"markdown_tokens":282,"pruned_tokens":44}},
{"rank":9010,"domain":"fillout.com","origin":"https://fillout.com","robots":{"status":"ok","http":200,"url":"https://www.fillout.com/robots.txt","bytes":122,"groups":1},"bots":{"GPTBot":["allowed","*",0,1],"ChatGPT-User":["allowed","*",0,1],"OAI-SearchBot":["allowed","*",0,1],"ClaudeBot":["allowed","*",0,1],"Claude-User":["allowed","*",0,1],"Claude-SearchBot":["allowed","*",0,1],"anthropic-ai":["allowed","*",0,1],"PerplexityBot":["allowed","*",0,1],"Perplexity-User":["allowed","*",0,1],"Google-Extended":["allowed","*",0,1],"Applebot-Extended":["allowed","*",0,1],"CCBot":["allowed","*",0,1],"Bytespider":["allowed","*",0,1],"Amazonbot":["allowed","*",0,1],"meta-externalagent":["allowed","*",0,1],"cohere-ai":["allowed","*",0,1],"Diffbot":["allowed","*",0,1],"ContextFetchBot":["allowed","*",0,1]},"llms":{"status":"present","http":200,"bytes":54756},"home":{"status":"ok","http":200,"url":"https://www.fillout.com/","html_tokens":52944,"markdown_tokens":278,"pruned_tokens":92}},
{"rank":9011,"domain":"fitsmallbusiness.com","origin":"https://fitsmallbusiness.com","robots":{"status":"ok","http":200,"url":"https://fitsmallbusiness.com/robots.txt","bytes":145,"groups":1},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"present","http":200,"bytes":8214},"home":{"status":"http_error","http":403,"url":"https://fitsmallbusiness.com/"}},
{"rank":9012,"domain":"cadence.com","origin":"https://cadence.com","robots":{"status":"refused","http":403,"url":"https://cadence.com/robots.txt","bytes":5527,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":9013,"domain":"ssir.org","origin":"https://ssir.org","robots":{"status":"ok","http":200,"url":"https://ssir.org/robots.txt","bytes":158,"groups":1},"bots":{"GPTBot":["partial","*",1,0],"ChatGPT-User":["partial","*",1,0],"OAI-SearchBot":["partial","*",1,0],"ClaudeBot":["partial","*",1,0],"Claude-User":["partial","*",1,0],"Claude-SearchBot":["partial","*",1,0],"anthropic-ai":["partial","*",1,0],"PerplexityBot":["partial","*",1,0],"Perplexity-User":["partial","*",1,0],"Google-Extended":["partial","*",1,0],"Applebot-Extended":["partial","*",1,0],"CCBot":["partial","*",1,0],"Bytespider":["partial","*",1,0],"Amazonbot":["partial","*",1,0],"meta-externalagent":["partial","*",1,0],"cohere-ai":["partial","*",1,0],"Diffbot":["partial","*",1,0],"ContextFetchBot":["partial","*",1,0]},"llms":{"status":"missing","http":404,"bytes":79805},"home":{"status":"ok","http":200,"url":"https://ssir.org/","html_tokens":52610,"markdown_tokens":503,"pruned_tokens":135}},
{"rank":9014,"domain":"seti.org","origin":"https://seti.org","robots":{"status":"missing","http":404,"url":"https://www.seti.org/robots.txt","bytes":0,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"present","http":200,"bytes":292},"home":{"status":"ok","http":200,"url":"https://www.seti.org/","html_tokens":24125,"markdown_tokens":124,"pruned_tokens":123}},
{"rank":9015,"domain":"weeklystandard.com","origin":"https://weeklystandard.com","robots":{"status":"timeout","http":null,"url":"https://weeklystandard.com/robots.txt","bytes":0,"groups":0,"error":"timed out after 10 s"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":9016,"domain":"gate-777.net","origin":"https://gate-777.net","robots":{"status":"ok","http":200,"url":"https://gate-777.net/robots.txt","bytes":67,"groups":1},"bots":{"GPTBot":["partial","*",1,1],"ChatGPT-User":["partial","*",1,1],"OAI-SearchBot":["partial","*",1,1],"ClaudeBot":["partial","*",1,1],"Claude-User":["partial","*",1,1],"Claude-SearchBot":["partial","*",1,1],"anthropic-ai":["partial","*",1,1],"PerplexityBot":["partial","*",1,1],"Perplexity-User":["partial","*",1,1],"Google-Extended":["partial","*",1,1],"Applebot-Extended":["partial","*",1,1],"CCBot":["partial","*",1,1],"Bytespider":["partial","*",1,1],"Amazonbot":["partial","*",1,1],"meta-externalagent":["partial","*",1,1],"cohere-ai":["partial","*",1,1],"Diffbot":["partial","*",1,1],"ContextFetchBot":["partial","*",1,1]},"llms":{"status":"missing","http":404,"bytes":44904},"home":{"status":"ok","http":200,"url":"https://gate-777.net/","html_tokens":15938,"markdown_tokens":1975,"pruned_tokens":1201}},
{"rank":9017,"domain":"pidgin.im","origin":"https://pidgin.im","robots":{"status":"ok","http":200,"url":"https://pidgin.im/robots.txt","bytes":3183,"groups":1},"bots":{"GPTBot":["allowed","*",0,1],"ChatGPT-User":["allowed","*",0,1],"OAI-SearchBot":["allowed","*",0,1],"ClaudeBot":["allowed","*",0,1],"Claude-User":["allowed","*",0,1],"Claude-SearchBot":["allowed","*",0,1],"anthropic-ai":["allowed","*",0,1],"PerplexityBot":["allowed","*",0,1],"Perplexity-User":["allowed","*",0,1],"Google-Extended":["allowed","*",0,1],"Applebot-Extended":["allowed","*",0,1],"CCBot":["allowed","*",0,1],"Bytespider":["allowed","*",0,1],"Amazonbot":["allowed","*",0,1],"meta-externalagent":["allowed","*",0,1],"cohere-ai":["allowed","*",0,1],"Diffbot":["allowed","*",0,1],"ContextFetchBot":["allowed","*",0,1]},"llms":{"status":"missing","http":404,"bytes":12762},"home":{"status":"ok","http":200,"url":"https://pidgin.im/","html_tokens":5691,"markdown_tokens":331,"pruned_tokens":331}},
{"rank":9018,"domain":"czgsjz.com","origin":"http://czgsjz.com","robots":{"status":"missing","http":404,"url":"http://czgsjz.com/robots.txt","bytes":3749,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":3749},"home":{"status":"ok","http":200,"url":"http://czgsjz.com/","html_tokens":5612,"markdown_tokens":35,"pruned_tokens":35}},
{"rank":9019,"domain":"cookieconsent.com","origin":"https://cookieconsent.com","robots":{"status":"ok","http":200,"url":"https://www.cookieconsent.com/robots.txt","bytes":2186,"groups":1},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"http_error","http":502,"bytes":6440},"home":{"status":"ok","http":200,"url":"https://www.cookieconsent.com/","html_tokens":32596,"markdown_tokens":5648,"pruned_tokens":260}},
{"rank":9020,"domain":"qbnz.com","origin":"https://qbnz.com","robots":{"status":"timeout","http":null,"url":"https://qbnz.com/robots.txt","bytes":0,"groups":0,"error":"timed out after 10 s"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":9021,"domain":"gearbest.com","origin":"https://gearbest.com","robots":{"status":"unreachable","http":null,"url":"https://gearbest.com/robots.txt","bytes":0,"groups":0,"error":"cannot resolve gearbest.com"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":9022,"domain":"mo.us","origin":"https://mo.us","robots":{"status":"unreachable","http":null,"url":"https://mo.us/robots.txt","bytes":0,"groups":0,"error":"cannot resolve mo.us"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":9023,"domain":"gmpg.org","origin":"https://gmpg.org","robots":{"status":"ok","http":200,"url":"https://gmpg.org/robots.txt","bytes":25,"groups":1},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"missing","http":404,"bytes":9},"home":{"status":"ok","http":200,"url":"https://gmpg.org/","html_tokens":425,"markdown_tokens":155,"pruned_tokens":155}},
{"rank":9024,"domain":"b23.tv","origin":"https://b23.tv","robots":{"status":"ok","http":200,"url":"https://b23.tv/robots.txt","bytes":46,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"other_text","http":200,"bytes":46},"home":{"status":"http_error","http":404,"url":"https://b23.tv/"}},
{"rank":9025,"domain":"jyweihu.cn","origin":"http://jyweihu.cn","robots":{"status":"ok","http":200,"url":"http://jyweihu.cn/robots.txt","bytes":485,"groups":1},"bots":{"GPTBot":["partial","*",21,0],"ChatGPT-User":["partial","*",21,0],"OAI-SearchBot":["partial","*",21,0],"ClaudeBot":["partial","*",21,0],"Claude-User":["partial","*",21,0],"Claude-SearchBot":["partial","*",21,0],"anthropic-ai":["partial","*",21,0],"PerplexityBot":["partial","*",21,0],"Perplexity-User":["partial","*",21,0],"Google-Extended":["partial","*",21,0],"Applebot-Extended":["partial","*",21,0],"CCBot":["partial","*",21,0],"Bytespider":["partial","*",21,0],"Amazonbot":["partial","*",21,0],"meta-externalagent":["partial","*",21,0],"cohere-ai":["partial","*",21,0],"Diffbot":["partial","*",21,0],"ContextFetchBot":["partial","*",21,0]},"llms":{"status":"missing","http":404,"bytes":4724},"home":{"status":"ok","http":200,"url":"http://jyweihu.cn/","html_tokens":10872,"markdown_tokens":63,"pruned_tokens":63}},
{"rank":9026,"domain":"hmrc.gov.uk","origin":"https://hmrc.gov.uk","robots":{"status":"ok","http":200,"url":"https://www.hmrc.gov.uk/robots.txt","bytes":83,"groups":1},"bots":{"GPTBot":["partial","*",3,0],"ChatGPT-User":["partial","*",3,0],"OAI-SearchBot":["partial","*",3,0],"ClaudeBot":["partial","*",3,0],"Claude-User":["partial","*",3,0],"Claude-SearchBot":["partial","*",3,0],"anthropic-ai":["partial","*",3,0],"PerplexityBot":["partial","*",3,0],"Perplexity-User":["partial","*",3,0],"Google-Extended":["partial","*",3,0],"Applebot-Extended":["partial","*",3,0],"CCBot":["partial","*",3,0],"Bytespider":["partial","*",3,0],"Amazonbot":["partial","*",3,0],"meta-externalagent":["partial","*",3,0],"cohere-ai":["partial","*",3,0],"Diffbot":["partial","*",3,0],"ContextFetchBot":["partial","*",3,0]},"llms":{"status":"missing","http":404,"bytes":52491},"home":{"status":"ok","http":200,"url":"https://www.gov.uk/government/organisations/hm-revenue-customs","html_tokens":32631,"markdown_tokens":83,"pruned_tokens":83}},
{"rank":9027,"domain":"okayama.jp","origin":"https://okayama.jp","robots":{"status":"unreachable","http":null,"url":"https://okayama.jp/robots.txt","bytes":0,"groups":0,"error":"cannot resolve okayama.jp"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":9028,"domain":"gurufocus.com","origin":"https://gurufocus.com","robots":{"status":"ok","http":200,"url":"https://www.gurufocus.com/robots.txt","bytes":3461,"groups":2},"bots":{"GPTBot":["partial","*",49,0],"ChatGPT-User":["partial","*",49,0],"OAI-SearchBot":["partial","*",49,0],"ClaudeBot":["partial","*",49,0],"Claude-User":["partial","*",49,0],"Claude-SearchBot":["partial","*",49,0],"anthropic-ai":["partial","*",49,0],"PerplexityBot":["partial","*",49,0],"Perplexity-User":["partial","*",49,0],"Google-Extended":["partial","*",49,0],"Applebot-Extended":["partial","*",49,0],"CCBot":["partial","*",49,0],"Bytespider":["partial","*",49,0],"Amazonbot":["partial","*",49,0],"meta-externalagent":["partial","*",49,0],"cohere-ai":["partial","*",49,0],"Diffbot":["partial","*",49,0],"ContextFetchBot":["partial","*",49,0]},"llms":{"status":"present","http":200,"bytes":169486},"home":{"status":"http_error","http":403,"url":"https://gurufocus.com/"}},
{"rank":9029,"domain":"parisinfo.com","origin":"https://parisinfo.com","robots":{"status":"ok","http":200,"url":"https://parisjetaime.com/robots.txt","bytes":3712,"groups":28},"bots":{"GPTBot":["allowed","named",0,1],"ChatGPT-User":["allowed","named",0,1],"OAI-SearchBot":["allowed","named",0,1],"ClaudeBot":["allowed","named",0,1],"Claude-User":["partial","*",26,1],"Claude-SearchBot":["partial","*",26,1],"anthropic-ai":["allowed","named",0,1],"PerplexityBot":["allowed","named",0,1],"Perplexity-User":["allowed","named",0,1],"Google-Extended":["allowed","named",0,1],"Applebot-Extended":["allowed","named",0,1],"CCBot":["allowed","named",0,1],"Bytespider":["allowed","named",0,1],"Amazonbot":["allowed","named",0,1],"meta-externalagent":["allowed","named",0,1],"cohere-ai":["allowed","named",0,1],"Diffbot":["partial","*",26,1],"ContextFetchBot":["partial","*",26,1]},"llms":{"status":"missing","http":404,"bytes":321587},"home":{"status":"http_error","http":403,"url":"https://parisinfo.com/"}},
{"rank":9030,"domain":"atimes.com","origin":"http://atimes.com","robots":{"status":"missing","http":404,"url":"http://www.atimes.com/robots.txt","bytes":153,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":153},"home":{"status":"ok","http":200,"url":"http://www.atimes.com/","html_tokens":383,"markdown_tokens":22,"pruned_tokens":8}},
{"rank":9031,"domain":"hannovermesse.de","origin":"https://hannovermesse.de","robots":{"status":"html","http":200,"url":"https://www.hannovermesse.de/","bytes":90323,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":9032,"domain":"moonshot.cn","origin":"https://moonshot.cn","robots":{"status":"missing","http":404,"url":"https://www.moonshot.cn/robots.txt","bytes":153,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"html","http":200,"bytes":98191},"home":{"status":"ok","http":200,"url":"https://www.moonshot.cn/","html_tokens":23481,"markdown_tokens":50,"pruned_tokens":50}},
{"rank":9033,"domain":"nm.gov","origin":"https://nm.gov","robots":{"status":"ok","http":200,"url":"https://www.nm.gov/robots.txt","bytes":179,"groups":1},"bots":{"GPTBot":["partial","*",2,0],"ChatGPT-User":["partial","*",2,0],"OAI-SearchBot":["partial","*",2,0],"ClaudeBot":["partial","*",2,0],"Claude-User":["partial","*",2,0],"Claude-SearchBot":["partial","*",2,0],"anthropic-ai":["partial","*",2,0],"PerplexityBot":["partial","*",2,0],"Perplexity-User":["partial","*",2,0],"Google-Extended":["partial","*",2,0],"Applebot-Extended":["partial","*",2,0],"CCBot":["partial","*",2,0],"Bytespider":["partial","*",2,0],"Amazonbot":["partial","*",2,0],"meta-externalagent":["partial","*",2,0],"cohere-ai":["partial","*",2,0],"Diffbot":["partial","*",2,0],"ContextFetchBot":["partial","*",2,0]},"llms":{"status":"missing","http":404,"bytes":34207},"home":{"status":"ok","http":200,"url":"https://www.nm.gov/","html_tokens":24575,"markdown_tokens":498,"pruned_tokens":226}},
{"rank":9034,"domain":"governmentjobs.com","origin":"https://governmentjobs.com","robots":{"status":"ok","http":200,"url":"https://www.governmentjobs.com/robots.txt","bytes":332,"groups":2},"bots":{"GPTBot":["blocked","*",1,0],"ChatGPT-User":["blocked","*",1,0],"OAI-SearchBot":["blocked","*",1,0],"ClaudeBot":["blocked","*",1,0],"Claude-User":["blocked","*",1,0],"Claude-SearchBot":["blocked","*",1,0],"anthropic-ai":["blocked","*",1,0],"PerplexityBot":["blocked","*",1,0],"Perplexity-User":["blocked","*",1,0],"Google-Extended":["blocked","*",1,0],"Applebot-Extended":["blocked","*",1,0],"CCBot":["blocked","*",1,0],"Bytespider":["blocked","*",1,0],"Amazonbot":["blocked","*",1,0],"meta-externalagent":["blocked","*",1,0],"cohere-ai":["blocked","*",1,0],"Diffbot":["blocked","*",1,0],"ContextFetchBot":["blocked","*",1,0]},"llms":{"status":"disallowed"},"home":{"status":"disallowed"}},
{"rank":9035,"domain":"dccomics.com","origin":"https://dccomics.com","robots":{"status":"ok","http":200,"url":"https://www.dc.com/robots.txt","bytes":115,"groups":2},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["partial","*",1,0],"OAI-SearchBot":["partial","*",1,0],"ClaudeBot":["partial","*",1,0],"Claude-User":["partial","*",1,0],"Claude-SearchBot":["partial","*",1,0],"anthropic-ai":["partial","*",1,0],"PerplexityBot":["partial","*",1,0],"Perplexity-User":["partial","*",1,0],"Google-Extended":["partial","*",1,0],"Applebot-Extended":["partial","*",1,0],"CCBot":["partial","*",1,0],"Bytespider":["partial","*",1,0],"Amazonbot":["partial","*",1,0],"meta-externalagent":["partial","*",1,0],"cohere-ai":["partial","*",1,0],"Diffbot":["partial","*",1,0],"ContextFetchBot":["partial","*",1,0]},"llms":{"status":"http_error","http":403,"bytes":118},"home":{"status":"ok","http":200,"url":"https://www.dc.com/","html_tokens":60873,"markdown_tokens":148,"pruned_tokens":148}},
{"rank":9036,"domain":"iserv.de","origin":"https://iserv.de","robots":{"status":"ok","http":200,"url":"https://iserv.de/robots.txt","bytes":291,"groups":1},"bots":{"GPTBot":["partial","*",9,7],"ChatGPT-User":["partial","*",9,7],"OAI-SearchBot":["partial","*",9,7],"ClaudeBot":["partial","*",9,7],"Claude-User":["partial","*",9,7],"Claude-SearchBot":["partial","*",9,7],"anthropic-ai":["partial","*",9,7],"PerplexityBot":["partial","*",9,7],"Perplexity-User":["partial","*",9,7],"Google-Extended":["partial","*",9,7],"Applebot-Extended":["partial","*",9,7],"CCBot":["partial","*",9,7],"Bytespider":["partial","*",9,7],"Amazonbot":["partial","*",9,7],"meta-externalagent":["partial","*",9,7],"cohere-ai":["partial","*",9,7],"Diffbot":["partial","*",9,7],"ContextFetchBot":["partial","*",9,7]},"llms":{"status":"missing","http":404,"bytes":154931},"home":{"status":"ok","http":200,"url":"https://iserv.de/","html_tokens":60534,"markdown_tokens":570,"pruned_tokens":168}},
{"rank":9037,"domain":"panic.com","origin":"https://panic.com","robots":{"status":"ok","http":200,"url":"https://panic.com/robots.txt","bytes":30,"groups":1},"bots":{"GPTBot":["partial","*",1,0],"ChatGPT-User":["partial","*",1,0],"OAI-SearchBot":["partial","*",1,0],"ClaudeBot":["partial","*",1,0],"Claude-User":["partial","*",1,0],"Claude-SearchBot":["partial","*",1,0],"anthropic-ai":["partial","*",1,0],"PerplexityBot":["partial","*",1,0],"Perplexity-User":["partial","*",1,0],"Google-Extended":["partial","*",1,0],"Applebot-Extended":["partial","*",1,0],"CCBot":["partial","*",1,0],"Bytespider":["partial","*",1,0],"Amazonbot":["partial","*",1,0],"meta-externalagent":["partial","*",1,0],"cohere-ai":["partial","*",1,0],"Diffbot":["partial","*",1,0],"ContextFetchBot":["partial","*",1,0]},"llms":{"status":"missing","http":404,"bytes":507},"home":{"status":"ok","http":200,"url":"https://panic.com/","html_tokens":5235,"markdown_tokens":25,"pruned_tokens":25}},
{"rank":9038,"domain":"justin.tv","origin":"https://justin.tv","robots":{"status":"timeout","http":null,"url":"https://justin.tv/robots.txt","bytes":0,"groups":0,"error":"ConnectTimeout"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":9039,"domain":"simplepie.org","origin":"https://simplepie.org","robots":{"status":"ok","http":200,"url":"https://simplepie.org/robots.txt","bytes":29,"groups":1},"bots":{"GPTBot":["partial","*",1,0],"ChatGPT-User":["partial","*",1,0],"OAI-SearchBot":["partial","*",1,0],"ClaudeBot":["partial","*",1,0],"Claude-User":["partial","*",1,0],"Claude-SearchBot":["partial","*",1,0],"anthropic-ai":["partial","*",1,0],"PerplexityBot":["partial","*",1,0],"Perplexity-User":["partial","*",1,0],"Google-Extended":["partial","*",1,0],"Applebot-Extended":["partial","*",1,0],"CCBot":["partial","*",1,0],"Bytespider":["partial","*",1,0],"Amazonbot":["partial","*",1,0],"meta-externalagent":["partial","*",1,0],"cohere-ai":["partial","*",1,0],"Diffbot":["partial","*",1,0],"ContextFetchBot":["partial","*",1,0]},"llms":{"status":"missing","http":404,"bytes":6236},"home":{"status":"ok","http":200,"url":"https://simplepie.org/","html_tokens":1593,"markdown_tokens":69,"pruned_tokens":69}},
{"rank":9040,"domain":"suzhoudress.com","origin":"https://suzhoudress.com","robots":{"status":"ok","http":200,"url":"https://www.suzhoudress.com/robots.txt","bytes":3673,"groups":2},"bots":{"GPTBot":["partial","*",40,23],"ChatGPT-User":["partial","*",40,23],"OAI-SearchBot":["partial","*",40,23],"ClaudeBot":["partial","*",40,23],"Claude-User":["partial","*",40,23],"Claude-SearchBot":["partial","*",40,23],"anthropic-ai":["partial","*",40,23],"PerplexityBot":["partial","*",40,23],"Perplexity-User":["partial","*",40,23],"Google-Extended":["partial","*",40,23],"Applebot-Extended":["partial","*",40,23],"CCBot":["partial","*",40,23],"Bytespider":["partial","*",40,23],"Amazonbot":["partial","*",40,23],"meta-externalagent":["partial","*",40,23],"cohere-ai":["partial","*",40,23],"Diffbot":["partial","*",40,23],"ContextFetchBot":["partial","*",40,23]},"llms":{"status":"present","http":200,"bytes":5820},"home":{"status":"ok","http":200,"url":"https://www.suzhoudress.com/","html_tokens":224652,"markdown_tokens":304,"pruned_tokens":139}},
{"rank":9041,"domain":"looka.com","origin":"https://looka.com","robots":{"status":"ok","http":200,"url":"https://looka.com/robots.txt","bytes":112,"groups":1},"bots":{"GPTBot":["partial","*",1,1],"ChatGPT-User":["partial","*",1,1],"OAI-SearchBot":["partial","*",1,1],"ClaudeBot":["partial","*",1,1],"Claude-User":["partial","*",1,1],"Claude-SearchBot":["partial","*",1,1],"anthropic-ai":["partial","*",1,1],"PerplexityBot":["partial","*",1,1],"Perplexity-User":["partial","*",1,1],"Google-Extended":["partial","*",1,1],"Applebot-Extended":["partial","*",1,1],"CCBot":["partial","*",1,1],"Bytespider":["partial","*",1,1],"Amazonbot":["partial","*",1,1],"meta-externalagent":["partial","*",1,1],"cohere-ai":["partial","*",1,1],"Diffbot":["partial","*",1,1],"ContextFetchBot":["partial","*",1,1]},"llms":{"status":"missing","http":404,"bytes":40152},"home":{"status":"http_error","http":403,"url":"https://looka.com/"}},
{"rank":9042,"domain":"inpi.fr","origin":"https://www.inpi.fr","robots":{"status":"ok","http":200,"url":"https://www.inpi.fr/robots.txt","bytes":2471,"groups":1},"bots":{"GPTBot":["partial","*",56,18],"ChatGPT-User":["partial","*",56,18],"OAI-SearchBot":["partial","*",56,18],"ClaudeBot":["partial","*",56,18],"Claude-User":["partial","*",56,18],"Claude-SearchBot":["partial","*",56,18],"anthropic-ai":["partial","*",56,18],"PerplexityBot":["partial","*",56,18],"Perplexity-User":["partial","*",56,18],"Google-Extended":["partial","*",56,18],"Applebot-Extended":["partial","*",56,18],"CCBot":["partial","*",56,18],"Bytespider":["partial","*",56,18],"Amazonbot":["partial","*",56,18],"meta-externalagent":["partial","*",56,18],"cohere-ai":["partial","*",56,18],"Diffbot":["partial","*",56,18],"ContextFetchBot":["partial","*",56,18]},"llms":{"status":"http_error","http":408,"bytes":243},"home":{"status":"ok","http":200,"url":"https://www.inpi.fr/","html_tokens":40098,"markdown_tokens":597,"pruned_tokens":145}},
{"rank":9043,"domain":"eurowings.com","origin":"https://eurowings.com","robots":{"status":"ok","http":200,"url":"https://www.eurowings.com/robots.txt","bytes":3676,"groups":1},"bots":{"GPTBot":["partial","*",84,10],"ChatGPT-User":["partial","*",84,10],"OAI-SearchBot":["partial","*",84,10],"ClaudeBot":["partial","*",84,10],"Claude-User":["partial","*",84,10],"Claude-SearchBot":["partial","*",84,10],"anthropic-ai":["partial","*",84,10],"PerplexityBot":["partial","*",84,10],"Perplexity-User":["partial","*",84,10],"Google-Extended":["partial","*",84,10],"Applebot-Extended":["partial","*",84,10],"CCBot":["partial","*",84,10],"Bytespider":["partial","*",84,10],"Amazonbot":["partial","*",84,10],"meta-externalagent":["partial","*",84,10],"cohere-ai":["partial","*",84,10],"Diffbot":["partial","*",84,10],"ContextFetchBot":["partial","*",84,10]},"llms":{"status":"http_error","http":403,"bytes":5549},"home":{"status":"http_error","http":403,"url":"https://www.eurowings.com/"}},
{"rank":9044,"domain":"sonicwall.com","origin":"https://sonicwall.com","robots":{"status":"ok","http":200,"url":"https://www.sonicwall.com/robots.txt","bytes":1362,"groups":1},"bots":{"GPTBot":["partial","*",13,9],"ChatGPT-User":["partial","*",13,9],"OAI-SearchBot":["partial","*",13,9],"ClaudeBot":["partial","*",13,9],"Claude-User":["partial","*",13,9],"Claude-SearchBot":["partial","*",13,9],"anthropic-ai":["partial","*",13,9],"PerplexityBot":["partial","*",13,9],"Perplexity-User":["partial","*",13,9],"Google-Extended":["partial","*",13,9],"Applebot-Extended":["partial","*",13,9],"CCBot":["partial","*",13,9],"Bytespider":["partial","*",13,9],"Amazonbot":["partial","*",13,9],"meta-externalagent":["partial","*",13,9],"cohere-ai":["partial","*",13,9],"Diffbot":["partial","*",13,9],"ContextFetchBot":["partial","*",13,9]},"llms":{"status":"missing","http":404,"bytes":43996},"home":{"status":"ok","http":200,"url":"https://www.sonicwall.com/","html_tokens":290,"markdown_tokens":21,"pruned_tokens":21}},
{"rank":9045,"domain":"blogspot.com.ar","origin":"https://blogspot.com.ar","robots":{"status":"missing","http":404,"url":"https://blogspot.com.ar/robots.txt","bytes":1571,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":1569},"home":{"status":"http_error","http":404,"url":"https://blogspot.com.ar/"}},
{"rank":9046,"domain":"festo.com","origin":"https://festo.com","robots":{"status":"missing","http":404,"url":"https://festo.com/robots.txt","bytes":12219,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":12219},"home":{"status":"ok","http":200,"url":"https://festo.com/","html_tokens":3450,"markdown_tokens":0,"pruned_tokens":0}},
{"rank":9047,"domain":"dominos.com","origin":"https://dominos.com","robots":{"status":"ok","http":200,"url":"https://www.dominos.com/robots.txt","bytes":112,"groups":1},"bots":{"GPTBot":["partial","*",2,1],"ChatGPT-User":["partial","*",2,1],"OAI-SearchBot":["partial","*",2,1],"ClaudeBot":["partial","*",2,1],"Claude-User":["partial","*",2,1],"Claude-SearchBot":["partial","*",2,1],"anthropic-ai":["partial","*",2,1],"PerplexityBot":["partial","*",2,1],"Perplexity-User":["partial","*",2,1],"Google-Extended":["partial","*",2,1],"Applebot-Extended":["partial","*",2,1],"CCBot":["partial","*",2,1],"Bytespider":["partial","*",2,1],"Amazonbot":["partial","*",2,1],"meta-externalagent":["partial","*",2,1],"cohere-ai":["partial","*",2,1],"Diffbot":["partial","*",2,1],"ContextFetchBot":["partial","*",2,1]},"llms":{"status":"missing","http":404,"bytes":269130},"home":{"status":"ok","http":200,"url":"https://www.dominos.com/","html_tokens":53347,"markdown_tokens":967,"pruned_tokens":23}},
{"rank":9048,"domain":"opendatacommons.org","origin":"https://opendatacommons.org","robots":{"status":"ok","http":200,"url":"https://opendatacommons.org/robots.txt","bytes":1248,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":25098},"home":{"status":"ok","http":200,"url":"https://opendatacommons.org/","html_tokens":6636,"markdown_tokens":212,"pruned_tokens":42}},
{"rank":9049,"domain":"yiwu2016.com","origin":"http://yiwu2016.com","robots":{"status":"refused","http":403,"url":"http://yiwu2016.com/robots.txt","bytes":10,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":9050,"domain":"embrapa.br","origin":"https://embrapa.br","robots":{"status":"ok","http":200,"url":"https://www.embrapa.br/robots.txt","bytes":106252,"groups":2},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["partial","named",1625,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["partial","named",1625,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":128513},"home":{"status":"ok","http":200,"url":"https://www.embrapa.br/","html_tokens":62446,"markdown_tokens":197,"pruned_tokens":158}},
{"rank":9051,"domain":"airliquide.com","origin":"https://airliquide.com","robots":{"status":"ok","http":200,"url":"https://www.airliquide.com/robots.txt","bytes":3228,"groups":1},"bots":{"GPTBot":["partial","*",44,30],"ChatGPT-User":["partial","*",44,30],"OAI-SearchBot":["partial","*",44,30],"ClaudeBot":["partial","*",44,30],"Claude-User":["partial","*",44,30],"Claude-SearchBot":["partial","*",44,30],"anthropic-ai":["partial","*",44,30],"PerplexityBot":["partial","*",44,30],"Perplexity-User":["partial","*",44,30],"Google-Extended":["partial","*",44,30],"Applebot-Extended":["partial","*",44,30],"CCBot":["partial","*",44,30],"Bytespider":["partial","*",44,30],"Amazonbot":["partial","*",44,30],"meta-externalagent":["partial","*",44,30],"cohere-ai":["partial","*",44,30],"Diffbot":["partial","*",44,30],"ContextFetchBot":["partial","*",44,30]},"llms":{"status":"missing","http":404,"bytes":192},"home":{"status":"ok","http":200,"url":"https://www.airliquide.com/","html_tokens":41601,"markdown_tokens":669,"pruned_tokens":57}},
{"rank":9052,"domain":"gizmodo.com.au","origin":"https://gizmodo.com.au","robots":{"status":"html","http":200,"url":"https://gizmodo.com","bytes":392699,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":9053,"domain":"wetteronline.de","origin":"https://wetteronline.de","robots":{"status":"ok","http":200,"url":"https://www.wetteronline.de/robots.txt","bytes":58,"groups":2},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"html","http":200,"bytes":46194},"home":{"status":"ok","http":200,"url":"https://www.wetteronline.de/","html_tokens":82399,"markdown_tokens":17,"pruned_tokens":17}},
{"rank":9054,"domain":"ours.com","origin":"https://ours.com","robots":{"status":"ok","http":200,"url":"https://ours.com/robots.txt","bytes":258,"groups":2},"bots":{"GPTBot":["partial","*",3,0],"ChatGPT-User":["partial","*",3,0],"OAI-SearchBot":["partial","*",3,0],"ClaudeBot":["partial","*",3,0],"Claude-User":["partial","*",3,0],"Claude-SearchBot":["partial","*",3,0],"anthropic-ai":["partial","*",3,0],"PerplexityBot":["partial","*",3,0],"Perplexity-User":["partial","*",3,0],"Google-Extended":["partial","*",3,0],"Applebot-Extended":["partial","*",3,0],"CCBot":["partial","*",3,0],"Bytespider":["partial","*",3,0],"Amazonbot":["partial","*",3,0],"meta-externalagent":["partial","*",3,0],"cohere-ai":["partial","*",3,0],"Diffbot":["partial","*",3,0],"ContextFetchBot":["partial","*",3,0]},"llms":{"status":"missing","http":404,"bytes":355},"home":{"status":"ok","http":200,"url":"https://ours.com/","html_tokens":5881,"markdown_tokens":229,"pruned_tokens":108}},
{"rank":9055,"domain":"coolworldcn.com","origin":"https://coolworldcn.com","robots":{"status":"unreachable","http":null,"url":"https://coolworldcn.com/robots.txt","bytes":0,"groups":0,"error":"[SSL: CERTIFICATE_VERIFY_FAILED] certificate verify failed: unable to get local issuer certificate (_ssl.c:1028)"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":9056,"domain":"js-sannai.com","origin":"http://js-sannai.com","robots":{"status":"ok","http":200,"url":"http://js-sannai.com/robots.txt","bytes":247,"groups":1},"bots":{"GPTBot":["partial","*",10,0],"ChatGPT-User":["partial","*",10,0],"OAI-SearchBot":["partial","*",10,0],"ClaudeBot":["partial","*",10,0],"Claude-User":["partial","*",10,0],"Claude-SearchBot":["partial","*",10,0],"anthropic-ai":["partial","*",10,0],"PerplexityBot":["partial","*",10,0],"Perplexity-User":["partial","*",10,0],"Google-Extended":["partial","*",10,0],"Applebot-Extended":["partial","*",10,0],"CCBot":["partial","*",10,0],"Bytespider":["partial","*",10,0],"Amazonbot":["partial","*",10,0],"meta-externalagent":["partial","*",10,0],"cohere-ai":["partial","*",10,0],"Diffbot":["partial","*",10,0],"ContextFetchBot":["partial","*",10,0]},"llms":{"status":"missing","http":404,"bytes":3693},"home":{"status":"ok","http":200,"url":"http://js-sannai.com/","html_tokens":2645,"markdown_tokens":41,"pruned_tokens":41}},
{"rank":9057,"domain":"cnppri.com.cn","origin":"https://cnppri.com.cn","robots":{"status":"unreachable","http":null,"url":"https://cnppri.com.cn/robots.txt","bytes":0,"groups":0,"error":"[SSL: CERTIFICATE_VERIFY_FAILED] certificate verify failed: Hostname mismatch, certificate is not valid for 'cnppri.com.cn'. (_ssl.c:1028)"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":9058,"domain":"gemini.com","origin":"https://gemini.com","robots":{"status":"ok","http":200,"url":"https://www.gemini.com/robots.txt","bytes":377,"groups":1},"bots":{"GPTBot":["partial","*",6,1],"ChatGPT-User":["partial","*",6,1],"OAI-SearchBot":["partial","*",6,1],"ClaudeBot":["partial","*",6,1],"Claude-User":["partial","*",6,1],"Claude-SearchBot":["partial","*",6,1],"anthropic-ai":["partial","*",6,1],"PerplexityBot":["partial","*",6,1],"Perplexity-User":["partial","*",6,1],"Google-Extended":["partial","*",6,1],"Applebot-Extended":["partial","*",6,1],"CCBot":["partial","*",6,1],"Bytespider":["partial","*",6,1],"Amazonbot":["partial","*",6,1],"meta-externalagent":["partial","*",6,1],"cohere-ai":["partial","*",6,1],"Diffbot":["partial","*",6,1],"ContextFetchBot":["partial","*",6,1]},"llms":{"status":"crawl_delay","error":"Crawl-delay 600 s"},"home":{"status":"crawl_delay","error":"Crawl-delay 600 s"}},
{"rank":9059,"domain":"nikkan.co.jp","origin":"https://nikkan.co.jp","robots":{"status":"missing","http":404,"url":"https://nikkan.co.jp/robots.txt","bytes":134,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":134},"home":{"status":"http_error","http":404,"url":"https://nikkan.co.jp/"}},
{"rank":9060,"domain":"cohere.com","origin":"https://cohere.com","robots":{"status":"ok","http":200,"url":"https://cohere.com/robots.txt","bytes":352,"groups":1},"bots":{"GPTBot":["partial","*",12,1],"ChatGPT-User":["partial","*",12,1],"OAI-SearchBot":["partial","*",12,1],"ClaudeBot":["partial","*",12,1],"Claude-User":["partial","*",12,1],"Claude-SearchBot":["partial","*",12,1],"anthropic-ai":["partial","*",12,1],"PerplexityBot":["partial","*",12,1],"Perplexity-User":["partial","*",12,1],"Google-Extended":["partial","*",12,1],"Applebot-Extended":["partial","*",12,1],"CCBot":["partial","*",12,1],"Bytespider":["partial","*",12,1],"Amazonbot":["partial","*",12,1],"meta-externalagent":["partial","*",12,1],"cohere-ai":["partial","*",12,1],"Diffbot":["partial","*",12,1],"ContextFetchBot":["partial","*",12,1]},"llms":{"status":"present","http":200,"bytes":4805},"home":{"status":"ok","http":200,"url":"https://cohere.com/","html_tokens":147887,"markdown_tokens":49,"pruned_tokens":49}},
{"rank":9061,"domain":"newsis.com","origin":"https://newsis.com","robots":{"status":"ok","http":200,"url":"https://www.newsis.com/robots.txt","bytes":1239,"groups":2},"bots":{"GPTBot":["partial","*",3,1],"ChatGPT-User":["partial","*",3,1],"OAI-SearchBot":["partial","*",3,1],"ClaudeBot":["partial","*",3,1],"Claude-User":["partial","*",3,1],"Claude-SearchBot":["partial","*",3,1],"anthropic-ai":["partial","*",3,1],"PerplexityBot":["partial","*",3,1],"Perplexity-User":["partial","*",3,1],"Google-Extended":["partial","*",3,1],"Applebot-Extended":["partial","*",3,1],"CCBot":["partial","*",3,1],"Bytespider":["partial","*",3,1],"Amazonbot":["partial","*",3,1],"meta-externalagent":["partial","*",3,1],"cohere-ai":["partial","*",3,1],"Diffbot":["partial","*",3,1],"ContextFetchBot":["partial","*",3,1]},"llms":{"status":"html","http":200,"bytes":87680},"home":{"status":"ok","http":200,"url":"https://www.newsis.com/","html_tokens":47035,"markdown_tokens":302,"pruned_tokens":168}},
{"rank":9062,"domain":"real-money-pokies.net","origin":"https://real-money-pokies.net","robots":{"status":"ok","http":200,"url":"https://real-money-pokies.net/robots.txt","bytes":546,"groups":1},"bots":{"GPTBot":["partial","*",19,0],"ChatGPT-User":["partial","*",19,0],"OAI-SearchBot":["partial","*",19,0],"ClaudeBot":["partial","*",19,0],"Claude-User":["partial","*",19,0],"Claude-SearchBot":["partial","*",19,0],"anthropic-ai":["partial","*",19,0],"PerplexityBot":["partial","*",19,0],"Perplexity-User":["partial","*",19,0],"Google-Extended":["partial","*",19,0],"Applebot-Extended":["partial","*",19,0],"CCBot":["partial","*",19,0],"Bytespider":["partial","*",19,0],"Amazonbot":["partial","*",19,0],"meta-externalagent":["partial","*",19,0],"cohere-ai":["partial","*",19,0],"Diffbot":["partial","*",19,0],"ContextFetchBot":["partial","*",19,0]},"llms":{"status":"missing","http":404,"bytes":15},"home":{"status":"http_error","http":404,"url":"https://real-money-pokies.net/"}},
{"rank":9063,"domain":"flathub.org","origin":"https://flathub.org","robots":{"status":"ok","http":200,"url":"https://flathub.org/robots.txt","bytes":337,"groups":8},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["allowed","*",0,1],"OAI-SearchBot":["allowed","*",0,1],"ClaudeBot":["allowed","*",0,1],"Claude-User":["allowed","*",0,1],"Claude-SearchBot":["allowed","*",0,1],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["allowed","*",0,1],"Perplexity-User":["allowed","*",0,1],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["allowed","*",0,1],"CCBot":["blocked","named",1,0],"Bytespider":["allowed","*",0,1],"Amazonbot":["allowed","*",0,1],"meta-externalagent":["allowed","*",0,1],"cohere-ai":["allowed","*",0,1],"Diffbot":["allowed","*",0,1],"ContextFetchBot":["allowed","*",0,1]},"llms":{"status":"missing","http":404,"bytes":6626},"home":{"status":"ok","http":200,"url":"https://flathub.org/en","html_tokens":199218,"markdown_tokens":20,"pruned_tokens":20}},
{"rank":9064,"domain":"uakron.edu","origin":"https://uakron.edu","robots":{"status":"ok","http":200,"url":"https://www.uakron.edu/robots.txt","bytes":1709,"groups":3},"bots":{"GPTBot":["partial","*",47,0],"ChatGPT-User":["partial","*",47,0],"OAI-SearchBot":["allowed","named",0,1],"ClaudeBot":["partial","*",47,0],"Claude-User":["partial","*",47,0],"Claude-SearchBot":["partial","*",47,0],"anthropic-ai":["partial","*",47,0],"PerplexityBot":["partial","*",47,0],"Perplexity-User":["partial","*",47,0],"Google-Extended":["partial","*",47,0],"Applebot-Extended":["partial","*",47,0],"CCBot":["partial","*",47,0],"Bytespider":["partial","*",47,0],"Amazonbot":["partial","*",47,0],"meta-externalagent":["partial","*",47,0],"cohere-ai":["partial","*",47,0],"Diffbot":["partial","*",47,0],"ContextFetchBot":["partial","*",47,0]},"llms":{"status":"missing","http":404,"bytes":136791},"home":{"status":"ok","http":200,"url":"https://www.uakron.edu/","html_tokens":47693,"markdown_tokens":289,"pruned_tokens":134}},
{"rank":9065,"domain":"hnzhonglinda.com","origin":"http://hnzhonglinda.com","robots":{"status":"html","http":200,"url":"http://hnzhonglinda.com/robots.txt","bytes":717,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":9066,"domain":"tcs.com","origin":"https://tcs.com","robots":{"status":"refused","http":403,"url":"https://www.tcs.com/robots.txt","bytes":379,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":9067,"domain":"bloomingdales.com","origin":"https://bloomingdales.com","robots":{"status":"refused","http":403,"url":"https://www.bloomingdales.com/robots.txt","bytes":12237,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":9068,"domain":"theicct.org","origin":"https://theicct.org","robots":{"status":"ok","http":200,"url":"https://theicct.org/robots.txt","bytes":264,"groups":1},"bots":{"GPTBot":["partial","*",2,0],"ChatGPT-User":["partial","*",2,0],"OAI-SearchBot":["partial","*",2,0],"ClaudeBot":["partial","*",2,0],"Claude-User":["partial","*",2,0],"Claude-SearchBot":["partial","*",2,0],"anthropic-ai":["partial","*",2,0],"PerplexityBot":["partial","*",2,0],"Perplexity-User":["partial","*",2,0],"Google-Extended":["partial","*",2,0],"Applebot-Extended":["partial","*",2,0],"CCBot":["partial","*",2,0],"Bytespider":["partial","*",2,0],"Amazonbot":["partial","*",2,0],"meta-externalagent":["partial","*",2,0],"cohere-ai":["partial","*",2,0],"Diffbot":["partial","*",2,0],"ContextFetchBot":["partial","*",2,0]},"llms":{"status":"http_error","http":403,"bytes":6169},"home":{"status":"http_error","http":403,"url":"https://theicct.org/"}},
{"rank":9069,"domain":"nycourts.gov","origin":"https://nycourts.gov","robots":{"status":"refused","http":403,"url":"https://www.nycourts.gov/robots.txt","bytes":5554,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":9070,"domain":"rowman.com","origin":"https://rowman.com","robots":{"status":"refused","http":403,"url":"https://rowman.com/robots.txt","bytes":1233,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":9071,"domain":"decider.com","origin":"https://decider.com","robots":{"status":"ok","http":200,"url":"https://decider.com/robots.txt","bytes":6151,"groups":65},"bots":{"GPTBot":["partial","*",92,2],"ChatGPT-User":["partial","*",92,2],"OAI-SearchBot":["partial","*",92,2],"ClaudeBot":["blocked","named",1,0],"Claude-User":["blocked","named",1,0],"Claude-SearchBot":["blocked","named",1,0],"anthropic-ai":["blocked","named",2,0],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["partial","*",92,2],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",3,0],"cohere-ai":["blocked","named",2,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",92,2]},"llms":{"status":"missing","http":404,"bytes":238358},"home":{"status":"ok","http":200,"url":"https://decider.com/","html_tokens":848250,"markdown_tokens":421,"pruned_tokens":137}},
{"rank":9072,"domain":"unian.net","origin":"https://unian.net","robots":{"status":"ok","http":200,"url":"https://www.unian.net/robots.txt","bytes":1975,"groups":6},"bots":{"GPTBot":["partial","*",34,6],"ChatGPT-User":["partial","*",34,6],"OAI-SearchBot":["partial","*",34,6],"ClaudeBot":["partial","*",34,6],"Claude-User":["partial","*",34,6],"Claude-SearchBot":["partial","*",34,6],"anthropic-ai":["partial","*",34,6],"PerplexityBot":["partial","*",34,6],"Perplexity-User":["partial","*",34,6],"Google-Extended":["partial","*",34,6],"Applebot-Extended":["partial","*",34,6],"CCBot":["partial","*",34,6],"Bytespider":["partial","*",34,6],"Amazonbot":["partial","*",34,6],"meta-externalagent":["partial","*",34,6],"cohere-ai":["partial","*",34,6],"Diffbot":["partial","*",34,6],"ContextFetchBot":["partial","*",34,6]},"llms":{"status":"missing","http":404,"bytes":71431},"home":{"status":"ok","http":200,"url":"https://www.unian.net/","html_tokens":37121,"markdown_tokens":118,"pruned_tokens":118}},
{"rank":9073,"domain":"23andme.org","origin":"https://23andme.org","robots":{"status":"ok","http":200,"url":"https://www.23andme.org/robots.txt","bytes":3030,"groups":4},"bots":{"GPTBot":["partial","*",14,1],"ChatGPT-User":["partial","named",8,1],"OAI-SearchBot":["partial","named",8,1],"ClaudeBot":["partial","*",14,1],"Claude-User":["partial","named",8,1],"Claude-SearchBot":["partial","named",8,1],"anthropic-ai":["partial","*",14,1],"PerplexityBot":["partial","named",8,1],"Perplexity-User":["partial","named",8,1],"Google-Extended":["partial","*",14,1],"Applebot-Extended":["partial","*",14,1],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["partial","*",14,1],"meta-externalagent":["partial","*",14,1],"cohere-ai":["partial","*",14,1],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",14,1]},"llms":{"status":"present","http":200,"bytes":6639},"home":{"status":"ok","http":200,"url":"https://www.23andme.org/","html_tokens":32016,"markdown_tokens":2970,"pruned_tokens":553}},
{"rank":9074,"domain":"desktopnexus.com","origin":"https://desktopnexus.com","robots":{"status":"ok","http":200,"url":"https://desktopnexus.com/robots.txt","bytes":426,"groups":4},"bots":{"GPTBot":["partial","*",14,0],"ChatGPT-User":["partial","*",14,0],"OAI-SearchBot":["partial","*",14,0],"ClaudeBot":["partial","*",14,0],"Claude-User":["partial","*",14,0],"Claude-SearchBot":["partial","*",14,0],"anthropic-ai":["partial","*",14,0],"PerplexityBot":["partial","*",14,0],"Perplexity-User":["partial","*",14,0],"Google-Extended":["partial","*",14,0],"Applebot-Extended":["partial","*",14,0],"CCBot":["partial","*",14,0],"Bytespider":["partial","*",14,0],"Amazonbot":["partial","*",14,0],"meta-externalagent":["partial","*",14,0],"cohere-ai":["partial","*",14,0],"Diffbot":["partial","*",14,0],"ContextFetchBot":["partial","*",14,0]},"llms":{"status":"missing","http":404,"bytes":9666},"home":{"status":"ok","http":200,"url":"https://desktopnexus.com/","html_tokens":31437,"markdown_tokens":61,"pruned_tokens":60}},
{"rank":9075,"domain":"uclm.es","origin":"https://www.uclm.es","robots":{"status":"refused","http":403,"url":"https://www.uclm.es/robots.txt","bytes":9402,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":9076,"domain":"sendibm3.com","origin":"https://sendibm3.com","robots":{"status":"ok","http":200,"url":"https://sendibm3.com/robots.txt","bytes":102,"groups":3},"bots":{"GPTBot":["blocked","*",1,0],"ChatGPT-User":["blocked","*",1,0],"OAI-SearchBot":["blocked","*",1,0],"ClaudeBot":["blocked","*",1,0],"Claude-User":["blocked","*",1,0],"Claude-SearchBot":["blocked","*",1,0],"anthropic-ai":["blocked","*",1,0],"PerplexityBot":["blocked","*",1,0],"Perplexity-User":["blocked","*",1,0],"Google-Extended":["blocked","*",1,0],"Applebot-Extended":["blocked","*",1,0],"CCBot":["blocked","*",1,0],"Bytespider":["blocked","*",1,0],"Amazonbot":["blocked","*",1,0],"meta-externalagent":["blocked","*",1,0],"cohere-ai":["blocked","*",1,0],"Diffbot":["blocked","*",1,0],"ContextFetchBot":["blocked","*",1,0]},"llms":{"status":"disallowed"},"home":{"status":"disallowed"}},
{"rank":9077,"domain":"lepetitjournal.com","origin":"https://lepetitjournal.com","robots":{"status":"ok","http":200,"url":"https://lepetitjournal.com/robots.txt","bytes":5339,"groups":1},"bots":{"GPTBot":["partial","*",12,1],"ChatGPT-User":["partial","*",12,1],"OAI-SearchBot":["partial","*",12,1],"ClaudeBot":["partial","*",12,1],"Claude-User":["partial","*",12,1],"Claude-SearchBot":["partial","*",12,1],"anthropic-ai":["partial","*",12,1],"PerplexityBot":["partial","*",12,1],"Perplexity-User":["partial","*",12,1],"Google-Extended":["partial","*",12,1],"Applebot-Extended":["partial","*",12,1],"CCBot":["partial","*",12,1],"Bytespider":["partial","*",12,1],"Amazonbot":["partial","*",12,1],"meta-externalagent":["partial","*",12,1],"cohere-ai":["partial","*",12,1],"Diffbot":["partial","*",12,1],"ContextFetchBot":["partial","*",12,1]},"llms":{"status":"missing","http":404,"bytes":125026},"home":{"status":"ok","http":200,"url":"https://lepetitjournal.com/","html_tokens":215954,"markdown_tokens":66,"pruned_tokens":66}},
{"rank":9078,"domain":"marist.edu","origin":"https://marist.edu","robots":{"status":"ok","http":200,"url":"https://www.marist.edu/robots.txt","bytes":1766,"groups":1},"bots":{"GPTBot":["partial","*",36,0],"ChatGPT-User":["partial","*",36,0],"OAI-SearchBot":["partial","*",36,0],"ClaudeBot":["partial","*",36,0],"Claude-User":["partial","*",36,0],"Claude-SearchBot":["partial","*",36,0],"anthropic-ai":["partial","*",36,0],"PerplexityBot":["partial","*",36,0],"Perplexity-User":["partial","*",36,0],"Google-Extended":["partial","*",36,0],"Applebot-Extended":["partial","*",36,0],"CCBot":["partial","*",36,0],"Bytespider":["partial","*",36,0],"Amazonbot":["partial","*",36,0],"meta-externalagent":["partial","*",36,0],"cohere-ai":["partial","*",36,0],"Diffbot":["partial","*",36,0],"ContextFetchBot":["partial","*",36,0]},"llms":{"status":"missing","http":404,"bytes":296923},"home":{"status":"ok","http":200,"url":"https://www.marist.edu/","html_tokens":114854,"markdown_tokens":1147,"pruned_tokens":329}},
{"rank":9079,"domain":"mstdn.social","origin":"https://mstdn.social","robots":{"status":"ok","http":200,"url":"https://mstdn.social/robots.txt","bytes":231,"groups":2},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["partial","*",3,0],"OAI-SearchBot":["partial","*",3,0],"ClaudeBot":["partial","*",3,0],"Claude-User":["partial","*",3,0],"Claude-SearchBot":["partial","*",3,0],"anthropic-ai":["partial","*",3,0],"PerplexityBot":["partial","*",3,0],"Perplexity-User":["partial","*",3,0],"Google-Extended":["partial","*",3,0],"Applebot-Extended":["partial","*",3,0],"CCBot":["partial","*",3,0],"Bytespider":["partial","*",3,0],"Amazonbot":["partial","*",3,0],"meta-externalagent":["partial","*",3,0],"cohere-ai":["partial","*",3,0],"Diffbot":["partial","*",3,0],"ContextFetchBot":["partial","*",3,0]},"llms":{"status":"missing","http":404,"bytes":1539},"home":{"status":"ok","http":200,"url":"https://mstdn.social/","html_tokens":13533,"markdown_tokens":34,"pruned_tokens":34}},
{"rank":9080,"domain":"basketball-reference.com","origin":"https://basketball-reference.com","robots":{"status":"ok","http":200,"url":"https://www.basketball-reference.com/robots.txt?&cf=1987","bytes":766,"groups":5},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["partial","*",16,0],"OAI-SearchBot":["partial","*",16,0],"ClaudeBot":["partial","*",16,0],"Claude-User":["partial","*",16,0],"Claude-SearchBot":["partial","*",16,0],"anthropic-ai":["partial","*",16,0],"PerplexityBot":["partial","*",16,0],"Perplexity-User":["partial","*",16,0],"Google-Extended":["partial","*",16,0],"Applebot-Extended":["partial","*",16,0],"CCBot":["partial","*",16,0],"Bytespider":["partial","*",16,0],"Amazonbot":["partial","*",16,0],"meta-externalagent":["partial","*",16,0],"cohere-ai":["partial","*",16,0],"Diffbot":["partial","*",16,0],"ContextFetchBot":["partial","*",16,0]},"llms":{"status":"missing","http":404,"bytes":93043},"home":{"status":"ok","http":200,"url":"https://www.basketball-reference.com/?&cf=1987","html_tokens":87409,"markdown_tokens":1420,"pruned_tokens":105}},
{"rank":9081,"domain":"lumalabs.ai","origin":"https://lumalabs.ai","robots":{"status":"ok","http":200,"url":"https://lumalabs.ai/robots.txt","bytes":761,"groups":1},"bots":{"GPTBot":["partial","*",4,1],"ChatGPT-User":["partial","*",4,1],"OAI-SearchBot":["partial","*",4,1],"ClaudeBot":["partial","*",4,1],"Claude-User":["partial","*",4,1],"Claude-SearchBot":["partial","*",4,1],"anthropic-ai":["partial","*",4,1],"PerplexityBot":["partial","*",4,1],"Perplexity-User":["partial","*",4,1],"Google-Extended":["partial","*",4,1],"Applebot-Extended":["partial","*",4,1],"CCBot":["partial","*",4,1],"Bytespider":["partial","*",4,1],"Amazonbot":["partial","*",4,1],"meta-externalagent":["partial","*",4,1],"cohere-ai":["partial","*",4,1],"Diffbot":["partial","*",4,1],"ContextFetchBot":["partial","*",4,1]},"llms":{"status":"present","http":200,"bytes":2886},"home":{"status":"not_html","error":"unsupported content type text/markdown"}},
{"rank":9082,"domain":"ultipro.com","origin":"https://ultipro.com","robots":{"status":"refused","http":403,"url":"https://ultipro.com/robots.txt","bytes":14238,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":9083,"domain":"global.toshiba","origin":"https://www.global.toshiba","robots":{"status":"missing","http":404,"url":"https://www.global.toshiba/robots.txt","bytes":11460,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":11460},"home":{"status":"ok","http":200,"url":"https://www.global.toshiba/jp/top.html","html_tokens":31894,"markdown_tokens":23,"pruned_tokens":23}},
{"rank":9084,"domain":"oceana.org","origin":"https://oceana.org","robots":{"status":"refused","http":403,"url":"https://oceana.org/robots.txt","bytes":560,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":9085,"domain":"hothardware.com","origin":"https://hothardware.com","robots":{"status":"refused","http":406,"url":"https://hothardware.com/robots.txt","bytes":1346,"groups":0,"error":"HTTP 406"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":9086,"domain":"kobobooks.com","origin":"https://kobobooks.com","robots":{"status":"ok","http":200,"url":"https://www.kobo.com/robots.txt","bytes":3240,"groups":1},"bots":{"GPTBot":["partial","*",95,48],"ChatGPT-User":["partial","*",95,48],"OAI-SearchBot":["partial","*",95,48],"ClaudeBot":["partial","*",95,48],"Claude-User":["partial","*",95,48],"Claude-SearchBot":["partial","*",95,48],"anthropic-ai":["partial","*",95,48],"PerplexityBot":["partial","*",95,48],"Perplexity-User":["partial","*",95,48],"Google-Extended":["partial","*",95,48],"Applebot-Extended":["partial","*",95,48],"CCBot":["partial","*",95,48],"Bytespider":["partial","*",95,48],"Amazonbot":["partial","*",95,48],"meta-externalagent":["partial","*",95,48],"cohere-ai":["partial","*",95,48],"Diffbot":["partial","*",95,48],"ContextFetchBot":["partial","*",95,48]},"llms":{"status":"http_error","http":403,"bytes":84272},"home":{"status":"http_error","http":403,"url":"https://www.kobo.com/"}},
{"rank":9087,"domain":"cengage.com","origin":"https://cengage.com","robots":{"status":"ok","http":200,"url":"https://www.cengage.com/robots.txt","bytes":4060,"groups":3},"bots":{"GPTBot":["partial","named",13,0],"ChatGPT-User":["partial","*",64,2],"OAI-SearchBot":["partial","*",64,2],"ClaudeBot":["partial","*",64,2],"Claude-User":["partial","*",64,2],"Claude-SearchBot":["partial","*",64,2],"anthropic-ai":["partial","*",64,2],"PerplexityBot":["partial","*",64,2],"Perplexity-User":["partial","*",64,2],"Google-Extended":["partial","*",64,2],"Applebot-Extended":["partial","*",64,2],"CCBot":["partial","*",64,2],"Bytespider":["partial","*",64,2],"Amazonbot":["partial","*",64,2],"meta-externalagent":["partial","named",13,0],"cohere-ai":["partial","*",64,2],"Diffbot":["partial","*",64,2],"ContextFetchBot":["partial","*",64,2]},"llms":{"status":"missing","http":404,"bytes":142516},"home":{"status":"ok","http":200,"url":"https://www.cengage.com/","html_tokens":43143,"markdown_tokens":268,"pruned_tokens":190}},
{"rank":9088,"domain":"geb.net.cn","origin":"https://geb.net.cn","robots":{"status":"unreachable","http":null,"url":"https://geb.net.cn/robots.txt","bytes":0,"groups":0,"error":"cannot resolve geb.net.cn"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":9089,"domain":"tyxsyy.com","origin":"https://tyxsyy.com","robots":{"status":"unreachable","http":null,"url":"https://tyxsyy.com/robots.txt","bytes":0,"groups":0,"error":"cannot resolve tyxsyy.com"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":9090,"domain":"wyzowl.com","origin":"https://wyzowl.com","robots":{"status":"ok","http":200,"url":"https://wyzowl.com/robots.txt","bytes":64,"groups":1},"bots":{"GPTBot":["allowed","*",0,1],"ChatGPT-User":["allowed","*",0,1],"OAI-SearchBot":["allowed","*",0,1],"ClaudeBot":["allowed","*",0,1],"Claude-User":["allowed","*",0,1],"Claude-SearchBot":["allowed","*",0,1],"anthropic-ai":["allowed","*",0,1],"PerplexityBot":["allowed","*",0,1],"Perplexity-User":["allowed","*",0,1],"Google-Extended":["allowed","*",0,1],"Applebot-Extended":["allowed","*",0,1],"CCBot":["allowed","*",0,1],"Bytespider":["allowed","*",0,1],"Amazonbot":["allowed","*",0,1],"meta-externalagent":["allowed","*",0,1],"cohere-ai":["allowed","*",0,1],"Diffbot":["allowed","*",0,1],"ContextFetchBot":["allowed","*",0,1]},"llms":{"status":"missing","http":404,"bytes":25355},"home":{"status":"http_error","http":403,"url":"https://wyzowl.com/"}},
{"rank":9091,"domain":"petition.parliament.uk","origin":"https://petition.parliament.uk","robots":{"status":"ok","http":200,"url":"https://petition.parliament.uk/robots.txt","bytes":416,"groups":1},"bots":{"GPTBot":["partial","*",15,0],"ChatGPT-User":["partial","*",15,0],"OAI-SearchBot":["partial","*",15,0],"ClaudeBot":["partial","*",15,0],"Claude-User":["partial","*",15,0],"Claude-SearchBot":["partial","*",15,0],"anthropic-ai":["partial","*",15,0],"PerplexityBot":["partial","*",15,0],"Perplexity-User":["partial","*",15,0],"Google-Extended":["partial","*",15,0],"Applebot-Extended":["partial","*",15,0],"CCBot":["partial","*",15,0],"Bytespider":["partial","*",15,0],"Amazonbot":["partial","*",15,0],"meta-externalagent":["partial","*",15,0],"cohere-ai":["partial","*",15,0],"Diffbot":["partial","*",15,0],"ContextFetchBot":["partial","*",15,0]},"llms":{"status":"missing","http":404,"bytes":13606},"home":{"status":"ok","http":200,"url":"https://petition.parliament.uk/","html_tokens":7473,"markdown_tokens":324,"pruned_tokens":63}},
{"rank":9092,"domain":"jobvite.com","origin":"https://jobvite.com","robots":{"status":"refused","http":403,"url":"https://jobvite.com/robots.txt","bytes":560,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":9093,"domain":"umassmed.edu","origin":"https://umassmed.edu","robots":{"status":"ok","http":200,"url":"https://www.umassmed.edu/robots.txt","bytes":25900,"groups":3},"bots":{"GPTBot":["partial","*",36,0],"ChatGPT-User":["partial","*",36,0],"OAI-SearchBot":["partial","*",36,0],"ClaudeBot":["partial","*",36,0],"Claude-User":["partial","*",36,0],"Claude-SearchBot":["partial","*",36,0],"anthropic-ai":["partial","*",36,0],"PerplexityBot":["partial","*",36,0],"Perplexity-User":["partial","*",36,0],"Google-Extended":["partial","*",36,0],"Applebot-Extended":["partial","*",36,0],"CCBot":["partial","*",36,0],"Bytespider":["partial","*",36,0],"Amazonbot":["partial","*",36,0],"meta-externalagent":["partial","*",36,0],"cohere-ai":["partial","*",36,0],"Diffbot":["partial","*",36,0],"ContextFetchBot":["partial","*",36,0]},"llms":{"status":"missing","http":404,"bytes":5633},"home":{"status":"http_error","http":403,"url":"https://www.umassmed.edu/"}},
{"rank":9094,"domain":"thecvf.com","origin":"https://thecvf.com","robots":{"status":"ok","http":200,"url":"https://www.thecvf.com/robots.txt","bytes":185,"groups":1},"bots":{"GPTBot":["partial","*",8,0],"ChatGPT-User":["partial","*",8,0],"OAI-SearchBot":["partial","*",8,0],"ClaudeBot":["partial","*",8,0],"Claude-User":["partial","*",8,0],"Claude-SearchBot":["partial","*",8,0],"anthropic-ai":["partial","*",8,0],"PerplexityBot":["partial","*",8,0],"Perplexity-User":["partial","*",8,0],"Google-Extended":["partial","*",8,0],"Applebot-Extended":["partial","*",8,0],"CCBot":["partial","*",8,0],"Bytespider":["partial","*",8,0],"Amazonbot":["partial","*",8,0],"meta-externalagent":["partial","*",8,0],"cohere-ai":["partial","*",8,0],"Diffbot":["partial","*",8,0],"ContextFetchBot":["partial","*",8,0]},"llms":{"status":"missing","http":404,"bytes":315},"home":{"status":"ok","http":200,"url":"https://www.thecvf.com/","html_tokens":8181,"markdown_tokens":34,"pruned_tokens":34}},
{"rank":9095,"domain":"sun.ac.za","origin":"https://sun.ac.za","robots":{"status":"timeout","http":null,"url":"https://sun.ac.za/robots.txt","bytes":0,"groups":0,"error":"ConnectTimeout"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":9096,"domain":"minobrnauki.gov.ru","origin":"https://minobrnauki.gov.ru","robots":{"status":"ok","http":200,"url":"https://minobrnauki.gov.ru/robots.txt","bytes":2310,"groups":1},"bots":{"GPTBot":["partial","*",65,5],"ChatGPT-User":["partial","*",65,5],"OAI-SearchBot":["partial","*",65,5],"ClaudeBot":["partial","*",65,5],"Claude-User":["partial","*",65,5],"Claude-SearchBot":["partial","*",65,5],"anthropic-ai":["partial","*",65,5],"PerplexityBot":["partial","*",65,5],"Perplexity-User":["partial","*",65,5],"Google-Extended":["partial","*",65,5],"Applebot-Extended":["partial","*",65,5],"CCBot":["partial","*",65,5],"Bytespider":["partial","*",65,5],"Amazonbot":["partial","*",65,5],"meta-externalagent":["partial","*",65,5],"cohere-ai":["partial","*",65,5],"Diffbot":["partial","*",65,5],"ContextFetchBot":["partial","*",65,5]},"llms":{"status":"missing","http":404,"bytes":48181},"home":{"status":"ok","http":200,"url":"https://minobrnauki.gov.ru/","html_tokens":20691,"markdown_tokens":204,"pruned_tokens":159}},
{"rank":9097,"domain":"come.to","origin":"https://come.to","robots":{"status":"ok","http":200,"url":"https://come.to/robots.txt","bytes":266,"groups":1},"bots":{"GPTBot":["partial","*",9,0],"ChatGPT-User":["partial","*",9,0],"OAI-SearchBot":["partial","*",9,0],"ClaudeBot":["partial","*",9,0],"Claude-User":["partial","*",9,0],"Claude-SearchBot":["partial","*",9,0],"anthropic-ai":["partial","*",9,0],"PerplexityBot":["partial","*",9,0],"Perplexity-User":["partial","*",9,0],"Google-Extended":["partial","*",9,0],"Applebot-Extended":["partial","*",9,0],"CCBot":["partial","*",9,0],"Bytespider":["partial","*",9,0],"Amazonbot":["partial","*",9,0],"meta-externalagent":["partial","*",9,0],"cohere-ai":["partial","*",9,0],"Diffbot":["partial","*",9,0],"ContextFetchBot":["partial","*",9,0]},"llms":{"status":"missing","http":404,"bytes":7899},"home":{"status":"ok","http":200,"url":"https://welcome.to/","html_tokens":69936,"markdown_tokens":770,"pruned_tokens":102}},
{"rank":9098,"domain":"hbzy.edu.cn","origin":"https://hbzy.edu.cn","robots":{"status":"missing","http":404,"url":"https://hbzy.edu.cn/robots.txt","bytes":1693,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":1693},"home":{"status":"ok","http":200,"url":"https://hbzy.edu.cn/","html_tokens":11123,"markdown_tokens":22,"pruned_tokens":22}},
{"rank":9099,"domain":"gusto.com","origin":"https://gusto.com","robots":{"status":"refused","http":403,"url":"https://gusto.com/robots.txt","bytes":5504,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":9100,"domain":"bscscan.com","origin":"https://bscscan.com","robots":{"status":"ok","http":200,"url":"https://bscscan.com/robots.txt","bytes":514,"groups":1},"bots":{"GPTBot":["partial","*",8,0],"ChatGPT-User":["partial","*",8,0],"OAI-SearchBot":["partial","*",8,0],"ClaudeBot":["partial","*",8,0],"Claude-User":["partial","*",8,0],"Claude-SearchBot":["partial","*",8,0],"anthropic-ai":["partial","*",8,0],"PerplexityBot":["partial","*",8,0],"Perplexity-User":["partial","*",8,0],"Google-Extended":["partial","*",8,0],"Applebot-Extended":["partial","*",8,0],"CCBot":["partial","*",8,0],"Bytespider":["partial","*",8,0],"Amazonbot":["partial","*",8,0],"meta-externalagent":["partial","*",8,0],"cohere-ai":["partial","*",8,0],"Diffbot":["partial","*",8,0],"ContextFetchBot":["partial","*",8,0]},"llms":{"status":"present","http":200,"bytes":5472},"home":{"status":"http_error","http":403,"url":"https://bscscan.com/"}},
{"rank":9101,"domain":"loopnet.com","origin":"https://loopnet.com","robots":{"status":"refused","http":403,"url":"https://loopnet.com/robots.txt","bytes":375,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":9102,"domain":"sfweekly.com","origin":"https://sfweekly.com","robots":{"status":"ok","http":200,"url":"https://sfweekly.com/robots.txt","bytes":4341,"groups":31},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["blocked","named",1,0],"OAI-SearchBot":["blocked","named",1,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",28,0],"Claude-SearchBot":["partial","*",28,0],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["partial","*",28,0],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["partial","*",28,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",28,0]},"llms":{"status":"http_error","http":429,"bytes":64},"home":{"status":"not_html","error":"unsupported content type text/plain"}},
{"rank":9103,"domain":"americanrhetoric.com","origin":"https://americanrhetoric.com","robots":{"status":"ok","http":200,"url":"https://www.americanrhetoric.com/robots.txt","bytes":711,"groups":17},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["blocked","named",1,0],"OAI-SearchBot":["allowed","*",0,1],"ClaudeBot":["allowed","*",0,1],"Claude-User":["allowed","*",0,1],"Claude-SearchBot":["allowed","*",0,1],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["allowed","*",0,1],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["allowed","*",0,1],"ContextFetchBot":["allowed","*",0,1]},"llms":{"status":"missing","http":404,"bytes":940},"home":{"status":"ok","http":200,"url":"https://americanrhetoric.com/","html_tokens":74086,"markdown_tokens":655,"pruned_tokens":90}},
{"rank":9104,"domain":"fco.gov.uk","origin":"http://fco.gov.uk","robots":{"status":"ok","http":200,"url":"http://fco.gov.uk/robots.txt","bytes":63,"groups":1},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"missing","http":404,"bytes":1729},"home":{"status":"ok","http":200,"url":"https://www.gov.uk/government/organisations/foreign-commonwealth-office","html_tokens":17714,"markdown_tokens":28,"pruned_tokens":21}},
{"rank":9105,"domain":"aliqbwl.com","origin":"http://aliqbwl.com","robots":{"status":"missing","http":404,"url":"http://aliqbwl.com/robots.txt","bytes":208,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":206},"home":{"status":"http_error","http":403,"url":"http://aliqbwl.com/"}},
{"rank":9106,"domain":"insighttimer.com","origin":"https://insighttimer.com","robots":{"status":"ok","http":200,"url":"https://insighttimer.com/robots.txt","bytes":188,"groups":1},"bots":{"GPTBot":["partial","*",3,2],"ChatGPT-User":["partial","*",3,2],"OAI-SearchBot":["partial","*",3,2],"ClaudeBot":["partial","*",3,2],"Claude-User":["partial","*",3,2],"Claude-SearchBot":["partial","*",3,2],"anthropic-ai":["partial","*",3,2],"PerplexityBot":["partial","*",3,2],"Perplexity-User":["partial","*",3,2],"Google-Extended":["partial","*",3,2],"Applebot-Extended":["partial","*",3,2],"CCBot":["partial","*",3,2],"Bytespider":["partial","*",3,2],"Amazonbot":["partial","*",3,2],"meta-externalagent":["partial","*",3,2],"cohere-ai":["partial","*",3,2],"Diffbot":["partial","*",3,2],"ContextFetchBot":["partial","*",3,2]},"llms":{"status":"present","http":200,"bytes":25950},"home":{"status":"ok","http":200,"url":"https://insighttimer.com/","html_tokens":2129,"markdown_tokens":0,"pruned_tokens":0}},
{"rank":9107,"domain":"fox8.com","origin":"https://fox8.com","robots":{"status":"ok","http":200,"url":"https://fox8.com/robots.txt","bytes":2218,"groups":38},"bots":{"GPTBot":["blocked","named",2,0],"ChatGPT-User":["blocked","named",1,0],"OAI-SearchBot":["blocked","named",1,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",12,1],"Claude-SearchBot":["partial","*",12,1],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["blocked","named",2,0],"Perplexity-User":["partial","*",12,1],"Google-Extended":["blocked","named",2,0],"Applebot-Extended":["blocked","named",2,0],"CCBot":["blocked","named",2,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["partial","*",12,1],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",12,1]},"llms":{"status":"http_error","http":403,"bytes":6514},"home":{"status":"http_error","http":403,"url":"https://fox8.com/"}},
{"rank":9108,"domain":"dmv.org","origin":"https://dmv.org","robots":{"status":"unreachable","http":null,"url":"https://dmv.org/robots.txt","bytes":0,"groups":0,"error":"cannot resolve dmv.org"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":9109,"domain":"google.bf","origin":"https://google.bf","robots":{"status":"ok","http":200,"url":"https://www.google.bf/robots.txt","bytes":6624,"groups":4},"bots":{"GPTBot":["partial","*",169,64],"ChatGPT-User":["partial","*",169,64],"OAI-SearchBot":["partial","*",169,64],"ClaudeBot":["partial","*",169,64],"Claude-User":["partial","*",169,64],"Claude-SearchBot":["partial","*",169,64],"anthropic-ai":["partial","*",169,64],"PerplexityBot":["partial","*",169,64],"Perplexity-User":["partial","*",169,64],"Google-Extended":["partial","*",169,64],"Applebot-Extended":["partial","*",169,64],"CCBot":["partial","*",169,64],"Bytespider":["partial","*",169,64],"Amazonbot":["partial","*",169,64],"meta-externalagent":["partial","*",169,64],"cohere-ai":["partial","*",169,64],"Diffbot":["partial","*",169,64],"ContextFetchBot":["partial","*",169,64]},"llms":{"status":"missing","http":404,"bytes":1569},"home":{"status":"ok","http":200,"url":"https://www.google.com/","html_tokens":21312,"markdown_tokens":30,"pruned_tokens":30}},
{"rank":9110,"domain":"guancha.cn","origin":"https://guancha.cn","robots":{"status":"unreachable","http":null,"url":"https://guancha.cn/robots.txt","bytes":0,"groups":0,"error":"All connection attempts failed"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":9111,"domain":"gmo.media","origin":"https://gmo.media","robots":{"status":"ok","http":200,"url":"https://www.gmo.media/robots.txt","bytes":231,"groups":1},"bots":{"GPTBot":["partial","*",1,0],"ChatGPT-User":["partial","*",1,0],"OAI-SearchBot":["partial","*",1,0],"ClaudeBot":["partial","*",1,0],"Claude-User":["partial","*",1,0],"Claude-SearchBot":["partial","*",1,0],"anthropic-ai":["partial","*",1,0],"PerplexityBot":["partial","*",1,0],"Perplexity-User":["partial","*",1,0],"Google-Extended":["partial","*",1,0],"Applebot-Extended":["partial","*",1,0],"CCBot":["partial","*",1,0],"Bytespider":["partial","*",1,0],"Amazonbot":["partial","*",1,0],"meta-externalagent":["partial","*",1,0],"cohere-ai":["partial","*",1,0],"Diffbot":["partial","*",1,0],"ContextFetchBot":["partial","*",1,0]},"llms":{"status":"missing","http":404,"bytes":35689},"home":{"status":"ok","http":200,"url":"https://www.gmo.media/","html_tokens":53,"markdown_tokens":0,"pruned_tokens":0}},
{"rank":9112,"domain":"lamag.com","origin":"https://lamag.com","robots":{"status":"missing","http":404,"url":"https://lamag.com/robots.txt","bytes":167,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":85911},"home":{"status":"ok","http":200,"url":"https://lamag.com/","html_tokens":64094,"markdown_tokens":159,"pruned_tokens":159}},
{"rank":9113,"domain":"questia.com","origin":"https://questia.com","robots":{"status":"timeout","http":null,"url":"https://questia.com/robots.txt","bytes":0,"groups":0,"error":"ConnectTimeout"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":9114,"domain":"kerala.gov.in","origin":"https://kerala.gov.in","robots":{"status":"server_error","http":503,"url":"https://kerala.gov.in/robots.txt","bytes":803,"groups":0,"error":"HTTP 503"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":9115,"domain":"thegospelcoalition.org","origin":"https://thegospelcoalition.org","robots":{"status":"ok","http":200,"url":"https://www.thegospelcoalition.org/robots.txt","bytes":77,"groups":1},"bots":{"GPTBot":["partial","*",2,0],"ChatGPT-User":["partial","*",2,0],"OAI-SearchBot":["partial","*",2,0],"ClaudeBot":["partial","*",2,0],"Claude-User":["partial","*",2,0],"Claude-SearchBot":["partial","*",2,0],"anthropic-ai":["partial","*",2,0],"PerplexityBot":["partial","*",2,0],"Perplexity-User":["partial","*",2,0],"Google-Extended":["partial","*",2,0],"Applebot-Extended":["partial","*",2,0],"CCBot":["partial","*",2,0],"Bytespider":["partial","*",2,0],"Amazonbot":["partial","*",2,0],"meta-externalagent":["partial","*",2,0],"cohere-ai":["partial","*",2,0],"Diffbot":["partial","*",2,0],"ContextFetchBot":["partial","*",2,0]},"llms":{"status":"crawl_delay","error":"Crawl-delay 600 s"},"home":{"status":"crawl_delay","error":"Crawl-delay 600 s"}},
{"rank":9116,"domain":"renaultgroup.com","origin":"https://renaultgroup.com","robots":{"status":"ok","http":200,"url":"https://www.renaultgroup.com/robots.txt","bytes":3143,"groups":16},"bots":{"GPTBot":["partial","named",7,1],"ChatGPT-User":["partial","named",7,1],"OAI-SearchBot":["partial","named",7,1],"ClaudeBot":["partial","named",7,1],"Claude-User":["partial","*",12,1],"Claude-SearchBot":["partial","*",12,1],"anthropic-ai":["partial","named",7,1],"PerplexityBot":["partial","named",7,1],"Perplexity-User":["partial","*",12,1],"Google-Extended":["partial","named",7,1],"Applebot-Extended":["partial","named",7,1],"CCBot":["partial","named",7,1],"Bytespider":["partial","named",7,1],"Amazonbot":["partial","named",7,1],"meta-externalagent":["partial","*",12,1],"cohere-ai":["partial","*",12,1],"Diffbot":["partial","*",12,1],"ContextFetchBot":["partial","*",12,1]},"llms":{"status":"missing","http":404,"bytes":229454},"home":{"status":"ok","http":200,"url":"https://www.renaultgroup.com/","html_tokens":87051,"markdown_tokens":15,"pruned_tokens":15}},
{"rank":9117,"domain":"access-board.gov","origin":"https://www.access-board.gov","robots":{"status":"ok","http":200,"url":"https://www.access-board.gov/robots.txt","bytes":64,"groups":1},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"missing","http":404,"bytes":16913},"home":{"status":"ok","http":200,"url":"https://www.access-board.gov/","html_tokens":9334,"markdown_tokens":155,"pruned_tokens":137}},
{"rank":9118,"domain":"giige.online","origin":"https://giige.online","robots":{"status":"unreachable","http":null,"url":"https://giige.online/robots.txt","bytes":0,"groups":0,"error":"cannot resolve giige.online"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":9119,"domain":"qzgs.net","origin":"https://qzgs.net","robots":{"status":"unreachable","http":null,"url":"https://qzgs.net/robots.txt","bytes":0,"groups":0,"error":"cannot resolve qzgs.net"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":9120,"domain":"cyberscoop.com","origin":"https://cyberscoop.com","robots":{"status":"ok","http":200,"url":"https://cyberscoop.com/robots.txt","bytes":683,"groups":2},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"missing","http":404,"bytes":58208},"home":{"status":"ok","http":200,"url":"https://cyberscoop.com/","html_tokens":35797,"markdown_tokens":167,"pruned_tokens":167}},
{"rank":9121,"domain":"publicintegrity.org","origin":"https://publicintegrity.org","robots":{"status":"ok","http":200,"url":"https://publicintegrity.org/robots.txt","bytes":225,"groups":1},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"missing","http":404,"bytes":141267},"home":{"status":"ok","http":200,"url":"https://publicintegrity.org/","html_tokens":55205,"markdown_tokens":100,"pruned_tokens":100}},
{"rank":9122,"domain":"riaa.com","origin":"https://riaa.com","robots":{"status":"ok","http":200,"url":"https://www.riaa.com/robots.txt","bytes":53,"groups":1},"bots":{"GPTBot":["partial","*",1,0],"ChatGPT-User":["partial","*",1,0],"OAI-SearchBot":["partial","*",1,0],"ClaudeBot":["partial","*",1,0],"Claude-User":["partial","*",1,0],"Claude-SearchBot":["partial","*",1,0],"anthropic-ai":["partial","*",1,0],"PerplexityBot":["partial","*",1,0],"Perplexity-User":["partial","*",1,0],"Google-Extended":["partial","*",1,0],"Applebot-Extended":["partial","*",1,0],"CCBot":["partial","*",1,0],"Bytespider":["partial","*",1,0],"Amazonbot":["partial","*",1,0],"meta-externalagent":["partial","*",1,0],"cohere-ai":["partial","*",1,0],"Diffbot":["partial","*",1,0],"ContextFetchBot":["partial","*",1,0]},"llms":{"status":"http_error","http":403,"bytes":5539},"home":{"status":"http_error","http":403,"url":"https://riaa.com/"}},
{"rank":9123,"domain":"asylum94.com","origin":"https://asylum94.com","robots":{"status":"ok","http":200,"url":"https://asylum94.com/robots.txt","bytes":66,"groups":1},"bots":{"GPTBot":["allowed","*",0,1],"ChatGPT-User":["allowed","*",0,1],"OAI-SearchBot":["allowed","*",0,1],"ClaudeBot":["allowed","*",0,1],"Claude-User":["allowed","*",0,1],"Claude-SearchBot":["allowed","*",0,1],"anthropic-ai":["allowed","*",0,1],"PerplexityBot":["allowed","*",0,1],"Perplexity-User":["allowed","*",0,1],"Google-Extended":["allowed","*",0,1],"Applebot-Extended":["allowed","*",0,1],"CCBot":["allowed","*",0,1],"Bytespider":["allowed","*",0,1],"Amazonbot":["allowed","*",0,1],"meta-externalagent":["allowed","*",0,1],"cohere-ai":["allowed","*",0,1],"Diffbot":["allowed","*",0,1],"ContextFetchBot":["allowed","*",0,1]},"llms":{"status":"missing","http":404,"bytes":641},"home":{"status":"ok","http":200,"url":"https://asylum94.com/","html_tokens":44507,"markdown_tokens":26635,"pruned_tokens":146}},
{"rank":9124,"domain":"net318.cn","origin":"https://net318.cn","robots":{"status":"unreachable","http":null,"url":"https://net318.cn/robots.txt","bytes":0,"groups":0,"error":"cannot resolve net318.cn"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":9125,"domain":"freepressjournal.in","origin":"https://freepressjournal.in","robots":{"status":"ok","http":200,"url":"https://www.freepressjournal.in/robots.txt","bytes":1102,"groups":1},"bots":{"GPTBot":["partial","*",38,1],"ChatGPT-User":["partial","*",38,1],"OAI-SearchBot":["partial","*",38,1],"ClaudeBot":["partial","*",38,1],"Claude-User":["partial","*",38,1],"Claude-SearchBot":["partial","*",38,1],"anthropic-ai":["partial","*",38,1],"PerplexityBot":["partial","*",38,1],"Perplexity-User":["partial","*",38,1],"Google-Extended":["partial","*",38,1],"Applebot-Extended":["partial","*",38,1],"CCBot":["partial","*",38,1],"Bytespider":["partial","*",38,1],"Amazonbot":["partial","*",38,1],"meta-externalagent":["partial","*",38,1],"cohere-ai":["partial","*",38,1],"Diffbot":["partial","*",38,1],"ContextFetchBot":["partial","*",38,1]},"llms":{"status":"html","http":200,"bytes":0},"home":{"status":"ok","http":200,"url":"https://www.freepressjournal.in/","html_tokens":46780,"markdown_tokens":29,"pruned_tokens":29}},
{"rank":9126,"domain":"digital-photography-school.com","origin":"https://digital-photography-school.com","robots":{"status":"ok","http":200,"url":"https://digital-photography-school.com/robots.txt","bytes":162,"groups":1},"bots":{"GPTBot":["partial","*",7,0],"ChatGPT-User":["partial","*",7,0],"OAI-SearchBot":["partial","*",7,0],"ClaudeBot":["partial","*",7,0],"Claude-User":["partial","*",7,0],"Claude-SearchBot":["partial","*",7,0],"anthropic-ai":["partial","*",7,0],"PerplexityBot":["partial","*",7,0],"Perplexity-User":["partial","*",7,0],"Google-Extended":["partial","*",7,0],"Applebot-Extended":["partial","*",7,0],"CCBot":["partial","*",7,0],"Bytespider":["partial","*",7,0],"Amazonbot":["partial","*",7,0],"meta-externalagent":["partial","*",7,0],"cohere-ai":["partial","*",7,0],"Diffbot":["partial","*",7,0],"ContextFetchBot":["partial","*",7,0]},"llms":{"status":"html","http":200,"bytes":130784},"home":{"status":"ok","http":200,"url":"https://digital-photography-school.com/","html_tokens":32482,"markdown_tokens":760,"pruned_tokens":7}},
{"rank":9127,"domain":"thewindowsclub.com","origin":"https://thewindowsclub.com","robots":{"status":"ok","http":200,"url":"https://thewindowsclub.com/robots.txt","bytes":447,"groups":1},"bots":{"GPTBot":["partial","*",15,0],"ChatGPT-User":["partial","*",15,0],"OAI-SearchBot":["partial","*",15,0],"ClaudeBot":["partial","*",15,0],"Claude-User":["partial","*",15,0],"Claude-SearchBot":["partial","*",15,0],"anthropic-ai":["partial","*",15,0],"PerplexityBot":["partial","*",15,0],"Perplexity-User":["partial","*",15,0],"Google-Extended":["partial","*",15,0],"Applebot-Extended":["partial","*",15,0],"CCBot":["partial","*",15,0],"Bytespider":["partial","*",15,0],"Amazonbot":["partial","*",15,0],"meta-externalagent":["partial","*",15,0],"cohere-ai":["partial","*",15,0],"Diffbot":["partial","*",15,0],"ContextFetchBot":["partial","*",15,0]},"llms":{"status":"other_text","http":200,"bytes":2431},"home":{"status":"http_error","http":403,"url":"https://thewindowsclub.com/"}},
{"rank":9128,"domain":"hhu.de","origin":"https://hhu.de","robots":{"status":"ok","http":200,"url":"https://www.hhu.de/robots.txt","bytes":905,"groups":4},"bots":{"GPTBot":["partial","*",18,7],"ChatGPT-User":["partial","*",18,7],"OAI-SearchBot":["partial","*",18,7],"ClaudeBot":["partial","*",18,7],"Claude-User":["partial","*",18,7],"Claude-SearchBot":["partial","*",18,7],"anthropic-ai":["partial","*",18,7],"PerplexityBot":["partial","*",18,7],"Perplexity-User":["partial","*",18,7],"Google-Extended":["partial","*",18,7],"Applebot-Extended":["partial","*",18,7],"CCBot":["partial","*",18,7],"Bytespider":["blocked","named",1,0],"Amazonbot":["partial","*",18,7],"meta-externalagent":["partial","*",18,7],"cohere-ai":["partial","*",18,7],"Diffbot":["partial","*",18,7],"ContextFetchBot":["partial","*",18,7]},"llms":{"status":"missing","http":404,"bytes":4465},"home":{"status":"ok","http":200,"url":"https://www.hhu.de/","html_tokens":135908,"markdown_tokens":17,"pruned_tokens":17}},
{"rank":9129,"domain":"zahranicneonlinecasinobonusbezvkladu.com","origin":"https://zahranicneonlinecasinobonusbezvkladu.com","robots":{"status":"ok","http":200,"url":"https://zahranicneonlinecasinobonusbezvkladu.com/robots.txt","bytes":2770,"groups":33},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["allowed","*",0,2],"OAI-SearchBot":["allowed","*",0,2],"ClaudeBot":["blocked","named",1,0],"Claude-User":["allowed","*",0,2],"Claude-SearchBot":["allowed","*",0,2],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["allowed","*",0,2],"Perplexity-User":["allowed","*",0,2],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["allowed","*",0,2]},"llms":{"status":"missing","http":404,"bytes":669},"home":{"status":"ok","http":200,"url":"https://zahranicneonlinecasinobonusbezvkladu.com/","html_tokens":52760,"markdown_tokens":28071,"pruned_tokens":252}},
{"rank":9130,"domain":"gtk.org","origin":"https://gtk.org","robots":{"status":"ok","http":200,"url":"https://gtk.org/robots.txt","bytes":41,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":6885},"home":{"status":"ok","http":200,"url":"https://gtk.org/","html_tokens":11552,"markdown_tokens":354,"pruned_tokens":342}},
{"rank":9131,"domain":"jsyixi.com","origin":"https://jsyixi.com","robots":{"status":"missing","http":404,"url":"https://jsyixi.com/robots.txt","bytes":0,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":0},"home":{"status":"http_error","http":404,"url":"https://jsyixi.com/"}},
{"rank":9132,"domain":"yppcn.com","origin":"http://yppcn.com","robots":{"status":"missing","http":404,"url":"http://yppcn.com/robots.txt","bytes":1163,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":1163},"home":{"status":"ok","http":200,"url":"http://yppcn.com/","html_tokens":203,"markdown_tokens":30,"pruned_tokens":30}},
{"rank":9133,"domain":"mapion.co.jp","origin":"https://mapion.co.jp","robots":{"status":"ok","http":200,"url":"https://www.mapion.co.jp/robots.txt","bytes":1068,"groups":2},"bots":{"GPTBot":["partial","*",22,10],"ChatGPT-User":["partial","*",22,10],"OAI-SearchBot":["partial","*",22,10],"ClaudeBot":["partial","*",22,10],"Claude-User":["partial","*",22,10],"Claude-SearchBot":["partial","*",22,10],"anthropic-ai":["partial","*",22,10],"PerplexityBot":["partial","*",22,10],"Perplexity-User":["partial","*",22,10],"Google-Extended":["partial","*",22,10],"Applebot-Extended":["partial","*",22,10],"CCBot":["partial","*",22,10],"Bytespider":["partial","*",22,10],"Amazonbot":["partial","*",22,10],"meta-externalagent":["partial","*",22,10],"cohere-ai":["partial","*",22,10],"Diffbot":["partial","*",22,10],"ContextFetchBot":["partial","*",22,10]},"llms":{"status":"missing","http":404,"bytes":6321},"home":{"status":"ok","http":200,"url":"https://www.mapion.co.jp/","html_tokens":12090,"markdown_tokens":33,"pruned_tokens":33}},
{"rank":9134,"domain":"breastcancer.org","origin":"https://breastcancer.org","robots":{"status":"ok","http":200,"url":"https://www.breastcancer.org/robots.txt","bytes":724,"groups":1},"bots":{"GPTBot":["partial","*",18,1],"ChatGPT-User":["partial","*",18,1],"OAI-SearchBot":["partial","*",18,1],"ClaudeBot":["partial","*",18,1],"Claude-User":["partial","*",18,1],"Claude-SearchBot":["partial","*",18,1],"anthropic-ai":["partial","*",18,1],"PerplexityBot":["partial","*",18,1],"Perplexity-User":["partial","*",18,1],"Google-Extended":["partial","*",18,1],"Applebot-Extended":["partial","*",18,1],"CCBot":["partial","*",18,1],"Bytespider":["partial","*",18,1],"Amazonbot":["partial","*",18,1],"meta-externalagent":["partial","*",18,1],"cohere-ai":["partial","*",18,1],"Diffbot":["partial","*",18,1],"ContextFetchBot":["partial","*",18,1]},"llms":{"status":"missing","http":404,"bytes":8732},"home":{"status":"ok","http":200,"url":"https://www.breastcancer.org/","html_tokens":18071,"markdown_tokens":23,"pruned_tokens":23}},
{"rank":9135,"domain":"chemistryworld.com","origin":"https://chemistryworld.com","robots":{"status":"ok","http":200,"url":"https://www.chemistryworld.com/robots.txt","bytes":802,"groups":1},"bots":{"GPTBot":["partial","*",30,0],"ChatGPT-User":["partial","*",30,0],"OAI-SearchBot":["partial","*",30,0],"ClaudeBot":["partial","*",30,0],"Claude-User":["partial","*",30,0],"Claude-SearchBot":["partial","*",30,0],"anthropic-ai":["partial","*",30,0],"PerplexityBot":["partial","*",30,0],"Perplexity-User":["partial","*",30,0],"Google-Extended":["partial","*",30,0],"Applebot-Extended":["partial","*",30,0],"CCBot":["partial","*",30,0],"Bytespider":["partial","*",30,0],"Amazonbot":["partial","*",30,0],"meta-externalagent":["partial","*",30,0],"cohere-ai":["partial","*",30,0],"Diffbot":["partial","*",30,0],"ContextFetchBot":["partial","*",30,0]},"llms":{"status":"missing","http":404,"bytes":79918},"home":{"status":"ok","http":200,"url":"https://www.chemistryworld.com/","html_tokens":45362,"markdown_tokens":533,"pruned_tokens":84}},
{"rank":9136,"domain":"xn--pssw46az6cqs3a.com","origin":"http://xn--pssw46az6cqs3a.com","robots":{"status":"missing","http":404,"url":"http://xn--pssw46az6cqs3a.com/robots.txt","bytes":4735,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":4731},"home":{"status":"ok","http":200,"url":"http://xn--pssw46az6cqs3a.com/","html_tokens":6005,"markdown_tokens":138,"pruned_tokens":138}},
{"rank":9137,"domain":"qidian.com","origin":"https://qidian.com","robots":{"status":"html","http":202,"url":"https://www.qidian.com","bytes":209,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":9138,"domain":"messe-stuttgart.de","origin":"https://messe-stuttgart.de","robots":{"status":"ok","http":200,"url":"https://www.messe-stuttgart.de/robots.txt","bytes":155,"groups":1},"bots":{"GPTBot":["partial","*",4,0],"ChatGPT-User":["partial","*",4,0],"OAI-SearchBot":["partial","*",4,0],"ClaudeBot":["partial","*",4,0],"Claude-User":["partial","*",4,0],"Claude-SearchBot":["partial","*",4,0],"anthropic-ai":["partial","*",4,0],"PerplexityBot":["partial","*",4,0],"Perplexity-User":["partial","*",4,0],"Google-Extended":["partial","*",4,0],"Applebot-Extended":["partial","*",4,0],"CCBot":["partial","*",4,0],"Bytespider":["partial","*",4,0],"Amazonbot":["partial","*",4,0],"meta-externalagent":["partial","*",4,0],"cohere-ai":["partial","*",4,0],"Diffbot":["partial","*",4,0],"ContextFetchBot":["partial","*",4,0]},"llms":{"status":"missing","http":404,"bytes":101099},"home":{"status":"ok","http":200,"url":"https://www.messe-stuttgart.de/","html_tokens":35158,"markdown_tokens":195,"pruned_tokens":195}},
{"rank":9139,"domain":"reactos.org","origin":"https://reactos.org","robots":{"status":"ok","http":200,"url":"https://reactos.org/robots.txt","bytes":0,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":12280},"home":{"status":"ok","http":200,"url":"https://reactos.org/","html_tokens":4827,"markdown_tokens":463,"pruned_tokens":220}},
{"rank":9140,"domain":"kddziot.online","origin":"https://kddziot.online","robots":{"status":"unreachable","http":null,"url":"https://kddziot.online/robots.txt","bytes":0,"groups":0,"error":"cannot resolve kddziot.online"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":9141,"domain":"vedos.cz","origin":"https://vedos.cz","robots":{"status":"ok","http":200,"url":"https://vedos.cz/robots.txt","bytes":115,"groups":1},"bots":{"GPTBot":["partial","*",1,1],"ChatGPT-User":["partial","*",1,1],"OAI-SearchBot":["partial","*",1,1],"ClaudeBot":["partial","*",1,1],"Claude-User":["partial","*",1,1],"Claude-SearchBot":["partial","*",1,1],"anthropic-ai":["partial","*",1,1],"PerplexityBot":["partial","*",1,1],"Perplexity-User":["partial","*",1,1],"Google-Extended":["partial","*",1,1],"Applebot-Extended":["partial","*",1,1],"CCBot":["partial","*",1,1],"Bytespider":["partial","*",1,1],"Amazonbot":["partial","*",1,1],"meta-externalagent":["partial","*",1,1],"cohere-ai":["partial","*",1,1],"Diffbot":["partial","*",1,1],"ContextFetchBot":["partial","*",1,1]},"llms":{"status":"other_text","http":200,"bytes":38621},"home":{"status":"ok","http":200,"url":"https://vedos.cz/","html_tokens":60683,"markdown_tokens":353,"pruned_tokens":140}},
{"rank":9142,"domain":"xenforo.com","origin":"https://xenforo.com","robots":{"status":"refused","http":403,"url":"https://xenforo.com/robots.txt","bytes":5527,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":9143,"domain":"hi.is","origin":"https://hi.is","robots":{"status":"ok","http":200,"url":"https://hi.is/robots.txt","bytes":2441,"groups":12},"bots":{"GPTBot":["partial","*",34,18],"ChatGPT-User":["partial","*",34,18],"OAI-SearchBot":["partial","*",34,18],"ClaudeBot":["partial","*",34,18],"Claude-User":["partial","*",34,18],"Claude-SearchBot":["partial","*",34,18],"anthropic-ai":["partial","*",34,18],"PerplexityBot":["partial","*",34,18],"Perplexity-User":["partial","*",34,18],"Google-Extended":["partial","*",34,18],"Applebot-Extended":["partial","*",34,18],"CCBot":["partial","*",34,18],"Bytespider":["partial","*",34,18],"Amazonbot":["partial","*",34,18],"meta-externalagent":["partial","*",34,18],"cohere-ai":["partial","*",34,18],"Diffbot":["partial","*",34,18],"ContextFetchBot":["partial","*",34,18]},"llms":{"status":"missing","http":404,"bytes":179},"home":{"status":"ok","http":200,"url":"https://hi.is/","html_tokens":79871,"markdown_tokens":35,"pruned_tokens":35}},
{"rank":9144,"domain":"bricklink.com","origin":"https://bricklink.com","robots":{"status":"ok","http":200,"url":"https://www.bricklink.com/robots.txt","bytes":713,"groups":4},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["partial","*",12,1],"OAI-SearchBot":["partial","*",12,1],"ClaudeBot":["partial","*",12,1],"Claude-User":["partial","*",12,1],"Claude-SearchBot":["partial","*",12,1],"anthropic-ai":["partial","*",12,1],"PerplexityBot":["partial","*",12,1],"Perplexity-User":["partial","*",12,1],"Google-Extended":["partial","*",12,1],"Applebot-Extended":["partial","*",12,1],"CCBot":["partial","*",12,1],"Bytespider":["partial","*",12,1],"Amazonbot":["partial","*",12,1],"meta-externalagent":["partial","*",12,1],"cohere-ai":["partial","*",12,1],"Diffbot":["partial","*",12,1],"ContextFetchBot":["partial","*",12,1]},"llms":{"status":"missing","http":404,"bytes":12694},"home":{"status":"ok","http":200,"url":"https://www.bricklink.com/v2/main.page","html_tokens":17030,"markdown_tokens":89,"pruned_tokens":89}},
{"rank":9145,"domain":"elcorreo.com","origin":"https://elcorreo.com","robots":{"status":"ok","http":200,"url":"https://www.elcorreo.com/robots.txt","bytes":2666,"groups":6},"bots":{"GPTBot":["partial","named",1,3],"ChatGPT-User":["partial","named",1,3],"OAI-SearchBot":["partial","named",1,3],"ClaudeBot":["partial","named",1,3],"Claude-User":["partial","named",1,3],"Claude-SearchBot":["partial","named",1,3],"anthropic-ai":["partial","*",46,3],"PerplexityBot":["partial","named",1,3],"Perplexity-User":["partial","named",1,3],"Google-Extended":["partial","*",46,3],"Applebot-Extended":["partial","named",1,3],"CCBot":["partial","named",1,3],"Bytespider":["partial","named",1,3],"Amazonbot":["partial","named",1,3],"meta-externalagent":["partial","named",1,3],"cohere-ai":["partial","named",1,3],"Diffbot":["partial","*",46,3],"ContextFetchBot":["partial","*",46,3]},"llms":{"status":"missing","http":404,"bytes":238797},"home":{"status":"ok","http":200,"url":"https://www.elcorreo.com/","html_tokens":140903,"markdown_tokens":1296,"pruned_tokens":128}},
{"rank":9146,"domain":"publuu.com","origin":"https://publuu.com","robots":{"status":"ok","http":200,"url":"https://publuu.com/robots.txt","bytes":348,"groups":3},"bots":{"GPTBot":["partial","*",8,1],"ChatGPT-User":["partial","*",8,1],"OAI-SearchBot":["partial","*",8,1],"ClaudeBot":["partial","*",8,1],"Claude-User":["partial","*",8,1],"Claude-SearchBot":["partial","*",8,1],"anthropic-ai":["partial","*",8,1],"PerplexityBot":["partial","*",8,1],"Perplexity-User":["partial","*",8,1],"Google-Extended":["partial","*",8,1],"Applebot-Extended":["partial","*",8,1],"CCBot":["partial","*",8,1],"Bytespider":["partial","*",8,1],"Amazonbot":["partial","*",8,1],"meta-externalagent":["partial","*",8,1],"cohere-ai":["partial","*",8,1],"Diffbot":["partial","*",8,1],"ContextFetchBot":["partial","*",8,1]},"llms":{"status":"missing","http":404,"bytes":90504},"home":{"status":"ok","http":200,"url":"https://publuu.com/","html_tokens":37228,"markdown_tokens":1218,"pruned_tokens":618}},
{"rank":9147,"domain":"focus.cn","origin":"https://focus.cn","robots":{"status":"html","http":200,"url":"https://house.focus.cn/","bytes":277145,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":9148,"domain":"surfline.com","origin":"https://surfline.com","robots":{"status":"ok","http":200,"url":"https://www.surfline.com/robots.txt","bytes":4244,"groups":47},"bots":{"GPTBot":["partial","*",114,0],"ChatGPT-User":["partial","*",114,0],"OAI-SearchBot":["partial","*",114,0],"ClaudeBot":["partial","*",114,0],"Claude-User":["partial","*",114,0],"Claude-SearchBot":["partial","*",114,0],"anthropic-ai":["partial","*",114,0],"PerplexityBot":["partial","*",114,0],"Perplexity-User":["partial","*",114,0],"Google-Extended":["partial","*",114,0],"Applebot-Extended":["partial","*",114,0],"CCBot":["blocked","named",1,0],"Bytespider":["partial","*",114,0],"Amazonbot":["partial","*",114,0],"meta-externalagent":["partial","*",114,0],"cohere-ai":["partial","*",114,0],"Diffbot":["partial","*",114,0],"ContextFetchBot":["partial","*",114,0]},"llms":{"status":"crawl_delay","error":"Crawl-delay 500 s"},"home":{"status":"crawl_delay","error":"Crawl-delay 500 s"}},
{"rank":9149,"domain":"cpubenchmark.net","origin":"https://cpubenchmark.net","robots":{"status":"ok","http":200,"url":"https://cpubenchmark.net/robots.txt","bytes":2676,"groups":32},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["allowed","*",0,1],"OAI-SearchBot":["allowed","*",0,1],"ClaudeBot":["blocked","named",1,0],"Claude-User":["allowed","*",0,1],"Claude-SearchBot":["allowed","*",0,1],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["allowed","*",0,1],"Perplexity-User":["allowed","*",0,1],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["allowed","*",0,1]},"llms":{"status":"missing","http":404,"bytes":42860},"home":{"status":"ok","http":200,"url":"https://www.cpubenchmark.net/","html_tokens":21237,"markdown_tokens":550,"pruned_tokens":150}},
{"rank":9150,"domain":"sp1-brevo.net","origin":"https://sp1-brevo.net","robots":{"status":"server_error","http":526,"url":"https://sp1-brevo.net/robots.txt","bytes":7701,"groups":0,"error":"HTTP 526"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":9151,"domain":"rapidshare.de","origin":"https://rapidshare.de","robots":{"status":"unreachable","http":null,"url":"https://rapidshare.de/robots.txt","bytes":0,"groups":0,"error":"rapidshare.de is not a public address"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":9152,"domain":"xn--ykts2c47u5r4a.com","origin":"http://xn--ykts2c47u5r4a.com","robots":{"status":"missing","http":404,"url":"http://xn--ykts2c47u5r4a.com/robots.txt","bytes":4847,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":4843},"home":{"status":"ok","http":200,"url":"http://xn--ykts2c47u5r4a.com/","html_tokens":20471,"markdown_tokens":253,"pruned_tokens":39}},
{"rank":9153,"domain":"alisolution.cn","origin":"https://alisolution.cn","robots":{"status":"unreachable","http":null,"url":"https://alisolution.cn/robots.txt","bytes":0,"groups":0,"error":"cannot resolve alisolution.cn"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":9154,"domain":"journaldemontreal.com","origin":"https://journaldemontreal.com","robots":{"status":"ok","http":200,"url":"https://www.journaldemontreal.com/robots.txt","bytes":6529,"groups":153},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["blocked","named",1,0],"OAI-SearchBot":["partial","*",1,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",1,0],"Claude-SearchBot":["blocked","named",1,0],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["blocked","named",1,0],"Google-Extended":["partial","*",1,0],"Applebot-Extended":["partial","*",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["partial","*",1,0],"Amazonbot":["partial","*",1,0],"meta-externalagent":["partial","*",1,0],"cohere-ai":["partial","*",1,0],"Diffbot":["partial","*",1,0],"ContextFetchBot":["partial","*",1,0]},"llms":{"status":"missing","http":404,"bytes":285},"home":{"status":"ok","http":200,"url":"https://www.journaldemontreal.com/","html_tokens":70790,"markdown_tokens":23,"pruned_tokens":23}},
{"rank":9155,"domain":"focustaiwan.tw","origin":"https://focustaiwan.tw","robots":{"status":"ok","http":200,"url":"https://focustaiwan.tw/robots.txt","bytes":553,"groups":1},"bots":{"GPTBot":["partial","*",16,0],"ChatGPT-User":["partial","*",16,0],"OAI-SearchBot":["partial","*",16,0],"ClaudeBot":["partial","*",16,0],"Claude-User":["partial","*",16,0],"Claude-SearchBot":["partial","*",16,0],"anthropic-ai":["partial","*",16,0],"PerplexityBot":["partial","*",16,0],"Perplexity-User":["partial","*",16,0],"Google-Extended":["partial","*",16,0],"Applebot-Extended":["partial","*",16,0],"CCBot":["partial","*",16,0],"Bytespider":["partial","*",16,0],"Amazonbot":["partial","*",16,0],"meta-externalagent":["partial","*",16,0],"cohere-ai":["partial","*",16,0],"Diffbot":["partial","*",16,0],"ContextFetchBot":["partial","*",16,0]},"llms":{"status":"missing","http":404,"bytes":117121},"home":{"status":"ok","http":200,"url":"https://focustaiwan.tw/","html_tokens":25471,"markdown_tokens":64,"pruned_tokens":64}},
{"rank":9156,"domain":"competethemes.com","origin":"https://competethemes.com","robots":{"status":"ok","http":200,"url":"https://competethemes.com/robots.txt","bytes":164,"groups":1},"bots":{"GPTBot":["partial","*",2,1],"ChatGPT-User":["partial","*",2,1],"OAI-SearchBot":["partial","*",2,1],"ClaudeBot":["partial","*",2,1],"Claude-User":["partial","*",2,1],"Claude-SearchBot":["partial","*",2,1],"anthropic-ai":["partial","*",2,1],"PerplexityBot":["partial","*",2,1],"Perplexity-User":["partial","*",2,1],"Google-Extended":["partial","*",2,1],"Applebot-Extended":["partial","*",2,1],"CCBot":["partial","*",2,1],"Bytespider":["partial","*",2,1],"Amazonbot":["partial","*",2,1],"meta-externalagent":["partial","*",2,1],"cohere-ai":["partial","*",2,1],"Diffbot":["partial","*",2,1],"ContextFetchBot":["partial","*",2,1]},"llms":{"status":"html","http":200,"bytes":64735},"home":{"status":"ok","http":200,"url":"https://www.competethemes.com/","html_tokens":16181,"markdown_tokens":85,"pruned_tokens":85}},
{"rank":9157,"domain":"kennesaw.edu","origin":"https://kennesaw.edu","robots":{"status":"ok","http":200,"url":"https://www.kennesaw.edu/robots.txt","bytes":62,"groups":1},"bots":{"GPTBot":["partial","*",2,0],"ChatGPT-User":["partial","*",2,0],"OAI-SearchBot":["partial","*",2,0],"ClaudeBot":["partial","*",2,0],"Claude-User":["partial","*",2,0],"Claude-SearchBot":["partial","*",2,0],"anthropic-ai":["partial","*",2,0],"PerplexityBot":["partial","*",2,0],"Perplexity-User":["partial","*",2,0],"Google-Extended":["partial","*",2,0],"Applebot-Extended":["partial","*",2,0],"CCBot":["partial","*",2,0],"Bytespider":["partial","*",2,0],"Amazonbot":["partial","*",2,0],"meta-externalagent":["partial","*",2,0],"cohere-ai":["partial","*",2,0],"Diffbot":["partial","*",2,0],"ContextFetchBot":["partial","*",2,0]},"llms":{"status":"missing","http":404,"bytes":33757},"home":{"status":"ok","http":200,"url":"https://www.kennesaw.edu/","html_tokens":17953,"markdown_tokens":268,"pruned_tokens":155}},
{"rank":9158,"domain":"chocolatey.org","origin":"https://chocolatey.org","robots":{"status":"missing","http":404,"url":"https://chocolatey.org/robots.txt","bytes":52750,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":52750},"home":{"status":"ok","http":200,"url":"https://chocolatey.org/","html_tokens":28668,"markdown_tokens":743,"pruned_tokens":278}},
{"rank":9159,"domain":"bankcomm.com","origin":"https://bankcomm.com","robots":{"status":"unreachable","http":null,"url":"https://bankcomm.com/robots.txt","bytes":0,"groups":0,"error":"[SSL: UNSAFE_LEGACY_RENEGOTIATION_DISABLED] unsafe legacy renegotiation disabled (_ssl.c:1028)"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":9160,"domain":"urocentrum-kosice.sk","origin":"https://urocentrum-kosice.sk","robots":{"status":"ok","http":200,"url":"https://urocentrum-kosice.sk/robots.txt","bytes":74,"groups":1},"bots":{"GPTBot":["allowed","*",0,1],"ChatGPT-User":["allowed","*",0,1],"OAI-SearchBot":["allowed","*",0,1],"ClaudeBot":["allowed","*",0,1],"Claude-User":["allowed","*",0,1],"Claude-SearchBot":["allowed","*",0,1],"anthropic-ai":["allowed","*",0,1],"PerplexityBot":["allowed","*",0,1],"Perplexity-User":["allowed","*",0,1],"Google-Extended":["allowed","*",0,1],"Applebot-Extended":["allowed","*",0,1],"CCBot":["allowed","*",0,1],"Bytespider":["allowed","*",0,1],"Amazonbot":["allowed","*",0,1],"meta-externalagent":["allowed","*",0,1],"cohere-ai":["allowed","*",0,1],"Diffbot":["allowed","*",0,1],"ContextFetchBot":["allowed","*",0,1]},"llms":{"status":"missing","http":404,"bytes":649},"home":{"status":"ok","http":200,"url":"https://urocentrum-kosice.sk/","html_tokens":38043,"markdown_tokens":24836,"pruned_tokens":52}},
{"rank":9161,"domain":"mas.gov.sg","origin":"https://www.mas.gov.sg","robots":{"status":"ok","http":200,"url":"https://www.mas.gov.sg/robots.txt","bytes":74,"groups":1},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"html","http":200,"bytes":853817},"home":{"status":"ok","http":200,"url":"https://www.mas.gov.sg/","html_tokens":213454,"markdown_tokens":76,"pruned_tokens":76}},
{"rank":9162,"domain":"kansas.com","origin":"https://kansas.com","robots":{"status":"unreachable","http":null,"url":"https://kansas.com/robots.txt","bytes":0,"groups":0,"error":"<StreamReset stream_id:1, error_code:2, remote_reset:True>"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":9163,"domain":"dolcegabbana.com","origin":"https://dolcegabbana.com","robots":{"status":"ok","http":200,"url":"https://www.dolcegabbana.com/robots.txt","bytes":30805,"groups":2},"bots":{"GPTBot":["partial","*",932,7],"ChatGPT-User":["partial","*",932,7],"OAI-SearchBot":["partial","*",932,7],"ClaudeBot":["partial","*",932,7],"Claude-User":["partial","*",932,7],"Claude-SearchBot":["partial","*",932,7],"anthropic-ai":["partial","*",932,7],"PerplexityBot":["partial","*",932,7],"Perplexity-User":["partial","*",932,7],"Google-Extended":["partial","*",932,7],"Applebot-Extended":["partial","*",932,7],"CCBot":["partial","*",932,7],"Bytespider":["partial","*",932,7],"Amazonbot":["partial","*",932,7],"meta-externalagent":["partial","*",932,7],"cohere-ai":["partial","*",932,7],"Diffbot":["partial","*",932,7],"ContextFetchBot":["partial","*",932,7]},"llms":{"status":"missing","http":404,"bytes":2987623},"home":{"status":"ok","http":200,"url":"https://www.dolcegabbana.com/en-us/","html_tokens":820475,"markdown_tokens":50,"pruned_tokens":50}},
{"rank":9164,"domain":"mrbet777.org","origin":"https://mrbet777.org","robots":{"status":"refused","http":401,"url":"https://mrbet777.org/robots.txt","bytes":539,"groups":0,"error":"HTTP 401"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":9165,"domain":"chinaxxscp.org","origin":"https://chinaxxscp.org","robots":{"status":"timeout","http":null,"url":"https://chinaxxscp.org/robots.txt","bytes":0,"groups":0,"error":"ConnectTimeout"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":9166,"domain":"cecece.com.cn","origin":"http://cecece.com.cn","robots":{"status":"missing","http":404,"url":"http://cecece.com.cn/robots.txt","bytes":4715,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":4711},"home":{"status":"disallowed","error":"robots.txt of stopnote.vhostgo.com not readable (unreachable)"}},
{"rank":9167,"domain":"makeagif.com","origin":"https://makeagif.com","robots":{"status":"ok","http":200,"url":"https://makeagif.com/robots.txt","bytes":20959,"groups":1},"bots":{"GPTBot":["partial","*",4,1005],"ChatGPT-User":["partial","*",4,1005],"OAI-SearchBot":["partial","*",4,1005],"ClaudeBot":["partial","*",4,1005],"Claude-User":["partial","*",4,1005],"Claude-SearchBot":["partial","*",4,1005],"anthropic-ai":["partial","*",4,1005],"PerplexityBot":["partial","*",4,1005],"Perplexity-User":["partial","*",4,1005],"Google-Extended":["partial","*",4,1005],"Applebot-Extended":["partial","*",4,1005],"CCBot":["partial","*",4,1005],"Bytespider":["partial","*",4,1005],"Amazonbot":["partial","*",4,1005],"meta-externalagent":["partial","*",4,1005],"cohere-ai":["partial","*",4,1005],"Diffbot":["partial","*",4,1005],"ContextFetchBot":["partial","*",4,1005]},"llms":{"status":"missing","http":404,"bytes":88708},"home":{"status":"ok","http":200,"url":"https://makeagif.com/","html_tokens":28715,"markdown_tokens":41,"pruned_tokens":41}},
{"rank":9168,"domain":"ha.com","origin":"https://ha.com","robots":{"status":"ok","http":200,"url":"https://www.ha.com/robots.txt","bytes":1511,"groups":19},"bots":{"GPTBot":["partial","*",13,0],"ChatGPT-User":["partial","*",13,0],"OAI-SearchBot":["partial","*",13,0],"ClaudeBot":["partial","*",13,0],"Claude-User":["partial","*",13,0],"Claude-SearchBot":["partial","*",13,0],"anthropic-ai":["partial","*",13,0],"PerplexityBot":["partial","*",13,0],"Perplexity-User":["partial","*",13,0],"Google-Extended":["partial","*",13,0],"Applebot-Extended":["partial","*",13,0],"CCBot":["partial","*",13,0],"Bytespider":["partial","*",13,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["partial","*",13,0],"cohere-ai":["partial","*",13,0],"Diffbot":["partial","*",13,0],"ContextFetchBot":["partial","*",13,0]},"llms":{"status":"missing","http":404,"bytes":57401},"home":{"status":"http_error","http":403,"url":"https://www.ha.com/"}},
{"rank":9169,"domain":"scotiabank.com","origin":"https://scotiabank.com","robots":{"status":"ok","http":200,"url":"https://www.scotiabank.com/robots.txt","bytes":1126,"groups":1},"bots":{"GPTBot":["partial","*",24,0],"ChatGPT-User":["partial","*",24,0],"OAI-SearchBot":["partial","*",24,0],"ClaudeBot":["partial","*",24,0],"Claude-User":["partial","*",24,0],"Claude-SearchBot":["partial","*",24,0],"anthropic-ai":["partial","*",24,0],"PerplexityBot":["partial","*",24,0],"Perplexity-User":["partial","*",24,0],"Google-Extended":["partial","*",24,0],"Applebot-Extended":["partial","*",24,0],"CCBot":["partial","*",24,0],"Bytespider":["partial","*",24,0],"Amazonbot":["partial","*",24,0],"meta-externalagent":["partial","*",24,0],"cohere-ai":["partial","*",24,0],"Diffbot":["partial","*",24,0],"ContextFetchBot":["partial","*",24,0]},"llms":{"status":"missing","http":404,"bytes":192979},"home":{"status":"ok","http":200,"url":"https://www.scotiabank.com/global/en/global-site.html","html_tokens":13934,"markdown_tokens":97,"pruned_tokens":97}},
{"rank":9170,"domain":"volunteermatch.org","origin":"https://volunteermatch.org","robots":{"status":"missing","http":404,"url":"https://volunteermatch.org/robots.txt","bytes":152,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"html","http":200,"bytes":202830},"home":{"status":"ok","http":200,"url":"https://www.idealist.org/volunteermatch","html_tokens":50693,"markdown_tokens":116,"pruned_tokens":116}},
{"rank":9171,"domain":"ybx-emc.com","origin":"http://ybx-emc.com","robots":{"status":"missing","http":404,"url":"http://ybx-emc.com/robots.txt","bytes":5072,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":5068},"home":{"status":"ok","http":200,"url":"http://www.ybx-emc.com","html_tokens":4979,"markdown_tokens":135,"pruned_tokens":135}},
{"rank":9172,"domain":"eudemon365.net","origin":"https://eudemon365.net","robots":{"status":"timeout","http":null,"url":"https://eudemon365.net/robots.txt","bytes":0,"groups":0,"error":"ConnectTimeout"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":9173,"domain":"federalnewsnetwork.com","origin":"https://federalnewsnetwork.com","robots":{"status":"ok","http":200,"url":"https://federalnewsnetwork.com/robots.txt","bytes":4431,"groups":46},"bots":{"GPTBot":["blocked","named",2,0],"ChatGPT-User":["blocked","named",1,0],"OAI-SearchBot":["partial","*",2,1],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",2,1],"Claude-SearchBot":["partial","*",2,1],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["partial","*",2,1],"Google-Extended":["blocked","named",2,0],"Applebot-Extended":["partial","*",2,1],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["partial","*",2,1],"meta-externalagent":["partial","*",2,1],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",2,1]},"llms":{"status":"missing","http":404,"bytes":3244},"home":{"status":"ok","http":200,"url":"https://federalnewsnetwork.com/","html_tokens":54415,"markdown_tokens":729,"pruned_tokens":143}},
{"rank":9174,"domain":"tb.cn","origin":"https://tb.cn","robots":{"status":"unreachable","http":null,"url":"https://tb.cn/robots.txt","bytes":0,"groups":0,"error":"cannot resolve tb.cn"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":9175,"domain":"rsl.ru","origin":"https://rsl.ru","robots":{"status":"ok","http":200,"url":"https://www.rsl.ru/robots.txt","bytes":515,"groups":1},"bots":{"GPTBot":["partial","*",9,6],"ChatGPT-User":["partial","*",9,6],"OAI-SearchBot":["partial","*",9,6],"ClaudeBot":["partial","*",9,6],"Claude-User":["partial","*",9,6],"Claude-SearchBot":["partial","*",9,6],"anthropic-ai":["partial","*",9,6],"PerplexityBot":["partial","*",9,6],"Perplexity-User":["partial","*",9,6],"Google-Extended":["partial","*",9,6],"Applebot-Extended":["partial","*",9,6],"CCBot":["partial","*",9,6],"Bytespider":["partial","*",9,6],"Amazonbot":["partial","*",9,6],"meta-externalagent":["partial","*",9,6],"cohere-ai":["partial","*",9,6],"Diffbot":["partial","*",9,6],"ContextFetchBot":["partial","*",9,6]},"llms":{"status":"html","http":200,"bytes":37727},"home":{"status":"ok","http":200,"url":"https://www.rsl.ru/","html_tokens":66199,"markdown_tokens":13,"pruned_tokens":13}},
{"rank":9176,"domain":"310s309s.cn","origin":"http://310s309s.cn","robots":{"status":"ok","http":200,"url":"http://310s309s.cn/robots.txt","bytes":485,"groups":1},"bots":{"GPTBot":["partial","*",21,0],"ChatGPT-User":["partial","*",21,0],"OAI-SearchBot":["partial","*",21,0],"ClaudeBot":["partial","*",21,0],"Claude-User":["partial","*",21,0],"Claude-SearchBot":["partial","*",21,0],"anthropic-ai":["partial","*",21,0],"PerplexityBot":["partial","*",21,0],"Perplexity-User":["partial","*",21,0],"Google-Extended":["partial","*",21,0],"Applebot-Extended":["partial","*",21,0],"CCBot":["partial","*",21,0],"Bytespider":["partial","*",21,0],"Amazonbot":["partial","*",21,0],"meta-externalagent":["partial","*",21,0],"cohere-ai":["partial","*",21,0],"Diffbot":["partial","*",21,0],"ContextFetchBot":["partial","*",21,0]},"llms":{"status":"missing","http":404,"bytes":5068},"home":{"status":"ok","http":200,"url":"http://310s309s.cn/","html_tokens":7533,"markdown_tokens":173,"pruned_tokens":173}},
{"rank":9177,"domain":"rescuetime.com","origin":"https://rescuetime.com","robots":{"status":"ok","http":200,"url":"https://www.rescuetime.com/robots.txt","bytes":558,"groups":1},"bots":{"GPTBot":["partial","*",20,2],"ChatGPT-User":["partial","*",20,2],"OAI-SearchBot":["partial","*",20,2],"ClaudeBot":["partial","*",20,2],"Claude-User":["partial","*",20,2],"Claude-SearchBot":["partial","*",20,2],"anthropic-ai":["partial","*",20,2],"PerplexityBot":["partial","*",20,2],"Perplexity-User":["partial","*",20,2],"Google-Extended":["partial","*",20,2],"Applebot-Extended":["partial","*",20,2],"CCBot":["partial","*",20,2],"Bytespider":["partial","*",20,2],"Amazonbot":["partial","*",20,2],"meta-externalagent":["partial","*",20,2],"cohere-ai":["partial","*",20,2],"Diffbot":["partial","*",20,2],"ContextFetchBot":["partial","*",20,2]},"llms":{"status":"missing","http":404,"bytes":163820},"home":{"status":"ok","http":200,"url":"https://www.rescuetime.com/","html_tokens":25236,"markdown_tokens":980,"pruned_tokens":469}},
{"rank":9178,"domain":"oxfordre.com","origin":"https://oxfordre.com","robots":{"status":"ok","http":200,"url":"https://oxfordre.com/robots.txt","bytes":4269,"groups":2},"bots":{"GPTBot":["partial","*",61,3],"ChatGPT-User":["partial","*",61,3],"OAI-SearchBot":["partial","*",61,3],"ClaudeBot":["partial","*",61,3],"Claude-User":["partial","*",61,3],"Claude-SearchBot":["partial","*",61,3],"anthropic-ai":["partial","*",61,3],"PerplexityBot":["partial","*",61,3],"Perplexity-User":["partial","*",61,3],"Google-Extended":["partial","*",61,3],"Applebot-Extended":["partial","*",61,3],"CCBot":["partial","*",61,3],"Bytespider":["partial","*",61,3],"Amazonbot":["partial","*",61,3],"meta-externalagent":["partial","*",61,3],"cohere-ai":["partial","*",61,3],"Diffbot":["partial","*",61,3],"ContextFetchBot":["partial","*",61,3]},"llms":{"status":"http_error","http":403,"bytes":5611},"home":{"status":"http_error","http":403,"url":"https://academic.oup.com/oxford-research-encyclopedias"}},
{"rank":9179,"domain":"upol.cz","origin":"https://upol.cz","robots":{"status":"ok","http":200,"url":"https://upol.cz/robots.txt","bytes":67,"groups":1},"bots":{"GPTBot":["partial","*",3,0],"ChatGPT-User":["partial","*",3,0],"OAI-SearchBot":["partial","*",3,0],"ClaudeBot":["partial","*",3,0],"Claude-User":["partial","*",3,0],"Claude-SearchBot":["partial","*",3,0],"anthropic-ai":["partial","*",3,0],"PerplexityBot":["partial","*",3,0],"Perplexity-User":["partial","*",3,0],"Google-Extended":["partial","*",3,0],"Applebot-Extended":["partial","*",3,0],"CCBot":["partial","*",3,0],"Bytespider":["partial","*",3,0],"Amazonbot":["partial","*",3,0],"meta-externalagent":["partial","*",3,0],"cohere-ai":["partial","*",3,0],"Diffbot":["partial","*",3,0],"ContextFetchBot":["partial","*",3,0]},"llms":{"status":"missing","http":404,"bytes":2793},"home":{"status":"ok","http":200,"url":"https://www.upol.cz/","html_tokens":14430,"markdown_tokens":125,"pruned_tokens":125}},
{"rank":9180,"domain":"ovh.co.uk","origin":"https://ovh.co.uk","robots":{"status":"ok","http":200,"url":"https://www.ovh.co.uk/robots.txt","bytes":1234,"groups":6},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["blocked","named",1,0],"OAI-SearchBot":["partial","*",27,0],"ClaudeBot":["partial","*",27,0],"Claude-User":["partial","*",27,0],"Claude-SearchBot":["partial","*",27,0],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["partial","*",27,0],"Perplexity-User":["partial","*",27,0],"Google-Extended":["partial","*",27,0],"Applebot-Extended":["partial","*",27,0],"CCBot":["blocked","named",1,0],"Bytespider":["partial","*",27,0],"Amazonbot":["partial","*",27,0],"meta-externalagent":["partial","*",27,0],"cohere-ai":["partial","*",27,0],"Diffbot":["partial","*",27,0],"ContextFetchBot":["partial","*",27,0]},"llms":{"status":"missing","http":404,"bytes":146},"home":{"status":"ok","http":200,"url":"https://www.ovhcloud.com/en-gb/","html_tokens":268428,"markdown_tokens":487,"pruned_tokens":107}},
{"rank":9181,"domain":"alphotography.sk","origin":"https://alphotography.sk","robots":{"status":"ok","http":200,"url":"https://alphotography.sk/robots.txt","bytes":70,"groups":1},"bots":{"GPTBot":["allowed","*",0,1],"ChatGPT-User":["allowed","*",0,1],"OAI-SearchBot":["allowed","*",0,1],"ClaudeBot":["allowed","*",0,1],"Claude-User":["allowed","*",0,1],"Claude-SearchBot":["allowed","*",0,1],"anthropic-ai":["allowed","*",0,1],"PerplexityBot":["allowed","*",0,1],"Perplexity-User":["allowed","*",0,1],"Google-Extended":["allowed","*",0,1],"Applebot-Extended":["allowed","*",0,1],"CCBot":["allowed","*",0,1],"Bytespider":["allowed","*",0,1],"Amazonbot":["allowed","*",0,1],"meta-externalagent":["allowed","*",0,1],"cohere-ai":["allowed","*",0,1],"Diffbot":["allowed","*",0,1],"ContextFetchBot":["allowed","*",0,1]},"llms":{"status":"missing","http":404,"bytes":645},"home":{"status":"ok","http":200,"url":"https://alphotography.sk/","html_tokens":44424,"markdown_tokens":25541,"pruned_tokens":123}},
{"rank":9182,"domain":"shunyuanlawyer.com","origin":"http://shunyuanlawyer.com","robots":{"status":"missing","http":404,"url":"http://shunyuanlawyer.com/robots.txt","bytes":1231,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":1231},"home":{"status":"ok","http":200,"url":"http://shunyuanlawyer.com/","html_tokens":10821,"markdown_tokens":73,"pruned_tokens":27}},
{"rank":9183,"domain":"panotoday.net","origin":"https://panotoday.net","robots":{"status":"unreachable","http":null,"url":"https://panotoday.net/robots.txt","bytes":0,"groups":0,"error":"cannot resolve panotoday.net"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":9184,"domain":"classicfm.com","origin":"https://classicfm.com","robots":{"status":"ok","http":200,"url":"https://www.classicfm.com/robots.txt","bytes":670,"groups":13},"bots":{"GPTBot":["partial","*",3,0],"ChatGPT-User":["partial","*",3,0],"OAI-SearchBot":["partial","*",3,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",3,0],"Claude-SearchBot":["partial","*",3,0],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["partial","*",3,0],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["partial","*",3,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["partial","*",3,0],"meta-externalagent":["partial","*",3,0],"cohere-ai":["partial","*",3,0],"Diffbot":["partial","*",3,0],"ContextFetchBot":["partial","*",3,0]},"llms":{"status":"missing","http":404,"bytes":53989},"home":{"status":"ok","http":200,"url":"https://www.classicfm.com/","html_tokens":54836,"markdown_tokens":26,"pruned_tokens":26}},
{"rank":9185,"domain":"bruegel.org","origin":"https://bruegel.org","robots":{"status":"ok","http":200,"url":"https://bruegel.org/robots.txt","bytes":2027,"groups":1},"bots":{"GPTBot":["partial","*",34,18],"ChatGPT-User":["partial","*",34,18],"OAI-SearchBot":["partial","*",34,18],"ClaudeBot":["partial","*",34,18],"Claude-User":["partial","*",34,18],"Claude-SearchBot":["partial","*",34,18],"anthropic-ai":["partial","*",34,18],"PerplexityBot":["partial","*",34,18],"Perplexity-User":["partial","*",34,18],"Google-Extended":["partial","*",34,18],"Applebot-Extended":["partial","*",34,18],"CCBot":["partial","*",34,18],"Bytespider":["partial","*",34,18],"Amazonbot":["partial","*",34,18],"meta-externalagent":["partial","*",34,18],"cohere-ai":["partial","*",34,18],"Diffbot":["partial","*",34,18],"ContextFetchBot":["partial","*",34,18]},"llms":{"status":"missing","http":404,"bytes":550},"home":{"status":"ok","http":200,"url":"https://www.bruegel.org/","html_tokens":27753,"markdown_tokens":599,"pruned_tokens":255}},
{"rank":9186,"domain":"virtualave.net","origin":"https://virtualave.net","robots":{"status":"refused","http":403,"url":"https://www.networksolutions.com/ipage?brand=ipage&siteID=100&channelID=P99C100S653N0B5A1D0E0000V113","bytes":12504,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":9187,"domain":"hbhzs.org.cn","origin":"http://hbhzs.org.cn","robots":{"status":"missing","http":404,"url":"http://hbhzs.org.cn/robots.txt","bytes":146,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":146},"home":{"status":"ok","http":200,"url":"http://hbhzs.org.cn/","html_tokens":35,"markdown_tokens":6,"pruned_tokens":6}},
{"rank":9188,"domain":"staradvertiser.com","origin":"https://staradvertiser.com","robots":{"status":"ok","http":200,"url":"https://staradvertiser.com/robots.txt","bytes":2810,"groups":57},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["blocked","named",1,0],"OAI-SearchBot":["blocked","named",1,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"http_error","http":403,"bytes":5571},"home":{"status":"http_error","http":403,"url":"https://staradvertiser.com/"}},
{"rank":9189,"domain":"fair.org","origin":"https://fair.org","robots":{"status":"refused","http":403,"url":"https://fair.org/robots.txt","bytes":5524,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":9190,"domain":"royal-game-slots.com","origin":"https://royal-game-slots.com","robots":{"status":"ok","http":200,"url":"https://royal-game-slots.com/robots.txt","bytes":67,"groups":1},"bots":{"GPTBot":["partial","*",1,1],"ChatGPT-User":["partial","*",1,1],"OAI-SearchBot":["partial","*",1,1],"ClaudeBot":["partial","*",1,1],"Claude-User":["partial","*",1,1],"Claude-SearchBot":["partial","*",1,1],"anthropic-ai":["partial","*",1,1],"PerplexityBot":["partial","*",1,1],"Perplexity-User":["partial","*",1,1],"Google-Extended":["partial","*",1,1],"Applebot-Extended":["partial","*",1,1],"CCBot":["partial","*",1,1],"Bytespider":["partial","*",1,1],"Amazonbot":["partial","*",1,1],"meta-externalagent":["partial","*",1,1],"cohere-ai":["partial","*",1,1],"Diffbot":["partial","*",1,1],"ContextFetchBot":["partial","*",1,1]},"llms":{"status":"missing","http":404,"bytes":51168},"home":{"status":"ok","http":200,"url":"https://royal-game-slots.com/","html_tokens":20609,"markdown_tokens":4439,"pruned_tokens":79}},
{"rank":9191,"domain":"beinsports.com","origin":"https://beinsports.com","robots":{"status":"ok","http":200,"url":"https://www.beinsports.com/robots.txt","bytes":834,"groups":1},"bots":{"GPTBot":["partial","*",35,0],"ChatGPT-User":["partial","*",35,0],"OAI-SearchBot":["partial","*",35,0],"ClaudeBot":["partial","*",35,0],"Claude-User":["partial","*",35,0],"Claude-SearchBot":["partial","*",35,0],"anthropic-ai":["partial","*",35,0],"PerplexityBot":["partial","*",35,0],"Perplexity-User":["partial","*",35,0],"Google-Extended":["partial","*",35,0],"Applebot-Extended":["partial","*",35,0],"CCBot":["partial","*",35,0],"Bytespider":["partial","*",35,0],"Amazonbot":["partial","*",35,0],"meta-externalagent":["partial","*",35,0],"cohere-ai":["partial","*",35,0],"Diffbot":["partial","*",35,0],"ContextFetchBot":["partial","*",35,0]},"llms":{"status":"missing","http":404,"bytes":9},"home":{"status":"ok","http":200,"url":"https://www.beinsports.com/en-us","html_tokens":245574,"markdown_tokens":77,"pruned_tokens":77}},
{"rank":9192,"domain":"nationalww2museum.org","origin":"https://www.nationalww2museum.org","robots":{"status":"refused","http":403,"url":"https://www.nationalww2museum.org/robots.txt","bytes":560,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":9193,"domain":"springerprofessional.de","origin":"https://springerprofessional.de","robots":{"status":"ok","http":200,"url":"https://www.springerprofessional.de/robots.txt","bytes":1083,"groups":5},"bots":{"GPTBot":["partial","*",35,1],"ChatGPT-User":["partial","*",35,1],"OAI-SearchBot":["partial","*",35,1],"ClaudeBot":["partial","*",35,1],"Claude-User":["partial","*",35,1],"Claude-SearchBot":["partial","*",35,1],"anthropic-ai":["partial","*",35,1],"PerplexityBot":["partial","*",35,1],"Perplexity-User":["partial","*",35,1],"Google-Extended":["partial","*",35,1],"Applebot-Extended":["partial","*",35,1],"CCBot":["partial","*",35,1],"Bytespider":["partial","*",35,1],"Amazonbot":["partial","*",35,1],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["partial","*",35,1],"Diffbot":["partial","*",35,1],"ContextFetchBot":["partial","*",35,1]},"llms":{"status":"missing","http":404,"bytes":77095},"home":{"status":"ok","http":200,"url":"https://www.springerprofessional.de/","html_tokens":69302,"markdown_tokens":1899,"pruned_tokens":93}},
{"rank":9194,"domain":"fluco.store","origin":"https://fluco.store","robots":{"status":"ok","http":200,"url":"https://fluco.store/robots.txt","bytes":74,"groups":1},"bots":{"GPTBot":["partial","*",1,0],"ChatGPT-User":["partial","*",1,0],"OAI-SearchBot":["partial","*",1,0],"ClaudeBot":["partial","*",1,0],"Claude-User":["partial","*",1,0],"Claude-SearchBot":["partial","*",1,0],"anthropic-ai":["partial","*",1,0],"PerplexityBot":["partial","*",1,0],"Perplexity-User":["partial","*",1,0],"Google-Extended":["partial","*",1,0],"Applebot-Extended":["partial","*",1,0],"CCBot":["partial","*",1,0],"Bytespider":["partial","*",1,0],"Amazonbot":["partial","*",1,0],"meta-externalagent":["partial","*",1,0],"cohere-ai":["partial","*",1,0],"Diffbot":["partial","*",1,0],"ContextFetchBot":["partial","*",1,0]},"llms":{"status":"missing","http":404,"bytes":640},"home":{"status":"ok","http":200,"url":"https://fluco.store/","html_tokens":19716,"markdown_tokens":995,"pruned_tokens":134}},
{"rank":9195,"domain":"libcom.org","origin":"https://libcom.org","robots":{"status":"ok","http":200,"url":"https://libcom.org/robots.txt","bytes":2027,"groups":1},"bots":{"GPTBot":["partial","*",34,18],"ChatGPT-User":["partial","*",34,18],"OAI-SearchBot":["partial","*",34,18],"ClaudeBot":["partial","*",34,18],"Claude-User":["partial","*",34,18],"Claude-SearchBot":["partial","*",34,18],"anthropic-ai":["partial","*",34,18],"PerplexityBot":["partial","*",34,18],"Perplexity-User":["partial","*",34,18],"Google-Extended":["partial","*",34,18],"Applebot-Extended":["partial","*",34,18],"CCBot":["partial","*",34,18],"Bytespider":["partial","*",34,18],"Amazonbot":["partial","*",34,18],"meta-externalagent":["partial","*",34,18],"cohere-ai":["partial","*",34,18],"Diffbot":["partial","*",34,18],"ContextFetchBot":["partial","*",34,18]},"llms":{"status":"missing","http":404,"bytes":551},"home":{"status":"ok","http":200,"url":"https://libcom.org/","html_tokens":13458,"markdown_tokens":82,"pruned_tokens":82}},
{"rank":9196,"domain":"videohelp.com","origin":"https://videohelp.com","robots":{"status":"ok","http":200,"url":"https://www.videohelp.com/robots.txt","bytes":2058,"groups":3},"bots":{"GPTBot":["allowed","named",0,0],"ChatGPT-User":["partial","*",62,0],"OAI-SearchBot":["partial","*",62,0],"ClaudeBot":["allowed","named",0,0],"Claude-User":["partial","*",62,0],"Claude-SearchBot":["partial","*",62,0],"anthropic-ai":["partial","*",62,0],"PerplexityBot":["partial","*",62,0],"Perplexity-User":["partial","*",62,0],"Google-Extended":["partial","*",62,0],"Applebot-Extended":["partial","*",62,0],"CCBot":["partial","*",62,0],"Bytespider":["partial","*",62,0],"Amazonbot":["allowed","named",0,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["partial","*",62,0],"Diffbot":["partial","*",62,0],"ContextFetchBot":["partial","*",62,0]},"llms":{"status":"missing","http":404,"bytes":29459},"home":{"status":"ok","http":200,"url":"https://www.videohelp.com/","html_tokens":30627,"markdown_tokens":713,"pruned_tokens":125}},
{"rank":9197,"domain":"paessler.com","origin":"https://paessler.com","robots":{"status":"ok","http":200,"url":"https://www.paessler.com/robots.txt","bytes":843,"groups":2},"bots":{"GPTBot":["partial","*",2,0],"ChatGPT-User":["partial","*",2,0],"OAI-SearchBot":["partial","*",2,0],"ClaudeBot":["partial","*",2,0],"Claude-User":["partial","*",2,0],"Claude-SearchBot":["partial","*",2,0],"anthropic-ai":["partial","*",2,0],"PerplexityBot":["partial","*",2,0],"Perplexity-User":["partial","*",2,0],"Google-Extended":["partial","*",2,0],"Applebot-Extended":["partial","*",2,0],"CCBot":["partial","*",2,0],"Bytespider":["partial","*",2,0],"Amazonbot":["partial","*",2,0],"meta-externalagent":["partial","*",2,0],"cohere-ai":["partial","*",2,0],"Diffbot":["partial","*",2,0],"ContextFetchBot":["partial","*",2,0]},"llms":{"status":"missing","http":404,"bytes":250441},"home":{"status":"ok","http":200,"url":"https://www.paessler.com/","html_tokens":35975,"markdown_tokens":397,"pruned_tokens":90}},
{"rank":9198,"domain":"railway.app","origin":"https://railway.app","robots":{"status":"ok","http":200,"url":"https://railway.com/robots.txt","bytes":520,"groups":1},"bots":{"GPTBot":["partial","*",13,1],"ChatGPT-User":["partial","*",13,1],"OAI-SearchBot":["partial","*",13,1],"ClaudeBot":["partial","*",13,1],"Claude-User":["partial","*",13,1],"Claude-SearchBot":["partial","*",13,1],"anthropic-ai":["partial","*",13,1],"PerplexityBot":["partial","*",13,1],"Perplexity-User":["partial","*",13,1],"Google-Extended":["partial","*",13,1],"Applebot-Extended":["partial","*",13,1],"CCBot":["partial","*",13,1],"Bytespider":["partial","*",13,1],"Amazonbot":["partial","*",13,1],"meta-externalagent":["partial","*",13,1],"cohere-ai":["partial","*",13,1],"Diffbot":["partial","*",13,1],"ContextFetchBot":["partial","*",13,1]},"llms":{"status":"present","http":200,"bytes":6625},"home":{"status":"ok","http":200,"url":"https://railway.com/","html_tokens":56598,"markdown_tokens":250,"pruned_tokens":111}},
{"rank":9199,"domain":"nero.com","origin":"https://nero.com","robots":{"status":"ok","http":200,"url":"https://www.nero.com/robots.txt","bytes":919,"groups":1},"bots":{"GPTBot":["partial","*",29,0],"ChatGPT-User":["partial","*",29,0],"OAI-SearchBot":["partial","*",29,0],"ClaudeBot":["partial","*",29,0],"Claude-User":["partial","*",29,0],"Claude-SearchBot":["partial","*",29,0],"anthropic-ai":["partial","*",29,0],"PerplexityBot":["partial","*",29,0],"Perplexity-User":["partial","*",29,0],"Google-Extended":["partial","*",29,0],"Applebot-Extended":["partial","*",29,0],"CCBot":["partial","*",29,0],"Bytespider":["partial","*",29,0],"Amazonbot":["partial","*",29,0],"meta-externalagent":["partial","*",29,0],"cohere-ai":["partial","*",29,0],"Diffbot":["partial","*",29,0],"ContextFetchBot":["partial","*",29,0]},"llms":{"status":"html","http":200,"bytes":149714},"home":{"status":"ok","http":200,"url":"https://www.nero.com/enu/","html_tokens":37394,"markdown_tokens":1052,"pruned_tokens":421}},
{"rank":9200,"domain":"casino-lastschrift.com","origin":"https://casino-lastschrift.com","robots":{"status":"ok","http":200,"url":"https://casino-lastschrift.com/robots.txt","bytes":549,"groups":1},"bots":{"GPTBot":["partial","*",19,0],"ChatGPT-User":["partial","*",19,0],"OAI-SearchBot":["partial","*",19,0],"ClaudeBot":["partial","*",19,0],"Claude-User":["partial","*",19,0],"Claude-SearchBot":["partial","*",19,0],"anthropic-ai":["partial","*",19,0],"PerplexityBot":["partial","*",19,0],"Perplexity-User":["partial","*",19,0],"Google-Extended":["partial","*",19,0],"Applebot-Extended":["partial","*",19,0],"CCBot":["partial","*",19,0],"Bytespider":["partial","*",19,0],"Amazonbot":["partial","*",19,0],"meta-externalagent":["partial","*",19,0],"cohere-ai":["partial","*",19,0],"Diffbot":["partial","*",19,0],"ContextFetchBot":["partial","*",19,0]},"llms":{"status":"missing","http":404,"bytes":19516},"home":{"status":"ok","http":200,"url":"https://casino-lastschrift.com/","html_tokens":15624,"markdown_tokens":1317,"pruned_tokens":573}},
{"rank":9201,"domain":"ethnologue.com","origin":"https://ethnologue.com","robots":{"status":"refused","http":403,"url":"https://ethnologue.com/robots.txt","bytes":5552,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":9202,"domain":"adroll.com","origin":"https://adroll.com","robots":{"status":"ok","http":200,"url":"https://www.adroll.com/robots.txt","bytes":2669,"groups":1},"bots":{"GPTBot":["partial","*",70,0],"ChatGPT-User":["partial","*",70,0],"OAI-SearchBot":["partial","*",70,0],"ClaudeBot":["partial","*",70,0],"Claude-User":["partial","*",70,0],"Claude-SearchBot":["partial","*",70,0],"anthropic-ai":["partial","*",70,0],"PerplexityBot":["partial","*",70,0],"Perplexity-User":["partial","*",70,0],"Google-Extended":["partial","*",70,0],"Applebot-Extended":["partial","*",70,0],"CCBot":["partial","*",70,0],"Bytespider":["partial","*",70,0],"Amazonbot":["partial","*",70,0],"meta-externalagent":["partial","*",70,0],"cohere-ai":["partial","*",70,0],"Diffbot":["partial","*",70,0],"ContextFetchBot":["partial","*",70,0]},"llms":{"status":"present","http":200,"bytes":2159},"home":{"status":"ok","http":200,"url":"https://www.adroll.com/","html_tokens":11130,"markdown_tokens":176,"pruned_tokens":78}},
{"rank":9203,"domain":"hallmark.com","origin":"https://hallmark.com","robots":{"status":"ok","http":200,"url":"https://www.hallmark.com/robots.txt","bytes":1437,"groups":2},"bots":{"GPTBot":["partial","*",52,0],"ChatGPT-User":["partial","*",52,0],"OAI-SearchBot":["partial","*",52,0],"ClaudeBot":["partial","*",52,0],"Claude-User":["partial","*",52,0],"Claude-SearchBot":["partial","*",52,0],"anthropic-ai":["partial","*",52,0],"PerplexityBot":["partial","*",52,0],"Perplexity-User":["partial","*",52,0],"Google-Extended":["partial","*",52,0],"Applebot-Extended":["partial","*",52,0],"CCBot":["partial","*",52,0],"Bytespider":["partial","*",52,0],"Amazonbot":["partial","*",52,0],"meta-externalagent":["partial","*",52,0],"cohere-ai":["partial","*",52,0],"Diffbot":["partial","*",52,0],"ContextFetchBot":["partial","*",52,0]},"llms":{"status":"missing","http":404,"bytes":80135},"home":{"status":"ok","http":200,"url":"https://www.hallmark.com/","html_tokens":255257,"markdown_tokens":79,"pruned_tokens":50}},
{"rank":9204,"domain":"wargaming.net","origin":"https://wargaming.net","robots":{"status":"ok","http":200,"url":"https://wargaming.net/robots.txt","bytes":140,"groups":1},"bots":{"GPTBot":["partial","*",4,0],"ChatGPT-User":["partial","*",4,0],"OAI-SearchBot":["partial","*",4,0],"ClaudeBot":["partial","*",4,0],"Claude-User":["partial","*",4,0],"Claude-SearchBot":["partial","*",4,0],"anthropic-ai":["partial","*",4,0],"PerplexityBot":["partial","*",4,0],"Perplexity-User":["partial","*",4,0],"Google-Extended":["partial","*",4,0],"Applebot-Extended":["partial","*",4,0],"CCBot":["partial","*",4,0],"Bytespider":["partial","*",4,0],"Amazonbot":["partial","*",4,0],"meta-externalagent":["partial","*",4,0],"cohere-ai":["partial","*",4,0],"Diffbot":["partial","*",4,0],"ContextFetchBot":["partial","*",4,0]},"llms":{"status":"missing","http":404,"bytes":47541},"home":{"status":"ok","http":200,"url":"https://wargaming.net/en","html_tokens":19106,"markdown_tokens":22,"pruned_tokens":22}},
{"rank":9205,"domain":"dhm.de","origin":"https://dhm.de","robots":{"status":"ok","http":200,"url":"https://www.dhm.de/robots.txt","bytes":615,"groups":1},"bots":{"GPTBot":["partial","*",11,0],"ChatGPT-User":["partial","*",11,0],"OAI-SearchBot":["partial","*",11,0],"ClaudeBot":["partial","*",11,0],"Claude-User":["partial","*",11,0],"Claude-SearchBot":["partial","*",11,0],"anthropic-ai":["partial","*",11,0],"PerplexityBot":["partial","*",11,0],"Perplexity-User":["partial","*",11,0],"Google-Extended":["partial","*",11,0],"Applebot-Extended":["partial","*",11,0],"CCBot":["partial","*",11,0],"Bytespider":["partial","*",11,0],"Amazonbot":["partial","*",11,0],"meta-externalagent":["partial","*",11,0],"cohere-ai":["partial","*",11,0],"Diffbot":["partial","*",11,0],"ContextFetchBot":["partial","*",11,0]},"llms":{"status":"missing","http":404,"bytes":111892},"home":{"status":"ok","http":200,"url":"https://www.dhm.de/","html_tokens":33269,"markdown_tokens":52,"pruned_tokens":52}},
{"rank":9206,"domain":"autohotkey.com","origin":"https://autohotkey.com","robots":{"status":"refused","http":403,"url":"https://www.autohotkey.com/robots.txt","bytes":5428,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":9207,"domain":"biodiversitylibrary.org","origin":"https://biodiversitylibrary.org","robots":{"status":"refused","http":403,"url":"https://biodiversitylibrary.org/robots.txt","bytes":5433,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":9208,"domain":"uokik.gov.pl","origin":"https://uokik.gov.pl","robots":{"status":"ok","http":200,"url":"https://uokik.gov.pl/robots.txt","bytes":5375,"groups":4},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["partial","*",144,19],"OAI-SearchBot":["partial","*",144,19],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",144,19],"Claude-SearchBot":["partial","*",144,19],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["partial","*",144,19],"Perplexity-User":["partial","*",144,19],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["partial","*",144,19],"ContextFetchBot":["partial","*",144,19]},"llms":{"status":"other_text","http":200,"bytes":1477},"home":{"status":"ok","http":200,"url":"https://uokik.gov.pl/","html_tokens":25186,"markdown_tokens":80,"pruned_tokens":80}},
{"rank":9209,"domain":"swissre.com","origin":"https://swissre.com","robots":{"status":"ok","http":200,"url":"https://www.swissre.com/robots.txt","bytes":783,"groups":1},"bots":{"GPTBot":["partial","*",1,3],"ChatGPT-User":["partial","*",1,3],"OAI-SearchBot":["partial","*",1,3],"ClaudeBot":["partial","*",1,3],"Claude-User":["partial","*",1,3],"Claude-SearchBot":["partial","*",1,3],"anthropic-ai":["partial","*",1,3],"PerplexityBot":["partial","*",1,3],"Perplexity-User":["partial","*",1,3],"Google-Extended":["partial","*",1,3],"Applebot-Extended":["partial","*",1,3],"CCBot":["partial","*",1,3],"Bytespider":["partial","*",1,3],"Amazonbot":["partial","*",1,3],"meta-externalagent":["partial","*",1,3],"cohere-ai":["partial","*",1,3],"Diffbot":["partial","*",1,3],"ContextFetchBot":["partial","*",1,3]},"llms":{"status":"missing","http":404,"bytes":14754},"home":{"status":"http_error","http":403,"url":"https://www.swissre.com/"}},
{"rank":9210,"domain":"protonmail.com","origin":"https://protonmail.com","robots":{"status":"ok","http":200,"url":"https://proton.me/robots.txt","bytes":181,"groups":1},"bots":{"GPTBot":["partial","*",1,3],"ChatGPT-User":["partial","*",1,3],"OAI-SearchBot":["partial","*",1,3],"ClaudeBot":["partial","*",1,3],"Claude-User":["partial","*",1,3],"Claude-SearchBot":["partial","*",1,3],"anthropic-ai":["partial","*",1,3],"PerplexityBot":["partial","*",1,3],"Perplexity-User":["partial","*",1,3],"Google-Extended":["partial","*",1,3],"Applebot-Extended":["partial","*",1,3],"CCBot":["partial","*",1,3],"Bytespider":["partial","*",1,3],"Amazonbot":["partial","*",1,3],"meta-externalagent":["partial","*",1,3],"cohere-ai":["partial","*",1,3],"Diffbot":["partial","*",1,3],"ContextFetchBot":["partial","*",1,3]},"llms":{"status":"missing","http":404,"bytes":240245},"home":{"status":"ok","http":200,"url":"https://proton.me/mail","html_tokens":97023,"markdown_tokens":866,"pruned_tokens":193}},
{"rank":9211,"domain":"pages10.com","origin":"https://pages10.com","robots":{"status":"ok","http":200,"url":"https://pages10.com/robots.txt","bytes":72,"groups":2},"bots":{"GPTBot":["partial","*",1,0],"ChatGPT-User":["partial","*",1,0],"OAI-SearchBot":["partial","*",1,0],"ClaudeBot":["partial","*",1,0],"Claude-User":["partial","*",1,0],"Claude-SearchBot":["partial","*",1,0],"anthropic-ai":["partial","*",1,0],"PerplexityBot":["partial","*",1,0],"Perplexity-User":["partial","*",1,0],"Google-Extended":["partial","*",1,0],"Applebot-Extended":["partial","*",1,0],"CCBot":["partial","*",1,0],"Bytespider":["partial","*",1,0],"Amazonbot":["partial","*",1,0],"meta-externalagent":["partial","*",1,0],"cohere-ai":["partial","*",1,0],"Diffbot":["partial","*",1,0],"ContextFetchBot":["partial","*",1,0]},"llms":{"status":"missing","http":404,"bytes":529},"home":{"status":"ok","http":200,"url":"https://pages10.com/","html_tokens":1884,"markdown_tokens":30,"pruned_tokens":30}},
{"rank":9212,"domain":"liepin.com","origin":"https://liepin.com","robots":{"status":"ok","http":200,"url":"https://www.liepin.com/robots.txt","bytes":178,"groups":1},"bots":{"GPTBot":["partial","*",7,0],"ChatGPT-User":["partial","*",7,0],"OAI-SearchBot":["partial","*",7,0],"ClaudeBot":["partial","*",7,0],"Claude-User":["partial","*",7,0],"Claude-SearchBot":["partial","*",7,0],"anthropic-ai":["partial","*",7,0],"PerplexityBot":["partial","*",7,0],"Perplexity-User":["partial","*",7,0],"Google-Extended":["partial","*",7,0],"Applebot-Extended":["partial","*",7,0],"CCBot":["partial","*",7,0],"Bytespider":["partial","*",7,0],"Amazonbot":["partial","*",7,0],"meta-externalagent":["partial","*",7,0],"cohere-ai":["partial","*",7,0],"Diffbot":["partial","*",7,0],"ContextFetchBot":["partial","*",7,0]},"llms":{"status":"missing","http":404,"bytes":4933},"home":{"status":"ok","http":200,"url":"https://www.liepin.com/","html_tokens":85525,"markdown_tokens":26,"pruned_tokens":26}},
{"rank":9213,"domain":"xcfybj.com","origin":"https://xcfybj.com","robots":{"status":"timeout","http":null,"url":"https://xcfybj.com/robots.txt","bytes":0,"groups":0,"error":"ConnectTimeout"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":9214,"domain":"ewtnnews.com","origin":"https://ewtnnews.com","robots":{"status":"ok","http":200,"url":"https://ewtnnews.com/robots.txt","bytes":2676,"groups":32},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["allowed","*",0,1],"OAI-SearchBot":["allowed","*",0,1],"ClaudeBot":["blocked","named",1,0],"Claude-User":["allowed","*",0,1],"Claude-SearchBot":["allowed","*",0,1],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["allowed","*",0,1],"Perplexity-User":["allowed","*",0,1],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["allowed","*",0,1]},"llms":{"status":"missing","http":404,"bytes":67401},"home":{"status":"ok","http":200,"url":"https://www.ewtnnews.com/","html_tokens":256318,"markdown_tokens":314,"pruned_tokens":157}},
{"rank":9215,"domain":"globalpost.com","origin":"https://globalpost.com","robots":{"status":"ok","http":200,"url":"https://globalpost.com/robots.txt","bytes":452,"groups":3},"bots":{"GPTBot":["partial","*",9,1],"ChatGPT-User":["partial","*",9,1],"OAI-SearchBot":["partial","*",9,1],"ClaudeBot":["partial","*",9,1],"Claude-User":["partial","*",9,1],"Claude-SearchBot":["partial","*",9,1],"anthropic-ai":["partial","*",9,1],"PerplexityBot":["partial","*",9,1],"Perplexity-User":["partial","*",9,1],"Google-Extended":["partial","*",9,1],"Applebot-Extended":["partial","*",9,1],"CCBot":["partial","*",9,1],"Bytespider":["partial","*",9,1],"Amazonbot":["partial","*",9,1],"meta-externalagent":["partial","*",9,1],"cohere-ai":["partial","*",9,1],"Diffbot":["partial","*",9,1],"ContextFetchBot":["partial","*",9,1]},"llms":{"status":"other_text","http":200,"bytes":4962},"home":{"status":"ok","http":200,"url":"https://globalpost.com/","html_tokens":31232,"markdown_tokens":106,"pruned_tokens":106}},
{"rank":9216,"domain":"toptenreviews.com","origin":"https://toptenreviews.com","robots":{"status":"ok","http":200,"url":"https://www.toptenreviews.com/robots.txt","bytes":1488,"groups":4},"bots":{"GPTBot":["partial","*",18,0],"ChatGPT-User":["partial","*",18,0],"OAI-SearchBot":["partial","*",18,0],"ClaudeBot":["partial","*",18,0],"Claude-User":["partial","*",18,0],"Claude-SearchBot":["partial","*",18,0],"anthropic-ai":["partial","*",18,0],"PerplexityBot":["partial","*",18,0],"Perplexity-User":["partial","*",18,0],"Google-Extended":["partial","*",18,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["partial","*",18,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["partial","*",18,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",18,0]},"llms":{"status":"missing","http":404,"bytes":420337},"home":{"status":"ok","http":200,"url":"https://www.toptenreviews.com/","html_tokens":183329,"markdown_tokens":35,"pruned_tokens":35}},
{"rank":9217,"domain":"yidianzixun.com","origin":"https://yidianzixun.com","robots":{"status":"ok","http":200,"url":"https://yidianzixun.com/robots.txt","bytes":25,"groups":1},"bots":{"GPTBot":["blocked","*",1,0],"ChatGPT-User":["blocked","*",1,0],"OAI-SearchBot":["blocked","*",1,0],"ClaudeBot":["blocked","*",1,0],"Claude-User":["blocked","*",1,0],"Claude-SearchBot":["blocked","*",1,0],"anthropic-ai":["blocked","*",1,0],"PerplexityBot":["blocked","*",1,0],"Perplexity-User":["blocked","*",1,0],"Google-Extended":["blocked","*",1,0],"Applebot-Extended":["blocked","*",1,0],"CCBot":["blocked","*",1,0],"Bytespider":["blocked","*",1,0],"Amazonbot":["blocked","*",1,0],"meta-externalagent":["blocked","*",1,0],"cohere-ai":["blocked","*",1,0],"Diffbot":["blocked","*",1,0],"ContextFetchBot":["blocked","*",1,0]},"llms":{"status":"disallowed"},"home":{"status":"disallowed"}},
{"rank":9218,"domain":"xn--wbr89i17glxijyay55ao40c5eza.com","origin":"http://xn--wbr89i17glxijyay55ao40c5eza.com","robots":{"status":"missing","http":404,"url":"http://xn--wbr89i17glxijyay55ao40c5eza.com/robots.txt","bytes":281,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":281},"home":{"status":"disallowed","error":"robots.txt of stopnote.vhostgo.com not readable (unreachable)"}},
{"rank":9219,"domain":"marieclaire.co.uk","origin":"https://marieclaire.co.uk","robots":{"status":"ok","http":200,"url":"https://www.marieclaire.co.uk/robots.txt","bytes":1865,"groups":6},"bots":{"GPTBot":["partial","*",24,0],"ChatGPT-User":["partial","*",24,0],"OAI-SearchBot":["partial","*",24,0],"ClaudeBot":["partial","*",24,0],"Claude-User":["partial","*",24,0],"Claude-SearchBot":["partial","*",24,0],"anthropic-ai":["partial","*",24,0],"PerplexityBot":["partial","*",24,0],"Perplexity-User":["partial","*",24,0],"Google-Extended":["partial","*",24,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["partial","*",24,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",2,0],"cohere-ai":["partial","*",24,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",24,0]},"llms":{"status":"missing","http":404,"bytes":942182},"home":{"status":"ok","http":200,"url":"https://www.marieclaire.co.uk/","html_tokens":374233,"markdown_tokens":1778,"pruned_tokens":172}},
{"rank":9220,"domain":"energysavingtrust.org.uk","origin":"https://energysavingtrust.org.uk","robots":{"status":"refused","http":403,"url":"https://energysavingtrust.org.uk/robots.txt","bytes":4918,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":9221,"domain":"rateyourmusic.com","origin":"https://rateyourmusic.com","robots":{"status":"ok","http":200,"url":"https://rateyourmusic.com/robots.txt","bytes":5230,"groups":59},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["blocked","named",1,0],"OAI-SearchBot":["blocked","named",1,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["blocked","named",1,0],"Claude-SearchBot":["blocked","named",1,0],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["blocked","named",1,0],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["blocked","*",1,0]},"llms":{"status":"disallowed"},"home":{"status":"disallowed"}},
{"rank":9222,"domain":"abuking777.com","origin":"https://abuking777.com","robots":{"status":"ok","http":200,"url":"https://abuking777.com/robots.txt","bytes":1973,"groups":1},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":643},"home":{"status":"ok","http":200,"url":"https://abuking777.com/","html_tokens":15468,"markdown_tokens":2078,"pruned_tokens":151}},
{"rank":9223,"domain":"ovocnehracieautomatyzadarmo.com","origin":"https://ovocnehracieautomatyzadarmo.com","robots":{"status":"ok","http":200,"url":"https://ovocnehracieautomatyzadarmo.com/robots.txt","bytes":2761,"groups":33},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["allowed","*",0,2],"OAI-SearchBot":["allowed","*",0,2],"ClaudeBot":["blocked","named",1,0],"Claude-User":["allowed","*",0,2],"Claude-SearchBot":["allowed","*",0,2],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["allowed","*",0,2],"Perplexity-User":["allowed","*",0,2],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["allowed","*",0,2]},"llms":{"status":"missing","http":404,"bytes":660},"home":{"status":"ok","http":200,"url":"https://ovocnehracieautomatyzadarmo.com/","html_tokens":39354,"markdown_tokens":22439,"pruned_tokens":141}},
{"rank":9224,"domain":"korculkovo.sk","origin":"https://korculkovo.sk","robots":{"status":"ok","http":200,"url":"https://korculkovo.sk/robots.txt","bytes":67,"groups":1},"bots":{"GPTBot":["allowed","*",0,1],"ChatGPT-User":["allowed","*",0,1],"OAI-SearchBot":["allowed","*",0,1],"ClaudeBot":["allowed","*",0,1],"Claude-User":["allowed","*",0,1],"Claude-SearchBot":["allowed","*",0,1],"anthropic-ai":["allowed","*",0,1],"PerplexityBot":["allowed","*",0,1],"Perplexity-User":["allowed","*",0,1],"Google-Extended":["allowed","*",0,1],"Applebot-Extended":["allowed","*",0,1],"CCBot":["allowed","*",0,1],"Bytespider":["allowed","*",0,1],"Amazonbot":["allowed","*",0,1],"meta-externalagent":["allowed","*",0,1],"cohere-ai":["allowed","*",0,1],"Diffbot":["allowed","*",0,1],"ContextFetchBot":["allowed","*",0,1]},"llms":{"status":"missing","http":404,"bytes":642},"home":{"status":"ok","http":200,"url":"https://korculkovo.sk/","html_tokens":40171,"markdown_tokens":23222,"pruned_tokens":121}},
{"rank":9225,"domain":"duo.com","origin":"https://duo.com","robots":{"status":"ok","http":200,"url":"https://duo.com/robots.txt","bytes":80,"groups":1},"bots":{"GPTBot":["partial","*",1,0],"ChatGPT-User":["partial","*",1,0],"OAI-SearchBot":["partial","*",1,0],"ClaudeBot":["partial","*",1,0],"Claude-User":["partial","*",1,0],"Claude-SearchBot":["partial","*",1,0],"anthropic-ai":["partial","*",1,0],"PerplexityBot":["partial","*",1,0],"Perplexity-User":["partial","*",1,0],"Google-Extended":["partial","*",1,0],"Applebot-Extended":["partial","*",1,0],"CCBot":["partial","*",1,0],"Bytespider":["partial","*",1,0],"Amazonbot":["partial","*",1,0],"meta-externalagent":["partial","*",1,0],"cohere-ai":["partial","*",1,0],"Diffbot":["partial","*",1,0],"ContextFetchBot":["partial","*",1,0]},"llms":{"status":"present","http":200,"bytes":266835},"home":{"status":"ok","http":200,"url":"https://duo.com/","html_tokens":33304,"markdown_tokens":80,"pruned_tokens":80}},
{"rank":9226,"domain":"correctiv.org","origin":"https://correctiv.org","robots":{"status":"ok","http":200,"url":"https://correctiv.org/robots.txt","bytes":326,"groups":1},"bots":{"GPTBot":["partial","*",5,0],"ChatGPT-User":["partial","*",5,0],"OAI-SearchBot":["partial","*",5,0],"ClaudeBot":["partial","*",5,0],"Claude-User":["partial","*",5,0],"Claude-SearchBot":["partial","*",5,0],"anthropic-ai":["partial","*",5,0],"PerplexityBot":["partial","*",5,0],"Perplexity-User":["partial","*",5,0],"Google-Extended":["partial","*",5,0],"Applebot-Extended":["partial","*",5,0],"CCBot":["partial","*",5,0],"Bytespider":["partial","*",5,0],"Amazonbot":["partial","*",5,0],"meta-externalagent":["partial","*",5,0],"cohere-ai":["partial","*",5,0],"Diffbot":["partial","*",5,0],"ContextFetchBot":["partial","*",5,0]},"llms":{"status":"missing","http":404,"bytes":12819},"home":{"status":"ok","http":200,"url":"https://correctiv.org/","html_tokens":76190,"markdown_tokens":173,"pruned_tokens":121}},
{"rank":9227,"domain":"vogue.fr","origin":"https://vogue.fr","robots":{"status":"ok","http":200,"url":"https://www.vogue.fr/robots.txt","bytes":2097,"groups":3},"bots":{"GPTBot":["partial","*",14,12],"ChatGPT-User":["partial","*",14,12],"OAI-SearchBot":["partial","*",14,12],"ClaudeBot":["blocked","named",1,0],"Claude-User":["blocked","named",1,0],"Claude-SearchBot":["blocked","named",1,0],"anthropic-ai":["partial","*",14,12],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["blocked","named",1,0],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",14,12]},"llms":{"status":"missing","http":404,"bytes":425676},"home":{"status":"ok","http":200,"url":"https://www.vogue.fr/","html_tokens":383198,"markdown_tokens":62,"pruned_tokens":57}},
{"rank":9228,"domain":"orenobl.ru","origin":"https://orenobl.ru","robots":{"status":"refused","http":403,"url":"https://orenobl.ru/robots.txt","bytes":4543,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":9229,"domain":"qemu.org","origin":"https://qemu.org","robots":{"status":"missing","http":404,"url":"https://www.qemu.org/robots.txt","bytes":3725,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":3725},"home":{"status":"ok","http":200,"url":"https://www.qemu.org/","html_tokens":2492,"markdown_tokens":70,"pruned_tokens":70}},
{"rank":9230,"domain":"bjwlxy.cn","origin":"https://bjwlxy.cn","robots":{"status":"refused","http":403,"url":"https://bjwlxy.cn/robots.txt","bytes":199,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":9231,"domain":"betterstack.com","origin":"https://betterstack.com","robots":{"status":"ok","http":200,"url":"https://betterstack.com/robots.txt","bytes":552,"groups":2},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"missing","http":404,"bytes":81657},"home":{"status":"not_html","error":"unsupported content type text/markdown"}},
{"rank":9232,"domain":"marketbeat.com","origin":"https://marketbeat.com","robots":{"status":"ok","http":200,"url":"https://www.marketbeat.com/robots.txt","bytes":1562,"groups":4},"bots":{"GPTBot":["partial","*",38,10],"ChatGPT-User":["partial","*",38,10],"OAI-SearchBot":["partial","*",38,10],"ClaudeBot":["partial","*",38,10],"Claude-User":["partial","*",38,10],"Claude-SearchBot":["partial","*",38,10],"anthropic-ai":["partial","*",38,10],"PerplexityBot":["partial","*",38,10],"Perplexity-User":["partial","*",38,10],"Google-Extended":["partial","*",38,10],"Applebot-Extended":["partial","*",38,10],"CCBot":["partial","*",38,10],"Bytespider":["partial","*",38,10],"Amazonbot":["partial","*",38,10],"meta-externalagent":["partial","*",38,10],"cohere-ai":["partial","*",38,10],"Diffbot":["partial","*",38,10],"ContextFetchBot":["partial","*",38,10]},"llms":{"status":"present","http":200,"bytes":2719},"home":{"status":"ok","http":200,"url":"https://www.marketbeat.com/","html_tokens":68722,"markdown_tokens":447,"pruned_tokens":448}},
{"rank":9233,"domain":"archive.today","origin":"https://archive.today","robots":{"status":"ok","http":200,"url":"https://archive.today/robots.txt","bytes":445,"groups":1},"bots":{"GPTBot":["partial","*",14,2],"ChatGPT-User":["partial","*",14,2],"OAI-SearchBot":["partial","*",14,2],"ClaudeBot":["partial","*",14,2],"Claude-User":["partial","*",14,2],"Claude-SearchBot":["partial","*",14,2],"anthropic-ai":["partial","*",14,2],"PerplexityBot":["partial","*",14,2],"Perplexity-User":["partial","*",14,2],"Google-Extended":["partial","*",14,2],"Applebot-Extended":["partial","*",14,2],"CCBot":["partial","*",14,2],"Bytespider":["partial","*",14,2],"Amazonbot":["partial","*",14,2],"meta-externalagent":["partial","*",14,2],"cohere-ai":["partial","*",14,2],"Diffbot":["partial","*",14,2],"ContextFetchBot":["partial","*",14,2]},"llms":{"status":"missing","http":404,"bytes":13},"home":{"status":"ok","http":200,"url":"https://archive.ph/","html_tokens":3546,"markdown_tokens":154,"pruned_tokens":38}},
{"rank":9234,"domain":"zaobao.com.sg","origin":"https://zaobao.com.sg","robots":{"status":"ok","http":200,"url":"https://www.zaobao.com.sg/robots.txt","bytes":2995,"groups":22},"bots":{"GPTBot":["allowed","named",0,1],"ChatGPT-User":["allowed","named",0,1],"OAI-SearchBot":["allowed","named",0,1],"ClaudeBot":["allowed","named",0,1],"Claude-User":["allowed","named",0,1],"Claude-SearchBot":["allowed","named",0,1],"anthropic-ai":["allowed","named",0,1],"PerplexityBot":["allowed","named",0,1],"Perplexity-User":["allowed","named",0,1],"Google-Extended":["allowed","named",0,1],"Applebot-Extended":["partial","*",41,18],"CCBot":["partial","*",41,18],"Bytespider":["partial","*",41,18],"Amazonbot":["partial","*",41,18],"meta-externalagent":["partial","*",41,18],"cohere-ai":["partial","*",41,18],"Diffbot":["partial","*",41,18],"ContextFetchBot":["partial","*",41,18]},"llms":{"status":"missing","http":404,"bytes":9},"home":{"status":"ok","http":200,"url":"https://www.zaobao.com.sg","html_tokens":63838,"markdown_tokens":10,"pruned_tokens":10}},
{"rank":9235,"domain":"ca-2000.com","origin":"http://ca-2000.com","robots":{"status":"ok","http":200,"url":"http://ca-2000.com/robots.txt","bytes":249,"groups":1},"bots":{"GPTBot":["partial","*",10,0],"ChatGPT-User":["partial","*",10,0],"OAI-SearchBot":["partial","*",10,0],"ClaudeBot":["partial","*",10,0],"Claude-User":["partial","*",10,0],"Claude-SearchBot":["partial","*",10,0],"anthropic-ai":["partial","*",10,0],"PerplexityBot":["partial","*",10,0],"Perplexity-User":["partial","*",10,0],"Google-Extended":["partial","*",10,0],"Applebot-Extended":["partial","*",10,0],"CCBot":["partial","*",10,0],"Bytespider":["partial","*",10,0],"Amazonbot":["partial","*",10,0],"meta-externalagent":["partial","*",10,0],"cohere-ai":["partial","*",10,0],"Diffbot":["partial","*",10,0],"ContextFetchBot":["partial","*",10,0]},"llms":{"status":"missing","http":404,"bytes":3749},"home":{"status":"ok","http":200,"url":"http://ca-2000.com/","html_tokens":3706,"markdown_tokens":189,"pruned_tokens":130}},
{"rank":9236,"domain":"sonesta.com","origin":"https://sonesta.com","robots":{"status":"ok","http":200,"url":"https://www.sonesta.com/robots.txt","bytes":224,"groups":4},"bots":{"GPTBot":["allowed","*",0,1],"ChatGPT-User":["allowed","*",0,1],"OAI-SearchBot":["allowed","*",0,1],"ClaudeBot":["allowed","*",0,1],"Claude-User":["allowed","*",0,1],"Claude-SearchBot":["allowed","*",0,1],"anthropic-ai":["allowed","*",0,1],"PerplexityBot":["allowed","*",0,1],"Perplexity-User":["allowed","*",0,1],"Google-Extended":["allowed","*",0,1],"Applebot-Extended":["allowed","*",0,1],"CCBot":["allowed","*",0,1],"Bytespider":["allowed","*",0,1],"Amazonbot":["allowed","*",0,1],"meta-externalagent":["allowed","*",0,1],"cohere-ai":["allowed","*",0,1],"Diffbot":["allowed","*",0,1],"ContextFetchBot":["allowed","*",0,1]},"llms":{"status":"missing","http":404,"bytes":767937},"home":{"status":"ok","http":200,"url":"https://www.sonesta.com/","html_tokens":227851,"markdown_tokens":121,"pruned_tokens":50}},
{"rank":9237,"domain":"sipc.org","origin":"https://sipc.org","robots":{"status":"missing","http":404,"url":"https://www.sipc.org/robots.txt","bytes":104180,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":104180},"home":{"status":"ok","http":200,"url":"https://www.sipc.org/","html_tokens":28537,"markdown_tokens":180,"pruned_tokens":116}},
{"rank":9238,"domain":"atf.gov","origin":"https://atf.gov","robots":{"status":"refused","http":403,"url":"https://www.atf.gov/robots.txt","bytes":379,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":9239,"domain":"laughingsquid.com","origin":"https://laughingsquid.com","robots":{"status":"ok","http":200,"url":"https://laughingsquid.com/robots.txt","bytes":281,"groups":1},"bots":{"GPTBot":["partial","*",3,0],"ChatGPT-User":["partial","*",3,0],"OAI-SearchBot":["partial","*",3,0],"ClaudeBot":["partial","*",3,0],"Claude-User":["partial","*",3,0],"Claude-SearchBot":["partial","*",3,0],"anthropic-ai":["partial","*",3,0],"PerplexityBot":["partial","*",3,0],"Perplexity-User":["partial","*",3,0],"Google-Extended":["partial","*",3,0],"Applebot-Extended":["partial","*",3,0],"CCBot":["partial","*",3,0],"Bytespider":["partial","*",3,0],"Amazonbot":["partial","*",3,0],"meta-externalagent":["partial","*",3,0],"cohere-ai":["partial","*",3,0],"Diffbot":["partial","*",3,0],"ContextFetchBot":["partial","*",3,0]},"llms":{"status":"other_text","http":200,"bytes":2328},"home":{"status":"ok","http":200,"url":"https://laughingsquid.com/","html_tokens":81368,"markdown_tokens":4,"pruned_tokens":4}},
{"rank":9240,"domain":"blockchain.info","origin":"https://blockchain.info","robots":{"status":"ok","http":200,"url":"https://www.blockchain.com/robots.txt","bytes":703,"groups":2},"bots":{"GPTBot":["partial","*",6,3],"ChatGPT-User":["partial","*",6,3],"OAI-SearchBot":["partial","*",6,3],"ClaudeBot":["partial","*",6,3],"Claude-User":["partial","*",6,3],"Claude-SearchBot":["partial","*",6,3],"anthropic-ai":["partial","*",6,3],"PerplexityBot":["partial","*",6,3],"Perplexity-User":["partial","*",6,3],"Google-Extended":["partial","*",6,3],"Applebot-Extended":["partial","*",6,3],"CCBot":["partial","*",6,3],"Bytespider":["partial","*",6,3],"Amazonbot":["partial","*",6,3],"meta-externalagent":["partial","*",6,3],"cohere-ai":["partial","*",6,3],"Diffbot":["partial","*",6,3],"ContextFetchBot":["partial","*",6,3]},"llms":{"status":"missing","http":404,"bytes":650177},"home":{"status":"http_error","http":403,"url":"https://www.blockchain.com/explorer"}},
{"rank":9241,"domain":"farawaygleese.com","origin":"https://farawaygleese.com","robots":{"status":"timeout","http":null,"url":"https://farawaygleese.com/robots.txt","bytes":0,"groups":0,"error":"timed out after 10 s"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":9242,"domain":"nbcolympics.com","origin":"https://nbcolympics.com","robots":{"status":"ok","http":200,"url":"https://www.nbcolympics.com/robots.txt","bytes":2205,"groups":1},"bots":{"GPTBot":["partial","*",34,18],"ChatGPT-User":["partial","*",34,18],"OAI-SearchBot":["partial","*",34,18],"ClaudeBot":["partial","*",34,18],"Claude-User":["partial","*",34,18],"Claude-SearchBot":["partial","*",34,18],"anthropic-ai":["partial","*",34,18],"PerplexityBot":["partial","*",34,18],"Perplexity-User":["partial","*",34,18],"Google-Extended":["partial","*",34,18],"Applebot-Extended":["partial","*",34,18],"CCBot":["partial","*",34,18],"Bytespider":["partial","*",34,18],"Amazonbot":["partial","*",34,18],"meta-externalagent":["partial","*",34,18],"cohere-ai":["partial","*",34,18],"Diffbot":["partial","*",34,18],"ContextFetchBot":["partial","*",34,18]},"llms":{"status":"missing","http":404,"bytes":193},"home":{"status":"ok","http":200,"url":"https://www.nbcolympics.com/","html_tokens":244083,"markdown_tokens":5,"pruned_tokens":5}},
{"rank":9243,"domain":"dreevoo.com","origin":"https://dreevoo.com","robots":{"status":"ok","http":200,"url":"https://dreevoo.com/robots.txt","bytes":270,"groups":8},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"missing","http":404,"bytes":153},"home":{"status":"ok","http":200,"url":"https://dreevoo.com/","html_tokens":18240,"markdown_tokens":919,"pruned_tokens":49}},
{"rank":9244,"domain":"school4-dinsk.ru","origin":"https://school4-dinsk.ru","robots":{"status":"refused","http":403,"url":"https://school4-dinsk.ru/robots.txt","bytes":4916,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":9245,"domain":"embassy.gov.au","origin":"https://embassy.gov.au","robots":{"status":"missing","http":404,"url":"https://embassy.gov.au/robots.txt","bytes":1606,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":1606},"home":{"status":"disallowed","error":"robots.txt of www.dfat.gov.au not readable (unreachable)"}},
{"rank":9246,"domain":"haixinlai.cn","origin":"http://haixinlai.cn","robots":{"status":"missing","http":404,"url":"http://haixinlai.cn/robots.txt","bytes":4715,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":4711},"home":{"status":"disallowed","error":"robots.txt of stopnote.vhostgo.com not readable (unreachable)"}},
{"rank":9247,"domain":"lshhxy.cn","origin":"https://lshhxy.cn","robots":{"status":"timeout","http":null,"url":"https://lshhxy.cn/robots.txt","bytes":0,"groups":0,"error":"ConnectTimeout"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":9248,"domain":"pl.pinterest.com","origin":"https://pl.pinterest.com","robots":{"status":"ok","http":200,"url":"https://pl.pinterest.com/robots.txt","bytes":32145,"groups":8},"bots":{"GPTBot":["blocked","*",1,0],"ChatGPT-User":["blocked","*",1,0],"OAI-SearchBot":["blocked","*",1,0],"ClaudeBot":["blocked","*",1,0],"Claude-User":["blocked","*",1,0],"Claude-SearchBot":["blocked","*",1,0],"anthropic-ai":["blocked","*",1,0],"PerplexityBot":["blocked","*",1,0],"Perplexity-User":["blocked","*",1,0],"Google-Extended":["blocked","*",1,0],"Applebot-Extended":["blocked","*",1,0],"CCBot":["blocked","*",1,0],"Bytespider":["blocked","*",1,0],"Amazonbot":["blocked","*",1,0],"meta-externalagent":["blocked","*",1,0],"cohere-ai":["blocked","*",1,0],"Diffbot":["blocked","*",1,0],"ContextFetchBot":["blocked","*",1,0]},"llms":{"status":"disallowed"},"home":{"status":"disallowed"}},
{"rank":9249,"domain":"gocardless.com","origin":"https://gocardless.com","robots":{"status":"ok","http":200,"url":"https://gocardless.com/robots.txt","bytes":140,"groups":1},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":107452},"home":{"status":"ok","http":200,"url":"https://gocardless.com/","html_tokens":437705,"markdown_tokens":1299,"pruned_tokens":1106}},
{"rank":9250,"domain":"vietnam.vn","origin":"https://vietnam.vn","robots":{"status":"ok","http":200,"url":"https://www.vietnam.vn/robots.txt","bytes":3628,"groups":51},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["allowed","named",0,1],"OAI-SearchBot":["allowed","named",0,1],"ClaudeBot":["blocked","named",1,0],"Claude-User":["allowed","named",0,1],"Claude-SearchBot":["allowed","named",0,1],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["allowed","named",0,1],"Perplexity-User":["allowed","named",0,1],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["partial","*",2,2],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",2,2]},"llms":{"status":"missing","http":404,"bytes":106543},"home":{"status":"ok","http":200,"url":"https://www.vietnam.vn/","html_tokens":406329,"markdown_tokens":227,"pruned_tokens":163}},
{"rank":9251,"domain":"biometricupdate.com","origin":"https://biometricupdate.com","robots":{"status":"ok","http":200,"url":"https://www.biometricupdate.com/robots.txt","bytes":262,"groups":6},"bots":{"GPTBot":["partial","*",1,1],"ChatGPT-User":["partial","*",1,1],"OAI-SearchBot":["partial","*",1,1],"ClaudeBot":["partial","*",1,1],"Claude-User":["partial","*",1,1],"Claude-SearchBot":["partial","*",1,1],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["partial","*",1,1],"Perplexity-User":["partial","*",1,1],"Google-Extended":["partial","*",1,1],"Applebot-Extended":["partial","*",1,1],"CCBot":["partial","*",1,1],"Bytespider":["blocked","named",1,0],"Amazonbot":["partial","*",1,1],"meta-externalagent":["partial","*",1,1],"cohere-ai":["partial","*",1,1],"Diffbot":["partial","*",1,1],"ContextFetchBot":["partial","*",1,1]},"llms":{"status":"other_text","http":200,"bytes":671276},"home":{"status":"ok","http":200,"url":"https://www.biometricupdate.com/","html_tokens":39252,"markdown_tokens":163,"pruned_tokens":163}},
{"rank":9252,"domain":"basspro.com","origin":"https://basspro.com","robots":{"status":"refused","http":403,"url":"https://basspro.com/robots.txt","bytes":373,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":9253,"domain":"circle.com","origin":"https://circle.com","robots":{"status":"ok","http":200,"url":"https://www.circle.com/robots.txt","bytes":5299,"groups":1},"bots":{"GPTBot":["partial","*",94,0],"ChatGPT-User":["partial","*",94,0],"OAI-SearchBot":["partial","*",94,0],"ClaudeBot":["partial","*",94,0],"Claude-User":["partial","*",94,0],"Claude-SearchBot":["partial","*",94,0],"anthropic-ai":["partial","*",94,0],"PerplexityBot":["partial","*",94,0],"Perplexity-User":["partial","*",94,0],"Google-Extended":["partial","*",94,0],"Applebot-Extended":["partial","*",94,0],"CCBot":["partial","*",94,0],"Bytespider":["partial","*",94,0],"Amazonbot":["partial","*",94,0],"meta-externalagent":["partial","*",94,0],"cohere-ai":["partial","*",94,0],"Diffbot":["partial","*",94,0],"ContextFetchBot":["partial","*",94,0]},"llms":{"status":"present","http":200,"bytes":12258},"home":{"status":"ok","http":200,"url":"https://www.circle.com/","html_tokens":110542,"markdown_tokens":2197,"pruned_tokens":646}},
{"rank":9254,"domain":"bakermckenzie.com","origin":"https://bakermckenzie.com","robots":{"status":"ok","http":200,"url":"https://www.bakermckenzie.com/robots.txt","bytes":1803,"groups":6},"bots":{"GPTBot":["partial","*",3,0],"ChatGPT-User":["partial","*",3,0],"OAI-SearchBot":["partial","*",3,0],"ClaudeBot":["partial","*",3,0],"Claude-User":["partial","*",3,0],"Claude-SearchBot":["partial","*",3,0],"anthropic-ai":["partial","*",3,0],"PerplexityBot":["partial","*",3,0],"Perplexity-User":["partial","*",3,0],"Google-Extended":["partial","*",3,0],"Applebot-Extended":["partial","*",3,0],"CCBot":["partial","*",3,0],"Bytespider":["partial","*",3,0],"Amazonbot":["partial","*",3,0],"meta-externalagent":["partial","*",3,0],"cohere-ai":["partial","*",3,0],"Diffbot":["partial","*",3,0],"ContextFetchBot":["partial","*",3,0]},"llms":{"status":"missing","http":404,"bytes":0},"home":{"status":"ok","http":200,"url":"https://www.bakermckenzie.com/en/","html_tokens":13239,"markdown_tokens":115,"pruned_tokens":115}},
{"rank":9255,"domain":"wpmucdn.com","origin":"http://wpmucdn.com","robots":{"status":"missing","http":404,"url":"https://wpmudev.com/cdn/robots.txt","bytes":297913,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":297913},"home":{"status":"ok","http":200,"url":"https://wpmudev.com/cdn/","html_tokens":140785,"markdown_tokens":533,"pruned_tokens":173}},
{"rank":9256,"domain":"zengyuan.org","origin":"http://zengyuan.org","robots":{"status":"missing","http":404,"url":"http://zengyuan.org/robots.txt","bytes":138,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":138},"home":{"status":"not_html","error":"unsupported content type application/octet-stream"}},
{"rank":9257,"domain":"winespectator.com","origin":"https://winespectator.com","robots":{"status":"ok","http":200,"url":"https://www.winespectator.com/robots.txt","bytes":725,"groups":16},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["blocked","named",1,0],"OAI-SearchBot":["blocked","named",1,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",5,0],"Claude-SearchBot":["partial","*",5,0],"anthropic-ai":["partial","*",5,0],"PerplexityBot":["partial","*",5,0],"Perplexity-User":["partial","*",5,0],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["partial","*",5,0],"CCBot":["blocked","named",1,0],"Bytespider":["partial","*",5,0],"Amazonbot":["partial","*",5,0],"meta-externalagent":["partial","*",5,0],"cohere-ai":["partial","*",5,0],"Diffbot":["partial","*",5,0],"ContextFetchBot":["partial","*",5,0]},"llms":{"status":"missing","http":404,"bytes":87637},"home":{"status":"ok","http":200,"url":"https://www.winespectator.com/","html_tokens":71646,"markdown_tokens":43,"pruned_tokens":43}},
{"rank":9258,"domain":"uniovi.es","origin":"https://www.uniovi.es","robots":{"status":"ok","http":200,"url":"https://www.uniovi.es/robots.txt","bytes":301,"groups":1},"bots":{"GPTBot":["partial","*",11,0],"ChatGPT-User":["partial","*",11,0],"OAI-SearchBot":["partial","*",11,0],"ClaudeBot":["partial","*",11,0],"Claude-User":["partial","*",11,0],"Claude-SearchBot":["partial","*",11,0],"anthropic-ai":["partial","*",11,0],"PerplexityBot":["partial","*",11,0],"Perplexity-User":["partial","*",11,0],"Google-Extended":["partial","*",11,0],"Applebot-Extended":["partial","*",11,0],"CCBot":["partial","*",11,0],"Bytespider":["partial","*",11,0],"Amazonbot":["partial","*",11,0],"meta-externalagent":["partial","*",11,0],"cohere-ai":["partial","*",11,0],"Diffbot":["partial","*",11,0],"ContextFetchBot":["partial","*",11,0]},"llms":{"status":"missing","http":404,"bytes":649},"home":{"status":"ok","http":200,"url":"https://www.uniovi.es/","html_tokens":46008,"markdown_tokens":79,"pruned_tokens":79}},
{"rank":9259,"domain":"icom.museum","origin":"https://icom.museum","robots":{"status":"ok","http":200,"url":"https://icom.museum/robots.txt","bytes":172,"groups":1},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"missing","http":404,"bytes":116467},"home":{"status":"ok","http":200,"url":"https://icom.museum/en/","html_tokens":47183,"markdown_tokens":310,"pruned_tokens":310}},
{"rank":9260,"domain":"shopee.vn","origin":"https://shopee.vn","robots":{"status":"ok","http":200,"url":"https://shopee.vn/robots.txt","bytes":1808,"groups":2},"bots":{"GPTBot":["partial","*",16,0],"ChatGPT-User":["partial","*",16,0],"OAI-SearchBot":["partial","*",16,0],"ClaudeBot":["partial","*",16,0],"Claude-User":["partial","*",16,0],"Claude-SearchBot":["partial","*",16,0],"anthropic-ai":["partial","*",16,0],"PerplexityBot":["partial","*",16,0],"Perplexity-User":["partial","*",16,0],"Google-Extended":["partial","*",16,0],"Applebot-Extended":["partial","*",16,0],"CCBot":["partial","*",16,0],"Bytespider":["partial","*",16,0],"Amazonbot":["partial","*",16,0],"meta-externalagent":["partial","*",16,0],"cohere-ai":["partial","*",16,0],"Diffbot":["partial","*",16,0],"ContextFetchBot":["partial","*",16,0]},"llms":{"status":"html","http":200,"bytes":202937},"home":{"status":"ok","http":200,"url":"https://shopee.vn/","html_tokens":50735,"markdown_tokens":11,"pruned_tokens":11}},
{"rank":9261,"domain":"docsend.com","origin":"https://docsend.com","robots":{"status":"refused","http":403,"url":"https://docsend.com/robots.txt","bytes":919,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":9262,"domain":"unimib.it","origin":"https://www.unimib.it","robots":{"status":"ok","http":200,"url":"https://www.unimib.it/robots.txt","bytes":1973,"groups":1},"bots":{"GPTBot":["partial","*",26,18],"ChatGPT-User":["partial","*",26,18],"OAI-SearchBot":["partial","*",26,18],"ClaudeBot":["partial","*",26,18],"Claude-User":["partial","*",26,18],"Claude-SearchBot":["partial","*",26,18],"anthropic-ai":["partial","*",26,18],"PerplexityBot":["partial","*",26,18],"Perplexity-User":["partial","*",26,18],"Google-Extended":["partial","*",26,18],"Applebot-Extended":["partial","*",26,18],"CCBot":["partial","*",26,18],"Bytespider":["partial","*",26,18],"Amazonbot":["partial","*",26,18],"meta-externalagent":["partial","*",26,18],"cohere-ai":["partial","*",26,18],"Diffbot":["partial","*",26,18],"ContextFetchBot":["partial","*",26,18]},"llms":{"status":"missing","http":404,"bytes":548},"home":{"status":"ok","http":200,"url":"https://www.unimib.it/","html_tokens":27586,"markdown_tokens":296,"pruned_tokens":170}},
{"rank":9263,"domain":"tympanus.net","origin":"https://tympanus.net","robots":{"status":"ok","http":200,"url":"https://tympanus.net/robots.txt","bytes":2791,"groups":33},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["partial","*",2,1],"OAI-SearchBot":["partial","*",2,1],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",2,1],"Claude-SearchBot":["partial","*",2,1],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["partial","*",2,1],"Perplexity-User":["partial","*",2,1],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",2,1]},"llms":{"status":"missing","http":404,"bytes":1441},"home":{"status":"http_error","http":403,"url":"https://tympanus.net/"}},
{"rank":9264,"domain":"cyysj.com","origin":"http://cyysj.com","robots":{"status":"ok","http":200,"url":"http://cyysj.com/robots.txt","bytes":249,"groups":1},"bots":{"GPTBot":["partial","*",10,0],"ChatGPT-User":["partial","*",10,0],"OAI-SearchBot":["partial","*",10,0],"ClaudeBot":["partial","*",10,0],"Claude-User":["partial","*",10,0],"Claude-SearchBot":["partial","*",10,0],"anthropic-ai":["partial","*",10,0],"PerplexityBot":["partial","*",10,0],"Perplexity-User":["partial","*",10,0],"Google-Extended":["partial","*",10,0],"Applebot-Extended":["partial","*",10,0],"CCBot":["partial","*",10,0],"Bytespider":["partial","*",10,0],"Amazonbot":["partial","*",10,0],"meta-externalagent":["partial","*",10,0],"cohere-ai":["partial","*",10,0],"Diffbot":["partial","*",10,0],"ContextFetchBot":["partial","*",10,0]},"llms":{"status":"missing","http":404,"bytes":3749},"home":{"status":"ok","http":200,"url":"http://cyysj.com/","html_tokens":12826,"markdown_tokens":68,"pruned_tokens":68}},
{"rank":9265,"domain":"wxkjpump.cn","origin":"http://wxkjpump.cn","robots":{"status":"ok","http":200,"url":"http://wxkjpump.cn/robots.txt","bytes":249,"groups":1},"bots":{"GPTBot":["partial","*",10,0],"ChatGPT-User":["partial","*",10,0],"OAI-SearchBot":["partial","*",10,0],"ClaudeBot":["partial","*",10,0],"Claude-User":["partial","*",10,0],"Claude-SearchBot":["partial","*",10,0],"anthropic-ai":["partial","*",10,0],"PerplexityBot":["partial","*",10,0],"Perplexity-User":["partial","*",10,0],"Google-Extended":["partial","*",10,0],"Applebot-Extended":["partial","*",10,0],"CCBot":["partial","*",10,0],"Bytespider":["partial","*",10,0],"Amazonbot":["partial","*",10,0],"meta-externalagent":["partial","*",10,0],"cohere-ai":["partial","*",10,0],"Diffbot":["partial","*",10,0],"ContextFetchBot":["partial","*",10,0]},"llms":{"status":"html","http":200,"bytes":3747},"home":{"status":"ok","http":200,"url":"http://wxkjpump.cn/","html_tokens":2482,"markdown_tokens":128,"pruned_tokens":128}},
{"rank":9266,"domain":"uni-oldenburg.de","origin":"https://uni-oldenburg.de","robots":{"status":"html","http":200,"url":"https://uol.de/","bytes":163647,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":9267,"domain":"wetter.com","origin":"https://wetter.com","robots":{"status":"ok","http":200,"url":"https://www.wetter.com/robots.txt","bytes":5022,"groups":7},"bots":{"GPTBot":["partial","*",62,61],"ChatGPT-User":["partial","*",62,61],"OAI-SearchBot":["partial","*",62,61],"ClaudeBot":["partial","*",62,61],"Claude-User":["partial","*",62,61],"Claude-SearchBot":["partial","*",62,61],"anthropic-ai":["partial","*",62,61],"PerplexityBot":["partial","*",62,61],"Perplexity-User":["partial","*",62,61],"Google-Extended":["partial","*",62,61],"Applebot-Extended":["partial","*",62,61],"CCBot":["partial","*",62,61],"Bytespider":["partial","*",62,61],"Amazonbot":["partial","*",62,61],"meta-externalagent":["partial","*",62,61],"cohere-ai":["partial","*",62,61],"Diffbot":["partial","*",62,61],"ContextFetchBot":["partial","*",62,61]},"llms":{"status":"missing","http":404,"bytes":216436},"home":{"status":"ok","http":200,"url":"https://www.wetter.com/","html_tokens":123164,"markdown_tokens":1217,"pruned_tokens":463}},
{"rank":9268,"domain":"stellarium.org","origin":"https://stellarium.org","robots":{"status":"missing","http":404,"url":"https://stellarium.org/robots.txt","bytes":7876,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":7876},"home":{"status":"ok","http":200,"url":"https://stellarium.org/","html_tokens":6379,"markdown_tokens":1619,"pruned_tokens":131}},
{"rank":9269,"domain":"ppc.go.jp","origin":"https://www.ppc.go.jp","robots":{"status":"ok","http":200,"url":"https://www.ppc.go.jp/robots.txt","bytes":205,"groups":2},"bots":{"GPTBot":["partial","*",5,0],"ChatGPT-User":["partial","*",5,0],"OAI-SearchBot":["partial","*",5,0],"ClaudeBot":["partial","*",5,0],"Claude-User":["partial","*",5,0],"Claude-SearchBot":["partial","*",5,0],"anthropic-ai":["partial","*",5,0],"PerplexityBot":["partial","*",5,0],"Perplexity-User":["partial","*",5,0],"Google-Extended":["partial","*",5,0],"Applebot-Extended":["partial","*",5,0],"CCBot":["partial","*",5,0],"Bytespider":["partial","*",5,0],"Amazonbot":["partial","*",5,0],"meta-externalagent":["partial","*",5,0],"cohere-ai":["partial","*",5,0],"Diffbot":["partial","*",5,0],"ContextFetchBot":["partial","*",5,0]},"llms":{"status":"missing","http":404,"bytes":39126},"home":{"status":"ok","http":200,"url":"https://www.ppc.go.jp/","html_tokens":20427,"markdown_tokens":43,"pruned_tokens":43}},
{"rank":9270,"domain":"icaew.com","origin":"https://icaew.com","robots":{"status":"ok","http":200,"url":"https://www.icaew.com/robots.txt","bytes":723,"groups":1},"bots":{"GPTBot":["partial","*",16,0],"ChatGPT-User":["partial","*",16,0],"OAI-SearchBot":["partial","*",16,0],"ClaudeBot":["partial","*",16,0],"Claude-User":["partial","*",16,0],"Claude-SearchBot":["partial","*",16,0],"anthropic-ai":["partial","*",16,0],"PerplexityBot":["partial","*",16,0],"Perplexity-User":["partial","*",16,0],"Google-Extended":["partial","*",16,0],"Applebot-Extended":["partial","*",16,0],"CCBot":["partial","*",16,0],"Bytespider":["partial","*",16,0],"Amazonbot":["partial","*",16,0],"meta-externalagent":["partial","*",16,0],"cohere-ai":["partial","*",16,0],"Diffbot":["partial","*",16,0],"ContextFetchBot":["partial","*",16,0]},"llms":{"status":"missing","http":404,"bytes":479313},"home":{"status":"ok","http":200,"url":"https://www.icaew.com/","html_tokens":128139,"markdown_tokens":414,"pruned_tokens":141}},
{"rank":9271,"domain":"orpha.net","origin":"https://www.orpha.net","robots":{"status":"html","http":200,"url":"https://www.orpha.net/robots.txt","bytes":5349,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":9272,"domain":"sinaimg.cn","origin":"https://sinaimg.cn","robots":{"status":"missing","http":404,"url":"https://sinaimg.cn/robots.txt","bytes":14,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":14},"home":{"status":"ok","http":200,"url":"https://sinaimg.cn/","html_tokens":63,"markdown_tokens":5,"pruned_tokens":5}},
{"rank":9273,"domain":"zgyxlc.com","origin":"https://zgyxlc.com","robots":{"status":"missing","http":404,"url":"https://zgyxlc.com/robots.txt","bytes":315,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":315},"home":{"status":"http_error","http":404,"url":"https://zgyxlc.com/"}},
{"rank":9274,"domain":"igamingbusiness.com","origin":"https://igamingbusiness.com","robots":{"status":"ok","http":200,"url":"https://igamingbusiness.com/robots.txt","bytes":235,"groups":1},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"missing","http":404,"bytes":366278},"home":{"status":"ok","http":200,"url":"https://igamingbusiness.com/","html_tokens":156969,"markdown_tokens":476,"pruned_tokens":210}},
{"rank":9275,"domain":"cargill.com","origin":"https://cargill.com","robots":{"status":"refused","http":403,"url":"https://www.cargill.com/robots.txt","bytes":383,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":9276,"domain":"komen.org","origin":"https://komen.org","robots":{"status":"refused","http":403,"url":"https://komen.org/robots.txt","bytes":1345,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":9277,"domain":"home.ne.jp","origin":"https://home.ne.jp","robots":{"status":"unreachable","http":null,"url":"https://home.ne.jp/robots.txt","bytes":0,"groups":0,"error":"cannot resolve home.ne.jp"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":9278,"domain":"hansard.parliament.uk","origin":"https://hansard.parliament.uk","robots":{"status":"refused","http":403,"url":"https://hansard.parliament.uk/robots.txt","bytes":5559,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":9279,"domain":"globalvoices.org","origin":"https://globalvoices.org","robots":{"status":"ok","http":200,"url":"https://globalvoices.org/robots.txt","bytes":296,"groups":1},"bots":{"GPTBot":["partial","*",3,0],"ChatGPT-User":["partial","*",3,0],"OAI-SearchBot":["partial","*",3,0],"ClaudeBot":["partial","*",3,0],"Claude-User":["partial","*",3,0],"Claude-SearchBot":["partial","*",3,0],"anthropic-ai":["partial","*",3,0],"PerplexityBot":["partial","*",3,0],"Perplexity-User":["partial","*",3,0],"Google-Extended":["partial","*",3,0],"Applebot-Extended":["partial","*",3,0],"CCBot":["partial","*",3,0],"Bytespider":["partial","*",3,0],"Amazonbot":["partial","*",3,0],"meta-externalagent":["partial","*",3,0],"cohere-ai":["partial","*",3,0],"Diffbot":["partial","*",3,0],"ContextFetchBot":["partial","*",3,0]},"llms":{"status":"missing","http":404,"bytes":120840},"home":{"status":"ok","http":200,"url":"https://globalvoices.org/","html_tokens":46429,"markdown_tokens":335,"pruned_tokens":155}},
{"rank":9280,"domain":"personio.de","origin":"https://personio.de","robots":{"status":"refused","http":429,"url":"https://personio.de/robots.txt","bytes":34218,"groups":0,"error":"HTTP 429"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":9281,"domain":"live365.com","origin":"https://live365.com","robots":{"status":"ok","http":200,"url":"https://live365.com/robots.txt","bytes":201,"groups":1},"bots":{"GPTBot":["partial","*",5,0],"ChatGPT-User":["partial","*",5,0],"OAI-SearchBot":["partial","*",5,0],"ClaudeBot":["partial","*",5,0],"Claude-User":["partial","*",5,0],"Claude-SearchBot":["partial","*",5,0],"anthropic-ai":["partial","*",5,0],"PerplexityBot":["partial","*",5,0],"Perplexity-User":["partial","*",5,0],"Google-Extended":["partial","*",5,0],"Applebot-Extended":["partial","*",5,0],"CCBot":["partial","*",5,0],"Bytespider":["partial","*",5,0],"Amazonbot":["partial","*",5,0],"meta-externalagent":["partial","*",5,0],"cohere-ai":["partial","*",5,0],"Diffbot":["partial","*",5,0],"ContextFetchBot":["partial","*",5,0]},"llms":{"status":"html","http":200,"bytes":247489},"home":{"status":"ok","http":200,"url":"https://live365.com/","html_tokens":61870,"markdown_tokens":132,"pruned_tokens":95}},
{"rank":9282,"domain":"naderman.de","origin":"https://naderman.de","robots":{"status":"ok","http":200,"url":"https://naderman.de/robots.txt","bytes":0,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":274},"home":{"status":"ok","http":200,"url":"https://naderman.de/","html_tokens":3588,"markdown_tokens":99,"pruned_tokens":99}},
{"rank":9283,"domain":"sportsbusinessjournal.com","origin":"https://sportsbusinessjournal.com","robots":{"status":"ok","http":200,"url":"https://www.sportsbusinessjournal.com/robots.txt","bytes":6488,"groups":93},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["blocked","named",1,0],"OAI-SearchBot":["blocked","named",1,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["blocked","named",1,0],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["blocked","named",1,0],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":112161},"home":{"status":"ok","http":200,"url":"https://www.sportsbusinessjournal.com/","html_tokens":142729,"markdown_tokens":179,"pruned_tokens":70}},
{"rank":9284,"domain":"bancaditalia.it","origin":"https://www.bancaditalia.it","robots":{"status":"ok","http":200,"url":"https://www.bancaditalia.it/homepage/robots.txt","bytes":82986,"groups":7},"bots":{"GPTBot":["partial","*",814,0],"ChatGPT-User":["partial","*",814,0],"OAI-SearchBot":["partial","*",814,0],"ClaudeBot":["partial","*",814,0],"Claude-User":["partial","*",814,0],"Claude-SearchBot":["partial","*",814,0],"anthropic-ai":["partial","*",814,0],"PerplexityBot":["partial","*",814,0],"Perplexity-User":["partial","*",814,0],"Google-Extended":["partial","*",814,0],"Applebot-Extended":["partial","*",814,0],"CCBot":["partial","*",814,0],"Bytespider":["partial","*",814,0],"Amazonbot":["partial","*",814,0],"meta-externalagent":["partial","*",814,0],"cohere-ai":["partial","*",814,0],"Diffbot":["partial","*",814,0],"ContextFetchBot":["partial","*",814,0]},"llms":{"status":"missing","http":404,"bytes":86299},"home":{"status":"ok","http":200,"url":"https://www.bancaditalia.it/","html_tokens":40820,"markdown_tokens":542,"pruned_tokens":77}},
{"rank":9285,"domain":"magnumphotos.com","origin":"https://magnumphotos.com","robots":{"status":"ok","http":200,"url":"https://www.magnumphotos.com/robots.txt","bytes":67,"groups":1},"bots":{"GPTBot":["partial","*",1,1],"ChatGPT-User":["partial","*",1,1],"OAI-SearchBot":["partial","*",1,1],"ClaudeBot":["partial","*",1,1],"Claude-User":["partial","*",1,1],"Claude-SearchBot":["partial","*",1,1],"anthropic-ai":["partial","*",1,1],"PerplexityBot":["partial","*",1,1],"Perplexity-User":["partial","*",1,1],"Google-Extended":["partial","*",1,1],"Applebot-Extended":["partial","*",1,1],"CCBot":["partial","*",1,1],"Bytespider":["partial","*",1,1],"Amazonbot":["partial","*",1,1],"meta-externalagent":["partial","*",1,1],"cohere-ai":["partial","*",1,1],"Diffbot":["partial","*",1,1],"ContextFetchBot":["partial","*",1,1]},"llms":{"status":"missing","http":404,"bytes":94575},"home":{"status":"ok","http":200,"url":"https://www.magnumphotos.com/","html_tokens":27441,"markdown_tokens":147,"pruned_tokens":115}},
{"rank":9286,"domain":"returntoschoolroadmap.org","origin":"https://returntoschoolroadmap.org","robots":{"status":"refused","http":403,"url":"https://returntoschoolroadmap.org/robots.txt","bytes":4925,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":9287,"domain":"asahi.co.jp","origin":"https://asahi.co.jp","robots":{"status":"ok","http":200,"url":"https://asahi.co.jp/robots.txt","bytes":177,"groups":1},"bots":{"GPTBot":["partial","*",2,0],"ChatGPT-User":["partial","*",2,0],"OAI-SearchBot":["partial","*",2,0],"ClaudeBot":["partial","*",2,0],"Claude-User":["partial","*",2,0],"Claude-SearchBot":["partial","*",2,0],"anthropic-ai":["partial","*",2,0],"PerplexityBot":["partial","*",2,0],"Perplexity-User":["partial","*",2,0],"Google-Extended":["partial","*",2,0],"Applebot-Extended":["partial","*",2,0],"CCBot":["partial","*",2,0],"Bytespider":["partial","*",2,0],"Amazonbot":["partial","*",2,0],"meta-externalagent":["partial","*",2,0],"cohere-ai":["partial","*",2,0],"Diffbot":["partial","*",2,0],"ContextFetchBot":["partial","*",2,0]},"llms":{"status":"missing","http":404,"bytes":4727},"home":{"status":"ok","http":200,"url":"https://asahi.co.jp/","html_tokens":12255,"markdown_tokens":11,"pruned_tokens":11}},
{"rank":9288,"domain":"chicagoreader.com","origin":"https://chicagoreader.com","robots":{"status":"ok","http":200,"url":"https://chicagoreader.com/robots.txt","bytes":1728,"groups":22},"bots":{"GPTBot":["partial","named",1,1],"ChatGPT-User":["partial","named",1,1],"OAI-SearchBot":["blocked","named",1,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["blocked","named",1,0],"Claude-SearchBot":["blocked","named",1,0],"anthropic-ai":["partial","named",1,1],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["partial","*",21,0],"Google-Extended":["partial","named",1,1],"Applebot-Extended":["partial","*",21,0],"CCBot":["partial","named",1,1],"Bytespider":["blocked","named",1,0],"Amazonbot":["partial","*",21,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["partial","named",1,1],"Diffbot":["partial","*",21,0],"ContextFetchBot":["partial","*",21,0]},"llms":{"status":"missing","http":404,"bytes":123442},"home":{"status":"ok","http":200,"url":"https://chicagoreader.com/","html_tokens":89571,"markdown_tokens":138,"pruned_tokens":74}},
{"rank":9289,"domain":"royalgames.casino","origin":"https://royalgames.casino","robots":{"status":"ok","http":200,"url":"https://royalgames.casino/robots.txt","bytes":67,"groups":1},"bots":{"GPTBot":["partial","*",1,1],"ChatGPT-User":["partial","*",1,1],"OAI-SearchBot":["partial","*",1,1],"ClaudeBot":["partial","*",1,1],"Claude-User":["partial","*",1,1],"Claude-SearchBot":["partial","*",1,1],"anthropic-ai":["partial","*",1,1],"PerplexityBot":["partial","*",1,1],"Perplexity-User":["partial","*",1,1],"Google-Extended":["partial","*",1,1],"Applebot-Extended":["partial","*",1,1],"CCBot":["partial","*",1,1],"Bytespider":["partial","*",1,1],"Amazonbot":["partial","*",1,1],"meta-externalagent":["partial","*",1,1],"cohere-ai":["partial","*",1,1],"Diffbot":["partial","*",1,1],"ContextFetchBot":["partial","*",1,1]},"llms":{"status":"missing","http":404,"bytes":51015},"home":{"status":"ok","http":200,"url":"https://royalgames.casino/","html_tokens":17872,"markdown_tokens":2262,"pruned_tokens":306}},
{"rank":9290,"domain":"dx.com","origin":"http://dx.com","robots":{"status":"missing","http":404,"url":"http://dx.com/robots.txt","bytes":0,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":0},"home":{"status":"not_html","error":"unsupported content type text/markdown"}},
{"rank":9291,"domain":"petco.com","origin":"https://petco.com","robots":{"status":"refused","http":403,"url":"https://petco.com/robots.txt","bytes":4764,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":9292,"domain":"lsm.lv","origin":"https://lsm.lv","robots":{"status":"ok","http":200,"url":"https://www.lsm.lv/robots.txt","bytes":73,"groups":1},"bots":{"GPTBot":["partial","*",1,0],"ChatGPT-User":["partial","*",1,0],"OAI-SearchBot":["partial","*",1,0],"ClaudeBot":["partial","*",1,0],"Claude-User":["partial","*",1,0],"Claude-SearchBot":["partial","*",1,0],"anthropic-ai":["partial","*",1,0],"PerplexityBot":["partial","*",1,0],"Perplexity-User":["partial","*",1,0],"Google-Extended":["partial","*",1,0],"Applebot-Extended":["partial","*",1,0],"CCBot":["partial","*",1,0],"Bytespider":["partial","*",1,0],"Amazonbot":["partial","*",1,0],"meta-externalagent":["partial","*",1,0],"cohere-ai":["partial","*",1,0],"Diffbot":["partial","*",1,0],"ContextFetchBot":["partial","*",1,0]},"llms":{"status":"missing","http":404,"bytes":85132},"home":{"status":"ok","http":200,"url":"https://www.lsm.lv/","html_tokens":87709,"markdown_tokens":107,"pruned_tokens":107}},
{"rank":9293,"domain":"dallasobserver.com","origin":"https://dallasobserver.com","robots":{"status":"ok","http":200,"url":"https://www.dallasobserver.com/robots.txt","bytes":1418,"groups":2},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["blocked","named",1,0],"OAI-SearchBot":["blocked","named",1,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["blocked","named",1,0],"Claude-SearchBot":["blocked","named",1,0],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["blocked","named",1,0],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",1,1]},"llms":{"status":"missing","http":404,"bytes":115598},"home":{"status":"ok","http":200,"url":"https://www.dallasobserver.com/","html_tokens":73167,"markdown_tokens":62,"pruned_tokens":62}},
{"rank":9294,"domain":"neo4j.com","origin":"https://neo4j.com","robots":{"status":"ok","http":200,"url":"https://neo4j.com/robots.txt","bytes":1265,"groups":1},"bots":{"GPTBot":["partial","*",11,23],"ChatGPT-User":["partial","*",11,23],"OAI-SearchBot":["partial","*",11,23],"ClaudeBot":["partial","*",11,23],"Claude-User":["partial","*",11,23],"Claude-SearchBot":["partial","*",11,23],"anthropic-ai":["partial","*",11,23],"PerplexityBot":["partial","*",11,23],"Perplexity-User":["partial","*",11,23],"Google-Extended":["partial","*",11,23],"Applebot-Extended":["partial","*",11,23],"CCBot":["partial","*",11,23],"Bytespider":["partial","*",11,23],"Amazonbot":["partial","*",11,23],"meta-externalagent":["partial","*",11,23],"cohere-ai":["partial","*",11,23],"Diffbot":["partial","*",11,23],"ContextFetchBot":["partial","*",11,23]},"llms":{"status":"present","http":200,"bytes":15308},"home":{"status":"ok","http":200,"url":"https://neo4j.com/","html_tokens":71336,"markdown_tokens":501,"pruned_tokens":93}},
{"rank":9295,"domain":"centrum.cz","origin":"https://centrum.cz","robots":{"status":"ok","http":200,"url":"https://www.centrum.cz/robots.txt","bytes":64,"groups":1},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"missing","http":404,"bytes":663320},"home":{"status":"ok","http":200,"url":"https://www.centrum.cz/","html_tokens":386843,"markdown_tokens":81,"pruned_tokens":81}},
{"rank":9296,"domain":"brocku.ca","origin":"https://brocku.ca","robots":{"status":"refused","http":403,"url":"https://brocku.ca/robots.txt","bytes":5610,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":9297,"domain":"aber.ac.uk","origin":"https://aber.ac.uk","robots":{"status":"ok","http":200,"url":"https://www.aber.ac.uk/robots.txt","bytes":871,"groups":1},"bots":{"GPTBot":["partial","*",18,0],"ChatGPT-User":["partial","*",18,0],"OAI-SearchBot":["partial","*",18,0],"ClaudeBot":["partial","*",18,0],"Claude-User":["partial","*",18,0],"Claude-SearchBot":["partial","*",18,0],"anthropic-ai":["partial","*",18,0],"PerplexityBot":["partial","*",18,0],"Perplexity-User":["partial","*",18,0],"Google-Extended":["partial","*",18,0],"Applebot-Extended":["partial","*",18,0],"CCBot":["partial","*",18,0],"Bytespider":["partial","*",18,0],"Amazonbot":["partial","*",18,0],"meta-externalagent":["partial","*",18,0],"cohere-ai":["partial","*",18,0],"Diffbot":["partial","*",18,0],"ContextFetchBot":["partial","*",18,0]},"llms":{"status":"missing","http":404,"bytes":41379},"home":{"status":"ok","http":200,"url":"https://www.aber.ac.uk/en/","html_tokens":9541,"markdown_tokens":184,"pruned_tokens":147}},
{"rank":9298,"domain":"gl-cs.com","origin":"http://gl-cs.com","robots":{"status":"refused","http":403,"url":"http://gl-cs.com/robots.txt","bytes":1157,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":9299,"domain":"forbesindia.com","origin":"https://www.forbesindia.com","robots":{"status":"ok","http":200,"url":"https://www.forbesindia.com/robots.txt","bytes":366,"groups":1},"bots":{"GPTBot":["partial","*",8,1],"ChatGPT-User":["partial","*",8,1],"OAI-SearchBot":["partial","*",8,1],"ClaudeBot":["partial","*",8,1],"Claude-User":["partial","*",8,1],"Claude-SearchBot":["partial","*",8,1],"anthropic-ai":["partial","*",8,1],"PerplexityBot":["partial","*",8,1],"Perplexity-User":["partial","*",8,1],"Google-Extended":["partial","*",8,1],"Applebot-Extended":["partial","*",8,1],"CCBot":["partial","*",8,1],"Bytespider":["partial","*",8,1],"Amazonbot":["partial","*",8,1],"meta-externalagent":["partial","*",8,1],"cohere-ai":["partial","*",8,1],"Diffbot":["partial","*",8,1],"ContextFetchBot":["partial","*",8,1]},"llms":{"status":"missing","http":404,"bytes":358882},"home":{"status":"ok","http":200,"url":"https://www.forbesindia.com/","html_tokens":109137,"markdown_tokens":101,"pruned_tokens":101}},
{"rank":9300,"domain":"miamioh.edu","origin":"https://miamioh.edu","robots":{"status":"ok","http":200,"url":"https://miamioh.edu/robots.txt","bytes":750,"groups":18},"bots":{"GPTBot":["allowed","named",0,1],"ChatGPT-User":["allowed","named",0,1],"OAI-SearchBot":["allowed","named",0,1],"ClaudeBot":["allowed","named",0,1],"Claude-User":["allowed","*",0,1],"Claude-SearchBot":["allowed","*",0,1],"anthropic-ai":["allowed","named",0,1],"PerplexityBot":["allowed","named",0,1],"Perplexity-User":["allowed","*",0,1],"Google-Extended":["allowed","named",0,1],"Applebot-Extended":["allowed","*",0,1],"CCBot":["allowed","*",0,1],"Bytespider":["allowed","*",0,1],"Amazonbot":["allowed","*",0,1],"meta-externalagent":["allowed","*",0,1],"cohere-ai":["allowed","*",0,1],"Diffbot":["allowed","*",0,1],"ContextFetchBot":["allowed","*",0,1]},"llms":{"status":"present","http":200,"bytes":15513},"home":{"status":"ok","http":200,"url":"https://miamioh.edu/","html_tokens":37027,"markdown_tokens":338,"pruned_tokens":89}},
{"rank":9301,"domain":"csus.edu","origin":"https://csus.edu","robots":{"status":"ok","http":200,"url":"https://www.csus.edu/robots.txt","bytes":361,"groups":1},"bots":{"GPTBot":["partial","*",18,0],"ChatGPT-User":["partial","*",18,0],"OAI-SearchBot":["partial","*",18,0],"ClaudeBot":["partial","*",18,0],"Claude-User":["partial","*",18,0],"Claude-SearchBot":["partial","*",18,0],"anthropic-ai":["partial","*",18,0],"PerplexityBot":["partial","*",18,0],"Perplexity-User":["partial","*",18,0],"Google-Extended":["partial","*",18,0],"Applebot-Extended":["partial","*",18,0],"CCBot":["partial","*",18,0],"Bytespider":["partial","*",18,0],"Amazonbot":["partial","*",18,0],"meta-externalagent":["partial","*",18,0],"cohere-ai":["partial","*",18,0],"Diffbot":["partial","*",18,0],"ContextFetchBot":["partial","*",18,0]},"llms":{"status":"missing","http":404,"bytes":27645},"home":{"status":"ok","http":200,"url":"https://www.csus.edu/","html_tokens":11662,"markdown_tokens":180,"pruned_tokens":159}},
{"rank":9302,"domain":"jskingtex.com","origin":"https://jskingtex.com","robots":{"status":"ok","http":200,"url":"https://jskingtex.com/robots.txt","bytes":485,"groups":1},"bots":{"GPTBot":["partial","*",21,0],"ChatGPT-User":["partial","*",21,0],"OAI-SearchBot":["partial","*",21,0],"ClaudeBot":["partial","*",21,0],"Claude-User":["partial","*",21,0],"Claude-SearchBot":["partial","*",21,0],"anthropic-ai":["partial","*",21,0],"PerplexityBot":["partial","*",21,0],"Perplexity-User":["partial","*",21,0],"Google-Extended":["partial","*",21,0],"Applebot-Extended":["partial","*",21,0],"CCBot":["partial","*",21,0],"Bytespider":["partial","*",21,0],"Amazonbot":["partial","*",21,0],"meta-externalagent":["partial","*",21,0],"cohere-ai":["partial","*",21,0],"Diffbot":["partial","*",21,0],"ContextFetchBot":["partial","*",21,0]},"llms":{"status":"missing","http":404,"bytes":4730},"home":{"status":"ok","http":200,"url":"https://jskingtex.com/","html_tokens":10474,"markdown_tokens":92,"pruned_tokens":92}},
{"rank":9303,"domain":"phillymag.com","origin":"https://phillymag.com","robots":{"status":"ok","http":200,"url":"https://www.phillymag.com/robots.txt","bytes":718,"groups":3},"bots":{"GPTBot":["partial","*",17,0],"ChatGPT-User":["partial","*",17,0],"OAI-SearchBot":["partial","*",17,0],"ClaudeBot":["partial","*",17,0],"Claude-User":["partial","*",17,0],"Claude-SearchBot":["partial","*",17,0],"anthropic-ai":["partial","*",17,0],"PerplexityBot":["partial","*",17,0],"Perplexity-User":["partial","*",17,0],"Google-Extended":["partial","*",17,0],"Applebot-Extended":["partial","*",17,0],"CCBot":["partial","*",17,0],"Bytespider":["partial","*",17,0],"Amazonbot":["partial","*",17,0],"meta-externalagent":["partial","*",17,0],"cohere-ai":["partial","*",17,0],"Diffbot":["partial","*",17,0],"ContextFetchBot":["partial","*",17,0]},"llms":{"status":"timeout","error":"timed out after 10 s"},"home":{"status":"disallowed","error":"robots.txt of www.phillymag.com not readable (timeout)"}},
{"rank":9304,"domain":"lohud.com","origin":"https://lohud.com","robots":{"status":"ok","http":200,"url":"https://www.lohud.com/robots.txt","bytes":12647,"groups":288},"bots":{"GPTBot":["partial","named",1,4],"ChatGPT-User":["partial","named",1,4],"OAI-SearchBot":["partial","named",1,4],"ClaudeBot":["blocked","named",1,0],"Claude-User":["blocked","named",1,0],"Claude-SearchBot":["blocked","named",1,0],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["partial","named",1,4],"Perplexity-User":["blocked","named",1,0],"Google-Extended":["partial","named",1,4],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",2,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",32,0]},"llms":{"status":"http_error","http":402,"bytes":254},"home":{"status":"http_error","http":402,"url":"https://www.lohud.com/"}},
{"rank":9305,"domain":"shacknews.com","origin":"https://shacknews.com","robots":{"status":"ok","http":200,"url":"https://www.shacknews.com/robots.txt","bytes":1001,"groups":4},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["partial","*",18,1],"OAI-SearchBot":["partial","*",18,1],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",18,1],"Claude-SearchBot":["partial","*",18,1],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["partial","*",18,1],"Perplexity-User":["partial","*",18,1],"Google-Extended":["partial","*",18,1],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["partial","*",18,1],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",18,1]},"llms":{"status":"missing","http":404,"bytes":390},"home":{"status":"ok","http":200,"url":"https://www.shacknews.com/","html_tokens":24740,"markdown_tokens":113,"pruned_tokens":113}},
{"rank":9306,"domain":"vnu.edu.vn","origin":"https://vnu.edu.vn","robots":{"status":"ok","http":200,"url":"https://vnu.edu.vn/robots.txt","bytes":206,"groups":1},"bots":{"GPTBot":["partial","*",6,6],"ChatGPT-User":["partial","*",6,6],"OAI-SearchBot":["partial","*",6,6],"ClaudeBot":["partial","*",6,6],"Claude-User":["partial","*",6,6],"Claude-SearchBot":["partial","*",6,6],"anthropic-ai":["partial","*",6,6],"PerplexityBot":["partial","*",6,6],"Perplexity-User":["partial","*",6,6],"Google-Extended":["partial","*",6,6],"Applebot-Extended":["partial","*",6,6],"CCBot":["partial","*",6,6],"Bytespider":["partial","*",6,6],"Amazonbot":["partial","*",6,6],"meta-externalagent":["partial","*",6,6],"cohere-ai":["partial","*",6,6],"Diffbot":["partial","*",6,6],"ContextFetchBot":["partial","*",6,6]},"llms":{"status":"html","http":200,"bytes":314600},"home":{"status":"ok","http":200,"url":"https://vnu.edu.vn/","html_tokens":160585,"markdown_tokens":51,"pruned_tokens":51}},
{"rank":9307,"domain":"cyclingnews.com","origin":"https://cyclingnews.com","robots":{"status":"ok","http":200,"url":"https://www.cyclingnews.com/robots.txt","bytes":1679,"groups":5},"bots":{"GPTBot":["partial","*",19,0],"ChatGPT-User":["partial","*",19,0],"OAI-SearchBot":["partial","*",19,0],"ClaudeBot":["partial","*",19,0],"Claude-User":["partial","*",19,0],"Claude-SearchBot":["partial","*",19,0],"anthropic-ai":["partial","*",19,0],"PerplexityBot":["partial","*",19,0],"Perplexity-User":["partial","*",19,0],"Google-Extended":["partial","*",19,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["partial","*",19,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["partial","*",19,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",19,0]},"llms":{"status":"missing","http":404,"bytes":1218831},"home":{"status":"ok","http":200,"url":"https://www.cyclingnews.com/","html_tokens":490534,"markdown_tokens":1727,"pruned_tokens":163}},
{"rank":9308,"domain":"windsurf.com","origin":"https://windsurf.com","robots":{"status":"ok","http":200,"url":"https://windsurf.com/robots.txt","bytes":170,"groups":1},"bots":{"GPTBot":["allowed","*",0,3],"ChatGPT-User":["allowed","*",0,3],"OAI-SearchBot":["allowed","*",0,3],"ClaudeBot":["allowed","*",0,3],"Claude-User":["allowed","*",0,3],"Claude-SearchBot":["allowed","*",0,3],"anthropic-ai":["allowed","*",0,3],"PerplexityBot":["allowed","*",0,3],"Perplexity-User":["allowed","*",0,3],"Google-Extended":["allowed","*",0,3],"Applebot-Extended":["allowed","*",0,3],"CCBot":["allowed","*",0,3],"Bytespider":["allowed","*",0,3],"Amazonbot":["allowed","*",0,3],"meta-externalagent":["allowed","*",0,3],"cohere-ai":["allowed","*",0,3],"Diffbot":["allowed","*",0,3],"ContextFetchBot":["allowed","*",0,3]},"llms":{"status":"html","http":200,"bytes":67028},"home":{"status":"ok","http":200,"url":"https://devin.ai/desktop","html_tokens":258456,"markdown_tokens":374,"pruned_tokens":23}},
{"rank":9309,"domain":"theodysseyonline.com","origin":"https://theodysseyonline.com","robots":{"status":"ok","http":200,"url":"https://www.theodysseyonline.com/robots.txt","bytes":519,"groups":2},"bots":{"GPTBot":["partial","*",4,0],"ChatGPT-User":["partial","*",4,0],"OAI-SearchBot":["partial","*",4,0],"ClaudeBot":["partial","*",4,0],"Claude-User":["partial","*",4,0],"Claude-SearchBot":["partial","*",4,0],"anthropic-ai":["partial","*",4,0],"PerplexityBot":["partial","*",4,0],"Perplexity-User":["partial","*",4,0],"Google-Extended":["partial","*",4,0],"Applebot-Extended":["partial","*",4,0],"CCBot":["partial","*",4,0],"Bytespider":["partial","*",4,0],"Amazonbot":["partial","*",4,0],"meta-externalagent":["partial","*",4,0],"cohere-ai":["partial","*",4,0],"Diffbot":["partial","*",4,0],"ContextFetchBot":["partial","*",4,0]},"llms":{"status":"missing","http":404,"bytes":161575},"home":{"status":"ok","http":200,"url":"https://www.theodysseyonline.com/","html_tokens":138725,"markdown_tokens":2044,"pruned_tokens":104}},
{"rank":9310,"domain":"portotheme.com","origin":"https://portotheme.com","robots":{"status":"refused","http":403,"url":"https://portotheme.com/robots.txt","bytes":5530,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":9311,"domain":"clouparking.cn","origin":"https://clouparking.cn","robots":{"status":"unreachable","http":null,"url":"https://clouparking.cn/robots.txt","bytes":0,"groups":0,"error":"clouparking.cn is not a public address"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":9312,"domain":"ulinkcollege.com","origin":"https://www.ulinkcollege.com","robots":{"status":"server_error","http":500,"url":"https://www.ulinkcollege.com/robots.txt","bytes":27046,"groups":0,"error":"HTTP 500"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":9313,"domain":"mediamarkt.de","origin":"https://mediamarkt.de","robots":{"status":"ok","http":200,"url":"https://www.mediamarkt.de/robots.txt","bytes":4194,"groups":44},"bots":{"GPTBot":["allowed","named",0,1],"ChatGPT-User":["allowed","named",0,1],"OAI-SearchBot":["allowed","named",0,1],"ClaudeBot":["allowed","named",0,1],"Claude-User":["allowed","named",0,1],"Claude-SearchBot":["allowed","named",0,1],"anthropic-ai":["allowed","named",0,1],"PerplexityBot":["allowed","named",0,1],"Perplexity-User":["allowed","named",0,1],"Google-Extended":["allowed","named",0,1],"Applebot-Extended":["allowed","named",0,1],"CCBot":["blocked","named",1,0],"Bytespider":["allowed","named",0,1],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["allowed","named",0,1],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",28,13]},"llms":{"status":"present","http":200,"bytes":18324},"home":{"status":"http_error","http":403,"url":"https://www.mediamarkt.de/"}},
{"rank":9314,"domain":"heraldo.es","origin":"https://heraldo.es","robots":{"status":"ok","http":200,"url":"https://www.heraldo.es/robots.txt","bytes":6619,"groups":117},"bots":{"GPTBot":["partial","named",1,2],"ChatGPT-User":["partial","*",13,0],"OAI-SearchBot":["partial","*",13,0],"ClaudeBot":["partial","named",1,2],"Claude-User":["partial","*",13,0],"Claude-SearchBot":["partial","*",13,0],"anthropic-ai":["partial","named",1,2],"PerplexityBot":["partial","*",13,0],"Perplexity-User":["partial","*",13,0],"Google-Extended":["partial","*",13,0],"Applebot-Extended":["partial","named",1,2],"CCBot":["partial","named",1,2],"Bytespider":["partial","named",1,2],"Amazonbot":["partial","named",1,2],"meta-externalagent":["partial","named",1,2],"cohere-ai":["partial","named",1,2],"Diffbot":["partial","named",1,2],"ContextFetchBot":["partial","*",13,0]},"llms":{"status":"missing","http":404,"bytes":49007},"home":{"status":"ok","http":200,"url":"https://www.heraldo.es/","html_tokens":65010,"markdown_tokens":14,"pruned_tokens":14}},
{"rank":9315,"domain":"usbr.gov","origin":"https://usbr.gov","robots":{"status":"unreachable","http":null,"url":"https://usbr.gov/robots.txt","bytes":0,"groups":0,"error":"ReadError"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":9316,"domain":"ecu.edu.au","origin":"https://ecu.edu.au","robots":{"status":"refused","http":403,"url":"https://www.ecu.edu.au/robots.txt","bytes":5530,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":9317,"domain":"webbyawards.com","origin":"https://webbyawards.com","robots":{"status":"ok","http":200,"url":"https://www.webbyawards.com/robots.txt","bytes":213,"groups":1},"bots":{"GPTBot":["partial","*",5,1],"ChatGPT-User":["partial","*",5,1],"OAI-SearchBot":["partial","*",5,1],"ClaudeBot":["partial","*",5,1],"Claude-User":["partial","*",5,1],"Claude-SearchBot":["partial","*",5,1],"anthropic-ai":["partial","*",5,1],"PerplexityBot":["partial","*",5,1],"Perplexity-User":["partial","*",5,1],"Google-Extended":["partial","*",5,1],"Applebot-Extended":["partial","*",5,1],"CCBot":["partial","*",5,1],"Bytespider":["partial","*",5,1],"Amazonbot":["partial","*",5,1],"meta-externalagent":["partial","*",5,1],"cohere-ai":["partial","*",5,1],"Diffbot":["partial","*",5,1],"ContextFetchBot":["partial","*",5,1]},"llms":{"status":"http_error","http":403,"bytes":5568},"home":{"status":"http_error","http":403,"url":"https://webbyawards.com/"}},
{"rank":9318,"domain":"dbpedia.org","origin":"https://www.dbpedia.org","robots":{"status":"ok","http":200,"url":"https://www.dbpedia.org/robots.txt","bytes":174,"groups":1},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"html","http":200,"bytes":116497},"home":{"status":"ok","http":200,"url":"https://www.dbpedia.org/","html_tokens":29126,"markdown_tokens":454,"pruned_tokens":399}},
{"rank":9319,"domain":"apple.com.cn","origin":"https://apple.com.cn","robots":{"status":"ok","http":200,"url":"https://www.apple.com.cn/robots.txt","bytes":1038,"groups":1},"bots":{"GPTBot":["partial","*",27,1],"ChatGPT-User":["partial","*",27,1],"OAI-SearchBot":["partial","*",27,1],"ClaudeBot":["partial","*",27,1],"Claude-User":["partial","*",27,1],"Claude-SearchBot":["partial","*",27,1],"anthropic-ai":["partial","*",27,1],"PerplexityBot":["partial","*",27,1],"Perplexity-User":["partial","*",27,1],"Google-Extended":["partial","*",27,1],"Applebot-Extended":["partial","*",27,1],"CCBot":["partial","*",27,1],"Bytespider":["partial","*",27,1],"Amazonbot":["partial","*",27,1],"meta-externalagent":["partial","*",27,1],"cohere-ai":["partial","*",27,1],"Diffbot":["partial","*",27,1],"ContextFetchBot":["partial","*",27,1]},"llms":{"status":"missing","http":404,"bytes":97317},"home":{"status":"ok","http":200,"url":"https://www.apple.com.cn/","html_tokens":35250,"markdown_tokens":13,"pruned_tokens":13}},
{"rank":9320,"domain":"image.baidu.com","origin":"https://image.baidu.com","robots":{"status":"ok","http":200,"url":"https://image.baidu.com/robots.txt","bytes":648,"groups":15},"bots":{"GPTBot":["blocked","*",1,0],"ChatGPT-User":["blocked","*",1,0],"OAI-SearchBot":["blocked","*",1,0],"ClaudeBot":["blocked","*",1,0],"Claude-User":["blocked","*",1,0],"Claude-SearchBot":["blocked","*",1,0],"anthropic-ai":["blocked","*",1,0],"PerplexityBot":["blocked","*",1,0],"Perplexity-User":["blocked","*",1,0],"Google-Extended":["blocked","*",1,0],"Applebot-Extended":["blocked","*",1,0],"CCBot":["blocked","*",1,0],"Bytespider":["blocked","*",1,0],"Amazonbot":["blocked","*",1,0],"meta-externalagent":["blocked","*",1,0],"cohere-ai":["blocked","*",1,0],"Diffbot":["blocked","*",1,0],"ContextFetchBot":["blocked","*",1,0]},"llms":{"status":"disallowed"},"home":{"status":"disallowed"}},
{"rank":9321,"domain":"khzdq.com","origin":"http://khzdq.com","robots":{"status":"ok","http":200,"url":"http://khzdq.com/robots.txt","bytes":291,"groups":1},"bots":{"GPTBot":["partial","*",12,0],"ChatGPT-User":["partial","*",12,0],"OAI-SearchBot":["partial","*",12,0],"ClaudeBot":["partial","*",12,0],"Claude-User":["partial","*",12,0],"Claude-SearchBot":["partial","*",12,0],"anthropic-ai":["partial","*",12,0],"PerplexityBot":["partial","*",12,0],"Perplexity-User":["partial","*",12,0],"Google-Extended":["partial","*",12,0],"Applebot-Extended":["partial","*",12,0],"CCBot":["partial","*",12,0],"Bytespider":["partial","*",12,0],"Amazonbot":["partial","*",12,0],"meta-externalagent":["partial","*",12,0],"cohere-ai":["partial","*",12,0],"Diffbot":["partial","*",12,0],"ContextFetchBot":["partial","*",12,0]},"llms":{"status":"missing","http":404,"bytes":4724},"home":{"status":"ok","http":200,"url":"http://khzdq.com/","html_tokens":3607,"markdown_tokens":69,"pruned_tokens":12}},
{"rank":9322,"domain":"datacenterknowledge.com","origin":"https://datacenterknowledge.com","robots":{"status":"ok","http":200,"url":"https://www.datacenterknowledge.com/robots.txt","bytes":1644,"groups":2},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["partial","*",6,0],"OAI-SearchBot":["partial","*",6,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",6,0],"Claude-SearchBot":["partial","*",6,0],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["partial","*",6,0],"Perplexity-User":["partial","*",6,0],"Google-Extended":["partial","*",6,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",6,0]},"llms":{"status":"missing","http":404,"bytes":365007},"home":{"status":"ok","http":200,"url":"https://www.datacenterknowledge.com/","html_tokens":155992,"markdown_tokens":98,"pruned_tokens":99}},
{"rank":9323,"domain":"funky-fruits-slot.com","origin":"https://funky-fruits-slot.com","robots":{"status":"ok","http":200,"url":"https://udinslotace.com/robots.txt","bytes":71,"groups":1},"bots":{"GPTBot":["allowed","*",0,1],"ChatGPT-User":["allowed","*",0,1],"OAI-SearchBot":["allowed","*",0,1],"ClaudeBot":["allowed","*",0,1],"Claude-User":["allowed","*",0,1],"Claude-SearchBot":["allowed","*",0,1],"anthropic-ai":["allowed","*",0,1],"PerplexityBot":["allowed","*",0,1],"Perplexity-User":["allowed","*",0,1],"Google-Extended":["allowed","*",0,1],"Applebot-Extended":["allowed","*",0,1],"CCBot":["allowed","*",0,1],"Bytespider":["allowed","*",0,1],"Amazonbot":["allowed","*",0,1],"meta-externalagent":["allowed","*",0,1],"cohere-ai":["allowed","*",0,1],"Diffbot":["allowed","*",0,1],"ContextFetchBot":["allowed","*",0,1]},"llms":{"status":"missing","http":404,"bytes":2001},"home":{"status":"ok","http":200,"url":"https://udinslotace.com/","html_tokens":58090,"markdown_tokens":9,"pruned_tokens":9}},
{"rank":9324,"domain":"comedycentral.com","origin":"https://comedycentral.com","robots":{"status":"ok","http":200,"url":"https://www.cc.com/robots.txt","bytes":253,"groups":2},"bots":{"GPTBot":["partial","*",1,0],"ChatGPT-User":["partial","*",1,0],"OAI-SearchBot":["partial","*",1,0],"ClaudeBot":["partial","*",1,0],"Claude-User":["partial","*",1,0],"Claude-SearchBot":["partial","*",1,0],"anthropic-ai":["partial","*",1,0],"PerplexityBot":["partial","*",1,0],"Perplexity-User":["partial","*",1,0],"Google-Extended":["partial","*",1,0],"Applebot-Extended":["partial","*",1,0],"CCBot":["partial","*",1,0],"Bytespider":["partial","*",1,0],"Amazonbot":["partial","*",1,0],"meta-externalagent":["partial","*",1,0],"cohere-ai":["partial","*",1,0],"Diffbot":["partial","*",1,0],"ContextFetchBot":["partial","*",1,0]},"llms":{"status":"missing","http":404,"bytes":794765},"home":{"status":"ok","http":200,"url":"https://www.cc.com","html_tokens":787828,"markdown_tokens":1369,"pruned_tokens":157}},
{"rank":9325,"domain":"mitpressjournals.org","origin":"https://mitpressjournals.org","robots":{"status":"ok","http":200,"url":"https://direct.mit.edu/robots.txt","bytes":1331,"groups":2},"bots":{"GPTBot":["partial","*",36,3],"ChatGPT-User":["partial","*",36,3],"OAI-SearchBot":["partial","*",36,3],"ClaudeBot":["partial","*",36,3],"Claude-User":["partial","*",36,3],"Claude-SearchBot":["partial","*",36,3],"anthropic-ai":["partial","*",36,3],"PerplexityBot":["partial","*",36,3],"Perplexity-User":["partial","*",36,3],"Google-Extended":["partial","*",36,3],"Applebot-Extended":["partial","*",36,3],"CCBot":["partial","*",36,3],"Bytespider":["partial","*",36,3],"Amazonbot":["partial","*",36,3],"meta-externalagent":["partial","*",36,3],"cohere-ai":["partial","*",36,3],"Diffbot":["partial","*",36,3],"ContextFetchBot":["partial","*",36,3]},"llms":{"status":"http_error","http":403,"bytes":5525},"home":{"status":"http_error","http":403,"url":"https://direct.mit.edu/journals"}},
{"rank":9326,"domain":"legaldaily.com.cn","origin":"https://legaldaily.com.cn","robots":{"status":"unreachable","http":null,"url":"https://legaldaily.com.cn/robots.txt","bytes":0,"groups":0,"error":"cannot resolve legaldaily.com.cn"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":9327,"domain":"jxmls.com","origin":"http://jxmls.com","robots":{"status":"ok","http":200,"url":"http://jxmls.com/robots.txt","bytes":72,"groups":1},"bots":{"GPTBot":["partial","*",2,0],"ChatGPT-User":["partial","*",2,0],"OAI-SearchBot":["partial","*",2,0],"ClaudeBot":["partial","*",2,0],"Claude-User":["partial","*",2,0],"Claude-SearchBot":["partial","*",2,0],"anthropic-ai":["partial","*",2,0],"PerplexityBot":["partial","*",2,0],"Perplexity-User":["partial","*",2,0],"Google-Extended":["partial","*",2,0],"Applebot-Extended":["partial","*",2,0],"CCBot":["partial","*",2,0],"Bytespider":["partial","*",2,0],"Amazonbot":["partial","*",2,0],"meta-externalagent":["partial","*",2,0],"cohere-ai":["partial","*",2,0],"Diffbot":["partial","*",2,0],"ContextFetchBot":["partial","*",2,0]},"llms":{"status":"missing","http":404,"bytes":16925},"home":{"status":"ok","http":200,"url":"http://jxmls.com/","html_tokens":7773,"markdown_tokens":366,"pruned_tokens":115}},
{"rank":9328,"domain":"pzxbwg.com","origin":"https://pzxbwg.com","robots":{"status":"missing","http":404,"url":"https://pzxbwg.com/robots.txt","bytes":1163,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":1163},"home":{"status":"ok","http":200,"url":"https://pzxbwg.com/","html_tokens":2626,"markdown_tokens":19,"pruned_tokens":19}},
{"rank":9329,"domain":"tj.gov.cn","origin":"https://www.tj.gov.cn","robots":{"status":"missing","http":404,"url":"https://www.tj.gov.cn/robots.txt","bytes":136155,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":136155},"home":{"status":"ok","http":200,"url":"https://www.tj.gov.cn/","html_tokens":45669,"markdown_tokens":47,"pruned_tokens":40}},
{"rank":9330,"domain":"bjd.com.cn","origin":"https://www.bjd.com.cn","robots":{"status":"ok","http":200,"url":"https://www.bjd.com.cn/robots.txt","bytes":49,"groups":1},"bots":{"GPTBot":["partial","*",2,0],"ChatGPT-User":["partial","*",2,0],"OAI-SearchBot":["partial","*",2,0],"ClaudeBot":["partial","*",2,0],"Claude-User":["partial","*",2,0],"Claude-SearchBot":["partial","*",2,0],"anthropic-ai":["partial","*",2,0],"PerplexityBot":["partial","*",2,0],"Perplexity-User":["partial","*",2,0],"Google-Extended":["partial","*",2,0],"Applebot-Extended":["partial","*",2,0],"CCBot":["partial","*",2,0],"Bytespider":["partial","*",2,0],"Amazonbot":["partial","*",2,0],"meta-externalagent":["partial","*",2,0],"cohere-ai":["partial","*",2,0],"Diffbot":["partial","*",2,0],"ContextFetchBot":["partial","*",2,0]},"llms":{"status":"present","http":200,"bytes":5925},"home":{"status":"ok","http":200,"url":"https://www.bjd.com.cn/","html_tokens":27129,"markdown_tokens":10,"pruned_tokens":10}},
{"rank":9331,"domain":"specificfeeds.com","origin":"https://specificfeeds.com","robots":{"status":"ok","http":200,"url":"https://follow.it/robots.txt","bytes":294,"groups":1},"bots":{"GPTBot":["partial","*",5,0],"ChatGPT-User":["partial","*",5,0],"OAI-SearchBot":["partial","*",5,0],"ClaudeBot":["partial","*",5,0],"Claude-User":["partial","*",5,0],"Claude-SearchBot":["partial","*",5,0],"anthropic-ai":["partial","*",5,0],"PerplexityBot":["partial","*",5,0],"Perplexity-User":["partial","*",5,0],"Google-Extended":["partial","*",5,0],"Applebot-Extended":["partial","*",5,0],"CCBot":["partial","*",5,0],"Bytespider":["partial","*",5,0],"Amazonbot":["partial","*",5,0],"meta-externalagent":["partial","*",5,0],"cohere-ai":["partial","*",5,0],"Diffbot":["partial","*",5,0],"ContextFetchBot":["partial","*",5,0]},"llms":{"status":"html","http":200,"bytes":334616},"home":{"status":"ok","http":200,"url":"https://follow.it/","html_tokens":46298,"markdown_tokens":337,"pruned_tokens":146}},
{"rank":9332,"domain":"createsend1.com","origin":"https://createsend1.com","robots":{"status":"ok","http":200,"url":"https://createsend1.com/robots.txt","bytes":26,"groups":1},"bots":{"GPTBot":["blocked","*",1,0],"ChatGPT-User":["blocked","*",1,0],"OAI-SearchBot":["blocked","*",1,0],"ClaudeBot":["blocked","*",1,0],"Claude-User":["blocked","*",1,0],"Claude-SearchBot":["blocked","*",1,0],"anthropic-ai":["blocked","*",1,0],"PerplexityBot":["blocked","*",1,0],"Perplexity-User":["blocked","*",1,0],"Google-Extended":["blocked","*",1,0],"Applebot-Extended":["blocked","*",1,0],"CCBot":["blocked","*",1,0],"Bytespider":["blocked","*",1,0],"Amazonbot":["blocked","*",1,0],"meta-externalagent":["blocked","*",1,0],"cohere-ai":["blocked","*",1,0],"Diffbot":["blocked","*",1,0],"ContextFetchBot":["blocked","*",1,0]},"llms":{"status":"disallowed"},"home":{"status":"disallowed"}},
{"rank":9333,"domain":"ycwb.com","origin":"https://www.ycwb.com","robots":{"status":"html","http":200,"url":"https://www.ycwb.com/robots.txt","bytes":4321,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":9334,"domain":"laprovence.com","origin":"https://laprovence.com","robots":{"status":"html","http":200,"url":"https://www.laprovence.com/","bytes":930120,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":9335,"domain":"edmontonjournal.com","origin":"https://edmontonjournal.com","robots":{"status":"ok","http":200,"url":"https://edmontonjournal.com/robots.txt","bytes":1245,"groups":15},"bots":{"GPTBot":["partial","*",3,6],"ChatGPT-User":["partial","*",3,6],"OAI-SearchBot":["partial","*",3,6],"ClaudeBot":["partial","*",3,6],"Claude-User":["partial","*",3,6],"Claude-SearchBot":["partial","*",3,6],"anthropic-ai":["partial","*",3,6],"PerplexityBot":["partial","*",3,6],"Perplexity-User":["partial","*",3,6],"Google-Extended":["partial","*",3,6],"Applebot-Extended":["partial","*",3,6],"CCBot":["partial","*",3,6],"Bytespider":["partial","*",3,6],"Amazonbot":["partial","*",3,6],"meta-externalagent":["partial","*",3,6],"cohere-ai":["partial","*",3,6],"Diffbot":["partial","*",3,6],"ContextFetchBot":["partial","*",3,6]},"llms":{"status":"missing","http":404,"bytes":287993},"home":{"status":"ok","http":200,"url":"https://edmontonjournal.com/","html_tokens":128847,"markdown_tokens":143,"pruned_tokens":143}},
{"rank":9336,"domain":"arthromedix.online","origin":"https://arthromedix.online","robots":{"status":"ok","http":200,"url":"https://arthromedix.online/robots.txt","bytes":73,"groups":1},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"missing","http":404,"bytes":647},"home":{"status":"ok","http":200,"url":"https://arthromedix.online/","html_tokens":3335,"markdown_tokens":1517,"pruned_tokens":144}},
{"rank":9337,"domain":"spacedaily.com","origin":"https://spacedaily.com","robots":{"status":"ok","http":200,"url":"https://spacedaily.com/robots.txt","bytes":511,"groups":2},"bots":{"GPTBot":["partial","*",10,0],"ChatGPT-User":["partial","*",10,0],"OAI-SearchBot":["partial","*",10,0],"ClaudeBot":["partial","*",10,0],"Claude-User":["partial","*",10,0],"Claude-SearchBot":["partial","*",10,0],"anthropic-ai":["partial","*",10,0],"PerplexityBot":["partial","*",10,0],"Perplexity-User":["partial","*",10,0],"Google-Extended":["partial","*",10,0],"Applebot-Extended":["partial","*",10,0],"CCBot":["partial","*",10,0],"Bytespider":["partial","*",10,0],"Amazonbot":["partial","*",10,0],"meta-externalagent":["partial","*",10,0],"cohere-ai":["partial","*",10,0],"Diffbot":["partial","*",10,0],"ContextFetchBot":["partial","*",10,0]},"llms":{"status":"missing","http":404,"bytes":56153},"home":{"status":"ok","http":200,"url":"https://spacedaily.com/","html_tokens":23235,"markdown_tokens":68,"pruned_tokens":68}},
{"rank":9338,"domain":"socrf.com","origin":"http://socrf.com","robots":{"status":"missing","http":404,"url":"http://socrf.com/robots.txt","bytes":4725,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":4721},"home":{"status":"ok","http":200,"url":"http://socrf.com/","html_tokens":6087,"markdown_tokens":184,"pruned_tokens":166}},
{"rank":9339,"domain":"df.gov.br","origin":"https://df.gov.br","robots":{"status":"timeout","http":null,"url":"https://df.gov.br/robots.txt","bytes":0,"groups":0,"error":"ConnectTimeout"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":9340,"domain":"holidaycheck.de","origin":"https://holidaycheck.de","robots":{"status":"ok","http":200,"url":"https://www.holidaycheck.de/robots.txt","bytes":6178,"groups":9},"bots":{"GPTBot":["partial","*",160,0],"ChatGPT-User":["allowed","named",0,1],"OAI-SearchBot":["partial","*",160,0],"ClaudeBot":["allowed","named",0,0],"Claude-User":["partial","*",160,0],"Claude-SearchBot":["partial","*",160,0],"anthropic-ai":["partial","*",160,0],"PerplexityBot":["partial","*",160,0],"Perplexity-User":["partial","*",160,0],"Google-Extended":["partial","*",160,0],"Applebot-Extended":["partial","*",160,0],"CCBot":["partial","*",160,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["partial","*",160,0],"meta-externalagent":["partial","*",160,0],"cohere-ai":["partial","*",160,0],"Diffbot":["partial","*",160,0],"ContextFetchBot":["partial","*",160,0]},"llms":{"status":"missing","http":404,"bytes":84391},"home":{"status":"ok","http":200,"url":"https://www.holidaycheck.de/","html_tokens":165032,"markdown_tokens":582,"pruned_tokens":372}},
{"rank":9341,"domain":"nl.ca","origin":"https://nl.ca","robots":{"status":"unreachable","http":null,"url":"https://nl.ca/robots.txt","bytes":0,"groups":0,"error":"cannot resolve nl.ca"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":9342,"domain":"annualreports.com","origin":"https://annualreports.com","robots":{"status":"ok","http":200,"url":"https://www.annualreports.com/robots.txt","bytes":889,"groups":14},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["blocked","named",1,0],"OAI-SearchBot":["blocked","named",1,0],"ClaudeBot":["partial","*",4,0],"Claude-User":["partial","*",4,0],"Claude-SearchBot":["partial","*",4,0],"anthropic-ai":["partial","*",4,0],"PerplexityBot":["partial","*",4,0],"Perplexity-User":["partial","*",4,0],"Google-Extended":["partial","*",4,0],"Applebot-Extended":["partial","*",4,0],"CCBot":["partial","*",4,0],"Bytespider":["partial","*",4,0],"Amazonbot":["partial","*",4,0],"meta-externalagent":["partial","*",4,0],"cohere-ai":["partial","*",4,0],"Diffbot":["partial","*",4,0],"ContextFetchBot":["partial","*",4,0]},"llms":{"status":"html","http":200,"bytes":19401},"home":{"status":"ok","http":200,"url":"https://www.annualreports.com/","html_tokens":4833,"markdown_tokens":81,"pruned_tokens":81}},
{"rank":9343,"domain":"historytoday.com","origin":"https://historytoday.com","robots":{"status":"ok","http":200,"url":"https://historytoday.com/robots.txt","bytes":0,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":194},"home":{"status":"ok","http":200,"url":"https://www.historytoday.com/","html_tokens":16349,"markdown_tokens":288,"pruned_tokens":62}},
{"rank":9344,"domain":"rmi.org","origin":"https://rmi.org","robots":{"status":"refused","http":403,"url":"https://rmi.org/robots.txt","bytes":560,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":9345,"domain":"yjsry.com","origin":"https://yjsry.com","robots":{"status":"missing","http":404,"url":"https://yjsry.com/robots.txt","bytes":1231,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":1231},"home":{"status":"ok","http":200,"url":"https://yjsry.com/","html_tokens":13661,"markdown_tokens":198,"pruned_tokens":171}},
{"rank":9346,"domain":"books2read.com","origin":"https://books2read.com","robots":{"status":"ok","http":200,"url":"https://books2read.com/robots.txt","bytes":84,"groups":1},"bots":{"GPTBot":["partial","*",1,1],"ChatGPT-User":["partial","*",1,1],"OAI-SearchBot":["partial","*",1,1],"ClaudeBot":["partial","*",1,1],"Claude-User":["partial","*",1,1],"Claude-SearchBot":["partial","*",1,1],"anthropic-ai":["partial","*",1,1],"PerplexityBot":["partial","*",1,1],"Perplexity-User":["partial","*",1,1],"Google-Extended":["partial","*",1,1],"Applebot-Extended":["partial","*",1,1],"CCBot":["partial","*",1,1],"Bytespider":["partial","*",1,1],"Amazonbot":["partial","*",1,1],"meta-externalagent":["partial","*",1,1],"cohere-ai":["partial","*",1,1],"Diffbot":["partial","*",1,1],"ContextFetchBot":["partial","*",1,1]},"llms":{"status":"missing","http":404,"bytes":13977},"home":{"status":"ok","http":200,"url":"https://books2read.com/","html_tokens":4392,"markdown_tokens":123,"pruned_tokens":115}},
{"rank":9347,"domain":"geology.com","origin":"https://geology.com","robots":{"status":"ok","http":200,"url":"https://geology.com/robots.txt","bytes":24,"groups":1},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"missing","http":404,"bytes":4808},"home":{"status":"ok","http":200,"url":"https://geology.com/","html_tokens":14046,"markdown_tokens":1371,"pruned_tokens":58}},
{"rank":9348,"domain":"insurancebusinessmag.com","origin":"https://insurancebusinessmag.com","robots":{"status":"ok","http":200,"url":"https://www.insurancebusinessmag.com/robots.txt","bytes":560,"groups":2},"bots":{"GPTBot":["partial","*",15,0],"ChatGPT-User":["allowed","named",0,0],"OAI-SearchBot":["partial","*",15,0],"ClaudeBot":["partial","*",15,0],"Claude-User":["allowed","named",0,0],"Claude-SearchBot":["partial","*",15,0],"anthropic-ai":["partial","*",15,0],"PerplexityBot":["partial","*",15,0],"Perplexity-User":["allowed","named",0,0],"Google-Extended":["partial","*",15,0],"Applebot-Extended":["partial","*",15,0],"CCBot":["partial","*",15,0],"Bytespider":["partial","*",15,0],"Amazonbot":["partial","*",15,0],"meta-externalagent":["partial","*",15,0],"cohere-ai":["partial","*",15,0],"Diffbot":["partial","*",15,0],"ContextFetchBot":["partial","*",15,0]},"llms":{"status":"missing","http":404,"bytes":7282},"home":{"status":"ok","http":200,"url":"https://www.insurancebusinessmag.com/us/","html_tokens":34103,"markdown_tokens":148,"pruned_tokens":148}},
{"rank":9349,"domain":"casethemes.net","origin":"https://casethemes.net","robots":{"status":"ok","http":200,"url":"https://casethemes.net/robots.txt","bytes":320,"groups":1},"bots":{"GPTBot":["partial","*",6,1],"ChatGPT-User":["partial","*",6,1],"OAI-SearchBot":["partial","*",6,1],"ClaudeBot":["partial","*",6,1],"Claude-User":["partial","*",6,1],"Claude-SearchBot":["partial","*",6,1],"anthropic-ai":["partial","*",6,1],"PerplexityBot":["partial","*",6,1],"Perplexity-User":["partial","*",6,1],"Google-Extended":["partial","*",6,1],"Applebot-Extended":["partial","*",6,1],"CCBot":["partial","*",6,1],"Bytespider":["partial","*",6,1],"Amazonbot":["partial","*",6,1],"meta-externalagent":["partial","*",6,1],"cohere-ai":["partial","*",6,1],"Diffbot":["partial","*",6,1],"ContextFetchBot":["partial","*",6,1]},"llms":{"status":"missing","http":404,"bytes":131876},"home":{"status":"ok","http":200,"url":"https://casethemes.net/","html_tokens":116734,"markdown_tokens":520,"pruned_tokens":236}},
{"rank":9350,"domain":"google.gl","origin":"https://google.gl","robots":{"status":"ok","http":200,"url":"https://www.google.gl/robots.txt","bytes":6624,"groups":4},"bots":{"GPTBot":["partial","*",169,64],"ChatGPT-User":["partial","*",169,64],"OAI-SearchBot":["partial","*",169,64],"ClaudeBot":["partial","*",169,64],"Claude-User":["partial","*",169,64],"Claude-SearchBot":["partial","*",169,64],"anthropic-ai":["partial","*",169,64],"PerplexityBot":["partial","*",169,64],"Perplexity-User":["partial","*",169,64],"Google-Extended":["partial","*",169,64],"Applebot-Extended":["partial","*",169,64],"CCBot":["partial","*",169,64],"Bytespider":["partial","*",169,64],"Amazonbot":["partial","*",169,64],"meta-externalagent":["partial","*",169,64],"cohere-ai":["partial","*",169,64],"Diffbot":["partial","*",169,64],"ContextFetchBot":["partial","*",169,64]},"llms":{"status":"missing","http":404,"bytes":1569},"home":{"status":"ok","http":200,"url":"https://www.google.com/","html_tokens":21477,"markdown_tokens":30,"pruned_tokens":30}},
{"rank":9351,"domain":"toyota.co.jp","origin":"https://www.toyota.co.jp","robots":{"status":"ok","http":200,"url":"https://www.toyota.co.jp/robots.txt","bytes":4487,"groups":1},"bots":{"GPTBot":["partial","*",65,0],"ChatGPT-User":["partial","*",65,0],"OAI-SearchBot":["partial","*",65,0],"ClaudeBot":["partial","*",65,0],"Claude-User":["partial","*",65,0],"Claude-SearchBot":["partial","*",65,0],"anthropic-ai":["partial","*",65,0],"PerplexityBot":["partial","*",65,0],"Perplexity-User":["partial","*",65,0],"Google-Extended":["partial","*",65,0],"Applebot-Extended":["partial","*",65,0],"CCBot":["partial","*",65,0],"Bytespider":["partial","*",65,0],"Amazonbot":["partial","*",65,0],"meta-externalagent":["partial","*",65,0],"cohere-ai":["partial","*",65,0],"Diffbot":["partial","*",65,0],"ContextFetchBot":["partial","*",65,0]},"llms":{"status":"missing","http":404,"bytes":2898},"home":{"status":"ok","http":200,"url":"https://global.toyota/jp/","html_tokens":4092,"markdown_tokens":103,"pruned_tokens":83}},
{"rank":9352,"domain":"whathifi.com","origin":"https://whathifi.com","robots":{"status":"ok","http":200,"url":"https://www.whathifi.com/robots.txt","bytes":1976,"groups":5},"bots":{"GPTBot":["partial","*",24,0],"ChatGPT-User":["partial","*",24,0],"OAI-SearchBot":["partial","*",24,0],"ClaudeBot":["partial","*",24,0],"Claude-User":["partial","*",24,0],"Claude-SearchBot":["partial","*",24,0],"anthropic-ai":["partial","*",24,0],"PerplexityBot":["partial","*",24,0],"Perplexity-User":["partial","*",24,0],"Google-Extended":["partial","*",24,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["partial","*",24,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["partial","*",24,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",24,0]},"llms":{"status":"missing","http":404,"bytes":1245738},"home":{"status":"ok","http":200,"url":"https://www.whathifi.com/","html_tokens":413685,"markdown_tokens":27,"pruned_tokens":27}},
{"rank":9353,"domain":"origin.com","origin":"https://origin.com","robots":{"status":"html","http":200,"url":"https://www.ea.com/games/library/pc-download","bytes":219600,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":9354,"domain":"google.je","origin":"https://google.je","robots":{"status":"ok","http":200,"url":"https://www.google.je/robots.txt","bytes":6624,"groups":4},"bots":{"GPTBot":["partial","*",169,64],"ChatGPT-User":["partial","*",169,64],"OAI-SearchBot":["partial","*",169,64],"ClaudeBot":["partial","*",169,64],"Claude-User":["partial","*",169,64],"Claude-SearchBot":["partial","*",169,64],"anthropic-ai":["partial","*",169,64],"PerplexityBot":["partial","*",169,64],"Perplexity-User":["partial","*",169,64],"Google-Extended":["partial","*",169,64],"Applebot-Extended":["partial","*",169,64],"CCBot":["partial","*",169,64],"Bytespider":["partial","*",169,64],"Amazonbot":["partial","*",169,64],"meta-externalagent":["partial","*",169,64],"cohere-ai":["partial","*",169,64],"Diffbot":["partial","*",169,64],"ContextFetchBot":["partial","*",169,64]},"llms":{"status":"missing","http":404,"bytes":1569},"home":{"status":"ok","http":200,"url":"https://www.google.com/","html_tokens":21442,"markdown_tokens":30,"pruned_tokens":30}},
{"rank":9355,"domain":"goldenlioncasino.io","origin":"https://goldenlioncasino.io","robots":{"status":"ok","http":200,"url":"https://goldenlioncasino.io/robots.txt","bytes":645,"groups":5},"bots":{"GPTBot":["partial","*",3,16],"ChatGPT-User":["partial","*",3,16],"OAI-SearchBot":["partial","*",3,16],"ClaudeBot":["partial","*",3,16],"Claude-User":["partial","*",3,16],"Claude-SearchBot":["partial","*",3,16],"anthropic-ai":["partial","*",3,16],"PerplexityBot":["partial","*",3,16],"Perplexity-User":["partial","*",3,16],"Google-Extended":["partial","*",3,16],"Applebot-Extended":["partial","*",3,16],"CCBot":["partial","*",3,16],"Bytespider":["partial","*",3,16],"Amazonbot":["partial","*",3,16],"meta-externalagent":["partial","*",3,16],"cohere-ai":["partial","*",3,16],"Diffbot":["partial","*",3,16],"ContextFetchBot":["partial","*",3,16]},"llms":{"status":"missing","http":404,"bytes":529},"home":{"status":"ok","http":200,"url":"https://goldenlioncasino.io/","html_tokens":18201,"markdown_tokens":5544,"pruned_tokens":88}},
{"rank":9356,"domain":"ems.com.cn","origin":"https://www.ems.com.cn","robots":{"status":"refused","http":405,"url":"https://www.ems.com.cn/robots.txt","bytes":64367,"groups":0,"error":"HTTP 405"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":9357,"domain":"infomoney.com.br","origin":"https://infomoney.com.br","robots":{"status":"ok","http":200,"url":"https://www.infomoney.com.br/robots.txt","bytes":247,"groups":1},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"missing","http":404,"bytes":384520},"home":{"status":"ok","http":200,"url":"https://www.infomoney.com.br/","html_tokens":150035,"markdown_tokens":35,"pruned_tokens":35}},
{"rank":9358,"domain":"abilitynet.org.uk","origin":"https://abilitynet.org.uk","robots":{"status":"ok","http":200,"url":"https://abilitynet.org.uk/robots.txt","bytes":2253,"groups":1},"bots":{"GPTBot":["partial","*",38,32],"ChatGPT-User":["partial","*",38,32],"OAI-SearchBot":["partial","*",38,32],"ClaudeBot":["partial","*",38,32],"Claude-User":["partial","*",38,32],"Claude-SearchBot":["partial","*",38,32],"anthropic-ai":["partial","*",38,32],"PerplexityBot":["partial","*",38,32],"Perplexity-User":["partial","*",38,32],"Google-Extended":["partial","*",38,32],"Applebot-Extended":["partial","*",38,32],"CCBot":["partial","*",38,32],"Bytespider":["partial","*",38,32],"Amazonbot":["partial","*",38,32],"meta-externalagent":["partial","*",38,32],"cohere-ai":["partial","*",38,32],"Diffbot":["partial","*",38,32],"ContextFetchBot":["partial","*",38,32]},"llms":{"status":"missing","http":404,"bytes":188},"home":{"status":"ok","http":200,"url":"https://abilitynet.org.uk/","html_tokens":20155,"markdown_tokens":666,"pruned_tokens":137}},
{"rank":9359,"domain":"bang-olufsen.com","origin":"https://bang-olufsen.com","robots":{"status":"ok","http":200,"url":"https://beoecommercestoragelive.blob.core.windows.net/redio/robots/robots_botld.txt","bytes":168,"groups":2},"bots":{"GPTBot":["partial","*",1,1],"ChatGPT-User":["partial","*",1,1],"OAI-SearchBot":["partial","*",1,1],"ClaudeBot":["partial","*",1,1],"Claude-User":["partial","*",1,1],"Claude-SearchBot":["partial","*",1,1],"anthropic-ai":["partial","*",1,1],"PerplexityBot":["partial","*",1,1],"Perplexity-User":["partial","*",1,1],"Google-Extended":["partial","*",1,1],"Applebot-Extended":["partial","*",1,1],"CCBot":["partial","*",1,1],"Bytespider":["partial","*",1,1],"Amazonbot":["partial","*",1,1],"meta-externalagent":["partial","*",1,1],"cohere-ai":["partial","*",1,1],"Diffbot":["partial","*",1,1],"ContextFetchBot":["partial","*",1,1]},"llms":{"status":"disallowed"},"home":{"status":"ok","http":200,"url":"https://www.bang-olufsen.com/en/us","html_tokens":189753,"markdown_tokens":51,"pruned_tokens":51}},
{"rank":9360,"domain":"rozhlas.cz","origin":"https://rozhlas.cz","robots":{"status":"ok","http":200,"url":"https://www.rozhlas.cz/robots.txt","bytes":2577,"groups":1},"bots":{"GPTBot":["partial","*",53,32],"ChatGPT-User":["partial","*",53,32],"OAI-SearchBot":["partial","*",53,32],"ClaudeBot":["partial","*",53,32],"Claude-User":["partial","*",53,32],"Claude-SearchBot":["partial","*",53,32],"anthropic-ai":["partial","*",53,32],"PerplexityBot":["partial","*",53,32],"Perplexity-User":["partial","*",53,32],"Google-Extended":["partial","*",53,32],"Applebot-Extended":["partial","*",53,32],"CCBot":["partial","*",53,32],"Bytespider":["partial","*",53,32],"Amazonbot":["partial","*",53,32],"meta-externalagent":["partial","*",53,32],"cohere-ai":["partial","*",53,32],"Diffbot":["partial","*",53,32],"ContextFetchBot":["partial","*",53,32]},"llms":{"status":"missing","http":404,"bytes":291},"home":{"status":"ok","http":200,"url":"https://portal.rozhlas.cz","html_tokens":100648,"markdown_tokens":48,"pruned_tokens":48}},
{"rank":9361,"domain":"bloody-disgusting.com","origin":"https://bloody-disgusting.com","robots":{"status":"refused","http":403,"url":"https://bloody-disgusting.com/robots.txt","bytes":5580,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":9362,"domain":"mymrbet.com","origin":"https://mymrbet.com","robots":{"status":"ok","http":200,"url":"https://mymrbet.com/robots.txt","bytes":69,"groups":1},"bots":{"GPTBot":["allowed","*",0,1],"ChatGPT-User":["allowed","*",0,1],"OAI-SearchBot":["allowed","*",0,1],"ClaudeBot":["allowed","*",0,1],"Claude-User":["allowed","*",0,1],"Claude-SearchBot":["allowed","*",0,1],"anthropic-ai":["allowed","*",0,1],"PerplexityBot":["allowed","*",0,1],"Perplexity-User":["allowed","*",0,1],"Google-Extended":["allowed","*",0,1],"Applebot-Extended":["allowed","*",0,1],"CCBot":["allowed","*",0,1],"Bytespider":["allowed","*",0,1],"Amazonbot":["allowed","*",0,1],"meta-externalagent":["allowed","*",0,1],"cohere-ai":["allowed","*",0,1],"Diffbot":["allowed","*",0,1],"ContextFetchBot":["allowed","*",0,1]},"llms":{"status":"missing","http":404,"bytes":1163},"home":{"status":"ok","http":200,"url":"https://mymrbet.com/","html_tokens":72841,"markdown_tokens":806,"pruned_tokens":160}},
{"rank":9363,"domain":"servethehome.com","origin":"https://servethehome.com","robots":{"status":"missing","http":404,"url":"https://www.servethehome.com/robots.txt","bytes":162,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":104151},"home":{"status":"ok","http":200,"url":"https://www.servethehome.com/","html_tokens":32104,"markdown_tokens":772,"pruned_tokens":175}},
{"rank":9364,"domain":"sfmoma.org","origin":"https://sfmoma.org","robots":{"status":"ok","http":200,"url":"https://www.sfmoma.org/robots.txt","bytes":232,"groups":1},"bots":{"GPTBot":["partial","*",9,0],"ChatGPT-User":["partial","*",9,0],"OAI-SearchBot":["partial","*",9,0],"ClaudeBot":["partial","*",9,0],"Claude-User":["partial","*",9,0],"Claude-SearchBot":["partial","*",9,0],"anthropic-ai":["partial","*",9,0],"PerplexityBot":["partial","*",9,0],"Perplexity-User":["partial","*",9,0],"Google-Extended":["partial","*",9,0],"Applebot-Extended":["partial","*",9,0],"CCBot":["partial","*",9,0],"Bytespider":["partial","*",9,0],"Amazonbot":["partial","*",9,0],"meta-externalagent":["partial","*",9,0],"cohere-ai":["partial","*",9,0],"Diffbot":["partial","*",9,0],"ContextFetchBot":["partial","*",9,0]},"llms":{"status":"http_error","http":403,"bytes":5541},"home":{"status":"http_error","http":403,"url":"https://sfmoma.org/"}},
{"rank":9365,"domain":"mono-project.com","origin":"https://mono-project.com","robots":{"status":"missing","http":404,"url":"https://www.mono-project.com/robots.txt","bytes":6820,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":6820},"home":{"status":"ok","http":200,"url":"https://www.mono-project.com/","html_tokens":2807,"markdown_tokens":501,"pruned_tokens":200}},
{"rank":9366,"domain":"generalmills.com","origin":"https://generalmills.com","robots":{"status":"ok","http":200,"url":"https://www.generalmills.com/robots.txt","bytes":337,"groups":1},"bots":{"GPTBot":["partial","*",10,0],"ChatGPT-User":["partial","*",10,0],"OAI-SearchBot":["partial","*",10,0],"ClaudeBot":["partial","*",10,0],"Claude-User":["partial","*",10,0],"Claude-SearchBot":["partial","*",10,0],"anthropic-ai":["partial","*",10,0],"PerplexityBot":["partial","*",10,0],"Perplexity-User":["partial","*",10,0],"Google-Extended":["partial","*",10,0],"Applebot-Extended":["partial","*",10,0],"CCBot":["partial","*",10,0],"Bytespider":["partial","*",10,0],"Amazonbot":["partial","*",10,0],"meta-externalagent":["partial","*",10,0],"cohere-ai":["partial","*",10,0],"Diffbot":["partial","*",10,0],"ContextFetchBot":["partial","*",10,0]},"llms":{"status":"missing","http":404,"bytes":2915960},"home":{"status":"ok","http":200,"url":"https://www.generalmills.com/","html_tokens":75680,"markdown_tokens":123,"pruned_tokens":61}},
{"rank":9367,"domain":"eventbrite.es","origin":"https://eventbrite.es","robots":{"status":"ok","http":200,"url":"https://eventbrite.es/robots.txt","bytes":7499,"groups":135},"bots":{"GPTBot":["partial","*",51,2],"ChatGPT-User":["partial","*",51,2],"OAI-SearchBot":["partial","*",51,2],"ClaudeBot":["partial","*",51,2],"Claude-User":["partial","*",51,2],"Claude-SearchBot":["partial","*",51,2],"anthropic-ai":["partial","*",51,2],"PerplexityBot":["partial","*",51,2],"Perplexity-User":["partial","*",51,2],"Google-Extended":["partial","*",51,2],"Applebot-Extended":["partial","*",51,2],"CCBot":["blocked","named",1,0],"Bytespider":["partial","*",51,2],"Amazonbot":["partial","*",51,2],"meta-externalagent":["partial","*",51,2],"cohere-ai":["partial","*",51,2],"Diffbot":["partial","*",51,2],"ContextFetchBot":["partial","*",51,2]},"llms":{"status":"missing","http":404,"bytes":78627},"home":{"status":"ok","http":200,"url":"https://www.eventbrite.es","html_tokens":53482,"markdown_tokens":63,"pruned_tokens":63}},
{"rank":9368,"domain":"qooh.me","origin":"https://qooh.me","robots":{"status":"server_error","http":521,"url":"https://qooh.me/robots.txt","bytes":6942,"groups":0,"error":"HTTP 521"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":9369,"domain":"commercialappeal.com","origin":"https://commercialappeal.com","robots":{"status":"ok","http":200,"url":"https://www.commercialappeal.com/robots.txt","bytes":12691,"groups":288},"bots":{"GPTBot":["partial","named",1,4],"ChatGPT-User":["partial","named",1,4],"OAI-SearchBot":["partial","named",1,4],"ClaudeBot":["blocked","named",1,0],"Claude-User":["blocked","named",1,0],"Claude-SearchBot":["blocked","named",1,0],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["partial","named",1,4],"Perplexity-User":["blocked","named",1,0],"Google-Extended":["partial","named",1,4],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",2,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",32,0]},"llms":{"status":"http_error","http":402,"bytes":254},"home":{"status":"http_error","http":402,"url":"https://www.commercialappeal.com/"}},
{"rank":9370,"domain":"dove.com","origin":"https://dove.com","robots":{"status":"ok","http":200,"url":"https://www.dove.com/robots.txt","bytes":2814,"groups":1},"bots":{"GPTBot":["partial","*",5,1],"ChatGPT-User":["partial","*",5,1],"OAI-SearchBot":["partial","*",5,1],"ClaudeBot":["partial","*",5,1],"Claude-User":["partial","*",5,1],"Claude-SearchBot":["partial","*",5,1],"anthropic-ai":["partial","*",5,1],"PerplexityBot":["partial","*",5,1],"Perplexity-User":["partial","*",5,1],"Google-Extended":["partial","*",5,1],"Applebot-Extended":["partial","*",5,1],"CCBot":["partial","*",5,1],"Bytespider":["partial","*",5,1],"Amazonbot":["partial","*",5,1],"meta-externalagent":["partial","*",5,1],"cohere-ai":["partial","*",5,1],"Diffbot":["partial","*",5,1],"ContextFetchBot":["partial","*",5,1]},"llms":{"status":"present","http":200,"bytes":22532},"home":{"status":"ok","http":200,"url":"https://www.dove.com/us/en/home.html","html_tokens":518408,"markdown_tokens":162,"pruned_tokens":162}},
{"rank":9371,"domain":"carnival.com","origin":"https://carnival.com","robots":{"status":"ok","http":200,"url":"https://www.carnival.com/robots.txt","bytes":408,"groups":1},"bots":{"GPTBot":["partial","*",5,5],"ChatGPT-User":["partial","*",5,5],"OAI-SearchBot":["partial","*",5,5],"ClaudeBot":["partial","*",5,5],"Claude-User":["partial","*",5,5],"Claude-SearchBot":["partial","*",5,5],"anthropic-ai":["partial","*",5,5],"PerplexityBot":["partial","*",5,5],"Perplexity-User":["partial","*",5,5],"Google-Extended":["partial","*",5,5],"Applebot-Extended":["partial","*",5,5],"CCBot":["partial","*",5,5],"Bytespider":["partial","*",5,5],"Amazonbot":["partial","*",5,5],"meta-externalagent":["partial","*",5,5],"cohere-ai":["partial","*",5,5],"Diffbot":["partial","*",5,5],"ContextFetchBot":["partial","*",5,5]},"llms":{"status":"html","http":200,"bytes":541},"home":{"status":"ok","http":200,"url":"https://www.carnival.com/","html_tokens":8856,"markdown_tokens":325,"pruned_tokens":16}},
{"rank":9372,"domain":"dibiz.com","origin":"https://www.dibiz.com","robots":{"status":"ok","http":200,"url":"https://www.dibiz.com/robots.txt","bytes":227,"groups":1},"bots":{"GPTBot":["partial","*",2,0],"ChatGPT-User":["partial","*",2,0],"OAI-SearchBot":["partial","*",2,0],"ClaudeBot":["partial","*",2,0],"Claude-User":["partial","*",2,0],"Claude-SearchBot":["partial","*",2,0],"anthropic-ai":["partial","*",2,0],"PerplexityBot":["partial","*",2,0],"Perplexity-User":["partial","*",2,0],"Google-Extended":["partial","*",2,0],"Applebot-Extended":["partial","*",2,0],"CCBot":["partial","*",2,0],"Bytespider":["partial","*",2,0],"Amazonbot":["partial","*",2,0],"meta-externalagent":["partial","*",2,0],"cohere-ai":["partial","*",2,0],"Diffbot":["partial","*",2,0],"ContextFetchBot":["partial","*",2,0]},"llms":{"status":"http_error","http":500,"bytes":182},"home":{"status":"ok","http":200,"url":"https://www.dibiz.com/","html_tokens":9769,"markdown_tokens":247,"pruned_tokens":115}},
{"rank":9373,"domain":"frbsf.org","origin":"https://frbsf.org","robots":{"status":"ok","http":200,"url":"https://www.frbsf.org/robots.txt","bytes":117,"groups":1},"bots":{"GPTBot":["partial","*",1,1],"ChatGPT-User":["partial","*",1,1],"OAI-SearchBot":["partial","*",1,1],"ClaudeBot":["partial","*",1,1],"Claude-User":["partial","*",1,1],"Claude-SearchBot":["partial","*",1,1],"anthropic-ai":["partial","*",1,1],"PerplexityBot":["partial","*",1,1],"Perplexity-User":["partial","*",1,1],"Google-Extended":["partial","*",1,1],"Applebot-Extended":["partial","*",1,1],"CCBot":["partial","*",1,1],"Bytespider":["partial","*",1,1],"Amazonbot":["partial","*",1,1],"meta-externalagent":["partial","*",1,1],"cohere-ai":["partial","*",1,1],"Diffbot":["partial","*",1,1],"ContextFetchBot":["partial","*",1,1]},"llms":{"status":"missing","http":404,"bytes":103526},"home":{"status":"ok","http":200,"url":"https://www.frbsf.org/","html_tokens":40067,"markdown_tokens":506,"pruned_tokens":106}},
{"rank":9374,"domain":"veneto.it","origin":"https://veneto.it","robots":{"status":"unreachable","http":null,"url":"https://veneto.it/robots.txt","bytes":0,"groups":0,"error":"cannot resolve veneto.it"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":9375,"domain":"ukcop26.org","origin":"https://ukcop26.org","robots":{"status":"refused","http":405,"url":"https://webarchive.nationalarchives.gov.uk/ukgwa/20230401054904/https://ukcop26.org/robots.txt","bytes":2379,"groups":0,"error":"HTTP 405"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":9376,"domain":"mattermost.com","origin":"https://mattermost.com","robots":{"status":"ok","http":200,"url":"https://mattermost.com/robots.txt","bytes":74,"groups":1},"bots":{"GPTBot":["allowed","*",0,1],"ChatGPT-User":["allowed","*",0,1],"OAI-SearchBot":["allowed","*",0,1],"ClaudeBot":["allowed","*",0,1],"Claude-User":["allowed","*",0,1],"Claude-SearchBot":["allowed","*",0,1],"anthropic-ai":["allowed","*",0,1],"PerplexityBot":["allowed","*",0,1],"Perplexity-User":["allowed","*",0,1],"Google-Extended":["allowed","*",0,1],"Applebot-Extended":["allowed","*",0,1],"CCBot":["allowed","*",0,1],"Bytespider":["allowed","*",0,1],"Amazonbot":["allowed","*",0,1],"meta-externalagent":["allowed","*",0,1],"cohere-ai":["allowed","*",0,1],"Diffbot":["allowed","*",0,1],"ContextFetchBot":["allowed","*",0,1]},"llms":{"status":"missing","http":404,"bytes":222969},"home":{"status":"ok","http":200,"url":"https://mattermost.com/","html_tokens":82461,"markdown_tokens":1460,"pruned_tokens":235}},
{"rank":9377,"domain":"home.nl","origin":"https://www.home.nl","robots":{"status":"refused","http":403,"url":"https://www.home.nl/robots.txt","bytes":111,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":9378,"domain":"floridastateparks.org","origin":"https://floridastateparks.org","robots":{"status":"refused","http":403,"url":"https://floridastateparks.org/robots.txt","bytes":4554,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":9379,"domain":"biblia.com","origin":"https://biblia.com","robots":{"status":"ok","http":200,"url":"https://biblia.com/robots.txt","bytes":70,"groups":1},"bots":{"GPTBot":["partial","*",1,0],"ChatGPT-User":["partial","*",1,0],"OAI-SearchBot":["partial","*",1,0],"ClaudeBot":["partial","*",1,0],"Claude-User":["partial","*",1,0],"Claude-SearchBot":["partial","*",1,0],"anthropic-ai":["partial","*",1,0],"PerplexityBot":["partial","*",1,0],"Perplexity-User":["partial","*",1,0],"Google-Extended":["partial","*",1,0],"Applebot-Extended":["partial","*",1,0],"CCBot":["partial","*",1,0],"Bytespider":["partial","*",1,0],"Amazonbot":["partial","*",1,0],"meta-externalagent":["partial","*",1,0],"cohere-ai":["partial","*",1,0],"Diffbot":["partial","*",1,0],"ContextFetchBot":["partial","*",1,0]},"llms":{"status":"missing","http":404,"bytes":33209},"home":{"status":"ok","http":200,"url":"https://biblia.com/","html_tokens":38474,"markdown_tokens":2114,"pruned_tokens":151}},
{"rank":9380,"domain":"bowdoin.edu","origin":"https://bowdoin.edu","robots":{"status":"ok","http":200,"url":"https://www.bowdoin.edu/robots.txt","bytes":772,"groups":15},"bots":{"GPTBot":["partial","*",6,0],"ChatGPT-User":["partial","*",6,0],"OAI-SearchBot":["partial","*",6,0],"ClaudeBot":["partial","*",6,0],"Claude-User":["partial","*",6,0],"Claude-SearchBot":["partial","*",6,0],"anthropic-ai":["partial","*",6,0],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["partial","*",6,0],"Google-Extended":["partial","*",6,0],"Applebot-Extended":["partial","*",6,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",6,0]},"llms":{"status":"missing","http":404,"bytes":16930},"home":{"status":"ok","http":200,"url":"https://www.bowdoin.edu/","html_tokens":31440,"markdown_tokens":418,"pruned_tokens":122}},
{"rank":9381,"domain":"zyxel.com","origin":"https://www.zyxel.com","robots":{"status":"ok","http":200,"url":"https://www.zyxel.com/robots.txt","bytes":1867,"groups":1},"bots":{"GPTBot":["partial","*",28,18],"ChatGPT-User":["partial","*",28,18],"OAI-SearchBot":["partial","*",28,18],"ClaudeBot":["partial","*",28,18],"Claude-User":["partial","*",28,18],"Claude-SearchBot":["partial","*",28,18],"anthropic-ai":["partial","*",28,18],"PerplexityBot":["partial","*",28,18],"Perplexity-User":["partial","*",28,18],"Google-Extended":["partial","*",28,18],"Applebot-Extended":["partial","*",28,18],"CCBot":["partial","*",28,18],"Bytespider":["partial","*",28,18],"Amazonbot":["partial","*",28,18],"meta-externalagent":["partial","*",28,18],"cohere-ai":["partial","*",28,18],"Diffbot":["partial","*",28,18],"ContextFetchBot":["partial","*",28,18]},"llms":{"status":"missing","http":404,"bytes":187},"home":{"status":"ok","http":200,"url":"https://www.zyxel.com/us/en-us/home","html_tokens":30787,"markdown_tokens":317,"pruned_tokens":147}},
{"rank":9382,"domain":"xmshiteng.com","origin":"https://xmshiteng.com","robots":{"status":"unreachable","http":null,"url":"https://xmshiteng.com/robots.txt","bytes":0,"groups":0,"error":"cannot resolve xmshiteng.com"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":9383,"domain":"cp24.com","origin":"https://cp24.com","robots":{"status":"ok","http":200,"url":"https://www.cp24.com/robots.txt","bytes":1226,"groups":12},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["blocked","named",1,0],"OAI-SearchBot":["blocked","named",1,0],"ClaudeBot":["partial","*",22,1],"Claude-User":["partial","*",22,1],"Claude-SearchBot":["partial","*",22,1],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["partial","*",22,1],"Google-Extended":["partial","*",22,1],"Applebot-Extended":["partial","*",22,1],"CCBot":["blocked","named",1,0],"Bytespider":["partial","*",22,1],"Amazonbot":["partial","*",22,1],"meta-externalagent":["partial","*",22,1],"cohere-ai":["partial","*",22,1],"Diffbot":["partial","*",22,1],"ContextFetchBot":["partial","*",22,1]},"llms":{"status":"missing","http":404,"bytes":95862},"home":{"status":"ok","http":200,"url":"https://www.cp24.com/","html_tokens":108793,"markdown_tokens":329,"pruned_tokens":142}},
{"rank":9384,"domain":"spigotmc.org","origin":"https://spigotmc.org","robots":{"status":"ok","http":200,"url":"https://www.spigotmc.org/robots.txt","bytes":195,"groups":2},"bots":{"GPTBot":["partial","*",2,0],"ChatGPT-User":["partial","*",2,0],"OAI-SearchBot":["partial","*",2,0],"ClaudeBot":["partial","*",2,0],"Claude-User":["partial","*",2,0],"Claude-SearchBot":["partial","*",2,0],"anthropic-ai":["partial","*",2,0],"PerplexityBot":["partial","*",2,0],"Perplexity-User":["partial","*",2,0],"Google-Extended":["partial","*",2,0],"Applebot-Extended":["partial","*",2,0],"CCBot":["partial","*",2,0],"Bytespider":["partial","*",2,0],"Amazonbot":["partial","*",2,0],"meta-externalagent":["partial","*",2,0],"cohere-ai":["partial","*",2,0],"Diffbot":["partial","*",2,0],"ContextFetchBot":["partial","*",2,0]},"llms":{"status":"missing","http":404,"bytes":44529},"home":{"status":"ok","http":200,"url":"https://www.spigotmc.org/","html_tokens":22759,"markdown_tokens":2287,"pruned_tokens":101}},
{"rank":9385,"domain":"casinobonusbezvkladu.com","origin":"https://casinobonusbezvkladu.com","robots":{"status":"ok","http":200,"url":"https://casinobonusbezvkladu.com/robots.txt","bytes":2754,"groups":33},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["allowed","*",0,2],"OAI-SearchBot":["allowed","*",0,2],"ClaudeBot":["blocked","named",1,0],"Claude-User":["allowed","*",0,2],"Claude-SearchBot":["allowed","*",0,2],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["allowed","*",0,2],"Perplexity-User":["allowed","*",0,2],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["allowed","*",0,2]},"llms":{"status":"missing","http":404,"bytes":653},"home":{"status":"ok","http":200,"url":"https://casinobonusbezvkladu.com/","html_tokens":44547,"markdown_tokens":25970,"pruned_tokens":115}},
{"rank":9386,"domain":"blazethemes.com","origin":"https://blazethemes.com","robots":{"status":"ok","http":200,"url":"https://blazethemes.com/robots.txt","bytes":375,"groups":2},"bots":{"GPTBot":["allowed","*",0,1],"ChatGPT-User":["allowed","*",0,1],"OAI-SearchBot":["allowed","*",0,1],"ClaudeBot":["allowed","*",0,1],"Claude-User":["allowed","*",0,1],"Claude-SearchBot":["allowed","*",0,1],"anthropic-ai":["allowed","*",0,1],"PerplexityBot":["allowed","*",0,1],"Perplexity-User":["allowed","*",0,1],"Google-Extended":["allowed","*",0,1],"Applebot-Extended":["allowed","*",0,1],"CCBot":["allowed","*",0,1],"Bytespider":["allowed","*",0,1],"Amazonbot":["allowed","*",0,1],"meta-externalagent":["allowed","*",0,1],"cohere-ai":["allowed","*",0,1],"Diffbot":["allowed","*",0,1],"ContextFetchBot":["allowed","*",0,1]},"llms":{"status":"other_text","http":200,"bytes":4866},"home":{"status":"ok","http":200,"url":"https://blazethemes.com/","html_tokens":22636,"markdown_tokens":1067,"pruned_tokens":195}},
{"rank":9387,"domain":"forbes.com.mx","origin":"https://forbes.com.mx","robots":{"status":"ok","http":200,"url":"https://forbes.com.mx/robots.txt","bytes":957,"groups":3},"bots":{"GPTBot":["partial","*",20,1],"ChatGPT-User":["partial","*",20,1],"OAI-SearchBot":["partial","*",20,1],"ClaudeBot":["partial","*",20,1],"Claude-User":["partial","*",20,1],"Claude-SearchBot":["partial","*",20,1],"anthropic-ai":["partial","*",20,1],"PerplexityBot":["partial","*",20,1],"Perplexity-User":["partial","*",20,1],"Google-Extended":["partial","*",20,1],"Applebot-Extended":["partial","*",20,1],"CCBot":["partial","*",20,1],"Bytespider":["partial","*",20,1],"Amazonbot":["partial","*",20,1],"meta-externalagent":["partial","*",20,1],"cohere-ai":["partial","*",20,1],"Diffbot":["partial","*",20,1],"ContextFetchBot":["partial","*",20,1]},"llms":{"status":"other_text","http":200,"bytes":9751},"home":{"status":"ok","http":200,"url":"https://forbes.com.mx/","html_tokens":66731,"markdown_tokens":110,"pruned_tokens":110}},
{"rank":9388,"domain":"todayonline.com","origin":"https://todayonline.com","robots":{"status":"ok","http":200,"url":"https://www.todayonline.com/robots.txt","bytes":25,"groups":1},"bots":{"GPTBot":["blocked","*",1,0],"ChatGPT-User":["blocked","*",1,0],"OAI-SearchBot":["blocked","*",1,0],"ClaudeBot":["blocked","*",1,0],"Claude-User":["blocked","*",1,0],"Claude-SearchBot":["blocked","*",1,0],"anthropic-ai":["blocked","*",1,0],"PerplexityBot":["blocked","*",1,0],"Perplexity-User":["blocked","*",1,0],"Google-Extended":["blocked","*",1,0],"Applebot-Extended":["blocked","*",1,0],"CCBot":["blocked","*",1,0],"Bytespider":["blocked","*",1,0],"Amazonbot":["blocked","*",1,0],"meta-externalagent":["blocked","*",1,0],"cohere-ai":["blocked","*",1,0],"Diffbot":["blocked","*",1,0],"ContextFetchBot":["blocked","*",1,0]},"llms":{"status":"disallowed"},"home":{"status":"disallowed"}},
{"rank":9389,"domain":"infogr.am","origin":"https://infogr.am","robots":{"status":"ok","http":200,"url":"https://infogram.com/robots.txt","bytes":97,"groups":1},"bots":{"GPTBot":["partial","*",2,0],"ChatGPT-User":["partial","*",2,0],"OAI-SearchBot":["partial","*",2,0],"ClaudeBot":["partial","*",2,0],"Claude-User":["partial","*",2,0],"Claude-SearchBot":["partial","*",2,0],"anthropic-ai":["partial","*",2,0],"PerplexityBot":["partial","*",2,0],"Perplexity-User":["partial","*",2,0],"Google-Extended":["partial","*",2,0],"Applebot-Extended":["partial","*",2,0],"CCBot":["partial","*",2,0],"Bytespider":["partial","*",2,0],"Amazonbot":["partial","*",2,0],"meta-externalagent":["partial","*",2,0],"cohere-ai":["partial","*",2,0],"Diffbot":["partial","*",2,0],"ContextFetchBot":["partial","*",2,0]},"llms":{"status":"missing","http":404,"bytes":410273},"home":{"status":"ok","http":200,"url":"https://infogram.com/","html_tokens":126760,"markdown_tokens":320,"pruned_tokens":142}},
{"rank":9390,"domain":"otzywy.com","origin":"https://otzywy.com","robots":{"status":"ok","http":200,"url":"https://www.otzywy.com/robots.txt","bytes":604,"groups":2},"bots":{"GPTBot":["partial","*",7,0],"ChatGPT-User":["partial","*",7,0],"OAI-SearchBot":["partial","*",7,0],"ClaudeBot":["partial","*",7,0],"Claude-User":["partial","*",7,0],"Claude-SearchBot":["partial","*",7,0],"anthropic-ai":["partial","*",7,0],"PerplexityBot":["partial","*",7,0],"Perplexity-User":["partial","*",7,0],"Google-Extended":["partial","*",7,0],"Applebot-Extended":["partial","*",7,0],"CCBot":["partial","*",7,0],"Bytespider":["partial","*",7,0],"Amazonbot":["partial","*",7,0],"meta-externalagent":["partial","*",7,0],"cohere-ai":["partial","*",7,0],"Diffbot":["partial","*",7,0],"ContextFetchBot":["partial","*",7,0]},"llms":{"status":"missing","http":404,"bytes":40685},"home":{"status":"ok","http":200,"url":"https://www.otzywy.com/","html_tokens":29326,"markdown_tokens":985,"pruned_tokens":166}},
{"rank":9391,"domain":"ucl.ac.be","origin":"https://ucl.ac.be","robots":{"status":"unreachable","http":null,"url":"https://ucl.ac.be/robots.txt","bytes":0,"groups":0,"error":"cannot resolve ucl.ac.be"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":9392,"domain":"themarysue.com","origin":"https://themarysue.com","robots":{"status":"refused","http":403,"url":"https://themarysue.com/robots.txt","bytes":4908,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":9393,"domain":"hse.ie","origin":"https://hse.ie","robots":{"status":"ok","http":200,"url":"https://www.hse.ie/robots.txt","bytes":139,"groups":1},"bots":{"GPTBot":["partial","*",2,1],"ChatGPT-User":["partial","*",2,1],"OAI-SearchBot":["partial","*",2,1],"ClaudeBot":["partial","*",2,1],"Claude-User":["partial","*",2,1],"Claude-SearchBot":["partial","*",2,1],"anthropic-ai":["partial","*",2,1],"PerplexityBot":["partial","*",2,1],"Perplexity-User":["partial","*",2,1],"Google-Extended":["partial","*",2,1],"Applebot-Extended":["partial","*",2,1],"CCBot":["partial","*",2,1],"Bytespider":["partial","*",2,1],"Amazonbot":["partial","*",2,1],"meta-externalagent":["partial","*",2,1],"cohere-ai":["partial","*",2,1],"Diffbot":["partial","*",2,1],"ContextFetchBot":["partial","*",2,1]},"llms":{"status":"missing","http":404,"bytes":37340},"home":{"status":"ok","http":200,"url":"https://www.hse.ie/","html_tokens":16419,"markdown_tokens":304,"pruned_tokens":81}},
{"rank":9394,"domain":"lawsociety.org.uk","origin":"https://lawsociety.org.uk","robots":{"status":"refused","http":403,"url":"https://www.lawsociety.org.uk/robots.txt","bytes":29883,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":9395,"domain":"v.gd","origin":"https://v.gd","robots":{"status":"ok","http":200,"url":"https://v.gd/robots.txt","bytes":0,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":1409},"home":{"status":"ok","http":200,"url":"https://v.gd/","html_tokens":768,"markdown_tokens":20,"pruned_tokens":20}},
{"rank":9396,"domain":"hnkgdh.cn","origin":"https://hnkgdh.cn","robots":{"status":"missing","http":404,"url":"https://hnkgdh.cn/robots.txt","bytes":4710,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":4706},"home":{"status":"ok","http":200,"url":"https://hnkgdh.cn/","html_tokens":335,"markdown_tokens":0,"pruned_tokens":0}},
{"rank":9397,"domain":"tonyrobbins.com","origin":"https://tonyrobbins.com","robots":{"status":"ok","http":200,"url":"https://www.tonyrobbins.com/robots.txt","bytes":200,"groups":1},"bots":{"GPTBot":["partial","*",7,1],"ChatGPT-User":["partial","*",7,1],"OAI-SearchBot":["partial","*",7,1],"ClaudeBot":["partial","*",7,1],"Claude-User":["partial","*",7,1],"Claude-SearchBot":["partial","*",7,1],"anthropic-ai":["partial","*",7,1],"PerplexityBot":["partial","*",7,1],"Perplexity-User":["partial","*",7,1],"Google-Extended":["partial","*",7,1],"Applebot-Extended":["partial","*",7,1],"CCBot":["partial","*",7,1],"Bytespider":["partial","*",7,1],"Amazonbot":["partial","*",7,1],"meta-externalagent":["partial","*",7,1],"cohere-ai":["partial","*",7,1],"Diffbot":["partial","*",7,1],"ContextFetchBot":["partial","*",7,1]},"llms":{"status":"html","http":200,"bytes":699467},"home":{"status":"ok","http":200,"url":"https://www.tonyrobbins.com/","html_tokens":174817,"markdown_tokens":344,"pruned_tokens":153}},
{"rank":9398,"domain":"henleyglobal.com","origin":"https://henleyglobal.com","robots":{"status":"ok","http":200,"url":"https://www.henleyglobal.com/robots.txt","bytes":674,"groups":2},"bots":{"GPTBot":["partial","*",15,0],"ChatGPT-User":["partial","*",15,0],"OAI-SearchBot":["partial","*",15,0],"ClaudeBot":["partial","*",15,0],"Claude-User":["partial","*",15,0],"Claude-SearchBot":["partial","*",15,0],"anthropic-ai":["partial","*",15,0],"PerplexityBot":["partial","*",15,0],"Perplexity-User":["partial","*",15,0],"Google-Extended":["partial","*",15,0],"Applebot-Extended":["partial","*",15,0],"CCBot":["partial","*",15,0],"Bytespider":["partial","*",15,0],"Amazonbot":["partial","*",15,0],"meta-externalagent":["partial","*",15,0],"cohere-ai":["partial","*",15,0],"Diffbot":["partial","*",15,0],"ContextFetchBot":["partial","*",15,0]},"llms":{"status":"missing","http":404,"bytes":87936},"home":{"status":"ok","http":200,"url":"https://www.henleyglobal.com/","html_tokens":34890,"markdown_tokens":259,"pruned_tokens":71}},
{"rank":9399,"domain":"ipinfo.io","origin":"https://ipinfo.io","robots":{"status":"ok","http":200,"url":"https://ipinfo.io/robots.txt","bytes":793,"groups":1},"bots":{"GPTBot":["partial","*",18,0],"ChatGPT-User":["partial","*",18,0],"OAI-SearchBot":["partial","*",18,0],"ClaudeBot":["partial","*",18,0],"Claude-User":["partial","*",18,0],"Claude-SearchBot":["partial","*",18,0],"anthropic-ai":["partial","*",18,0],"PerplexityBot":["partial","*",18,0],"Perplexity-User":["partial","*",18,0],"Google-Extended":["partial","*",18,0],"Applebot-Extended":["partial","*",18,0],"CCBot":["partial","*",18,0],"Bytespider":["partial","*",18,0],"Amazonbot":["partial","*",18,0],"meta-externalagent":["partial","*",18,0],"cohere-ai":["partial","*",18,0],"Diffbot":["partial","*",18,0],"ContextFetchBot":["partial","*",18,0]},"llms":{"status":"missing","http":404,"bytes":34516},"home":{"status":"ok","http":200,"url":"https://ipinfo.io/","html_tokens":63336,"markdown_tokens":607,"pruned_tokens":72}},
{"rank":9400,"domain":"turbosquid.com","origin":"https://turbosquid.com","robots":{"status":"ok","http":200,"url":"https://www.turbosquid.com/robots.txt","bytes":5025,"groups":5},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["partial","*",191,0],"OAI-SearchBot":["partial","*",191,0],"ClaudeBot":["partial","*",191,0],"Claude-User":["partial","*",191,0],"Claude-SearchBot":["partial","*",191,0],"anthropic-ai":["partial","*",191,0],"PerplexityBot":["partial","*",191,0],"Perplexity-User":["partial","*",191,0],"Google-Extended":["partial","*",191,0],"Applebot-Extended":["partial","*",191,0],"CCBot":["blocked","named",1,0],"Bytespider":["partial","*",191,0],"Amazonbot":["partial","*",191,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["partial","*",191,0],"Diffbot":["partial","*",191,0],"ContextFetchBot":["partial","*",191,0]},"llms":{"status":"http_error","http":403,"bytes":782},"home":{"status":"http_error","http":403,"url":"https://www.turbosquid.com/"}},
{"rank":9401,"domain":"tv.com","origin":"https://tv.com","robots":{"status":"unreachable","http":null,"url":"https://tv.com/robots.txt","bytes":0,"groups":0,"error":"cannot resolve tv.com"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":9402,"domain":"tu-freiberg.de","origin":"https://tu-freiberg.de","robots":{"status":"ok","http":200,"url":"https://tu-freiberg.de/robots.txt","bytes":2189,"groups":1},"bots":{"GPTBot":["partial","*",54,18],"ChatGPT-User":["partial","*",54,18],"OAI-SearchBot":["partial","*",54,18],"ClaudeBot":["partial","*",54,18],"Claude-User":["partial","*",54,18],"Claude-SearchBot":["partial","*",54,18],"anthropic-ai":["partial","*",54,18],"PerplexityBot":["partial","*",54,18],"Perplexity-User":["partial","*",54,18],"Google-Extended":["partial","*",54,18],"Applebot-Extended":["partial","*",54,18],"CCBot":["partial","*",54,18],"Bytespider":["partial","*",54,18],"Amazonbot":["partial","*",54,18],"meta-externalagent":["partial","*",54,18],"cohere-ai":["partial","*",54,18],"Diffbot":["partial","*",54,18],"ContextFetchBot":["partial","*",54,18]},"llms":{"status":"missing","http":404,"bytes":188},"home":{"status":"ok","http":200,"url":"https://tu-freiberg.de/","html_tokens":44748,"markdown_tokens":420,"pruned_tokens":36}},
{"rank":9403,"domain":"advfn.com","origin":"https://advfn.com","robots":{"status":"ok","http":200,"url":"https://www.advfn.com/robots.txt","bytes":549,"groups":3},"bots":{"GPTBot":["partial","*",16,1],"ChatGPT-User":["partial","*",16,1],"OAI-SearchBot":["partial","*",16,1],"ClaudeBot":["partial","*",16,1],"Claude-User":["partial","*",16,1],"Claude-SearchBot":["partial","*",16,1],"anthropic-ai":["partial","*",16,1],"PerplexityBot":["partial","*",16,1],"Perplexity-User":["partial","*",16,1],"Google-Extended":["partial","*",16,1],"Applebot-Extended":["partial","*",16,1],"CCBot":["partial","*",16,1],"Bytespider":["partial","*",16,1],"Amazonbot":["partial","*",16,1],"meta-externalagent":["partial","*",16,1],"cohere-ai":["partial","*",16,1],"Diffbot":["partial","*",16,1],"ContextFetchBot":["partial","*",16,1]},"llms":{"status":"missing","http":404,"bytes":110833},"home":{"status":"http_error","http":403,"url":"https://advfn.com/"}},
{"rank":9404,"domain":"edaily.co.kr","origin":"https://edaily.co.kr","robots":{"status":"ok","http":200,"url":"https://www.edaily.co.kr/robots.txt","bytes":811,"groups":2},"bots":{"GPTBot":["partial","*",1,0],"ChatGPT-User":["partial","*",1,0],"OAI-SearchBot":["partial","*",1,0],"ClaudeBot":["partial","*",1,0],"Claude-User":["partial","*",1,0],"Claude-SearchBot":["partial","*",1,0],"anthropic-ai":["partial","*",1,0],"PerplexityBot":["partial","*",1,0],"Perplexity-User":["partial","*",1,0],"Google-Extended":["partial","*",1,0],"Applebot-Extended":["partial","*",1,0],"CCBot":["partial","*",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["partial","*",1,0],"meta-externalagent":["partial","*",1,0],"cohere-ai":["partial","*",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",1,0]},"llms":{"status":"html","http":200,"bytes":145009},"home":{"status":"ok","http":200,"url":"https://www.edaily.co.kr/","html_tokens":29745,"markdown_tokens":16,"pruned_tokens":16}},
{"rank":9405,"domain":"thieme-connect.de","origin":"https://thieme-connect.de","robots":{"status":"ok","http":200,"url":"https://thieme-connect.de/robots.txt","bytes":1931,"groups":43},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["blocked","named",1,0],"OAI-SearchBot":["blocked","named",1,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",8,0],"Claude-SearchBot":["partial","*",8,0],"anthropic-ai":["partial","*",8,0],"PerplexityBot":["partial","*",8,0],"Perplexity-User":["partial","*",8,0],"Google-Extended":["partial","*",8,0],"Applebot-Extended":["partial","*",8,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",2,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["partial","*",8,0],"Diffbot":["partial","*",8,0],"ContextFetchBot":["partial","*",8,0]},"llms":{"status":"html","http":200,"bytes":4767},"home":{"status":"ok","http":200,"url":"https://thieme-connect.de/","html_tokens":1192,"markdown_tokens":94,"pruned_tokens":66}},
{"rank":9406,"domain":"pw.edu.pl","origin":"https://pw.edu.pl","robots":{"status":"ok","http":200,"url":"https://pw.edu.pl/robots.txt","bytes":2141,"groups":1},"bots":{"GPTBot":["partial","*",35,22],"ChatGPT-User":["partial","*",35,22],"OAI-SearchBot":["partial","*",35,22],"ClaudeBot":["partial","*",35,22],"Claude-User":["partial","*",35,22],"Claude-SearchBot":["partial","*",35,22],"anthropic-ai":["partial","*",35,22],"PerplexityBot":["partial","*",35,22],"Perplexity-User":["partial","*",35,22],"Google-Extended":["partial","*",35,22],"Applebot-Extended":["partial","*",35,22],"CCBot":["partial","*",35,22],"Bytespider":["partial","*",35,22],"Amazonbot":["partial","*",35,22],"meta-externalagent":["partial","*",35,22],"cohere-ai":["partial","*",35,22],"Diffbot":["partial","*",35,22],"ContextFetchBot":["partial","*",35,22]},"llms":{"status":"missing","http":404,"bytes":182},"home":{"status":"ok","http":200,"url":"https://pw.edu.pl/","html_tokens":20388,"markdown_tokens":761,"pruned_tokens":112}},
{"rank":9407,"domain":"unilad.com","origin":"https://unilad.com","robots":{"status":"ok","http":200,"url":"https://www.unilad.com/robots.txt","bytes":210,"groups":2},"bots":{"GPTBot":["partial","*",2,1],"ChatGPT-User":["partial","*",2,1],"OAI-SearchBot":["partial","*",2,1],"ClaudeBot":["partial","*",2,1],"Claude-User":["partial","*",2,1],"Claude-SearchBot":["partial","*",2,1],"anthropic-ai":["partial","*",2,1],"PerplexityBot":["partial","*",2,1],"Perplexity-User":["partial","*",2,1],"Google-Extended":["partial","*",2,1],"Applebot-Extended":["partial","*",2,1],"CCBot":["partial","*",2,1],"Bytespider":["partial","*",2,1],"Amazonbot":["partial","*",2,1],"meta-externalagent":["partial","*",2,1],"cohere-ai":["partial","*",2,1],"Diffbot":["partial","*",2,1],"ContextFetchBot":["partial","*",2,1]},"llms":{"status":"missing","http":404,"bytes":48031},"home":{"status":"ok","http":200,"url":"https://www.unilad.com/","html_tokens":70826,"markdown_tokens":264,"pruned_tokens":129}},
{"rank":9408,"domain":"afi.com","origin":"https://afi.com","robots":{"status":"ok","http":200,"url":"https://afi.com/robots.txt","bytes":2676,"groups":32},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["allowed","*",0,1],"OAI-SearchBot":["allowed","*",0,1],"ClaudeBot":["blocked","named",1,0],"Claude-User":["allowed","*",0,1],"Claude-SearchBot":["allowed","*",0,1],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["allowed","*",0,1],"Perplexity-User":["allowed","*",0,1],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["allowed","*",0,1]},"llms":{"status":"http_error","http":403,"bytes":4540},"home":{"status":"http_error","http":403,"url":"https://afi.com/"}},
{"rank":9409,"domain":"shodan.io","origin":"https://shodan.io","robots":{"status":"ok","http":200,"url":"https://www.shodan.io/robots.txt","bytes":49,"groups":1},"bots":{"GPTBot":["partial","*",1,0],"ChatGPT-User":["partial","*",1,0],"OAI-SearchBot":["partial","*",1,0],"ClaudeBot":["partial","*",1,0],"Claude-User":["partial","*",1,0],"Claude-SearchBot":["partial","*",1,0],"anthropic-ai":["partial","*",1,0],"PerplexityBot":["partial","*",1,0],"Perplexity-User":["partial","*",1,0],"Google-Extended":["partial","*",1,0],"Applebot-Extended":["partial","*",1,0],"CCBot":["partial","*",1,0],"Bytespider":["partial","*",1,0],"Amazonbot":["partial","*",1,0],"meta-externalagent":["partial","*",1,0],"cohere-ai":["partial","*",1,0],"Diffbot":["partial","*",1,0],"ContextFetchBot":["partial","*",1,0]},"llms":{"status":"present","http":200,"bytes":5122},"home":{"status":"ok","http":200,"url":"https://www.shodan.io/","html_tokens":3221,"markdown_tokens":406,"pruned_tokens":244}},
{"rank":9410,"domain":"northumbria.ac.uk","origin":"https://northumbria.ac.uk","robots":{"status":"ok","http":200,"url":"https://www.northumbria.ac.uk/robots.txt","bytes":581,"groups":1},"bots":{"GPTBot":["partial","*",18,0],"ChatGPT-User":["partial","*",18,0],"OAI-SearchBot":["partial","*",18,0],"ClaudeBot":["partial","*",18,0],"Claude-User":["partial","*",18,0],"Claude-SearchBot":["partial","*",18,0],"anthropic-ai":["partial","*",18,0],"PerplexityBot":["partial","*",18,0],"Perplexity-User":["partial","*",18,0],"Google-Extended":["partial","*",18,0],"Applebot-Extended":["partial","*",18,0],"CCBot":["partial","*",18,0],"Bytespider":["partial","*",18,0],"Amazonbot":["partial","*",18,0],"meta-externalagent":["partial","*",18,0],"cohere-ai":["partial","*",18,0],"Diffbot":["partial","*",18,0],"ContextFetchBot":["partial","*",18,0]},"llms":{"status":"missing","http":404,"bytes":1245},"home":{"status":"ok","http":200,"url":"https://www.northumbria.ac.uk/","html_tokens":57579,"markdown_tokens":218,"pruned_tokens":87}},
{"rank":9411,"domain":"wheel-of-fortune-pokie.com","origin":"https://wheel-of-fortune-pokie.com","robots":{"status":"ok","http":200,"url":"https://wheel-of-fortune-pokie.com/robots.txt","bytes":557,"groups":1},"bots":{"GPTBot":["partial","*",19,0],"ChatGPT-User":["partial","*",19,0],"OAI-SearchBot":["partial","*",19,0],"ClaudeBot":["partial","*",19,0],"Claude-User":["partial","*",19,0],"Claude-SearchBot":["partial","*",19,0],"anthropic-ai":["partial","*",19,0],"PerplexityBot":["partial","*",19,0],"Perplexity-User":["partial","*",19,0],"Google-Extended":["partial","*",19,0],"Applebot-Extended":["partial","*",19,0],"CCBot":["partial","*",19,0],"Bytespider":["partial","*",19,0],"Amazonbot":["partial","*",19,0],"meta-externalagent":["partial","*",19,0],"cohere-ai":["partial","*",19,0],"Diffbot":["partial","*",19,0],"ContextFetchBot":["partial","*",19,0]},"llms":{"status":"missing","http":404,"bytes":36964},"home":{"status":"ok","http":200,"url":"https://wheel-of-fortune-pokie.com/","html_tokens":29466,"markdown_tokens":2181,"pruned_tokens":1127}},
{"rank":9412,"domain":"jjjshg.com","origin":"http://jjjshg.com","robots":{"status":"ok","http":200,"url":"http://jjjshg.com/robots.txt","bytes":247,"groups":1},"bots":{"GPTBot":["partial","*",10,0],"ChatGPT-User":["partial","*",10,0],"OAI-SearchBot":["partial","*",10,0],"ClaudeBot":["partial","*",10,0],"Claude-User":["partial","*",10,0],"Claude-SearchBot":["partial","*",10,0],"anthropic-ai":["partial","*",10,0],"PerplexityBot":["partial","*",10,0],"Perplexity-User":["partial","*",10,0],"Google-Extended":["partial","*",10,0],"Applebot-Extended":["partial","*",10,0],"CCBot":["partial","*",10,0],"Bytespider":["partial","*",10,0],"Amazonbot":["partial","*",10,0],"meta-externalagent":["partial","*",10,0],"cohere-ai":["partial","*",10,0],"Diffbot":["partial","*",10,0],"ContextFetchBot":["partial","*",10,0]},"llms":{"status":"missing","http":404,"bytes":3693},"home":{"status":"ok","http":200,"url":"http://jjjshg.com/","html_tokens":5794,"markdown_tokens":127,"pruned_tokens":127}},
{"rank":9413,"domain":"yhj.com.cn","origin":"https://yhj.com.cn","robots":{"status":"unreachable","http":null,"url":"https://yhj.com.cn/robots.txt","bytes":0,"groups":0,"error":"cannot resolve yhj.com.cn"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":9414,"domain":"sg-host.com","origin":"https://sg-host.com","robots":{"status":"unreachable","http":null,"url":"https://sg-host.com/robots.txt","bytes":0,"groups":0,"error":"[SSL: CERTIFICATE_VERIFY_FAILED] certificate verify failed: Hostname mismatch, certificate is not valid for 'sg-host.com'. (_ssl.c:1028)"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":9415,"domain":"nationthailand.com","origin":"https://nationthailand.com","robots":{"status":"ok","http":200,"url":"https://www.nationthailand.com/robots.txt","bytes":232,"groups":1},"bots":{"GPTBot":["partial","*",3,0],"ChatGPT-User":["partial","*",3,0],"OAI-SearchBot":["partial","*",3,0],"ClaudeBot":["partial","*",3,0],"Claude-User":["partial","*",3,0],"Claude-SearchBot":["partial","*",3,0],"anthropic-ai":["partial","*",3,0],"PerplexityBot":["partial","*",3,0],"Perplexity-User":["partial","*",3,0],"Google-Extended":["partial","*",3,0],"Applebot-Extended":["partial","*",3,0],"CCBot":["partial","*",3,0],"Bytespider":["partial","*",3,0],"Amazonbot":["partial","*",3,0],"meta-externalagent":["partial","*",3,0],"cohere-ai":["partial","*",3,0],"Diffbot":["partial","*",3,0],"ContextFetchBot":["partial","*",3,0]},"llms":{"status":"present","http":200,"bytes":13076},"home":{"status":"ok","http":200,"url":"https://www.nationthailand.com/","html_tokens":38417,"markdown_tokens":85,"pruned_tokens":85}},
{"rank":9416,"domain":"stryker.com","origin":"https://stryker.com","robots":{"status":"ok","http":200,"url":"https://www.stryker.com/robots.txt","bytes":158,"groups":1},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":2868},"home":{"status":"ok","http":200,"url":"https://www.stryker.com/","html_tokens":12129,"markdown_tokens":7,"pruned_tokens":7}},
{"rank":9417,"domain":"princess.com","origin":"https://princess.com","robots":{"status":"ok","http":200,"url":"https://www.princess.com/robots.txt","bytes":1874,"groups":1},"bots":{"GPTBot":["partial","*",34,1],"ChatGPT-User":["partial","named",34,1],"OAI-SearchBot":["partial","named",34,1],"ClaudeBot":["partial","*",34,1],"Claude-User":["partial","named",34,1],"Claude-SearchBot":["partial","named",34,1],"anthropic-ai":["partial","*",34,1],"PerplexityBot":["partial","named",34,1],"Perplexity-User":["partial","named",34,1],"Google-Extended":["partial","*",34,1],"Applebot-Extended":["partial","*",34,1],"CCBot":["partial","*",34,1],"Bytespider":["partial","*",34,1],"Amazonbot":["partial","*",34,1],"meta-externalagent":["partial","*",34,1],"cohere-ai":["partial","*",34,1],"Diffbot":["partial","*",34,1],"ContextFetchBot":["partial","*",34,1]},"llms":{"status":"other_text","http":200,"bytes":1769},"home":{"status":"ok","http":200,"url":"https://www.princess.com/","html_tokens":223653,"markdown_tokens":437,"pruned_tokens":71}},
{"rank":9418,"domain":"ispsystem.com","origin":"https://ispsystem.com","robots":{"status":"ok","http":200,"url":"https://www.ispsystem.com/robots.txt","bytes":7366,"groups":2},"bots":{"GPTBot":["partial","*",118,19],"ChatGPT-User":["partial","*",118,19],"OAI-SearchBot":["partial","*",118,19],"ClaudeBot":["partial","*",118,19],"Claude-User":["partial","*",118,19],"Claude-SearchBot":["partial","*",118,19],"anthropic-ai":["partial","*",118,19],"PerplexityBot":["partial","*",118,19],"Perplexity-User":["partial","*",118,19],"Google-Extended":["partial","*",118,19],"Applebot-Extended":["partial","*",118,19],"CCBot":["partial","*",118,19],"Bytespider":["partial","*",118,19],"Amazonbot":["partial","*",118,19],"meta-externalagent":["partial","*",118,19],"cohere-ai":["partial","*",118,19],"Diffbot":["partial","*",118,19],"ContextFetchBot":["partial","*",118,19]},"llms":{"status":"present","http":200,"bytes":1309},"home":{"status":"ok","http":200,"url":"https://www.ispsystem.com/","html_tokens":32151,"markdown_tokens":628,"pruned_tokens":148}},
{"rank":9419,"domain":"xylem.com","origin":"https://xylem.com","robots":{"status":"refused","http":403,"url":"https://www.xylem.com/robots.txt","bytes":4542,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":9420,"domain":"trustwave.com","origin":"https://trustwave.com","robots":{"status":"ok","http":200,"url":"https://www.levelblue.com/robots.txt","bytes":960,"groups":1},"bots":{"GPTBot":["partial","*",13,13],"ChatGPT-User":["partial","*",13,13],"OAI-SearchBot":["partial","*",13,13],"ClaudeBot":["partial","*",13,13],"Claude-User":["partial","*",13,13],"Claude-SearchBot":["partial","*",13,13],"anthropic-ai":["partial","*",13,13],"PerplexityBot":["partial","*",13,13],"Perplexity-User":["partial","*",13,13],"Google-Extended":["partial","*",13,13],"Applebot-Extended":["partial","*",13,13],"CCBot":["partial","*",13,13],"Bytespider":["partial","*",13,13],"Amazonbot":["partial","*",13,13],"meta-externalagent":["partial","*",13,13],"cohere-ai":["partial","*",13,13],"Diffbot":["partial","*",13,13],"ContextFetchBot":["partial","*",13,13]},"llms":{"status":"missing","http":404,"bytes":55243},"home":{"status":"ok","http":200,"url":"https://www.levelblue.com/","html_tokens":26436,"markdown_tokens":550,"pruned_tokens":119}},
{"rank":9421,"domain":"ownernetworks.com","origin":"https://ownernetworks.com","robots":{"status":"ok","http":200,"url":"https://ownernetworks.com/robots.txt","bytes":88,"groups":1},"bots":{"GPTBot":["partial","*",3,1],"ChatGPT-User":["partial","*",3,1],"OAI-SearchBot":["partial","*",3,1],"ClaudeBot":["partial","*",3,1],"Claude-User":["partial","*",3,1],"Claude-SearchBot":["partial","*",3,1],"anthropic-ai":["partial","*",3,1],"PerplexityBot":["partial","*",3,1],"Perplexity-User":["partial","*",3,1],"Google-Extended":["partial","*",3,1],"Applebot-Extended":["partial","*",3,1],"CCBot":["partial","*",3,1],"Bytespider":["partial","*",3,1],"Amazonbot":["partial","*",3,1],"meta-externalagent":["partial","*",3,1],"cohere-ai":["partial","*",3,1],"Diffbot":["partial","*",3,1],"ContextFetchBot":["partial","*",3,1]},"llms":{"status":"missing","http":404,"bytes":103},"home":{"status":"ok","http":200,"url":"https://www.ownernetworks.com/default.aspx","html_tokens":27674,"markdown_tokens":928,"pruned_tokens":63}},
{"rank":9422,"domain":"frost.com","origin":"https://frost.com","robots":{"status":"ok","http":200,"url":"https://frost.com/robots.txt","bytes":540,"groups":2},"bots":{"GPTBot":["partial","*",5,1],"ChatGPT-User":["partial","*",5,1],"OAI-SearchBot":["partial","*",5,1],"ClaudeBot":["partial","*",5,1],"Claude-User":["partial","*",5,1],"Claude-SearchBot":["partial","*",5,1],"anthropic-ai":["partial","*",5,1],"PerplexityBot":["partial","*",5,1],"Perplexity-User":["partial","*",5,1],"Google-Extended":["partial","*",5,1],"Applebot-Extended":["partial","*",5,1],"CCBot":["partial","*",5,1],"Bytespider":["partial","*",5,1],"Amazonbot":["partial","*",5,1],"meta-externalagent":["partial","*",5,1],"cohere-ai":["partial","*",5,1],"Diffbot":["partial","*",5,1],"ContextFetchBot":["partial","*",5,1]},"llms":{"status":"present","http":200,"bytes":11839},"home":{"status":"ok","http":200,"url":"https://www.frost.com/","html_tokens":207318,"markdown_tokens":319,"pruned_tokens":159}},
{"rank":9423,"domain":"zenhabits.net","origin":"https://zenhabits.net","robots":{"status":"ok","http":200,"url":"https://zenhabits.net/robots.txt","bytes":1248,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":6128},"home":{"status":"ok","http":200,"url":"https://zenhabits.net/","html_tokens":2867,"markdown_tokens":1158,"pruned_tokens":145}},
{"rank":9424,"domain":"maps.google.at","origin":"https://maps.google.at","robots":{"status":"ok","http":200,"url":"https://maps.google.at/robots.txt","bytes":6624,"groups":4},"bots":{"GPTBot":["partial","*",169,64],"ChatGPT-User":["partial","*",169,64],"OAI-SearchBot":["partial","*",169,64],"ClaudeBot":["partial","*",169,64],"Claude-User":["partial","*",169,64],"Claude-SearchBot":["partial","*",169,64],"anthropic-ai":["partial","*",169,64],"PerplexityBot":["partial","*",169,64],"Perplexity-User":["partial","*",169,64],"Google-Extended":["partial","*",169,64],"Applebot-Extended":["partial","*",169,64],"CCBot":["partial","*",169,64],"Bytespider":["partial","*",169,64],"Amazonbot":["partial","*",169,64],"meta-externalagent":["partial","*",169,64],"cohere-ai":["partial","*",169,64],"Diffbot":["partial","*",169,64],"ContextFetchBot":["partial","*",169,64]},"llms":{"status":"missing","http":404,"bytes":1569},"home":{"status":"ok","http":200,"url":"https://www.google.at/maps","html_tokens":54519,"markdown_tokens":30,"pruned_tokens":30}},
{"rank":9425,"domain":"thcxjsjt.com","origin":"https://www.thcxjsjt.com","robots":{"status":"missing","http":404,"url":"https://www.thcxjsjt.com/robots.txt","bytes":1163,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":1163},"home":{"status":"ok","http":200,"url":"https://www.thcxjsjt.com/","html_tokens":73558,"markdown_tokens":506,"pruned_tokens":151}},
{"rank":9426,"domain":"brak.de","origin":"https://brak.de","robots":{"status":"ok","http":200,"url":"https://www.brak.de/robots.txt","bytes":147,"groups":1},"bots":{"GPTBot":["partial","*",2,1],"ChatGPT-User":["partial","*",2,1],"OAI-SearchBot":["partial","*",2,1],"ClaudeBot":["partial","*",2,1],"Claude-User":["partial","*",2,1],"Claude-SearchBot":["partial","*",2,1],"anthropic-ai":["partial","*",2,1],"PerplexityBot":["partial","*",2,1],"Perplexity-User":["partial","*",2,1],"Google-Extended":["partial","*",2,1],"Applebot-Extended":["partial","*",2,1],"CCBot":["partial","*",2,1],"Bytespider":["partial","*",2,1],"Amazonbot":["partial","*",2,1],"meta-externalagent":["partial","*",2,1],"cohere-ai":["partial","*",2,1],"Diffbot":["partial","*",2,1],"ContextFetchBot":["partial","*",2,1]},"llms":{"status":"missing","http":404,"bytes":185171},"home":{"status":"ok","http":200,"url":"https://www.brak.de/","html_tokens":80538,"markdown_tokens":267,"pruned_tokens":154}},
{"rank":9427,"domain":"scribbr.com","origin":"https://scribbr.com","robots":{"status":"ok","http":200,"url":"https://www.scribbr.com/robots.txt","bytes":1200,"groups":15},"bots":{"GPTBot":["partial","*",11,2],"ChatGPT-User":["partial","*",11,2],"OAI-SearchBot":["partial","*",11,2],"ClaudeBot":["partial","named",1,0],"Claude-User":["partial","*",11,2],"Claude-SearchBot":["partial","*",11,2],"anthropic-ai":["partial","*",11,2],"PerplexityBot":["partial","*",11,2],"Perplexity-User":["partial","*",11,2],"Google-Extended":["partial","*",11,2],"Applebot-Extended":["partial","*",11,2],"CCBot":["partial","*",11,2],"Bytespider":["partial","*",11,2],"Amazonbot":["partial","*",11,2],"meta-externalagent":["partial","*",11,2],"cohere-ai":["partial","*",11,2],"Diffbot":["partial","*",11,2],"ContextFetchBot":["partial","*",11,2]},"llms":{"status":"missing","http":404,"bytes":123787},"home":{"status":"http_error","http":403,"url":"https://scribbr.com/"}},
{"rank":9428,"domain":"commentcamarche.net","origin":"https://commentcamarche.net","robots":{"status":"html","http":200,"url":"https://www.commentcamarche.net/","bytes":214548,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":9429,"domain":"abcactionnews.com","origin":"https://abcactionnews.com","robots":{"status":"ok","http":200,"url":"https://www.abcactionnews.com/robots.txt","bytes":395,"groups":4},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["partial","*",5,0],"OAI-SearchBot":["partial","*",5,0],"ClaudeBot":["partial","*",5,0],"Claude-User":["partial","*",5,0],"Claude-SearchBot":["partial","*",5,0],"anthropic-ai":["partial","*",5,0],"PerplexityBot":["partial","*",5,0],"Perplexity-User":["partial","*",5,0],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["partial","*",5,0],"CCBot":["partial","*",5,0],"Bytespider":["partial","*",5,0],"Amazonbot":["partial","*",5,0],"meta-externalagent":["partial","*",5,0],"cohere-ai":["partial","*",5,0],"Diffbot":["partial","*",5,0],"ContextFetchBot":["partial","*",5,0]},"llms":{"status":"missing","http":404,"bytes":79499},"home":{"status":"ok","http":200,"url":"https://www.tampabay28.com/","html_tokens":66502,"markdown_tokens":87,"pruned_tokens":87}},
{"rank":9430,"domain":"txdot.gov","origin":"https://www.txdot.gov","robots":{"status":"missing","http":404,"url":"https://www.txdot.gov/robots.txt","bytes":147832,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":147832},"home":{"status":"ok","http":200,"url":"https://www.txdot.gov/","html_tokens":46654,"markdown_tokens":267,"pruned_tokens":130}},
{"rank":9431,"domain":"premierinn.com","origin":"https://premierinn.com","robots":{"status":"unreachable","http":null,"url":"https://premierinn.com/robots.txt","bytes":0,"groups":0,"error":"<StreamReset stream_id:1, error_code:2, remote_reset:True>"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":9432,"domain":"turner.com","origin":"https://turner.com","robots":{"status":"unreachable","http":null,"url":"https://turner.com/robots.txt","bytes":0,"groups":0,"error":"[SSL: CERTIFICATE_VERIFY_FAILED] certificate verify failed: Hostname mismatch, certificate is not valid for 'turner.com'. (_ssl.c:1028)"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":9433,"domain":"ahk.de","origin":"https://ahk.de","robots":{"status":"ok","http":200,"url":"https://www.ahk.de/robots.txt","bytes":118,"groups":1},"bots":{"GPTBot":["partial","*",3,0],"ChatGPT-User":["partial","*",3,0],"OAI-SearchBot":["partial","*",3,0],"ClaudeBot":["partial","*",3,0],"Claude-User":["partial","*",3,0],"Claude-SearchBot":["partial","*",3,0],"anthropic-ai":["partial","*",3,0],"PerplexityBot":["partial","*",3,0],"Perplexity-User":["partial","*",3,0],"Google-Extended":["partial","*",3,0],"Applebot-Extended":["partial","*",3,0],"CCBot":["partial","*",3,0],"Bytespider":["partial","*",3,0],"Amazonbot":["partial","*",3,0],"meta-externalagent":["partial","*",3,0],"cohere-ai":["partial","*",3,0],"Diffbot":["partial","*",3,0],"ContextFetchBot":["partial","*",3,0]},"llms":{"status":"missing","http":404,"bytes":168704},"home":{"status":"ok","http":200,"url":"https://www.ahk.de/de","html_tokens":48077,"markdown_tokens":234,"pruned_tokens":79}},
{"rank":9434,"domain":"onefootball.com","origin":"https://onefootball.com","robots":{"status":"ok","http":200,"url":"https://onefootball.com/robots.txt","bytes":136,"groups":1},"bots":{"GPTBot":["partial","*",4,0],"ChatGPT-User":["partial","*",4,0],"OAI-SearchBot":["partial","*",4,0],"ClaudeBot":["partial","*",4,0],"Claude-User":["partial","*",4,0],"Claude-SearchBot":["partial","*",4,0],"anthropic-ai":["partial","*",4,0],"PerplexityBot":["partial","*",4,0],"Perplexity-User":["partial","*",4,0],"Google-Extended":["partial","*",4,0],"Applebot-Extended":["partial","*",4,0],"CCBot":["partial","*",4,0],"Bytespider":["partial","*",4,0],"Amazonbot":["partial","*",4,0],"meta-externalagent":["partial","*",4,0],"cohere-ai":["partial","*",4,0],"Diffbot":["partial","*",4,0],"ContextFetchBot":["partial","*",4,0]},"llms":{"status":"missing","http":404,"bytes":89146},"home":{"status":"ok","http":200,"url":"https://onefootball.com/en/home","html_tokens":87957,"markdown_tokens":67,"pruned_tokens":67}},
{"rank":9435,"domain":"webnode.es","origin":"https://webnode.es","robots":{"status":"missing","http":404,"url":"https://webnode.es/robots.txt","bytes":0,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"present","http":200,"bytes":3499},"home":{"status":"ok","http":200,"url":"https://www.webnode.com/es/","html_tokens":61886,"markdown_tokens":82,"pruned_tokens":82}},
{"rank":9436,"domain":"trendforce.com","origin":"https://trendforce.com","robots":{"status":"ok","http":200,"url":"https://www.trendforce.com/robots.txt","bytes":405,"groups":1},"bots":{"GPTBot":["partial","*",6,0],"ChatGPT-User":["partial","*",6,0],"OAI-SearchBot":["partial","*",6,0],"ClaudeBot":["partial","*",6,0],"Claude-User":["partial","*",6,0],"Claude-SearchBot":["partial","*",6,0],"anthropic-ai":["partial","*",6,0],"PerplexityBot":["partial","*",6,0],"Perplexity-User":["partial","*",6,0],"Google-Extended":["partial","*",6,0],"Applebot-Extended":["partial","*",6,0],"CCBot":["partial","*",6,0],"Bytespider":["partial","*",6,0],"Amazonbot":["partial","*",6,0],"meta-externalagent":["partial","*",6,0],"cohere-ai":["partial","*",6,0],"Diffbot":["partial","*",6,0],"ContextFetchBot":["partial","*",6,0]},"llms":{"status":"missing","http":404,"bytes":123645},"home":{"status":"ok","http":200,"url":"https://www.trendforce.com/","html_tokens":35752,"markdown_tokens":278,"pruned_tokens":87}},
{"rank":9437,"domain":"wu.ac.at","origin":"https://wu.ac.at","robots":{"status":"ok","http":200,"url":"https://www.wu.ac.at/robots.txt","bytes":113,"groups":1},"bots":{"GPTBot":["partial","*",4,0],"ChatGPT-User":["partial","*",4,0],"OAI-SearchBot":["partial","*",4,0],"ClaudeBot":["partial","*",4,0],"Claude-User":["partial","*",4,0],"Claude-SearchBot":["partial","*",4,0],"anthropic-ai":["partial","*",4,0],"PerplexityBot":["partial","*",4,0],"Perplexity-User":["partial","*",4,0],"Google-Extended":["partial","*",4,0],"Applebot-Extended":["partial","*",4,0],"CCBot":["partial","*",4,0],"Bytespider":["partial","*",4,0],"Amazonbot":["partial","*",4,0],"meta-externalagent":["partial","*",4,0],"cohere-ai":["partial","*",4,0],"Diffbot":["partial","*",4,0],"ContextFetchBot":["partial","*",4,0]},"llms":{"status":"missing","http":404,"bytes":101313},"home":{"status":"ok","http":200,"url":"https://www.wu.ac.at/","html_tokens":29696,"markdown_tokens":164,"pruned_tokens":164}},
{"rank":9438,"domain":"lycos.co.uk","origin":"http://lycos.co.uk","robots":{"status":"html","http":200,"url":"http://lycos.co.uk/robots.txt","bytes":94,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":9439,"domain":"adoptium.net","origin":"https://adoptium.net","robots":{"status":"ok","http":200,"url":"https://adoptium.net/robots.txt","bytes":82,"groups":1},"bots":{"GPTBot":["partial","*",1,1],"ChatGPT-User":["partial","*",1,1],"OAI-SearchBot":["partial","*",1,1],"ClaudeBot":["partial","*",1,1],"Claude-User":["partial","*",1,1],"Claude-SearchBot":["partial","*",1,1],"anthropic-ai":["partial","*",1,1],"PerplexityBot":["partial","*",1,1],"Perplexity-User":["partial","*",1,1],"Google-Extended":["partial","*",1,1],"Applebot-Extended":["partial","*",1,1],"CCBot":["partial","*",1,1],"Bytespider":["partial","*",1,1],"Amazonbot":["partial","*",1,1],"meta-externalagent":["partial","*",1,1],"cohere-ai":["partial","*",1,1],"Diffbot":["partial","*",1,1],"ContextFetchBot":["partial","*",1,1]},"llms":{"status":"missing","http":404,"bytes":60149},"home":{"status":"ok","http":200,"url":"https://adoptium.net/","html_tokens":51151,"markdown_tokens":526,"pruned_tokens":356}},
{"rank":9440,"domain":"aszmr.com","origin":"http://aszmr.com","robots":{"status":"missing","http":404,"url":"http://aszmr.com/robots.txt","bytes":5082,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":5078},"home":{"status":"ok","http":200,"url":"http://aszmr.com/","html_tokens":13566,"markdown_tokens":86,"pruned_tokens":86}},
{"rank":9441,"domain":"tzsjysg.com","origin":"http://tzsjysg.com","robots":{"status":"ok","http":200,"url":"http://tzsjysg.com/robots.txt","bytes":249,"groups":1},"bots":{"GPTBot":["partial","*",10,0],"ChatGPT-User":["partial","*",10,0],"OAI-SearchBot":["partial","*",10,0],"ClaudeBot":["partial","*",10,0],"Claude-User":["partial","*",10,0],"Claude-SearchBot":["partial","*",10,0],"anthropic-ai":["partial","*",10,0],"PerplexityBot":["partial","*",10,0],"Perplexity-User":["partial","*",10,0],"Google-Extended":["partial","*",10,0],"Applebot-Extended":["partial","*",10,0],"CCBot":["partial","*",10,0],"Bytespider":["partial","*",10,0],"Amazonbot":["partial","*",10,0],"meta-externalagent":["partial","*",10,0],"cohere-ai":["partial","*",10,0],"Diffbot":["partial","*",10,0],"ContextFetchBot":["partial","*",10,0]},"llms":{"status":"missing","http":404,"bytes":3749},"home":{"status":"ok","http":200,"url":"http://tzsjysg.com/","html_tokens":15583,"markdown_tokens":70,"pruned_tokens":70}},
{"rank":9442,"domain":"tianlonggroup.com.cn","origin":"http://tianlonggroup.com.cn","robots":{"status":"missing","http":404,"url":"http://tianlonggroup.com.cn/robots.txt","bytes":5044,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":5040},"home":{"status":"ok","http":200,"url":"http://tianlonggroup.com.cn/","html_tokens":5729,"markdown_tokens":24,"pruned_tokens":24}},
{"rank":9443,"domain":"onepeloton.com","origin":"https://onepeloton.com","robots":{"status":"ok","http":200,"url":"https://www.onepeloton.com/robots.txt","bytes":186,"groups":1},"bots":{"GPTBot":["partial","*",5,0],"ChatGPT-User":["partial","*",5,0],"OAI-SearchBot":["partial","*",5,0],"ClaudeBot":["partial","*",5,0],"Claude-User":["partial","*",5,0],"Claude-SearchBot":["partial","*",5,0],"anthropic-ai":["partial","*",5,0],"PerplexityBot":["partial","*",5,0],"Perplexity-User":["partial","*",5,0],"Google-Extended":["partial","*",5,0],"Applebot-Extended":["partial","*",5,0],"CCBot":["partial","*",5,0],"Bytespider":["partial","*",5,0],"Amazonbot":["partial","*",5,0],"meta-externalagent":["partial","*",5,0],"cohere-ai":["partial","*",5,0],"Diffbot":["partial","*",5,0],"ContextFetchBot":["partial","*",5,0]},"llms":{"status":"missing","http":404,"bytes":250968},"home":{"status":"ok","http":200,"url":"https://www.onepeloton.com/","html_tokens":119520,"markdown_tokens":1227,"pruned_tokens":54}},
{"rank":9444,"domain":"opentelemetry.io","origin":"https://opentelemetry.io","robots":{"status":"ok","http":200,"url":"https://opentelemetry.io/robots.txt","bytes":14,"groups":1},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"present","http":200,"bytes":2151},"home":{"status":"ok","http":200,"url":"https://opentelemetry.io/","html_tokens":8338,"markdown_tokens":278,"pruned_tokens":221}},
{"rank":9445,"domain":"unrwa.org","origin":"https://unrwa.org","robots":{"status":"refused","http":403,"url":"https://unrwa.org/robots.txt","bytes":5504,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":9446,"domain":"truste.com","origin":"https://truste.com","robots":{"status":"ok","http":200,"url":"https://trustarc.com/robots.txt","bytes":213,"groups":1},"bots":{"GPTBot":["partial","*",4,0],"ChatGPT-User":["partial","*",4,0],"OAI-SearchBot":["partial","*",4,0],"ClaudeBot":["partial","*",4,0],"Claude-User":["partial","*",4,0],"Claude-SearchBot":["partial","*",4,0],"anthropic-ai":["partial","*",4,0],"PerplexityBot":["partial","*",4,0],"Perplexity-User":["partial","*",4,0],"Google-Extended":["partial","*",4,0],"Applebot-Extended":["partial","*",4,0],"CCBot":["partial","*",4,0],"Bytespider":["partial","*",4,0],"Amazonbot":["partial","*",4,0],"meta-externalagent":["partial","*",4,0],"cohere-ai":["partial","*",4,0],"Diffbot":["partial","*",4,0],"ContextFetchBot":["partial","*",4,0]},"llms":{"status":"http_error","http":403,"bytes":5543},"home":{"status":"http_error","http":403,"url":"https://trustarc.com/"}},
{"rank":9447,"domain":"popmatters.com","origin":"https://popmatters.com","robots":{"status":"ok","http":200,"url":"https://popmatters.com/robots.txt","bytes":395,"groups":1},"bots":{"GPTBot":["partial","*",3,1],"ChatGPT-User":["partial","*",3,1],"OAI-SearchBot":["partial","*",3,1],"ClaudeBot":["partial","*",3,1],"Claude-User":["partial","*",3,1],"Claude-SearchBot":["partial","*",3,1],"anthropic-ai":["partial","*",3,1],"PerplexityBot":["partial","*",3,1],"Perplexity-User":["partial","*",3,1],"Google-Extended":["partial","*",3,1],"Applebot-Extended":["partial","*",3,1],"CCBot":["partial","*",3,1],"Bytespider":["partial","*",3,1],"Amazonbot":["partial","*",3,1],"meta-externalagent":["partial","*",3,1],"cohere-ai":["partial","*",3,1],"Diffbot":["partial","*",3,1],"ContextFetchBot":["partial","*",3,1]},"llms":{"status":"html","http":200,"bytes":365574},"home":{"status":"ok","http":200,"url":"https://www.popmatters.com/","html_tokens":146927,"markdown_tokens":48,"pruned_tokens":48}},
{"rank":9448,"domain":"cesseon.com","origin":"https://cesseon.com","robots":{"status":"unreachable","http":null,"url":"https://cesseon.com/robots.txt","bytes":0,"groups":0,"error":"cannot resolve cesseon.com"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":9449,"domain":"yzrsbxg.com","origin":"http://yzrsbxg.com","robots":{"status":"ok","http":200,"url":"http://yzrsbxg.com/robots.txt","bytes":249,"groups":1},"bots":{"GPTBot":["partial","*",10,0],"ChatGPT-User":["partial","*",10,0],"OAI-SearchBot":["partial","*",10,0],"ClaudeBot":["partial","*",10,0],"Claude-User":["partial","*",10,0],"Claude-SearchBot":["partial","*",10,0],"anthropic-ai":["partial","*",10,0],"PerplexityBot":["partial","*",10,0],"Perplexity-User":["partial","*",10,0],"Google-Extended":["partial","*",10,0],"Applebot-Extended":["partial","*",10,0],"CCBot":["partial","*",10,0],"Bytespider":["partial","*",10,0],"Amazonbot":["partial","*",10,0],"meta-externalagent":["partial","*",10,0],"cohere-ai":["partial","*",10,0],"Diffbot":["partial","*",10,0],"ContextFetchBot":["partial","*",10,0]},"llms":{"status":"missing","http":404,"bytes":3749},"home":{"status":"ok","http":200,"url":"http://yzrsbxg.com/","html_tokens":5621,"markdown_tokens":77,"pruned_tokens":77}},
{"rank":9450,"domain":"mountvernon.org","origin":"https://mountvernon.org","robots":{"status":"ok","http":200,"url":"https://www.mountvernon.org/robots.txt","bytes":2130,"groups":1},"bots":{"GPTBot":["partial","*",36,18],"ChatGPT-User":["partial","*",36,18],"OAI-SearchBot":["partial","*",36,18],"ClaudeBot":["partial","*",36,18],"Claude-User":["partial","*",36,18],"Claude-SearchBot":["partial","*",36,18],"anthropic-ai":["partial","*",36,18],"PerplexityBot":["partial","*",36,18],"Perplexity-User":["partial","*",36,18],"Google-Extended":["partial","*",36,18],"Applebot-Extended":["partial","*",36,18],"CCBot":["partial","*",36,18],"Bytespider":["partial","*",36,18],"Amazonbot":["partial","*",36,18],"meta-externalagent":["partial","*",36,18],"cohere-ai":["partial","*",36,18],"Diffbot":["partial","*",36,18],"ContextFetchBot":["partial","*",36,18]},"llms":{"status":"missing","http":404,"bytes":192},"home":{"status":"http_error","http":403,"url":"https://mountvernon.org/"}},
{"rank":9451,"domain":"cafemom.com","origin":"https://cafemom.com","robots":{"status":"missing","http":404,"url":"https://cafemom.com/robots.txt","bytes":915,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":139996},"home":{"status":"ok","http":200,"url":"https://cafemom.com/","html_tokens":53085,"markdown_tokens":49,"pruned_tokens":45}},
{"rank":9452,"domain":"onlinecasinobonusbezvkladu.com","origin":"https://onlinecasinobonusbezvkladu.com","robots":{"status":"ok","http":200,"url":"https://onlinecasinobonusbezvkladu.com/robots.txt","bytes":84,"groups":1},"bots":{"GPTBot":["allowed","*",0,1],"ChatGPT-User":["allowed","*",0,1],"OAI-SearchBot":["allowed","*",0,1],"ClaudeBot":["allowed","*",0,1],"Claude-User":["allowed","*",0,1],"Claude-SearchBot":["allowed","*",0,1],"anthropic-ai":["allowed","*",0,1],"PerplexityBot":["allowed","*",0,1],"Perplexity-User":["allowed","*",0,1],"Google-Extended":["allowed","*",0,1],"Applebot-Extended":["allowed","*",0,1],"CCBot":["allowed","*",0,1],"Bytespider":["allowed","*",0,1],"Amazonbot":["allowed","*",0,1],"meta-externalagent":["allowed","*",0,1],"cohere-ai":["allowed","*",0,1],"Diffbot":["allowed","*",0,1],"ContextFetchBot":["allowed","*",0,1]},"llms":{"status":"missing","http":404,"bytes":659},"home":{"status":"ok","http":200,"url":"https://onlinecasinobonusbezvkladu.com/","html_tokens":40364,"markdown_tokens":23018,"pruned_tokens":148}},
{"rank":9453,"domain":"jybczy.com","origin":"http://jybczy.com","robots":{"status":"ok","http":200,"url":"http://jybczy.com/robots.txt","bytes":485,"groups":1},"bots":{"GPTBot":["partial","*",21,0],"ChatGPT-User":["partial","*",21,0],"OAI-SearchBot":["partial","*",21,0],"ClaudeBot":["partial","*",21,0],"Claude-User":["partial","*",21,0],"Claude-SearchBot":["partial","*",21,0],"anthropic-ai":["partial","*",21,0],"PerplexityBot":["partial","*",21,0],"Perplexity-User":["partial","*",21,0],"Google-Extended":["partial","*",21,0],"Applebot-Extended":["partial","*",21,0],"CCBot":["partial","*",21,0],"Bytespider":["partial","*",21,0],"Amazonbot":["partial","*",21,0],"meta-externalagent":["partial","*",21,0],"cohere-ai":["partial","*",21,0],"Diffbot":["partial","*",21,0],"ContextFetchBot":["partial","*",21,0]},"llms":{"status":"missing","http":404,"bytes":4723},"home":{"status":"error","http":200,"url":"http://jybczy.com/","error":"offset must be a timedelta strictly between -timedelta(hours=24) and timedelta(hours=24)."}},
{"rank":9454,"domain":"ocnk.net","origin":"https://ocnk.net","robots":{"status":"missing","http":404,"url":"https://www.ocnk.net/robots.txt","bytes":1939,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"unreachable","error":"Server disconnected without sending a response."},"home":{"status":"ok","http":200,"url":"https://www.ocnk.net/","html_tokens":22129,"markdown_tokens":13,"pruned_tokens":13}},
{"rank":9455,"domain":"bnnvara.nl","origin":"https://bnnvara.nl","robots":{"status":"ok","http":200,"url":"https://bnnvara.nl/robots.txt","bytes":14,"groups":1},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"missing","http":404,"bytes":126255},"home":{"status":"ok","http":200,"url":"https://www.bnnvara.nl","html_tokens":56523,"markdown_tokens":16,"pruned_tokens":16}},
{"rank":9456,"domain":"ca.com","origin":"https://ca.com","robots":{"status":"missing","http":404,"url":"https://www.broadcom.com/404-ca-technologies","bytes":16609,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":16609},"home":{"status":"ok","http":200,"url":"https://www.broadcom.com/","html_tokens":4186,"markdown_tokens":0,"pruned_tokens":0}},
{"rank":9457,"domain":"chlitchi.com","origin":"https://chlitchi.com","robots":{"status":"missing","http":404,"url":"https://chlitchi.com/robots.txt","bytes":3168,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":3164},"home":{"status":"ok","http":200,"url":"https://chlitchi.com/","html_tokens":2236,"markdown_tokens":52,"pruned_tokens":52}},
{"rank":9458,"domain":"latamairlines.com","origin":"https://latamairlines.com","robots":{"status":"refused","http":403,"url":"https://www.latamairlines.com/robots.txt","bytes":1331,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":9459,"domain":"downtoearth.org.in","origin":"https://downtoearth.org.in","robots":{"status":"ok","http":200,"url":"https://www.downtoearth.org.in/robots.txt","bytes":131,"groups":1},"bots":{"GPTBot":["allowed","*",0,1],"ChatGPT-User":["allowed","*",0,1],"OAI-SearchBot":["allowed","*",0,1],"ClaudeBot":["allowed","*",0,1],"Claude-User":["allowed","*",0,1],"Claude-SearchBot":["allowed","*",0,1],"anthropic-ai":["allowed","*",0,1],"PerplexityBot":["allowed","*",0,1],"Perplexity-User":["allowed","*",0,1],"Google-Extended":["allowed","*",0,1],"Applebot-Extended":["allowed","*",0,1],"CCBot":["allowed","*",0,1],"Bytespider":["allowed","*",0,1],"Amazonbot":["allowed","*",0,1],"meta-externalagent":["allowed","*",0,1],"cohere-ai":["allowed","*",0,1],"Diffbot":["allowed","*",0,1],"ContextFetchBot":["allowed","*",0,1]},"llms":{"status":"missing","http":404,"bytes":719231},"home":{"status":"ok","http":200,"url":"https://www.downtoearth.org.in/","html_tokens":292269,"markdown_tokens":10,"pruned_tokens":10}},
{"rank":9460,"domain":"motherearthnews.com","origin":"https://motherearthnews.com","robots":{"status":"ok","http":200,"url":"https://www.motherearthnews.com/robots.txt","bytes":927,"groups":6},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["partial","*",16,1],"OAI-SearchBot":["partial","*",16,1],"ClaudeBot":["partial","*",16,1],"Claude-User":["partial","*",16,1],"Claude-SearchBot":["partial","*",16,1],"anthropic-ai":["partial","*",16,1],"PerplexityBot":["partial","*",16,1],"Perplexity-User":["partial","*",16,1],"Google-Extended":["partial","*",16,1],"Applebot-Extended":["partial","*",16,1],"CCBot":["partial","*",16,1],"Bytespider":["partial","*",16,1],"Amazonbot":["partial","*",16,1],"meta-externalagent":["partial","*",16,1],"cohere-ai":["partial","*",16,1],"Diffbot":["partial","*",16,1],"ContextFetchBot":["partial","*",16,1]},"llms":{"status":"other_text","http":200,"bytes":686741},"home":{"status":"ok","http":200,"url":"https://www.motherearthnews.com/","html_tokens":111890,"markdown_tokens":3588,"pruned_tokens":147}},
{"rank":9461,"domain":"webring.com","origin":"https://webring.com","robots":{"status":"ok","http":200,"url":"https://webring.com/robots.txt","bytes":314,"groups":1},"bots":{"GPTBot":["partial","*",12,0],"ChatGPT-User":["partial","*",12,0],"OAI-SearchBot":["partial","*",12,0],"ClaudeBot":["partial","*",12,0],"Claude-User":["partial","*",12,0],"Claude-SearchBot":["partial","*",12,0],"anthropic-ai":["partial","*",12,0],"PerplexityBot":["partial","*",12,0],"Perplexity-User":["partial","*",12,0],"Google-Extended":["partial","*",12,0],"Applebot-Extended":["partial","*",12,0],"CCBot":["partial","*",12,0],"Bytespider":["partial","*",12,0],"Amazonbot":["partial","*",12,0],"meta-externalagent":["partial","*",12,0],"cohere-ai":["partial","*",12,0],"Diffbot":["partial","*",12,0],"ContextFetchBot":["partial","*",12,0]},"llms":{"status":"html","http":200,"bytes":99872},"home":{"status":"ok","http":200,"url":"https://webring.com/","html_tokens":24964,"markdown_tokens":649,"pruned_tokens":260}},
{"rank":9462,"domain":"simplyhired.com","origin":"https://simplyhired.com","robots":{"status":"refused","http":403,"url":"https://www.simplyhired.com/","bytes":5527,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":9463,"domain":"astonmartin.com","origin":"https://astonmartin.com","robots":{"status":"ok","http":200,"url":"https://www.astonmartin.com/en/robots.txt","bytes":579,"groups":1},"bots":{"GPTBot":["partial","*",1,1],"ChatGPT-User":["partial","*",1,1],"OAI-SearchBot":["partial","*",1,1],"ClaudeBot":["partial","*",1,1],"Claude-User":["partial","*",1,1],"Claude-SearchBot":["partial","*",1,1],"anthropic-ai":["partial","*",1,1],"PerplexityBot":["partial","*",1,1],"Perplexity-User":["partial","*",1,1],"Google-Extended":["partial","*",1,1],"Applebot-Extended":["partial","*",1,1],"CCBot":["partial","*",1,1],"Bytespider":["partial","*",1,1],"Amazonbot":["partial","*",1,1],"meta-externalagent":["partial","*",1,1],"cohere-ai":["partial","*",1,1],"Diffbot":["partial","*",1,1],"ContextFetchBot":["partial","*",1,1]},"llms":{"status":"missing","http":404,"bytes":89274},"home":{"status":"ok","http":200,"url":"https://www.astonmartin.com/en/welcome","html_tokens":3517,"markdown_tokens":5,"pruned_tokens":5}},
{"rank":9464,"domain":"illinoisstate.edu","origin":"https://illinoisstate.edu","robots":{"status":"ok","http":200,"url":"https://illinoisstate.edu/robots.txt","bytes":200,"groups":1},"bots":{"GPTBot":["partial","*",4,0],"ChatGPT-User":["partial","*",4,0],"OAI-SearchBot":["partial","*",4,0],"ClaudeBot":["partial","*",4,0],"Claude-User":["partial","*",4,0],"Claude-SearchBot":["partial","*",4,0],"anthropic-ai":["partial","*",4,0],"PerplexityBot":["partial","*",4,0],"Perplexity-User":["partial","*",4,0],"Google-Extended":["partial","*",4,0],"Applebot-Extended":["partial","*",4,0],"CCBot":["partial","*",4,0],"Bytespider":["partial","*",4,0],"Amazonbot":["partial","*",4,0],"meta-externalagent":["partial","*",4,0],"cohere-ai":["partial","*",4,0],"Diffbot":["partial","*",4,0],"ContextFetchBot":["partial","*",4,0]},"llms":{"status":"missing","http":404,"bytes":19128},"home":{"status":"ok","http":200,"url":"https://illinoisstate.edu/","html_tokens":13631,"markdown_tokens":115,"pruned_tokens":40}},
{"rank":9465,"domain":"kmk.org","origin":"https://kmk.org","robots":{"status":"server_error","http":500,"url":"https://kmk.org/robots.txt","bytes":5063,"groups":0,"error":"HTTP 500"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":9466,"domain":"kw.com","origin":"https://kw.com","robots":{"status":"refused","http":403,"url":"https://kw.com/robots.txt","bytes":5522,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":9467,"domain":"scamadviser.com","origin":"https://scamadviser.com","robots":{"status":"ok","http":200,"url":"https://www.scamadviser.com/robots.txt","bytes":921,"groups":1},"bots":{"GPTBot":["partial","*",20,0],"ChatGPT-User":["partial","*",20,0],"OAI-SearchBot":["partial","*",20,0],"ClaudeBot":["partial","*",20,0],"Claude-User":["partial","*",20,0],"Claude-SearchBot":["partial","*",20,0],"anthropic-ai":["partial","*",20,0],"PerplexityBot":["partial","*",20,0],"Perplexity-User":["partial","*",20,0],"Google-Extended":["partial","*",20,0],"Applebot-Extended":["partial","*",20,0],"CCBot":["partial","*",20,0],"Bytespider":["partial","*",20,0],"Amazonbot":["partial","*",20,0],"meta-externalagent":["partial","*",20,0],"cohere-ai":["partial","*",20,0],"Diffbot":["partial","*",20,0],"ContextFetchBot":["partial","*",20,0]},"llms":{"status":"missing","http":404,"bytes":36274},"home":{"status":"ok","http":200,"url":"https://www.scamadviser.com","html_tokens":48245,"markdown_tokens":2239,"pruned_tokens":37}},
{"rank":9468,"domain":"distractify.com","origin":"https://distractify.com","robots":{"status":"ok","http":200,"url":"https://www.distractify.com/robots.txt","bytes":166,"groups":2},"bots":{"GPTBot":["allowed","*",0,1],"ChatGPT-User":["allowed","*",0,1],"OAI-SearchBot":["allowed","*",0,1],"ClaudeBot":["allowed","*",0,1],"Claude-User":["allowed","*",0,1],"Claude-SearchBot":["allowed","*",0,1],"anthropic-ai":["allowed","*",0,1],"PerplexityBot":["allowed","*",0,1],"Perplexity-User":["allowed","*",0,1],"Google-Extended":["allowed","*",0,1],"Applebot-Extended":["allowed","*",0,1],"CCBot":["allowed","*",0,1],"Bytespider":["allowed","*",0,1],"Amazonbot":["allowed","*",0,1],"meta-externalagent":["allowed","*",0,1],"cohere-ai":["allowed","*",0,1],"Diffbot":["allowed","*",0,1],"ContextFetchBot":["allowed","*",0,1]},"llms":{"status":"missing","http":404,"bytes":54754},"home":{"status":"ok","http":200,"url":"https://www.distractify.com/","html_tokens":21980,"markdown_tokens":403,"pruned_tokens":23}},
{"rank":9469,"domain":"signonsandiego.com","origin":"http://signonsandiego.com","robots":{"status":"html","http":200,"url":"https://www.sandiegouniontribune.com/","bytes":272756,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":9470,"domain":"economy.gov.ru","origin":"https://economy.gov.ru","robots":{"status":"timeout","http":null,"url":"https://economy.gov.ru/robots.txt","bytes":0,"groups":0,"error":"ConnectTimeout"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":9471,"domain":"ezboard.com","origin":"https://ezboard.com","robots":{"status":"ok","http":200,"url":"https://ezboard.com/robots.txt","bytes":959,"groups":1},"bots":{"GPTBot":["partial","*",27,6],"ChatGPT-User":["partial","*",27,6],"OAI-SearchBot":["partial","*",27,6],"ClaudeBot":["partial","*",27,6],"Claude-User":["partial","*",27,6],"Claude-SearchBot":["partial","*",27,6],"anthropic-ai":["partial","*",27,6],"PerplexityBot":["partial","*",27,6],"Perplexity-User":["partial","*",27,6],"Google-Extended":["partial","*",27,6],"Applebot-Extended":["partial","*",27,6],"CCBot":["partial","*",27,6],"Bytespider":["partial","*",27,6],"Amazonbot":["partial","*",27,6],"meta-externalagent":["partial","*",27,6],"cohere-ai":["partial","*",27,6],"Diffbot":["partial","*",27,6],"ContextFetchBot":["partial","*",27,6]},"llms":{"status":"html","http":200,"bytes":4211},"home":{"status":"ok","http":200,"url":"https://ezboard.com/","html_tokens":1288,"markdown_tokens":0,"pruned_tokens":0}},
{"rank":9472,"domain":"uji.es","origin":"https://www.uji.es","robots":{"status":"ok","http":200,"url":"https://www.uji.es/robots.txt","bytes":65,"groups":2},"bots":{"GPTBot":["partial","*",1,0],"ChatGPT-User":["partial","*",1,0],"OAI-SearchBot":["partial","*",1,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",1,0],"Claude-SearchBot":["partial","*",1,0],"anthropic-ai":["partial","*",1,0],"PerplexityBot":["partial","*",1,0],"Perplexity-User":["partial","*",1,0],"Google-Extended":["partial","*",1,0],"Applebot-Extended":["partial","*",1,0],"CCBot":["partial","*",1,0],"Bytespider":["partial","*",1,0],"Amazonbot":["partial","*",1,0],"meta-externalagent":["partial","*",1,0],"cohere-ai":["partial","*",1,0],"Diffbot":["partial","*",1,0],"ContextFetchBot":["partial","*",1,0]},"llms":{"status":"missing","http":404,"bytes":206},"home":{"status":"ok","http":200,"url":"https://www.uji.es/","html_tokens":11623,"markdown_tokens":243,"pruned_tokens":97}},
{"rank":9473,"domain":"jsczkt.cn","origin":"http://jsczkt.cn","robots":{"status":"ok","http":200,"url":"http://jsczkt.cn/robots.txt","bytes":247,"groups":1},"bots":{"GPTBot":["partial","*",10,0],"ChatGPT-User":["partial","*",10,0],"OAI-SearchBot":["partial","*",10,0],"ClaudeBot":["partial","*",10,0],"Claude-User":["partial","*",10,0],"Claude-SearchBot":["partial","*",10,0],"anthropic-ai":["partial","*",10,0],"PerplexityBot":["partial","*",10,0],"Perplexity-User":["partial","*",10,0],"Google-Extended":["partial","*",10,0],"Applebot-Extended":["partial","*",10,0],"CCBot":["partial","*",10,0],"Bytespider":["partial","*",10,0],"Amazonbot":["partial","*",10,0],"meta-externalagent":["partial","*",10,0],"cohere-ai":["partial","*",10,0],"Diffbot":["partial","*",10,0],"ContextFetchBot":["partial","*",10,0]},"llms":{"status":"missing","http":404,"bytes":3693},"home":{"status":"ok","http":200,"url":"http://jsczkt.cn/","html_tokens":3030,"markdown_tokens":196,"pruned_tokens":175}},
{"rank":9474,"domain":"edubirdie.com","origin":"https://edubirdie.com","robots":{"status":"ok","http":200,"url":"https://edubirdie.com/robots.txt","bytes":932,"groups":1},"bots":{"GPTBot":["partial","*",32,5],"ChatGPT-User":["partial","*",32,5],"OAI-SearchBot":["partial","*",32,5],"ClaudeBot":["partial","*",32,5],"Claude-User":["partial","*",32,5],"Claude-SearchBot":["partial","*",32,5],"anthropic-ai":["partial","*",32,5],"PerplexityBot":["partial","*",32,5],"Perplexity-User":["partial","*",32,5],"Google-Extended":["partial","*",32,5],"Applebot-Extended":["partial","*",32,5],"CCBot":["partial","*",32,5],"Bytespider":["partial","*",32,5],"Amazonbot":["partial","*",32,5],"meta-externalagent":["partial","*",32,5],"cohere-ai":["partial","*",32,5],"Diffbot":["partial","*",32,5],"ContextFetchBot":["partial","*",32,5]},"llms":{"status":"present","http":200,"bytes":5003},"home":{"status":"ok","http":200,"url":"https://edubirdie.com/","html_tokens":42480,"markdown_tokens":3948,"pruned_tokens":692}},
{"rank":9475,"domain":"decathlon.fr","origin":"https://decathlon.fr","robots":{"status":"ok","http":200,"url":"https://www.decathlon.fr/robots.txt","bytes":61786,"groups":22},"bots":{"GPTBot":["partial","*",51,6],"ChatGPT-User":["partial","*",51,6],"OAI-SearchBot":["partial","*",51,6],"ClaudeBot":["partial","named",60,1007],"Claude-User":["partial","*",51,6],"Claude-SearchBot":["partial","*",51,6],"anthropic-ai":["partial","*",51,6],"PerplexityBot":["partial","*",51,6],"Perplexity-User":["partial","*",51,6],"Google-Extended":["partial","*",51,6],"Applebot-Extended":["partial","*",51,6],"CCBot":["partial","*",51,6],"Bytespider":["partial","*",51,6],"Amazonbot":["partial","*",51,6],"meta-externalagent":["partial","*",51,6],"cohere-ai":["partial","*",51,6],"Diffbot":["partial","*",51,6],"ContextFetchBot":["partial","*",51,6]},"llms":{"status":"missing","http":404,"bytes":73903},"home":{"status":"http_error","http":403,"url":"https://www.decathlon.fr/"}},
{"rank":9476,"domain":"indiatvnews.com","origin":"https://indiatvnews.com","robots":{"status":"ok","http":200,"url":"https://www.indiatvnews.com/robots.txt","bytes":721,"groups":6},"bots":{"GPTBot":["partial","*",13,1],"ChatGPT-User":["partial","*",13,1],"OAI-SearchBot":["partial","*",13,1],"ClaudeBot":["partial","*",13,1],"Claude-User":["partial","*",13,1],"Claude-SearchBot":["partial","*",13,1],"anthropic-ai":["partial","*",13,1],"PerplexityBot":["partial","*",13,1],"Perplexity-User":["partial","*",13,1],"Google-Extended":["partial","*",13,1],"Applebot-Extended":["partial","*",13,1],"CCBot":["partial","*",13,1],"Bytespider":["partial","*",13,1],"Amazonbot":["partial","*",13,1],"meta-externalagent":["partial","*",13,1],"cohere-ai":["partial","*",13,1],"Diffbot":["partial","*",13,1],"ContextFetchBot":["partial","*",13,1]},"llms":{"status":"missing","http":404,"bytes":232556},"home":{"status":"ok","http":200,"url":"https://www.indiatvnews.com/","html_tokens":100022,"markdown_tokens":98,"pruned_tokens":39}},
{"rank":9477,"domain":"gettr.com","origin":"https://gettr.com","robots":{"status":"ok","http":200,"url":"https://gettr.com/robots.txt","bytes":648,"groups":6},"bots":{"GPTBot":["allowed","*",0,1],"ChatGPT-User":["allowed","*",0,1],"OAI-SearchBot":["allowed","*",0,1],"ClaudeBot":["allowed","*",0,1],"Claude-User":["allowed","*",0,1],"Claude-SearchBot":["allowed","*",0,1],"anthropic-ai":["allowed","*",0,1],"PerplexityBot":["allowed","*",0,1],"Perplexity-User":["allowed","*",0,1],"Google-Extended":["allowed","*",0,1],"Applebot-Extended":["allowed","*",0,1],"CCBot":["allowed","*",0,1],"Bytespider":["allowed","*",0,1],"Amazonbot":["allowed","*",0,1],"meta-externalagent":["allowed","*",0,1],"cohere-ai":["allowed","*",0,1],"Diffbot":["allowed","*",0,1],"ContextFetchBot":["allowed","*",0,1]},"llms":{"status":"html","http":200,"bytes":3552},"home":{"status":"ok","http":200,"url":"https://gettr.com/","html_tokens":888,"markdown_tokens":18,"pruned_tokens":18}},
{"rank":9478,"domain":"tzfhbwg.com","origin":"http://tzfhbwg.com","robots":{"status":"html","http":200,"url":"http://tzfhbwg.com/robots.txt","bytes":3747,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":9479,"domain":"jingbianhairong.com","origin":"https://jingbianhairong.com","robots":{"status":"timeout","http":null,"url":"https://jingbianhairong.com/robots.txt","bytes":0,"groups":0,"error":"ConnectTimeout"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":9480,"domain":"capcom.co.jp","origin":"https://www.capcom.co.jp","robots":{"status":"missing","http":404,"url":"https://www.capcom.co.jp/robots.txt","bytes":10943,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":10943},"home":{"status":"ok","http":200,"url":"https://www.capcom.co.jp/","html_tokens":5762,"markdown_tokens":45,"pruned_tokens":45}},
{"rank":9481,"domain":"standardmedia.co.ke","origin":"https://standardmedia.co.ke","robots":{"status":"ok","http":200,"url":"https://standardmedia.co.ke/robots.txt","bytes":517,"groups":1},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"missing","http":404,"bytes":1923},"home":{"status":"ok","http":200,"url":"https://standardmedia.co.ke/","html_tokens":120235,"markdown_tokens":83,"pruned_tokens":83}},
{"rank":9482,"domain":"weekand.com","origin":"https://weekand.com","robots":{"status":"ok","http":200,"url":"https://www.weekand.com/robots.txt","bytes":907,"groups":3},"bots":{"GPTBot":["partial","*",8,0],"ChatGPT-User":["partial","*",8,0],"OAI-SearchBot":["partial","*",8,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["blocked","named",1,0],"Claude-SearchBot":["partial","*",8,0],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["blocked","named",1,0],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["partial","*",8,0],"meta-externalagent":["partial","*",8,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",8,0]},"llms":{"status":"missing","http":404,"bytes":51306},"home":{"status":"ok","http":200,"url":"https://www.weekand.com/","html_tokens":95072,"markdown_tokens":26,"pruned_tokens":26}},
{"rank":9483,"domain":"superpages.com","origin":"https://superpages.com","robots":{"status":"refused","http":403,"url":"https://www.superpages.com/","bytes":4908,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":9484,"domain":"nxtbook.com","origin":"https://nxtbook.com","robots":{"status":"refused","http":403,"url":"https://nxtbook.com/robots.txt","bytes":118,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":9485,"domain":"webstarts.com","origin":"https://webstarts.com","robots":{"status":"ok","http":200,"url":"https://www.webstarts.com/robots.txt","bytes":70,"groups":1},"bots":{"GPTBot":["allowed","*",0,1],"ChatGPT-User":["allowed","*",0,1],"OAI-SearchBot":["allowed","*",0,1],"ClaudeBot":["allowed","*",0,1],"Claude-User":["allowed","*",0,1],"Claude-SearchBot":["allowed","*",0,1],"anthropic-ai":["allowed","*",0,1],"PerplexityBot":["allowed","*",0,1],"Perplexity-User":["allowed","*",0,1],"Google-Extended":["allowed","*",0,1],"Applebot-Extended":["allowed","*",0,1],"CCBot":["allowed","*",0,1],"Bytespider":["allowed","*",0,1],"Amazonbot":["allowed","*",0,1],"meta-externalagent":["allowed","*",0,1],"cohere-ai":["allowed","*",0,1],"Diffbot":["allowed","*",0,1],"ContextFetchBot":["allowed","*",0,1]},"llms":{"status":"missing","http":404,"bytes":1299},"home":{"status":"ok","http":200,"url":"https://www.webstarts.com/","html_tokens":16792,"markdown_tokens":642,"pruned_tokens":209}},
{"rank":9486,"domain":"fontawesome.io","origin":"https://fontawesome.io","robots":{"status":"html","http":200,"url":"https://fontawesome.com/","bytes":21105,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":9487,"domain":"zj-xingyi.com","origin":"http://zj-xingyi.com","robots":{"status":"ok","http":200,"url":"http://zj-xingyi.com/robots.txt","bytes":249,"groups":1},"bots":{"GPTBot":["partial","*",10,0],"ChatGPT-User":["partial","*",10,0],"OAI-SearchBot":["partial","*",10,0],"ClaudeBot":["partial","*",10,0],"Claude-User":["partial","*",10,0],"Claude-SearchBot":["partial","*",10,0],"anthropic-ai":["partial","*",10,0],"PerplexityBot":["partial","*",10,0],"Perplexity-User":["partial","*",10,0],"Google-Extended":["partial","*",10,0],"Applebot-Extended":["partial","*",10,0],"CCBot":["partial","*",10,0],"Bytespider":["partial","*",10,0],"Amazonbot":["partial","*",10,0],"meta-externalagent":["partial","*",10,0],"cohere-ai":["partial","*",10,0],"Diffbot":["partial","*",10,0],"ContextFetchBot":["partial","*",10,0]},"llms":{"status":"missing","http":404,"bytes":3749},"home":{"status":"ok","http":200,"url":"http://zj-xingyi.com/","html_tokens":6673,"markdown_tokens":109,"pruned_tokens":109}},
{"rank":9488,"domain":"pokerstrategy.com","origin":"https://pokerstrategy.com","robots":{"status":"ok","http":200,"url":"https://www.pokerstrategy.com/robots.txt","bytes":1459,"groups":1},"bots":{"GPTBot":["partial","*",16,1],"ChatGPT-User":["partial","*",16,1],"OAI-SearchBot":["partial","*",16,1],"ClaudeBot":["partial","*",16,1],"Claude-User":["partial","*",16,1],"Claude-SearchBot":["partial","*",16,1],"anthropic-ai":["partial","*",16,1],"PerplexityBot":["partial","*",16,1],"Perplexity-User":["partial","*",16,1],"Google-Extended":["partial","*",16,1],"Applebot-Extended":["partial","*",16,1],"CCBot":["partial","*",16,1],"Bytespider":["partial","*",16,1],"Amazonbot":["partial","*",16,1],"meta-externalagent":["partial","*",16,1],"cohere-ai":["partial","*",16,1],"Diffbot":["partial","*",16,1],"ContextFetchBot":["partial","*",16,1]},"llms":{"status":"missing","http":404,"bytes":43981},"home":{"status":"ok","http":200,"url":"https://www.pokerstrategy.com/","html_tokens":119359,"markdown_tokens":316,"pruned_tokens":180}},
{"rank":9489,"domain":"zhsumarc.com","origin":"https://zhsumarc.com","robots":{"status":"html","http":200,"url":"https://zhsumarc.com/","bytes":12590,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":9490,"domain":"leafly.com","origin":"https://leafly.com","robots":{"status":"ok","http":200,"url":"https://www.leafly.com/robots.txt","bytes":2647,"groups":7},"bots":{"GPTBot":["partial","named",1,26],"ChatGPT-User":["partial","*",18,1],"OAI-SearchBot":["partial","*",18,1],"ClaudeBot":["partial","*",18,1],"Claude-User":["partial","*",18,1],"Claude-SearchBot":["partial","*",18,1],"anthropic-ai":["partial","*",18,1],"PerplexityBot":["partial","*",18,1],"Perplexity-User":["partial","*",18,1],"Google-Extended":["partial","*",18,1],"Applebot-Extended":["partial","*",18,1],"CCBot":["partial","*",18,1],"Bytespider":["partial","*",18,1],"Amazonbot":["partial","*",18,1],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["partial","*",18,1],"Diffbot":["partial","*",18,1],"ContextFetchBot":["partial","*",18,1]},"llms":{"status":"missing","http":404,"bytes":91519},"home":{"status":"ok","http":200,"url":"https://www.leafly.com/","html_tokens":62447,"markdown_tokens":176,"pruned_tokens":176}},
{"rank":9491,"domain":"jugendherberge.de","origin":"https://jugendherberge.de","robots":{"status":"ok","http":200,"url":"https://www.jugendherberge.de/robots.txt","bytes":489,"groups":1},"bots":{"GPTBot":["partial","*",12,3],"ChatGPT-User":["partial","*",12,3],"OAI-SearchBot":["partial","*",12,3],"ClaudeBot":["partial","*",12,3],"Claude-User":["partial","*",12,3],"Claude-SearchBot":["partial","*",12,3],"anthropic-ai":["partial","*",12,3],"PerplexityBot":["partial","*",12,3],"Perplexity-User":["partial","*",12,3],"Google-Extended":["partial","*",12,3],"Applebot-Extended":["partial","*",12,3],"CCBot":["partial","*",12,3],"Bytespider":["partial","*",12,3],"Amazonbot":["partial","*",12,3],"meta-externalagent":["partial","*",12,3],"cohere-ai":["partial","*",12,3],"Diffbot":["partial","*",12,3],"ContextFetchBot":["partial","*",12,3]},"llms":{"status":"html","http":200,"bytes":81599},"home":{"status":"ok","http":200,"url":"https://www.jugendherberge.de/","html_tokens":109141,"markdown_tokens":458,"pruned_tokens":413}},
{"rank":9492,"domain":"sourceforge.jp","origin":"http://sourceforge.jp","robots":{"status":"ok","http":200,"url":"http://sourceforge.jp/robots.txt","bytes":0,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"html","http":200,"bytes":40295},"home":{"status":"ok","http":200,"url":"http://sourceforge.jp/","html_tokens":10074,"markdown_tokens":0,"pruned_tokens":0}},
{"rank":9493,"domain":"pointblog.net","origin":"https://pointblog.net","robots":{"status":"ok","http":200,"url":"https://pointblog.net/robots.txt","bytes":72,"groups":2},"bots":{"GPTBot":["partial","*",1,0],"ChatGPT-User":["partial","*",1,0],"OAI-SearchBot":["partial","*",1,0],"ClaudeBot":["partial","*",1,0],"Claude-User":["partial","*",1,0],"Claude-SearchBot":["partial","*",1,0],"anthropic-ai":["partial","*",1,0],"PerplexityBot":["partial","*",1,0],"Perplexity-User":["partial","*",1,0],"Google-Extended":["partial","*",1,0],"Applebot-Extended":["partial","*",1,0],"CCBot":["partial","*",1,0],"Bytespider":["partial","*",1,0],"Amazonbot":["partial","*",1,0],"meta-externalagent":["partial","*",1,0],"cohere-ai":["partial","*",1,0],"Diffbot":["partial","*",1,0],"ContextFetchBot":["partial","*",1,0]},"llms":{"status":"missing","http":404,"bytes":529},"home":{"status":"ok","http":200,"url":"https://pointblog.net/","html_tokens":1885,"markdown_tokens":48,"pruned_tokens":48}},
{"rank":9494,"domain":"somethingawful.com","origin":"https://somethingawful.com","robots":{"status":"ok","http":200,"url":"https://www.somethingawful.com/robots.txt","bytes":379,"groups":3},"bots":{"GPTBot":["partial","*",7,0],"ChatGPT-User":["partial","*",7,0],"OAI-SearchBot":["partial","*",7,0],"ClaudeBot":["partial","*",7,0],"Claude-User":["partial","*",7,0],"Claude-SearchBot":["partial","*",7,0],"anthropic-ai":["partial","*",7,0],"PerplexityBot":["partial","*",7,0],"Perplexity-User":["partial","*",7,0],"Google-Extended":["partial","*",7,0],"Applebot-Extended":["partial","*",7,0],"CCBot":["partial","*",7,0],"Bytespider":["partial","*",7,0],"Amazonbot":["partial","*",7,0],"meta-externalagent":["partial","*",7,0],"cohere-ai":["partial","*",7,0],"Diffbot":["partial","*",7,0],"ContextFetchBot":["partial","*",7,0]},"llms":{"status":"missing","http":404,"bytes":14020},"home":{"status":"ok","http":200,"url":"https://forums.somethingawful.com/","html_tokens":15782,"markdown_tokens":1021,"pruned_tokens":1021}},
{"rank":9495,"domain":"verified-project.com","origin":"https://verified-project.com","robots":{"status":"unreachable","http":null,"url":"https://verified-project.com/robots.txt","bytes":0,"groups":0,"error":"cannot resolve verified-project.com"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":9496,"domain":"celebritynetworth.com","origin":"https://celebritynetworth.com","robots":{"status":"ok","http":200,"url":"https://www.celebritynetworth.com/robots.txt","bytes":452,"groups":6},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["partial","*",7,2],"OAI-SearchBot":["partial","*",7,2],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",7,2],"Claude-SearchBot":["blocked","named",1,0],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["partial","*",7,2],"Google-Extended":["partial","*",7,2],"Applebot-Extended":["partial","*",7,2],"CCBot":["partial","*",7,2],"Bytespider":["partial","*",7,2],"Amazonbot":["partial","*",7,2],"meta-externalagent":["partial","*",7,2],"cohere-ai":["partial","*",7,2],"Diffbot":["partial","*",7,2],"ContextFetchBot":["partial","*",7,2]},"llms":{"status":"missing","http":404,"bytes":215002},"home":{"status":"ok","http":200,"url":"https://www.celebritynetworth.com/","html_tokens":64821,"markdown_tokens":12,"pruned_tokens":12}},
{"rank":9497,"domain":"stubhub.com","origin":"https://stubhub.com","robots":{"status":"ok","http":200,"url":"https://www.stubhub.com/robots.txt","bytes":14172,"groups":1},"bots":{"GPTBot":["partial","*",504,0],"ChatGPT-User":["partial","*",504,0],"OAI-SearchBot":["partial","*",504,0],"ClaudeBot":["partial","*",504,0],"Claude-User":["partial","*",504,0],"Claude-SearchBot":["partial","*",504,0],"anthropic-ai":["partial","*",504,0],"PerplexityBot":["partial","*",504,0],"Perplexity-User":["partial","*",504,0],"Google-Extended":["partial","*",504,0],"Applebot-Extended":["partial","*",504,0],"CCBot":["partial","*",504,0],"Bytespider":["partial","*",504,0],"Amazonbot":["partial","*",504,0],"meta-externalagent":["partial","*",504,0],"cohere-ai":["partial","*",504,0],"Diffbot":["partial","*",504,0],"ContextFetchBot":["partial","*",504,0]},"llms":{"status":"missing","http":404,"bytes":58996},"home":{"status":"http_error","http":403,"url":"https://www.stubhub.com/"}},
{"rank":9498,"domain":"taskandpurpose.com","origin":"https://taskandpurpose.com","robots":{"status":"ok","http":200,"url":"https://taskandpurpose.com/robots.txt","bytes":373,"groups":2},"bots":{"GPTBot":["partial","*",6,1],"ChatGPT-User":["partial","*",6,1],"OAI-SearchBot":["partial","*",6,1],"ClaudeBot":["partial","*",6,1],"Claude-User":["partial","*",6,1],"Claude-SearchBot":["partial","*",6,1],"anthropic-ai":["partial","*",6,1],"PerplexityBot":["partial","*",6,1],"Perplexity-User":["partial","*",6,1],"Google-Extended":["partial","*",6,1],"Applebot-Extended":["partial","*",6,1],"CCBot":["partial","*",6,1],"Bytespider":["partial","*",6,1],"Amazonbot":["partial","*",6,1],"meta-externalagent":["partial","*",6,1],"cohere-ai":["partial","*",6,1],"Diffbot":["partial","*",6,1],"ContextFetchBot":["partial","*",6,1]},"llms":{"status":"missing","http":404,"bytes":110058},"home":{"status":"ok","http":200,"url":"https://taskandpurpose.com/","html_tokens":46552,"markdown_tokens":343,"pruned_tokens":132}},
{"rank":9499,"domain":"bentleymotors.com","origin":"https://bentleymotors.com","robots":{"status":"refused","http":403,"url":"https://bentleymotors.com/robots.txt","bytes":381,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":9500,"domain":"puppet.com","origin":"https://puppet.com","robots":{"status":"refused","http":403,"url":"https://puppet.com/robots.txt","bytes":560,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":9501,"domain":"cyberpolice.cn","origin":"https://cyberpolice.cn","robots":{"status":"unreachable","http":null,"url":"https://cyberpolice.cn/robots.txt","bytes":0,"groups":0,"error":"cannot resolve cyberpolice.cn"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":9502,"domain":"ikea.cn","origin":"https://www.ikea.cn","robots":{"status":"ok","http":200,"url":"https://www.ikea.cn/robots.txt","bytes":719,"groups":1},"bots":{"GPTBot":["partial","*",24,0],"ChatGPT-User":["partial","*",24,0],"OAI-SearchBot":["partial","*",24,0],"ClaudeBot":["partial","*",24,0],"Claude-User":["partial","*",24,0],"Claude-SearchBot":["partial","*",24,0],"anthropic-ai":["partial","*",24,0],"PerplexityBot":["partial","*",24,0],"Perplexity-User":["partial","*",24,0],"Google-Extended":["partial","*",24,0],"Applebot-Extended":["partial","*",24,0],"CCBot":["partial","*",24,0],"Bytespider":["partial","*",24,0],"Amazonbot":["partial","*",24,0],"meta-externalagent":["partial","*",24,0],"cohere-ai":["partial","*",24,0],"Diffbot":["partial","*",24,0],"ContextFetchBot":["partial","*",24,0]},"llms":{"status":"missing","http":404,"bytes":9},"home":{"status":"ok","http":200,"url":"https://www.ikea.cn/cn/en/","html_tokens":30402,"markdown_tokens":89,"pruned_tokens":89}},
{"rank":9503,"domain":"thesun.ie","origin":"https://thesun.ie","robots":{"status":"ok","http":200,"url":"https://www.thesun.ie/robots.txt","bytes":4736,"groups":2},"bots":{"GPTBot":["partial","named",100,3],"ChatGPT-User":["partial","named",100,3],"OAI-SearchBot":["partial","named",100,3],"ClaudeBot":["blocked","*",1,0],"Claude-User":["blocked","*",1,0],"Claude-SearchBot":["blocked","*",1,0],"anthropic-ai":["blocked","*",1,0],"PerplexityBot":["blocked","*",1,0],"Perplexity-User":["blocked","*",1,0],"Google-Extended":["blocked","*",1,0],"Applebot-Extended":["blocked","*",1,0],"CCBot":["blocked","*",1,0],"Bytespider":["blocked","*",1,0],"Amazonbot":["blocked","*",1,0],"meta-externalagent":["blocked","*",1,0],"cohere-ai":["blocked","*",1,0],"Diffbot":["blocked","*",1,0],"ContextFetchBot":["blocked","*",1,0]},"llms":{"status":"disallowed"},"home":{"status":"disallowed"}},
{"rank":9504,"domain":"crowdfunder.co.uk","origin":"https://crowdfunder.co.uk","robots":{"status":"ok","http":200,"url":"https://www.crowdfunder.co.uk/robots.txt","bytes":357,"groups":2},"bots":{"GPTBot":["partial","*",9,1],"ChatGPT-User":["partial","*",9,1],"OAI-SearchBot":["partial","*",9,1],"ClaudeBot":["partial","*",9,1],"Claude-User":["partial","*",9,1],"Claude-SearchBot":["partial","*",9,1],"anthropic-ai":["partial","*",9,1],"PerplexityBot":["partial","*",9,1],"Perplexity-User":["partial","*",9,1],"Google-Extended":["partial","*",9,1],"Applebot-Extended":["partial","*",9,1],"CCBot":["partial","*",9,1],"Bytespider":["partial","*",9,1],"Amazonbot":["partial","*",9,1],"meta-externalagent":["partial","*",9,1],"cohere-ai":["partial","*",9,1],"Diffbot":["partial","*",9,1],"ContextFetchBot":["partial","*",9,1]},"llms":{"status":"missing","http":404,"bytes":47375},"home":{"status":"http_error","http":403,"url":"https://crowdfunder.co.uk/"}},
{"rank":9505,"domain":"viddler.com","origin":"https://viddler.com","robots":{"status":"ok","http":200,"url":"https://viddler.com/robots.txt","bytes":228,"groups":1},"bots":{"GPTBot":["partial","*",8,1],"ChatGPT-User":["partial","*",8,1],"OAI-SearchBot":["partial","*",8,1],"ClaudeBot":["partial","*",8,1],"Claude-User":["partial","*",8,1],"Claude-SearchBot":["partial","*",8,1],"anthropic-ai":["partial","*",8,1],"PerplexityBot":["partial","*",8,1],"Perplexity-User":["partial","*",8,1],"Google-Extended":["partial","*",8,1],"Applebot-Extended":["partial","*",8,1],"CCBot":["partial","*",8,1],"Bytespider":["partial","*",8,1],"Amazonbot":["partial","*",8,1],"meta-externalagent":["partial","*",8,1],"cohere-ai":["partial","*",8,1],"Diffbot":["partial","*",8,1],"ContextFetchBot":["partial","*",8,1]},"llms":{"status":"present","http":200,"bytes":9257},"home":{"status":"ok","http":200,"url":"https://viddler.com/","html_tokens":3206,"markdown_tokens":173,"pruned_tokens":91}},
{"rank":9506,"domain":"ascendoor.com","origin":"https://ascendoor.com","robots":{"status":"ok","http":200,"url":"https://ascendoor.com/robots.txt","bytes":160,"groups":1},"bots":{"GPTBot":["partial","*",3,1],"ChatGPT-User":["partial","*",3,1],"OAI-SearchBot":["partial","*",3,1],"ClaudeBot":["partial","*",3,1],"Claude-User":["partial","*",3,1],"Claude-SearchBot":["partial","*",3,1],"anthropic-ai":["partial","*",3,1],"PerplexityBot":["partial","*",3,1],"Perplexity-User":["partial","*",3,1],"Google-Extended":["partial","*",3,1],"Applebot-Extended":["partial","*",3,1],"CCBot":["partial","*",3,1],"Bytespider":["partial","*",3,1],"Amazonbot":["partial","*",3,1],"meta-externalagent":["partial","*",3,1],"cohere-ai":["partial","*",3,1],"Diffbot":["partial","*",3,1],"ContextFetchBot":["partial","*",3,1]},"llms":{"status":"other_text","http":200,"bytes":70376},"home":{"status":"ok","http":200,"url":"https://ascendoor.com/","html_tokens":36799,"markdown_tokens":472,"pruned_tokens":163}},
{"rank":9507,"domain":"fc2web.com","origin":"http://fc2web.com","robots":{"status":"html","http":200,"url":"http://fc2web.com/","bytes":258804,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":9508,"domain":"jeuneafrique.com","origin":"https://jeuneafrique.com","robots":{"status":"ok","http":200,"url":"https://www.jeuneafrique.com/robots.txt","bytes":1954,"groups":34},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["blocked","named",2,0],"OAI-SearchBot":["blocked","named",1,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",6,0],"Claude-SearchBot":["partial","*",6,0],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["blocked","named",1,0],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["partial","*",6,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["partial","*",6,0],"Diffbot":["partial","*",6,0],"ContextFetchBot":["partial","*",6,0]},"llms":{"status":"missing","http":404,"bytes":16357},"home":{"status":"ok","http":200,"url":"https://www.jeuneafrique.com/","html_tokens":82660,"markdown_tokens":384,"pruned_tokens":174}},
{"rank":9509,"domain":"vueling.com","origin":"https://vueling.com","robots":{"status":"ok","http":200,"url":"https://www.vueling.com/robots.txt","bytes":1209,"groups":3},"bots":{"GPTBot":["partial","named",12,1],"ChatGPT-User":["partial","named",12,1],"OAI-SearchBot":["partial","named",12,1],"ClaudeBot":["partial","named",12,1],"Claude-User":["partial","*",12,1],"Claude-SearchBot":["partial","named",12,1],"anthropic-ai":["partial","*",12,1],"PerplexityBot":["partial","named",12,1],"Perplexity-User":["partial","named",12,1],"Google-Extended":["partial","named",12,1],"Applebot-Extended":["partial","*",12,1],"CCBot":["partial","named",12,1],"Bytespider":["partial","*",12,1],"Amazonbot":["partial","*",12,1],"meta-externalagent":["partial","*",12,1],"cohere-ai":["partial","*",12,1],"Diffbot":["partial","*",12,1],"ContextFetchBot":["partial","*",12,1]},"llms":{"status":"present","http":200,"bytes":5081},"home":{"status":"ok","http":200,"url":"https://www.vueling.com/en","html_tokens":72218,"markdown_tokens":0,"pruned_tokens":0}},
{"rank":9510,"domain":"omnigroup.com","origin":"https://omnigroup.com","robots":{"status":"ok","http":200,"url":"https://www.omnigroup.com/robots.txt","bytes":34,"groups":1},"bots":{"GPTBot":["partial","*",1,0],"ChatGPT-User":["partial","*",1,0],"OAI-SearchBot":["partial","*",1,0],"ClaudeBot":["partial","*",1,0],"Claude-User":["partial","*",1,0],"Claude-SearchBot":["partial","*",1,0],"anthropic-ai":["partial","*",1,0],"PerplexityBot":["partial","*",1,0],"Perplexity-User":["partial","*",1,0],"Google-Extended":["partial","*",1,0],"Applebot-Extended":["partial","*",1,0],"CCBot":["partial","*",1,0],"Bytespider":["partial","*",1,0],"Amazonbot":["partial","*",1,0],"meta-externalagent":["partial","*",1,0],"cohere-ai":["partial","*",1,0],"Diffbot":["partial","*",1,0],"ContextFetchBot":["partial","*",1,0]},"llms":{"status":"missing","http":404,"bytes":21364},"home":{"status":"ok","http":200,"url":"https://www.omnigroup.com/","html_tokens":6439,"markdown_tokens":35,"pruned_tokens":35}},
{"rank":9511,"domain":"telia.com","origin":"https://telia.com","robots":{"status":"html","http":200,"url":"https://www.telia.se/","bytes":1092605,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":9512,"domain":"poe.com","origin":"https://poe.com","robots":{"status":"ok","http":200,"url":"https://poe.com/robots.txt","bytes":509,"groups":7},"bots":{"GPTBot":["partial","named",1,0],"ChatGPT-User":["partial","*",14,1],"OAI-SearchBot":["partial","*",14,1],"ClaudeBot":["partial","*",14,1],"Claude-User":["partial","*",14,1],"Claude-SearchBot":["partial","*",14,1],"anthropic-ai":["partial","*",14,1],"PerplexityBot":["partial","*",14,1],"Perplexity-User":["partial","*",14,1],"Google-Extended":["partial","named",1,0],"Applebot-Extended":["partial","*",14,1],"CCBot":["partial","*",14,1],"Bytespider":["partial","*",14,1],"Amazonbot":["partial","*",14,1],"meta-externalagent":["partial","*",14,1],"cohere-ai":["partial","*",14,1],"Diffbot":["partial","*",14,1],"ContextFetchBot":["partial","*",14,1]},"llms":{"status":"disallowed"},"home":{"status":"disallowed"}},
{"rank":9513,"domain":"joystiq.com","origin":"https://joystiq.com","robots":{"status":"html","http":200,"url":"https://www.engadget.com/category/gaming/","bytes":90419,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":9514,"domain":"openshot.org","origin":"https://openshot.org","robots":{"status":"ok","http":200,"url":"https://openshot.org/robots.txt","bytes":94,"groups":1},"bots":{"GPTBot":["partial","*",1,0],"ChatGPT-User":["partial","*",1,0],"OAI-SearchBot":["partial","*",1,0],"ClaudeBot":["partial","*",1,0],"Claude-User":["partial","*",1,0],"Claude-SearchBot":["partial","*",1,0],"anthropic-ai":["partial","*",1,0],"PerplexityBot":["partial","*",1,0],"Perplexity-User":["partial","*",1,0],"Google-Extended":["partial","*",1,0],"Applebot-Extended":["partial","*",1,0],"CCBot":["partial","*",1,0],"Bytespider":["partial","*",1,0],"Amazonbot":["partial","*",1,0],"meta-externalagent":["partial","*",1,0],"cohere-ai":["partial","*",1,0],"Diffbot":["partial","*",1,0],"ContextFetchBot":["partial","*",1,0]},"llms":{"status":"missing","http":404,"bytes":57487},"home":{"status":"ok","http":200,"url":"https://www.openshot.org/","html_tokens":18549,"markdown_tokens":133,"pruned_tokens":88}},
{"rank":9515,"domain":"doodlekit.com","origin":"https://www.doodlekit.com","robots":{"status":"ok","http":200,"url":"https://www.doodlekit.com/robots.txt","bytes":292,"groups":6},"bots":{"GPTBot":["blocked","*",6,0],"ChatGPT-User":["blocked","*",6,0],"OAI-SearchBot":["blocked","*",6,0],"ClaudeBot":["blocked","*",6,0],"Claude-User":["blocked","*",6,0],"Claude-SearchBot":["blocked","*",6,0],"anthropic-ai":["blocked","*",6,0],"PerplexityBot":["blocked","*",6,0],"Perplexity-User":["blocked","*",6,0],"Google-Extended":["blocked","*",6,0],"Applebot-Extended":["blocked","*",6,0],"CCBot":["blocked","*",6,0],"Bytespider":["blocked","*",6,0],"Amazonbot":["blocked","*",6,0],"meta-externalagent":["blocked","*",6,0],"cohere-ai":["blocked","*",6,0],"Diffbot":["blocked","*",6,0],"ContextFetchBot":["blocked","*",6,0]},"llms":{"status":"disallowed"},"home":{"status":"disallowed"}},
{"rank":9516,"domain":"observador.pt","origin":"https://observador.pt","robots":{"status":"ok","http":200,"url":"https://observador.pt/robots.txt","bytes":1206,"groups":4},"bots":{"GPTBot":["allowed","named",0,3],"ChatGPT-User":["allowed","named",0,3],"OAI-SearchBot":["allowed","named",0,3],"ClaudeBot":["allowed","named",0,3],"Claude-User":["partial","*",2,1],"Claude-SearchBot":["partial","*",2,1],"anthropic-ai":["allowed","named",0,3],"PerplexityBot":["partial","*",2,1],"Perplexity-User":["partial","*",2,1],"Google-Extended":["allowed","named",0,1],"Applebot-Extended":["allowed","named",0,3],"CCBot":["allowed","named",0,3],"Bytespider":["partial","named",1,2],"Amazonbot":["allowed","named",0,3],"meta-externalagent":["allowed","named",0,3],"cohere-ai":["allowed","named",0,3],"Diffbot":["allowed","named",0,3],"ContextFetchBot":["partial","*",2,1]},"llms":{"status":"missing","http":404,"bytes":160059},"home":{"status":"ok","http":200,"url":"https://observador.pt/","html_tokens":253736,"markdown_tokens":210,"pruned_tokens":54}},
{"rank":9517,"domain":"ngrok.com","origin":"https://ngrok.com","robots":{"status":"ok","http":200,"url":"https://ngrok.com/robots.txt","bytes":116,"groups":1},"bots":{"GPTBot":["allowed","*",0,1],"ChatGPT-User":["allowed","*",0,1],"OAI-SearchBot":["allowed","*",0,1],"ClaudeBot":["allowed","*",0,1],"Claude-User":["allowed","*",0,1],"Claude-SearchBot":["allowed","*",0,1],"anthropic-ai":["allowed","*",0,1],"PerplexityBot":["allowed","*",0,1],"Perplexity-User":["allowed","*",0,1],"Google-Extended":["allowed","*",0,1],"Applebot-Extended":["allowed","*",0,1],"CCBot":["allowed","*",0,1],"Bytespider":["allowed","*",0,1],"Amazonbot":["allowed","*",0,1],"meta-externalagent":["allowed","*",0,1],"cohere-ai":["allowed","*",0,1],"Diffbot":["allowed","*",0,1],"ContextFetchBot":["allowed","*",0,1]},"llms":{"status":"present","http":200,"bytes":27960},"home":{"status":"not_html","error":"unsupported content type text/markdown"}},
{"rank":9518,"domain":"msc.org","origin":"https://msc.org","robots":{"status":"ok","http":200,"url":"https://www.msc.org/robots.txt","bytes":1416,"groups":1},"bots":{"GPTBot":["partial","*",3,0],"ChatGPT-User":["partial","*",3,0],"OAI-SearchBot":["partial","*",3,0],"ClaudeBot":["partial","*",3,0],"Claude-User":["partial","*",3,0],"Claude-SearchBot":["partial","*",3,0],"anthropic-ai":["partial","*",3,0],"PerplexityBot":["partial","*",3,0],"Perplexity-User":["partial","*",3,0],"Google-Extended":["partial","*",3,0],"Applebot-Extended":["partial","*",3,0],"CCBot":["partial","*",3,0],"Bytespider":["partial","*",3,0],"Amazonbot":["partial","*",3,0],"meta-externalagent":["partial","*",3,0],"cohere-ai":["partial","*",3,0],"Diffbot":["partial","*",3,0],"ContextFetchBot":["partial","*",3,0]},"llms":{"status":"present","http":200,"bytes":759580},"home":{"status":"ok","http":200,"url":"https://www.msc.org/en-us","html_tokens":24779,"markdown_tokens":71,"pruned_tokens":71}},
{"rank":9519,"domain":"szys2022.cn","origin":"http://szys2022.cn","robots":{"status":"missing","http":404,"url":"http://szys2022.cn/robots.txt","bytes":1163,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":1163},"home":{"status":"ok","http":200,"url":"http://szys2022.cn/","html_tokens":772,"markdown_tokens":26,"pruned_tokens":26}},
{"rank":9520,"domain":"glovoapp.com","origin":"https://glovoapp.com","robots":{"status":"server_error","http":503,"url":"https://glovoapp.com/robots.txt","bytes":5811,"groups":0,"error":"HTTP 503"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":9521,"domain":"big.or.jp","origin":"https://www.big.or.jp","robots":{"status":"ok","http":200,"url":"https://www.big.or.jp/robots.txt","bytes":964,"groups":23},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["blocked","named",1,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":579},"home":{"status":"ok","http":200,"url":"https://www.big.or.jp/","html_tokens":145,"markdown_tokens":7,"pruned_tokens":7}},
{"rank":9522,"domain":"easyslimnow.store","origin":"https://easyslimnow.store","robots":{"status":"html","http":200,"url":"https://orlilab.com/","bytes":57052,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":9523,"domain":"kaskus.co.id","origin":"https://kaskus.co.id","robots":{"status":"ok","http":200,"url":"https://m.kaskus.co.id/robots.txt","bytes":323,"groups":1},"bots":{"GPTBot":["partial","*",14,0],"ChatGPT-User":["partial","*",14,0],"OAI-SearchBot":["partial","*",14,0],"ClaudeBot":["partial","*",14,0],"Claude-User":["partial","*",14,0],"Claude-SearchBot":["partial","*",14,0],"anthropic-ai":["partial","*",14,0],"PerplexityBot":["partial","*",14,0],"Perplexity-User":["partial","*",14,0],"Google-Extended":["partial","*",14,0],"Applebot-Extended":["partial","*",14,0],"CCBot":["partial","*",14,0],"Bytespider":["partial","*",14,0],"Amazonbot":["partial","*",14,0],"meta-externalagent":["partial","*",14,0],"cohere-ai":["partial","*",14,0],"Diffbot":["partial","*",14,0],"ContextFetchBot":["partial","*",14,0]},"llms":{"status":"missing","http":404,"bytes":3095},"home":{"status":"ok","http":200,"url":"https://m.kaskus.co.id/home","html_tokens":19399,"markdown_tokens":11,"pruned_tokens":11}},
{"rank":9524,"domain":"historynet.com","origin":"https://historynet.com","robots":{"status":"ok","http":200,"url":"https://historynet.com/robots.txt","bytes":255,"groups":2},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"missing","http":404,"bytes":105124},"home":{"status":"ok","http":200,"url":"https://historynet.com/","html_tokens":51278,"markdown_tokens":159,"pruned_tokens":160}},
{"rank":9525,"domain":"sprint.com","origin":"https://sprint.com","robots":{"status":"refused","http":403,"url":"https://sprint.com/robots.txt","bytes":374,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":9526,"domain":"tvzvezda.ru","origin":"https://tvzvezda.ru","robots":{"status":"ok","http":200,"url":"https://tvzvezda.ru/robots.txt","bytes":571,"groups":3},"bots":{"GPTBot":["partial","*",2,1],"ChatGPT-User":["partial","*",2,1],"OAI-SearchBot":["partial","*",2,1],"ClaudeBot":["partial","*",2,1],"Claude-User":["partial","*",2,1],"Claude-SearchBot":["partial","*",2,1],"anthropic-ai":["partial","*",2,1],"PerplexityBot":["partial","*",2,1],"Perplexity-User":["partial","*",2,1],"Google-Extended":["partial","*",2,1],"Applebot-Extended":["partial","*",2,1],"CCBot":["partial","*",2,1],"Bytespider":["partial","*",2,1],"Amazonbot":["partial","*",2,1],"meta-externalagent":["partial","*",2,1],"cohere-ai":["partial","*",2,1],"Diffbot":["partial","*",2,1],"ContextFetchBot":["partial","*",2,1]},"llms":{"status":"present","http":200,"bytes":31225},"home":{"status":"ok","http":200,"url":"https://tvzvezda.ru/","html_tokens":68069,"markdown_tokens":20,"pruned_tokens":20}},
{"rank":9527,"domain":"marieclaire.fr","origin":"https://marieclaire.fr","robots":{"status":"ok","http":200,"url":"https://www.marieclaire.fr/robots.txt","bytes":621,"groups":2},"bots":{"GPTBot":["partial","*",21,0],"ChatGPT-User":["partial","*",21,0],"OAI-SearchBot":["partial","*",21,0],"ClaudeBot":["partial","*",21,0],"Claude-User":["partial","*",21,0],"Claude-SearchBot":["partial","*",21,0],"anthropic-ai":["partial","*",21,0],"PerplexityBot":["partial","*",21,0],"Perplexity-User":["partial","*",21,0],"Google-Extended":["partial","*",21,0],"Applebot-Extended":["partial","*",21,0],"CCBot":["partial","*",21,0],"Bytespider":["partial","*",21,0],"Amazonbot":["partial","*",21,0],"meta-externalagent":["partial","*",21,0],"cohere-ai":["partial","*",21,0],"Diffbot":["partial","*",21,0],"ContextFetchBot":["partial","*",21,0]},"llms":{"status":"missing","http":404,"bytes":52606},"home":{"status":"ok","http":200,"url":"https://www.marieclaire.fr/","html_tokens":84313,"markdown_tokens":295,"pruned_tokens":44}},
{"rank":9528,"domain":"observer.co.uk","origin":"https://observer.co.uk","robots":{"status":"ok","http":200,"url":"https://observer.co.uk/robots.txt","bytes":1112,"groups":21},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["partial","named",2,0],"OAI-SearchBot":["partial","named",2,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","named",2,0],"Claude-SearchBot":["partial","named",2,0],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["partial","named",2,0],"Perplexity-User":["partial","named",2,0],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",2,0]},"llms":{"status":"missing","http":404,"bytes":194347},"home":{"status":"ok","http":200,"url":"https://observer.co.uk/","html_tokens":235989,"markdown_tokens":35,"pruned_tokens":9}},
{"rank":9529,"domain":"cabidigitallibrary.org","origin":"https://cabidigitallibrary.org","robots":{"status":"ok","http":200,"url":"https://www.cabidigitallibrary.org/robots.txt","bytes":821,"groups":3},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["partial","*",18,7],"OAI-SearchBot":["partial","*",18,7],"ClaudeBot":["partial","*",18,7],"Claude-User":["partial","*",18,7],"Claude-SearchBot":["partial","*",18,7],"anthropic-ai":["partial","*",18,7],"PerplexityBot":["partial","*",18,7],"Perplexity-User":["partial","*",18,7],"Google-Extended":["partial","*",18,7],"Applebot-Extended":["partial","*",18,7],"CCBot":["partial","*",18,7],"Bytespider":["partial","*",18,7],"Amazonbot":["partial","*",18,7],"meta-externalagent":["partial","*",18,7],"cohere-ai":["partial","*",18,7],"Diffbot":["partial","*",18,7],"ContextFetchBot":["partial","*",18,7]},"llms":{"status":"http_error","http":403,"bytes":5575},"home":{"status":"http_error","http":403,"url":"https://cabidigitallibrary.org/"}},
{"rank":9530,"domain":"hashthemes.com","origin":"https://hashthemes.com","robots":{"status":"ok","http":200,"url":"https://hashthemes.com/robots.txt","bytes":118,"groups":1},"bots":{"GPTBot":["partial","*",1,1],"ChatGPT-User":["partial","*",1,1],"OAI-SearchBot":["partial","*",1,1],"ClaudeBot":["partial","*",1,1],"Claude-User":["partial","*",1,1],"Claude-SearchBot":["partial","*",1,1],"anthropic-ai":["partial","*",1,1],"PerplexityBot":["partial","*",1,1],"Perplexity-User":["partial","*",1,1],"Google-Extended":["partial","*",1,1],"Applebot-Extended":["partial","*",1,1],"CCBot":["partial","*",1,1],"Bytespider":["partial","*",1,1],"Amazonbot":["partial","*",1,1],"meta-externalagent":["partial","*",1,1],"cohere-ai":["partial","*",1,1],"Diffbot":["partial","*",1,1],"ContextFetchBot":["partial","*",1,1]},"llms":{"status":"missing","http":404,"bytes":36642},"home":{"status":"ok","http":200,"url":"https://hashthemes.com/","html_tokens":20590,"markdown_tokens":517,"pruned_tokens":122}},
{"rank":9531,"domain":"incompetech.com","origin":"https://incompetech.com","robots":{"status":"ok","http":200,"url":"https://incompetech.com/robots.txt","bytes":103,"groups":1},"bots":{"GPTBot":["partial","*",4,0],"ChatGPT-User":["partial","*",4,0],"OAI-SearchBot":["partial","*",4,0],"ClaudeBot":["partial","*",4,0],"Claude-User":["partial","*",4,0],"Claude-SearchBot":["partial","*",4,0],"anthropic-ai":["partial","*",4,0],"PerplexityBot":["partial","*",4,0],"Perplexity-User":["partial","*",4,0],"Google-Extended":["partial","*",4,0],"Applebot-Extended":["partial","*",4,0],"CCBot":["partial","*",4,0],"Bytespider":["partial","*",4,0],"Amazonbot":["partial","*",4,0],"meta-externalagent":["partial","*",4,0],"cohere-ai":["partial","*",4,0],"Diffbot":["partial","*",4,0],"ContextFetchBot":["partial","*",4,0]},"llms":{"status":"present","http":200,"bytes":4244},"home":{"status":"ok","http":200,"url":"https://incompetech.com/","html_tokens":3472,"markdown_tokens":1872,"pruned_tokens":439}},
{"rank":9532,"domain":"meixingsoft.cn","origin":"https://meixingsoft.cn","robots":{"status":"unreachable","http":null,"url":"https://meixingsoft.cn/robots.txt","bytes":0,"groups":0,"error":"cannot resolve meixingsoft.cn"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":9533,"domain":"chunichi.co.jp","origin":"https://chunichi.co.jp","robots":{"status":"ok","http":200,"url":"https://www.chunichi.co.jp/robots.txt","bytes":2912,"groups":63},"bots":{"GPTBot":["partial","named",1,1],"ChatGPT-User":["partial","named",1,1],"OAI-SearchBot":["partial","named",1,1],"ClaudeBot":["blocked","named",1,0],"Claude-User":["blocked","named",1,0],"Claude-SearchBot":["blocked","named",1,0],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["blocked","named",1,0],"Google-Extended":["partial","named",1,1],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",3,1]},"llms":{"status":"missing","http":404,"bytes":23514},"home":{"status":"ok","http":200,"url":"https://www.chunichi.co.jp/","html_tokens":46732,"markdown_tokens":2,"pruned_tokens":2}},
{"rank":9534,"domain":"2checkout.com","origin":"https://2checkout.com","robots":{"status":"ok","http":200,"url":"https://www.2checkout.com/robots.txt","bytes":1970,"groups":2},"bots":{"GPTBot":["partial","*",28,0],"ChatGPT-User":["partial","*",28,0],"OAI-SearchBot":["partial","*",28,0],"ClaudeBot":["partial","*",28,0],"Claude-User":["partial","*",28,0],"Claude-SearchBot":["partial","*",28,0],"anthropic-ai":["partial","*",28,0],"PerplexityBot":["partial","*",28,0],"Perplexity-User":["partial","*",28,0],"Google-Extended":["partial","*",28,0],"Applebot-Extended":["partial","*",28,0],"CCBot":["partial","*",28,0],"Bytespider":["partial","*",28,0],"Amazonbot":["partial","*",28,0],"meta-externalagent":["partial","*",28,0],"cohere-ai":["partial","*",28,0],"Diffbot":["partial","*",28,0],"ContextFetchBot":["partial","*",28,0]},"llms":{"status":"missing","http":404,"bytes":149299},"home":{"status":"ok","http":200,"url":"https://2checkout.com/","html_tokens":53,"markdown_tokens":0,"pruned_tokens":0}},
{"rank":9535,"domain":"fox59.com","origin":"https://fox59.com","robots":{"status":"ok","http":200,"url":"https://fox59.com/robots.txt","bytes":2297,"groups":40},"bots":{"GPTBot":["blocked","named",2,0],"ChatGPT-User":["blocked","named",1,0],"OAI-SearchBot":["blocked","named",1,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",12,1],"Claude-SearchBot":["partial","*",12,1],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["blocked","named",2,0],"Perplexity-User":["partial","*",12,1],"Google-Extended":["blocked","named",2,0],"Applebot-Extended":["blocked","named",2,0],"CCBot":["blocked","named",2,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["partial","*",12,1],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",12,1]},"llms":{"status":"http_error","http":403,"bytes":6514},"home":{"status":"http_error","http":403,"url":"https://fox59.com/"}},
{"rank":9536,"domain":"westernsydney.edu.au","origin":"https://westernsydney.edu.au","robots":{"status":"ok","http":200,"url":"https://www.westernsydney.edu.au/robots.txt","bytes":2939,"groups":1},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":7382},"home":{"status":"ok","http":200,"url":"https://www.westernsydney.edu.au/","html_tokens":51227,"markdown_tokens":175,"pruned_tokens":79}},
{"rank":9537,"domain":"coventry.ac.uk","origin":"https://www.coventry.ac.uk","robots":{"status":"ok","http":200,"url":"https://www.coventry.ac.uk/robots.txt","bytes":2701,"groups":1},"bots":{"GPTBot":["partial","*",59,14],"ChatGPT-User":["partial","*",59,14],"OAI-SearchBot":["partial","*",59,14],"ClaudeBot":["partial","*",59,14],"Claude-User":["partial","*",59,14],"Claude-SearchBot":["partial","*",59,14],"anthropic-ai":["partial","*",59,14],"PerplexityBot":["partial","*",59,14],"Perplexity-User":["partial","*",59,14],"Google-Extended":["partial","*",59,14],"Applebot-Extended":["partial","*",59,14],"CCBot":["partial","*",59,14],"Bytespider":["partial","*",59,14],"Amazonbot":["partial","*",59,14],"meta-externalagent":["partial","*",59,14],"cohere-ai":["partial","*",59,14],"Diffbot":["partial","*",59,14],"ContextFetchBot":["partial","*",59,14]},"llms":{"status":"missing","http":404,"bytes":103},"home":{"status":"ok","http":200,"url":"https://www.coventry.ac.uk/","html_tokens":20780,"markdown_tokens":199,"pruned_tokens":90}},
{"rank":9538,"domain":"lobstermania-slot.com","origin":"https://lobstermania-slot.com","robots":{"status":"ok","http":200,"url":"https://lobstermania-slot.com/robots.txt","bytes":546,"groups":1},"bots":{"GPTBot":["partial","*",19,0],"ChatGPT-User":["partial","*",19,0],"OAI-SearchBot":["partial","*",19,0],"ClaudeBot":["partial","*",19,0],"Claude-User":["partial","*",19,0],"Claude-SearchBot":["partial","*",19,0],"anthropic-ai":["partial","*",19,0],"PerplexityBot":["partial","*",19,0],"Perplexity-User":["partial","*",19,0],"Google-Extended":["partial","*",19,0],"Applebot-Extended":["partial","*",19,0],"CCBot":["partial","*",19,0],"Bytespider":["partial","*",19,0],"Amazonbot":["partial","*",19,0],"meta-externalagent":["partial","*",19,0],"cohere-ai":["partial","*",19,0],"Diffbot":["partial","*",19,0],"ContextFetchBot":["partial","*",19,0]},"llms":{"status":"missing","http":404,"bytes":24218},"home":{"status":"ok","http":200,"url":"https://lobstermania-slot.com/","html_tokens":19857,"markdown_tokens":922,"pruned_tokens":481}},
{"rank":9539,"domain":"univ-nantes.fr","origin":"https://univ-nantes.fr","robots":{"status":"ok","http":200,"url":"https://www.univ-nantes.fr/robots.txt","bytes":341,"groups":1},"bots":{"GPTBot":["partial","*",10,0],"ChatGPT-User":["partial","*",10,0],"OAI-SearchBot":["partial","*",10,0],"ClaudeBot":["partial","*",10,0],"Claude-User":["partial","*",10,0],"Claude-SearchBot":["partial","*",10,0],"anthropic-ai":["partial","*",10,0],"PerplexityBot":["partial","*",10,0],"Perplexity-User":["partial","*",10,0],"Google-Extended":["partial","*",10,0],"Applebot-Extended":["partial","*",10,0],"CCBot":["partial","*",10,0],"Bytespider":["partial","*",10,0],"Amazonbot":["partial","*",10,0],"meta-externalagent":["partial","*",10,0],"cohere-ai":["partial","*",10,0],"Diffbot":["partial","*",10,0],"ContextFetchBot":["partial","*",10,0]},"llms":{"status":"missing","http":404,"bytes":352117},"home":{"status":"ok","http":200,"url":"https://www.univ-nantes.fr/","html_tokens":95737,"markdown_tokens":591,"pruned_tokens":173}},
{"rank":9540,"domain":"126.com","origin":"https://126.com","robots":{"status":"missing","http":404,"url":"https://126.com/robots.txt","bytes":2660,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":2660},"home":{"status":"ok","http":200,"url":"https://126.com/","html_tokens":4876,"markdown_tokens":50,"pruned_tokens":50}},
{"rank":9541,"domain":"nagano.jp","origin":"https://nagano.jp","robots":{"status":"unreachable","http":null,"url":"https://nagano.jp/robots.txt","bytes":0,"groups":0,"error":"cannot resolve nagano.jp"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":9542,"domain":"knmi.nl","origin":"https://knmi.nl","robots":{"status":"ok","http":200,"url":"https://www.knmi.nl/robots.txt","bytes":99,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"html","http":200,"bytes":1722},"home":{"status":"ok","http":200,"url":"https://www.knmi.nl/home","html_tokens":12624,"markdown_tokens":86,"pruned_tokens":70}},
{"rank":9543,"domain":"wxhdlhj.cn","origin":"http://wxhdlhj.cn","robots":{"status":"ok","http":200,"url":"http://wxhdlhj.cn/robots.txt","bytes":485,"groups":1},"bots":{"GPTBot":["partial","*",21,0],"ChatGPT-User":["partial","*",21,0],"OAI-SearchBot":["partial","*",21,0],"ClaudeBot":["partial","*",21,0],"Claude-User":["partial","*",21,0],"Claude-SearchBot":["partial","*",21,0],"anthropic-ai":["partial","*",21,0],"PerplexityBot":["partial","*",21,0],"Perplexity-User":["partial","*",21,0],"Google-Extended":["partial","*",21,0],"Applebot-Extended":["partial","*",21,0],"CCBot":["partial","*",21,0],"Bytespider":["partial","*",21,0],"Amazonbot":["partial","*",21,0],"meta-externalagent":["partial","*",21,0],"cohere-ai":["partial","*",21,0],"Diffbot":["partial","*",21,0],"ContextFetchBot":["partial","*",21,0]},"llms":{"status":"missing","http":404,"bytes":4724},"home":{"status":"ok","http":200,"url":"http://wxhdlhj.cn/","html_tokens":9649,"markdown_tokens":121,"pruned_tokens":121}},
{"rank":9544,"domain":"tntsports.co.uk","origin":"https://tntsports.co.uk","robots":{"status":"ok","http":200,"url":"https://www.tntsports.co.uk/robots.txt","bytes":4076,"groups":5},"bots":{"GPTBot":["partial","*",5,0],"ChatGPT-User":["partial","*",5,0],"OAI-SearchBot":["partial","*",5,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",5,0],"Claude-SearchBot":["partial","*",5,0],"anthropic-ai":["partial","*",5,0],"PerplexityBot":["partial","*",5,0],"Perplexity-User":["partial","*",5,0],"Google-Extended":["partial","*",5,0],"Applebot-Extended":["partial","*",5,0],"CCBot":["partial","*",5,0],"Bytespider":["partial","*",5,0],"Amazonbot":["partial","*",5,0],"meta-externalagent":["partial","*",5,0],"cohere-ai":["partial","*",5,0],"Diffbot":["partial","*",5,0],"ContextFetchBot":["partial","*",5,0]},"llms":{"status":"missing","http":404,"bytes":197106},"home":{"status":"ok","http":200,"url":"https://www.tntsports.co.uk/","html_tokens":236436,"markdown_tokens":35,"pruned_tokens":35}},
{"rank":9545,"domain":"yenisafak.com","origin":"https://yenisafak.com","robots":{"status":"ok","http":200,"url":"https://www.yenisafak.com/robots.txt","bytes":746,"groups":1},"bots":{"GPTBot":["partial","*",14,1],"ChatGPT-User":["partial","*",14,1],"OAI-SearchBot":["partial","*",14,1],"ClaudeBot":["partial","*",14,1],"Claude-User":["partial","*",14,1],"Claude-SearchBot":["partial","*",14,1],"anthropic-ai":["partial","*",14,1],"PerplexityBot":["partial","*",14,1],"Perplexity-User":["partial","*",14,1],"Google-Extended":["partial","*",14,1],"Applebot-Extended":["partial","*",14,1],"CCBot":["partial","*",14,1],"Bytespider":["partial","*",14,1],"Amazonbot":["partial","*",14,1],"meta-externalagent":["partial","*",14,1],"cohere-ai":["partial","*",14,1],"Diffbot":["partial","*",14,1],"ContextFetchBot":["partial","*",14,1]},"llms":{"status":"missing","http":404,"bytes":251079},"home":{"status":"ok","http":200,"url":"https://www.yenisafak.com/","html_tokens":468739,"markdown_tokens":2920,"pruned_tokens":174}},
{"rank":9546,"domain":"studyfinds.org","origin":"https://studyfinds.org","robots":{"status":"ok","http":200,"url":"https://studyfinds.com/robots.txt","bytes":564,"groups":3},"bots":{"GPTBot":["partial","*",2,1],"ChatGPT-User":["partial","*",2,1],"OAI-SearchBot":["partial","*",2,1],"ClaudeBot":["partial","*",2,1],"Claude-User":["partial","*",2,1],"Claude-SearchBot":["partial","*",2,1],"anthropic-ai":["partial","*",2,1],"PerplexityBot":["partial","*",2,1],"Perplexity-User":["partial","*",2,1],"Google-Extended":["partial","*",2,1],"Applebot-Extended":["partial","*",2,1],"CCBot":["partial","*",2,1],"Bytespider":["partial","*",2,1],"Amazonbot":["partial","*",2,1],"meta-externalagent":["partial","*",2,1],"cohere-ai":["partial","*",2,1],"Diffbot":["partial","*",2,1],"ContextFetchBot":["partial","*",2,1]},"llms":{"status":"missing","http":404,"bytes":202780},"home":{"status":"ok","http":200,"url":"https://studyfinds.com/","html_tokens":71804,"markdown_tokens":293,"pruned_tokens":140}},
{"rank":9547,"domain":"qifu66.com","origin":"http://qifu66.com","robots":{"status":"server_error","http":500,"url":"http://qifu66.com/mycustompage.htm?aspxerrorpath=/robots.txt","bytes":1773,"groups":0,"error":"HTTP 500"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":9548,"domain":"dunkindonuts.com","origin":"https://dunkindonuts.com","robots":{"status":"ok","http":200,"url":"https://www.dunkindonuts.com/robots.txt","bytes":74,"groups":1},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"missing","http":404,"bytes":63808},"home":{"status":"ok","http":200,"url":"https://www.dunkindonuts.com/","html_tokens":27315,"markdown_tokens":43,"pruned_tokens":43}},
{"rank":9549,"domain":"backblazeb2.com","origin":"https://backblazeb2.com","robots":{"status":"ok","http":200,"url":"https://backblazeb2.com/robots.txt","bytes":26,"groups":1},"bots":{"GPTBot":["blocked","*",1,0],"ChatGPT-User":["blocked","*",1,0],"OAI-SearchBot":["blocked","*",1,0],"ClaudeBot":["blocked","*",1,0],"Claude-User":["blocked","*",1,0],"Claude-SearchBot":["blocked","*",1,0],"anthropic-ai":["blocked","*",1,0],"PerplexityBot":["blocked","*",1,0],"Perplexity-User":["blocked","*",1,0],"Google-Extended":["blocked","*",1,0],"Applebot-Extended":["blocked","*",1,0],"CCBot":["blocked","*",1,0],"Bytespider":["blocked","*",1,0],"Amazonbot":["blocked","*",1,0],"meta-externalagent":["blocked","*",1,0],"cohere-ai":["blocked","*",1,0],"Diffbot":["blocked","*",1,0],"ContextFetchBot":["blocked","*",1,0]},"llms":{"status":"disallowed"},"home":{"status":"disallowed"}},
{"rank":9550,"domain":"dp.ru","origin":"https://dp.ru","robots":{"status":"ok","http":200,"url":"https://www.dp.ru/robots.txt","bytes":461,"groups":3},"bots":{"GPTBot":["partial","*",2,0],"ChatGPT-User":["partial","*",2,0],"OAI-SearchBot":["partial","*",2,0],"ClaudeBot":["partial","*",2,0],"Claude-User":["partial","*",2,0],"Claude-SearchBot":["partial","*",2,0],"anthropic-ai":["partial","*",2,0],"PerplexityBot":["partial","*",2,0],"Perplexity-User":["partial","*",2,0],"Google-Extended":["partial","*",2,0],"Applebot-Extended":["partial","*",2,0],"CCBot":["partial","*",2,0],"Bytespider":["partial","*",2,0],"Amazonbot":["partial","*",2,0],"meta-externalagent":["partial","*",2,0],"cohere-ai":["partial","*",2,0],"Diffbot":["partial","*",2,0],"ContextFetchBot":["partial","*",2,0]},"llms":{"status":"missing","http":404,"bytes":40623},"home":{"status":"ok","http":200,"url":"https://www.dp.ru/","html_tokens":20327,"markdown_tokens":133,"pruned_tokens":133}},
{"rank":9551,"domain":"xysa.com.cn","origin":"https://xysa.com.cn","robots":{"status":"timeout","http":null,"url":"https://xysa.com.cn/robots.txt","bytes":0,"groups":0,"error":"ConnectTimeout"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":9552,"domain":"amplitude.com","origin":"https://amplitude.com","robots":{"status":"ok","http":200,"url":"https://amplitude.com/robots.txt","bytes":2385,"groups":16},"bots":{"GPTBot":["partial","named",1,1],"ChatGPT-User":["partial","named",1,1],"OAI-SearchBot":["partial","named",1,1],"ClaudeBot":["partial","named",1,1],"Claude-User":["partial","*",1,1],"Claude-SearchBot":["partial","*",1,1],"anthropic-ai":["partial","named",1,1],"PerplexityBot":["partial","named",1,1],"Perplexity-User":["partial","named",1,1],"Google-Extended":["partial","named",1,1],"Applebot-Extended":["partial","named",1,1],"CCBot":["partial","named",1,1],"Bytespider":["blocked","named",1,0],"Amazonbot":["partial","named",1,1],"meta-externalagent":["partial","named",1,1],"cohere-ai":["partial","named",1,1],"Diffbot":["partial","*",1,1],"ContextFetchBot":["partial","*",1,1]},"llms":{"status":"present","http":200,"bytes":14777},"home":{"status":"ok","http":200,"url":"https://amplitude.com/","html_tokens":97871,"markdown_tokens":340,"pruned_tokens":96}},
{"rank":9553,"domain":"wtsp.com","origin":"https://wtsp.com","robots":{"status":"ok","http":200,"url":"https://www.wtsp.com/robots.txt","bytes":444,"groups":2},"bots":{"GPTBot":["partial","*",10,0],"ChatGPT-User":["partial","*",10,0],"OAI-SearchBot":["partial","*",10,0],"ClaudeBot":["partial","*",10,0],"Claude-User":["partial","*",10,0],"Claude-SearchBot":["partial","*",10,0],"anthropic-ai":["partial","*",10,0],"PerplexityBot":["partial","*",10,0],"Perplexity-User":["partial","*",10,0],"Google-Extended":["partial","*",10,0],"Applebot-Extended":["partial","*",10,0],"CCBot":["partial","*",10,0],"Bytespider":["partial","*",10,0],"Amazonbot":["partial","*",10,0],"meta-externalagent":["partial","*",10,0],"cohere-ai":["partial","*",10,0],"Diffbot":["partial","*",10,0],"ContextFetchBot":["partial","*",10,0]},"llms":{"status":"missing","http":404,"bytes":6832},"home":{"status":"ok","http":200,"url":"https://www.wtsp.com/","html_tokens":97470,"markdown_tokens":1650,"pruned_tokens":168}},
{"rank":9554,"domain":"sdjpzy.com","origin":"http://sdjpzy.com","robots":{"status":"missing","http":404,"url":"http://sdjpzy.com/robots.txt","bytes":315,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":315},"home":{"status":"http_error","http":404,"url":"http://sdjpzy.com/"}},
{"rank":9555,"domain":"slickdeals.net","origin":"https://slickdeals.net","robots":{"status":"ok","http":200,"url":"https://slickdeals.net/robots.txt","bytes":3043,"groups":5},"bots":{"GPTBot":["partial","*",111,2],"ChatGPT-User":["partial","*",111,2],"OAI-SearchBot":["partial","*",111,2],"ClaudeBot":["partial","*",111,2],"Claude-User":["partial","*",111,2],"Claude-SearchBot":["partial","*",111,2],"anthropic-ai":["partial","*",111,2],"PerplexityBot":["partial","*",111,2],"Perplexity-User":["partial","*",111,2],"Google-Extended":["partial","*",111,2],"Applebot-Extended":["partial","*",111,2],"CCBot":["partial","*",111,2],"Bytespider":["partial","*",111,2],"Amazonbot":["partial","*",111,2],"meta-externalagent":["partial","*",111,2],"cohere-ai":["partial","*",111,2],"Diffbot":["partial","*",111,2],"ContextFetchBot":["partial","*",111,2]},"llms":{"status":"missing","http":404,"bytes":49068},"home":{"status":"ok","http":200,"url":"https://slickdeals.net/","html_tokens":710625,"markdown_tokens":7,"pruned_tokens":7}},
{"rank":9556,"domain":"thescottishsun.co.uk","origin":"https://thescottishsun.co.uk","robots":{"status":"ok","http":200,"url":"https://www.thescottishsun.co.uk/robots.txt","bytes":5039,"groups":2},"bots":{"GPTBot":["partial","named",104,3],"ChatGPT-User":["partial","named",104,3],"OAI-SearchBot":["partial","named",104,3],"ClaudeBot":["blocked","*",1,0],"Claude-User":["blocked","*",1,0],"Claude-SearchBot":["blocked","*",1,0],"anthropic-ai":["blocked","*",1,0],"PerplexityBot":["blocked","*",1,0],"Perplexity-User":["blocked","*",1,0],"Google-Extended":["blocked","*",1,0],"Applebot-Extended":["blocked","*",1,0],"CCBot":["blocked","*",1,0],"Bytespider":["blocked","*",1,0],"Amazonbot":["blocked","*",1,0],"meta-externalagent":["blocked","*",1,0],"cohere-ai":["blocked","*",1,0],"Diffbot":["blocked","*",1,0],"ContextFetchBot":["blocked","*",1,0]},"llms":{"status":"disallowed"},"home":{"status":"disallowed"}},
{"rank":9557,"domain":"lowyinstitute.org","origin":"https://lowyinstitute.org","robots":{"status":"ok","http":200,"url":"https://www.lowyinstitute.org/robots.txt","bytes":142,"groups":1},"bots":{"GPTBot":["partial","*",2,1],"ChatGPT-User":["partial","*",2,1],"OAI-SearchBot":["partial","*",2,1],"ClaudeBot":["partial","*",2,1],"Claude-User":["partial","*",2,1],"Claude-SearchBot":["partial","*",2,1],"anthropic-ai":["partial","*",2,1],"PerplexityBot":["partial","*",2,1],"Perplexity-User":["partial","*",2,1],"Google-Extended":["partial","*",2,1],"Applebot-Extended":["partial","*",2,1],"CCBot":["partial","*",2,1],"Bytespider":["partial","*",2,1],"Amazonbot":["partial","*",2,1],"meta-externalagent":["partial","*",2,1],"cohere-ai":["partial","*",2,1],"Diffbot":["partial","*",2,1],"ContextFetchBot":["partial","*",2,1]},"llms":{"status":"html","http":200,"bytes":119109},"home":{"status":"ok","http":200,"url":"https://www.lowyinstitute.org/","html_tokens":72967,"markdown_tokens":87,"pruned_tokens":87}},
{"rank":9558,"domain":"google.com.tj","origin":"https://google.com.tj","robots":{"status":"ok","http":200,"url":"https://www.google.com.tj/robots.txt","bytes":6624,"groups":4},"bots":{"GPTBot":["partial","*",169,64],"ChatGPT-User":["partial","*",169,64],"OAI-SearchBot":["partial","*",169,64],"ClaudeBot":["partial","*",169,64],"Claude-User":["partial","*",169,64],"Claude-SearchBot":["partial","*",169,64],"anthropic-ai":["partial","*",169,64],"PerplexityBot":["partial","*",169,64],"Perplexity-User":["partial","*",169,64],"Google-Extended":["partial","*",169,64],"Applebot-Extended":["partial","*",169,64],"CCBot":["partial","*",169,64],"Bytespider":["partial","*",169,64],"Amazonbot":["partial","*",169,64],"meta-externalagent":["partial","*",169,64],"cohere-ai":["partial","*",169,64],"Diffbot":["partial","*",169,64],"ContextFetchBot":["partial","*",169,64]},"llms":{"status":"missing","http":404,"bytes":1569},"home":{"status":"ok","http":200,"url":"https://www.google.com/","html_tokens":21317,"markdown_tokens":30,"pruned_tokens":30}},
{"rank":9559,"domain":"ampblogs.com","origin":"https://ampblogs.com","robots":{"status":"ok","http":200,"url":"https://ampblogs.com/robots.txt","bytes":72,"groups":2},"bots":{"GPTBot":["partial","*",1,0],"ChatGPT-User":["partial","*",1,0],"OAI-SearchBot":["partial","*",1,0],"ClaudeBot":["partial","*",1,0],"Claude-User":["partial","*",1,0],"Claude-SearchBot":["partial","*",1,0],"anthropic-ai":["partial","*",1,0],"PerplexityBot":["partial","*",1,0],"Perplexity-User":["partial","*",1,0],"Google-Extended":["partial","*",1,0],"Applebot-Extended":["partial","*",1,0],"CCBot":["partial","*",1,0],"Bytespider":["partial","*",1,0],"Amazonbot":["partial","*",1,0],"meta-externalagent":["partial","*",1,0],"cohere-ai":["partial","*",1,0],"Diffbot":["partial","*",1,0],"ContextFetchBot":["partial","*",1,0]},"llms":{"status":"missing","http":404,"bytes":529},"home":{"status":"ok","http":200,"url":"https://ampblogs.com/","html_tokens":1878,"markdown_tokens":48,"pruned_tokens":48}},
{"rank":9560,"domain":"windfinder.com","origin":"https://windfinder.com","robots":{"status":"ok","http":200,"url":"https://www.windfinder.com/robots.txt","bytes":1528,"groups":7},"bots":{"GPTBot":["partial","*",42,0],"ChatGPT-User":["partial","*",42,0],"OAI-SearchBot":["partial","*",42,0],"ClaudeBot":["partial","*",42,0],"Claude-User":["partial","*",42,0],"Claude-SearchBot":["partial","*",42,0],"anthropic-ai":["partial","*",42,0],"PerplexityBot":["partial","*",42,0],"Perplexity-User":["partial","*",42,0],"Google-Extended":["partial","*",42,0],"Applebot-Extended":["partial","*",42,0],"CCBot":["partial","*",42,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["partial","*",42,0],"cohere-ai":["partial","*",42,0],"Diffbot":["partial","*",42,0],"ContextFetchBot":["partial","*",42,0]},"llms":{"status":"present","http":200,"bytes":5408},"home":{"status":"ok","http":200,"url":"https://www.windfinder.com/","html_tokens":13068,"markdown_tokens":27,"pruned_tokens":27}},
{"rank":9561,"domain":"ejsino.net","origin":"https://ejsino.net","robots":{"status":"missing","http":404,"url":"https://ejsino.net/robots.txt","bytes":578,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":576},"home":{"status":"ok","http":200,"url":"https://ejsino.net/","html_tokens":139,"markdown_tokens":56,"pruned_tokens":56}},
{"rank":9562,"domain":"ca369.cn","origin":"https://www.ca369.cn","robots":{"status":"html","http":200,"url":"https://www.ca369.cn/robots.txt","bytes":16527,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":9563,"domain":"to.it","origin":"https://to.it","robots":{"status":"unreachable","http":null,"url":"https://to.it/robots.txt","bytes":0,"groups":0,"error":"cannot resolve to.it"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":9564,"domain":"bgu.ac.il","origin":"https://bgu.ac.il","robots":{"status":"ok","http":200,"url":"https://www.bgu.ac.il//robots.txt","bytes":192,"groups":1},"bots":{"GPTBot":["partial","*",5,1],"ChatGPT-User":["partial","*",5,1],"OAI-SearchBot":["partial","*",5,1],"ClaudeBot":["partial","*",5,1],"Claude-User":["partial","*",5,1],"Claude-SearchBot":["partial","*",5,1],"anthropic-ai":["partial","*",5,1],"PerplexityBot":["partial","*",5,1],"Perplexity-User":["partial","*",5,1],"Google-Extended":["partial","*",5,1],"Applebot-Extended":["partial","*",5,1],"CCBot":["partial","*",5,1],"Bytespider":["partial","*",5,1],"Amazonbot":["partial","*",5,1],"meta-externalagent":["partial","*",5,1],"cohere-ai":["partial","*",5,1],"Diffbot":["partial","*",5,1],"ContextFetchBot":["partial","*",5,1]},"llms":{"status":"missing","http":404,"bytes":0},"home":{"status":"ok","http":200,"url":"https://www.bgu.ac.il","html_tokens":637,"markdown_tokens":8,"pruned_tokens":8}},
{"rank":9565,"domain":"everything2.com","origin":"https://everything2.com","robots":{"status":"ok","http":200,"url":"https://everything2.com/robots.txt","bytes":806,"groups":12},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["partial","*",5,4],"OAI-SearchBot":["partial","*",5,4],"ClaudeBot":["partial","*",5,4],"Claude-User":["partial","*",5,4],"Claude-SearchBot":["partial","*",5,4],"anthropic-ai":["partial","*",5,4],"PerplexityBot":["partial","*",5,4],"Perplexity-User":["partial","*",5,4],"Google-Extended":["partial","*",5,4],"Applebot-Extended":["partial","*",5,4],"CCBot":["partial","*",5,4],"Bytespider":["partial","*",5,4],"Amazonbot":["partial","*",5,4],"meta-externalagent":["partial","*",5,4],"cohere-ai":["partial","*",5,4],"Diffbot":["partial","*",5,4],"ContextFetchBot":["partial","*",5,4]},"llms":{"status":"html","http":200,"bytes":28748},"home":{"status":"ok","http":200,"url":"https://everything2.com/","html_tokens":7495,"markdown_tokens":0,"pruned_tokens":0}},
{"rank":9566,"domain":"h-net.org","origin":"https://h-net.org","robots":{"status":"ok","http":200,"url":"https://www.h-net.org/robots.txt","bytes":353,"groups":9},"bots":{"GPTBot":["partial","*",2,0],"ChatGPT-User":["partial","*",2,0],"OAI-SearchBot":["partial","*",2,0],"ClaudeBot":["partial","*",2,0],"Claude-User":["partial","*",2,0],"Claude-SearchBot":["partial","*",2,0],"anthropic-ai":["partial","*",2,0],"PerplexityBot":["partial","*",2,0],"Perplexity-User":["partial","*",2,0],"Google-Extended":["partial","*",2,0],"Applebot-Extended":["partial","*",2,0],"CCBot":["partial","*",2,0],"Bytespider":["partial","*",2,0],"Amazonbot":["partial","*",2,0],"meta-externalagent":["partial","*",2,0],"cohere-ai":["partial","*",2,0],"Diffbot":["partial","*",2,0],"ContextFetchBot":["partial","*",2,0]},"llms":{"status":"missing","http":404,"bytes":236},"home":{"status":"ok","http":200,"url":"https://networks.h-net.org/","html_tokens":11462,"markdown_tokens":74,"pruned_tokens":53}},
{"rank":9567,"domain":"pingan.com","origin":"https://pingan.com","robots":{"status":"ok","http":200,"url":"https://pingan.com/robots.txt","bytes":621,"groups":1},"bots":{"GPTBot":["partial","*",14,0],"ChatGPT-User":["partial","*",14,0],"OAI-SearchBot":["partial","*",14,0],"ClaudeBot":["partial","*",14,0],"Claude-User":["partial","*",14,0],"Claude-SearchBot":["partial","*",14,0],"anthropic-ai":["partial","*",14,0],"PerplexityBot":["partial","*",14,0],"Perplexity-User":["partial","*",14,0],"Google-Extended":["partial","*",14,0],"Applebot-Extended":["partial","*",14,0],"CCBot":["partial","*",14,0],"Bytespider":["partial","*",14,0],"Amazonbot":["partial","*",14,0],"meta-externalagent":["partial","*",14,0],"cohere-ai":["partial","*",14,0],"Diffbot":["partial","*",14,0],"ContextFetchBot":["partial","*",14,0]},"llms":{"status":"other_text","http":200,"bytes":6101},"home":{"status":"ok","http":200,"url":"https://www.pingan.com/","html_tokens":33173,"markdown_tokens":13,"pruned_tokens":13}},
{"rank":9568,"domain":"aas.org","origin":"https://aas.org","robots":{"status":"ok","http":200,"url":"https://aas.org/robots.txt","bytes":1706,"groups":1},"bots":{"GPTBot":["partial","*",26,18],"ChatGPT-User":["partial","*",26,18],"OAI-SearchBot":["partial","*",26,18],"ClaudeBot":["partial","*",26,18],"Claude-User":["partial","*",26,18],"Claude-SearchBot":["partial","*",26,18],"anthropic-ai":["partial","*",26,18],"PerplexityBot":["partial","*",26,18],"Perplexity-User":["partial","*",26,18],"Google-Extended":["partial","*",26,18],"Applebot-Extended":["partial","*",26,18],"CCBot":["partial","*",26,18],"Bytespider":["partial","*",26,18],"Amazonbot":["partial","*",26,18],"meta-externalagent":["partial","*",26,18],"cohere-ai":["partial","*",26,18],"Diffbot":["partial","*",26,18],"ContextFetchBot":["partial","*",26,18]},"llms":{"status":"http_error","http":403,"bytes":5773},"home":{"status":"http_error","http":403,"url":"https://aas.org/"}},
{"rank":9569,"domain":"mot.gov.cn","origin":"https://www.mot.gov.cn","robots":{"status":"missing","http":404,"url":"https://www.mot.gov.cn/robots.txt","bytes":287,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":287},"home":{"status":"ok","http":200,"url":"https://www.mot.gov.cn/","html_tokens":21430,"markdown_tokens":78,"pruned_tokens":78}},
{"rank":9570,"domain":"nrao.edu","origin":"https://nrao.edu","robots":{"status":"ok","http":200,"url":"https://www.nrao.edu/robots.txt","bytes":356,"groups":2},"bots":{"GPTBot":["partial","*",9,0],"ChatGPT-User":["partial","*",9,0],"OAI-SearchBot":["partial","*",9,0],"ClaudeBot":["partial","*",9,0],"Claude-User":["partial","*",9,0],"Claude-SearchBot":["partial","*",9,0],"anthropic-ai":["partial","*",9,0],"PerplexityBot":["partial","*",9,0],"Perplexity-User":["partial","*",9,0],"Google-Extended":["partial","*",9,0],"Applebot-Extended":["partial","*",9,0],"CCBot":["partial","*",9,0],"Bytespider":["partial","*",9,0],"Amazonbot":["partial","*",9,0],"meta-externalagent":["partial","*",9,0],"cohere-ai":["partial","*",9,0],"Diffbot":["partial","*",9,0],"ContextFetchBot":["partial","*",9,0]},"llms":{"status":"missing","http":404,"bytes":3114},"home":{"status":"ok","http":200,"url":"https://public.nrao.edu","html_tokens":24573,"markdown_tokens":628,"pruned_tokens":341}},
{"rank":9571,"domain":"remax.com","origin":"https://remax.com","robots":{"status":"ok","http":200,"url":"https://www.remax.com/robots.txt","bytes":1584,"groups":3},"bots":{"GPTBot":["partial","named",7,1],"ChatGPT-User":["partial","named",7,1],"OAI-SearchBot":["partial","named",7,1],"ClaudeBot":["partial","named",7,1],"Claude-User":["partial","named",7,1],"Claude-SearchBot":["partial","named",7,1],"anthropic-ai":["partial","*",7,1],"PerplexityBot":["partial","named",7,1],"Perplexity-User":["partial","named",7,1],"Google-Extended":["partial","named",7,1],"Applebot-Extended":["partial","named",7,1],"CCBot":["partial","named",7,1],"Bytespider":["partial","*",7,1],"Amazonbot":["partial","*",7,1],"meta-externalagent":["partial","*",7,1],"cohere-ai":["partial","named",7,1],"Diffbot":["partial","*",7,1],"ContextFetchBot":["partial","*",7,1]},"llms":{"status":"http_error","http":403,"bytes":919},"home":{"status":"http_error","http":403,"url":"https://www.remax.com/"}},
{"rank":9572,"domain":"maharashtra.gov.in","origin":"https://maharashtra.gov.in","robots":{"status":"ok","http":200,"url":"https://maharashtra.gov.in/robots.txt","bytes":23,"groups":1},"bots":{"GPTBot":["allowed","*",0,1],"ChatGPT-User":["allowed","*",0,1],"OAI-SearchBot":["allowed","*",0,1],"ClaudeBot":["allowed","*",0,1],"Claude-User":["allowed","*",0,1],"Claude-SearchBot":["allowed","*",0,1],"anthropic-ai":["allowed","*",0,1],"PerplexityBot":["allowed","*",0,1],"Perplexity-User":["allowed","*",0,1],"Google-Extended":["allowed","*",0,1],"Applebot-Extended":["allowed","*",0,1],"CCBot":["allowed","*",0,1],"Bytespider":["allowed","*",0,1],"Amazonbot":["allowed","*",0,1],"meta-externalagent":["allowed","*",0,1],"cohere-ai":["allowed","*",0,1],"Diffbot":["allowed","*",0,1],"ContextFetchBot":["allowed","*",0,1]},"llms":{"status":"missing","http":404,"bytes":0},"home":{"status":"ok","http":200,"url":"https://maharashtra.gov.in/","html_tokens":106256,"markdown_tokens":56169,"pruned_tokens":157}},
{"rank":9573,"domain":"broadway.com","origin":"https://broadway.com","robots":{"status":"ok","http":200,"url":"https://www.broadway.com/robots.txt","bytes":107,"groups":1},"bots":{"GPTBot":["partial","*",2,1],"ChatGPT-User":["partial","*",2,1],"OAI-SearchBot":["partial","*",2,1],"ClaudeBot":["partial","*",2,1],"Claude-User":["partial","*",2,1],"Claude-SearchBot":["partial","*",2,1],"anthropic-ai":["partial","*",2,1],"PerplexityBot":["partial","*",2,1],"Perplexity-User":["partial","*",2,1],"Google-Extended":["partial","*",2,1],"Applebot-Extended":["partial","*",2,1],"CCBot":["partial","*",2,1],"Bytespider":["partial","*",2,1],"Amazonbot":["partial","*",2,1],"meta-externalagent":["partial","*",2,1],"cohere-ai":["partial","*",2,1],"Diffbot":["partial","*",2,1],"ContextFetchBot":["partial","*",2,1]},"llms":{"status":"missing","http":404,"bytes":61046},"home":{"status":"ok","http":200,"url":"https://www.broadway.com/","html_tokens":116828,"markdown_tokens":52,"pruned_tokens":52}},
{"rank":9574,"domain":"stylecaster.com","origin":"https://stylecaster.com","robots":{"status":"ok","http":200,"url":"https://stylecaster.com/robots.txt","bytes":1217,"groups":25},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["blocked","named",1,0],"OAI-SearchBot":["blocked","named",1,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",9,1],"Claude-SearchBot":["partial","*",9,1],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["partial","*",9,1],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["partial","*",9,1],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",9,1]},"llms":{"status":"missing","http":404,"bytes":310955},"home":{"status":"ok","http":200,"url":"https://stylecaster.com/","html_tokens":114505,"markdown_tokens":153,"pruned_tokens":84}},
{"rank":9575,"domain":"amadeus.com","origin":"https://amadeus.com","robots":{"status":"ok","http":200,"url":"https://amadeus.com/robots.txt","bytes":64,"groups":1},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"present","http":200,"bytes":10765},"home":{"status":"ok","http":200,"url":"https://amadeus.com/","html_tokens":260,"markdown_tokens":21,"pruned_tokens":21}},
{"rank":9576,"domain":"mysite.com","origin":"http://mysite.com","robots":{"status":"ok","http":200,"url":"http://mysite.com/robots.txt","bytes":125,"groups":1},"bots":{"GPTBot":["partial","*",2,0],"ChatGPT-User":["partial","*",2,0],"OAI-SearchBot":["partial","*",2,0],"ClaudeBot":["partial","*",2,0],"Claude-User":["partial","*",2,0],"Claude-SearchBot":["partial","*",2,0],"anthropic-ai":["partial","*",2,0],"PerplexityBot":["partial","*",2,0],"Perplexity-User":["partial","*",2,0],"Google-Extended":["partial","*",2,0],"Applebot-Extended":["partial","*",2,0],"CCBot":["partial","*",2,0],"Bytespider":["partial","*",2,0],"Amazonbot":["partial","*",2,0],"meta-externalagent":["partial","*",2,0],"cohere-ai":["partial","*",2,0],"Diffbot":["partial","*",2,0],"ContextFetchBot":["partial","*",2,0]},"llms":{"status":"missing","http":404,"bytes":11033},"home":{"status":"ok","http":200,"url":"http://mysite.com/","html_tokens":5007,"markdown_tokens":224,"pruned_tokens":81}},
{"rank":9577,"domain":"gbnews.com","origin":"https://gbnews.com","robots":{"status":"ok","http":200,"url":"https://www.gbnews.com/robots.txt","bytes":1475,"groups":2},"bots":{"GPTBot":["partial","*",20,1],"ChatGPT-User":["partial","*",20,1],"OAI-SearchBot":["partial","*",20,1],"ClaudeBot":["partial","*",20,1],"Claude-User":["partial","*",20,1],"Claude-SearchBot":["partial","*",20,1],"anthropic-ai":["partial","*",20,1],"PerplexityBot":["partial","*",20,1],"Perplexity-User":["partial","*",20,1],"Google-Extended":["partial","*",20,1],"Applebot-Extended":["partial","*",20,1],"CCBot":["partial","*",20,1],"Bytespider":["partial","*",20,1],"Amazonbot":["partial","*",20,1],"meta-externalagent":["partial","*",20,1],"cohere-ai":["partial","*",20,1],"Diffbot":["partial","*",20,1],"ContextFetchBot":["partial","*",20,1]},"llms":{"status":"present","http":200,"bytes":120449},"home":{"status":"ok","http":200,"url":"https://www.gbnews.com/","html_tokens":205873,"markdown_tokens":113,"pruned_tokens":113}},
{"rank":9578,"domain":"sport1.de","origin":"https://sport1.de","robots":{"status":"ok","http":200,"url":"https://www.sport1.de/robots.txt","bytes":462,"groups":1},"bots":{"GPTBot":["partial","*",12,1],"ChatGPT-User":["partial","*",12,1],"OAI-SearchBot":["partial","*",12,1],"ClaudeBot":["partial","*",12,1],"Claude-User":["partial","*",12,1],"Claude-SearchBot":["partial","*",12,1],"anthropic-ai":["partial","*",12,1],"PerplexityBot":["partial","*",12,1],"Perplexity-User":["partial","*",12,1],"Google-Extended":["partial","*",12,1],"Applebot-Extended":["partial","*",12,1],"CCBot":["partial","*",12,1],"Bytespider":["partial","*",12,1],"Amazonbot":["partial","*",12,1],"meta-externalagent":["partial","*",12,1],"cohere-ai":["partial","*",12,1],"Diffbot":["partial","*",12,1],"ContextFetchBot":["partial","*",12,1]},"llms":{"status":"missing","http":404,"bytes":399445},"home":{"status":"ok","http":200,"url":"https://www.sport1.de/","html_tokens":156054,"markdown_tokens":20,"pruned_tokens":20}},
{"rank":9579,"domain":"hanselman.com","origin":"https://hanselman.com","robots":{"status":"ok","http":200,"url":"https://www.hanselman.com/robots.txt","bytes":258,"groups":1},"bots":{"GPTBot":["partial","*",5,0],"ChatGPT-User":["partial","*",5,0],"OAI-SearchBot":["partial","*",5,0],"ClaudeBot":["partial","*",5,0],"Claude-User":["partial","*",5,0],"Claude-SearchBot":["partial","*",5,0],"anthropic-ai":["partial","*",5,0],"PerplexityBot":["partial","*",5,0],"Perplexity-User":["partial","*",5,0],"Google-Extended":["partial","*",5,0],"Applebot-Extended":["partial","*",5,0],"CCBot":["partial","*",5,0],"Bytespider":["partial","*",5,0],"Amazonbot":["partial","*",5,0],"meta-externalagent":["partial","*",5,0],"cohere-ai":["partial","*",5,0],"Diffbot":["partial","*",5,0],"ContextFetchBot":["partial","*",5,0]},"llms":{"status":"missing","http":404,"bytes":0},"home":{"status":"ok","http":200,"url":"https://www.hanselman.com/","html_tokens":4153,"markdown_tokens":250,"pruned_tokens":83}},
{"rank":9580,"domain":"princehotels.co.jp","origin":"https://princehotels.co.jp","robots":{"status":"ok","http":200,"url":"https://www.princehotels.co.jp/robots.txt","bytes":50,"groups":1},"bots":{"GPTBot":["partial","*",1,0],"ChatGPT-User":["partial","*",1,0],"OAI-SearchBot":["partial","*",1,0],"ClaudeBot":["partial","*",1,0],"Claude-User":["partial","*",1,0],"Claude-SearchBot":["partial","*",1,0],"anthropic-ai":["partial","*",1,0],"PerplexityBot":["partial","*",1,0],"Perplexity-User":["partial","*",1,0],"Google-Extended":["partial","*",1,0],"Applebot-Extended":["partial","*",1,0],"CCBot":["partial","*",1,0],"Bytespider":["partial","*",1,0],"Amazonbot":["partial","*",1,0],"meta-externalagent":["partial","*",1,0],"cohere-ai":["partial","*",1,0],"Diffbot":["partial","*",1,0],"ContextFetchBot":["partial","*",1,0]},"llms":{"status":"missing","http":404,"bytes":36190},"home":{"status":"ok","http":200,"url":"https://www.princehotels.co.jp/","html_tokens":31981,"markdown_tokens":47,"pruned_tokens":47}},
{"rank":9581,"domain":"sfstandard.com","origin":"https://sfstandard.com","robots":{"status":"ok","http":200,"url":"https://sfstandard.com/robots.txt","bytes":3742,"groups":94},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["partial","*",5,0],"OAI-SearchBot":["partial","*",5,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",5,0],"Claude-SearchBot":["partial","*",5,0],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["partial","*",5,0],"Google-Extended":["partial","*",5,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",2,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",5,0]},"llms":{"status":"missing","http":404,"bytes":135263},"home":{"status":"ok","http":200,"url":"https://sfstandard.com/","html_tokens":219992,"markdown_tokens":200,"pruned_tokens":33}},
{"rank":9582,"domain":"bookriot.com","origin":"https://bookriot.com","robots":{"status":"ok","http":200,"url":"https://bookriot.com/robots.txt","bytes":214,"groups":1},"bots":{"GPTBot":["partial","*",6,0],"ChatGPT-User":["partial","*",6,0],"OAI-SearchBot":["partial","*",6,0],"ClaudeBot":["partial","*",6,0],"Claude-User":["partial","*",6,0],"Claude-SearchBot":["partial","*",6,0],"anthropic-ai":["partial","*",6,0],"PerplexityBot":["partial","*",6,0],"Perplexity-User":["partial","*",6,0],"Google-Extended":["partial","*",6,0],"Applebot-Extended":["partial","*",6,0],"CCBot":["partial","*",6,0],"Bytespider":["partial","*",6,0],"Amazonbot":["partial","*",6,0],"meta-externalagent":["partial","*",6,0],"cohere-ai":["partial","*",6,0],"Diffbot":["partial","*",6,0],"ContextFetchBot":["partial","*",6,0]},"llms":{"status":"missing","http":404,"bytes":108680},"home":{"status":"ok","http":200,"url":"https://bookriot.com/","html_tokens":51696,"markdown_tokens":101,"pruned_tokens":101}},
{"rank":9583,"domain":"mega-moolah-play.com","origin":"https://mega-moolah-play.com","robots":{"status":"ok","http":200,"url":"https://mega-moolah-play.com/robots.txt","bytes":544,"groups":1},"bots":{"GPTBot":["partial","*",19,0],"ChatGPT-User":["partial","*",19,0],"OAI-SearchBot":["partial","*",19,0],"ClaudeBot":["partial","*",19,0],"Claude-User":["partial","*",19,0],"Claude-SearchBot":["partial","*",19,0],"anthropic-ai":["partial","*",19,0],"PerplexityBot":["partial","*",19,0],"Perplexity-User":["partial","*",19,0],"Google-Extended":["partial","*",19,0],"Applebot-Extended":["partial","*",19,0],"CCBot":["partial","*",19,0],"Bytespider":["partial","*",19,0],"Amazonbot":["partial","*",19,0],"meta-externalagent":["partial","*",19,0],"cohere-ai":["partial","*",19,0],"Diffbot":["partial","*",19,0],"ContextFetchBot":["partial","*",19,0]},"llms":{"status":"missing","http":404,"bytes":75560},"home":{"status":"ok","http":200,"url":"https://mega-moolah-play.com/","html_tokens":30767,"markdown_tokens":2957,"pruned_tokens":702}},
{"rank":9584,"domain":"ticketspice.com","origin":"https://ticketspice.com","robots":{"status":"ok","http":200,"url":"https://www.ticketspice.com/robots.txt","bytes":117,"groups":2},"bots":{"GPTBot":["allowed","*",0,1],"ChatGPT-User":["allowed","*",0,1],"OAI-SearchBot":["allowed","*",0,1],"ClaudeBot":["allowed","*",0,1],"Claude-User":["allowed","*",0,1],"Claude-SearchBot":["allowed","*",0,1],"anthropic-ai":["allowed","*",0,1],"PerplexityBot":["allowed","*",0,1],"Perplexity-User":["allowed","*",0,1],"Google-Extended":["allowed","*",0,1],"Applebot-Extended":["allowed","*",0,1],"CCBot":["allowed","*",0,1],"Bytespider":["allowed","*",0,1],"Amazonbot":["allowed","*",0,1],"meta-externalagent":["allowed","*",0,1],"cohere-ai":["allowed","*",0,1],"Diffbot":["allowed","*",0,1],"ContextFetchBot":["allowed","*",0,1]},"llms":{"status":"present","http":200,"bytes":9493},"home":{"status":"ok","http":200,"url":"https://www.ticketspice.com/","html_tokens":127752,"markdown_tokens":2247,"pruned_tokens":632}},
{"rank":9585,"domain":"tmweb.ru","origin":"https://tmweb.ru","robots":{"status":"html","http":200,"url":"https://timeweb.com/","bytes":504954,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":9586,"domain":"thanhnien.vn","origin":"https://thanhnien.vn","robots":{"status":"ok","http":200,"url":"https://thanhnien.vn/robots.txt","bytes":165,"groups":1},"bots":{"GPTBot":["partial","*",2,1],"ChatGPT-User":["partial","*",2,1],"OAI-SearchBot":["partial","*",2,1],"ClaudeBot":["partial","*",2,1],"Claude-User":["partial","*",2,1],"Claude-SearchBot":["partial","*",2,1],"anthropic-ai":["partial","*",2,1],"PerplexityBot":["partial","*",2,1],"Perplexity-User":["partial","*",2,1],"Google-Extended":["partial","*",2,1],"Applebot-Extended":["partial","*",2,1],"CCBot":["partial","*",2,1],"Bytespider":["partial","*",2,1],"Amazonbot":["partial","*",2,1],"meta-externalagent":["partial","*",2,1],"cohere-ai":["partial","*",2,1],"Diffbot":["partial","*",2,1],"ContextFetchBot":["partial","*",2,1]},"llms":{"status":"present","http":200,"bytes":2483},"home":{"status":"ok","http":200,"url":"https://thanhnien.vn/","html_tokens":66923,"markdown_tokens":60,"pruned_tokens":60}},
{"rank":9587,"domain":"wftv.com","origin":"https://wftv.com","robots":{"status":"ok","http":200,"url":"https://www.wftv.com/robots.txt","bytes":242,"groups":2},"bots":{"GPTBot":["partial","*",1,0],"ChatGPT-User":["partial","*",1,0],"OAI-SearchBot":["partial","*",1,0],"ClaudeBot":["partial","*",1,0],"Claude-User":["partial","*",1,0],"Claude-SearchBot":["partial","*",1,0],"anthropic-ai":["partial","*",1,0],"PerplexityBot":["partial","*",1,0],"Perplexity-User":["partial","*",1,0],"Google-Extended":["partial","*",1,0],"Applebot-Extended":["partial","*",1,0],"CCBot":["partial","*",1,0],"Bytespider":["partial","*",1,0],"Amazonbot":["partial","*",1,0],"meta-externalagent":["partial","*",1,0],"cohere-ai":["partial","*",1,0],"Diffbot":["partial","*",1,0],"ContextFetchBot":["partial","*",1,0]},"llms":{"status":"missing","http":404,"bytes":328030},"home":{"status":"ok","http":200,"url":"https://www.wftv.com/","html_tokens":103783,"markdown_tokens":30,"pruned_tokens":30}},
{"rank":9588,"domain":"bunny.net","origin":"https://bunny.net","robots":{"status":"ok","http":200,"url":"https://bunny.net/robots.txt","bytes":86,"groups":1},"bots":{"GPTBot":["allowed","*",0,1],"ChatGPT-User":["allowed","*",0,1],"OAI-SearchBot":["allowed","*",0,1],"ClaudeBot":["allowed","*",0,1],"Claude-User":["allowed","*",0,1],"Claude-SearchBot":["allowed","*",0,1],"anthropic-ai":["allowed","*",0,1],"PerplexityBot":["allowed","*",0,1],"Perplexity-User":["allowed","*",0,1],"Google-Extended":["allowed","*",0,1],"Applebot-Extended":["allowed","*",0,1],"CCBot":["allowed","*",0,1],"Bytespider":["allowed","*",0,1],"Amazonbot":["allowed","*",0,1],"meta-externalagent":["allowed","*",0,1],"cohere-ai":["allowed","*",0,1],"Diffbot":["allowed","*",0,1],"ContextFetchBot":["allowed","*",0,1]},"llms":{"status":"present","http":200,"bytes":3365},"home":{"status":"not_html","error":"unsupported content type text/markdown"}},
{"rank":9589,"domain":"goodyear.com","origin":"https://goodyear.com","robots":{"status":"ok","http":200,"url":"https://www.goodyear.com/robots.txt","bytes":746,"groups":9},"bots":{"GPTBot":["partial","*",16,0],"ChatGPT-User":["partial","*",16,0],"OAI-SearchBot":["partial","*",16,0],"ClaudeBot":["partial","*",16,0],"Claude-User":["partial","*",16,0],"Claude-SearchBot":["partial","*",16,0],"anthropic-ai":["partial","*",16,0],"PerplexityBot":["partial","*",16,0],"Perplexity-User":["partial","*",16,0],"Google-Extended":["partial","*",16,0],"Applebot-Extended":["partial","*",16,0],"CCBot":["partial","*",16,0],"Bytespider":["partial","*",16,0],"Amazonbot":["partial","*",16,0],"meta-externalagent":["partial","*",16,0],"cohere-ai":["partial","*",16,0],"Diffbot":["partial","*",16,0],"ContextFetchBot":["partial","*",16,0]},"llms":{"status":"http_error","http":500,"bytes":188299},"home":{"status":"ok","http":200,"url":"https://www.goodyear.com/en-us","html_tokens":147625,"markdown_tokens":28,"pruned_tokens":28}},
{"rank":9590,"domain":"minnpost.com","origin":"https://minnpost.com","robots":{"status":"ok","http":200,"url":"https://www.minnpost.com/robots.txt","bytes":523,"groups":14},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["blocked","named",1,0],"OAI-SearchBot":["blocked","named",1,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["blocked","named",1,0],"Claude-SearchBot":["blocked","named",1,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":179806},"home":{"status":"ok","http":200,"url":"https://www.minnpost.com/","html_tokens":87748,"markdown_tokens":213,"pruned_tokens":41}},
{"rank":9591,"domain":"lds.org","origin":"https://lds.org","robots":{"status":"ok","http":200,"url":"https://www.churchofjesuschrist.org/robots.txt","bytes":15329,"groups":1},"bots":{"GPTBot":["partial","*",15,0],"ChatGPT-User":["partial","*",15,0],"OAI-SearchBot":["partial","*",15,0],"ClaudeBot":["partial","*",15,0],"Claude-User":["partial","*",15,0],"Claude-SearchBot":["partial","*",15,0],"anthropic-ai":["partial","*",15,0],"PerplexityBot":["partial","*",15,0],"Perplexity-User":["partial","*",15,0],"Google-Extended":["partial","*",15,0],"Applebot-Extended":["partial","*",15,0],"CCBot":["partial","*",15,0],"Bytespider":["partial","*",15,0],"Amazonbot":["partial","*",15,0],"meta-externalagent":["partial","*",15,0],"cohere-ai":["partial","*",15,0],"Diffbot":["partial","*",15,0],"ContextFetchBot":["partial","*",15,0]},"llms":{"status":"http_error","http":503,"bytes":221},"home":{"status":"ok","http":200,"url":"https://www.churchofjesuschrist.org/?lang=eng","html_tokens":20544,"markdown_tokens":1133,"pruned_tokens":132}},
{"rank":9592,"domain":"engineeringtoolbox.com","origin":"https://engineeringtoolbox.com","robots":{"status":"missing","http":404,"url":"https://www.engineeringtoolbox.com/robots.txt","bytes":27150,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":27150},"home":{"status":"ok","http":200,"url":"https://www.engineeringtoolbox.com/","html_tokens":14738,"markdown_tokens":1033,"pruned_tokens":152}},
{"rank":9593,"domain":"app.br","origin":"https://app.br","robots":{"status":"unreachable","http":null,"url":"https://app.br/robots.txt","bytes":0,"groups":0,"error":"cannot resolve app.br"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":9594,"domain":"selfridges.com","origin":"https://selfridges.com","robots":{"status":"refused","http":403,"url":"https://selfridges.com/robots.txt","bytes":4547,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":9595,"domain":"bestcolleges.com","origin":"https://bestcolleges.com","robots":{"status":"ok","http":200,"url":"https://www.bestcolleges.com/robots.txt","bytes":130,"groups":1},"bots":{"GPTBot":["partial","*",3,0],"ChatGPT-User":["partial","*",3,0],"OAI-SearchBot":["partial","*",3,0],"ClaudeBot":["partial","*",3,0],"Claude-User":["partial","*",3,0],"Claude-SearchBot":["partial","*",3,0],"anthropic-ai":["partial","*",3,0],"PerplexityBot":["partial","*",3,0],"Perplexity-User":["partial","*",3,0],"Google-Extended":["partial","*",3,0],"Applebot-Extended":["partial","*",3,0],"CCBot":["partial","*",3,0],"Bytespider":["partial","*",3,0],"Amazonbot":["partial","*",3,0],"meta-externalagent":["partial","*",3,0],"cohere-ai":["partial","*",3,0],"Diffbot":["partial","*",3,0],"ContextFetchBot":["partial","*",3,0]},"llms":{"status":"missing","http":404,"bytes":97227},"home":{"status":"ok","http":200,"url":"https://www.bestcolleges.com/","html_tokens":127500,"markdown_tokens":223,"pruned_tokens":104}},
{"rank":9596,"domain":"fidoalliance.org","origin":"https://fidoalliance.org","robots":{"status":"ok","http":200,"url":"https://fidoalliance.org/robots.txt","bytes":135,"groups":1},"bots":{"GPTBot":["partial","*",1,1],"ChatGPT-User":["partial","*",1,1],"OAI-SearchBot":["partial","*",1,1],"ClaudeBot":["partial","*",1,1],"Claude-User":["partial","*",1,1],"Claude-SearchBot":["partial","*",1,1],"anthropic-ai":["partial","*",1,1],"PerplexityBot":["partial","*",1,1],"Perplexity-User":["partial","*",1,1],"Google-Extended":["partial","*",1,1],"Applebot-Extended":["partial","*",1,1],"CCBot":["partial","*",1,1],"Bytespider":["partial","*",1,1],"Amazonbot":["partial","*",1,1],"meta-externalagent":["partial","*",1,1],"cohere-ai":["partial","*",1,1],"Diffbot":["partial","*",1,1],"ContextFetchBot":["partial","*",1,1]},"llms":{"status":"http_error","http":403,"bytes":5569},"home":{"status":"http_error","http":403,"url":"https://fidoalliance.org/"}},
{"rank":9597,"domain":"jvc.com","origin":"https://jvc.com","robots":{"status":"ok","http":200,"url":"https://www.jvc.com/robots.txt","bytes":2584,"groups":13},"bots":{"GPTBot":["allowed","named",0,1],"ChatGPT-User":["allowed","named",0,1],"OAI-SearchBot":["partial","*",5,0],"ClaudeBot":["allowed","named",0,1],"Claude-User":["partial","*",5,0],"Claude-SearchBot":["partial","*",5,0],"anthropic-ai":["allowed","named",0,1],"PerplexityBot":["allowed","named",0,1],"Perplexity-User":["partial","*",5,0],"Google-Extended":["allowed","named",0,1],"Applebot-Extended":["allowed","named",0,1],"CCBot":["allowed","named",0,1],"Bytespider":["allowed","named",0,1],"Amazonbot":["allowed","named",0,1],"meta-externalagent":["partial","*",5,0],"cohere-ai":["partial","*",5,0],"Diffbot":["partial","*",5,0],"ContextFetchBot":["partial","*",5,0]},"llms":{"status":"present","http":200,"bytes":988},"home":{"status":"ok","http":200,"url":"https://www.jvc.com/","html_tokens":3338,"markdown_tokens":18,"pruned_tokens":18}},
{"rank":9598,"domain":"tribuna.com","origin":"https://tribuna.com","robots":{"status":"ok","http":200,"url":"https://tribuna.com/robots.txt","bytes":4629,"groups":1},"bots":{"GPTBot":["partial","*",25,1],"ChatGPT-User":["partial","*",25,1],"OAI-SearchBot":["partial","*",25,1],"ClaudeBot":["partial","*",25,1],"Claude-User":["partial","*",25,1],"Claude-SearchBot":["partial","*",25,1],"anthropic-ai":["partial","*",25,1],"PerplexityBot":["partial","*",25,1],"Perplexity-User":["partial","*",25,1],"Google-Extended":["partial","*",25,1],"Applebot-Extended":["partial","*",25,1],"CCBot":["partial","*",25,1],"Bytespider":["partial","*",25,1],"Amazonbot":["partial","*",25,1],"meta-externalagent":["partial","*",25,1],"cohere-ai":["partial","*",25,1],"Diffbot":["partial","*",25,1],"ContextFetchBot":["partial","*",25,1]},"llms":{"status":"missing","http":404,"bytes":1542391},"home":{"status":"http_error","http":403,"url":"https://tribuna.com/"}},
{"rank":9599,"domain":"dayoo.com","origin":"https://dayoo.com","robots":{"status":"ok","http":200,"url":"https://www.dayoo.com/robots.txt","bytes":176,"groups":1},"bots":{"GPTBot":["partial","*",10,0],"ChatGPT-User":["partial","*",10,0],"OAI-SearchBot":["partial","*",10,0],"ClaudeBot":["partial","*",10,0],"Claude-User":["partial","*",10,0],"Claude-SearchBot":["partial","*",10,0],"anthropic-ai":["partial","*",10,0],"PerplexityBot":["partial","*",10,0],"Perplexity-User":["partial","*",10,0],"Google-Extended":["partial","*",10,0],"Applebot-Extended":["partial","*",10,0],"CCBot":["partial","*",10,0],"Bytespider":["partial","*",10,0],"Amazonbot":["partial","*",10,0],"meta-externalagent":["partial","*",10,0],"cohere-ai":["partial","*",10,0],"Diffbot":["partial","*",10,0],"ContextFetchBot":["partial","*",10,0]},"llms":{"status":"html","http":200,"bytes":14660},"home":{"status":"ok","http":200,"url":"https://www.dayoo.com/","html_tokens":19825,"markdown_tokens":12,"pruned_tokens":12}},
{"rank":9600,"domain":"whattoexpect.com","origin":"https://whattoexpect.com","robots":{"status":"ok","http":200,"url":"https://www.whattoexpect.com/robots.txt","bytes":225856,"groups":3},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["blocked","named",1,0],"OAI-SearchBot":["blocked","named",1,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["blocked","named",1,0],"Claude-SearchBot":["blocked","named",1,0],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["blocked","named",1,0],"Google-Extended":["partial","named",2752,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",20,2]},"llms":{"status":"missing","http":404,"bytes":341426},"home":{"status":"ok","http":200,"url":"https://www.whattoexpect.com/","html_tokens":137296,"markdown_tokens":75,"pruned_tokens":75}},
{"rank":9601,"domain":"bitbucket.io","origin":"https://bitbucket.io","robots":{"status":"ok","http":200,"url":"https://bitbucket.io/robots.txt","bytes":889,"groups":2},"bots":{"GPTBot":["partial","*",38,0],"ChatGPT-User":["partial","*",38,0],"OAI-SearchBot":["partial","*",38,0],"ClaudeBot":["partial","*",38,0],"Claude-User":["partial","*",38,0],"Claude-SearchBot":["partial","*",38,0],"anthropic-ai":["partial","*",38,0],"PerplexityBot":["partial","*",38,0],"Perplexity-User":["partial","*",38,0],"Google-Extended":["partial","*",38,0],"Applebot-Extended":["partial","*",38,0],"CCBot":["partial","*",38,0],"Bytespider":["partial","*",38,0],"Amazonbot":["partial","*",38,0],"meta-externalagent":["partial","*",38,0],"cohere-ai":["partial","*",38,0],"Diffbot":["partial","*",38,0],"ContextFetchBot":["partial","*",38,0]},"llms":{"status":"missing","http":404,"bytes":13814},"home":{"status":"ok","http":200,"url":"https://bitbucket.io/product/workspace/repositories/","html_tokens":2928,"markdown_tokens":0,"pruned_tokens":0}},
{"rank":9602,"domain":"tampabay28.com","origin":"https://tampabay28.com","robots":{"status":"ok","http":200,"url":"https://www.tampabay28.com/robots.txt","bytes":395,"groups":4},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["partial","*",5,0],"OAI-SearchBot":["partial","*",5,0],"ClaudeBot":["partial","*",5,0],"Claude-User":["partial","*",5,0],"Claude-SearchBot":["partial","*",5,0],"anthropic-ai":["partial","*",5,0],"PerplexityBot":["partial","*",5,0],"Perplexity-User":["partial","*",5,0],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["partial","*",5,0],"CCBot":["partial","*",5,0],"Bytespider":["partial","*",5,0],"Amazonbot":["partial","*",5,0],"meta-externalagent":["partial","*",5,0],"cohere-ai":["partial","*",5,0],"Diffbot":["partial","*",5,0],"ContextFetchBot":["partial","*",5,0]},"llms":{"status":"missing","http":404,"bytes":79499},"home":{"status":"ok","http":200,"url":"https://www.tampabay28.com/","html_tokens":66502,"markdown_tokens":87,"pruned_tokens":87}},
{"rank":9603,"domain":"unmissions.org","origin":"https://unmissions.org","robots":{"status":"ok","http":200,"url":"https://unmissions.org/robots.txt","bytes":2189,"groups":1},"bots":{"GPTBot":["partial","*",36,32],"ChatGPT-User":["partial","*",36,32],"OAI-SearchBot":["partial","*",36,32],"ClaudeBot":["partial","*",36,32],"Claude-User":["partial","*",36,32],"Claude-SearchBot":["partial","*",36,32],"anthropic-ai":["partial","*",36,32],"PerplexityBot":["partial","*",36,32],"Perplexity-User":["partial","*",36,32],"Google-Extended":["partial","*",36,32],"Applebot-Extended":["partial","*",36,32],"CCBot":["partial","*",36,32],"Bytespider":["partial","*",36,32],"Amazonbot":["partial","*",36,32],"meta-externalagent":["partial","*",36,32],"cohere-ai":["partial","*",36,32],"Diffbot":["partial","*",36,32],"ContextFetchBot":["partial","*",36,32]},"llms":{"status":"missing","http":404,"bytes":34818},"home":{"status":"ok","http":200,"url":"https://www.unmissions.org/en","html_tokens":82827,"markdown_tokens":252,"pruned_tokens":131}},
{"rank":9604,"domain":"guitarworld.com","origin":"https://guitarworld.com","robots":{"status":"ok","http":200,"url":"https://www.guitarworld.com/robots.txt","bytes":1722,"groups":5},"bots":{"GPTBot":["partial","*",21,0],"ChatGPT-User":["partial","*",21,0],"OAI-SearchBot":["partial","*",21,0],"ClaudeBot":["partial","*",21,0],"Claude-User":["partial","*",21,0],"Claude-SearchBot":["partial","*",21,0],"anthropic-ai":["partial","*",21,0],"PerplexityBot":["partial","*",21,0],"Perplexity-User":["partial","*",21,0],"Google-Extended":["partial","*",21,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["partial","*",21,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["partial","*",21,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",21,0]},"llms":{"status":"missing","http":404,"bytes":1130751},"home":{"status":"ok","http":200,"url":"https://www.guitarworld.com/","html_tokens":412771,"markdown_tokens":72,"pruned_tokens":72}},
{"rank":9605,"domain":"google.tg","origin":"https://google.tg","robots":{"status":"ok","http":200,"url":"https://www.google.tg/robots.txt","bytes":6624,"groups":4},"bots":{"GPTBot":["partial","*",169,64],"ChatGPT-User":["partial","*",169,64],"OAI-SearchBot":["partial","*",169,64],"ClaudeBot":["partial","*",169,64],"Claude-User":["partial","*",169,64],"Claude-SearchBot":["partial","*",169,64],"anthropic-ai":["partial","*",169,64],"PerplexityBot":["partial","*",169,64],"Perplexity-User":["partial","*",169,64],"Google-Extended":["partial","*",169,64],"Applebot-Extended":["partial","*",169,64],"CCBot":["partial","*",169,64],"Bytespider":["partial","*",169,64],"Amazonbot":["partial","*",169,64],"meta-externalagent":["partial","*",169,64],"cohere-ai":["partial","*",169,64],"Diffbot":["partial","*",169,64],"ContextFetchBot":["partial","*",169,64]},"llms":{"status":"missing","http":404,"bytes":1569},"home":{"status":"ok","http":200,"url":"https://www.google.com/","html_tokens":21282,"markdown_tokens":30,"pruned_tokens":30}},
{"rank":9606,"domain":"germany.travel","origin":"https://germany.travel","robots":{"status":"ok","http":200,"url":"https://www.germany.travel/robots.txt","bytes":22,"groups":1},"bots":{"GPTBot":["allowed","*",0,1],"ChatGPT-User":["allowed","*",0,1],"OAI-SearchBot":["allowed","*",0,1],"ClaudeBot":["allowed","*",0,1],"Claude-User":["allowed","*",0,1],"Claude-SearchBot":["allowed","*",0,1],"anthropic-ai":["allowed","*",0,1],"PerplexityBot":["allowed","*",0,1],"Perplexity-User":["allowed","*",0,1],"Google-Extended":["allowed","*",0,1],"Applebot-Extended":["allowed","*",0,1],"CCBot":["allowed","*",0,1],"Bytespider":["allowed","*",0,1],"Amazonbot":["allowed","*",0,1],"meta-externalagent":["allowed","*",0,1],"cohere-ai":["allowed","*",0,1],"Diffbot":["allowed","*",0,1],"ContextFetchBot":["allowed","*",0,1]},"llms":{"status":"missing","http":404,"bytes":49387},"home":{"status":"ok","http":200,"url":"https://www.germany.travel/en/home.html","html_tokens":78318,"markdown_tokens":307,"pruned_tokens":100}},
{"rank":9607,"domain":"slimgogo.store","origin":"https://slimgogo.store","robots":{"status":"ok","http":200,"url":"https://slimgogo.store/robots.txt","bytes":75,"groups":1},"bots":{"GPTBot":["partial","*",1,0],"ChatGPT-User":["partial","*",1,0],"OAI-SearchBot":["partial","*",1,0],"ClaudeBot":["partial","*",1,0],"Claude-User":["partial","*",1,0],"Claude-SearchBot":["partial","*",1,0],"anthropic-ai":["partial","*",1,0],"PerplexityBot":["partial","*",1,0],"Perplexity-User":["partial","*",1,0],"Google-Extended":["partial","*",1,0],"Applebot-Extended":["partial","*",1,0],"CCBot":["partial","*",1,0],"Bytespider":["partial","*",1,0],"Amazonbot":["partial","*",1,0],"meta-externalagent":["partial","*",1,0],"cohere-ai":["partial","*",1,0],"Diffbot":["partial","*",1,0],"ContextFetchBot":["partial","*",1,0]},"llms":{"status":"missing","http":404,"bytes":643},"home":{"status":"ok","http":200,"url":"https://slimgogo.store/","html_tokens":27339,"markdown_tokens":1988,"pruned_tokens":120}},
{"rank":9608,"domain":"caprover.com","origin":"https://caprover.com","robots":{"status":"ok","http":200,"url":"https://caprover.com/robots.txt","bytes":65,"groups":1},"bots":{"GPTBot":["allowed","*",0,1],"ChatGPT-User":["allowed","*",0,1],"OAI-SearchBot":["allowed","*",0,1],"ClaudeBot":["allowed","*",0,1],"Claude-User":["allowed","*",0,1],"Claude-SearchBot":["allowed","*",0,1],"anthropic-ai":["allowed","*",0,1],"PerplexityBot":["allowed","*",0,1],"Perplexity-User":["allowed","*",0,1],"Google-Extended":["allowed","*",0,1],"Applebot-Extended":["allowed","*",0,1],"CCBot":["allowed","*",0,1],"Bytespider":["allowed","*",0,1],"Amazonbot":["allowed","*",0,1],"meta-externalagent":["allowed","*",0,1],"cohere-ai":["allowed","*",0,1],"Diffbot":["allowed","*",0,1],"ContextFetchBot":["allowed","*",0,1]},"llms":{"status":"missing","http":404,"bytes":10839},"home":{"status":"ok","http":200,"url":"https://caprover.com/","html_tokens":10560,"markdown_tokens":435,"pruned_tokens":42}},
{"rank":9609,"domain":"ibaraki.jp","origin":"https://ibaraki.jp","robots":{"status":"unreachable","http":null,"url":"https://ibaraki.jp/robots.txt","bytes":0,"groups":0,"error":"cannot resolve ibaraki.jp"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":9610,"domain":"barclays.co.uk","origin":"https://barclays.co.uk","robots":{"status":"ok","http":200,"url":"https://www.barclays.co.uk/robots.txt","bytes":486,"groups":1},"bots":{"GPTBot":["partial","*",6,0],"ChatGPT-User":["partial","*",6,0],"OAI-SearchBot":["partial","*",6,0],"ClaudeBot":["partial","*",6,0],"Claude-User":["partial","*",6,0],"Claude-SearchBot":["partial","*",6,0],"anthropic-ai":["partial","*",6,0],"PerplexityBot":["partial","*",6,0],"Perplexity-User":["partial","*",6,0],"Google-Extended":["partial","*",6,0],"Applebot-Extended":["partial","*",6,0],"CCBot":["partial","*",6,0],"Bytespider":["partial","*",6,0],"Amazonbot":["partial","*",6,0],"meta-externalagent":["partial","*",6,0],"cohere-ai":["partial","*",6,0],"Diffbot":["partial","*",6,0],"ContextFetchBot":["partial","*",6,0]},"llms":{"status":"html","http":200,"bytes":303344},"home":{"status":"ok","http":200,"url":"https://www.barclays.co.uk/","html_tokens":85514,"markdown_tokens":462,"pruned_tokens":46}},
{"rank":9611,"domain":"miyagi.jp","origin":"https://miyagi.jp","robots":{"status":"unreachable","http":null,"url":"https://miyagi.jp/robots.txt","bytes":0,"groups":0,"error":"cannot resolve miyagi.jp"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":9612,"domain":"dost.gov.ph","origin":"https://dost.gov.ph","robots":{"status":"ok","http":200,"url":"https://dost.gov.ph/robots.txt","bytes":23,"groups":1},"bots":{"GPTBot":["allowed","*",0,1],"ChatGPT-User":["allowed","*",0,1],"OAI-SearchBot":["allowed","*",0,1],"ClaudeBot":["allowed","*",0,1],"Claude-User":["allowed","*",0,1],"Claude-SearchBot":["allowed","*",0,1],"anthropic-ai":["allowed","*",0,1],"PerplexityBot":["allowed","*",0,1],"Perplexity-User":["allowed","*",0,1],"Google-Extended":["allowed","*",0,1],"Applebot-Extended":["allowed","*",0,1],"CCBot":["allowed","*",0,1],"Bytespider":["allowed","*",0,1],"Amazonbot":["allowed","*",0,1],"meta-externalagent":["allowed","*",0,1],"cohere-ai":["allowed","*",0,1],"Diffbot":["allowed","*",0,1],"ContextFetchBot":["allowed","*",0,1]},"llms":{"status":"html","http":200,"bytes":51736},"home":{"status":"ok","http":200,"url":"https://dost.gov.ph/","html_tokens":25949,"markdown_tokens":240,"pruned_tokens":120}},
{"rank":9613,"domain":"thefork.com","origin":"https://thefork.com","robots":{"status":"ok","http":200,"url":"https://www.thefork.com/robots.txt","bytes":5152,"groups":1},"bots":{"GPTBot":["partial","*",184,1],"ChatGPT-User":["partial","*",184,1],"OAI-SearchBot":["partial","*",184,1],"ClaudeBot":["partial","*",184,1],"Claude-User":["partial","*",184,1],"Claude-SearchBot":["partial","*",184,1],"anthropic-ai":["partial","*",184,1],"PerplexityBot":["partial","*",184,1],"Perplexity-User":["partial","*",184,1],"Google-Extended":["partial","*",184,1],"Applebot-Extended":["partial","*",184,1],"CCBot":["partial","*",184,1],"Bytespider":["partial","*",184,1],"Amazonbot":["partial","*",184,1],"meta-externalagent":["partial","*",184,1],"cohere-ai":["partial","*",184,1],"Diffbot":["partial","*",184,1],"ContextFetchBot":["partial","*",184,1]},"llms":{"status":"missing","http":404,"bytes":260250},"home":{"status":"http_error","http":403,"url":"https://www.thefork.com/"}},
{"rank":9614,"domain":"oec.world","origin":"https://oec.world","robots":{"status":"ok","http":200,"url":"https://oec.world/robots.txt","bytes":684,"groups":9},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":124227},"home":{"status":"ok","http":200,"url":"https://oec.world/en","html_tokens":206379,"markdown_tokens":226,"pruned_tokens":115}},
{"rank":9615,"domain":"froxlor.org","origin":"https://froxlor.org","robots":{"status":"ok","http":200,"url":"https://froxlor.org/robots.txt","bytes":26,"groups":1},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"missing","http":404,"bytes":6603},"home":{"status":"ok","http":200,"url":"https://froxlor.org/","html_tokens":5059,"markdown_tokens":588,"pruned_tokens":62}},
{"rank":9616,"domain":"curator.io","origin":"https://curator.io","robots":{"status":"ok","http":200,"url":"https://curator.io/robots.txt","bytes":63,"groups":1},"bots":{"GPTBot":["allowed","*",0,1],"ChatGPT-User":["allowed","*",0,1],"OAI-SearchBot":["allowed","*",0,1],"ClaudeBot":["allowed","*",0,1],"Claude-User":["allowed","*",0,1],"Claude-SearchBot":["allowed","*",0,1],"anthropic-ai":["allowed","*",0,1],"PerplexityBot":["allowed","*",0,1],"Perplexity-User":["allowed","*",0,1],"Google-Extended":["allowed","*",0,1],"Applebot-Extended":["allowed","*",0,1],"CCBot":["allowed","*",0,1],"Bytespider":["allowed","*",0,1],"Amazonbot":["allowed","*",0,1],"meta-externalagent":["allowed","*",0,1],"cohere-ai":["allowed","*",0,1],"Diffbot":["allowed","*",0,1],"ContextFetchBot":["allowed","*",0,1]},"llms":{"status":"present","http":200,"bytes":7230},"home":{"status":"ok","http":200,"url":"https://curator.io/","html_tokens":280666,"markdown_tokens":933,"pruned_tokens":45}},
{"rank":9617,"domain":"thunderstruck-slots.com","origin":"https://thunderstruck-slots.com","robots":{"status":"ok","http":200,"url":"https://thunderstruck-slots.com/robots.txt","bytes":550,"groups":1},"bots":{"GPTBot":["partial","*",19,0],"ChatGPT-User":["partial","*",19,0],"OAI-SearchBot":["partial","*",19,0],"ClaudeBot":["partial","*",19,0],"Claude-User":["partial","*",19,0],"Claude-SearchBot":["partial","*",19,0],"anthropic-ai":["partial","*",19,0],"PerplexityBot":["partial","*",19,0],"Perplexity-User":["partial","*",19,0],"Google-Extended":["partial","*",19,0],"Applebot-Extended":["partial","*",19,0],"CCBot":["partial","*",19,0],"Bytespider":["partial","*",19,0],"Amazonbot":["partial","*",19,0],"meta-externalagent":["partial","*",19,0],"cohere-ai":["partial","*",19,0],"Diffbot":["partial","*",19,0],"ContextFetchBot":["partial","*",19,0]},"llms":{"status":"missing","http":404,"bytes":21461},"home":{"status":"ok","http":200,"url":"https://thunderstruck-slots.com/","html_tokens":18712,"markdown_tokens":804,"pruned_tokens":252}},
{"rank":9618,"domain":"webnode.fr","origin":"https://webnode.fr","robots":{"status":"missing","http":404,"url":"https://webnode.fr/robots.txt","bytes":0,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"present","http":200,"bytes":3499},"home":{"status":"ok","http":200,"url":"https://www.webnode.com/fr/","html_tokens":62797,"markdown_tokens":102,"pruned_tokens":102}},
{"rank":9619,"domain":"kormany.hu","origin":"https://kormany.hu","robots":{"status":"ok","http":200,"url":"https://kormany.hu/robots.txt","bytes":35,"groups":1},"bots":{"GPTBot":["partial","*",1,0],"ChatGPT-User":["partial","*",1,0],"OAI-SearchBot":["partial","*",1,0],"ClaudeBot":["partial","*",1,0],"Claude-User":["partial","*",1,0],"Claude-SearchBot":["partial","*",1,0],"anthropic-ai":["partial","*",1,0],"PerplexityBot":["partial","*",1,0],"Perplexity-User":["partial","*",1,0],"Google-Extended":["partial","*",1,0],"Applebot-Extended":["partial","*",1,0],"CCBot":["partial","*",1,0],"Bytespider":["partial","*",1,0],"Amazonbot":["partial","*",1,0],"meta-externalagent":["partial","*",1,0],"cohere-ai":["partial","*",1,0],"Diffbot":["partial","*",1,0],"ContextFetchBot":["partial","*",1,0]},"llms":{"status":"missing","http":404,"bytes":717415},"home":{"status":"ok","http":200,"url":"https://kormany.hu/","html_tokens":208632,"markdown_tokens":501,"pruned_tokens":111}},
{"rank":9620,"domain":"treasurydirect.gov","origin":"https://treasurydirect.gov","robots":{"status":"ok","http":200,"url":"https://treasurydirect.gov/robots.txt","bytes":128,"groups":2},"bots":{"GPTBot":["blocked","*",1,0],"ChatGPT-User":["blocked","*",1,0],"OAI-SearchBot":["blocked","*",1,0],"ClaudeBot":["blocked","*",1,0],"Claude-User":["blocked","*",1,0],"Claude-SearchBot":["blocked","*",1,0],"anthropic-ai":["blocked","*",1,0],"PerplexityBot":["blocked","*",1,0],"Perplexity-User":["blocked","*",1,0],"Google-Extended":["blocked","*",1,0],"Applebot-Extended":["blocked","*",1,0],"CCBot":["blocked","*",1,0],"Bytespider":["blocked","*",1,0],"Amazonbot":["blocked","*",1,0],"meta-externalagent":["blocked","*",1,0],"cohere-ai":["blocked","*",1,0],"Diffbot":["blocked","*",1,0],"ContextFetchBot":["blocked","*",1,0]},"llms":{"status":"disallowed"},"home":{"status":"disallowed"}},
{"rank":9621,"domain":"vonage.com","origin":"https://vonage.com","robots":{"status":"ok","http":200,"url":"https://www.vonage.com/robots.txt","bytes":94,"groups":1},"bots":{"GPTBot":["partial","*",2,0],"ChatGPT-User":["partial","*",2,0],"OAI-SearchBot":["partial","*",2,0],"ClaudeBot":["partial","*",2,0],"Claude-User":["partial","*",2,0],"Claude-SearchBot":["partial","*",2,0],"anthropic-ai":["partial","*",2,0],"PerplexityBot":["partial","*",2,0],"Perplexity-User":["partial","*",2,0],"Google-Extended":["partial","*",2,0],"Applebot-Extended":["partial","*",2,0],"CCBot":["partial","*",2,0],"Bytespider":["partial","*",2,0],"Amazonbot":["partial","*",2,0],"meta-externalagent":["partial","*",2,0],"cohere-ai":["partial","*",2,0],"Diffbot":["partial","*",2,0],"ContextFetchBot":["partial","*",2,0]},"llms":{"status":"present","http":200,"bytes":5873},"home":{"status":"ok","http":200,"url":"https://www.vonage.com/","html_tokens":67207,"markdown_tokens":74,"pruned_tokens":74}},
{"rank":9622,"domain":"streamlabs.com","origin":"https://streamlabs.com","robots":{"status":"ok","http":200,"url":"https://streamlabs.com/robots.txt","bytes":388,"groups":1},"bots":{"GPTBot":["partial","*",9,2],"ChatGPT-User":["partial","*",9,2],"OAI-SearchBot":["partial","*",9,2],"ClaudeBot":["partial","*",9,2],"Claude-User":["partial","*",9,2],"Claude-SearchBot":["partial","*",9,2],"anthropic-ai":["partial","*",9,2],"PerplexityBot":["partial","*",9,2],"Perplexity-User":["partial","*",9,2],"Google-Extended":["partial","*",9,2],"Applebot-Extended":["partial","*",9,2],"CCBot":["partial","*",9,2],"Bytespider":["partial","*",9,2],"Amazonbot":["partial","*",9,2],"meta-externalagent":["partial","*",9,2],"cohere-ai":["partial","*",9,2],"Diffbot":["partial","*",9,2],"ContextFetchBot":["partial","*",9,2]},"llms":{"status":"present","http":200,"bytes":8402},"home":{"status":"ok","http":200,"url":"https://streamlabs.com/","html_tokens":7741,"markdown_tokens":0,"pruned_tokens":0}},
{"rank":9623,"domain":"furusato-tax.jp","origin":"https://furusato-tax.jp","robots":{"status":"ok","http":200,"url":"https://www.furusato-tax.jp/robots.txt","bytes":263,"groups":5},"bots":{"GPTBot":["partial","*",3,0],"ChatGPT-User":["partial","*",3,0],"OAI-SearchBot":["partial","*",3,0],"ClaudeBot":["partial","*",3,0],"Claude-User":["partial","*",3,0],"Claude-SearchBot":["partial","*",3,0],"anthropic-ai":["partial","*",3,0],"PerplexityBot":["partial","*",3,0],"Perplexity-User":["partial","*",3,0],"Google-Extended":["partial","*",3,0],"Applebot-Extended":["partial","*",3,0],"CCBot":["partial","*",3,0],"Bytespider":["partial","*",3,0],"Amazonbot":["partial","*",3,0],"meta-externalagent":["partial","*",3,0],"cohere-ai":["partial","*",3,0],"Diffbot":["partial","*",3,0],"ContextFetchBot":["partial","*",3,0]},"llms":{"status":"missing","http":404,"bytes":119608},"home":{"status":"ok","http":200,"url":"https://www.furusato-tax.jp/","html_tokens":185646,"markdown_tokens":159,"pruned_tokens":159}},
{"rank":9624,"domain":"shopinfo.jp","origin":"https://shopinfo.jp","robots":{"status":"unreachable","http":null,"url":"https://shopinfo.jp/robots.txt","bytes":0,"groups":0,"error":"cannot resolve shopinfo.jp"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":9625,"domain":"wsp.com","origin":"https://wsp.com","robots":{"status":"ok","http":200,"url":"https://www.wsp.com/robots.txt","bytes":2579,"groups":1},"bots":{"GPTBot":["partial","*",25,0],"ChatGPT-User":["partial","*",25,0],"OAI-SearchBot":["partial","*",25,0],"ClaudeBot":["partial","*",25,0],"Claude-User":["partial","*",25,0],"Claude-SearchBot":["partial","*",25,0],"anthropic-ai":["partial","*",25,0],"PerplexityBot":["partial","*",25,0],"Perplexity-User":["partial","*",25,0],"Google-Extended":["partial","*",25,0],"Applebot-Extended":["partial","*",25,0],"CCBot":["partial","*",25,0],"Bytespider":["partial","*",25,0],"Amazonbot":["partial","*",25,0],"meta-externalagent":["partial","*",25,0],"cohere-ai":["partial","*",25,0],"Diffbot":["partial","*",25,0],"ContextFetchBot":["partial","*",25,0]},"llms":{"status":"missing","http":404,"bytes":1245},"home":{"status":"ok","http":200,"url":"https://www.wsp.com/en-us","html_tokens":79694,"markdown_tokens":64,"pruned_tokens":64}},
{"rank":9626,"domain":"google.tl","origin":"https://google.tl","robots":{"status":"ok","http":200,"url":"https://www.google.tl/robots.txt","bytes":6624,"groups":4},"bots":{"GPTBot":["partial","*",169,64],"ChatGPT-User":["partial","*",169,64],"OAI-SearchBot":["partial","*",169,64],"ClaudeBot":["partial","*",169,64],"Claude-User":["partial","*",169,64],"Claude-SearchBot":["partial","*",169,64],"anthropic-ai":["partial","*",169,64],"PerplexityBot":["partial","*",169,64],"Perplexity-User":["partial","*",169,64],"Google-Extended":["partial","*",169,64],"Applebot-Extended":["partial","*",169,64],"CCBot":["partial","*",169,64],"Bytespider":["partial","*",169,64],"Amazonbot":["partial","*",169,64],"meta-externalagent":["partial","*",169,64],"cohere-ai":["partial","*",169,64],"Diffbot":["partial","*",169,64],"ContextFetchBot":["partial","*",169,64]},"llms":{"status":"missing","http":404,"bytes":1569},"home":{"status":"ok","http":200,"url":"https://www.google.com/","html_tokens":21294,"markdown_tokens":30,"pruned_tokens":30}},
{"rank":9627,"domain":"gwern.net","origin":"https://gwern.net","robots":{"status":"ok","http":200,"url":"https://gwern.net/robots.txt","bytes":890,"groups":2},"bots":{"GPTBot":["partial","*",12,0],"ChatGPT-User":["partial","*",12,0],"OAI-SearchBot":["partial","*",12,0],"ClaudeBot":["partial","*",12,0],"Claude-User":["partial","*",12,0],"Claude-SearchBot":["partial","*",12,0],"anthropic-ai":["partial","*",12,0],"PerplexityBot":["partial","*",12,0],"Perplexity-User":["partial","*",12,0],"Google-Extended":["partial","*",12,0],"Applebot-Extended":["partial","*",12,0],"CCBot":["partial","*",12,0],"Bytespider":["partial","*",12,0],"Amazonbot":["partial","*",12,0],"meta-externalagent":["partial","*",12,0],"cohere-ai":["partial","*",12,0],"Diffbot":["partial","*",12,0],"ContextFetchBot":["partial","*",12,0]},"llms":{"status":"other_text","http":200,"bytes":38167},"home":{"status":"not_html","error":"unsupported content type text/markdown"}},
{"rank":9628,"domain":"zhaw.ch","origin":"https://zhaw.ch","robots":{"status":"ok","http":200,"url":"https://www.zhaw.ch/robots.txt","bytes":235,"groups":1},"bots":{"GPTBot":["partial","*",5,5],"ChatGPT-User":["partial","*",5,5],"OAI-SearchBot":["partial","*",5,5],"ClaudeBot":["partial","*",5,5],"Claude-User":["partial","*",5,5],"Claude-SearchBot":["partial","*",5,5],"anthropic-ai":["partial","*",5,5],"PerplexityBot":["partial","*",5,5],"Perplexity-User":["partial","*",5,5],"Google-Extended":["partial","*",5,5],"Applebot-Extended":["partial","*",5,5],"CCBot":["partial","*",5,5],"Bytespider":["partial","*",5,5],"Amazonbot":["partial","*",5,5],"meta-externalagent":["partial","*",5,5],"cohere-ai":["partial","*",5,5],"Diffbot":["partial","*",5,5],"ContextFetchBot":["partial","*",5,5]},"llms":{"status":"missing","http":404,"bytes":86331},"home":{"status":"ok","http":200,"url":"https://www.zhaw.ch/de/hochschule","html_tokens":43339,"markdown_tokens":266,"pruned_tokens":153}},
{"rank":9629,"domain":"kwork.ru","origin":"https://kwork.ru","robots":{"status":"ok","http":200,"url":"https://kwork.ru/robots.txt","bytes":479,"groups":1},"bots":{"GPTBot":["partial","*",21,0],"ChatGPT-User":["partial","*",21,0],"OAI-SearchBot":["partial","*",21,0],"ClaudeBot":["partial","*",21,0],"Claude-User":["partial","*",21,0],"Claude-SearchBot":["partial","*",21,0],"anthropic-ai":["partial","*",21,0],"PerplexityBot":["partial","*",21,0],"Perplexity-User":["partial","*",21,0],"Google-Extended":["partial","*",21,0],"Applebot-Extended":["partial","*",21,0],"CCBot":["partial","*",21,0],"Bytespider":["partial","*",21,0],"Amazonbot":["partial","*",21,0],"meta-externalagent":["partial","*",21,0],"cohere-ai":["partial","*",21,0],"Diffbot":["partial","*",21,0],"ContextFetchBot":["partial","*",21,0]},"llms":{"status":"missing","http":404,"bytes":52547},"home":{"status":"ok","http":200,"url":"https://kwork.ru/","html_tokens":18413,"markdown_tokens":60,"pruned_tokens":60}},
{"rank":9630,"domain":"genproc.gov.ru","origin":"https://genproc.gov.ru","robots":{"status":"timeout","http":null,"url":"https://genproc.gov.ru/robots.txt","bytes":0,"groups":0,"error":"ConnectTimeout"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":9631,"domain":"in2p3.fr","origin":"https://www.in2p3.fr","robots":{"status":"html","http":200,"url":"https://www.in2p3.cnrs.fr","bytes":244,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":9632,"domain":"privacyinternational.org","origin":"https://privacyinternational.org","robots":{"status":"ok","http":200,"url":"https://privacyinternational.org/robots.txt","bytes":1652,"groups":1},"bots":{"GPTBot":["partial","*",24,18],"ChatGPT-User":["partial","*",24,18],"OAI-SearchBot":["partial","*",24,18],"ClaudeBot":["partial","*",24,18],"Claude-User":["partial","*",24,18],"Claude-SearchBot":["partial","*",24,18],"anthropic-ai":["partial","*",24,18],"PerplexityBot":["partial","*",24,18],"Perplexity-User":["partial","*",24,18],"Google-Extended":["partial","*",24,18],"Applebot-Extended":["partial","*",24,18],"CCBot":["partial","*",24,18],"Bytespider":["partial","*",24,18],"Amazonbot":["partial","*",24,18],"meta-externalagent":["partial","*",24,18],"cohere-ai":["partial","*",24,18],"Diffbot":["partial","*",24,18],"ContextFetchBot":["partial","*",24,18]},"llms":{"status":"missing","http":404,"bytes":662},"home":{"status":"ok","http":200,"url":"https://privacyinternational.org/","html_tokens":13384,"markdown_tokens":223,"pruned_tokens":52}},
{"rank":9633,"domain":"fi.edu","origin":"https://fi.edu","robots":{"status":"ok","http":200,"url":"https://fi.edu/robots.txt","bytes":1249,"groups":1},"bots":{"GPTBot":["partial","*",24,18],"ChatGPT-User":["partial","*",24,18],"OAI-SearchBot":["partial","*",24,18],"ClaudeBot":["partial","*",24,18],"Claude-User":["partial","*",24,18],"Claude-SearchBot":["partial","*",24,18],"anthropic-ai":["partial","*",24,18],"PerplexityBot":["partial","*",24,18],"Perplexity-User":["partial","*",24,18],"Google-Extended":["partial","*",24,18],"Applebot-Extended":["partial","*",24,18],"CCBot":["partial","*",24,18],"Bytespider":["partial","*",24,18],"Amazonbot":["partial","*",24,18],"meta-externalagent":["partial","*",24,18],"cohere-ai":["partial","*",24,18],"Diffbot":["partial","*",24,18],"ContextFetchBot":["partial","*",24,18]},"llms":{"status":"missing","http":404,"bytes":53778},"home":{"status":"ok","http":200,"url":"https://fi.edu/en","html_tokens":19416,"markdown_tokens":223,"pruned_tokens":66}},
{"rank":9634,"domain":"rzhevnews.ru","origin":"https://rzhevnews.ru","robots":{"status":"ok","http":200,"url":"https://rzhevnews.ru/robots.txt","bytes":129,"groups":1},"bots":{"GPTBot":["partial","*",2,1],"ChatGPT-User":["partial","*",2,1],"OAI-SearchBot":["partial","*",2,1],"ClaudeBot":["partial","*",2,1],"Claude-User":["partial","*",2,1],"Claude-SearchBot":["partial","*",2,1],"anthropic-ai":["partial","*",2,1],"PerplexityBot":["partial","*",2,1],"Perplexity-User":["partial","*",2,1],"Google-Extended":["partial","*",2,1],"Applebot-Extended":["partial","*",2,1],"CCBot":["partial","*",2,1],"Bytespider":["partial","*",2,1],"Amazonbot":["partial","*",2,1],"meta-externalagent":["partial","*",2,1],"cohere-ai":["partial","*",2,1],"Diffbot":["partial","*",2,1],"ContextFetchBot":["partial","*",2,1]},"llms":{"status":"html","http":200,"bytes":21358},"home":{"status":"ok","http":200,"url":"https://rzhevnews.ru/","html_tokens":4999,"markdown_tokens":213,"pruned_tokens":163}},
{"rank":9635,"domain":"blacklivesmatter.com","origin":"https://blacklivesmatter.com","robots":{"status":"refused","http":403,"url":"https://blacklivesmatter.com/robots.txt","bytes":5537,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":9636,"domain":"bezformata.com","origin":"https://bezformata.com","robots":{"status":"refused","http":403,"url":"https://bezformata.com/robots.txt","bytes":5517,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":9637,"domain":"macports.org","origin":"https://macports.org","robots":{"status":"missing","http":404,"url":"https://www.macports.org/robots.txt","bytes":279,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":279},"home":{"status":"ok","http":200,"url":"https://www.macports.org/","html_tokens":2386,"markdown_tokens":790,"pruned_tokens":128}},
{"rank":9638,"domain":"pressgazette.co.uk","origin":"https://pressgazette.co.uk","robots":{"status":"refused","http":403,"url":"https://pressgazette.co.uk/robots.txt","bytes":560,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":9639,"domain":"mines.edu","origin":"https://mines.edu","robots":{"status":"ok","http":200,"url":"https://www.mines.edu/robots.txt","bytes":316,"groups":5},"bots":{"GPTBot":["partial","*",3,0],"ChatGPT-User":["partial","*",3,0],"OAI-SearchBot":["partial","*",3,0],"ClaudeBot":["partial","*",3,0],"Claude-User":["partial","*",3,0],"Claude-SearchBot":["partial","*",3,0],"anthropic-ai":["partial","*",3,0],"PerplexityBot":["partial","*",3,0],"Perplexity-User":["partial","*",3,0],"Google-Extended":["partial","*",3,0],"Applebot-Extended":["partial","*",3,0],"CCBot":["partial","*",3,0],"Bytespider":["partial","*",3,0],"Amazonbot":["partial","*",3,0],"meta-externalagent":["partial","*",3,0],"cohere-ai":["partial","*",3,0],"Diffbot":["partial","*",3,0],"ContextFetchBot":["partial","*",3,0]},"llms":{"status":"missing","http":404,"bytes":47755},"home":{"status":"ok","http":200,"url":"https://www.mines.edu/","html_tokens":25316,"markdown_tokens":1128,"pruned_tokens":156}},
{"rank":9640,"domain":"wkyc.com","origin":"https://wkyc.com","robots":{"status":"ok","http":200,"url":"https://www.wkyc.com/robots.txt","bytes":444,"groups":2},"bots":{"GPTBot":["partial","*",10,0],"ChatGPT-User":["partial","*",10,0],"OAI-SearchBot":["partial","*",10,0],"ClaudeBot":["partial","*",10,0],"Claude-User":["partial","*",10,0],"Claude-SearchBot":["partial","*",10,0],"anthropic-ai":["partial","*",10,0],"PerplexityBot":["partial","*",10,0],"Perplexity-User":["partial","*",10,0],"Google-Extended":["partial","*",10,0],"Applebot-Extended":["partial","*",10,0],"CCBot":["partial","*",10,0],"Bytespider":["partial","*",10,0],"Amazonbot":["partial","*",10,0],"meta-externalagent":["partial","*",10,0],"cohere-ai":["partial","*",10,0],"Diffbot":["partial","*",10,0],"ContextFetchBot":["partial","*",10,0]},"llms":{"status":"missing","http":404,"bytes":6836},"home":{"status":"ok","http":200,"url":"https://www.wkyc.com/","html_tokens":344585,"markdown_tokens":3282,"pruned_tokens":168}},
{"rank":9641,"domain":"schiphol.nl","origin":"https://schiphol.nl","robots":{"status":"ok","http":200,"url":"https://www.schiphol.nl/robots.txt","bytes":811,"groups":1},"bots":{"GPTBot":["partial","*",21,0],"ChatGPT-User":["partial","*",21,0],"OAI-SearchBot":["partial","*",21,0],"ClaudeBot":["partial","*",21,0],"Claude-User":["partial","*",21,0],"Claude-SearchBot":["partial","*",21,0],"anthropic-ai":["partial","*",21,0],"PerplexityBot":["partial","*",21,0],"Perplexity-User":["partial","*",21,0],"Google-Extended":["partial","*",21,0],"Applebot-Extended":["partial","*",21,0],"CCBot":["partial","*",21,0],"Bytespider":["partial","*",21,0],"Amazonbot":["partial","*",21,0],"meta-externalagent":["partial","*",21,0],"cohere-ai":["partial","*",21,0],"Diffbot":["partial","*",21,0],"ContextFetchBot":["partial","*",21,0]},"llms":{"status":"missing","http":404,"bytes":22508},"home":{"status":"http_error","http":403,"url":"https://www.schiphol.nl/en/"}},
{"rank":9642,"domain":"heavens-above.com","origin":"https://heavens-above.com","robots":{"status":"ok","http":200,"url":"https://heavens-above.com/robots.txt","bytes":563,"groups":1},"bots":{"GPTBot":["partial","*",19,0],"ChatGPT-User":["partial","*",19,0],"OAI-SearchBot":["partial","*",19,0],"ClaudeBot":["partial","*",19,0],"Claude-User":["partial","*",19,0],"Claude-SearchBot":["partial","*",19,0],"anthropic-ai":["partial","*",19,0],"PerplexityBot":["partial","*",19,0],"Perplexity-User":["partial","*",19,0],"Google-Extended":["partial","*",19,0],"Applebot-Extended":["partial","*",19,0],"CCBot":["partial","*",19,0],"Bytespider":["partial","*",19,0],"Amazonbot":["partial","*",19,0],"meta-externalagent":["partial","*",19,0],"cohere-ai":["partial","*",19,0],"Diffbot":["partial","*",19,0],"ContextFetchBot":["partial","*",19,0]},"llms":{"status":"missing","http":404,"bytes":1285},"home":{"status":"ok","http":200,"url":"https://heavens-above.com/","html_tokens":3777,"markdown_tokens":107,"pruned_tokens":23}},
{"rank":9643,"domain":"seg-social.es","origin":"https://www.seg-social.es","robots":{"status":"ok","http":200,"url":"https://www.seg-social.es/robots.txt","bytes":124,"groups":1},"bots":{"GPTBot":["partial","*",2,1],"ChatGPT-User":["partial","*",2,1],"OAI-SearchBot":["partial","*",2,1],"ClaudeBot":["partial","*",2,1],"Claude-User":["partial","*",2,1],"Claude-SearchBot":["partial","*",2,1],"anthropic-ai":["partial","*",2,1],"PerplexityBot":["partial","*",2,1],"Perplexity-User":["partial","*",2,1],"Google-Extended":["partial","*",2,1],"Applebot-Extended":["partial","*",2,1],"CCBot":["partial","*",2,1],"Bytespider":["partial","*",2,1],"Amazonbot":["partial","*",2,1],"meta-externalagent":["partial","*",2,1],"cohere-ai":["partial","*",2,1],"Diffbot":["partial","*",2,1],"ContextFetchBot":["partial","*",2,1]},"llms":{"status":"missing","http":404,"bytes":2972},"home":{"status":"ok","http":200,"url":"https://www.seg-social.es/wps/portal/wss/internet/Inicio","html_tokens":564,"markdown_tokens":0,"pruned_tokens":0}},
{"rank":9644,"domain":"atari.com","origin":"https://atari.com","robots":{"status":"ok","http":200,"url":"https://atari.com/robots.txt","bytes":3623,"groups":2},"bots":{"GPTBot":["partial","*",40,23],"ChatGPT-User":["partial","*",40,23],"OAI-SearchBot":["partial","*",40,23],"ClaudeBot":["partial","*",40,23],"Claude-User":["partial","*",40,23],"Claude-SearchBot":["partial","*",40,23],"anthropic-ai":["partial","*",40,23],"PerplexityBot":["partial","*",40,23],"Perplexity-User":["partial","*",40,23],"Google-Extended":["partial","*",40,23],"Applebot-Extended":["partial","*",40,23],"CCBot":["partial","*",40,23],"Bytespider":["partial","*",40,23],"Amazonbot":["partial","*",40,23],"meta-externalagent":["partial","*",40,23],"cohere-ai":["partial","*",40,23],"Diffbot":["partial","*",40,23],"ContextFetchBot":["partial","*",40,23]},"llms":{"status":"present","http":200,"bytes":5678},"home":{"status":"http_error","http":403,"url":"https://atari.com/"}},
{"rank":9645,"domain":"nzn-oa.com","origin":"https://nzn-oa.com","robots":{"status":"ok","http":200,"url":"https://nzn-oa.com/robots.txt","bytes":26,"groups":1},"bots":{"GPTBot":["blocked","*",1,0],"ChatGPT-User":["blocked","*",1,0],"OAI-SearchBot":["blocked","*",1,0],"ClaudeBot":["blocked","*",1,0],"Claude-User":["blocked","*",1,0],"Claude-SearchBot":["blocked","*",1,0],"anthropic-ai":["blocked","*",1,0],"PerplexityBot":["blocked","*",1,0],"Perplexity-User":["blocked","*",1,0],"Google-Extended":["blocked","*",1,0],"Applebot-Extended":["blocked","*",1,0],"CCBot":["blocked","*",1,0],"Bytespider":["blocked","*",1,0],"Amazonbot":["blocked","*",1,0],"meta-externalagent":["blocked","*",1,0],"cohere-ai":["blocked","*",1,0],"Diffbot":["blocked","*",1,0],"ContextFetchBot":["blocked","*",1,0]},"llms":{"status":"disallowed"},"home":{"status":"disallowed"}},
{"rank":9646,"domain":"xdjxpt.com","origin":"http://xdjxpt.com","robots":{"status":"html","http":200,"url":"http://xhjxpt.com/robots.txt","bytes":887,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":9647,"domain":"wweek.com","origin":"https://wweek.com","robots":{"status":"ok","http":200,"url":"https://www.wweek.com/robots.txt","bytes":820,"groups":1},"bots":{"GPTBot":["partial","*",9,0],"ChatGPT-User":["partial","*",9,0],"OAI-SearchBot":["partial","*",9,0],"ClaudeBot":["partial","*",9,0],"Claude-User":["partial","*",9,0],"Claude-SearchBot":["partial","*",9,0],"anthropic-ai":["partial","*",9,0],"PerplexityBot":["partial","*",9,0],"Perplexity-User":["partial","*",9,0],"Google-Extended":["partial","*",9,0],"Applebot-Extended":["partial","*",9,0],"CCBot":["partial","*",9,0],"Bytespider":["partial","*",9,0],"Amazonbot":["partial","*",9,0],"meta-externalagent":["partial","*",9,0],"cohere-ai":["partial","*",9,0],"Diffbot":["partial","*",9,0],"ContextFetchBot":["partial","*",9,0]},"llms":{"status":"missing","http":404,"bytes":112402},"home":{"status":"ok","http":200,"url":"https://www.wweek.com/homepage/","html_tokens":47076,"markdown_tokens":19,"pruned_tokens":19}},
{"rank":9648,"domain":"talkshoe.com","origin":"https://talkshoe.com","robots":{"status":"ok","http":200,"url":"https://www.talkshoe.com/robots.txt","bytes":227,"groups":1},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"missing","http":404,"bytes":72168},"home":{"status":"ok","http":200,"url":"https://www.talkshoe.com/","html_tokens":29958,"markdown_tokens":828,"pruned_tokens":328}},
{"rank":9649,"domain":"fitday.com","origin":"https://fitday.com","robots":{"status":"ok","http":200,"url":"https://www.fitday.com/robots.txt","bytes":120,"groups":2},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["allowed","*",0,1],"OAI-SearchBot":["allowed","*",0,1],"ClaudeBot":["allowed","*",0,1],"Claude-User":["allowed","*",0,1],"Claude-SearchBot":["allowed","*",0,1],"anthropic-ai":["allowed","*",0,1],"PerplexityBot":["allowed","*",0,1],"Perplexity-User":["allowed","*",0,1],"Google-Extended":["allowed","*",0,1],"Applebot-Extended":["allowed","*",0,1],"CCBot":["allowed","*",0,1],"Bytespider":["allowed","*",0,1],"Amazonbot":["allowed","*",0,1],"meta-externalagent":["allowed","*",0,1],"cohere-ai":["allowed","*",0,1],"Diffbot":["allowed","*",0,1],"ContextFetchBot":["allowed","*",0,1]},"llms":{"status":"missing","http":404,"bytes":13},"home":{"status":"ok","http":200,"url":"https://www.fitday.com/","html_tokens":12358,"markdown_tokens":351,"pruned_tokens":145}},
{"rank":9650,"domain":"guim.co.uk","origin":"https://guim.co.uk","robots":{"status":"missing","http":404,"url":"https://guim.co.uk/robots.txt","bytes":9379,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":9379},"home":{"status":"error","http":200,"url":"https://guim.co.uk/","error":"unparseable HTML"}},
{"rank":9651,"domain":"abema.tv","origin":"https://abema.tv","robots":{"status":"missing","http":404,"url":"https://abema.tv/robots.txt","bytes":31529,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":31529},"home":{"status":"ok","http":200,"url":"https://abema.tv/","html_tokens":45577,"markdown_tokens":0,"pruned_tokens":0}},
{"rank":9652,"domain":"utilitydive.com","origin":"https://utilitydive.com","robots":{"status":"ok","http":200,"url":"https://www.utilitydive.com/robots.txt","bytes":4954,"groups":13},"bots":{"GPTBot":["partial","*",15,2],"ChatGPT-User":["partial","*",15,2],"OAI-SearchBot":["partial","*",15,2],"ClaudeBot":["partial","*",15,2],"Claude-User":["partial","*",15,2],"Claude-SearchBot":["partial","*",15,2],"anthropic-ai":["partial","*",15,2],"PerplexityBot":["partial","*",15,2],"Perplexity-User":["partial","*",15,2],"Google-Extended":["partial","*",15,2],"Applebot-Extended":["partial","*",15,2],"CCBot":["partial","*",15,2],"Bytespider":["partial","*",15,2],"Amazonbot":["partial","named",17,2],"meta-externalagent":["partial","*",15,2],"cohere-ai":["partial","*",15,2],"Diffbot":["partial","*",15,2],"ContextFetchBot":["partial","*",15,2]},"llms":{"status":"missing","http":404,"bytes":137875},"home":{"status":"ok","http":200,"url":"https://www.utilitydive.com/","html_tokens":72236,"markdown_tokens":1316,"pruned_tokens":137}},
{"rank":9653,"domain":"autism.org.uk","origin":"https://autism.org.uk","robots":{"status":"ok","http":200,"url":"https://www.autism.org.uk/robots.txt","bytes":204,"groups":1},"bots":{"GPTBot":["partial","*",6,0],"ChatGPT-User":["partial","*",6,0],"OAI-SearchBot":["partial","*",6,0],"ClaudeBot":["partial","*",6,0],"Claude-User":["partial","*",6,0],"Claude-SearchBot":["partial","*",6,0],"anthropic-ai":["partial","*",6,0],"PerplexityBot":["partial","*",6,0],"Perplexity-User":["partial","*",6,0],"Google-Extended":["partial","*",6,0],"Applebot-Extended":["partial","*",6,0],"CCBot":["partial","*",6,0],"Bytespider":["partial","*",6,0],"Amazonbot":["partial","*",6,0],"meta-externalagent":["partial","*",6,0],"cohere-ai":["partial","*",6,0],"Diffbot":["partial","*",6,0],"ContextFetchBot":["partial","*",6,0]},"llms":{"status":"missing","http":404,"bytes":632013},"home":{"status":"ok","http":200,"url":"https://www.autism.org.uk/","html_tokens":181536,"markdown_tokens":73,"pruned_tokens":73}},
{"rank":9654,"domain":"pcloud.link","origin":"https://pcloud.link","robots":{"status":"unreachable","http":null,"url":"https://pcloud.link/robots.txt","bytes":0,"groups":0,"error":"cannot resolve pcloud.link"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":9655,"domain":"signify.com","origin":"https://signify.com","robots":{"status":"ok","http":200,"url":"https://www.signify.com/robots.txt","bytes":9738,"groups":1},"bots":{"GPTBot":["partial","*",5,0],"ChatGPT-User":["partial","*",5,0],"OAI-SearchBot":["partial","*",5,0],"ClaudeBot":["partial","*",5,0],"Claude-User":["partial","*",5,0],"Claude-SearchBot":["partial","*",5,0],"anthropic-ai":["partial","*",5,0],"PerplexityBot":["partial","*",5,0],"Perplexity-User":["partial","*",5,0],"Google-Extended":["partial","*",5,0],"Applebot-Extended":["partial","*",5,0],"CCBot":["partial","*",5,0],"Bytespider":["partial","*",5,0],"Amazonbot":["partial","*",5,0],"meta-externalagent":["partial","*",5,0],"cohere-ai":["partial","*",5,0],"Diffbot":["partial","*",5,0],"ContextFetchBot":["partial","*",5,0]},"llms":{"status":"missing","http":404,"bytes":75},"home":{"status":"ok","http":200,"url":"https://www.signify.com/en-us","html_tokens":69115,"markdown_tokens":91,"pruned_tokens":91}},
{"rank":9656,"domain":"bjhwsj.com","origin":"https://bjhwsj.com","robots":{"status":"unreachable","http":null,"url":"https://bjhwsj.com/robots.txt","bytes":0,"groups":0,"error":"cannot resolve bjhwsj.com"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":9657,"domain":"rgj.com","origin":"https://rgj.com","robots":{"status":"ok","http":200,"url":"https://www.rgj.com/robots.txt","bytes":12639,"groups":288},"bots":{"GPTBot":["partial","named",1,4],"ChatGPT-User":["partial","named",1,4],"OAI-SearchBot":["partial","named",1,4],"ClaudeBot":["blocked","named",1,0],"Claude-User":["blocked","named",1,0],"Claude-SearchBot":["blocked","named",1,0],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["partial","named",1,4],"Perplexity-User":["blocked","named",1,0],"Google-Extended":["partial","named",1,4],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",2,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",32,0]},"llms":{"status":"http_error","http":402,"bytes":254},"home":{"status":"http_error","http":402,"url":"https://www.rgj.com/"}},
{"rank":9658,"domain":"gd-sanhe.com","origin":"http://gd-sanhe.com","robots":{"status":"missing","http":404,"url":"http://gd-sanhe.com/robots.txt","bytes":1342,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":1342},"home":{"status":"ok","http":200,"url":"http://gd-sanhe.com/","html_tokens":8096,"markdown_tokens":36,"pruned_tokens":31}},
{"rank":9659,"domain":"ramseysolutions.com","origin":"https://ramseysolutions.com","robots":{"status":"ok","http":200,"url":"https://ramseysolutions.com/robots.txt","bytes":482,"groups":1},"bots":{"GPTBot":["partial","*",13,2],"ChatGPT-User":["partial","*",13,2],"OAI-SearchBot":["partial","*",13,2],"ClaudeBot":["partial","*",13,2],"Claude-User":["partial","*",13,2],"Claude-SearchBot":["partial","*",13,2],"anthropic-ai":["partial","*",13,2],"PerplexityBot":["partial","*",13,2],"Perplexity-User":["partial","*",13,2],"Google-Extended":["partial","*",13,2],"Applebot-Extended":["partial","*",13,2],"CCBot":["partial","*",13,2],"Bytespider":["partial","*",13,2],"Amazonbot":["partial","*",13,2],"meta-externalagent":["partial","*",13,2],"cohere-ai":["partial","*",13,2],"Diffbot":["partial","*",13,2],"ContextFetchBot":["partial","*",13,2]},"llms":{"status":"missing","http":404,"bytes":381105},"home":{"status":"ok","http":200,"url":"https://www.ramseysolutions.com/","html_tokens":128772,"markdown_tokens":873,"pruned_tokens":109}},
{"rank":9660,"domain":"lovdata.no","origin":"https://lovdata.no","robots":{"status":"ok","http":200,"url":"https://lovdata.no/robots.txt","bytes":959,"groups":6},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["blocked","named",1,0],"OAI-SearchBot":["blocked","*",1,0],"ClaudeBot":["blocked","*",1,0],"Claude-User":["blocked","*",1,0],"Claude-SearchBot":["blocked","*",1,0],"anthropic-ai":["blocked","*",1,0],"PerplexityBot":["blocked","*",1,0],"Perplexity-User":["blocked","*",1,0],"Google-Extended":["blocked","*",1,0],"Applebot-Extended":["blocked","*",1,0],"CCBot":["blocked","*",1,0],"Bytespider":["blocked","*",1,0],"Amazonbot":["blocked","*",1,0],"meta-externalagent":["blocked","*",1,0],"cohere-ai":["blocked","*",1,0],"Diffbot":["blocked","*",1,0],"ContextFetchBot":["blocked","*",1,0]},"llms":{"status":"disallowed"},"home":{"status":"disallowed"}},
{"rank":9661,"domain":"pragprog.com","origin":"https://pragprog.com","robots":{"status":"missing","http":404,"url":"https://pragprog.com/robots.txt","bytes":24406,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":24406},"home":{"status":"ok","http":200,"url":"https://pragprog.com/","html_tokens":7470,"markdown_tokens":148,"pruned_tokens":148}},
{"rank":9662,"domain":"fastercapital.com","origin":"https://fastercapital.com","robots":{"status":"ok","http":200,"url":"https://fastercapital.com/robots.txt","bytes":2356,"groups":1},"bots":{"GPTBot":["partial","*",3,0],"ChatGPT-User":["partial","*",3,0],"OAI-SearchBot":["partial","*",3,0],"ClaudeBot":["partial","*",3,0],"Claude-User":["partial","*",3,0],"Claude-SearchBot":["partial","*",3,0],"anthropic-ai":["partial","*",3,0],"PerplexityBot":["partial","*",3,0],"Perplexity-User":["partial","*",3,0],"Google-Extended":["partial","*",3,0],"Applebot-Extended":["partial","*",3,0],"CCBot":["partial","*",3,0],"Bytespider":["partial","*",3,0],"Amazonbot":["partial","*",3,0],"meta-externalagent":["partial","*",3,0],"cohere-ai":["partial","*",3,0],"Diffbot":["partial","*",3,0],"ContextFetchBot":["partial","*",3,0]},"llms":{"status":"present","http":200,"bytes":2807},"home":{"status":"ok","http":200,"url":"https://fastercapital.com/","html_tokens":88508,"markdown_tokens":190,"pruned_tokens":103}},
{"rank":9663,"domain":"afar.com","origin":"https://afar.com","robots":{"status":"ok","http":200,"url":"https://www.afar.com/robots.txt","bytes":289,"groups":1},"bots":{"GPTBot":["partial","*",11,0],"ChatGPT-User":["partial","*",11,0],"OAI-SearchBot":["partial","*",11,0],"ClaudeBot":["partial","*",11,0],"Claude-User":["partial","*",11,0],"Claude-SearchBot":["partial","*",11,0],"anthropic-ai":["partial","*",11,0],"PerplexityBot":["partial","*",11,0],"Perplexity-User":["partial","*",11,0],"Google-Extended":["partial","*",11,0],"Applebot-Extended":["partial","*",11,0],"CCBot":["partial","*",11,0],"Bytespider":["partial","*",11,0],"Amazonbot":["partial","*",11,0],"meta-externalagent":["partial","*",11,0],"cohere-ai":["partial","*",11,0],"Diffbot":["partial","*",11,0],"ContextFetchBot":["partial","*",11,0]},"llms":{"status":"missing","http":404,"bytes":113780},"home":{"status":"ok","http":200,"url":"https://www.afar.com/","html_tokens":94755,"markdown_tokens":843,"pruned_tokens":159}},
{"rank":9664,"domain":"teslarati.com","origin":"https://teslarati.com","robots":{"status":"ok","http":200,"url":"https://www.teslarati.com/robots.txt","bytes":330,"groups":1},"bots":{"GPTBot":["partial","*",7,1],"ChatGPT-User":["partial","*",7,1],"OAI-SearchBot":["partial","*",7,1],"ClaudeBot":["partial","*",7,1],"Claude-User":["partial","*",7,1],"Claude-SearchBot":["partial","*",7,1],"anthropic-ai":["partial","*",7,1],"PerplexityBot":["partial","*",7,1],"Perplexity-User":["partial","*",7,1],"Google-Extended":["partial","*",7,1],"Applebot-Extended":["partial","*",7,1],"CCBot":["partial","*",7,1],"Bytespider":["partial","*",7,1],"Amazonbot":["partial","*",7,1],"meta-externalagent":["partial","*",7,1],"cohere-ai":["partial","*",7,1],"Diffbot":["partial","*",7,1],"ContextFetchBot":["partial","*",7,1]},"llms":{"status":"missing","http":404,"bytes":80390},"home":{"status":"ok","http":200,"url":"https://www.teslarati.com/","html_tokens":50206,"markdown_tokens":47,"pruned_tokens":40}},
{"rank":9665,"domain":"bz-berlin.de","origin":"https://bz-berlin.de","robots":{"status":"ok","http":200,"url":"https://www.bz-berlin.de/robots.txt","bytes":2584,"groups":2},"bots":{"GPTBot":["partial","*",9,0],"ChatGPT-User":["partial","*",9,0],"OAI-SearchBot":["partial","*",9,0],"ClaudeBot":["partial","*",9,0],"Claude-User":["partial","*",9,0],"Claude-SearchBot":["partial","*",9,0],"anthropic-ai":["partial","*",9,0],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["blocked","named",1,0],"Google-Extended":["partial","*",9,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["partial","*",9,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",9,0]},"llms":{"status":"missing","http":404,"bytes":0},"home":{"status":"ok","http":200,"url":"https://www.bz-berlin.de/","html_tokens":116287,"markdown_tokens":1032,"pruned_tokens":105}},
{"rank":9666,"domain":"lacoste.com","origin":"https://lacoste.com","robots":{"status":"refused","http":403,"url":"https://lacoste.com/robots.txt","bytes":375,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":9667,"domain":"aecom.com","origin":"https://aecom.com","robots":{"status":"refused","http":403,"url":"https://aecom.com/robots.txt","bytes":560,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":9668,"domain":"scielo.cl","origin":"https://scielo.cl","robots":{"status":"ok","http":200,"url":"https://scielo.cl/robots.txt","bytes":553,"groups":10},"bots":{"GPTBot":["blocked","*",1,0],"ChatGPT-User":["blocked","*",1,0],"OAI-SearchBot":["blocked","*",1,0],"ClaudeBot":["blocked","*",1,0],"Claude-User":["blocked","*",1,0],"Claude-SearchBot":["blocked","*",1,0],"anthropic-ai":["blocked","*",1,0],"PerplexityBot":["blocked","*",1,0],"Perplexity-User":["blocked","*",1,0],"Google-Extended":["blocked","*",1,0],"Applebot-Extended":["blocked","*",1,0],"CCBot":["blocked","*",1,0],"Bytespider":["blocked","*",1,0],"Amazonbot":["blocked","*",1,0],"meta-externalagent":["blocked","*",1,0],"cohere-ai":["blocked","*",1,0],"Diffbot":["blocked","*",1,0],"ContextFetchBot":["blocked","*",1,0]},"llms":{"status":"disallowed"},"home":{"status":"disallowed"}},
{"rank":9669,"domain":"google.tk","origin":"https://google.tk","robots":{"status":"ok","http":200,"url":"https://www.google.tk/robots.txt","bytes":6624,"groups":4},"bots":{"GPTBot":["partial","*",169,64],"ChatGPT-User":["partial","*",169,64],"OAI-SearchBot":["partial","*",169,64],"ClaudeBot":["partial","*",169,64],"Claude-User":["partial","*",169,64],"Claude-SearchBot":["partial","*",169,64],"anthropic-ai":["partial","*",169,64],"PerplexityBot":["partial","*",169,64],"Perplexity-User":["partial","*",169,64],"Google-Extended":["partial","*",169,64],"Applebot-Extended":["partial","*",169,64],"CCBot":["partial","*",169,64],"Bytespider":["partial","*",169,64],"Amazonbot":["partial","*",169,64],"meta-externalagent":["partial","*",169,64],"cohere-ai":["partial","*",169,64],"Diffbot":["partial","*",169,64],"ContextFetchBot":["partial","*",169,64]},"llms":{"status":"missing","http":404,"bytes":1569},"home":{"status":"ok","http":200,"url":"https://www.google.tk/","html_tokens":21495,"markdown_tokens":32,"pruned_tokens":32}},
{"rank":9670,"domain":"mj12bot.com","origin":"https://mj12bot.com","robots":{"status":"missing","http":404,"url":"https://mj12bot.com/robots.txt","bytes":236,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":236},"home":{"status":"ok","http":200,"url":"https://mj12bot.com/","html_tokens":3406,"markdown_tokens":1705,"pruned_tokens":189}},
{"rank":9671,"domain":"red-gate.com","origin":"https://red-gate.com","robots":{"status":"ok","http":200,"url":"https://www.red-gate.com/robots.txt","bytes":6022,"groups":1},"bots":{"GPTBot":["partial","*",153,0],"ChatGPT-User":["partial","*",153,0],"OAI-SearchBot":["partial","*",153,0],"ClaudeBot":["partial","*",153,0],"Claude-User":["partial","*",153,0],"Claude-SearchBot":["partial","*",153,0],"anthropic-ai":["partial","*",153,0],"PerplexityBot":["partial","*",153,0],"Perplexity-User":["partial","*",153,0],"Google-Extended":["partial","*",153,0],"Applebot-Extended":["partial","*",153,0],"CCBot":["partial","*",153,0],"Bytespider":["partial","*",153,0],"Amazonbot":["partial","*",153,0],"meta-externalagent":["partial","*",153,0],"cohere-ai":["partial","*",153,0],"Diffbot":["partial","*",153,0],"ContextFetchBot":["partial","*",153,0]},"llms":{"status":"present","http":200,"bytes":5724},"home":{"status":"ok","http":200,"url":"https://www.red-gate.com/","html_tokens":79268,"markdown_tokens":71,"pruned_tokens":71}},
{"rank":9672,"domain":"karelia.ru","origin":"http://karelia.ru","robots":{"status":"missing","http":404,"url":"http://karelia.ru/robots.txt","bytes":5382,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":5382},"home":{"status":"ok","http":200,"url":"http://karelia.ru/","html_tokens":100,"markdown_tokens":14,"pruned_tokens":14}},
{"rank":9673,"domain":"xfce.org","origin":"https://xfce.org","robots":{"status":"ok","http":200,"url":"https://xfce.org/robots.txt","bytes":71,"groups":1},"bots":{"GPTBot":["partial","*",3,0],"ChatGPT-User":["partial","*",3,0],"OAI-SearchBot":["partial","*",3,0],"ClaudeBot":["partial","*",3,0],"Claude-User":["partial","*",3,0],"Claude-SearchBot":["partial","*",3,0],"anthropic-ai":["partial","*",3,0],"PerplexityBot":["partial","*",3,0],"Perplexity-User":["partial","*",3,0],"Google-Extended":["partial","*",3,0],"Applebot-Extended":["partial","*",3,0],"CCBot":["partial","*",3,0],"Bytespider":["partial","*",3,0],"Amazonbot":["partial","*",3,0],"meta-externalagent":["partial","*",3,0],"cohere-ai":["partial","*",3,0],"Diffbot":["partial","*",3,0],"ContextFetchBot":["partial","*",3,0]},"llms":{"status":"missing","http":404,"bytes":5647},"home":{"status":"ok","http":200,"url":"https://xfce.org/","html_tokens":2836,"markdown_tokens":52,"pruned_tokens":52}},
{"rank":9674,"domain":"videohive.net","origin":"https://videohive.net","robots":{"status":"ok","http":200,"url":"https://videohive.net/robots.txt","bytes":781,"groups":2},"bots":{"GPTBot":["partial","*",27,0],"ChatGPT-User":["partial","*",27,0],"OAI-SearchBot":["partial","*",27,0],"ClaudeBot":["partial","*",27,0],"Claude-User":["partial","*",27,0],"Claude-SearchBot":["partial","*",27,0],"anthropic-ai":["partial","*",27,0],"PerplexityBot":["partial","*",27,0],"Perplexity-User":["partial","*",27,0],"Google-Extended":["partial","*",27,0],"Applebot-Extended":["partial","*",27,0],"CCBot":["blocked","named",1,0],"Bytespider":["partial","*",27,0],"Amazonbot":["partial","*",27,0],"meta-externalagent":["partial","*",27,0],"cohere-ai":["partial","*",27,0],"Diffbot":["partial","*",27,0],"ContextFetchBot":["partial","*",27,0]},"llms":{"status":"missing","http":404,"bytes":21},"home":{"status":"ok","http":200,"url":"https://videohive.net/","html_tokens":138052,"markdown_tokens":67,"pruned_tokens":67}},
{"rank":9675,"domain":"libertaddigital.com","origin":"https://libertaddigital.com","robots":{"status":"ok","http":200,"url":"https://www.libertaddigital.com/robots.txt","bytes":1821,"groups":1},"bots":{"GPTBot":["partial","*",32,0],"ChatGPT-User":["partial","*",32,0],"OAI-SearchBot":["partial","*",32,0],"ClaudeBot":["partial","*",32,0],"Claude-User":["partial","*",32,0],"Claude-SearchBot":["partial","*",32,0],"anthropic-ai":["partial","*",32,0],"PerplexityBot":["partial","*",32,0],"Perplexity-User":["partial","*",32,0],"Google-Extended":["partial","*",32,0],"Applebot-Extended":["partial","*",32,0],"CCBot":["partial","*",32,0],"Bytespider":["partial","*",32,0],"Amazonbot":["partial","*",32,0],"meta-externalagent":["partial","*",32,0],"cohere-ai":["partial","*",32,0],"Diffbot":["partial","*",32,0],"ContextFetchBot":["partial","*",32,0]},"llms":{"status":"missing","http":404,"bytes":404},"home":{"status":"ok","http":200,"url":"https://www.libertaddigital.com/","html_tokens":133369,"markdown_tokens":148,"pruned_tokens":148}},
{"rank":9676,"domain":"wcpo.com","origin":"https://wcpo.com","robots":{"status":"ok","http":200,"url":"https://www.wcpo.com/robots.txt","bytes":438,"groups":4},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["partial","*",6,0],"OAI-SearchBot":["partial","*",6,0],"ClaudeBot":["partial","*",6,0],"Claude-User":["partial","*",6,0],"Claude-SearchBot":["partial","*",6,0],"anthropic-ai":["partial","*",6,0],"PerplexityBot":["partial","*",6,0],"Perplexity-User":["partial","*",6,0],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["partial","*",6,0],"CCBot":["partial","*",6,0],"Bytespider":["partial","*",6,0],"Amazonbot":["partial","*",6,0],"meta-externalagent":["partial","*",6,0],"cohere-ai":["partial","*",6,0],"Diffbot":["partial","*",6,0],"ContextFetchBot":["partial","*",6,0]},"llms":{"status":"missing","http":404,"bytes":73240},"home":{"status":"ok","http":200,"url":"https://www.wcpo.com/","html_tokens":66618,"markdown_tokens":17,"pruned_tokens":17}},
{"rank":9677,"domain":"shopbop.com","origin":"https://shopbop.com","robots":{"status":"ok","http":200,"url":"https://www.shopbop.com/robots.txt","bytes":1608,"groups":1},"bots":{"GPTBot":["partial","*",60,0],"ChatGPT-User":["partial","*",60,0],"OAI-SearchBot":["partial","*",60,0],"ClaudeBot":["partial","*",60,0],"Claude-User":["partial","*",60,0],"Claude-SearchBot":["partial","*",60,0],"anthropic-ai":["partial","*",60,0],"PerplexityBot":["partial","*",60,0],"Perplexity-User":["partial","*",60,0],"Google-Extended":["partial","*",60,0],"Applebot-Extended":["partial","*",60,0],"CCBot":["partial","*",60,0],"Bytespider":["partial","*",60,0],"Amazonbot":["partial","*",60,0],"meta-externalagent":["partial","*",60,0],"cohere-ai":["partial","*",60,0],"Diffbot":["partial","*",60,0],"ContextFetchBot":["partial","*",60,0]},"llms":{"status":"missing","http":404,"bytes":431},"home":{"status":"ok","http":200,"url":"https://www.shopbop.com/","html_tokens":173802,"markdown_tokens":18,"pruned_tokens":18}},
{"rank":9678,"domain":"polldaddy.com","origin":"https://polldaddy.com","robots":{"status":"ok","http":200,"url":"https://polldaddy.com/robots.txt","bytes":200,"groups":1},"bots":{"GPTBot":["partial","*",9,1],"ChatGPT-User":["partial","*",9,1],"OAI-SearchBot":["partial","*",9,1],"ClaudeBot":["partial","*",9,1],"Claude-User":["partial","*",9,1],"Claude-SearchBot":["partial","*",9,1],"anthropic-ai":["partial","*",9,1],"PerplexityBot":["partial","*",9,1],"Perplexity-User":["partial","*",9,1],"Google-Extended":["partial","*",9,1],"Applebot-Extended":["partial","*",9,1],"CCBot":["partial","*",9,1],"Bytespider":["partial","*",9,1],"Amazonbot":["partial","*",9,1],"meta-externalagent":["partial","*",9,1],"cohere-ai":["partial","*",9,1],"Diffbot":["partial","*",9,1],"ContextFetchBot":["partial","*",9,1]},"llms":{"status":"missing","http":404,"bytes":29194},"home":{"status":"ok","http":200,"url":"https://crowdsignal.com","html_tokens":25837,"markdown_tokens":369,"pruned_tokens":160}},
{"rank":9679,"domain":"miraheze.org","origin":"https://miraheze.org","robots":{"status":"ok","http":200,"url":"https://miraheze.org/robots.txt","bytes":42,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":146},"home":{"status":"ok","http":200,"url":"https://miraheze.org/","html_tokens":7978,"markdown_tokens":351,"pruned_tokens":123}},
{"rank":9680,"domain":"worldvision.org","origin":"https://worldvision.org","robots":{"status":"ok","http":200,"url":"https://www.worldvision.org/robots.txt","bytes":4660,"groups":1},"bots":{"GPTBot":["partial","*",148,12],"ChatGPT-User":["partial","*",148,12],"OAI-SearchBot":["partial","*",148,12],"ClaudeBot":["partial","*",148,12],"Claude-User":["partial","*",148,12],"Claude-SearchBot":["partial","*",148,12],"anthropic-ai":["partial","*",148,12],"PerplexityBot":["partial","*",148,12],"Perplexity-User":["partial","*",148,12],"Google-Extended":["partial","*",148,12],"Applebot-Extended":["partial","*",148,12],"CCBot":["partial","*",148,12],"Bytespider":["partial","*",148,12],"Amazonbot":["partial","*",148,12],"meta-externalagent":["partial","*",148,12],"cohere-ai":["partial","*",148,12],"Diffbot":["partial","*",148,12],"ContextFetchBot":["partial","*",148,12]},"llms":{"status":"missing","http":404,"bytes":216070},"home":{"status":"ok","http":200,"url":"https://www.worldvision.org/","html_tokens":95352,"markdown_tokens":1053,"pruned_tokens":101}},
{"rank":9681,"domain":"blogspot.mx","origin":"https://blogspot.mx","robots":{"status":"ok","http":200,"url":"https://blogspot.mx/robots.txt","bytes":6624,"groups":4},"bots":{"GPTBot":["partial","*",169,64],"ChatGPT-User":["partial","*",169,64],"OAI-SearchBot":["partial","*",169,64],"ClaudeBot":["partial","*",169,64],"Claude-User":["partial","*",169,64],"Claude-SearchBot":["partial","*",169,64],"anthropic-ai":["partial","*",169,64],"PerplexityBot":["partial","*",169,64],"Perplexity-User":["partial","*",169,64],"Google-Extended":["partial","*",169,64],"Applebot-Extended":["partial","*",169,64],"CCBot":["partial","*",169,64],"Bytespider":["partial","*",169,64],"Amazonbot":["partial","*",169,64],"meta-externalagent":["partial","*",169,64],"cohere-ai":["partial","*",169,64],"Diffbot":["partial","*",169,64],"ContextFetchBot":["partial","*",169,64]},"llms":{"status":"missing","http":404,"bytes":1569},"home":{"status":"ok","http":200,"url":"https://www.google.com/","html_tokens":21304,"markdown_tokens":30,"pruned_tokens":30}},
{"rank":9682,"domain":"informationisbeautiful.net","origin":"https://informationisbeautiful.net","robots":{"status":"ok","http":200,"url":"https://informationisbeautiful.net/robots.txt","bytes":507,"groups":11},"bots":{"GPTBot":["allowed","named",0,1],"ChatGPT-User":["allowed","named",0,1],"OAI-SearchBot":["allowed","named",0,1],"ClaudeBot":["allowed","named",0,1],"Claude-User":["partial","*",1,0],"Claude-SearchBot":["partial","*",1,0],"anthropic-ai":["allowed","named",0,1],"PerplexityBot":["allowed","named",0,1],"Perplexity-User":["allowed","named",0,1],"Google-Extended":["allowed","named",0,1],"Applebot-Extended":["allowed","named",0,1],"CCBot":["allowed","named",0,1],"Bytespider":["partial","*",1,0],"Amazonbot":["partial","*",1,0],"meta-externalagent":["partial","*",1,0],"cohere-ai":["partial","*",1,0],"Diffbot":["partial","*",1,0],"ContextFetchBot":["partial","*",1,0]},"llms":{"status":"missing","http":404,"bytes":20495},"home":{"status":"ok","http":200,"url":"https://informationisbeautiful.net/","html_tokens":5783,"markdown_tokens":4,"pruned_tokens":4}},
{"rank":9683,"domain":"etracker.com","origin":"https://etracker.com","robots":{"status":"ok","http":200,"url":"https://www.etracker.com/robots.txt","bytes":487,"groups":3},"bots":{"GPTBot":["partial","*",7,1],"ChatGPT-User":["partial","*",7,1],"OAI-SearchBot":["partial","*",7,1],"ClaudeBot":["partial","*",7,1],"Claude-User":["partial","*",7,1],"Claude-SearchBot":["partial","*",7,1],"anthropic-ai":["partial","*",7,1],"PerplexityBot":["partial","*",7,1],"Perplexity-User":["partial","*",7,1],"Google-Extended":["partial","*",7,1],"Applebot-Extended":["partial","*",7,1],"CCBot":["partial","*",7,1],"Bytespider":["partial","*",7,1],"Amazonbot":["partial","*",7,1],"meta-externalagent":["partial","*",7,1],"cohere-ai":["partial","*",7,1],"Diffbot":["partial","*",7,1],"ContextFetchBot":["partial","*",7,1]},"llms":{"status":"missing","http":404,"bytes":66938},"home":{"status":"ok","http":200,"url":"https://www.etracker.com/","html_tokens":33876,"markdown_tokens":1094,"pruned_tokens":122}},
{"rank":9684,"domain":"myrecipes.com","origin":"https://myrecipes.com","robots":{"status":"ok","http":200,"url":"https://www.myrecipes.com/robots.txt","bytes":3471,"groups":6},"bots":{"GPTBot":["partial","named",1,0],"ChatGPT-User":["partial","named",1,0],"OAI-SearchBot":["partial","named",1,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["blocked","named",1,0],"Claude-SearchBot":["partial","named",1,2],"anthropic-ai":["partial","named",1,2],"PerplexityBot":["partial","named",1,2],"Perplexity-User":["blocked","named",1,0],"Google-Extended":["partial","named",1,2],"Applebot-Extended":["blocked","named",1,0],"CCBot":["partial","named",1,2],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["partial","named",1,2],"cohere-ai":["partial","named",1,2],"Diffbot":["partial","*",2,0],"ContextFetchBot":["partial","*",2,0]},"llms":{"status":"missing","http":404,"bytes":150339},"home":{"status":"ok","http":200,"url":"https://www.myrecipes.com/","html_tokens":210008,"markdown_tokens":27,"pruned_tokens":27}},
{"rank":9685,"domain":"barbican.org.uk","origin":"https://barbican.org.uk","robots":{"status":"ok","http":200,"url":"https://www.barbican.org.uk/robots.txt","bytes":2227,"groups":1},"bots":{"GPTBot":["partial","*",36,22],"ChatGPT-User":["partial","*",36,22],"OAI-SearchBot":["partial","*",36,22],"ClaudeBot":["partial","*",36,22],"Claude-User":["partial","*",36,22],"Claude-SearchBot":["partial","*",36,22],"anthropic-ai":["partial","*",36,22],"PerplexityBot":["partial","*",36,22],"Perplexity-User":["partial","*",36,22],"Google-Extended":["partial","*",36,22],"Applebot-Extended":["partial","*",36,22],"CCBot":["partial","*",36,22],"Bytespider":["partial","*",36,22],"Amazonbot":["partial","*",36,22],"meta-externalagent":["partial","*",36,22],"cohere-ai":["partial","*",36,22],"Diffbot":["partial","*",36,22],"ContextFetchBot":["partial","*",36,22]},"llms":{"status":"missing","http":404,"bytes":193},"home":{"status":"ok","http":200,"url":"https://www.barbican.org.uk/","html_tokens":49085,"markdown_tokens":181,"pruned_tokens":106}},
{"rank":9686,"domain":"subscriber.politicopro.com","origin":"https://subscriber.politicopro.com","robots":{"status":"html","http":200,"url":"https://login.politicopro.com?redirect=https://subscriber.politicopro.com/","bytes":5355,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":9687,"domain":"heathrow.com","origin":"https://heathrow.com","robots":{"status":"ok","http":200,"url":"https://www.heathrow.com/robots.txt","bytes":248,"groups":1},"bots":{"GPTBot":["partial","*",6,1],"ChatGPT-User":["partial","*",6,1],"OAI-SearchBot":["partial","*",6,1],"ClaudeBot":["partial","*",6,1],"Claude-User":["partial","*",6,1],"Claude-SearchBot":["partial","*",6,1],"anthropic-ai":["partial","*",6,1],"PerplexityBot":["partial","*",6,1],"Perplexity-User":["partial","*",6,1],"Google-Extended":["partial","*",6,1],"Applebot-Extended":["partial","*",6,1],"CCBot":["partial","*",6,1],"Bytespider":["partial","*",6,1],"Amazonbot":["partial","*",6,1],"meta-externalagent":["partial","*",6,1],"cohere-ai":["partial","*",6,1],"Diffbot":["partial","*",6,1],"ContextFetchBot":["partial","*",6,1]},"llms":{"status":"missing","http":404,"bytes":1093005},"home":{"status":"ok","http":200,"url":"https://www.heathrow.com/","html_tokens":284822,"markdown_tokens":129,"pruned_tokens":116}},
{"rank":9688,"domain":"xn--xhqv9jbtpfww.cn","origin":"http://xn--xhqv9jbtpfww.cn","robots":{"status":"missing","http":404,"url":"http://xn--xhqv9jbtpfww.cn/robots.txt","bytes":5076,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":5072},"home":{"status":"ok","http":200,"url":"http://xn--xhqv9jbtpfww.cn/","html_tokens":2193,"markdown_tokens":45,"pruned_tokens":45}},
{"rank":9689,"domain":"thedailymeal.com","origin":"https://thedailymeal.com","robots":{"status":"ok","http":200,"url":"https://thedailymeal.com/robots.txt","bytes":323,"groups":2},"bots":{"GPTBot":["partial","*",7,0],"ChatGPT-User":["partial","*",7,0],"OAI-SearchBot":["partial","*",7,0],"ClaudeBot":["partial","*",7,0],"Claude-User":["partial","*",7,0],"Claude-SearchBot":["partial","*",7,0],"anthropic-ai":["partial","*",7,0],"PerplexityBot":["partial","*",7,0],"Perplexity-User":["partial","*",7,0],"Google-Extended":["partial","*",7,0],"Applebot-Extended":["partial","*",7,0],"CCBot":["partial","*",7,0],"Bytespider":["partial","*",7,0],"Amazonbot":["partial","*",7,0],"meta-externalagent":["partial","*",7,0],"cohere-ai":["partial","*",7,0],"Diffbot":["partial","*",7,0],"ContextFetchBot":["partial","*",7,0]},"llms":{"status":"html","http":200,"bytes":96683},"home":{"status":"ok","http":200,"url":"https://www.thedailymeal.com/","html_tokens":24170,"markdown_tokens":1161,"pruned_tokens":168}},
{"rank":9690,"domain":"applovin.com","origin":"https://applovin.com","robots":{"status":"ok","http":200,"url":"https://applovin.com/robots.txt","bytes":218,"groups":1},"bots":{"GPTBot":["partial","*",3,1],"ChatGPT-User":["partial","*",3,1],"OAI-SearchBot":["partial","*",3,1],"ClaudeBot":["partial","*",3,1],"Claude-User":["partial","*",3,1],"Claude-SearchBot":["partial","*",3,1],"anthropic-ai":["partial","*",3,1],"PerplexityBot":["partial","*",3,1],"Perplexity-User":["partial","*",3,1],"Google-Extended":["partial","*",3,1],"Applebot-Extended":["partial","*",3,1],"CCBot":["partial","*",3,1],"Bytespider":["partial","*",3,1],"Amazonbot":["partial","*",3,1],"meta-externalagent":["partial","*",3,1],"cohere-ai":["partial","*",3,1],"Diffbot":["partial","*",3,1],"ContextFetchBot":["partial","*",3,1]},"llms":{"status":"html","http":200,"bytes":401824},"home":{"status":"ok","http":200,"url":"https://applovin.com/en","html_tokens":100244,"markdown_tokens":102,"pruned_tokens":48}},
{"rank":9691,"domain":"jamf.com","origin":"https://jamf.com","robots":{"status":"ok","http":200,"url":"https://www.jamf.com/robots.txt","bytes":412,"groups":2},"bots":{"GPTBot":["partial","*",1,0],"ChatGPT-User":["partial","*",1,0],"OAI-SearchBot":["partial","*",1,0],"ClaudeBot":["partial","*",1,0],"Claude-User":["partial","*",1,0],"Claude-SearchBot":["partial","*",1,0],"anthropic-ai":["partial","*",1,0],"PerplexityBot":["partial","*",1,0],"Perplexity-User":["partial","*",1,0],"Google-Extended":["partial","*",1,0],"Applebot-Extended":["partial","*",1,0],"CCBot":["partial","*",1,0],"Bytespider":["partial","*",1,0],"Amazonbot":["partial","*",1,0],"meta-externalagent":["partial","*",1,0],"cohere-ai":["partial","*",1,0],"Diffbot":["partial","*",1,0],"ContextFetchBot":["partial","*",1,0]},"llms":{"status":"missing","http":404,"bytes":123702},"home":{"status":"ok","http":200,"url":"https://www.jamf.com/","html_tokens":47606,"markdown_tokens":264,"pruned_tokens":210}},
{"rank":9692,"domain":"frieze.com","origin":"https://frieze.com","robots":{"status":"ok","http":200,"url":"https://www.frieze.com/robots.txt","bytes":81,"groups":1},"bots":{"GPTBot":["partial","*",3,0],"ChatGPT-User":["partial","*",3,0],"OAI-SearchBot":["partial","*",3,0],"ClaudeBot":["partial","*",3,0],"Claude-User":["partial","*",3,0],"Claude-SearchBot":["partial","*",3,0],"anthropic-ai":["partial","*",3,0],"PerplexityBot":["partial","*",3,0],"Perplexity-User":["partial","*",3,0],"Google-Extended":["partial","*",3,0],"Applebot-Extended":["partial","*",3,0],"CCBot":["partial","*",3,0],"Bytespider":["partial","*",3,0],"Amazonbot":["partial","*",3,0],"meta-externalagent":["partial","*",3,0],"cohere-ai":["partial","*",3,0],"Diffbot":["partial","*",3,0],"ContextFetchBot":["partial","*",3,0]},"llms":{"status":"missing","http":404,"bytes":21891},"home":{"status":"http_error","http":403,"url":"https://www.frieze.com/"}},
{"rank":9693,"domain":"deutschlandradio.de","origin":"https://deutschlandradio.de","robots":{"status":"ok","http":200,"url":"https://www.deutschlandradio.de/robots.txt","bytes":408,"groups":1},"bots":{"GPTBot":["partial","*",2,0],"ChatGPT-User":["partial","*",2,0],"OAI-SearchBot":["partial","*",2,0],"ClaudeBot":["partial","*",2,0],"Claude-User":["partial","*",2,0],"Claude-SearchBot":["partial","*",2,0],"anthropic-ai":["partial","*",2,0],"PerplexityBot":["partial","*",2,0],"Perplexity-User":["partial","*",2,0],"Google-Extended":["partial","*",2,0],"Applebot-Extended":["partial","*",2,0],"CCBot":["partial","*",2,0],"Bytespider":["partial","*",2,0],"Amazonbot":["partial","*",2,0],"meta-externalagent":["partial","*",2,0],"cohere-ai":["partial","*",2,0],"Diffbot":["partial","*",2,0],"ContextFetchBot":["partial","*",2,0]},"llms":{"status":"missing","http":404,"bytes":152486},"home":{"status":"ok","http":200,"url":"https://www.deutschlandradio.de/","html_tokens":129174,"markdown_tokens":63,"pruned_tokens":63}},
{"rank":9694,"domain":"apollo.io","origin":"https://apollo.io","robots":{"status":"ok","http":200,"url":"https://www.apollo.io/robots.txt","bytes":1685,"groups":29},"bots":{"GPTBot":["allowed","named",0,1],"ChatGPT-User":["allowed","named",0,1],"OAI-SearchBot":["allowed","named",0,1],"ClaudeBot":["allowed","named",0,1],"Claude-User":["partial","*",11,1],"Claude-SearchBot":["partial","*",11,1],"anthropic-ai":["allowed","named",0,1],"PerplexityBot":["allowed","named",0,1],"Perplexity-User":["partial","*",11,1],"Google-Extended":["allowed","named",0,1],"Applebot-Extended":["partial","*",11,1],"CCBot":["allowed","named",0,1],"Bytespider":["partial","*",11,1],"Amazonbot":["partial","*",11,1],"meta-externalagent":["partial","*",11,1],"cohere-ai":["allowed","named",0,1],"Diffbot":["partial","*",11,1],"ContextFetchBot":["partial","*",11,1]},"llms":{"status":"present","http":200,"bytes":37764},"home":{"status":"ok","http":200,"url":"https://www.apollo.io/","html_tokens":165315,"markdown_tokens":512,"pruned_tokens":219}},
{"rank":9695,"domain":"nation.africa","origin":"https://nation.africa","robots":{"status":"refused","http":403,"url":"https://nation.africa/robots.txt","bytes":4907,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":9696,"domain":"get-admission.to","origin":"https://get-admission.to","robots":{"status":"unreachable","http":null,"url":"https://get-admission.to/robots.txt","bytes":0,"groups":0,"error":"cannot resolve get-admission.to"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":9697,"domain":"penguin.de","origin":"https://penguin.de","robots":{"status":"ok","http":200,"url":"https://www.penguin.de/robots.txt","bytes":2250,"groups":38},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["partial","named",2,1],"OAI-SearchBot":["partial","named",2,1],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",6,1],"Claude-SearchBot":["partial","*",6,1],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["partial","named",2,1],"Perplexity-User":["partial","named",2,1],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["partial","*",6,1],"CCBot":["blocked","named",1,0],"Bytespider":["partial","*",6,1],"Amazonbot":["partial","named",2,1],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",6,1]},"llms":{"status":"missing","http":404,"bytes":79054},"home":{"status":"ok","http":200,"url":"https://www.penguin.de/","html_tokens":257861,"markdown_tokens":264,"pruned_tokens":159}},
{"rank":9698,"domain":"toledoblade.com","origin":"https://toledoblade.com","robots":{"status":"ok","http":200,"url":"https://www.toledoblade.com/robots.txt","bytes":716,"groups":3},"bots":{"GPTBot":["partial","*",15,0],"ChatGPT-User":["partial","*",15,0],"OAI-SearchBot":["partial","*",15,0],"ClaudeBot":["partial","*",15,0],"Claude-User":["partial","*",15,0],"Claude-SearchBot":["partial","*",15,0],"anthropic-ai":["partial","*",15,0],"PerplexityBot":["partial","*",15,0],"Perplexity-User":["partial","*",15,0],"Google-Extended":["partial","*",15,0],"Applebot-Extended":["partial","*",15,0],"CCBot":["partial","*",15,0],"Bytespider":["partial","*",15,0],"Amazonbot":["partial","*",15,0],"meta-externalagent":["partial","*",15,0],"cohere-ai":["partial","*",15,0],"Diffbot":["partial","*",15,0],"ContextFetchBot":["partial","*",15,0]},"llms":{"status":"html","http":200,"bytes":54942},"home":{"status":"ok","http":200,"url":"https://www.toledoblade.com/","html_tokens":24174,"markdown_tokens":353,"pruned_tokens":173}},
{"rank":9699,"domain":"scot.nhs.uk","origin":"https://scot.nhs.uk","robots":{"status":"ok","http":200,"url":"https://www.scot.nhs.uk/robots.txt","bytes":449,"groups":1},"bots":{"GPTBot":["partial","*",12,0],"ChatGPT-User":["partial","*",12,0],"OAI-SearchBot":["partial","*",12,0],"ClaudeBot":["partial","*",12,0],"Claude-User":["partial","*",12,0],"Claude-SearchBot":["partial","*",12,0],"anthropic-ai":["partial","*",12,0],"PerplexityBot":["partial","*",12,0],"Perplexity-User":["partial","*",12,0],"Google-Extended":["partial","*",12,0],"Applebot-Extended":["partial","*",12,0],"CCBot":["partial","*",12,0],"Bytespider":["partial","*",12,0],"Amazonbot":["partial","*",12,0],"meta-externalagent":["partial","*",12,0],"cohere-ai":["partial","*",12,0],"Diffbot":["partial","*",12,0],"ContextFetchBot":["partial","*",12,0]},"llms":{"status":"missing","http":404,"bytes":103994},"home":{"status":"error","error":"unsupported content encoding br"}},
{"rank":9700,"domain":"uwindsor.ca","origin":"https://uwindsor.ca","robots":{"status":"ok","http":200,"url":"https://www.uwindsor.ca/robots.txt","bytes":2240,"groups":1},"bots":{"GPTBot":["partial","*",38,0],"ChatGPT-User":["partial","*",38,0],"OAI-SearchBot":["partial","*",38,0],"ClaudeBot":["partial","*",38,0],"Claude-User":["partial","*",38,0],"Claude-SearchBot":["partial","*",38,0],"anthropic-ai":["partial","*",38,0],"PerplexityBot":["partial","*",38,0],"Perplexity-User":["partial","*",38,0],"Google-Extended":["partial","*",38,0],"Applebot-Extended":["partial","*",38,0],"CCBot":["partial","*",38,0],"Bytespider":["partial","*",38,0],"Amazonbot":["partial","*",38,0],"meta-externalagent":["partial","*",38,0],"cohere-ai":["partial","*",38,0],"Diffbot":["partial","*",38,0],"ContextFetchBot":["partial","*",38,0]},"llms":{"status":"missing","http":404,"bytes":40480},"home":{"status":"ok","http":200,"url":"https://www.uwindsor.ca/","html_tokens":10208,"markdown_tokens":46,"pruned_tokens":46}},
{"rank":9701,"domain":"10news.com","origin":"https://10news.com","robots":{"status":"ok","http":200,"url":"https://www.10news.com/robots.txt","bytes":387,"groups":4},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["partial","*",5,0],"OAI-SearchBot":["partial","*",5,0],"ClaudeBot":["partial","*",5,0],"Claude-User":["partial","*",5,0],"Claude-SearchBot":["partial","*",5,0],"anthropic-ai":["partial","*",5,0],"PerplexityBot":["partial","*",5,0],"Perplexity-User":["partial","*",5,0],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["partial","*",5,0],"CCBot":["partial","*",5,0],"Bytespider":["partial","*",5,0],"Amazonbot":["partial","*",5,0],"meta-externalagent":["partial","*",5,0],"cohere-ai":["partial","*",5,0],"Diffbot":["partial","*",5,0],"ContextFetchBot":["partial","*",5,0]},"llms":{"status":"missing","http":404,"bytes":74484},"home":{"status":"ok","http":200,"url":"https://www.10news.com/","html_tokens":168066,"markdown_tokens":12,"pruned_tokens":12}},
{"rank":9702,"domain":"envothemes.com","origin":"https://envothemes.com","robots":{"status":"ok","http":200,"url":"https://envothemes.com/robots.txt","bytes":107,"groups":2},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"missing","http":404,"bytes":35598},"home":{"status":"ok","http":200,"url":"https://envothemes.com/","html_tokens":42832,"markdown_tokens":317,"pruned_tokens":164}},
{"rank":9703,"domain":"cssn.cn","origin":"https://cssn.cn","robots":{"status":"missing","http":404,"url":"https://cssn.cn/robots.txt","bytes":22482,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":22482},"home":{"status":"ok","http":200,"url":"https://cssn.cn/","html_tokens":61410,"markdown_tokens":98,"pruned_tokens":98}},
{"rank":9704,"domain":"tim.blog","origin":"https://tim.blog","robots":{"status":"ok","http":200,"url":"https://tim.blog/robots.txt","bytes":217,"groups":1},"bots":{"GPTBot":["partial","*",1,1],"ChatGPT-User":["partial","*",1,1],"OAI-SearchBot":["partial","*",1,1],"ClaudeBot":["partial","*",1,1],"Claude-User":["partial","*",1,1],"Claude-SearchBot":["partial","*",1,1],"anthropic-ai":["partial","*",1,1],"PerplexityBot":["partial","*",1,1],"Perplexity-User":["partial","*",1,1],"Google-Extended":["partial","*",1,1],"Applebot-Extended":["partial","*",1,1],"CCBot":["partial","*",1,1],"Bytespider":["partial","*",1,1],"Amazonbot":["partial","*",1,1],"meta-externalagent":["partial","*",1,1],"cohere-ai":["partial","*",1,1],"Diffbot":["partial","*",1,1],"ContextFetchBot":["partial","*",1,1]},"llms":{"status":"missing","http":404,"bytes":212340},"home":{"status":"ok","http":200,"url":"https://tim.blog/","html_tokens":118387,"markdown_tokens":178,"pruned_tokens":118}},
{"rank":9705,"domain":"dradio.de","origin":"https://dradio.de","robots":{"status":"ok","http":200,"url":"https://www.deutschlandradio.de/robots.txt","bytes":408,"groups":1},"bots":{"GPTBot":["partial","*",2,0],"ChatGPT-User":["partial","*",2,0],"OAI-SearchBot":["partial","*",2,0],"ClaudeBot":["partial","*",2,0],"Claude-User":["partial","*",2,0],"Claude-SearchBot":["partial","*",2,0],"anthropic-ai":["partial","*",2,0],"PerplexityBot":["partial","*",2,0],"Perplexity-User":["partial","*",2,0],"Google-Extended":["partial","*",2,0],"Applebot-Extended":["partial","*",2,0],"CCBot":["partial","*",2,0],"Bytespider":["partial","*",2,0],"Amazonbot":["partial","*",2,0],"meta-externalagent":["partial","*",2,0],"cohere-ai":["partial","*",2,0],"Diffbot":["partial","*",2,0],"ContextFetchBot":["partial","*",2,0]},"llms":{"status":"missing","http":404,"bytes":152486},"home":{"status":"ok","http":200,"url":"https://www.deutschlandradio.de/","html_tokens":129174,"markdown_tokens":63,"pruned_tokens":63}},
{"rank":9706,"domain":"ntfy.sh","origin":"https://ntfy.sh","robots":{"status":"missing","http":404,"url":"https://ntfy.sh/robots.txt","bytes":51,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":51},"home":{"status":"ok","http":200,"url":"https://ntfy.sh/","html_tokens":56016,"markdown_tokens":531,"pruned_tokens":96}},
{"rank":9707,"domain":"dyson.com","origin":"https://dyson.com","robots":{"status":"ok","http":200,"url":"https://www.dyson.com/robots.txt","bytes":559,"groups":1},"bots":{"GPTBot":["partial","*",13,1],"ChatGPT-User":["partial","*",13,1],"OAI-SearchBot":["partial","*",13,1],"ClaudeBot":["partial","*",13,1],"Claude-User":["partial","*",13,1],"Claude-SearchBot":["partial","*",13,1],"anthropic-ai":["partial","*",13,1],"PerplexityBot":["partial","*",13,1],"Perplexity-User":["partial","*",13,1],"Google-Extended":["partial","*",13,1],"Applebot-Extended":["partial","*",13,1],"CCBot":["partial","*",13,1],"Bytespider":["partial","*",13,1],"Amazonbot":["partial","*",13,1],"meta-externalagent":["partial","*",13,1],"cohere-ai":["partial","*",13,1],"Diffbot":["partial","*",13,1],"ContextFetchBot":["partial","*",13,1]},"llms":{"status":"present","http":200,"bytes":13033},"home":{"status":"http_error","http":403,"url":"https://www.dyson.com/"}},
{"rank":9708,"domain":"elgato.com","origin":"https://elgato.com","robots":{"status":"ok","http":200,"url":"https://www.elgato.com/robots.txt","bytes":2656,"groups":8},"bots":{"GPTBot":["partial","*",4,2],"ChatGPT-User":["partial","*",4,2],"OAI-SearchBot":["partial","*",4,2],"ClaudeBot":["partial","*",4,2],"Claude-User":["partial","*",4,2],"Claude-SearchBot":["partial","*",4,2],"anthropic-ai":["partial","*",4,2],"PerplexityBot":["partial","*",4,2],"Perplexity-User":["partial","*",4,2],"Google-Extended":["partial","*",4,2],"Applebot-Extended":["partial","*",4,2],"CCBot":["partial","*",4,2],"Bytespider":["partial","*",4,2],"Amazonbot":["partial","*",4,2],"meta-externalagent":["partial","*",4,2],"cohere-ai":["partial","*",4,2],"Diffbot":["partial","*",4,2],"ContextFetchBot":["partial","*",4,2]},"llms":{"status":"present","http":200,"bytes":106143},"home":{"status":"ok","http":200,"url":"https://www.elgato.com/us/en","html_tokens":166274,"markdown_tokens":46,"pruned_tokens":46}},
{"rank":9709,"domain":"pro.br","origin":"https://pro.br","robots":{"status":"unreachable","http":null,"url":"https://pro.br/robots.txt","bytes":0,"groups":0,"error":"cannot resolve pro.br"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":9710,"domain":"n9.cl","origin":"https://n9.cl","robots":{"status":"ok","http":200,"url":"https://n9.cl/robots.txt","bytes":49,"groups":1},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"missing","http":404,"bytes":0},"home":{"status":"ok","http":200,"url":"https://n9.cl/","html_tokens":10791,"markdown_tokens":818,"pruned_tokens":300}},
{"rank":9711,"domain":"urlz.fr","origin":"https://urlz.fr","robots":{"status":"ok","http":200,"url":"https://urlz.fr/robots.txt","bytes":23,"groups":1},"bots":{"GPTBot":["allowed","*",0,1],"ChatGPT-User":["allowed","*",0,1],"OAI-SearchBot":["allowed","*",0,1],"ClaudeBot":["allowed","*",0,1],"Claude-User":["allowed","*",0,1],"Claude-SearchBot":["allowed","*",0,1],"anthropic-ai":["allowed","*",0,1],"PerplexityBot":["allowed","*",0,1],"Perplexity-User":["allowed","*",0,1],"Google-Extended":["allowed","*",0,1],"Applebot-Extended":["allowed","*",0,1],"CCBot":["allowed","*",0,1],"Bytespider":["allowed","*",0,1],"Amazonbot":["allowed","*",0,1],"meta-externalagent":["allowed","*",0,1],"cohere-ai":["allowed","*",0,1],"Diffbot":["allowed","*",0,1],"ContextFetchBot":["allowed","*",0,1]},"llms":{"status":"missing","http":404,"bytes":11979},"home":{"status":"ok","http":200,"url":"https://urlz.fr/","html_tokens":2963,"markdown_tokens":105,"pruned_tokens":43}},
{"rank":9712,"domain":"univ-lille.fr","origin":"https://univ-lille.fr","robots":{"status":"html","http":200,"url":"https://www.univ-lille.fr","bytes":132697,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":9713,"domain":"ezgif.com","origin":"https://ezgif.com","robots":{"status":"ok","http":200,"url":"https://ezgif.com/robots.txt","bytes":131,"groups":1},"bots":{"GPTBot":["partial","*",4,1],"ChatGPT-User":["partial","*",4,1],"OAI-SearchBot":["partial","*",4,1],"ClaudeBot":["partial","*",4,1],"Claude-User":["partial","*",4,1],"Claude-SearchBot":["partial","*",4,1],"anthropic-ai":["partial","*",4,1],"PerplexityBot":["partial","*",4,1],"Perplexity-User":["partial","*",4,1],"Google-Extended":["partial","*",4,1],"Applebot-Extended":["partial","*",4,1],"CCBot":["partial","*",4,1],"Bytespider":["partial","*",4,1],"Amazonbot":["partial","*",4,1],"meta-externalagent":["partial","*",4,1],"cohere-ai":["partial","*",4,1],"Diffbot":["partial","*",4,1],"ContextFetchBot":["partial","*",4,1]},"llms":{"status":"missing","http":404,"bytes":146},"home":{"status":"ok","http":200,"url":"https://ezgif.com/","html_tokens":4633,"markdown_tokens":1100,"pruned_tokens":164}},
{"rank":9714,"domain":"terra.es","origin":"http://terra.es","robots":{"status":"html","http":200,"url":"https://www.terra.com/","bytes":377019,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":9715,"domain":"dffl.com.cn","origin":"https://dffl.com.cn","robots":{"status":"missing","http":404,"url":"https://dffl.com.cn/robots.txt","bytes":118,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":1163},"home":{"status":"ok","http":200,"url":"https://dffl.com.cn/","html_tokens":2555,"markdown_tokens":102,"pruned_tokens":102}},
{"rank":9716,"domain":"laliga.com","origin":"https://laliga.com","robots":{"status":"ok","http":200,"url":"https://www.laliga.com/robots.txt","bytes":1484,"groups":1},"bots":{"GPTBot":["partial","*",3,67],"ChatGPT-User":["partial","*",3,67],"OAI-SearchBot":["partial","*",3,67],"ClaudeBot":["partial","*",3,67],"Claude-User":["partial","*",3,67],"Claude-SearchBot":["partial","*",3,67],"anthropic-ai":["partial","*",3,67],"PerplexityBot":["partial","*",3,67],"Perplexity-User":["partial","*",3,67],"Google-Extended":["partial","*",3,67],"Applebot-Extended":["partial","*",3,67],"CCBot":["partial","*",3,67],"Bytespider":["partial","*",3,67],"Amazonbot":["partial","*",3,67],"meta-externalagent":["partial","*",3,67],"cohere-ai":["partial","*",3,67],"Diffbot":["partial","*",3,67],"ContextFetchBot":["partial","*",3,67]},"llms":{"status":"other_text","http":200,"bytes":4400},"home":{"status":"timeout","error":"timed out after 15 s"}},
{"rank":9717,"domain":"pokernews.com","origin":"https://pokernews.com","robots":{"status":"ok","http":200,"url":"https://www.pokernews.com/robots.txt","bytes":5466,"groups":133},"bots":{"GPTBot":["partial","*",11,2],"ChatGPT-User":["partial","*",11,2],"OAI-SearchBot":["partial","*",11,2],"ClaudeBot":["partial","*",11,2],"Claude-User":["partial","*",11,2],"Claude-SearchBot":["partial","*",11,2],"anthropic-ai":["partial","*",11,2],"PerplexityBot":["partial","*",11,2],"Perplexity-User":["partial","*",11,2],"Google-Extended":["partial","*",11,2],"Applebot-Extended":["partial","*",11,2],"CCBot":["partial","*",11,2],"Bytespider":["partial","*",11,2],"Amazonbot":["partial","*",11,2],"meta-externalagent":["partial","*",11,2],"cohere-ai":["partial","*",11,2],"Diffbot":["partial","*",11,2],"ContextFetchBot":["partial","*",11,2]},"llms":{"status":"missing","http":404,"bytes":86989},"home":{"status":"ok","http":200,"url":"https://www.pokernews.com/","html_tokens":75056,"markdown_tokens":136,"pruned_tokens":111}},
{"rank":9718,"domain":"trustedshops.com","origin":"https://trustedshops.com","robots":{"status":"ok","http":200,"url":"https://www.trustedshops.com/robots.txt","bytes":2537,"groups":4},"bots":{"GPTBot":["partial","*",21,0],"ChatGPT-User":["partial","*",21,0],"OAI-SearchBot":["partial","*",21,0],"ClaudeBot":["partial","*",21,0],"Claude-User":["partial","*",21,0],"Claude-SearchBot":["partial","*",21,0],"anthropic-ai":["partial","*",21,0],"PerplexityBot":["partial","*",21,0],"Perplexity-User":["partial","*",21,0],"Google-Extended":["partial","*",21,0],"Applebot-Extended":["partial","*",21,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["partial","*",21,0],"meta-externalagent":["partial","*",21,0],"cohere-ai":["partial","*",21,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",21,0]},"llms":{"status":"missing","http":404,"bytes":1874},"home":{"status":"ok","http":200,"url":"https://www.trustedshops.com/","html_tokens":199952,"markdown_tokens":182,"pruned_tokens":22}},
{"rank":9719,"domain":"akzonobel.com","origin":"https://akzonobel.com","robots":{"status":"ok","http":200,"url":"https://www.akzonobel.com/robots.txt","bytes":2851,"groups":1},"bots":{"GPTBot":["partial","*",54,32],"ChatGPT-User":["partial","*",54,32],"OAI-SearchBot":["partial","*",54,32],"ClaudeBot":["partial","*",54,32],"Claude-User":["partial","*",54,32],"Claude-SearchBot":["partial","*",54,32],"anthropic-ai":["partial","*",54,32],"PerplexityBot":["partial","*",54,32],"Perplexity-User":["partial","*",54,32],"Google-Extended":["partial","*",54,32],"Applebot-Extended":["partial","*",54,32],"CCBot":["partial","*",54,32],"Bytespider":["partial","*",54,32],"Amazonbot":["partial","*",54,32],"meta-externalagent":["partial","*",54,32],"cohere-ai":["partial","*",54,32],"Diffbot":["partial","*",54,32],"ContextFetchBot":["partial","*",54,32]},"llms":{"status":"missing","http":404,"bytes":95702},"home":{"status":"ok","http":200,"url":"https://www.akzonobel.com/en","html_tokens":35984,"markdown_tokens":303,"pruned_tokens":143}},
{"rank":9720,"domain":"usinenouvelle.com","origin":"https://usinenouvelle.com","robots":{"status":"refused","http":403,"url":"https://www.usinenouvelle.com/robots.txt","bytes":777,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":9721,"domain":"kagi.com","origin":"https://kagi.com","robots":{"status":"ok","http":200,"url":"https://kagi.com/robots.txt","bytes":24,"groups":1},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"missing","http":404,"bytes":9398},"home":{"status":"ok","http":200,"url":"https://kagi.com/","html_tokens":14984,"markdown_tokens":1030,"pruned_tokens":726}},
{"rank":9722,"domain":"sxjtxy.com","origin":"http://sxjtxy.com","robots":{"status":"missing","http":404,"url":"http://sxjtxy.com/robots.txt","bytes":1163,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":1163},"home":{"status":"ok","http":200,"url":"http://sxjtxy.com/","html_tokens":7667,"markdown_tokens":131,"pruned_tokens":131}},
{"rank":9723,"domain":"birdeye.com","origin":"https://birdeye.com","robots":{"status":"ok","http":200,"url":"https://birdeye.com/robots.txt","bytes":853,"groups":3},"bots":{"GPTBot":["partial","*",20,0],"ChatGPT-User":["partial","*",20,0],"OAI-SearchBot":["partial","*",20,0],"ClaudeBot":["partial","*",20,0],"Claude-User":["partial","*",20,0],"Claude-SearchBot":["partial","*",20,0],"anthropic-ai":["partial","*",20,0],"PerplexityBot":["partial","*",20,0],"Perplexity-User":["partial","*",20,0],"Google-Extended":["partial","*",20,0],"Applebot-Extended":["partial","*",20,0],"CCBot":["partial","*",20,0],"Bytespider":["partial","*",20,0],"Amazonbot":["partial","*",20,0],"meta-externalagent":["partial","*",20,0],"cohere-ai":["partial","*",20,0],"Diffbot":["partial","*",20,0],"ContextFetchBot":["partial","*",20,0]},"llms":{"status":"missing","http":404,"bytes":82218},"home":{"status":"ok","http":200,"url":"https://birdeye.com/","html_tokens":93302,"markdown_tokens":426,"pruned_tokens":84}},
{"rank":9724,"domain":"hrdive.com","origin":"https://hrdive.com","robots":{"status":"ok","http":200,"url":"https://www.hrdive.com/robots.txt","bytes":4954,"groups":13},"bots":{"GPTBot":["partial","*",15,2],"ChatGPT-User":["partial","*",15,2],"OAI-SearchBot":["partial","*",15,2],"ClaudeBot":["partial","*",15,2],"Claude-User":["partial","*",15,2],"Claude-SearchBot":["partial","*",15,2],"anthropic-ai":["partial","*",15,2],"PerplexityBot":["partial","*",15,2],"Perplexity-User":["partial","*",15,2],"Google-Extended":["partial","*",15,2],"Applebot-Extended":["partial","*",15,2],"CCBot":["partial","*",15,2],"Bytespider":["partial","*",15,2],"Amazonbot":["partial","named",17,2],"meta-externalagent":["partial","*",15,2],"cohere-ai":["partial","*",15,2],"Diffbot":["partial","*",15,2],"ContextFetchBot":["partial","*",15,2]},"llms":{"status":"missing","http":404,"bytes":137485},"home":{"status":"ok","http":200,"url":"https://www.hrdive.com/","html_tokens":71652,"markdown_tokens":688,"pruned_tokens":161}},
{"rank":9725,"domain":"stadt-zuerich.ch","origin":"https://stadt-zuerich.ch","robots":{"status":"ok","http":200,"url":"https://www.stadt-zuerich.ch/robots.txt","bytes":2675,"groups":1},"bots":{"GPTBot":["partial","*",29,1],"ChatGPT-User":["partial","*",29,1],"OAI-SearchBot":["partial","*",29,1],"ClaudeBot":["partial","*",29,1],"Claude-User":["partial","*",29,1],"Claude-SearchBot":["partial","*",29,1],"anthropic-ai":["partial","*",29,1],"PerplexityBot":["partial","*",29,1],"Perplexity-User":["partial","*",29,1],"Google-Extended":["partial","*",29,1],"Applebot-Extended":["partial","*",29,1],"CCBot":["partial","*",29,1],"Bytespider":["partial","*",29,1],"Amazonbot":["partial","*",29,1],"meta-externalagent":["partial","*",29,1],"cohere-ai":["partial","*",29,1],"Diffbot":["partial","*",29,1],"ContextFetchBot":["partial","*",29,1]},"llms":{"status":"missing","http":404,"bytes":5605},"home":{"status":"ok","http":200,"url":"https://www.stadt-zuerich.ch/de.html","html_tokens":7066,"markdown_tokens":16,"pruned_tokens":16}},
{"rank":9726,"domain":"unice.fr","origin":"https://unice.fr","robots":{"status":"ok","http":200,"url":"https://unice.fr/robots.txt","bytes":226,"groups":1},"bots":{"GPTBot":["partial","*",8,0],"ChatGPT-User":["partial","*",8,0],"OAI-SearchBot":["partial","*",8,0],"ClaudeBot":["partial","*",8,0],"Claude-User":["partial","*",8,0],"Claude-SearchBot":["partial","*",8,0],"anthropic-ai":["partial","*",8,0],"PerplexityBot":["partial","*",8,0],"Perplexity-User":["partial","*",8,0],"Google-Extended":["partial","*",8,0],"Applebot-Extended":["partial","*",8,0],"CCBot":["partial","*",8,0],"Bytespider":["partial","*",8,0],"Amazonbot":["partial","*",8,0],"meta-externalagent":["partial","*",8,0],"cohere-ai":["partial","*",8,0],"Diffbot":["partial","*",8,0],"ContextFetchBot":["partial","*",8,0]},"llms":{"status":"missing","http":404,"bytes":15229},"home":{"status":"http_error","http":404,"url":"https://unice.fr/"}},
{"rank":9727,"domain":"grisoft.com","origin":"http://grisoft.com","robots":{"status":"ok","http":200,"url":"http://www.grisoft.com/robots.txt","bytes":440,"groups":9},"bots":{"GPTBot":["allowed","named",0,1],"ChatGPT-User":["allowed","named",0,1],"OAI-SearchBot":["allowed","named",0,1],"ClaudeBot":["allowed","named",0,1],"Claude-User":["allowed","named",0,1],"Claude-SearchBot":["partial","*",1,0],"anthropic-ai":["partial","*",1,0],"PerplexityBot":["partial","named",1,1],"Perplexity-User":["allowed","named",0,1],"Google-Extended":["partial","*",1,0],"Applebot-Extended":["partial","*",1,0],"CCBot":["partial","*",1,0],"Bytespider":["partial","*",1,0],"Amazonbot":["partial","*",1,0],"meta-externalagent":["partial","*",1,0],"cohere-ai":["partial","*",1,0],"Diffbot":["partial","*",1,0],"ContextFetchBot":["partial","*",1,0]},"llms":{"status":"disallowed","error":"robots.txt of www.avast.com not readable (refused)"},"home":{"status":"disallowed","error":"robots.txt of www.avg.com not readable (refused)"}},
{"rank":9728,"domain":"qiniu.com","origin":"https://qiniu.com","robots":{"status":"ok","http":200,"url":"https://www.qiniu.com/robots.txt","bytes":595,"groups":13},"bots":{"GPTBot":["partial","*",2,0],"ChatGPT-User":["partial","*",2,0],"OAI-SearchBot":["partial","*",2,0],"ClaudeBot":["partial","*",2,0],"Claude-User":["partial","*",2,0],"Claude-SearchBot":["partial","*",2,0],"anthropic-ai":["partial","*",2,0],"PerplexityBot":["partial","*",2,0],"Perplexity-User":["partial","*",2,0],"Google-Extended":["partial","*",2,0],"Applebot-Extended":["partial","*",2,0],"CCBot":["partial","*",2,0],"Bytespider":["partial","*",2,0],"Amazonbot":["partial","*",2,0],"meta-externalagent":["partial","*",2,0],"cohere-ai":["partial","*",2,0],"Diffbot":["partial","*",2,0],"ContextFetchBot":["partial","*",2,0]},"llms":{"status":"present","http":200,"bytes":10500},"home":{"status":"not_html","error":"unsupported content type text/markdown"}},
{"rank":9729,"domain":"hospitalitynet.org","origin":"https://hospitalitynet.org","robots":{"status":"ok","http":200,"url":"https://www.hospitalitynet.org/robots.txt","bytes":4623,"groups":17},"bots":{"GPTBot":["partial","named",15,1],"ChatGPT-User":["partial","named",5,1],"OAI-SearchBot":["partial","named",5,1],"ClaudeBot":["partial","named",15,1],"Claude-User":["partial","*",20,13],"Claude-SearchBot":["partial","*",20,13],"anthropic-ai":["partial","named",5,1],"PerplexityBot":["partial","named",15,1],"Perplexity-User":["partial","named",5,1],"Google-Extended":["partial","named",15,1],"Applebot-Extended":["partial","named",5,1],"CCBot":["partial","named",5,1],"Bytespider":["partial","named",5,1],"Amazonbot":["partial","named",5,1],"meta-externalagent":["partial","named",5,1],"cohere-ai":["partial","named",5,1],"Diffbot":["partial","named",5,1],"ContextFetchBot":["partial","*",20,13]},"llms":{"status":"present","http":200,"bytes":4147},"home":{"status":"http_error","http":403,"url":"https://www.hospitalitynet.org/"}},
{"rank":9730,"domain":"bicycling.com","origin":"https://bicycling.com","robots":{"status":"ok","http":200,"url":"https://bicycling.com/robots.txt","bytes":948,"groups":3},"bots":{"GPTBot":["partial","named",31,0],"ChatGPT-User":["partial","named",31,0],"OAI-SearchBot":["partial","named",31,0],"ClaudeBot":["partial","*",19,0],"Claude-User":["partial","*",19,0],"Claude-SearchBot":["partial","*",19,0],"anthropic-ai":["partial","*",19,0],"PerplexityBot":["partial","*",19,0],"Perplexity-User":["partial","*",19,0],"Google-Extended":["partial","*",19,0],"Applebot-Extended":["partial","*",19,0],"CCBot":["partial","*",19,0],"Bytespider":["partial","*",19,0],"Amazonbot":["partial","*",19,0],"meta-externalagent":["partial","*",19,0],"cohere-ai":["partial","*",19,0],"Diffbot":["partial","*",19,0],"ContextFetchBot":["partial","*",19,0]},"llms":{"status":"missing","http":404,"bytes":84},"home":{"status":"ok","http":200,"url":"https://www.bicycling.com/","html_tokens":184464,"markdown_tokens":55,"pruned_tokens":55}},
{"rank":9731,"domain":"teamusa.org","origin":"https://teamusa.org","robots":{"status":"unreachable","http":null,"url":"https://teamusa.org/robots.txt","bytes":0,"groups":0,"error":"[SSL: CERTIFICATE_VERIFY_FAILED] certificate verify failed: certificate has expired (_ssl.c:1028)"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":9732,"domain":"poder360.com.br","origin":"https://poder360.com.br","robots":{"status":"ok","http":200,"url":"https://www.poder360.com.br/robots.txt","bytes":238,"groups":1},"bots":{"GPTBot":["partial","*",4,1],"ChatGPT-User":["partial","*",4,1],"OAI-SearchBot":["partial","*",4,1],"ClaudeBot":["partial","*",4,1],"Claude-User":["partial","*",4,1],"Claude-SearchBot":["partial","*",4,1],"anthropic-ai":["partial","*",4,1],"PerplexityBot":["partial","*",4,1],"Perplexity-User":["partial","*",4,1],"Google-Extended":["partial","*",4,1],"Applebot-Extended":["partial","*",4,1],"CCBot":["partial","*",4,1],"Bytespider":["partial","*",4,1],"Amazonbot":["partial","*",4,1],"meta-externalagent":["partial","*",4,1],"cohere-ai":["partial","*",4,1],"Diffbot":["partial","*",4,1],"ContextFetchBot":["partial","*",4,1]},"llms":{"status":"missing","http":404,"bytes":161228},"home":{"status":"ok","http":200,"url":"https://www.poder360.com.br/","html_tokens":92882,"markdown_tokens":240,"pruned_tokens":122}},
{"rank":9733,"domain":"kron4.com","origin":"https://kron4.com","robots":{"status":"ok","http":200,"url":"https://www.kron4.com/robots.txt","bytes":2308,"groups":40},"bots":{"GPTBot":["blocked","named",2,0],"ChatGPT-User":["blocked","named",1,0],"OAI-SearchBot":["blocked","named",1,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",12,1],"Claude-SearchBot":["partial","*",12,1],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["blocked","named",2,0],"Perplexity-User":["partial","*",12,1],"Google-Extended":["blocked","named",2,0],"Applebot-Extended":["blocked","named",2,0],"CCBot":["blocked","named",2,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["partial","*",12,1],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",12,1]},"llms":{"status":"http_error","http":403,"bytes":6514},"home":{"status":"http_error","http":403,"url":"https://kron4.com/"}},
{"rank":9734,"domain":"lazaworx.com","origin":"https://lazaworx.com","robots":{"status":"ok","http":200,"url":"https://lazaworx.com/robots.txt","bytes":119,"groups":1},"bots":{"GPTBot":["partial","*",1,1],"ChatGPT-User":["partial","*",1,1],"OAI-SearchBot":["partial","*",1,1],"ClaudeBot":["partial","*",1,1],"Claude-User":["partial","*",1,1],"Claude-SearchBot":["partial","*",1,1],"anthropic-ai":["partial","*",1,1],"PerplexityBot":["partial","*",1,1],"Perplexity-User":["partial","*",1,1],"Google-Extended":["partial","*",1,1],"Applebot-Extended":["partial","*",1,1],"CCBot":["partial","*",1,1],"Bytespider":["partial","*",1,1],"Amazonbot":["partial","*",1,1],"meta-externalagent":["partial","*",1,1],"cohere-ai":["partial","*",1,1],"Diffbot":["partial","*",1,1],"ContextFetchBot":["partial","*",1,1]},"llms":{"status":"missing","http":404,"bytes":49271},"home":{"status":"ok","http":200,"url":"https://lazaworx.com/","html_tokens":34861,"markdown_tokens":3444,"pruned_tokens":76}},
{"rank":9735,"domain":"propochemu.ru","origin":"https://propochemu.ru","robots":{"status":"ok","http":200,"url":"https://propochemu.ru/robots.txt","bytes":188,"groups":1},"bots":{"GPTBot":["partial","*",7,0],"ChatGPT-User":["partial","*",7,0],"OAI-SearchBot":["partial","*",7,0],"ClaudeBot":["partial","*",7,0],"Claude-User":["partial","*",7,0],"Claude-SearchBot":["partial","*",7,0],"anthropic-ai":["partial","*",7,0],"PerplexityBot":["partial","*",7,0],"Perplexity-User":["partial","*",7,0],"Google-Extended":["partial","*",7,0],"Applebot-Extended":["partial","*",7,0],"CCBot":["partial","*",7,0],"Bytespider":["partial","*",7,0],"Amazonbot":["partial","*",7,0],"meta-externalagent":["partial","*",7,0],"cohere-ai":["partial","*",7,0],"Diffbot":["partial","*",7,0],"ContextFetchBot":["partial","*",7,0]},"llms":{"status":"html","http":200,"bytes":16170},"home":{"status":"ok","http":200,"url":"https://propochemu.ru/","html_tokens":3676,"markdown_tokens":234,"pruned_tokens":59}},
{"rank":9736,"domain":"subscribepage.io","origin":"https://subscribepage.io","robots":{"status":"html","http":200,"url":"https://subscribepage.io/robots.txt","bytes":5585,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":9737,"domain":"jamestown.org","origin":"https://jamestown.org","robots":{"status":"ok","http":200,"url":"https://jamestown.org/robots.txt","bytes":8309,"groups":32},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["blocked","named",1,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"crawl_delay","error":"Crawl-delay 100 s"},"home":{"status":"crawl_delay","error":"Crawl-delay 100 s"}},
{"rank":9738,"domain":"travelchannel.com","origin":"https://travelchannel.com","robots":{"status":"ok","http":200,"url":"https://www.travelchannel.com/robots.txt","bytes":74,"groups":1},"bots":{"GPTBot":["allowed","*",0,1],"ChatGPT-User":["allowed","*",0,1],"OAI-SearchBot":["allowed","*",0,1],"ClaudeBot":["allowed","*",0,1],"Claude-User":["allowed","*",0,1],"Claude-SearchBot":["allowed","*",0,1],"anthropic-ai":["allowed","*",0,1],"PerplexityBot":["allowed","*",0,1],"Perplexity-User":["allowed","*",0,1],"Google-Extended":["allowed","*",0,1],"Applebot-Extended":["allowed","*",0,1],"CCBot":["allowed","*",0,1],"Bytespider":["allowed","*",0,1],"Amazonbot":["allowed","*",0,1],"meta-externalagent":["allowed","*",0,1],"cohere-ai":["allowed","*",0,1],"Diffbot":["allowed","*",0,1],"ContextFetchBot":["allowed","*",0,1]},"llms":{"status":"html","http":200,"bytes":73566},"home":{"status":"ok","http":200,"url":"https://www.travelchannel.com/","html_tokens":18392,"markdown_tokens":0,"pruned_tokens":0}},
{"rank":9739,"domain":"wjx.top","origin":"https://wjx.top","robots":{"status":"ok","http":200,"url":"https://wjx.top/robots.txt","bytes":1508,"groups":1},"bots":{"GPTBot":["partial","*",55,0],"ChatGPT-User":["partial","*",55,0],"OAI-SearchBot":["partial","*",55,0],"ClaudeBot":["partial","*",55,0],"Claude-User":["partial","*",55,0],"Claude-SearchBot":["partial","*",55,0],"anthropic-ai":["partial","*",55,0],"PerplexityBot":["partial","*",55,0],"Perplexity-User":["partial","*",55,0],"Google-Extended":["partial","*",55,0],"Applebot-Extended":["partial","*",55,0],"CCBot":["partial","*",55,0],"Bytespider":["partial","*",55,0],"Amazonbot":["partial","*",55,0],"meta-externalagent":["partial","*",55,0],"cohere-ai":["partial","*",55,0],"Diffbot":["partial","*",55,0],"ContextFetchBot":["partial","*",55,0]},"llms":{"status":"missing","http":404,"bytes":1183},"home":{"status":"ok","http":200,"url":"https://www.wjx.cn/","html_tokens":13981,"markdown_tokens":10,"pruned_tokens":10}},
{"rank":9740,"domain":"christianbook.com","origin":"https://christianbook.com","robots":{"status":"ok","http":200,"url":"https://www.christianbook.com/robots.txt","bytes":8512,"groups":10},"bots":{"GPTBot":["partial","*",32,0],"ChatGPT-User":["partial","*",32,0],"OAI-SearchBot":["partial","*",32,0],"ClaudeBot":["partial","*",32,0],"Claude-User":["partial","*",32,0],"Claude-SearchBot":["partial","*",32,0],"anthropic-ai":["partial","*",32,0],"PerplexityBot":["partial","*",32,0],"Perplexity-User":["partial","*",32,0],"Google-Extended":["partial","*",32,0],"Applebot-Extended":["partial","*",32,0],"CCBot":["partial","*",32,0],"Bytespider":["partial","*",32,0],"Amazonbot":["partial","*",32,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["partial","*",32,0],"Diffbot":["partial","*",32,0],"ContextFetchBot":["partial","*",32,0]},"llms":{"status":"missing","http":404,"bytes":115054},"home":{"status":"ok","http":200,"url":"https://www.christianbook.com/","html_tokens":38687,"markdown_tokens":987,"pruned_tokens":37}},
{"rank":9741,"domain":"gs1.org","origin":"https://gs1.org","robots":{"status":"refused","http":403,"url":"https://www.gs1.org/robots.txt","bytes":1484,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":9742,"domain":"cyberduck.io","origin":"https://cyberduck.io","robots":{"status":"ok","http":200,"url":"https://cyberduck.io/robots.txt","bytes":25,"groups":1},"bots":{"GPTBot":["allowed","*",0,1],"ChatGPT-User":["allowed","*",0,1],"OAI-SearchBot":["allowed","*",0,1],"ClaudeBot":["allowed","*",0,1],"Claude-User":["allowed","*",0,1],"Claude-SearchBot":["allowed","*",0,1],"anthropic-ai":["allowed","*",0,1],"PerplexityBot":["allowed","*",0,1],"Perplexity-User":["allowed","*",0,1],"Google-Extended":["allowed","*",0,1],"Applebot-Extended":["allowed","*",0,1],"CCBot":["allowed","*",0,1],"Bytespider":["allowed","*",0,1],"Amazonbot":["allowed","*",0,1],"meta-externalagent":["allowed","*",0,1],"cohere-ai":["allowed","*",0,1],"Diffbot":["allowed","*",0,1],"ContextFetchBot":["allowed","*",0,1]},"llms":{"status":"missing","http":404,"bytes":5436},"home":{"status":"ok","http":200,"url":"https://cyberduck.io/","html_tokens":6610,"markdown_tokens":1042,"pruned_tokens":39}},
{"rank":9743,"domain":"radissonblu.com","origin":"https://radissonblu.com","robots":{"status":"missing","http":404,"url":"https://radissonblu.com/robots.txt","bytes":156,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"disallowed","error":"robots.txt of www.radissonhotels.com not readable (refused)"},"home":{"status":"disallowed","error":"robots.txt of www.radissonhotels.com not readable (refused)"}},
{"rank":9744,"domain":"fox5ny.com","origin":"https://fox5ny.com","robots":{"status":"ok","http":200,"url":"https://www.fox5ny.com/robots.txt","bytes":798,"groups":7},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["partial","*",11,0],"OAI-SearchBot":["partial","*",11,0],"ClaudeBot":["partial","*",11,0],"Claude-User":["partial","*",11,0],"Claude-SearchBot":["partial","*",11,0],"anthropic-ai":["partial","*",11,0],"PerplexityBot":["partial","*",11,0],"Perplexity-User":["partial","*",11,0],"Google-Extended":["partial","*",11,0],"Applebot-Extended":["partial","*",11,0],"CCBot":["partial","*",11,0],"Bytespider":["partial","*",11,0],"Amazonbot":["partial","*",11,0],"meta-externalagent":["partial","*",11,0],"cohere-ai":["partial","*",11,0],"Diffbot":["partial","*",11,0],"ContextFetchBot":["partial","*",11,0]},"llms":{"status":"missing","http":404,"bytes":0},"home":{"status":"ok","http":200,"url":"https://www.fox5ny.com/","html_tokens":99719,"markdown_tokens":42,"pruned_tokens":42}},
{"rank":9745,"domain":"kk.org","origin":"https://kk.org","robots":{"status":"ok","http":200,"url":"https://kk.org/robots.txt","bytes":163,"groups":5},"bots":{"GPTBot":["allowed","*",0,1],"ChatGPT-User":["allowed","*",0,1],"OAI-SearchBot":["allowed","*",0,1],"ClaudeBot":["allowed","*",0,1],"Claude-User":["allowed","*",0,1],"Claude-SearchBot":["allowed","*",0,1],"anthropic-ai":["allowed","*",0,1],"PerplexityBot":["allowed","*",0,1],"Perplexity-User":["allowed","*",0,1],"Google-Extended":["allowed","*",0,1],"Applebot-Extended":["allowed","*",0,1],"CCBot":["allowed","*",0,1],"Bytespider":["allowed","*",0,1],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["allowed","*",0,1],"Diffbot":["allowed","*",0,1],"ContextFetchBot":["allowed","*",0,1]},"llms":{"status":"missing","http":404,"bytes":162},"home":{"status":"ok","http":200,"url":"https://kk.org/","html_tokens":12582,"markdown_tokens":1357,"pruned_tokens":159}},
{"rank":9746,"domain":"jdxzyy.com","origin":"https://jdxzyy.com","robots":{"status":"refused","http":403,"url":"https://jdxzyy.com/robots.txt","bytes":138,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":9747,"domain":"okinawa.jp","origin":"https://okinawa.jp","robots":{"status":"unreachable","http":null,"url":"https://okinawa.jp/robots.txt","bytes":0,"groups":0,"error":"cannot resolve okinawa.jp"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":9748,"domain":"journaldesfemmes.fr","origin":"https://journaldesfemmes.fr","robots":{"status":"ok","http":200,"url":"http://www.journaldesfemmes.com/robots.txt","bytes":1524,"groups":7},"bots":{"GPTBot":["partial","*",36,1],"ChatGPT-User":["partial","*",36,1],"OAI-SearchBot":["partial","*",36,1],"ClaudeBot":["partial","*",36,1],"Claude-User":["partial","*",36,1],"Claude-SearchBot":["partial","*",36,1],"anthropic-ai":["partial","*",36,1],"PerplexityBot":["partial","*",36,1],"Perplexity-User":["partial","*",36,1],"Google-Extended":["partial","*",36,1],"Applebot-Extended":["partial","*",36,1],"CCBot":["partial","*",36,1],"Bytespider":["partial","*",36,1],"Amazonbot":["partial","*",36,1],"meta-externalagent":["partial","*",36,1],"cohere-ai":["partial","*",36,1],"Diffbot":["partial","*",36,1],"ContextFetchBot":["partial","*",36,1]},"llms":{"status":"missing","http":404,"bytes":14527},"home":{"status":"ok","http":200,"url":"https://www.journaldesfemmes.fr/","html_tokens":76171,"markdown_tokens":226,"pruned_tokens":134}},
{"rank":9749,"domain":"t3.com","origin":"https://t3.com","robots":{"status":"ok","http":200,"url":"https://www.t3.com/robots.txt","bytes":2256,"groups":5},"bots":{"GPTBot":["partial","*",22,0],"ChatGPT-User":["partial","*",22,0],"OAI-SearchBot":["partial","*",22,0],"ClaudeBot":["partial","*",22,0],"Claude-User":["partial","*",22,0],"Claude-SearchBot":["partial","*",22,0],"anthropic-ai":["partial","*",22,0],"PerplexityBot":["partial","*",22,0],"Perplexity-User":["partial","*",22,0],"Google-Extended":["partial","*",22,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["partial","*",22,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["partial","*",22,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",22,0]},"llms":{"status":"missing","http":404,"bytes":1361869},"home":{"status":"ok","http":200,"url":"https://www.t3.com/us","html_tokens":425964,"markdown_tokens":4221,"pruned_tokens":204}},
{"rank":9750,"domain":"leetchi.com","origin":"https://leetchi.com","robots":{"status":"refused","http":403,"url":"https://leetchi.com/robots.txt","bytes":728885,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":9751,"domain":"vodafone.co.uk","origin":"https://vodafone.co.uk","robots":{"status":"ok","http":200,"url":"https://www.vodafone.co.uk/robots.txt","bytes":3121,"groups":13},"bots":{"GPTBot":["partial","*",17,0],"ChatGPT-User":["partial","*",17,0],"OAI-SearchBot":["partial","*",17,0],"ClaudeBot":["partial","*",17,0],"Claude-User":["partial","*",17,0],"Claude-SearchBot":["partial","*",17,0],"anthropic-ai":["partial","*",17,0],"PerplexityBot":["partial","*",17,0],"Perplexity-User":["partial","*",17,0],"Google-Extended":["partial","*",17,0],"Applebot-Extended":["partial","*",17,0],"CCBot":["partial","*",17,0],"Bytespider":["partial","*",17,0],"Amazonbot":["partial","*",17,0],"meta-externalagent":["partial","*",17,0],"cohere-ai":["partial","*",17,0],"Diffbot":["partial","*",17,0],"ContextFetchBot":["partial","*",17,0]},"llms":{"status":"missing","http":404,"bytes":307007},"home":{"status":"ok","http":200,"url":"https://www.vodafone.co.uk/","html_tokens":483730,"markdown_tokens":4492,"pruned_tokens":399}},
{"rank":9752,"domain":"holland.com","origin":"https://holland.com","robots":{"status":"ok","http":200,"url":"https://www.holland.com/robots.txt","bytes":264,"groups":1},"bots":{"GPTBot":["partial","*",8,1],"ChatGPT-User":["partial","*",8,1],"OAI-SearchBot":["partial","*",8,1],"ClaudeBot":["partial","*",8,1],"Claude-User":["partial","*",8,1],"Claude-SearchBot":["partial","*",8,1],"anthropic-ai":["partial","*",8,1],"PerplexityBot":["partial","*",8,1],"Perplexity-User":["partial","*",8,1],"Google-Extended":["partial","*",8,1],"Applebot-Extended":["partial","*",8,1],"CCBot":["partial","*",8,1],"Bytespider":["partial","*",8,1],"Amazonbot":["partial","*",8,1],"meta-externalagent":["partial","*",8,1],"cohere-ai":["partial","*",8,1],"Diffbot":["partial","*",8,1],"ContextFetchBot":["partial","*",8,1]},"llms":{"status":"missing","http":404,"bytes":0},"home":{"status":"ok","http":200,"url":"https://www.holland.com/","html_tokens":129,"markdown_tokens":0,"pruned_tokens":0}},
{"rank":9753,"domain":"datawrapper.de","origin":"https://datawrapper.de","robots":{"status":"ok","http":200,"url":"https://www.datawrapper.de/robots.txt","bytes":23,"groups":1},"bots":{"GPTBot":["allowed","*",0,1],"ChatGPT-User":["allowed","*",0,1],"OAI-SearchBot":["allowed","*",0,1],"ClaudeBot":["allowed","*",0,1],"Claude-User":["allowed","*",0,1],"Claude-SearchBot":["allowed","*",0,1],"anthropic-ai":["allowed","*",0,1],"PerplexityBot":["allowed","*",0,1],"Perplexity-User":["allowed","*",0,1],"Google-Extended":["allowed","*",0,1],"Applebot-Extended":["allowed","*",0,1],"CCBot":["allowed","*",0,1],"Bytespider":["allowed","*",0,1],"Amazonbot":["allowed","*",0,1],"meta-externalagent":["allowed","*",0,1],"cohere-ai":["allowed","*",0,1],"Diffbot":["allowed","*",0,1],"ContextFetchBot":["allowed","*",0,1]},"llms":{"status":"missing","http":404,"bytes":634871},"home":{"status":"ok","http":200,"url":"https://www.datawrapper.de/","html_tokens":234703,"markdown_tokens":110,"pruned_tokens":110}},
{"rank":9754,"domain":"wi-fi.org","origin":"https://wi-fi.org","robots":{"status":"ok","http":200,"url":"https://www.wi-fi.org/robots.txt","bytes":2141,"groups":1},"bots":{"GPTBot":["partial","*",35,22],"ChatGPT-User":["partial","*",35,22],"OAI-SearchBot":["partial","*",35,22],"ClaudeBot":["partial","*",35,22],"Claude-User":["partial","*",35,22],"Claude-SearchBot":["partial","*",35,22],"anthropic-ai":["partial","*",35,22],"PerplexityBot":["partial","*",35,22],"Perplexity-User":["partial","*",35,22],"Google-Extended":["partial","*",35,22],"Applebot-Extended":["partial","*",35,22],"CCBot":["partial","*",35,22],"Bytespider":["partial","*",35,22],"Amazonbot":["partial","*",35,22],"meta-externalagent":["partial","*",35,22],"cohere-ai":["partial","*",35,22],"Diffbot":["partial","*",35,22],"ContextFetchBot":["partial","*",35,22]},"llms":{"status":"present","http":200,"bytes":5846},"home":{"status":"ok","http":200,"url":"https://www.wi-fi.org/","html_tokens":43904,"markdown_tokens":207,"pruned_tokens":207}},
{"rank":9755,"domain":"readyfor.jp","origin":"https://readyfor.jp","robots":{"status":"html","http":202,"url":"https://readyfor.jp/robots.txt","bytes":2423,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":9756,"domain":"gazetadopovo.com.br","origin":"https://gazetadopovo.com.br","robots":{"status":"ok","http":200,"url":"https://www.gazetadopovo.com.br/robots.txt","bytes":2044,"groups":16},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["partial","*",31,2],"OAI-SearchBot":["partial","*",31,2],"ClaudeBot":["partial","*",31,2],"Claude-User":["partial","*",31,2],"Claude-SearchBot":["partial","*",31,2],"anthropic-ai":["partial","*",31,2],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["blocked","named",1,0],"Google-Extended":["allowed","named",0,1],"Applebot-Extended":["partial","*",31,2],"CCBot":["blocked","named",1,0],"Bytespider":["partial","*",31,2],"Amazonbot":["partial","*",31,2],"meta-externalagent":["partial","*",31,2],"cohere-ai":["partial","*",31,2],"Diffbot":["partial","*",31,2],"ContextFetchBot":["partial","*",31,2]},"llms":{"status":"other_text","http":200,"bytes":6045},"home":{"status":"ok","http":200,"url":"https://www.gazetadopovo.com.br/","html_tokens":160660,"markdown_tokens":21,"pruned_tokens":21}},
{"rank":9757,"domain":"stat.gov.pl","origin":"https://stat.gov.pl","robots":{"status":"server_error","http":500,"url":"https://stat.gov.pl/robots.txt","bytes":10158,"groups":0,"error":"HTTP 500"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":9758,"domain":"monoskop.org","origin":"https://monoskop.org","robots":{"status":"ok","http":200,"url":"https://monoskop.org/robots.txt","bytes":1781,"groups":2},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["blocked","named",1,0],"OAI-SearchBot":["blocked","named",1,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["blocked","named",1,0],"Claude-SearchBot":["blocked","named",1,0],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["blocked","named",1,0],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",4,0]},"llms":{"status":"missing","http":404,"bytes":18293},"home":{"status":"ok","http":200,"url":"https://monoskop.org/Monoskop","html_tokens":16970,"markdown_tokens":3070,"pruned_tokens":16}},
{"rank":9759,"domain":"univ-grenoble-alpes.fr","origin":"https://univ-grenoble-alpes.fr","robots":{"status":"ok","http":200,"url":"https://univ-grenoble-alpes.fr/robots.txt","bytes":270,"groups":1},"bots":{"GPTBot":["partial","*",7,0],"ChatGPT-User":["partial","*",7,0],"OAI-SearchBot":["partial","*",7,0],"ClaudeBot":["partial","*",7,0],"Claude-User":["partial","*",7,0],"Claude-SearchBot":["partial","*",7,0],"anthropic-ai":["partial","*",7,0],"PerplexityBot":["partial","*",7,0],"Perplexity-User":["partial","*",7,0],"Google-Extended":["partial","*",7,0],"Applebot-Extended":["partial","*",7,0],"CCBot":["partial","*",7,0],"Bytespider":["partial","*",7,0],"Amazonbot":["partial","*",7,0],"meta-externalagent":["partial","*",7,0],"cohere-ai":["partial","*",7,0],"Diffbot":["partial","*",7,0],"ContextFetchBot":["partial","*",7,0]},"llms":{"status":"missing","http":404,"bytes":0},"home":{"status":"http_error","http":404,"url":"https://univ-grenoble-alpes.fr/"}},
{"rank":9760,"domain":"txhykj.com","origin":"http://txhykj.com","robots":{"status":"ok","http":200,"url":"http://txhykj.com/robots.txt","bytes":249,"groups":1},"bots":{"GPTBot":["partial","*",10,0],"ChatGPT-User":["partial","*",10,0],"OAI-SearchBot":["partial","*",10,0],"ClaudeBot":["partial","*",10,0],"Claude-User":["partial","*",10,0],"Claude-SearchBot":["partial","*",10,0],"anthropic-ai":["partial","*",10,0],"PerplexityBot":["partial","*",10,0],"Perplexity-User":["partial","*",10,0],"Google-Extended":["partial","*",10,0],"Applebot-Extended":["partial","*",10,0],"CCBot":["partial","*",10,0],"Bytespider":["partial","*",10,0],"Amazonbot":["partial","*",10,0],"meta-externalagent":["partial","*",10,0],"cohere-ai":["partial","*",10,0],"Diffbot":["partial","*",10,0],"ContextFetchBot":["partial","*",10,0]},"llms":{"status":"missing","http":404,"bytes":3749},"home":{"status":"ok","http":200,"url":"http://txhykj.com/","html_tokens":4361,"markdown_tokens":39,"pruned_tokens":39}},
{"rank":9761,"domain":"mbs.jp","origin":"https://mbs.jp","robots":{"status":"ok","http":200,"url":"https://www.mbs.jp/robots.txt","bytes":1247,"groups":30},"bots":{"GPTBot":["partial","named",1,0],"ChatGPT-User":["partial","named",1,0],"OAI-SearchBot":["partial","named",1,0],"ClaudeBot":["partial","named",1,0],"Claude-User":["partial","named",1,0],"Claude-SearchBot":["partial","named",1,0],"anthropic-ai":["partial","named",1,0],"PerplexityBot":["partial","named",1,0],"Perplexity-User":["partial","named",1,0],"Google-Extended":["partial","named",1,0],"Applebot-Extended":["partial","named",1,0],"CCBot":["partial","named",1,0],"Bytespider":["partial","named",1,0],"Amazonbot":["partial","named",1,0],"meta-externalagent":["partial","named",1,0],"cohere-ai":["partial","named",1,0],"Diffbot":["partial","named",1,0],"ContextFetchBot":["partial","*",1,0]},"llms":{"status":"missing","http":404,"bytes":65473},"home":{"status":"ok","http":200,"url":"https://www.mbs.jp/","html_tokens":10631,"markdown_tokens":10,"pruned_tokens":10}},
{"rank":9762,"domain":"redcross.org.uk","origin":"https://redcross.org.uk","robots":{"status":"ok","http":200,"url":"https://www.redcross.org.uk/robots.txt","bytes":110,"groups":1},"bots":{"GPTBot":["partial","*",3,0],"ChatGPT-User":["partial","*",3,0],"OAI-SearchBot":["partial","*",3,0],"ClaudeBot":["partial","*",3,0],"Claude-User":["partial","*",3,0],"Claude-SearchBot":["partial","*",3,0],"anthropic-ai":["partial","*",3,0],"PerplexityBot":["partial","*",3,0],"Perplexity-User":["partial","*",3,0],"Google-Extended":["partial","*",3,0],"Applebot-Extended":["partial","*",3,0],"CCBot":["partial","*",3,0],"Bytespider":["partial","*",3,0],"Amazonbot":["partial","*",3,0],"meta-externalagent":["partial","*",3,0],"cohere-ai":["partial","*",3,0],"Diffbot":["partial","*",3,0],"ContextFetchBot":["partial","*",3,0]},"llms":{"status":"missing","http":404,"bytes":12312},"home":{"status":"ok","http":200,"url":"https://www.redcross.org.uk/","html_tokens":430231,"markdown_tokens":145,"pruned_tokens":36}},
{"rank":9763,"domain":"mcmaster.com","origin":"https://mcmaster.com","robots":{"status":"ok","http":200,"url":"https://www.mcmaster.com/robots.txt","bytes":1499,"groups":3},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["partial","*",25,7],"OAI-SearchBot":["partial","*",25,7],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",25,7],"Claude-SearchBot":["partial","*",25,7],"anthropic-ai":["partial","*",25,7],"PerplexityBot":["partial","*",25,7],"Perplexity-User":["partial","*",25,7],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["partial","*",25,7],"ContextFetchBot":["partial","*",25,7]},"llms":{"status":"missing","http":410,"bytes":336},"home":{"status":"ok","http":200,"url":"https://www.mcmaster.com/","html_tokens":47810,"markdown_tokens":372,"pruned_tokens":107}},
{"rank":9764,"domain":"17173.com","origin":"https://17173.com","robots":{"status":"server_error","http":567,"url":"https://www.17173.com/robots.txt","bytes":7352,"groups":0,"error":"HTTP 567"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":9765,"domain":"js-lssy.com","origin":"http://js-lssy.com","robots":{"status":"ok","http":200,"url":"http://js-lssy.com/robots.txt","bytes":485,"groups":1},"bots":{"GPTBot":["partial","*",21,0],"ChatGPT-User":["partial","*",21,0],"OAI-SearchBot":["partial","*",21,0],"ClaudeBot":["partial","*",21,0],"Claude-User":["partial","*",21,0],"Claude-SearchBot":["partial","*",21,0],"anthropic-ai":["partial","*",21,0],"PerplexityBot":["partial","*",21,0],"Perplexity-User":["partial","*",21,0],"Google-Extended":["partial","*",21,0],"Applebot-Extended":["partial","*",21,0],"CCBot":["partial","*",21,0],"Bytespider":["partial","*",21,0],"Amazonbot":["partial","*",21,0],"meta-externalagent":["partial","*",21,0],"cohere-ai":["partial","*",21,0],"Diffbot":["partial","*",21,0],"ContextFetchBot":["partial","*",21,0]},"llms":{"status":"missing","http":404,"bytes":1444},"home":{"status":"ok","http":200,"url":"http://js-lssy.com/","html_tokens":10777,"markdown_tokens":138,"pruned_tokens":138}},
{"rank":9766,"domain":"nara.jp","origin":"https://nara.jp","robots":{"status":"unreachable","http":null,"url":"https://nara.jp/robots.txt","bytes":0,"groups":0,"error":"cannot resolve nara.jp"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":9767,"domain":"usertesting.com","origin":"https://usertesting.com","robots":{"status":"ok","http":200,"url":"https://www.usertesting.com/robots.txt","bytes":2005,"groups":1},"bots":{"GPTBot":["partial","*",33,18],"ChatGPT-User":["partial","*",33,18],"OAI-SearchBot":["partial","*",33,18],"ClaudeBot":["partial","*",33,18],"Claude-User":["partial","*",33,18],"Claude-SearchBot":["partial","*",33,18],"anthropic-ai":["partial","*",33,18],"PerplexityBot":["partial","*",33,18],"Perplexity-User":["partial","*",33,18],"Google-Extended":["partial","*",33,18],"Applebot-Extended":["partial","*",33,18],"CCBot":["partial","*",33,18],"Bytespider":["partial","*",33,18],"Amazonbot":["partial","*",33,18],"meta-externalagent":["partial","*",33,18],"cohere-ai":["partial","*",33,18],"Diffbot":["partial","*",33,18],"ContextFetchBot":["partial","*",33,18]},"llms":{"status":"missing","http":404,"bytes":193},"home":{"status":"ok","http":200,"url":"https://www.usertesting.com/","html_tokens":53297,"markdown_tokens":601,"pruned_tokens":34}},
{"rank":9768,"domain":"forever21.com","origin":"https://forever21.com","robots":{"status":"ok","http":200,"url":"https://www.forever21.com/robots.txt","bytes":5462,"groups":6},"bots":{"GPTBot":["partial","*",46,0],"ChatGPT-User":["partial","*",46,0],"OAI-SearchBot":["partial","*",46,0],"ClaudeBot":["partial","*",46,0],"Claude-User":["partial","*",46,0],"Claude-SearchBot":["partial","*",46,0],"anthropic-ai":["partial","*",46,0],"PerplexityBot":["partial","*",46,0],"Perplexity-User":["partial","*",46,0],"Google-Extended":["partial","*",46,0],"Applebot-Extended":["partial","*",46,0],"CCBot":["partial","*",46,0],"Bytespider":["partial","*",46,0],"Amazonbot":["partial","*",46,0],"meta-externalagent":["partial","*",46,0],"cohere-ai":["partial","*",46,0],"Diffbot":["partial","*",46,0],"ContextFetchBot":["partial","*",46,0]},"llms":{"status":"present","http":200,"bytes":5804},"home":{"status":"ok","http":200,"url":"https://www.forever21.com/","html_tokens":129463,"markdown_tokens":4,"pruned_tokens":4}},
{"rank":9769,"domain":"topwar.ru","origin":"https://topwar.ru","robots":{"status":"ok","http":200,"url":"https://topwar.ru/robots.txt","bytes":1681,"groups":2},"bots":{"GPTBot":["partial","*",27,0],"ChatGPT-User":["partial","*",27,0],"OAI-SearchBot":["partial","*",27,0],"ClaudeBot":["partial","*",27,0],"Claude-User":["partial","*",27,0],"Claude-SearchBot":["partial","*",27,0],"anthropic-ai":["partial","*",27,0],"PerplexityBot":["partial","*",27,0],"Perplexity-User":["partial","*",27,0],"Google-Extended":["partial","*",27,0],"Applebot-Extended":["partial","*",27,0],"CCBot":["partial","*",27,0],"Bytespider":["partial","*",27,0],"Amazonbot":["partial","*",27,0],"meta-externalagent":["partial","*",27,0],"cohere-ai":["partial","*",27,0],"Diffbot":["partial","*",27,0],"ContextFetchBot":["partial","*",27,0]},"llms":{"status":"missing","http":404,"bytes":1969},"home":{"status":"ok","http":200,"url":"https://topwar.ru/","html_tokens":30841,"markdown_tokens":48,"pruned_tokens":48}},
{"rank":9770,"domain":"appsto.re","origin":"https://appsto.re","robots":{"status":"unreachable","http":null,"url":"https://appsto.re/robots.txt","bytes":0,"groups":0,"error":"cannot resolve appsto.re"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":9771,"domain":"kapwing.com","origin":"https://kapwing.com","robots":{"status":"ok","http":200,"url":"https://www.kapwing.com/robots.txt","bytes":190,"groups":2},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"html","http":200,"bytes":12077},"home":{"status":"ok","http":200,"url":"https://www.kapwing.com/","html_tokens":53084,"markdown_tokens":246,"pruned_tokens":169}},
{"rank":9772,"domain":"irishcentral.com","origin":"https://irishcentral.com","robots":{"status":"ok","http":200,"url":"https://www.irishcentral.com/robots.txt","bytes":286,"groups":1},"bots":{"GPTBot":["partial","*",1,0],"ChatGPT-User":["partial","*",1,0],"OAI-SearchBot":["partial","*",1,0],"ClaudeBot":["partial","*",1,0],"Claude-User":["partial","*",1,0],"Claude-SearchBot":["partial","*",1,0],"anthropic-ai":["partial","*",1,0],"PerplexityBot":["partial","*",1,0],"Perplexity-User":["partial","*",1,0],"Google-Extended":["partial","*",1,0],"Applebot-Extended":["partial","*",1,0],"CCBot":["partial","*",1,0],"Bytespider":["partial","*",1,0],"Amazonbot":["partial","*",1,0],"meta-externalagent":["partial","*",1,0],"cohere-ai":["partial","*",1,0],"Diffbot":["partial","*",1,0],"ContextFetchBot":["partial","*",1,0]},"llms":{"status":"http_error","http":406,"bytes":0},"home":{"status":"ok","http":200,"url":"https://www.irishcentral.com/","html_tokens":84321,"markdown_tokens":85,"pruned_tokens":85}},
{"rank":9773,"domain":"archaeology.org","origin":"https://archaeology.org","robots":{"status":"ok","http":200,"url":"https://archaeology.org/robots.txt","bytes":173,"groups":1},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"other_text","http":200,"bytes":4611},"home":{"status":"ok","http":307,"url":"https://archaeology.org/","html_tokens":306,"markdown_tokens":23,"pruned_tokens":23}},
{"rank":9774,"domain":"sortiraparis.com","origin":"https://sortiraparis.com","robots":{"status":"ok","http":200,"url":"https://www.sortiraparis.com/robots.txt","bytes":144,"groups":2},"bots":{"GPTBot":["partial","*",2,0],"ChatGPT-User":["partial","*",2,0],"OAI-SearchBot":["partial","*",2,0],"ClaudeBot":["partial","*",2,0],"Claude-User":["partial","*",2,0],"Claude-SearchBot":["partial","*",2,0],"anthropic-ai":["partial","*",2,0],"PerplexityBot":["partial","*",2,0],"Perplexity-User":["partial","*",2,0],"Google-Extended":["partial","*",2,0],"Applebot-Extended":["partial","*",2,0],"CCBot":["partial","*",2,0],"Bytespider":["partial","*",2,0],"Amazonbot":["partial","*",2,0],"meta-externalagent":["partial","*",2,0],"cohere-ai":["partial","*",2,0],"Diffbot":["partial","*",2,0],"ContextFetchBot":["partial","*",2,0]},"llms":{"status":"missing","http":404,"bytes":54023},"home":{"status":"ok","http":200,"url":"https://www.sortiraparis.com/","html_tokens":229170,"markdown_tokens":165,"pruned_tokens":58}},
{"rank":9775,"domain":"judiciary.uk","origin":"https://judiciary.uk","robots":{"status":"ok","http":200,"url":"https://judiciary.uk/robots.txt","bytes":26,"groups":1},"bots":{"GPTBot":["blocked","*",1,0],"ChatGPT-User":["blocked","*",1,0],"OAI-SearchBot":["blocked","*",1,0],"ClaudeBot":["blocked","*",1,0],"Claude-User":["blocked","*",1,0],"Claude-SearchBot":["blocked","*",1,0],"anthropic-ai":["blocked","*",1,0],"PerplexityBot":["blocked","*",1,0],"Perplexity-User":["blocked","*",1,0],"Google-Extended":["blocked","*",1,0],"Applebot-Extended":["blocked","*",1,0],"CCBot":["blocked","*",1,0],"Bytespider":["blocked","*",1,0],"Amazonbot":["blocked","*",1,0],"meta-externalagent":["blocked","*",1,0],"cohere-ai":["blocked","*",1,0],"Diffbot":["blocked","*",1,0],"ContextFetchBot":["blocked","*",1,0]},"llms":{"status":"disallowed"},"home":{"status":"disallowed"}},
{"rank":9776,"domain":"gate777casino.net","origin":"https://gate777casino.net","robots":{"status":"unreachable","http":null,"url":"https://gate777casino.net/robots.txt","bytes":0,"groups":0,"error":"ConnectError"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":9777,"domain":"douyu.com","origin":"https://douyu.com","robots":{"status":"ok","http":200,"url":"https://www.douyu.com/robots.txt","bytes":106,"groups":1},"bots":{"GPTBot":["partial","*",5,0],"ChatGPT-User":["partial","*",5,0],"OAI-SearchBot":["partial","*",5,0],"ClaudeBot":["partial","*",5,0],"Claude-User":["partial","*",5,0],"Claude-SearchBot":["partial","*",5,0],"anthropic-ai":["partial","*",5,0],"PerplexityBot":["partial","*",5,0],"Perplexity-User":["partial","*",5,0],"Google-Extended":["partial","*",5,0],"Applebot-Extended":["partial","*",5,0],"CCBot":["partial","*",5,0],"Bytespider":["partial","*",5,0],"Amazonbot":["partial","*",5,0],"meta-externalagent":["partial","*",5,0],"cohere-ai":["partial","*",5,0],"Diffbot":["partial","*",5,0],"ContextFetchBot":["partial","*",5,0]},"llms":{"status":"missing","http":404,"bytes":5239},"home":{"status":"ok","http":200,"url":"https://www.douyu.com/","html_tokens":101591,"markdown_tokens":7,"pruned_tokens":7}},
{"rank":9778,"domain":"ynet.com","origin":"https://www.ynet.com","robots":{"status":"refused","http":403,"url":"https://www.ynet.com/robots.txt","bytes":1962,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":9779,"domain":"goholp.com","origin":"https://goholp.com","robots":{"status":"missing","http":404,"url":"https://goholp.com/robots.txt","bytes":1231,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":1231},"home":{"status":"ok","http":200,"url":"https://goholp.com/","html_tokens":6620,"markdown_tokens":125,"pruned_tokens":125}},
{"rank":9780,"domain":"txxddt.com","origin":"http://txxddt.com","robots":{"status":"ok","http":200,"url":"http://txxddt.com/robots.txt","bytes":247,"groups":1},"bots":{"GPTBot":["partial","*",10,0],"ChatGPT-User":["partial","*",10,0],"OAI-SearchBot":["partial","*",10,0],"ClaudeBot":["partial","*",10,0],"Claude-User":["partial","*",10,0],"Claude-SearchBot":["partial","*",10,0],"anthropic-ai":["partial","*",10,0],"PerplexityBot":["partial","*",10,0],"Perplexity-User":["partial","*",10,0],"Google-Extended":["partial","*",10,0],"Applebot-Extended":["partial","*",10,0],"CCBot":["partial","*",10,0],"Bytespider":["partial","*",10,0],"Amazonbot":["partial","*",10,0],"meta-externalagent":["partial","*",10,0],"cohere-ai":["partial","*",10,0],"Diffbot":["partial","*",10,0],"ContextFetchBot":["partial","*",10,0]},"llms":{"status":"missing","http":404,"bytes":3693},"home":{"status":"ok","http":200,"url":"http://txxddt.com/","html_tokens":8256,"markdown_tokens":104,"pruned_tokens":104}},
{"rank":9781,"domain":"goserver.host","origin":"http://goserver.host","robots":{"status":"missing","http":404,"url":"https://webgo.de/robots.txt","bytes":146,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":331399},"home":{"status":"ok","http":200,"url":"https://webgo.de/","html_tokens":133954,"markdown_tokens":1477,"pruned_tokens":139}},
{"rank":9782,"domain":"thefader.com","origin":"https://thefader.com","robots":{"status":"ok","http":200,"url":"https://www.thefader.com/robots.txt","bytes":388,"groups":1},"bots":{"GPTBot":["partial","*",4,0],"ChatGPT-User":["partial","*",4,0],"OAI-SearchBot":["partial","*",4,0],"ClaudeBot":["partial","*",4,0],"Claude-User":["partial","*",4,0],"Claude-SearchBot":["partial","*",4,0],"anthropic-ai":["partial","*",4,0],"PerplexityBot":["partial","*",4,0],"Perplexity-User":["partial","*",4,0],"Google-Extended":["partial","*",4,0],"Applebot-Extended":["partial","*",4,0],"CCBot":["partial","*",4,0],"Bytespider":["partial","*",4,0],"Amazonbot":["partial","*",4,0],"meta-externalagent":["partial","*",4,0],"cohere-ai":["partial","*",4,0],"Diffbot":["partial","*",4,0],"ContextFetchBot":["partial","*",4,0]},"llms":{"status":"missing","http":404,"bytes":12827},"home":{"status":"ok","http":200,"url":"https://www.thefader.com/","html_tokens":14579,"markdown_tokens":60,"pruned_tokens":60}},
{"rank":9783,"domain":"yorkshirepost.co.uk","origin":"https://yorkshirepost.co.uk","robots":{"status":"ok","http":200,"url":"https://www.yorkshirepost.co.uk/robots.txt","bytes":864,"groups":2},"bots":{"GPTBot":["partial","*",3,0],"ChatGPT-User":["partial","*",3,0],"OAI-SearchBot":["partial","*",3,0],"ClaudeBot":["partial","*",3,0],"Claude-User":["partial","*",3,0],"Claude-SearchBot":["partial","*",3,0],"anthropic-ai":["partial","*",3,0],"PerplexityBot":["partial","*",3,0],"Perplexity-User":["partial","*",3,0],"Google-Extended":["partial","*",3,0],"Applebot-Extended":["partial","*",3,0],"CCBot":["partial","*",3,0],"Bytespider":["partial","*",3,0],"Amazonbot":["partial","*",3,0],"meta-externalagent":["partial","*",3,0],"cohere-ai":["partial","*",3,0],"Diffbot":["partial","*",3,0],"ContextFetchBot":["partial","*",3,0]},"llms":{"status":"http_error","http":403,"bytes":5555},"home":{"status":"http_error","http":403,"url":"https://www.yorkshirepost.co.uk/"}},
{"rank":9784,"domain":"talosintelligence.com","origin":"https://talosintelligence.com","robots":{"status":"ok","http":200,"url":"https://talosintelligence.com/robots.txt","bytes":91,"groups":1},"bots":{"GPTBot":["allowed","*",0,1],"ChatGPT-User":["allowed","*",0,1],"OAI-SearchBot":["allowed","*",0,1],"ClaudeBot":["allowed","*",0,1],"Claude-User":["allowed","*",0,1],"Claude-SearchBot":["allowed","*",0,1],"anthropic-ai":["allowed","*",0,1],"PerplexityBot":["allowed","*",0,1],"Perplexity-User":["allowed","*",0,1],"Google-Extended":["allowed","*",0,1],"Applebot-Extended":["allowed","*",0,1],"CCBot":["allowed","*",0,1],"Bytespider":["allowed","*",0,1],"Amazonbot":["allowed","*",0,1],"meta-externalagent":["allowed","*",0,1],"cohere-ai":["allowed","*",0,1],"Diffbot":["allowed","*",0,1],"ContextFetchBot":["allowed","*",0,1]},"llms":{"status":"missing","http":404,"bytes":3865},"home":{"status":"ok","http":200,"url":"https://talosintelligence.com/","html_tokens":22465,"markdown_tokens":328,"pruned_tokens":166}},
{"rank":9785,"domain":"rentalcars.com","origin":"https://rentalcars.com","robots":{"status":"ok","http":200,"url":"https://www.rentalcars.com/robots.txt","bytes":3542,"groups":9},"bots":{"GPTBot":["partial","*",105,0],"ChatGPT-User":["partial","*",105,0],"OAI-SearchBot":["partial","*",105,0],"ClaudeBot":["partial","*",105,0],"Claude-User":["partial","*",105,0],"Claude-SearchBot":["partial","*",105,0],"anthropic-ai":["partial","*",105,0],"PerplexityBot":["partial","*",105,0],"Perplexity-User":["partial","*",105,0],"Google-Extended":["partial","*",105,0],"Applebot-Extended":["partial","*",105,0],"CCBot":["partial","*",105,0],"Bytespider":["partial","*",105,0],"Amazonbot":["partial","*",105,0],"meta-externalagent":["partial","*",105,0],"cohere-ai":["partial","*",105,0],"Diffbot":["partial","*",105,0],"ContextFetchBot":["partial","*",105,0]},"llms":{"status":"missing","http":404,"bytes":135129},"home":{"status":"ok","http":200,"url":"https://www.rentalcars.com/","html_tokens":91770,"markdown_tokens":409,"pruned_tokens":178}},
{"rank":9786,"domain":"pnp.ru","origin":"https://pnp.ru","robots":{"status":"ok","http":200,"url":"https://www.pnp.ru/robots.txt","bytes":1406,"groups":2},"bots":{"GPTBot":["partial","*",12,0],"ChatGPT-User":["partial","*",12,0],"OAI-SearchBot":["partial","*",12,0],"ClaudeBot":["partial","*",12,0],"Claude-User":["partial","*",12,0],"Claude-SearchBot":["partial","*",12,0],"anthropic-ai":["partial","*",12,0],"PerplexityBot":["partial","*",12,0],"Perplexity-User":["partial","*",12,0],"Google-Extended":["partial","*",12,0],"Applebot-Extended":["partial","*",12,0],"CCBot":["partial","*",12,0],"Bytespider":["partial","*",12,0],"Amazonbot":["partial","*",12,0],"meta-externalagent":["partial","*",12,0],"cohere-ai":["partial","*",12,0],"Diffbot":["partial","*",12,0],"ContextFetchBot":["partial","*",12,0]},"llms":{"status":"missing","http":404,"bytes":32717},"home":{"status":"ok","http":200,"url":"https://www.pnp.ru/","html_tokens":15074,"markdown_tokens":42,"pruned_tokens":42}},
{"rank":9787,"domain":"idf.org","origin":"https://idf.org","robots":{"status":"ok","http":200,"url":"https://idf.org/robots.txt","bytes":587,"groups":2},"bots":{"GPTBot":["partial","*",8,1],"ChatGPT-User":["partial","*",8,1],"OAI-SearchBot":["partial","*",8,1],"ClaudeBot":["partial","*",8,1],"Claude-User":["partial","*",8,1],"Claude-SearchBot":["partial","*",8,1],"anthropic-ai":["partial","*",8,1],"PerplexityBot":["partial","*",8,1],"Perplexity-User":["partial","*",8,1],"Google-Extended":["partial","*",8,1],"Applebot-Extended":["partial","*",8,1],"CCBot":["partial","*",8,1],"Bytespider":["partial","*",8,1],"Amazonbot":["partial","*",8,1],"meta-externalagent":["partial","*",8,1],"cohere-ai":["partial","*",8,1],"Diffbot":["partial","*",8,1],"ContextFetchBot":["partial","*",8,1]},"llms":{"status":"missing","http":404,"bytes":220531},"home":{"status":"ok","http":200,"url":"https://idf.org/","html_tokens":73790,"markdown_tokens":98,"pruned_tokens":85}},
{"rank":9788,"domain":"indelible.org","origin":"https://www.indelible.org","robots":{"status":"ok","http":200,"url":"https://www.indelible.org/robots.txt","bytes":47,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":9379},"home":{"status":"ok","http":200,"url":"https://www.indelible.org/","html_tokens":878,"markdown_tokens":115,"pruned_tokens":115}},
{"rank":9789,"domain":"jyjinsuisy.com","origin":"https://jyjinsuisy.com","robots":{"status":"ok","http":200,"url":"https://jyjinsuisy.com/robots.txt","bytes":485,"groups":1},"bots":{"GPTBot":["partial","*",21,0],"ChatGPT-User":["partial","*",21,0],"OAI-SearchBot":["partial","*",21,0],"ClaudeBot":["partial","*",21,0],"Claude-User":["partial","*",21,0],"Claude-SearchBot":["partial","*",21,0],"anthropic-ai":["partial","*",21,0],"PerplexityBot":["partial","*",21,0],"Perplexity-User":["partial","*",21,0],"Google-Extended":["partial","*",21,0],"Applebot-Extended":["partial","*",21,0],"CCBot":["partial","*",21,0],"Bytespider":["partial","*",21,0],"Amazonbot":["partial","*",21,0],"meta-externalagent":["partial","*",21,0],"cohere-ai":["partial","*",21,0],"Diffbot":["partial","*",21,0],"ContextFetchBot":["partial","*",21,0]},"llms":{"status":"missing","http":404,"bytes":4732},"home":{"status":"ok","http":200,"url":"https://jyjinsuisy.com/","html_tokens":16194,"markdown_tokens":108,"pruned_tokens":108}},
{"rank":9790,"domain":"tzcxhr.cn","origin":"http://tzcxhr.cn","robots":{"status":"ok","http":200,"url":"http://tzcxhr.cn/robots.txt","bytes":247,"groups":1},"bots":{"GPTBot":["partial","*",10,0],"ChatGPT-User":["partial","*",10,0],"OAI-SearchBot":["partial","*",10,0],"ClaudeBot":["partial","*",10,0],"Claude-User":["partial","*",10,0],"Claude-SearchBot":["partial","*",10,0],"anthropic-ai":["partial","*",10,0],"PerplexityBot":["partial","*",10,0],"Perplexity-User":["partial","*",10,0],"Google-Extended":["partial","*",10,0],"Applebot-Extended":["partial","*",10,0],"CCBot":["partial","*",10,0],"Bytespider":["partial","*",10,0],"Amazonbot":["partial","*",10,0],"meta-externalagent":["partial","*",10,0],"cohere-ai":["partial","*",10,0],"Diffbot":["partial","*",10,0],"ContextFetchBot":["partial","*",10,0]},"llms":{"status":"html","http":200,"bytes":67},"home":{"status":"ok","http":200,"url":"http://tzcxhr.cn/","html_tokens":7002,"markdown_tokens":124,"pruned_tokens":124}},
{"rank":9791,"domain":"gasbuddy.com","origin":"https://gasbuddy.com","robots":{"status":"ok","http":200,"url":"https://www.gasbuddy.com/robots.txt","bytes":6883,"groups":111},"bots":{"GPTBot":["partial","*",16,0],"ChatGPT-User":["partial","*",16,0],"OAI-SearchBot":["partial","*",16,0],"ClaudeBot":["partial","*",16,0],"Claude-User":["partial","*",16,0],"Claude-SearchBot":["partial","*",16,0],"anthropic-ai":["partial","*",16,0],"PerplexityBot":["partial","*",16,0],"Perplexity-User":["partial","*",16,0],"Google-Extended":["partial","*",16,0],"Applebot-Extended":["partial","*",16,0],"CCBot":["blocked","named",1,0],"Bytespider":["partial","*",16,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["partial","*",16,0],"cohere-ai":["partial","*",16,0],"Diffbot":["partial","*",16,0],"ContextFetchBot":["partial","*",16,0]},"llms":{"status":"missing","http":404,"bytes":1245},"home":{"status":"ok","http":200,"url":"https://www.gasbuddy.com/","html_tokens":27682,"markdown_tokens":75,"pruned_tokens":75}},
{"rank":9792,"domain":"undrr.org","origin":"https://undrr.org","robots":{"status":"ok","http":200,"url":"https://www.undrr.org/robots.txt","bytes":2157,"groups":1},"bots":{"GPTBot":["partial","*",35,22],"ChatGPT-User":["partial","*",35,22],"OAI-SearchBot":["partial","*",35,22],"ClaudeBot":["partial","*",35,22],"Claude-User":["partial","*",35,22],"Claude-SearchBot":["partial","*",35,22],"anthropic-ai":["partial","*",35,22],"PerplexityBot":["partial","*",35,22],"Perplexity-User":["partial","*",35,22],"Google-Extended":["partial","*",35,22],"Applebot-Extended":["partial","*",35,22],"CCBot":["partial","*",35,22],"Bytespider":["partial","*",35,22],"Amazonbot":["partial","*",35,22],"meta-externalagent":["partial","*",35,22],"cohere-ai":["partial","*",35,22],"Diffbot":["partial","*",35,22],"ContextFetchBot":["partial","*",35,22]},"llms":{"status":"missing","http":404,"bytes":5451},"home":{"status":"http_error","http":403,"url":"https://undrr.org/"}},
{"rank":9793,"domain":"notaires.fr","origin":"https://notaires.fr","robots":{"status":"ok","http":200,"url":"https://www.notaires.fr/robots.txt","bytes":1981,"groups":1},"bots":{"GPTBot":["partial","*",31,18],"ChatGPT-User":["partial","*",31,18],"OAI-SearchBot":["partial","*",31,18],"ClaudeBot":["partial","*",31,18],"Claude-User":["partial","*",31,18],"Claude-SearchBot":["partial","*",31,18],"anthropic-ai":["partial","*",31,18],"PerplexityBot":["partial","*",31,18],"Perplexity-User":["partial","*",31,18],"Google-Extended":["partial","*",31,18],"Applebot-Extended":["partial","*",31,18],"CCBot":["partial","*",31,18],"Bytespider":["partial","*",31,18],"Amazonbot":["partial","*",31,18],"meta-externalagent":["partial","*",31,18],"cohere-ai":["partial","*",31,18],"Diffbot":["partial","*",31,18],"ContextFetchBot":["partial","*",31,18]},"llms":{"status":"missing","http":404,"bytes":323},"home":{"status":"ok","http":200,"url":"https://www.notaires.fr/fr","html_tokens":41278,"markdown_tokens":244,"pruned_tokens":74}},
{"rank":9794,"domain":"philips-hue.com","origin":"https://philips-hue.com","robots":{"status":"ok","http":200,"url":"https://www.philips-hue.com/robots.txt","bytes":5895,"groups":5},"bots":{"GPTBot":["partial","*",51,0],"ChatGPT-User":["partial","*",51,0],"OAI-SearchBot":["partial","*",51,0],"ClaudeBot":["partial","*",51,0],"Claude-User":["partial","*",51,0],"Claude-SearchBot":["partial","*",51,0],"anthropic-ai":["partial","*",51,0],"PerplexityBot":["partial","*",51,0],"Perplexity-User":["partial","*",51,0],"Google-Extended":["partial","*",51,0],"Applebot-Extended":["partial","*",51,0],"CCBot":["partial","*",51,0],"Bytespider":["partial","*",51,0],"Amazonbot":["partial","*",51,0],"meta-externalagent":["partial","*",51,0],"cohere-ai":["partial","*",51,0],"Diffbot":["partial","*",51,0],"ContextFetchBot":["partial","*",51,0]},"llms":{"status":"missing","http":404,"bytes":28110},"home":{"status":"ok","http":200,"url":"https://www.philips-hue.com/en-us","html_tokens":145099,"markdown_tokens":69,"pruned_tokens":50}},
{"rank":9795,"domain":"education.gov.uk","origin":"https://education.gov.uk","robots":{"status":"html","http":200,"url":"https://www.gov.uk/government/organisations/department-for-education","bytes":134531,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":9796,"domain":"camcom.it","origin":"https://www.camcom.it","robots":{"status":"html","http":200,"url":"https://www.unioncamere.gov.it/sistema-camerale","bytes":88642,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":9797,"domain":"business-gazeta.ru","origin":"https://business-gazeta.ru","robots":{"status":"ok","http":200,"url":"https://www.business-gazeta.ru/robots.txt","bytes":752,"groups":2},"bots":{"GPTBot":["partial","*",10,0],"ChatGPT-User":["partial","*",10,0],"OAI-SearchBot":["partial","*",10,0],"ClaudeBot":["partial","*",10,0],"Claude-User":["partial","*",10,0],"Claude-SearchBot":["partial","*",10,0],"anthropic-ai":["partial","*",10,0],"PerplexityBot":["partial","*",10,0],"Perplexity-User":["partial","*",10,0],"Google-Extended":["partial","*",10,0],"Applebot-Extended":["partial","*",10,0],"CCBot":["partial","*",10,0],"Bytespider":["partial","*",10,0],"Amazonbot":["partial","*",10,0],"meta-externalagent":["partial","*",10,0],"cohere-ai":["partial","*",10,0],"Diffbot":["partial","*",10,0],"ContextFetchBot":["partial","*",10,0]},"llms":{"status":"missing","http":404,"bytes":146},"home":{"status":"not_html","error":"unsupported content type text/markdown"}},
{"rank":9798,"domain":"jimdoweb.com","origin":"https://www.jimdoweb.com","robots":{"status":"refused","http":403,"url":"https://www.jimdoweb.com/robots.txt","bytes":3553,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":9799,"domain":"newspicks.com","origin":"https://newspicks.com","robots":{"status":"ok","http":200,"url":"https://newspicks.com/robots.txt","bytes":3328,"groups":32},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["partial","named",2,4],"OAI-SearchBot":["partial","named",2,4],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","named",3,5],"Claude-SearchBot":["partial","named",3,5],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["partial","named",2,4],"Perplexity-User":["partial","named",2,4],"Google-Extended":["partial","named",11,10],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",6,0]},"llms":{"status":"missing","http":404,"bytes":28718},"home":{"status":"ok","http":200,"url":"https://newspicks.com/","html_tokens":63687,"markdown_tokens":7,"pruned_tokens":7}},
{"rank":9800,"domain":"myfwc.com","origin":"https://myfwc.com","robots":{"status":"ok","http":200,"url":"https://myfwc.com/robots.txt","bytes":536,"groups":3},"bots":{"GPTBot":["partial","*",9,0],"ChatGPT-User":["partial","*",9,0],"OAI-SearchBot":["partial","*",9,0],"ClaudeBot":["partial","*",9,0],"Claude-User":["partial","*",9,0],"Claude-SearchBot":["partial","*",9,0],"anthropic-ai":["partial","*",9,0],"PerplexityBot":["partial","*",9,0],"Perplexity-User":["partial","*",9,0],"Google-Extended":["partial","*",9,0],"Applebot-Extended":["partial","*",9,0],"CCBot":["partial","*",9,0],"Bytespider":["partial","*",9,0],"Amazonbot":["partial","*",9,0],"meta-externalagent":["partial","*",9,0],"cohere-ai":["partial","*",9,0],"Diffbot":["partial","*",9,0],"ContextFetchBot":["partial","*",9,0]},"llms":{"status":"missing","http":404,"bytes":0},"home":{"status":"ok","http":200,"url":"https://myfwc.com/","html_tokens":44460,"markdown_tokens":26,"pruned_tokens":26}},
{"rank":9801,"domain":"kstp.com","origin":"https://kstp.com","robots":{"status":"refused","http":403,"url":"https://kstp.com/robots.txt","bytes":560,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":9802,"domain":"timesfreepress.com","origin":"https://timesfreepress.com","robots":{"status":"ok","http":200,"url":"https://www.timesfreepress.com/robots.txt","bytes":9679,"groups":249},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["blocked","named",1,0],"OAI-SearchBot":["blocked","named",1,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["blocked","named",1,0],"Claude-SearchBot":["blocked","named",1,0],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["blocked","named",1,0],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["allowed","*",0,1]},"llms":{"status":"missing","http":404,"bytes":93540},"home":{"status":"ok","http":200,"url":"https://www.timesfreepress.com/","html_tokens":49721,"markdown_tokens":28,"pruned_tokens":28}},
{"rank":9803,"domain":"pravda.sk","origin":"https://pravda.sk","robots":{"status":"missing","http":404,"url":"https://www.pravda.sk/robots.txt","bytes":54459,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":54459},"home":{"status":"ok","http":200,"url":"https://www.pravda.sk/","html_tokens":279608,"markdown_tokens":1791,"pruned_tokens":167}},
{"rank":9804,"domain":"bafta.org","origin":"https://bafta.org","robots":{"status":"ok","http":200,"url":"https://www.bafta.org/robots.txt","bytes":171,"groups":1},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"http_error","http":403,"bytes":1355},"home":{"status":"http_error","http":403,"url":"https://www.bafta.org/"}},
{"rank":9805,"domain":"interq.or.jp","origin":"https://interq.or.jp","robots":{"status":"unreachable","http":null,"url":"https://interq.or.jp/robots.txt","bytes":0,"groups":0,"error":"cannot resolve interq.or.jp"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":9806,"domain":"facer.io","origin":"https://facer.io","robots":{"status":"ok","http":200,"url":"https://www.facer.io/robots.txt","bytes":1079,"groups":1},"bots":{"GPTBot":["partial","*",41,1],"ChatGPT-User":["partial","*",41,1],"OAI-SearchBot":["partial","*",41,1],"ClaudeBot":["partial","*",41,1],"Claude-User":["partial","*",41,1],"Claude-SearchBot":["partial","*",41,1],"anthropic-ai":["partial","*",41,1],"PerplexityBot":["partial","*",41,1],"Perplexity-User":["partial","*",41,1],"Google-Extended":["partial","*",41,1],"Applebot-Extended":["partial","*",41,1],"CCBot":["partial","*",41,1],"Bytespider":["partial","*",41,1],"Amazonbot":["partial","*",41,1],"meta-externalagent":["partial","*",41,1],"cohere-ai":["partial","*",41,1],"Diffbot":["partial","*",41,1],"ContextFetchBot":["partial","*",41,1]},"llms":{"status":"html","http":200,"bytes":22644},"home":{"status":"ok","http":200,"url":"https://www.facer.io/","html_tokens":5994,"markdown_tokens":1,"pruned_tokens":1}},
{"rank":9807,"domain":"printful.com","origin":"https://printful.com","robots":{"status":"ok","http":200,"url":"https://www.printful.com/robots.txt","bytes":1327,"groups":1},"bots":{"GPTBot":["partial","*",29,0],"ChatGPT-User":["partial","*",29,0],"OAI-SearchBot":["partial","*",29,0],"ClaudeBot":["partial","*",29,0],"Claude-User":["partial","*",29,0],"Claude-SearchBot":["partial","*",29,0],"anthropic-ai":["partial","*",29,0],"PerplexityBot":["partial","*",29,0],"Perplexity-User":["partial","*",29,0],"Google-Extended":["partial","*",29,0],"Applebot-Extended":["partial","*",29,0],"CCBot":["partial","*",29,0],"Bytespider":["partial","*",29,0],"Amazonbot":["partial","*",29,0],"meta-externalagent":["partial","*",29,0],"cohere-ai":["partial","*",29,0],"Diffbot":["partial","*",29,0],"ContextFetchBot":["partial","*",29,0]},"llms":{"status":"present","http":200,"bytes":268184},"home":{"status":"ok","http":200,"url":"https://www.printful.com/","html_tokens":101623,"markdown_tokens":64,"pruned_tokens":64}},
{"rank":9808,"domain":"mistore.jp","origin":"https://mistore.jp","robots":{"status":"timeout","http":null,"url":"https://mistore.jp/robots.txt","bytes":0,"groups":0,"error":"timed out after 10 s"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":9809,"domain":"disney.co.jp","origin":"https://www.disney.co.jp","robots":{"status":"ok","http":200,"url":"https://www.disney.co.jp/robots.txt","bytes":117,"groups":2},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":319498},"home":{"status":"ok","http":200,"url":"https://www.disney.co.jp/","html_tokens":128790,"markdown_tokens":51,"pruned_tokens":51}},
{"rank":9810,"domain":"delfi.ee","origin":"https://delfi.ee","robots":{"status":"ok","http":200,"url":"https://www.delfi.ee/robots.txt","bytes":1384,"groups":28},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["blocked","named",1,0],"OAI-SearchBot":["blocked","named",1,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",12,1],"Claude-SearchBot":["partial","*",12,1],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["partial","*",12,1],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",12,1]},"llms":{"status":"missing","http":404,"bytes":140858},"home":{"status":"ok","http":200,"url":"https://www.delfi.ee/","html_tokens":184675,"markdown_tokens":302,"pruned_tokens":175}},
{"rank":9811,"domain":"google.to","origin":"https://google.to","robots":{"status":"ok","http":200,"url":"https://www.google.to/robots.txt","bytes":6624,"groups":4},"bots":{"GPTBot":["partial","*",169,64],"ChatGPT-User":["partial","*",169,64],"OAI-SearchBot":["partial","*",169,64],"ClaudeBot":["partial","*",169,64],"Claude-User":["partial","*",169,64],"Claude-SearchBot":["partial","*",169,64],"anthropic-ai":["partial","*",169,64],"PerplexityBot":["partial","*",169,64],"Perplexity-User":["partial","*",169,64],"Google-Extended":["partial","*",169,64],"Applebot-Extended":["partial","*",169,64],"CCBot":["partial","*",169,64],"Bytespider":["partial","*",169,64],"Amazonbot":["partial","*",169,64],"meta-externalagent":["partial","*",169,64],"cohere-ai":["partial","*",169,64],"Diffbot":["partial","*",169,64],"ContextFetchBot":["partial","*",169,64]},"llms":{"status":"missing","http":404,"bytes":1569},"home":{"status":"ok","http":200,"url":"https://www.google.com/","html_tokens":21307,"markdown_tokens":30,"pruned_tokens":30}},
{"rank":9812,"domain":"planet.nl","origin":"https://planet.nl","robots":{"status":"html","http":200,"url":"https://www.kpn.com/","bytes":508274,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":9813,"domain":"collegeboard.com","origin":"https://collegeboard.com","robots":{"status":"html","http":200,"url":"https://www.collegeboard.org/","bytes":41480,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":9814,"domain":"els-cdn.com","origin":"https://els-cdn.com","robots":{"status":"unreachable","http":null,"url":"https://els-cdn.com/robots.txt","bytes":0,"groups":0,"error":"cannot resolve els-cdn.com"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":9815,"domain":"zpsyxx.com","origin":"http://zpsyxx.com","robots":{"status":"missing","http":404,"url":"http://zpsyxx.com/robots.txt","bytes":5071,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":5067},"home":{"status":"ok","http":200,"url":"http://zpsyxx.com/","html_tokens":2107,"markdown_tokens":16,"pruned_tokens":16}},
{"rank":9816,"domain":"converse.com","origin":"https://converse.com","robots":{"status":"refused","http":403,"url":"https://www.converse.com/robots.txt","bytes":384,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":9817,"domain":"phrases.org.uk","origin":"https://phrases.org.uk","robots":{"status":"ok","http":200,"url":"https://www.phrases.org.uk/robots.txt","bytes":132,"groups":1},"bots":{"GPTBot":["partial","*",3,0],"ChatGPT-User":["partial","*",3,0],"OAI-SearchBot":["partial","*",3,0],"ClaudeBot":["partial","*",3,0],"Claude-User":["partial","*",3,0],"Claude-SearchBot":["partial","*",3,0],"anthropic-ai":["partial","*",3,0],"PerplexityBot":["partial","*",3,0],"Perplexity-User":["partial","*",3,0],"Google-Extended":["partial","*",3,0],"Applebot-Extended":["partial","*",3,0],"CCBot":["partial","*",3,0],"Bytespider":["partial","*",3,0],"Amazonbot":["partial","*",3,0],"meta-externalagent":["partial","*",3,0],"cohere-ai":["partial","*",3,0],"Diffbot":["partial","*",3,0],"ContextFetchBot":["partial","*",3,0]},"llms":{"status":"missing","http":404,"bytes":21868},"home":{"status":"ok","http":200,"url":"https://www.phrases.org.uk/","html_tokens":11953,"markdown_tokens":935,"pruned_tokens":82}},
{"rank":9818,"domain":"bmz.de","origin":"https://bmz.de","robots":{"status":"ok","http":200,"url":"https://www.bmz.de/robots.txt","bytes":326,"groups":3},"bots":{"GPTBot":["partial","*",1,0],"ChatGPT-User":["partial","*",1,0],"OAI-SearchBot":["partial","*",1,0],"ClaudeBot":["partial","*",1,0],"Claude-User":["partial","*",1,0],"Claude-SearchBot":["partial","*",1,0],"anthropic-ai":["partial","*",1,0],"PerplexityBot":["partial","*",1,0],"Perplexity-User":["partial","*",1,0],"Google-Extended":["partial","*",1,0],"Applebot-Extended":["partial","*",1,0],"CCBot":["partial","*",1,0],"Bytespider":["partial","*",1,0],"Amazonbot":["partial","*",1,0],"meta-externalagent":["partial","*",1,0],"cohere-ai":["partial","*",1,0],"Diffbot":["partial","*",1,0],"ContextFetchBot":["partial","*",1,0]},"llms":{"status":"missing","http":404,"bytes":58375},"home":{"status":"ok","http":200,"url":"https://www.bmz.de/de","html_tokens":121649,"markdown_tokens":122,"pruned_tokens":122}},
{"rank":9819,"domain":"iwate.jp","origin":"https://iwate.jp","robots":{"status":"unreachable","http":null,"url":"https://iwate.jp/robots.txt","bytes":0,"groups":0,"error":"cannot resolve iwate.jp"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":9820,"domain":"acma.gov.au","origin":"https://acma.gov.au","robots":{"status":"unreachable","http":null,"url":"https://acma.gov.au/robots.txt","bytes":0,"groups":0,"error":"<StreamReset stream_id:1, error_code:2, remote_reset:True>"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":9821,"domain":"tbsnews.net","origin":"https://tbsnews.net","robots":{"status":"ok","http":200,"url":"https://www.tbsnews.net/robots.txt","bytes":2418,"groups":1},"bots":{"GPTBot":["partial","*",37,32],"ChatGPT-User":["partial","*",37,32],"OAI-SearchBot":["partial","*",37,32],"ClaudeBot":["partial","*",37,32],"Claude-User":["partial","*",37,32],"Claude-SearchBot":["partial","*",37,32],"anthropic-ai":["partial","*",37,32],"PerplexityBot":["partial","*",37,32],"Perplexity-User":["partial","*",37,32],"Google-Extended":["partial","*",37,32],"Applebot-Extended":["partial","*",37,32],"CCBot":["partial","*",37,32],"Bytespider":["partial","*",37,32],"Amazonbot":["partial","*",37,32],"meta-externalagent":["partial","*",37,32],"cohere-ai":["partial","*",37,32],"Diffbot":["partial","*",37,32],"ContextFetchBot":["partial","*",37,32]},"llms":{"status":"http_error","http":403,"bytes":530},"home":{"status":"ok","http":200,"url":"https://www.tbsnews.net/","html_tokens":41012,"markdown_tokens":24,"pruned_tokens":24}},
{"rank":9822,"domain":"avrotros.nl","origin":"https://avrotros.nl","robots":{"status":"missing","http":404,"url":"https://www.avrotros.nl/robots.txt/","bytes":78271,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":78269},"home":{"status":"ok","http":200,"url":"https://www.avrotros.nl/","html_tokens":94268,"markdown_tokens":75,"pruned_tokens":75}},
{"rank":9823,"domain":"umms.org","origin":"https://umms.org","robots":{"status":"ok","http":200,"url":"https://umms.org/robots.txt","bytes":2676,"groups":32},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["allowed","*",0,1],"OAI-SearchBot":["allowed","*",0,1],"ClaudeBot":["blocked","named",1,0],"Claude-User":["allowed","*",0,1],"Claude-SearchBot":["allowed","*",0,1],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["allowed","*",0,1],"Perplexity-User":["allowed","*",0,1],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["allowed","*",0,1]},"llms":{"status":"missing","http":404,"bytes":103},"home":{"status":"ok","http":200,"url":"https://www.umms.org/","html_tokens":10416,"markdown_tokens":53,"pruned_tokens":53}},
{"rank":9824,"domain":"std.com","origin":"https://std.com","robots":{"status":"ok","http":200,"url":"https://theworld.com/robots.txt","bytes":2189,"groups":1},"bots":{"GPTBot":["partial","*",36,32],"ChatGPT-User":["partial","*",36,32],"OAI-SearchBot":["partial","*",36,32],"ClaudeBot":["partial","*",36,32],"Claude-User":["partial","*",36,32],"Claude-SearchBot":["partial","*",36,32],"anthropic-ai":["partial","*",36,32],"PerplexityBot":["partial","*",36,32],"Perplexity-User":["partial","*",36,32],"Google-Extended":["partial","*",36,32],"Applebot-Extended":["partial","*",36,32],"CCBot":["partial","*",36,32],"Bytespider":["partial","*",36,32],"Amazonbot":["partial","*",36,32],"meta-externalagent":["partial","*",36,32],"cohere-ai":["partial","*",36,32],"Diffbot":["partial","*",36,32],"ContextFetchBot":["partial","*",36,32]},"llms":{"status":"missing","http":404,"bytes":9957},"home":{"status":"ok","http":200,"url":"https://theworld.com/","html_tokens":6533,"markdown_tokens":146,"pruned_tokens":83}},
{"rank":9825,"domain":"hyogo.jp","origin":"https://hyogo.jp","robots":{"status":"unreachable","http":null,"url":"https://hyogo.jp/robots.txt","bytes":0,"groups":0,"error":"cannot resolve hyogo.jp"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":9826,"domain":"usopen.org","origin":"https://usopen.org","robots":{"status":"unreachable","http":null,"url":"https://usopen.org/robots.txt","bytes":0,"groups":0,"error":"<StreamReset stream_id:1, error_code:2, remote_reset:True>"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":9827,"domain":"zanox.com","origin":"https://zanox.com","robots":{"status":"html","http":200,"url":"https://www.awin.com/?utm_medium=redirect&utm_source=www.zanox.com","bytes":296526,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":9828,"domain":"hofstra.edu","origin":"https://hofstra.edu","robots":{"status":"ok","http":200,"url":"https://www.hofstra.edu/robots.txt","bytes":14,"groups":1},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"missing","http":404,"bytes":68986},"home":{"status":"ok","http":200,"url":"https://www.hofstra.edu/","html_tokens":26636,"markdown_tokens":154,"pruned_tokens":54}},
{"rank":9829,"domain":"techstars.com","origin":"https://techstars.com","robots":{"status":"ok","http":200,"url":"https://www.techstars.com/robots.txt","bytes":71,"groups":1},"bots":{"GPTBot":["allowed","*",0,1],"ChatGPT-User":["allowed","*",0,1],"OAI-SearchBot":["allowed","*",0,1],"ClaudeBot":["allowed","*",0,1],"Claude-User":["allowed","*",0,1],"Claude-SearchBot":["allowed","*",0,1],"anthropic-ai":["allowed","*",0,1],"PerplexityBot":["allowed","*",0,1],"Perplexity-User":["allowed","*",0,1],"Google-Extended":["allowed","*",0,1],"Applebot-Extended":["allowed","*",0,1],"CCBot":["allowed","*",0,1],"Bytespider":["allowed","*",0,1],"Amazonbot":["allowed","*",0,1],"meta-externalagent":["allowed","*",0,1],"cohere-ai":["allowed","*",0,1],"Diffbot":["allowed","*",0,1],"ContextFetchBot":["allowed","*",0,1]},"llms":{"status":"missing","http":404,"bytes":30103},"home":{"status":"ok","http":200,"url":"https://www.techstars.com/","html_tokens":111331,"markdown_tokens":659,"pruned_tokens":377}},
{"rank":9830,"domain":"gismeteo.ru","origin":"https://gismeteo.ru","robots":{"status":"ok","http":200,"url":"https://www.gismeteo.ru/robots.txt","bytes":277,"groups":1},"bots":{"GPTBot":["partial","*",10,0],"ChatGPT-User":["partial","*",10,0],"OAI-SearchBot":["partial","*",10,0],"ClaudeBot":["partial","*",10,0],"Claude-User":["partial","*",10,0],"Claude-SearchBot":["partial","*",10,0],"anthropic-ai":["partial","*",10,0],"PerplexityBot":["partial","*",10,0],"Perplexity-User":["partial","*",10,0],"Google-Extended":["partial","*",10,0],"Applebot-Extended":["partial","*",10,0],"CCBot":["partial","*",10,0],"Bytespider":["partial","*",10,0],"Amazonbot":["partial","*",10,0],"meta-externalagent":["partial","*",10,0],"cohere-ai":["partial","*",10,0],"Diffbot":["partial","*",10,0],"ContextFetchBot":["partial","*",10,0]},"llms":{"status":"missing","http":404,"bytes":68516},"home":{"status":"ok","http":200,"url":"https://www.gismeteo.ru/","html_tokens":47480,"markdown_tokens":7,"pruned_tokens":7}},
{"rank":9831,"domain":"jsjlxf.cn","origin":"http://jsjlxf.cn","robots":{"status":"ok","http":200,"url":"http://jsjlxf.cn/robots.txt","bytes":249,"groups":1},"bots":{"GPTBot":["partial","*",10,0],"ChatGPT-User":["partial","*",10,0],"OAI-SearchBot":["partial","*",10,0],"ClaudeBot":["partial","*",10,0],"Claude-User":["partial","*",10,0],"Claude-SearchBot":["partial","*",10,0],"anthropic-ai":["partial","*",10,0],"PerplexityBot":["partial","*",10,0],"Perplexity-User":["partial","*",10,0],"Google-Extended":["partial","*",10,0],"Applebot-Extended":["partial","*",10,0],"CCBot":["partial","*",10,0],"Bytespider":["partial","*",10,0],"Amazonbot":["partial","*",10,0],"meta-externalagent":["partial","*",10,0],"cohere-ai":["partial","*",10,0],"Diffbot":["partial","*",10,0],"ContextFetchBot":["partial","*",10,0]},"llms":{"status":"missing","http":404,"bytes":3749},"home":{"status":"ok","http":200,"url":"http://jsjlxf.cn/","html_tokens":4341,"markdown_tokens":33,"pruned_tokens":33}},
{"rank":9832,"domain":"denik.cz","origin":"https://denik.cz","robots":{"status":"ok","http":200,"url":"https://www.denik.cz/robots.txt","bytes":1154,"groups":3},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["blocked","named",1,0],"OAI-SearchBot":["blocked","named",1,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",4,0],"Claude-SearchBot":["partial","*",4,0],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["partial","*",4,0],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",4,0]},"llms":{"status":"missing","http":404,"bytes":50837},"home":{"status":"ok","http":200,"url":"https://www.denik.cz/","html_tokens":66376,"markdown_tokens":82,"pruned_tokens":15}},
{"rank":9833,"domain":"weatherspark.com","origin":"https://weatherspark.com","robots":{"status":"ok","http":200,"url":"https://weatherspark.com/robots.txt","bytes":628,"groups":2},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["partial","*",12,0],"OAI-SearchBot":["partial","*",12,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",12,0],"Claude-SearchBot":["partial","*",12,0],"anthropic-ai":["partial","*",12,0],"PerplexityBot":["partial","*",12,0],"Perplexity-User":["partial","*",12,0],"Google-Extended":["partial","*",12,0],"Applebot-Extended":["partial","*",12,0],"CCBot":["partial","*",12,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["partial","*",12,0],"cohere-ai":["partial","*",12,0],"Diffbot":["partial","*",12,0],"ContextFetchBot":["partial","*",12,0]},"llms":{"status":"missing","http":404,"bytes":1141},"home":{"status":"ok","http":200,"url":"https://weatherspark.com/","html_tokens":27094,"markdown_tokens":28,"pruned_tokens":28}},
{"rank":9834,"domain":"creighton.edu","origin":"https://creighton.edu","robots":{"status":"ok","http":200,"url":"https://www.creighton.edu/robots.txt","bytes":1694,"groups":1},"bots":{"GPTBot":["partial","*",25,18],"ChatGPT-User":["partial","*",25,18],"OAI-SearchBot":["partial","*",25,18],"ClaudeBot":["partial","*",25,18],"Claude-User":["partial","*",25,18],"Claude-SearchBot":["partial","*",25,18],"anthropic-ai":["partial","*",25,18],"PerplexityBot":["partial","*",25,18],"Perplexity-User":["partial","*",25,18],"Google-Extended":["partial","*",25,18],"Applebot-Extended":["partial","*",25,18],"CCBot":["partial","*",25,18],"Bytespider":["partial","*",25,18],"Amazonbot":["partial","*",25,18],"meta-externalagent":["partial","*",25,18],"cohere-ai":["partial","*",25,18],"Diffbot":["partial","*",25,18],"ContextFetchBot":["partial","*",25,18]},"llms":{"status":"missing","http":404,"bytes":291},"home":{"status":"ok","http":200,"url":"https://www.creighton.edu/","html_tokens":21617,"markdown_tokens":218,"pruned_tokens":171}},
{"rank":9835,"domain":"nips.cc","origin":"https://nips.cc","robots":{"status":"ok","http":200,"url":"https://nips.cc/robots.txt","bytes":1038,"groups":10},"bots":{"GPTBot":["partial","named",1,0],"ChatGPT-User":["partial","named",1,0],"OAI-SearchBot":["partial","named",1,0],"ClaudeBot":["partial","*",8,0],"Claude-User":["partial","*",8,0],"Claude-SearchBot":["partial","*",8,0],"anthropic-ai":["partial","*",8,0],"PerplexityBot":["allowed","named",0,0],"Perplexity-User":["partial","*",8,0],"Google-Extended":["partial","*",8,0],"Applebot-Extended":["partial","*",8,0],"CCBot":["partial","*",8,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["allowed","named",0,0],"meta-externalagent":["blocked","named",2,0],"cohere-ai":["partial","*",8,0],"Diffbot":["partial","*",8,0],"ContextFetchBot":["partial","*",8,0]},"llms":{"status":"missing","http":404,"bytes":47680},"home":{"status":"ok","http":200,"url":"https://nips.cc/","html_tokens":43320,"markdown_tokens":1053,"pruned_tokens":130}},
{"rank":9836,"domain":"um.edu.my","origin":"https://um.edu.my","robots":{"status":"missing","http":404,"url":"https://um.edu.my/robots.txt","bytes":327,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":325},"home":{"status":"ok","http":200,"url":"https://um.edu.my/","html_tokens":22815,"markdown_tokens":195,"pruned_tokens":116}},
{"rank":9837,"domain":"chinhphu.vn","origin":"https://chinhphu.vn","robots":{"status":"ok","http":200,"url":"https://chinhphu.vn/robots.txt","bytes":23,"groups":1},"bots":{"GPTBot":["allowed","*",0,1],"ChatGPT-User":["allowed","*",0,1],"OAI-SearchBot":["allowed","*",0,1],"ClaudeBot":["allowed","*",0,1],"Claude-User":["allowed","*",0,1],"Claude-SearchBot":["allowed","*",0,1],"anthropic-ai":["allowed","*",0,1],"PerplexityBot":["allowed","*",0,1],"Perplexity-User":["allowed","*",0,1],"Google-Extended":["allowed","*",0,1],"Applebot-Extended":["allowed","*",0,1],"CCBot":["allowed","*",0,1],"Bytespider":["allowed","*",0,1],"Amazonbot":["allowed","*",0,1],"meta-externalagent":["allowed","*",0,1],"cohere-ai":["allowed","*",0,1],"Diffbot":["allowed","*",0,1],"ContextFetchBot":["allowed","*",0,1]},"llms":{"status":"missing","http":404,"bytes":1245},"home":{"status":"ok","http":200,"url":"https://chinhphu.vn/","html_tokens":48240,"markdown_tokens":397,"pruned_tokens":244}},
{"rank":9838,"domain":"excelsior.com.mx","origin":"https://excelsior.com.mx","robots":{"status":"ok","http":200,"url":"https://www.excelsior.com.mx/robots.txt","bytes":7796,"groups":44},"bots":{"GPTBot":["partial","*",139,33],"ChatGPT-User":["partial","*",139,33],"OAI-SearchBot":["partial","*",139,33],"ClaudeBot":["partial","*",139,33],"Claude-User":["partial","*",139,33],"Claude-SearchBot":["partial","*",139,33],"anthropic-ai":["partial","*",139,33],"PerplexityBot":["partial","*",139,33],"Perplexity-User":["partial","*",139,33],"Google-Extended":["partial","*",139,33],"Applebot-Extended":["partial","*",139,33],"CCBot":["partial","*",139,33],"Bytespider":["partial","*",139,33],"Amazonbot":["allowed","named",0,1],"meta-externalagent":["partial","*",139,33],"cohere-ai":["partial","*",139,33],"Diffbot":["partial","*",139,33],"ContextFetchBot":["partial","*",139,33]},"llms":{"status":"missing","http":404,"bytes":56781},"home":{"status":"ok","http":200,"url":"https://www.excelsior.com.mx/","html_tokens":37969,"markdown_tokens":142,"pruned_tokens":142}},
{"rank":9839,"domain":"nissanusa.com","origin":"https://nissanusa.com","robots":{"status":"ok","http":200,"url":"https://www.nissanusa.com/robots.txt","bytes":3678,"groups":3},"bots":{"GPTBot":["partial","*",73,0],"ChatGPT-User":["partial","*",73,0],"OAI-SearchBot":["partial","*",73,0],"ClaudeBot":["partial","*",73,0],"Claude-User":["partial","*",73,0],"Claude-SearchBot":["partial","*",73,0],"anthropic-ai":["partial","*",73,0],"PerplexityBot":["partial","*",73,0],"Perplexity-User":["partial","*",73,0],"Google-Extended":["partial","*",73,0],"Applebot-Extended":["partial","*",73,0],"CCBot":["partial","*",73,0],"Bytespider":["partial","*",73,0],"Amazonbot":["partial","*",73,0],"meta-externalagent":["partial","*",73,0],"cohere-ai":["partial","*",73,0],"Diffbot":["partial","*",73,0],"ContextFetchBot":["partial","*",73,0]},"llms":{"status":"present","http":200,"bytes":11288},"home":{"status":"ok","http":200,"url":"https://www.nissanusa.com/","html_tokens":151898,"markdown_tokens":137,"pruned_tokens":137}},
{"rank":9840,"domain":"aflcio.org","origin":"https://aflcio.org","robots":{"status":"ok","http":200,"url":"https://aflcio.org/robots.txt","bytes":2005,"groups":1},"bots":{"GPTBot":["partial","*",33,18],"ChatGPT-User":["partial","*",33,18],"OAI-SearchBot":["partial","*",33,18],"ClaudeBot":["partial","*",33,18],"Claude-User":["partial","*",33,18],"Claude-SearchBot":["partial","*",33,18],"anthropic-ai":["partial","*",33,18],"PerplexityBot":["partial","*",33,18],"Perplexity-User":["partial","*",33,18],"Google-Extended":["partial","*",33,18],"Applebot-Extended":["partial","*",33,18],"CCBot":["partial","*",33,18],"Bytespider":["partial","*",33,18],"Amazonbot":["partial","*",33,18],"meta-externalagent":["partial","*",33,18],"cohere-ai":["partial","*",33,18],"Diffbot":["partial","*",33,18],"ContextFetchBot":["partial","*",33,18]},"llms":{"status":"missing","http":404,"bytes":545},"home":{"status":"http_error","http":403,"url":"https://aflcio.org/"}},
{"rank":9841,"domain":"pi-hole.net","origin":"https://pi-hole.net","robots":{"status":"ok","http":200,"url":"https://pi-hole.net/robots.txt","bytes":112,"groups":1},"bots":{"GPTBot":["partial","*",1,1],"ChatGPT-User":["partial","*",1,1],"OAI-SearchBot":["partial","*",1,1],"ClaudeBot":["partial","*",1,1],"Claude-User":["partial","*",1,1],"Claude-SearchBot":["partial","*",1,1],"anthropic-ai":["partial","*",1,1],"PerplexityBot":["partial","*",1,1],"Perplexity-User":["partial","*",1,1],"Google-Extended":["partial","*",1,1],"Applebot-Extended":["partial","*",1,1],"CCBot":["partial","*",1,1],"Bytespider":["partial","*",1,1],"Amazonbot":["partial","*",1,1],"meta-externalagent":["partial","*",1,1],"cohere-ai":["partial","*",1,1],"Diffbot":["partial","*",1,1],"ContextFetchBot":["partial","*",1,1]},"llms":{"status":"missing","http":404,"bytes":49982},"home":{"status":"ok","http":200,"url":"https://pi-hole.net/","html_tokens":19955,"markdown_tokens":685,"pruned_tokens":442}},
{"rank":9842,"domain":"igg.me","origin":"https://igg.me","robots":{"status":"refused","http":403,"url":"https://www.indiegogo.com/","bytes":5504,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":9843,"domain":"omniglot.com","origin":"https://omniglot.com","robots":{"status":"ok","http":200,"url":"https://omniglot.com/robots.txt","bytes":3259,"groups":46},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["blocked","named",1,0],"OAI-SearchBot":["allowed","*",0,1],"ClaudeBot":["blocked","named",1,0],"Claude-User":["blocked","named",1,0],"Claude-SearchBot":["allowed","*",0,1],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["allowed","*",0,1],"Perplexity-User":["blocked","named",1,0],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["allowed","*",0,1]},"llms":{"status":"http_error","http":403,"bytes":5523},"home":{"status":"http_error","http":403,"url":"https://omniglot.com/"}},
{"rank":9844,"domain":"u-szeged.hu","origin":"https://u-szeged.hu","robots":{"status":"ok","http":200,"url":"https://u-szeged.hu/robots.txt","bytes":2104,"groups":30},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"missing","http":404,"bytes":64372},"home":{"status":"ok","http":200,"url":"https://u-szeged.hu/","html_tokens":36701,"markdown_tokens":136,"pruned_tokens":136}},
{"rank":9845,"domain":"crcpress.com","origin":"https://crcpress.com","robots":{"status":"ok","http":200,"url":"https://www.routledge.com/robots.txt","bytes":319,"groups":1},"bots":{"GPTBot":["partial","*",10,0],"ChatGPT-User":["partial","*",10,0],"OAI-SearchBot":["partial","*",10,0],"ClaudeBot":["partial","*",10,0],"Claude-User":["partial","*",10,0],"Claude-SearchBot":["partial","*",10,0],"anthropic-ai":["partial","*",10,0],"PerplexityBot":["partial","*",10,0],"Perplexity-User":["partial","*",10,0],"Google-Extended":["partial","*",10,0],"Applebot-Extended":["partial","*",10,0],"CCBot":["partial","*",10,0],"Bytespider":["partial","*",10,0],"Amazonbot":["partial","*",10,0],"meta-externalagent":["partial","*",10,0],"cohere-ai":["partial","*",10,0],"Diffbot":["partial","*",10,0],"ContextFetchBot":["partial","*",10,0]},"llms":{"status":"missing","http":404,"bytes":113083},"home":{"status":"ok","http":200,"url":"https://www.routledge.com/","html_tokens":32691,"markdown_tokens":267,"pruned_tokens":104}},
{"rank":9846,"domain":"segment.com","origin":"https://segment.com","robots":{"status":"ok","http":200,"url":"https://www.twilio.com/robots.txt","bytes":976,"groups":2},"bots":{"GPTBot":["partial","*",13,6],"ChatGPT-User":["partial","*",13,6],"OAI-SearchBot":["partial","*",13,6],"ClaudeBot":["partial","*",13,6],"Claude-User":["partial","*",13,6],"Claude-SearchBot":["partial","*",13,6],"anthropic-ai":["partial","*",13,6],"PerplexityBot":["partial","*",13,6],"Perplexity-User":["partial","*",13,6],"Google-Extended":["partial","*",13,6],"Applebot-Extended":["partial","*",13,6],"CCBot":["partial","*",13,6],"Bytespider":["partial","*",13,6],"Amazonbot":["partial","*",13,6],"meta-externalagent":["partial","*",13,6],"cohere-ai":["partial","*",13,6],"Diffbot":["partial","*",13,6],"ContextFetchBot":["partial","*",13,6]},"llms":{"status":"html","http":200,"bytes":725513},"home":{"status":"ok","http":200,"url":"https://www.twilio.com/en-us/segment","html_tokens":156049,"markdown_tokens":174,"pruned_tokens":173}},
{"rank":9847,"domain":"forvo.com","origin":"https://forvo.com","robots":{"status":"ok","http":200,"url":"https://forvo.com/robots.txt","bytes":1410,"groups":5},"bots":{"GPTBot":["partial","*",48,5],"ChatGPT-User":["partial","*",48,5],"OAI-SearchBot":["partial","*",48,5],"ClaudeBot":["partial","*",48,5],"Claude-User":["partial","*",48,5],"Claude-SearchBot":["partial","*",48,5],"anthropic-ai":["partial","*",48,5],"PerplexityBot":["partial","*",48,5],"Perplexity-User":["partial","*",48,5],"Google-Extended":["partial","*",48,5],"Applebot-Extended":["partial","*",48,5],"CCBot":["partial","*",48,5],"Bytespider":["partial","*",48,5],"Amazonbot":["partial","*",48,5],"meta-externalagent":["partial","*",48,5],"cohere-ai":["partial","*",48,5],"Diffbot":["partial","*",48,5],"ContextFetchBot":["partial","*",48,5]},"llms":{"status":"missing","http":404,"bytes":42919},"home":{"status":"http_error","http":403,"url":"https://forvo.com/"}},
{"rank":9848,"domain":"yonsei.ac.kr","origin":"https://yonsei.ac.kr","robots":{"status":"ok","http":200,"url":"https://yonsei.ac.kr/robots.txt","bytes":306,"groups":1},"bots":{"GPTBot":["partial","*",8,0],"ChatGPT-User":["partial","*",8,0],"OAI-SearchBot":["partial","*",8,0],"ClaudeBot":["partial","*",8,0],"Claude-User":["partial","*",8,0],"Claude-SearchBot":["partial","*",8,0],"anthropic-ai":["partial","*",8,0],"PerplexityBot":["partial","*",8,0],"Perplexity-User":["partial","*",8,0],"Google-Extended":["partial","*",8,0],"Applebot-Extended":["partial","*",8,0],"CCBot":["partial","*",8,0],"Bytespider":["partial","*",8,0],"Amazonbot":["partial","*",8,0],"meta-externalagent":["partial","*",8,0],"cohere-ai":["partial","*",8,0],"Diffbot":["partial","*",8,0],"ContextFetchBot":["partial","*",8,0]},"llms":{"status":"html","http":200,"bytes":993},"home":{"status":"ok","http":200,"url":"https://yonsei.ac.kr/","html_tokens":414,"markdown_tokens":0,"pruned_tokens":0}},
{"rank":9849,"domain":"cntxcl.com","origin":"http://cntxcl.com","robots":{"status":"ok","http":200,"url":"http://cntxcl.com/robots.txt","bytes":249,"groups":1},"bots":{"GPTBot":["partial","*",10,0],"ChatGPT-User":["partial","*",10,0],"OAI-SearchBot":["partial","*",10,0],"ClaudeBot":["partial","*",10,0],"Claude-User":["partial","*",10,0],"Claude-SearchBot":["partial","*",10,0],"anthropic-ai":["partial","*",10,0],"PerplexityBot":["partial","*",10,0],"Perplexity-User":["partial","*",10,0],"Google-Extended":["partial","*",10,0],"Applebot-Extended":["partial","*",10,0],"CCBot":["partial","*",10,0],"Bytespider":["partial","*",10,0],"Amazonbot":["partial","*",10,0],"meta-externalagent":["partial","*",10,0],"cohere-ai":["partial","*",10,0],"Diffbot":["partial","*",10,0],"ContextFetchBot":["partial","*",10,0]},"llms":{"status":"missing","http":404,"bytes":3749},"home":{"status":"ok","http":200,"url":"http://cntxcl.com/","html_tokens":3352,"markdown_tokens":59,"pruned_tokens":59}},
{"rank":9850,"domain":"shu.edu","origin":"https://www.shu.edu","robots":{"status":"ok","http":200,"url":"https://www.shu.edu/robots.txt","bytes":900,"groups":12},"bots":{"GPTBot":["partial","*",16,1],"ChatGPT-User":["partial","*",16,1],"OAI-SearchBot":["partial","*",16,1],"ClaudeBot":["partial","*",16,1],"Claude-User":["partial","*",16,1],"Claude-SearchBot":["partial","*",16,1],"anthropic-ai":["partial","*",16,1],"PerplexityBot":["partial","*",16,1],"Perplexity-User":["partial","*",16,1],"Google-Extended":["partial","*",16,1],"Applebot-Extended":["partial","*",16,1],"CCBot":["partial","*",16,1],"Bytespider":["partial","*",16,1],"Amazonbot":["partial","*",16,1],"meta-externalagent":["partial","*",16,1],"cohere-ai":["partial","*",16,1],"Diffbot":["partial","*",16,1],"ContextFetchBot":["partial","*",16,1]},"llms":{"status":"missing","http":404,"bytes":58201},"home":{"status":"ok","http":200,"url":"https://www.shu.edu/","html_tokens":15743,"markdown_tokens":195,"pruned_tokens":103}},
{"rank":9851,"domain":"tusk-casino.org","origin":"https://tusk-casino.org","robots":{"status":"timeout","http":null,"url":"https://tusk-casino.org/robots.txt","bytes":0,"groups":0,"error":"ConnectTimeout"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":9852,"domain":"petsmart.com","origin":"https://www.petsmart.com","robots":{"status":"ok","http":200,"url":"https://www.petsmart.com/robots.txt","bytes":1577,"groups":5},"bots":{"GPTBot":["partial","*",14,4],"ChatGPT-User":["partial","*",14,4],"OAI-SearchBot":["partial","named",5,4],"ClaudeBot":["partial","*",14,4],"Claude-User":["partial","*",14,4],"Claude-SearchBot":["partial","*",14,4],"anthropic-ai":["partial","*",14,4],"PerplexityBot":["partial","named",6,4],"Perplexity-User":["partial","*",14,4],"Google-Extended":["partial","*",14,4],"Applebot-Extended":["partial","*",14,4],"CCBot":["partial","*",14,4],"Bytespider":["partial","*",14,4],"Amazonbot":["partial","*",14,4],"meta-externalagent":["partial","*",14,4],"cohere-ai":["partial","*",14,4],"Diffbot":["partial","*",14,4],"ContextFetchBot":["partial","*",14,4]},"llms":{"status":"present","http":200,"bytes":1005181},"home":{"status":"ok","http":200,"url":"https://www.petsmart.com/","html_tokens":91038,"markdown_tokens":9,"pruned_tokens":9}},
{"rank":9853,"domain":"gizmochina.com","origin":"https://gizmochina.com","robots":{"status":"ok","http":200,"url":"https://www.gizmochina.com/robots.txt","bytes":220,"groups":1},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"missing","http":404,"bytes":507},"home":{"status":"http_error","http":403,"url":"https://gizmochina.com/"}},
{"rank":9854,"domain":"olsztyn.pl","origin":"https://olsztyn.pl","robots":{"status":"unreachable","http":null,"url":"https://olsztyn.pl/robots.txt","bytes":0,"groups":0,"error":"cannot resolve olsztyn.pl"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":9855,"domain":"khn.org","origin":"https://khn.org","robots":{"status":"ok","http":200,"url":"https://kffhealthnews.org/robots.txt","bytes":331,"groups":1},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"missing","http":404,"bytes":105013},"home":{"status":"ok","http":200,"url":"https://kffhealthnews.org/","html_tokens":59752,"markdown_tokens":759,"pruned_tokens":166}},
{"rank":9856,"domain":"wy.us","origin":"https://wy.us","robots":{"status":"unreachable","http":null,"url":"https://wy.us/robots.txt","bytes":0,"groups":0,"error":"cannot resolve wy.us"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":9857,"domain":"leovegasfi.com","origin":"https://leovegasfi.com","robots":{"status":"ok","http":200,"url":"https://leovegasfi.com/robots.txt","bytes":104,"groups":1},"bots":{"GPTBot":["partial","*",2,1],"ChatGPT-User":["partial","*",2,1],"OAI-SearchBot":["partial","*",2,1],"ClaudeBot":["partial","*",2,1],"Claude-User":["partial","*",2,1],"Claude-SearchBot":["partial","*",2,1],"anthropic-ai":["partial","*",2,1],"PerplexityBot":["partial","*",2,1],"Perplexity-User":["partial","*",2,1],"Google-Extended":["partial","*",2,1],"Applebot-Extended":["partial","*",2,1],"CCBot":["partial","*",2,1],"Bytespider":["partial","*",2,1],"Amazonbot":["partial","*",2,1],"meta-externalagent":["partial","*",2,1],"cohere-ai":["partial","*",2,1],"Diffbot":["partial","*",2,1],"ContextFetchBot":["partial","*",2,1]},"llms":{"status":"missing","http":404,"bytes":1183},"home":{"status":"ok","http":200,"url":"https://leovegasfi.com/","html_tokens":12304,"markdown_tokens":3241,"pruned_tokens":420}},
{"rank":9858,"domain":"eveeno.com","origin":"https://eveeno.com","robots":{"status":"ok","http":200,"url":"https://eveeno.com/robots.txt","bytes":221,"groups":1},"bots":{"GPTBot":["partial","*",5,0],"ChatGPT-User":["partial","*",5,0],"OAI-SearchBot":["partial","*",5,0],"ClaudeBot":["partial","*",5,0],"Claude-User":["partial","*",5,0],"Claude-SearchBot":["partial","*",5,0],"anthropic-ai":["partial","*",5,0],"PerplexityBot":["partial","*",5,0],"Perplexity-User":["partial","*",5,0],"Google-Extended":["partial","*",5,0],"Applebot-Extended":["partial","*",5,0],"CCBot":["partial","*",5,0],"Bytespider":["partial","*",5,0],"Amazonbot":["partial","*",5,0],"meta-externalagent":["partial","*",5,0],"cohere-ai":["partial","*",5,0],"Diffbot":["partial","*",5,0],"ContextFetchBot":["partial","*",5,0]},"llms":{"status":"missing","http":404,"bytes":9228},"home":{"status":"ok","http":200,"url":"https://eveeno.com/de/","html_tokens":9295,"markdown_tokens":491,"pruned_tokens":89}},
{"rank":9859,"domain":"robbast.nl","origin":"https://robbast.nl","robots":{"status":"ok","http":200,"url":"https://robbast.nl/robots.txt","bytes":36,"groups":1},"bots":{"GPTBot":["partial","*",1,0],"ChatGPT-User":["partial","*",1,0],"OAI-SearchBot":["partial","*",1,0],"ClaudeBot":["partial","*",1,0],"Claude-User":["partial","*",1,0],"Claude-SearchBot":["partial","*",1,0],"anthropic-ai":["partial","*",1,0],"PerplexityBot":["partial","*",1,0],"Perplexity-User":["partial","*",1,0],"Google-Extended":["partial","*",1,0],"Applebot-Extended":["partial","*",1,0],"CCBot":["partial","*",1,0],"Bytespider":["partial","*",1,0],"Amazonbot":["partial","*",1,0],"meta-externalagent":["partial","*",1,0],"cohere-ai":["partial","*",1,0],"Diffbot":["partial","*",1,0],"ContextFetchBot":["partial","*",1,0]},"llms":{"status":"missing","http":404,"bytes":9379},"home":{"status":"ok","http":200,"url":"https://robbast.nl/","html_tokens":1222,"markdown_tokens":132,"pruned_tokens":132}},
{"rank":9860,"domain":"nzn-crm.com","origin":"http://nzn-crm.com","robots":{"status":"missing","http":404,"url":"http://nzn-crm.com/robots.txt","bytes":315,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":315},"home":{"status":"http_error","http":404,"url":"http://nzn-crm.com/"}},
{"rank":9861,"domain":"xmhns.com","origin":"https://xmhns.com","robots":{"status":"unreachable","http":null,"url":"https://xmhns.com/robots.txt","bytes":0,"groups":0,"error":"cannot resolve xmhns.com"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":9862,"domain":"zgtxyh.com","origin":"http://zgtxyh.com","robots":{"status":"ok","http":200,"url":"http://zgtxyh.com/robots.txt","bytes":249,"groups":1},"bots":{"GPTBot":["partial","*",10,0],"ChatGPT-User":["partial","*",10,0],"OAI-SearchBot":["partial","*",10,0],"ClaudeBot":["partial","*",10,0],"Claude-User":["partial","*",10,0],"Claude-SearchBot":["partial","*",10,0],"anthropic-ai":["partial","*",10,0],"PerplexityBot":["partial","*",10,0],"Perplexity-User":["partial","*",10,0],"Google-Extended":["partial","*",10,0],"Applebot-Extended":["partial","*",10,0],"CCBot":["partial","*",10,0],"Bytespider":["partial","*",10,0],"Amazonbot":["partial","*",10,0],"meta-externalagent":["partial","*",10,0],"cohere-ai":["partial","*",10,0],"Diffbot":["partial","*",10,0],"ContextFetchBot":["partial","*",10,0]},"llms":{"status":"missing","http":404,"bytes":3749},"home":{"status":"ok","http":200,"url":"http://zgtxyh.com/","html_tokens":222,"markdown_tokens":7,"pruned_tokens":7}},
{"rank":9863,"domain":"enea.it","origin":"http://enea.it","robots":{"status":"ok","http":200,"url":"http://www.enea.it/robots.txt","bytes":764,"groups":1},"bots":{"GPTBot":["partial","*",15,0],"ChatGPT-User":["partial","*",15,0],"OAI-SearchBot":["partial","*",15,0],"ClaudeBot":["partial","*",15,0],"Claude-User":["partial","*",15,0],"Claude-SearchBot":["partial","*",15,0],"anthropic-ai":["partial","*",15,0],"PerplexityBot":["partial","*",15,0],"Perplexity-User":["partial","*",15,0],"Google-Extended":["partial","*",15,0],"Applebot-Extended":["partial","*",15,0],"CCBot":["partial","*",15,0],"Bytespider":["partial","*",15,0],"Amazonbot":["partial","*",15,0],"meta-externalagent":["partial","*",15,0],"cohere-ai":["partial","*",15,0],"Diffbot":["partial","*",15,0],"ContextFetchBot":["partial","*",15,0]},"llms":{"status":"disallowed","error":"robots.txt of www.enea.it not readable (unreachable)"},"home":{"status":"disallowed","error":"robots.txt of www.enea.it not readable (unreachable)"}},
{"rank":9864,"domain":"txqsyz.com","origin":"http://txqsyz.com","robots":{"status":"ok","http":200,"url":"http://txqsyz.com/robots.txt","bytes":249,"groups":1},"bots":{"GPTBot":["partial","*",10,0],"ChatGPT-User":["partial","*",10,0],"OAI-SearchBot":["partial","*",10,0],"ClaudeBot":["partial","*",10,0],"Claude-User":["partial","*",10,0],"Claude-SearchBot":["partial","*",10,0],"anthropic-ai":["partial","*",10,0],"PerplexityBot":["partial","*",10,0],"Perplexity-User":["partial","*",10,0],"Google-Extended":["partial","*",10,0],"Applebot-Extended":["partial","*",10,0],"CCBot":["partial","*",10,0],"Bytespider":["partial","*",10,0],"Amazonbot":["partial","*",10,0],"meta-externalagent":["partial","*",10,0],"cohere-ai":["partial","*",10,0],"Diffbot":["partial","*",10,0],"ContextFetchBot":["partial","*",10,0]},"llms":{"status":"missing","http":404,"bytes":3749},"home":{"status":"ok","http":200,"url":"http://txqsyz.com/","html_tokens":226,"markdown_tokens":7,"pruned_tokens":7}},
{"rank":9865,"domain":"themedia.jp","origin":"https://themedia.jp","robots":{"status":"unreachable","http":null,"url":"https://themedia.jp/robots.txt","bytes":0,"groups":0,"error":"cannot resolve themedia.jp"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":9866,"domain":"nicematin.com","origin":"https://nicematin.com","robots":{"status":"ok","http":200,"url":"https://www.nicematin.com/robots.txt","bytes":22686,"groups":179},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["partial","named",1,44],"OAI-SearchBot":["partial","named",1,44],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","named",1,44],"Claude-SearchBot":["partial","named",1,44],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["partial","named",1,44],"Perplexity-User":["partial","named",1,44],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["partial","named",1,44],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",18,0]},"llms":{"status":"timeout","error":"timed out after 10 s"},"home":{"status":"ok","http":200,"url":"https://www.nicematin.com/","html_tokens":85917,"markdown_tokens":204,"pruned_tokens":175}},
{"rank":9867,"domain":"socialblade.com","origin":"https://socialblade.com","robots":{"status":"ok","http":200,"url":"https://socialblade.com/robots.txt","bytes":33,"groups":1},"bots":{"GPTBot":["partial","*",1,0],"ChatGPT-User":["partial","*",1,0],"OAI-SearchBot":["partial","*",1,0],"ClaudeBot":["partial","*",1,0],"Claude-User":["partial","*",1,0],"Claude-SearchBot":["partial","*",1,0],"anthropic-ai":["partial","*",1,0],"PerplexityBot":["partial","*",1,0],"Perplexity-User":["partial","*",1,0],"Google-Extended":["partial","*",1,0],"Applebot-Extended":["partial","*",1,0],"CCBot":["partial","*",1,0],"Bytespider":["partial","*",1,0],"Amazonbot":["partial","*",1,0],"meta-externalagent":["partial","*",1,0],"cohere-ai":["partial","*",1,0],"Diffbot":["partial","*",1,0],"ContextFetchBot":["partial","*",1,0]},"llms":{"status":"missing","http":404,"bytes":35371},"home":{"status":"http_error","http":403,"url":"https://socialblade.com/"}},
{"rank":9868,"domain":"expertise.com","origin":"https://expertise.com","robots":{"status":"ok","http":200,"url":"https://www.expertise.com/robots.txt","bytes":285,"groups":2},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"http_error","http":403,"bytes":0},"home":{"status":"http_error","http":403,"url":"https://www.expertise.com/"}},
{"rank":9869,"domain":"txsmtyl.net","origin":"http://txsmtyl.net","robots":{"status":"missing","http":404,"url":"http://txsmtyl.net/robots.txt","bytes":4731,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":4727},"home":{"status":"ok","http":200,"url":"http://txsmtyl.net/","html_tokens":110,"markdown_tokens":0,"pruned_tokens":0}},
{"rank":9870,"domain":"bank.gov.ua","origin":"https://bank.gov.ua","robots":{"status":"ok","http":200,"url":"https://bank.gov.ua/robots.txt","bytes":165,"groups":1},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"missing","http":404,"bytes":114611},"home":{"status":"ok","http":200,"url":"https://bank.gov.ua/","html_tokens":77587,"markdown_tokens":7,"pruned_tokens":7}},
{"rank":9871,"domain":"museoreinasofia.es","origin":"https://museoreinasofia.es","robots":{"status":"ok","http":200,"url":"https://museoreinasofia.es/robots.txt","bytes":66,"groups":1},"bots":{"GPTBot":["partial","*",2,1],"ChatGPT-User":["partial","*",2,1],"OAI-SearchBot":["partial","*",2,1],"ClaudeBot":["partial","*",2,1],"Claude-User":["partial","*",2,1],"Claude-SearchBot":["partial","*",2,1],"anthropic-ai":["partial","*",2,1],"PerplexityBot":["partial","*",2,1],"Perplexity-User":["partial","*",2,1],"Google-Extended":["partial","*",2,1],"Applebot-Extended":["partial","*",2,1],"CCBot":["partial","*",2,1],"Bytespider":["partial","*",2,1],"Amazonbot":["partial","*",2,1],"meta-externalagent":["partial","*",2,1],"cohere-ai":["partial","*",2,1],"Diffbot":["partial","*",2,1],"ContextFetchBot":["partial","*",2,1]},"llms":{"status":"missing","http":404,"bytes":66632},"home":{"status":"ok","http":200,"url":"https://museoreinasofia.es/","html_tokens":26755,"markdown_tokens":655,"pruned_tokens":167}},
{"rank":9872,"domain":"tuni.fi","origin":"https://tuni.fi","robots":{"status":"missing","http":404,"url":"https://www.tuni.fi/robots.txt","bytes":120116,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":120114},"home":{"status":"ok","http":200,"url":"https://www.tuni.fi/fi","html_tokens":37009,"markdown_tokens":153,"pruned_tokens":114}},
{"rank":9873,"domain":"inist.fr","origin":"https://inist.fr","robots":{"status":"ok","http":200,"url":"https://www.inist.fr/robots.txt","bytes":170,"groups":1},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"missing","http":404,"bytes":46664},"home":{"status":"ok","http":200,"url":"https://www.inist.fr/","html_tokens":22593,"markdown_tokens":108,"pruned_tokens":108}},
{"rank":9874,"domain":"icecast.org","origin":"https://icecast.org","robots":{"status":"missing","http":404,"url":"https://icecast.org/robots.txt","bytes":146,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":146},"home":{"status":"ok","http":200,"url":"https://icecast.org/","html_tokens":2041,"markdown_tokens":910,"pruned_tokens":141}},
{"rank":9875,"domain":"huachu.com.cn","origin":"https://huachu.com.cn","robots":{"status":"unreachable","http":null,"url":"https://huachu.com.cn/robots.txt","bytes":0,"groups":0,"error":"ConnectError"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":9876,"domain":"superoffice.com","origin":"https://superoffice.com","robots":{"status":"ok","http":200,"url":"https://www.superoffice.com/robots.txt","bytes":616,"groups":1},"bots":{"GPTBot":["partial","*",5,0],"ChatGPT-User":["partial","*",5,0],"OAI-SearchBot":["partial","*",5,0],"ClaudeBot":["partial","*",5,0],"Claude-User":["partial","*",5,0],"Claude-SearchBot":["partial","*",5,0],"anthropic-ai":["partial","*",5,0],"PerplexityBot":["partial","*",5,0],"Perplexity-User":["partial","*",5,0],"Google-Extended":["partial","*",5,0],"Applebot-Extended":["partial","*",5,0],"CCBot":["partial","*",5,0],"Bytespider":["partial","*",5,0],"Amazonbot":["partial","*",5,0],"meta-externalagent":["partial","*",5,0],"cohere-ai":["partial","*",5,0],"Diffbot":["partial","*",5,0],"ContextFetchBot":["partial","*",5,0]},"llms":{"status":"missing","http":404,"bytes":115619},"home":{"status":"ok","http":200,"url":"https://www.superoffice.com/","html_tokens":44960,"markdown_tokens":1089,"pruned_tokens":441}},
{"rank":9877,"domain":"google.com.nf","origin":"https://google.com.nf","robots":{"status":"ok","http":200,"url":"https://www.google.com.nf/robots.txt","bytes":6624,"groups":4},"bots":{"GPTBot":["partial","*",169,64],"ChatGPT-User":["partial","*",169,64],"OAI-SearchBot":["partial","*",169,64],"ClaudeBot":["partial","*",169,64],"Claude-User":["partial","*",169,64],"Claude-SearchBot":["partial","*",169,64],"anthropic-ai":["partial","*",169,64],"PerplexityBot":["partial","*",169,64],"Perplexity-User":["partial","*",169,64],"Google-Extended":["partial","*",169,64],"Applebot-Extended":["partial","*",169,64],"CCBot":["partial","*",169,64],"Bytespider":["partial","*",169,64],"Amazonbot":["partial","*",169,64],"meta-externalagent":["partial","*",169,64],"cohere-ai":["partial","*",169,64],"Diffbot":["partial","*",169,64],"ContextFetchBot":["partial","*",169,64]},"llms":{"status":"missing","http":404,"bytes":1569},"home":{"status":"ok","http":200,"url":"https://www.google.com.nf/","html_tokens":21337,"markdown_tokens":32,"pruned_tokens":32}},
{"rank":9878,"domain":"theshop.jp","origin":"https://www.theshop.jp","robots":{"status":"ok","http":200,"url":"https://www.theshop.jp/robots.txt","bytes":808,"groups":3},"bots":{"GPTBot":["partial","*",6,1],"ChatGPT-User":["partial","*",6,1],"OAI-SearchBot":["partial","*",6,1],"ClaudeBot":["partial","*",6,1],"Claude-User":["partial","*",6,1],"Claude-SearchBot":["partial","*",6,1],"anthropic-ai":["partial","*",6,1],"PerplexityBot":["partial","*",6,1],"Perplexity-User":["partial","*",6,1],"Google-Extended":["partial","*",6,1],"Applebot-Extended":["partial","*",6,1],"CCBot":["partial","*",6,1],"Bytespider":["partial","*",6,1],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["partial","*",6,1],"cohere-ai":["partial","*",6,1],"Diffbot":["partial","*",6,1],"ContextFetchBot":["partial","*",6,1]},"llms":{"status":"html","http":200,"bytes":149241},"home":{"status":"ok","http":200,"url":"https://thebase.com/","html_tokens":35899,"markdown_tokens":151,"pruned_tokens":151}},
{"rank":9879,"domain":"clarionledger.com","origin":"https://clarionledger.com","robots":{"status":"ok","http":200,"url":"https://www.clarionledger.com/robots.txt","bytes":12679,"groups":288},"bots":{"GPTBot":["partial","named",1,4],"ChatGPT-User":["partial","named",1,4],"OAI-SearchBot":["partial","named",1,4],"ClaudeBot":["blocked","named",1,0],"Claude-User":["blocked","named",1,0],"Claude-SearchBot":["blocked","named",1,0],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["partial","named",1,4],"Perplexity-User":["blocked","named",1,0],"Google-Extended":["partial","named",1,4],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",2,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",32,0]},"llms":{"status":"http_error","http":402,"bytes":254},"home":{"status":"http_error","http":402,"url":"https://www.clarionledger.com/"}},
{"rank":9880,"domain":"emplifi.io","origin":"https://emplifi.io","robots":{"status":"ok","http":200,"url":"https://emplifi.io/robots.txt","bytes":397,"groups":2},"bots":{"GPTBot":["allowed","named",0,1],"ChatGPT-User":["allowed","named",0,1],"OAI-SearchBot":["allowed","named",0,1],"ClaudeBot":["allowed","named",0,1],"Claude-User":["partial","*",1,1],"Claude-SearchBot":["partial","*",1,1],"anthropic-ai":["partial","*",1,1],"PerplexityBot":["allowed","named",0,1],"Perplexity-User":["partial","*",1,1],"Google-Extended":["partial","*",1,1],"Applebot-Extended":["partial","*",1,1],"CCBot":["partial","*",1,1],"Bytespider":["partial","*",1,1],"Amazonbot":["partial","*",1,1],"meta-externalagent":["partial","*",1,1],"cohere-ai":["partial","*",1,1],"Diffbot":["partial","*",1,1],"ContextFetchBot":["partial","*",1,1]},"llms":{"status":"http_error","http":403,"bytes":5541},"home":{"status":"http_error","http":403,"url":"https://emplifi.io/"}},
{"rank":9881,"domain":"thejc.com","origin":"https://thejc.com","robots":{"status":"ok","http":200,"url":"https://www.thejc.com/robots.txt","bytes":231,"groups":1},"bots":{"GPTBot":["allowed","*",0,1],"ChatGPT-User":["allowed","*",0,1],"OAI-SearchBot":["allowed","*",0,1],"ClaudeBot":["allowed","*",0,1],"Claude-User":["allowed","*",0,1],"Claude-SearchBot":["allowed","*",0,1],"anthropic-ai":["allowed","*",0,1],"PerplexityBot":["allowed","*",0,1],"Perplexity-User":["allowed","*",0,1],"Google-Extended":["allowed","*",0,1],"Applebot-Extended":["allowed","*",0,1],"CCBot":["allowed","*",0,1],"Bytespider":["allowed","*",0,1],"Amazonbot":["allowed","*",0,1],"meta-externalagent":["allowed","*",0,1],"cohere-ai":["allowed","*",0,1],"Diffbot":["allowed","*",0,1],"ContextFetchBot":["allowed","*",0,1]},"llms":{"status":"missing","http":404,"bytes":128362},"home":{"status":"ok","http":200,"url":"https://www.thejc.com/","html_tokens":177367,"markdown_tokens":241,"pruned_tokens":110}},
{"rank":9882,"domain":"rationalwiki.org","origin":"https://rationalwiki.org","robots":{"status":"server_error","http":503,"url":"https://rationalwiki.org/robots.txt","bytes":175,"groups":0,"error":"HTTP 503"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":9883,"domain":"dblp.org","origin":"https://dblp.org","robots":{"status":"ok","http":200,"url":"https://dblp.org/robots.txt","bytes":1018,"groups":1},"bots":{"GPTBot":["partial","*",44,6],"ChatGPT-User":["partial","*",44,6],"OAI-SearchBot":["partial","*",44,6],"ClaudeBot":["partial","*",44,6],"Claude-User":["partial","*",44,6],"Claude-SearchBot":["partial","*",44,6],"anthropic-ai":["partial","*",44,6],"PerplexityBot":["partial","*",44,6],"Perplexity-User":["partial","*",44,6],"Google-Extended":["partial","*",44,6],"Applebot-Extended":["partial","*",44,6],"CCBot":["partial","*",44,6],"Bytespider":["partial","*",44,6],"Amazonbot":["partial","*",44,6],"meta-externalagent":["partial","*",44,6],"cohere-ai":["partial","*",44,6],"Diffbot":["partial","*",44,6],"ContextFetchBot":["partial","*",44,6]},"llms":{"status":"html","http":202,"bytes":3262},"home":{"status":"ok","http":202,"url":"https://dblp.org/","html_tokens":811,"markdown_tokens":29,"pruned_tokens":29}},
{"rank":9884,"domain":"bka.de","origin":"https://www.bka.de","robots":{"status":"ok","http":200,"url":"https://www.bka.de/robots.txt","bytes":685,"groups":3},"bots":{"GPTBot":["partial","*",10,4],"ChatGPT-User":["partial","*",10,4],"OAI-SearchBot":["partial","*",10,4],"ClaudeBot":["partial","*",10,4],"Claude-User":["partial","*",10,4],"Claude-SearchBot":["partial","*",10,4],"anthropic-ai":["partial","*",10,4],"PerplexityBot":["partial","*",10,4],"Perplexity-User":["partial","*",10,4],"Google-Extended":["partial","*",10,4],"Applebot-Extended":["partial","*",10,4],"CCBot":["partial","*",10,4],"Bytespider":["partial","*",10,4],"Amazonbot":["partial","*",10,4],"meta-externalagent":["partial","*",10,4],"cohere-ai":["partial","*",10,4],"Diffbot":["partial","*",10,4],"ContextFetchBot":["partial","*",10,4]},"llms":{"status":"crawl_delay","error":"Crawl-delay 180 s"},"home":{"status":"crawl_delay","error":"Crawl-delay 180 s"}},
{"rank":9885,"domain":"orlistate.online","origin":"https://orlistate.online","robots":{"status":"html","http":200,"url":"https://orlilab.com/","bytes":57052,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":9886,"domain":"geneanet.org","origin":"https://geneanet.org","robots":{"status":"refused","http":403,"url":"https://geneanet.org/robots.txt","bytes":5529,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":9887,"domain":"google.im","origin":"https://google.im","robots":{"status":"ok","http":200,"url":"https://www.google.im/robots.txt","bytes":6624,"groups":4},"bots":{"GPTBot":["partial","*",169,64],"ChatGPT-User":["partial","*",169,64],"OAI-SearchBot":["partial","*",169,64],"ClaudeBot":["partial","*",169,64],"Claude-User":["partial","*",169,64],"Claude-SearchBot":["partial","*",169,64],"anthropic-ai":["partial","*",169,64],"PerplexityBot":["partial","*",169,64],"Perplexity-User":["partial","*",169,64],"Google-Extended":["partial","*",169,64],"Applebot-Extended":["partial","*",169,64],"CCBot":["partial","*",169,64],"Bytespider":["partial","*",169,64],"Amazonbot":["partial","*",169,64],"meta-externalagent":["partial","*",169,64],"cohere-ai":["partial","*",169,64],"Diffbot":["partial","*",169,64],"ContextFetchBot":["partial","*",169,64]},"llms":{"status":"missing","http":404,"bytes":1569},"home":{"status":"ok","http":200,"url":"https://www.google.com/","html_tokens":21313,"markdown_tokens":30,"pruned_tokens":30}},
{"rank":9888,"domain":"mlsend.com","origin":"https://mlsend.com","robots":{"status":"html","http":200,"url":"https://www.mailerlite.com/","bytes":314473,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":9889,"domain":"mem.gov.cn","origin":"https://www.mem.gov.cn","robots":{"status":"missing","http":404,"url":"https://www.mem.gov.cn/robots.txt","bytes":146,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":146},"home":{"status":"ok","http":200,"url":"https://www.mem.gov.cn/","html_tokens":24621,"markdown_tokens":44,"pruned_tokens":44}},
{"rank":9890,"domain":"kfqzdfwzx.cn","origin":"http://kfqzdfwzx.cn","robots":{"status":"missing","http":404,"url":"http://kfqzdfwzx.cn/robots.txt","bytes":1231,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":1231},"home":{"status":"error","http":200,"url":"http://kfqzdfwzx.cn/","error":"unparseable HTML"}},
{"rank":9891,"domain":"allce.net","origin":"https://allce.net","robots":{"status":"ok","http":200,"url":"https://allce.net/robots.txt","bytes":485,"groups":1},"bots":{"GPTBot":["partial","*",21,0],"ChatGPT-User":["partial","*",21,0],"OAI-SearchBot":["partial","*",21,0],"ClaudeBot":["partial","*",21,0],"Claude-User":["partial","*",21,0],"Claude-SearchBot":["partial","*",21,0],"anthropic-ai":["partial","*",21,0],"PerplexityBot":["partial","*",21,0],"Perplexity-User":["partial","*",21,0],"Google-Extended":["partial","*",21,0],"Applebot-Extended":["partial","*",21,0],"CCBot":["partial","*",21,0],"Bytespider":["partial","*",21,0],"Amazonbot":["partial","*",21,0],"meta-externalagent":["partial","*",21,0],"cohere-ai":["partial","*",21,0],"Diffbot":["partial","*",21,0],"ContextFetchBot":["partial","*",21,0]},"llms":{"status":"missing","http":404,"bytes":4727},"home":{"status":"ok","http":200,"url":"https://allce.net/","html_tokens":4894,"markdown_tokens":141,"pruned_tokens":141}},
{"rank":9892,"domain":"novonordisk.com","origin":"https://novonordisk.com","robots":{"status":"ok","http":200,"url":"https://www.novonordisk.com/robots.txt","bytes":48,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":453},"home":{"status":"ok","http":200,"url":"https://www.novonordisk.com/","html_tokens":18444,"markdown_tokens":20,"pruned_tokens":20}},
{"rank":9893,"domain":"anantara.com","origin":"https://www.anantara.com","robots":{"status":"ok","http":200,"url":"https://www.anantara.com/robots.txt","bytes":180,"groups":1},"bots":{"GPTBot":["partial","*",5,1],"ChatGPT-User":["partial","*",5,1],"OAI-SearchBot":["partial","*",5,1],"ClaudeBot":["partial","*",5,1],"Claude-User":["partial","*",5,1],"Claude-SearchBot":["partial","*",5,1],"anthropic-ai":["partial","*",5,1],"PerplexityBot":["partial","*",5,1],"Perplexity-User":["partial","*",5,1],"Google-Extended":["partial","*",5,1],"Applebot-Extended":["partial","*",5,1],"CCBot":["partial","*",5,1],"Bytespider":["partial","*",5,1],"Amazonbot":["partial","*",5,1],"meta-externalagent":["partial","*",5,1],"cohere-ai":["partial","*",5,1],"Diffbot":["partial","*",5,1],"ContextFetchBot":["partial","*",5,1]},"llms":{"status":"other_text","http":200,"bytes":16976},"home":{"status":"http_error","http":403,"url":"https://www.anantara.com/"}},
{"rank":9894,"domain":"bitrix24.ru","origin":"https://bitrix24.ru","robots":{"status":"ok","http":200,"url":"https://www.bitrix24.ru/robots.txt","bytes":3594,"groups":3},"bots":{"GPTBot":["partial","named",16,5],"ChatGPT-User":["partial","named",16,5],"OAI-SearchBot":["partial","named",16,5],"ClaudeBot":["partial","named",16,5],"Claude-User":["partial","named",16,5],"Claude-SearchBot":["partial","named",16,5],"anthropic-ai":["partial","*",6,15],"PerplexityBot":["partial","named",16,5],"Perplexity-User":["partial","named",16,5],"Google-Extended":["partial","named",16,5],"Applebot-Extended":["partial","*",6,15],"CCBot":["partial","*",6,15],"Bytespider":["partial","*",6,15],"Amazonbot":["partial","*",6,15],"meta-externalagent":["partial","*",6,15],"cohere-ai":["partial","*",6,15],"Diffbot":["partial","*",6,15],"ContextFetchBot":["partial","*",6,15]},"llms":{"status":"present","http":200,"bytes":11378},"home":{"status":"ok","http":200,"url":"https://www.bitrix24.ru/","html_tokens":150215,"markdown_tokens":1337,"pruned_tokens":154}},
{"rank":9895,"domain":"rgedu.cn","origin":"https://rgedu.cn","robots":{"status":"ok","http":200,"url":"https://rgedu.cn/robots.txt","bytes":66,"groups":1},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"missing","http":404,"bytes":146},"home":{"status":"ok","http":200,"url":"https://rgedu.cn/","html_tokens":23261,"markdown_tokens":182,"pruned_tokens":175}},
{"rank":9896,"domain":"wxyz.com","origin":"https://wxyz.com","robots":{"status":"ok","http":200,"url":"https://www.wxyz.com/robots.txt","bytes":383,"groups":4},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["partial","*",5,0],"OAI-SearchBot":["partial","*",5,0],"ClaudeBot":["partial","*",5,0],"Claude-User":["partial","*",5,0],"Claude-SearchBot":["partial","*",5,0],"anthropic-ai":["partial","*",5,0],"PerplexityBot":["partial","*",5,0],"Perplexity-User":["partial","*",5,0],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["partial","*",5,0],"CCBot":["partial","*",5,0],"Bytespider":["partial","*",5,0],"Amazonbot":["partial","*",5,0],"meta-externalagent":["partial","*",5,0],"cohere-ai":["partial","*",5,0],"Diffbot":["partial","*",5,0],"ContextFetchBot":["partial","*",5,0]},"llms":{"status":"missing","http":404,"bytes":77263},"home":{"status":"ok","http":200,"url":"https://www.wxyz.com/","html_tokens":71563,"markdown_tokens":29,"pruned_tokens":29}},
{"rank":9897,"domain":"iltalehti.fi","origin":"https://iltalehti.fi","robots":{"status":"ok","http":200,"url":"https://www.iltalehti.fi/robots.txt","bytes":2879,"groups":64},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["blocked","named",1,0],"OAI-SearchBot":["blocked","named",1,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["blocked","named",1,0],"Claude-SearchBot":["blocked","named",1,0],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["blocked","named",1,0],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",1,0]},"llms":{"status":"missing","http":404,"bytes":138393},"home":{"status":"ok","http":200,"url":"https://www.iltalehti.fi/","html_tokens":87688,"markdown_tokens":244,"pruned_tokens":174}},
{"rank":9898,"domain":"lmu.edu","origin":"https://lmu.edu","robots":{"status":"ok","http":200,"url":"https://www.lmu.edu/robots.txt","bytes":316,"groups":2},"bots":{"GPTBot":["partial","*",2,0],"ChatGPT-User":["partial","*",2,0],"OAI-SearchBot":["partial","*",2,0],"ClaudeBot":["partial","*",2,0],"Claude-User":["partial","*",2,0],"Claude-SearchBot":["partial","*",2,0],"anthropic-ai":["partial","*",2,0],"PerplexityBot":["partial","*",2,0],"Perplexity-User":["partial","*",2,0],"Google-Extended":["partial","*",2,0],"Applebot-Extended":["partial","*",2,0],"CCBot":["partial","*",2,0],"Bytespider":["partial","*",2,0],"Amazonbot":["partial","*",2,0],"meta-externalagent":["partial","*",2,0],"cohere-ai":["partial","*",2,0],"Diffbot":["partial","*",2,0],"ContextFetchBot":["partial","*",2,0]},"llms":{"status":"missing","http":404,"bytes":101507},"home":{"status":"ok","http":200,"url":"https://www.lmu.edu/","html_tokens":31405,"markdown_tokens":362,"pruned_tokens":174}},
{"rank":9899,"domain":"globalspec.com","origin":"https://globalspec.com","robots":{"status":"ok","http":200,"url":"https://www.globalspec.com/robots.txt","bytes":2216,"groups":20},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["partial","*",54,2],"OAI-SearchBot":["partial","*",54,2],"ClaudeBot":["partial","*",54,2],"Claude-User":["partial","*",54,2],"Claude-SearchBot":["partial","*",54,2],"anthropic-ai":["partial","*",54,2],"PerplexityBot":["partial","*",54,2],"Perplexity-User":["partial","*",54,2],"Google-Extended":["partial","*",54,2],"Applebot-Extended":["partial","*",54,2],"CCBot":["partial","*",54,2],"Bytespider":["partial","*",54,2],"Amazonbot":["partial","*",54,2],"meta-externalagent":["partial","*",54,2],"cohere-ai":["partial","*",54,2],"Diffbot":["partial","*",54,2],"ContextFetchBot":["partial","*",54,2]},"llms":{"status":"missing","http":404,"bytes":66648},"home":{"status":"http_error","http":403,"url":"https://www.globalspec.com/"}},
{"rank":9900,"domain":"uwi.edu","origin":"https://uwi.edu","robots":{"status":"html","http":200,"url":"https://uwi.edu/robots.txt","bytes":8909,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":9901,"domain":"teamspeak.com","origin":"https://teamspeak.com","robots":{"status":"missing","http":404,"url":"https://teamspeak.com/robots.txt","bytes":41744,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":41744},"home":{"status":"ok","http":200,"url":"https://teamspeak.com/en/","html_tokens":14359,"markdown_tokens":163,"pruned_tokens":108}},
{"rank":9902,"domain":"gamma.site","origin":"https://gamma.site","robots":{"status":"ok","http":200,"url":"https://gamma.site/robots.txt","bytes":1248,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":76727},"home":{"status":"ok","http":200,"url":"https://gamma.site/","html_tokens":51140,"markdown_tokens":51,"pruned_tokens":51}},
{"rank":9903,"domain":"zapf.in","origin":"https://zapf.in","robots":{"status":"missing","http":404,"url":"https://zapf.in/robots.txt","bytes":19,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":19},"home":{"status":"ok","http":200,"url":"https://hertz.physik.hamburg/","html_tokens":4060,"markdown_tokens":1251,"pruned_tokens":671}},
{"rank":9904,"domain":"blueletterbible.org","origin":"https://blueletterbible.org","robots":{"status":"ok","http":200,"url":"https://www.blueletterbible.org/robots.txt","bytes":2190,"groups":2},"bots":{"GPTBot":["partial","*",57,1],"ChatGPT-User":["partial","*",57,1],"OAI-SearchBot":["partial","*",57,1],"ClaudeBot":["partial","*",57,1],"Claude-User":["partial","*",57,1],"Claude-SearchBot":["partial","*",57,1],"anthropic-ai":["partial","*",57,1],"PerplexityBot":["partial","*",57,1],"Perplexity-User":["partial","*",57,1],"Google-Extended":["partial","*",57,1],"Applebot-Extended":["partial","*",57,1],"CCBot":["partial","*",57,1],"Bytespider":["partial","*",57,1],"Amazonbot":["partial","*",57,1],"meta-externalagent":["partial","*",57,1],"cohere-ai":["partial","*",57,1],"Diffbot":["partial","*",57,1],"ContextFetchBot":["partial","*",57,1]},"llms":{"status":"http_error","http":502,"bytes":157},"home":{"status":"http_error","http":502,"url":"https://www.blueletterbible.org/"}},
{"rank":9905,"domain":"hntuji.com","origin":"https://hntuji.com","robots":{"status":"missing","http":404,"url":"https://hntuji.com/robots.txt","bytes":0,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":0},"home":{"status":"ok","http":200,"url":"https://hntuji.com/","html_tokens":89297,"markdown_tokens":84,"pruned_tokens":84}},
{"rank":9906,"domain":"kyoto.lg.jp","origin":"https://kyoto.lg.jp","robots":{"status":"unreachable","http":null,"url":"https://kyoto.lg.jp/robots.txt","bytes":0,"groups":0,"error":"cannot resolve kyoto.lg.jp"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":9907,"domain":"hmv.co.jp","origin":"https://www.hmv.co.jp","robots":{"status":"ok","http":200,"url":"https://www.hmv.co.jp/robots.txt","bytes":1767,"groups":7},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["blocked","named",1,0],"OAI-SearchBot":["blocked","named",1,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["blocked","named",1,0],"Claude-SearchBot":["partial","*",56,2],"anthropic-ai":["partial","*",56,2],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["partial","*",56,2],"Google-Extended":["partial","*",56,2],"Applebot-Extended":["partial","*",56,2],"CCBot":["partial","*",56,2],"Bytespider":["partial","*",56,2],"Amazonbot":["partial","*",56,2],"meta-externalagent":["partial","*",56,2],"cohere-ai":["partial","*",56,2],"Diffbot":["partial","*",56,2],"ContextFetchBot":["partial","*",56,2]},"llms":{"status":"missing","http":404,"bytes":3790},"home":{"status":"ok","http":200,"url":"https://www.hmv.co.jp/","html_tokens":85484,"markdown_tokens":5154,"pruned_tokens":171}},
{"rank":9908,"domain":"eng.br","origin":"https://eng.br","robots":{"status":"unreachable","http":null,"url":"https://eng.br/robots.txt","bytes":0,"groups":0,"error":"cannot resolve eng.br"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":9909,"domain":"xcar.com.cn","origin":"https://xcar.com.cn","robots":{"status":"ok","http":200,"url":"https://www.xcar.com.cn/robots.txt","bytes":1305,"groups":1},"bots":{"GPTBot":["partial","*",48,1],"ChatGPT-User":["partial","*",48,1],"OAI-SearchBot":["partial","*",48,1],"ClaudeBot":["partial","*",48,1],"Claude-User":["partial","*",48,1],"Claude-SearchBot":["partial","*",48,1],"anthropic-ai":["partial","*",48,1],"PerplexityBot":["partial","*",48,1],"Perplexity-User":["partial","*",48,1],"Google-Extended":["partial","*",48,1],"Applebot-Extended":["partial","*",48,1],"CCBot":["partial","*",48,1],"Bytespider":["partial","*",48,1],"Amazonbot":["partial","*",48,1],"meta-externalagent":["partial","*",48,1],"cohere-ai":["partial","*",48,1],"Diffbot":["partial","*",48,1],"ContextFetchBot":["partial","*",48,1]},"llms":{"status":"disallowed","error":"robots.txt of www.xcar.com.cn not readable (server_error)"},"home":{"status":"disallowed","error":"robots.txt of www.xcar.com.cn not readable (server_error)"}},
{"rank":9910,"domain":"aliexpress.ru","origin":"https://aliexpress.ru","robots":{"status":"ok","http":200,"url":"https://aliexpress.ru/robots.txt","bytes":5549,"groups":4},"bots":{"GPTBot":["partial","*",41,1],"ChatGPT-User":["partial","*",41,1],"OAI-SearchBot":["partial","*",41,1],"ClaudeBot":["partial","*",41,1],"Claude-User":["partial","*",41,1],"Claude-SearchBot":["partial","*",41,1],"anthropic-ai":["partial","*",41,1],"PerplexityBot":["partial","*",41,1],"Perplexity-User":["partial","*",41,1],"Google-Extended":["partial","*",41,1],"Applebot-Extended":["partial","*",41,1],"CCBot":["partial","*",41,1],"Bytespider":["partial","*",41,1],"Amazonbot":["partial","*",41,1],"meta-externalagent":["partial","*",41,1],"cohere-ai":["partial","*",41,1],"Diffbot":["partial","*",41,1],"ContextFetchBot":["partial","*",41,1]},"llms":{"status":"html","http":200,"bytes":13874},"home":{"status":"error","http":200,"url":"https://aliexpress.ru/","error":"unparseable HTML"}},
{"rank":9911,"domain":"biologists.org","origin":"https://biologists.org","robots":{"status":"timeout","http":null,"url":"https://biologists.org/robots.txt","bytes":0,"groups":0,"error":"ConnectTimeout"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":9912,"domain":"w3school.com.cn","origin":"https://w3school.com.cn","robots":{"status":"missing","http":404,"url":"https://www.w3school.com.cn/robots.txt","bytes":1163,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":1163},"home":{"status":"ok","http":200,"url":"https://www.w3school.com.cn/","html_tokens":6578,"markdown_tokens":387,"pruned_tokens":104}},
{"rank":9913,"domain":"fox5dc.com","origin":"https://fox5dc.com","robots":{"status":"ok","http":200,"url":"https://www.fox5dc.com/robots.txt","bytes":798,"groups":7},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["partial","*",11,0],"OAI-SearchBot":["partial","*",11,0],"ClaudeBot":["partial","*",11,0],"Claude-User":["partial","*",11,0],"Claude-SearchBot":["partial","*",11,0],"anthropic-ai":["partial","*",11,0],"PerplexityBot":["partial","*",11,0],"Perplexity-User":["partial","*",11,0],"Google-Extended":["partial","*",11,0],"Applebot-Extended":["partial","*",11,0],"CCBot":["partial","*",11,0],"Bytespider":["partial","*",11,0],"Amazonbot":["partial","*",11,0],"meta-externalagent":["partial","*",11,0],"cohere-ai":["partial","*",11,0],"Diffbot":["partial","*",11,0],"ContextFetchBot":["partial","*",11,0]},"llms":{"status":"missing","http":404,"bytes":0},"home":{"status":"ok","http":200,"url":"https://www.fox5dc.com/","html_tokens":154977,"markdown_tokens":287,"pruned_tokens":141}},
{"rank":9914,"domain":"google.com.pg","origin":"https://google.com.pg","robots":{"status":"ok","http":200,"url":"https://www.google.com.pg/robots.txt","bytes":6624,"groups":4},"bots":{"GPTBot":["partial","*",169,64],"ChatGPT-User":["partial","*",169,64],"OAI-SearchBot":["partial","*",169,64],"ClaudeBot":["partial","*",169,64],"Claude-User":["partial","*",169,64],"Claude-SearchBot":["partial","*",169,64],"anthropic-ai":["partial","*",169,64],"PerplexityBot":["partial","*",169,64],"Perplexity-User":["partial","*",169,64],"Google-Extended":["partial","*",169,64],"Applebot-Extended":["partial","*",169,64],"CCBot":["partial","*",169,64],"Bytespider":["partial","*",169,64],"Amazonbot":["partial","*",169,64],"meta-externalagent":["partial","*",169,64],"cohere-ai":["partial","*",169,64],"Diffbot":["partial","*",169,64],"ContextFetchBot":["partial","*",169,64]},"llms":{"status":"missing","http":404,"bytes":1569},"home":{"status":"ok","http":200,"url":"https://www.google.com/","html_tokens":21310,"markdown_tokens":30,"pruned_tokens":30}},
{"rank":9915,"domain":"bmuv.de","origin":"https://bmuv.de","robots":{"status":"ok","http":200,"url":"https://www.bundesumweltministerium.de/robots.txt","bytes":222,"groups":1},"bots":{"GPTBot":["partial","*",5,0],"ChatGPT-User":["partial","*",5,0],"OAI-SearchBot":["partial","*",5,0],"ClaudeBot":["partial","*",5,0],"Claude-User":["partial","*",5,0],"Claude-SearchBot":["partial","*",5,0],"anthropic-ai":["partial","*",5,0],"PerplexityBot":["partial","*",5,0],"Perplexity-User":["partial","*",5,0],"Google-Extended":["partial","*",5,0],"Applebot-Extended":["partial","*",5,0],"CCBot":["partial","*",5,0],"Bytespider":["partial","*",5,0],"Amazonbot":["partial","*",5,0],"meta-externalagent":["partial","*",5,0],"cohere-ai":["partial","*",5,0],"Diffbot":["partial","*",5,0],"ContextFetchBot":["partial","*",5,0]},"llms":{"status":"missing","http":404,"bytes":93329},"home":{"status":"ok","http":200,"url":"https://www.bundesumweltministerium.de/","html_tokens":48842,"markdown_tokens":145,"pruned_tokens":145}},
{"rank":9916,"domain":"residentadvisor.net","origin":"https://residentadvisor.net","robots":{"status":"ok","http":200,"url":"https://ra.co/robots.txt","bytes":1164,"groups":26},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["blocked","named",1,0],"OAI-SearchBot":["partial","*",10,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",10,0],"Claude-SearchBot":["partial","*",10,0],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["partial","*",10,0],"Google-Extended":["partial","*",10,0],"Applebot-Extended":["partial","*",10,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["partial","*",10,0],"cohere-ai":["partial","*",10,0],"Diffbot":["partial","*",10,0],"ContextFetchBot":["partial","*",10,0]},"llms":{"status":"missing","http":404,"bytes":101866},"home":{"status":"http_error","http":403,"url":"https://ra.co/"}},
{"rank":9917,"domain":"edu.gov.ru","origin":"https://edu.gov.ru","robots":{"status":"unreachable","http":null,"url":"https://edu.gov.ru/robots.txt","bytes":0,"groups":0,"error":"[SSL: CERTIFICATE_VERIFY_FAILED] certificate verify failed: self-signed certificate in certificate chain (_ssl.c:1028)"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":9918,"domain":"zjguode.com","origin":"http://zjguode.com","robots":{"status":"error","http":null,"url":"http://zjguode.com/robots.txt","bytes":0,"groups":0,"error":"more than 5 redirects"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":9919,"domain":"techpolicy.press","origin":"https://techpolicy.press","robots":{"status":"ok","http":200,"url":"https://www.techpolicy.press/robots.txt","bytes":223,"groups":1},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"html","http":200,"bytes":13367},"home":{"status":"ok","http":200,"url":"https://www.techpolicy.press/","html_tokens":79768,"markdown_tokens":60,"pruned_tokens":25}},
{"rank":9920,"domain":"www.gov.kz","origin":"https://www.gov.kz","robots":{"status":"server_error","http":500,"url":"https://www.gov.kz/robots.txt","bytes":15711,"groups":0,"error":"HTTP 500"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":9921,"domain":"cdiscount.com","origin":"https://cdiscount.com","robots":{"status":"ok","http":200,"url":"https://www.cdiscount.com/robots.txt","bytes":8269,"groups":4},"bots":{"GPTBot":["partial","*",244,4],"ChatGPT-User":["partial","*",244,4],"OAI-SearchBot":["partial","*",244,4],"ClaudeBot":["partial","*",244,4],"Claude-User":["partial","*",244,4],"Claude-SearchBot":["partial","*",244,4],"anthropic-ai":["partial","*",244,4],"PerplexityBot":["partial","*",244,4],"Perplexity-User":["partial","*",244,4],"Google-Extended":["partial","*",244,4],"Applebot-Extended":["partial","*",244,4],"CCBot":["partial","*",244,4],"Bytespider":["partial","*",244,4],"Amazonbot":["partial","*",244,4],"meta-externalagent":["partial","*",244,4],"cohere-ai":["partial","*",244,4],"Diffbot":["partial","*",244,4],"ContextFetchBot":["partial","*",244,4]},"llms":{"status":"missing","http":404,"bytes":169444},"home":{"status":"ok","http":200,"url":"https://www.cdiscount.com/","html_tokens":3490,"markdown_tokens":55,"pruned_tokens":55}},
{"rank":9922,"domain":"the7.io","origin":"https://the7.io","robots":{"status":"ok","http":200,"url":"https://the7.io/robots.txt","bytes":232,"groups":2},"bots":{"GPTBot":["partial","*",2,1],"ChatGPT-User":["partial","*",2,1],"OAI-SearchBot":["partial","*",2,1],"ClaudeBot":["partial","*",2,1],"Claude-User":["partial","*",2,1],"Claude-SearchBot":["partial","*",2,1],"anthropic-ai":["partial","*",2,1],"PerplexityBot":["partial","*",2,1],"Perplexity-User":["partial","*",2,1],"Google-Extended":["partial","*",2,1],"Applebot-Extended":["partial","*",2,1],"CCBot":["partial","*",2,1],"Bytespider":["partial","*",2,1],"Amazonbot":["partial","*",2,1],"meta-externalagent":["partial","*",2,1],"cohere-ai":["partial","*",2,1],"Diffbot":["partial","*",2,1],"ContextFetchBot":["partial","*",2,1]},"llms":{"status":"missing","http":404,"bytes":52144},"home":{"status":"ok","http":200,"url":"https://the7.io/","html_tokens":257310,"markdown_tokens":2156,"pruned_tokens":448}},
{"rank":9923,"domain":"gansu.gov.cn","origin":"https://gansu.gov.cn","robots":{"status":"missing","http":404,"url":"https://gansu.gov.cn/robots.txt","bytes":146,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":146},"home":{"status":"http_error","http":412,"url":"https://gansu.gov.cn/"}},
{"rank":9924,"domain":"w3layouts.com","origin":"https://w3layouts.com","robots":{"status":"ok","http":200,"url":"https://w3layouts.com/robots.txt","bytes":441,"groups":2},"bots":{"GPTBot":["partial","*",6,1],"ChatGPT-User":["partial","*",6,1],"OAI-SearchBot":["partial","*",6,1],"ClaudeBot":["partial","*",6,1],"Claude-User":["partial","*",6,1],"Claude-SearchBot":["partial","*",6,1],"anthropic-ai":["partial","*",6,1],"PerplexityBot":["partial","*",6,1],"Perplexity-User":["partial","*",6,1],"Google-Extended":["partial","*",6,1],"Applebot-Extended":["partial","*",6,1],"CCBot":["partial","*",6,1],"Bytespider":["partial","*",6,1],"Amazonbot":["partial","*",6,1],"meta-externalagent":["partial","*",6,1],"cohere-ai":["partial","*",6,1],"Diffbot":["partial","*",6,1],"ContextFetchBot":["partial","*",6,1]},"llms":{"status":"missing","http":404,"bytes":150453},"home":{"status":"ok","http":200,"url":"https://w3layouts.com/","html_tokens":56246,"markdown_tokens":124,"pruned_tokens":124}},
{"rank":9925,"domain":"ssh.com","origin":"https://ssh.com","robots":{"status":"ok","http":200,"url":"https://www.ssh.com/robots.txt","bytes":210,"groups":1},"bots":{"GPTBot":["partial","*",7,0],"ChatGPT-User":["partial","*",7,0],"OAI-SearchBot":["partial","*",7,0],"ClaudeBot":["partial","*",7,0],"Claude-User":["partial","*",7,0],"Claude-SearchBot":["partial","*",7,0],"anthropic-ai":["partial","*",7,0],"PerplexityBot":["partial","*",7,0],"Perplexity-User":["partial","*",7,0],"Google-Extended":["partial","*",7,0],"Applebot-Extended":["partial","*",7,0],"CCBot":["partial","*",7,0],"Bytespider":["partial","*",7,0],"Amazonbot":["partial","*",7,0],"meta-externalagent":["partial","*",7,0],"cohere-ai":["partial","*",7,0],"Diffbot":["partial","*",7,0],"ContextFetchBot":["partial","*",7,0]},"llms":{"status":"missing","http":404,"bytes":78482},"home":{"status":"ok","http":200,"url":"https://www.ssh.com/","html_tokens":47910,"markdown_tokens":837,"pruned_tokens":64}},
{"rank":9926,"domain":"sega.jp","origin":"https://sega.jp","robots":{"status":"missing","http":404,"url":"https://www.sega.jp/robots.txt","bytes":19905,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":19899},"home":{"status":"http_error","http":403,"url":"https://www.sega.jp/"}},
{"rank":9927,"domain":"svelte.dev","origin":"https://svelte.dev","robots":{"status":"ok","http":200,"url":"https://svelte.dev/robots.txt","bytes":24,"groups":1},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"present","http":200,"bytes":1676},"home":{"status":"ok","http":200,"url":"https://svelte.dev/","html_tokens":22720,"markdown_tokens":132,"pruned_tokens":132}},
{"rank":9928,"domain":"bestlifeonline.com","origin":"https://bestlifeonline.com","robots":{"status":"ok","http":200,"url":"https://bestlifeonline.com/robots.txt","bytes":3040,"groups":35},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["partial","*",4,1],"OAI-SearchBot":["partial","*",4,1],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",4,1],"Claude-SearchBot":["partial","*",4,1],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["partial","*",4,1],"Perplexity-User":["partial","*",4,1],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",4,1]},"llms":{"status":"missing","http":404,"bytes":95091},"home":{"status":"http_error","http":403,"url":"https://bestlifeonline.com/"}},
{"rank":9929,"domain":"prezly.com","origin":"https://prezly.com","robots":{"status":"ok","http":200,"url":"https://www.prezly.com/robots.txt","bytes":304,"groups":1},"bots":{"GPTBot":["partial","*",9,0],"ChatGPT-User":["partial","*",9,0],"OAI-SearchBot":["partial","*",9,0],"ClaudeBot":["partial","*",9,0],"Claude-User":["partial","*",9,0],"Claude-SearchBot":["partial","*",9,0],"anthropic-ai":["partial","*",9,0],"PerplexityBot":["partial","*",9,0],"Perplexity-User":["partial","*",9,0],"Google-Extended":["partial","*",9,0],"Applebot-Extended":["partial","*",9,0],"CCBot":["partial","*",9,0],"Bytespider":["partial","*",9,0],"Amazonbot":["partial","*",9,0],"meta-externalagent":["partial","*",9,0],"cohere-ai":["partial","*",9,0],"Diffbot":["partial","*",9,0],"ContextFetchBot":["partial","*",9,0]},"llms":{"status":"missing","http":404,"bytes":71047},"home":{"status":"ok","http":200,"url":"https://www.prezly.com/","html_tokens":43008,"markdown_tokens":610,"pruned_tokens":159}},
{"rank":9930,"domain":"kxlims.com","origin":"http://kxlims.com","robots":{"status":"refused","http":401,"url":"http://kxlims.com/robots.txt","bytes":0,"groups":0,"error":"HTTP 401"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":9931,"domain":"dfzyxx.com","origin":"http://dfzyxx.com","robots":{"status":"ok","http":200,"url":"http://dfzyxx.com/robots.txt","bytes":93,"groups":1},"bots":{"GPTBot":["partial","*",4,0],"ChatGPT-User":["partial","*",4,0],"OAI-SearchBot":["partial","*",4,0],"ClaudeBot":["partial","*",4,0],"Claude-User":["partial","*",4,0],"Claude-SearchBot":["partial","*",4,0],"anthropic-ai":["partial","*",4,0],"PerplexityBot":["partial","*",4,0],"Perplexity-User":["partial","*",4,0],"Google-Extended":["partial","*",4,0],"Applebot-Extended":["partial","*",4,0],"CCBot":["partial","*",4,0],"Bytespider":["partial","*",4,0],"Amazonbot":["partial","*",4,0],"meta-externalagent":["partial","*",4,0],"cohere-ai":["partial","*",4,0],"Diffbot":["partial","*",4,0],"ContextFetchBot":["partial","*",4,0]},"llms":{"status":"missing","http":404,"bytes":1308},"home":{"status":"ok","http":200,"url":"http://dfzyxx.com/","html_tokens":4669,"markdown_tokens":52,"pruned_tokens":24}},
{"rank":9932,"domain":"ionos.es","origin":"https://ionos.es","robots":{"status":"ok","http":200,"url":"https://www.ionos.es/robots.txt","bytes":449,"groups":2},"bots":{"GPTBot":["partial","*",8,1],"ChatGPT-User":["partial","*",8,1],"OAI-SearchBot":["partial","*",8,1],"ClaudeBot":["partial","*",8,1],"Claude-User":["partial","*",8,1],"Claude-SearchBot":["partial","*",8,1],"anthropic-ai":["partial","*",8,1],"PerplexityBot":["partial","*",8,1],"Perplexity-User":["partial","*",8,1],"Google-Extended":["partial","*",8,1],"Applebot-Extended":["partial","*",8,1],"CCBot":["partial","*",8,1],"Bytespider":["partial","*",8,1],"Amazonbot":["partial","*",8,1],"meta-externalagent":["partial","*",8,1],"cohere-ai":["partial","*",8,1],"Diffbot":["partial","*",8,1],"ContextFetchBot":["partial","*",8,1]},"llms":{"status":"missing","http":404,"bytes":621362},"home":{"status":"ok","http":200,"url":"https://www.ionos.es/","html_tokens":192636,"markdown_tokens":101,"pruned_tokens":82}},
{"rank":9933,"domain":"splashtop.com","origin":"https://splashtop.com","robots":{"status":"ok","http":200,"url":"https://www.splashtop.com/robots.txt","bytes":424,"groups":2},"bots":{"GPTBot":["partial","*",11,1],"ChatGPT-User":["partial","*",11,1],"OAI-SearchBot":["partial","*",11,1],"ClaudeBot":["partial","*",11,1],"Claude-User":["partial","*",11,1],"Claude-SearchBot":["partial","*",11,1],"anthropic-ai":["partial","*",11,1],"PerplexityBot":["partial","*",11,1],"Perplexity-User":["partial","*",11,1],"Google-Extended":["partial","*",11,1],"Applebot-Extended":["partial","*",11,1],"CCBot":["partial","*",11,1],"Bytespider":["partial","*",11,1],"Amazonbot":["partial","*",11,1],"meta-externalagent":["partial","*",11,1],"cohere-ai":["partial","*",11,1],"Diffbot":["partial","*",11,1],"ContextFetchBot":["partial","*",11,1]},"llms":{"status":"missing","http":404,"bytes":158919},"home":{"status":"ok","http":200,"url":"https://www.splashtop.com/","html_tokens":105017,"markdown_tokens":597,"pruned_tokens":158}},
{"rank":9934,"domain":"1prime.ru","origin":"https://1prime.ru","robots":{"status":"ok","http":200,"url":"https://1prime.ru/robots.txt","bytes":444,"groups":1},"bots":{"GPTBot":["partial","*",11,0],"ChatGPT-User":["partial","*",11,0],"OAI-SearchBot":["partial","*",11,0],"ClaudeBot":["partial","*",11,0],"Claude-User":["partial","*",11,0],"Claude-SearchBot":["partial","*",11,0],"anthropic-ai":["partial","*",11,0],"PerplexityBot":["partial","*",11,0],"Perplexity-User":["partial","*",11,0],"Google-Extended":["partial","*",11,0],"Applebot-Extended":["partial","*",11,0],"CCBot":["partial","*",11,0],"Bytespider":["partial","*",11,0],"Amazonbot":["partial","*",11,0],"meta-externalagent":["partial","*",11,0],"cohere-ai":["partial","*",11,0],"Diffbot":["partial","*",11,0],"ContextFetchBot":["partial","*",11,0]},"llms":{"status":"missing","http":404,"bytes":74799},"home":{"status":"ok","http":200,"url":"https://1prime.ru/","html_tokens":31048,"markdown_tokens":109,"pruned_tokens":109}},
{"rank":9935,"domain":"jotformeu.com","origin":"https://jotformeu.com","robots":{"status":"ok","http":200,"url":"https://www.jotformeu.com/robots.txt","bytes":615,"groups":4},"bots":{"GPTBot":["partial","named",1,1],"ChatGPT-User":["partial","*",1,1],"OAI-SearchBot":["partial","named",1,1],"ClaudeBot":["partial","named",1,1],"Claude-User":["partial","*",1,1],"Claude-SearchBot":["partial","*",1,1],"anthropic-ai":["partial","named",1,1],"PerplexityBot":["partial","named",1,1],"Perplexity-User":["partial","*",1,1],"Google-Extended":["partial","*",1,1],"Applebot-Extended":["partial","named",1,1],"CCBot":["partial","named",1,1],"Bytespider":["partial","named",1,1],"Amazonbot":["partial","*",1,1],"meta-externalagent":["partial","named",1,1],"cohere-ai":["partial","named",1,1],"Diffbot":["partial","named",1,1],"ContextFetchBot":["partial","*",1,1]},"llms":{"status":"disallowed"},"home":{"status":"disallowed"}},
{"rank":9936,"domain":"uni-wuppertal.de","origin":"https://uni-wuppertal.de","robots":{"status":"ok","http":200,"url":"https://www.uni-wuppertal.de/robots.txt","bytes":183,"groups":1},"bots":{"GPTBot":["partial","*",5,0],"ChatGPT-User":["partial","*",5,0],"OAI-SearchBot":["partial","*",5,0],"ClaudeBot":["partial","*",5,0],"Claude-User":["partial","*",5,0],"Claude-SearchBot":["partial","*",5,0],"anthropic-ai":["partial","*",5,0],"PerplexityBot":["partial","*",5,0],"Perplexity-User":["partial","*",5,0],"Google-Extended":["partial","*",5,0],"Applebot-Extended":["partial","*",5,0],"CCBot":["partial","*",5,0],"Bytespider":["partial","*",5,0],"Amazonbot":["partial","*",5,0],"meta-externalagent":["partial","*",5,0],"cohere-ai":["partial","*",5,0],"Diffbot":["partial","*",5,0],"ContextFetchBot":["partial","*",5,0]},"llms":{"status":"missing","http":404,"bytes":128564},"home":{"status":"unreachable","error":"Server disconnected without sending a response."}},
{"rank":9937,"domain":"delfi.lt","origin":"https://delfi.lt","robots":{"status":"ok","http":200,"url":"https://delfi.lt/robots.txt","bytes":3485,"groups":28},"bots":{"GPTBot":["partial","named",1,2],"ChatGPT-User":["partial","named",1,2],"OAI-SearchBot":["partial","named",1,2],"ClaudeBot":["partial","named",1,2],"Claude-User":["partial","*",4,1],"Claude-SearchBot":["partial","*",4,1],"anthropic-ai":["partial","named",1,2],"PerplexityBot":["partial","named",1,2],"Perplexity-User":["partial","*",4,1],"Google-Extended":["partial","named",1,2],"Applebot-Extended":["partial","named",1,2],"CCBot":["partial","named",1,2],"Bytespider":["partial","named",1,2],"Amazonbot":["partial","named",1,2],"meta-externalagent":["partial","named",1,2],"cohere-ai":["partial","named",1,2],"Diffbot":["partial","named",1,2],"ContextFetchBot":["partial","*",4,1]},"llms":{"status":"missing","http":404,"bytes":0},"home":{"status":"ok","http":200,"url":"https://www.delfi.lt/","html_tokens":120823,"markdown_tokens":441,"pruned_tokens":128}},
{"rank":9938,"domain":"hipaajournal.com","origin":"https://hipaajournal.com","robots":{"status":"ok","http":200,"url":"https://hipaajournal.com/robots.txt","bytes":461,"groups":6},"bots":{"GPTBot":["partial","*",1,0],"ChatGPT-User":["partial","*",1,0],"OAI-SearchBot":["partial","*",1,0],"ClaudeBot":["partial","*",1,0],"Claude-User":["partial","*",1,0],"Claude-SearchBot":["partial","*",1,0],"anthropic-ai":["partial","*",1,0],"PerplexityBot":["partial","*",1,0],"Perplexity-User":["partial","*",1,0],"Google-Extended":["partial","*",1,0],"Applebot-Extended":["partial","*",1,0],"CCBot":["partial","*",1,0],"Bytespider":["partial","*",1,0],"Amazonbot":["partial","*",1,0],"meta-externalagent":["partial","*",1,0],"cohere-ai":["partial","*",1,0],"Diffbot":["partial","*",1,0],"ContextFetchBot":["partial","*",1,0]},"llms":{"status":"present","http":200,"bytes":108929},"home":{"status":"http_error","http":403,"url":"https://hipaajournal.com/"}},
{"rank":9939,"domain":"roadtrippers.com","origin":"https://roadtrippers.com","robots":{"status":"ok","http":200,"url":"https://roadtrippers.com/robots.txt","bytes":301,"groups":2},"bots":{"GPTBot":["partial","*",5,1],"ChatGPT-User":["partial","*",5,1],"OAI-SearchBot":["partial","*",5,1],"ClaudeBot":["partial","*",5,1],"Claude-User":["partial","*",5,1],"Claude-SearchBot":["partial","*",5,1],"anthropic-ai":["partial","*",5,1],"PerplexityBot":["partial","*",5,1],"Perplexity-User":["partial","*",5,1],"Google-Extended":["partial","*",5,1],"Applebot-Extended":["partial","*",5,1],"CCBot":["partial","*",5,1],"Bytespider":["partial","*",5,1],"Amazonbot":["partial","*",5,1],"meta-externalagent":["partial","*",5,1],"cohere-ai":["partial","*",5,1],"Diffbot":["partial","*",5,1],"ContextFetchBot":["partial","*",5,1]},"llms":{"status":"http_error","http":403,"bytes":5569},"home":{"status":"http_error","http":403,"url":"https://roadtrippers.com/"}},
{"rank":9940,"domain":"omg.org","origin":"https://omg.org","robots":{"status":"ok","http":200,"url":"https://www.omg.org/robots.txt","bytes":1411,"groups":3},"bots":{"GPTBot":["partial","*",49,1],"ChatGPT-User":["partial","*",49,1],"OAI-SearchBot":["partial","*",49,1],"ClaudeBot":["partial","*",49,1],"Claude-User":["partial","*",49,1],"Claude-SearchBot":["partial","*",49,1],"anthropic-ai":["partial","*",49,1],"PerplexityBot":["partial","*",49,1],"Perplexity-User":["partial","*",49,1],"Google-Extended":["partial","*",49,1],"Applebot-Extended":["partial","*",49,1],"CCBot":["partial","*",49,1],"Bytespider":["partial","*",49,1],"Amazonbot":["partial","*",49,1],"meta-externalagent":["partial","*",49,1],"cohere-ai":["partial","*",49,1],"Diffbot":["partial","*",49,1],"ContextFetchBot":["partial","*",49,1]},"llms":{"status":"missing","http":404,"bytes":567},"home":{"status":"ok","http":200,"url":"https://www.omg.org/","html_tokens":19882,"markdown_tokens":87,"pruned_tokens":87}},
{"rank":9941,"domain":"0577vr.cn","origin":"http://0577vr.cn","robots":{"status":"ok","http":200,"url":"http://0577vr.cn/robots.txt","bytes":685,"groups":1},"bots":{"GPTBot":["partial","*",30,0],"ChatGPT-User":["partial","*",30,0],"OAI-SearchBot":["partial","*",30,0],"ClaudeBot":["partial","*",30,0],"Claude-User":["partial","*",30,0],"Claude-SearchBot":["partial","*",30,0],"anthropic-ai":["partial","*",30,0],"PerplexityBot":["partial","*",30,0],"Perplexity-User":["partial","*",30,0],"Google-Extended":["partial","*",30,0],"Applebot-Extended":["partial","*",30,0],"CCBot":["partial","*",30,0],"Bytespider":["partial","*",30,0],"Amazonbot":["partial","*",30,0],"meta-externalagent":["partial","*",30,0],"cohere-ai":["partial","*",30,0],"Diffbot":["partial","*",30,0],"ContextFetchBot":["partial","*",30,0]},"llms":{"status":"missing","http":404,"bytes":1231},"home":{"status":"ok","http":200,"url":"http://0577vr.cn/","html_tokens":5558,"markdown_tokens":138,"pruned_tokens":138}},
{"rank":9942,"domain":"yodobashi.com","origin":"https://yodobashi.com","robots":{"status":"unreachable","http":null,"url":"https://yodobashi.com/robots.txt","bytes":0,"groups":0,"error":"<StreamReset stream_id:1, error_code:2, remote_reset:True>"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":9943,"domain":"esmadrid.com","origin":"https://www.esmadrid.com","robots":{"status":"ok","http":200,"url":"https://www.esmadrid.com/robots.txt","bytes":6098,"groups":89},"bots":{"GPTBot":["partial","*",69,32],"ChatGPT-User":["partial","*",69,32],"OAI-SearchBot":["partial","*",69,32],"ClaudeBot":["partial","*",69,32],"Claude-User":["partial","*",69,32],"Claude-SearchBot":["partial","*",69,32],"anthropic-ai":["partial","*",69,32],"PerplexityBot":["partial","*",69,32],"Perplexity-User":["partial","*",69,32],"Google-Extended":["partial","*",69,32],"Applebot-Extended":["partial","*",69,32],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["partial","*",69,32],"Diffbot":["partial","*",69,32],"ContextFetchBot":["partial","*",69,32]},"llms":{"status":"present","http":200,"bytes":1418560},"home":{"status":"ok","http":200,"url":"https://www.esmadrid.com/","html_tokens":32455,"markdown_tokens":44,"pruned_tokens":44}},
{"rank":9944,"domain":"stackoverflow.blog","origin":"https://stackoverflow.blog","robots":{"status":"ok","http":200,"url":"https://stackoverflow.blog/robots.txt","bytes":30,"groups":1},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":11133},"home":{"status":"ok","http":200,"url":"https://stackoverflow.blog/","html_tokens":34620,"markdown_tokens":615,"pruned_tokens":69}},
{"rank":9945,"domain":"ascelibrary.org","origin":"https://ascelibrary.org","robots":{"status":"ok","http":200,"url":"https://ascelibrary.org/robots.txt","bytes":810,"groups":3},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["partial","*",18,7],"OAI-SearchBot":["partial","*",18,7],"ClaudeBot":["partial","*",18,7],"Claude-User":["partial","*",18,7],"Claude-SearchBot":["partial","*",18,7],"anthropic-ai":["partial","*",18,7],"PerplexityBot":["partial","*",18,7],"Perplexity-User":["partial","*",18,7],"Google-Extended":["partial","*",18,7],"Applebot-Extended":["partial","*",18,7],"CCBot":["partial","*",18,7],"Bytespider":["partial","*",18,7],"Amazonbot":["partial","*",18,7],"meta-externalagent":["partial","*",18,7],"cohere-ai":["partial","*",18,7],"Diffbot":["partial","*",18,7],"ContextFetchBot":["partial","*",18,7]},"llms":{"status":"http_error","http":403,"bytes":5568},"home":{"status":"http_error","http":403,"url":"https://ascelibrary.org/"}},
{"rank":9946,"domain":"unav.edu","origin":"https://unav.edu","robots":{"status":"ok","http":200,"url":"https://www.unav.edu/robots.txt","bytes":9735,"groups":1},"bots":{"GPTBot":["partial","*",152,0],"ChatGPT-User":["partial","*",152,0],"OAI-SearchBot":["partial","*",152,0],"ClaudeBot":["partial","*",152,0],"Claude-User":["partial","*",152,0],"Claude-SearchBot":["partial","*",152,0],"anthropic-ai":["partial","*",152,0],"PerplexityBot":["partial","*",152,0],"Perplexity-User":["partial","*",152,0],"Google-Extended":["partial","*",152,0],"Applebot-Extended":["partial","*",152,0],"CCBot":["partial","*",152,0],"Bytespider":["partial","*",152,0],"Amazonbot":["partial","*",152,0],"meta-externalagent":["partial","*",152,0],"cohere-ai":["partial","*",152,0],"Diffbot":["partial","*",152,0],"ContextFetchBot":["partial","*",152,0]},"llms":{"status":"html","http":200,"bytes":157684},"home":{"status":"ok","http":200,"url":"https://www.unav.edu/","html_tokens":103205,"markdown_tokens":234,"pruned_tokens":108}},
{"rank":9947,"domain":"sport-express.ru","origin":"https://sport-express.ru","robots":{"status":"ok","http":200,"url":"https://www.sport-express.ru/robots.txt","bytes":8105,"groups":3},"bots":{"GPTBot":["partial","*",10,0],"ChatGPT-User":["partial","*",10,0],"OAI-SearchBot":["partial","*",10,0],"ClaudeBot":["partial","*",10,0],"Claude-User":["partial","*",10,0],"Claude-SearchBot":["partial","*",10,0],"anthropic-ai":["partial","*",10,0],"PerplexityBot":["partial","*",10,0],"Perplexity-User":["partial","*",10,0],"Google-Extended":["partial","*",10,0],"Applebot-Extended":["partial","*",10,0],"CCBot":["partial","*",10,0],"Bytespider":["partial","*",10,0],"Amazonbot":["partial","*",10,0],"meta-externalagent":["partial","*",10,0],"cohere-ai":["partial","*",10,0],"Diffbot":["partial","*",10,0],"ContextFetchBot":["partial","*",10,0]},"llms":{"status":"http_error","http":403,"bytes":902},"home":{"status":"ok","http":200,"url":"https://www.sport-express.ru/","html_tokens":202962,"markdown_tokens":395,"pruned_tokens":295}},
{"rank":9948,"domain":"google.com.bn","origin":"https://google.com.bn","robots":{"status":"ok","http":200,"url":"https://www.google.com.bn/robots.txt","bytes":6624,"groups":4},"bots":{"GPTBot":["partial","*",169,64],"ChatGPT-User":["partial","*",169,64],"OAI-SearchBot":["partial","*",169,64],"ClaudeBot":["partial","*",169,64],"Claude-User":["partial","*",169,64],"Claude-SearchBot":["partial","*",169,64],"anthropic-ai":["partial","*",169,64],"PerplexityBot":["partial","*",169,64],"Perplexity-User":["partial","*",169,64],"Google-Extended":["partial","*",169,64],"Applebot-Extended":["partial","*",169,64],"CCBot":["partial","*",169,64],"Bytespider":["partial","*",169,64],"Amazonbot":["partial","*",169,64],"meta-externalagent":["partial","*",169,64],"cohere-ai":["partial","*",169,64],"Diffbot":["partial","*",169,64],"ContextFetchBot":["partial","*",169,64]},"llms":{"status":"missing","http":404,"bytes":1569},"home":{"status":"ok","http":200,"url":"https://www.google.com/","html_tokens":21349,"markdown_tokens":30,"pruned_tokens":30}},
{"rank":9949,"domain":"boell.org","origin":"https://boell.org","robots":{"status":"html","http":200,"url":"https://www.boell.de/en","bytes":122859,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":9950,"domain":"acortar.link","origin":"https://acortar.link","robots":{"status":"html","http":200,"url":"https://acortar.link","bytes":45067,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":9951,"domain":"ce.gov.br","origin":"https://ce.gov.br","robots":{"status":"refused","http":403,"url":"https://ce.gov.br/robots.txt","bytes":309,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":9952,"domain":"ebayinc.com","origin":"https://ebayinc.com","robots":{"status":"ok","http":200,"url":"https://ebayinc.com/robots.txt","bytes":35,"groups":1},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"http_error","http":403,"bytes":13},"home":{"status":"disallowed","error":"robots.txt of www.ebayinc.com not readable (refused)"}},
{"rank":9953,"domain":"llbean.com","origin":"https://llbean.com","robots":{"status":"ok","http":200,"url":"https://www.llbean.com/robots.txt","bytes":1972,"groups":1},"bots":{"GPTBot":["partial","*",62,0],"ChatGPT-User":["partial","*",62,0],"OAI-SearchBot":["partial","*",62,0],"ClaudeBot":["partial","*",62,0],"Claude-User":["partial","*",62,0],"Claude-SearchBot":["partial","*",62,0],"anthropic-ai":["partial","*",62,0],"PerplexityBot":["partial","*",62,0],"Perplexity-User":["partial","*",62,0],"Google-Extended":["partial","*",62,0],"Applebot-Extended":["partial","*",62,0],"CCBot":["partial","*",62,0],"Bytespider":["partial","*",62,0],"Amazonbot":["partial","*",62,0],"meta-externalagent":["partial","*",62,0],"cohere-ai":["partial","*",62,0],"Diffbot":["partial","*",62,0],"ContextFetchBot":["partial","*",62,0]},"llms":{"status":"missing","http":404,"bytes":98683},"home":{"status":"ok","http":200,"url":"https://www.llbean.com/","html_tokens":181740,"markdown_tokens":278,"pruned_tokens":144}},
{"rank":9954,"domain":"internetlivestats.com","origin":"https://internetlivestats.com","robots":{"status":"missing","http":404,"url":"https://www.internetlivestats.com/robots.txt","bytes":1251,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":1251},"home":{"status":"ok","http":200,"url":"https://www.internetlivestats.com/","html_tokens":7392,"markdown_tokens":34,"pruned_tokens":34}},
{"rank":9955,"domain":"aanda.org","origin":"https://aanda.org","robots":{"status":"ok","http":200,"url":"https://www.aanda.org/robots.txt","bytes":1084,"groups":2},"bots":{"GPTBot":["partial","*",16,0],"ChatGPT-User":["partial","*",16,0],"OAI-SearchBot":["partial","*",16,0],"ClaudeBot":["partial","*",16,0],"Claude-User":["partial","*",16,0],"Claude-SearchBot":["partial","*",16,0],"anthropic-ai":["partial","*",16,0],"PerplexityBot":["partial","*",16,0],"Perplexity-User":["partial","*",16,0],"Google-Extended":["partial","*",16,0],"Applebot-Extended":["partial","*",16,0],"CCBot":["partial","*",16,0],"Bytespider":["partial","*",16,0],"Amazonbot":["partial","*",16,0],"meta-externalagent":["partial","*",16,0],"cohere-ai":["partial","*",16,0],"Diffbot":["partial","*",16,0],"ContextFetchBot":["partial","*",16,0]},"llms":{"status":"missing","http":404,"bytes":2279},"home":{"status":"http_error","http":403,"url":"https://www.aanda.org/"}},
{"rank":9956,"domain":"first.org","origin":"https://first.org","robots":{"status":"ok","http":200,"url":"https://www.first.org/robots.txt","bytes":55,"groups":1},"bots":{"GPTBot":["partial","*",2,0],"ChatGPT-User":["partial","*",2,0],"OAI-SearchBot":["partial","*",2,0],"ClaudeBot":["partial","*",2,0],"Claude-User":["partial","*",2,0],"Claude-SearchBot":["partial","*",2,0],"anthropic-ai":["partial","*",2,0],"PerplexityBot":["partial","*",2,0],"Perplexity-User":["partial","*",2,0],"Google-Extended":["partial","*",2,0],"Applebot-Extended":["partial","*",2,0],"CCBot":["partial","*",2,0],"Bytespider":["partial","*",2,0],"Amazonbot":["partial","*",2,0],"meta-externalagent":["partial","*",2,0],"cohere-ai":["partial","*",2,0],"Diffbot":["partial","*",2,0],"ContextFetchBot":["partial","*",2,0]},"llms":{"status":"missing","http":404,"bytes":23352},"home":{"status":"ok","http":200,"url":"https://www.first.org/","html_tokens":7306,"markdown_tokens":298,"pruned_tokens":298}},
{"rank":9957,"domain":"the-decoder.com","origin":"https://the-decoder.com","robots":{"status":"ok","http":200,"url":"https://the-decoder.com/robots.txt","bytes":293,"groups":1},"bots":{"GPTBot":["partial","*",9,0],"ChatGPT-User":["partial","*",9,0],"OAI-SearchBot":["partial","*",9,0],"ClaudeBot":["partial","*",9,0],"Claude-User":["partial","*",9,0],"Claude-SearchBot":["partial","*",9,0],"anthropic-ai":["partial","*",9,0],"PerplexityBot":["partial","*",9,0],"Perplexity-User":["partial","*",9,0],"Google-Extended":["partial","*",9,0],"Applebot-Extended":["partial","*",9,0],"CCBot":["partial","*",9,0],"Bytespider":["partial","*",9,0],"Amazonbot":["partial","*",9,0],"meta-externalagent":["partial","*",9,0],"cohere-ai":["partial","*",9,0],"Diffbot":["partial","*",9,0],"ContextFetchBot":["partial","*",9,0]},"llms":{"status":"html","http":200,"bytes":151291},"home":{"status":"ok","http":200,"url":"https://the-decoder.com/","html_tokens":37812,"markdown_tokens":270,"pruned_tokens":151}},
{"rank":9958,"domain":"nightcafe.studio","origin":"https://nightcafe.studio","robots":{"status":"ok","http":200,"url":"https://nightcafe.studio/robots.txt","bytes":3658,"groups":2},"bots":{"GPTBot":["partial","*",40,23],"ChatGPT-User":["partial","*",40,23],"OAI-SearchBot":["partial","*",40,23],"ClaudeBot":["partial","*",40,23],"Claude-User":["partial","*",40,23],"Claude-SearchBot":["partial","*",40,23],"anthropic-ai":["partial","*",40,23],"PerplexityBot":["partial","*",40,23],"Perplexity-User":["partial","*",40,23],"Google-Extended":["partial","*",40,23],"Applebot-Extended":["partial","*",40,23],"CCBot":["partial","*",40,23],"Bytespider":["partial","*",40,23],"Amazonbot":["partial","*",40,23],"meta-externalagent":["partial","*",40,23],"cohere-ai":["partial","*",40,23],"Diffbot":["partial","*",40,23],"ContextFetchBot":["partial","*",40,23]},"llms":{"status":"present","http":200,"bytes":5738},"home":{"status":"http_error","http":403,"url":"https://creator.nightcafe.studio/"}},
{"rank":9959,"domain":"bcn.cl","origin":"https://bcn.cl","robots":{"status":"ok","http":200,"url":"https://c.bcn.cl/robots.txt","bytes":23,"groups":1},"bots":{"GPTBot":["allowed","*",0,1],"ChatGPT-User":["allowed","*",0,1],"OAI-SearchBot":["allowed","*",0,1],"ClaudeBot":["allowed","*",0,1],"Claude-User":["allowed","*",0,1],"Claude-SearchBot":["allowed","*",0,1],"anthropic-ai":["allowed","*",0,1],"PerplexityBot":["allowed","*",0,1],"Perplexity-User":["allowed","*",0,1],"Google-Extended":["allowed","*",0,1],"Applebot-Extended":["allowed","*",0,1],"CCBot":["allowed","*",0,1],"Bytespider":["allowed","*",0,1],"Amazonbot":["allowed","*",0,1],"meta-externalagent":["allowed","*",0,1],"cohere-ai":["allowed","*",0,1],"Diffbot":["allowed","*",0,1],"ContextFetchBot":["allowed","*",0,1]},"llms":{"status":"disallowed","error":"robots.txt of www.bcn.cl not readable (html)"},"home":{"status":"disallowed","error":"robots.txt of www.bcn.cl not readable (html)"}},
{"rank":9960,"domain":"marketwire.com","origin":"https://marketwire.com","robots":{"status":"refused","http":403,"url":"https://www.globenewswire.com/robots.txt","bytes":387,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":9961,"domain":"grabcad.com","origin":"https://grabcad.com","robots":{"status":"refused","http":403,"url":"https://grabcad.com/robots.txt","bytes":919,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":9962,"domain":"anglican.org","origin":"https://anglican.org","robots":{"status":"ok","http":200,"url":"https://anglican.org/robots.txt","bytes":94,"groups":1},"bots":{"GPTBot":["partial","*",3,0],"ChatGPT-User":["partial","*",3,0],"OAI-SearchBot":["partial","*",3,0],"ClaudeBot":["partial","*",3,0],"Claude-User":["partial","*",3,0],"Claude-SearchBot":["partial","*",3,0],"anthropic-ai":["partial","*",3,0],"PerplexityBot":["partial","*",3,0],"Perplexity-User":["partial","*",3,0],"Google-Extended":["partial","*",3,0],"Applebot-Extended":["partial","*",3,0],"CCBot":["partial","*",3,0],"Bytespider":["partial","*",3,0],"Amazonbot":["partial","*",3,0],"meta-externalagent":["partial","*",3,0],"cohere-ai":["partial","*",3,0],"Diffbot":["partial","*",3,0],"ContextFetchBot":["partial","*",3,0]},"llms":{"status":"missing","http":404,"bytes":196},"home":{"status":"ok","http":200,"url":"https://anglican.org/","html_tokens":936,"markdown_tokens":246,"pruned_tokens":246}},
{"rank":9963,"domain":"dgdkjg.com","origin":"https://dgdkjg.com","robots":{"status":"unreachable","http":null,"url":"https://dgdkjg.com/robots.txt","bytes":0,"groups":0,"error":"cannot resolve dgdkjg.com"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":9964,"domain":"kpn.com","origin":"https://kpn.com","robots":{"status":"ok","http":200,"url":"https://www.kpn.com/robots.txt","bytes":783,"groups":1},"bots":{"GPTBot":["partial","*",23,2],"ChatGPT-User":["partial","*",23,2],"OAI-SearchBot":["partial","*",23,2],"ClaudeBot":["partial","*",23,2],"Claude-User":["partial","*",23,2],"Claude-SearchBot":["partial","*",23,2],"anthropic-ai":["partial","*",23,2],"PerplexityBot":["partial","*",23,2],"Perplexity-User":["partial","*",23,2],"Google-Extended":["partial","*",23,2],"Applebot-Extended":["partial","*",23,2],"CCBot":["partial","*",23,2],"Bytespider":["partial","*",23,2],"Amazonbot":["partial","*",23,2],"meta-externalagent":["partial","*",23,2],"cohere-ai":["partial","*",23,2],"Diffbot":["partial","*",23,2],"ContextFetchBot":["partial","*",23,2]},"llms":{"status":"missing","http":404,"bytes":177704},"home":{"status":"ok","http":200,"url":"https://www.kpn.com/","html_tokens":127049,"markdown_tokens":109,"pruned_tokens":79}},
{"rank":9965,"domain":"tripadvisor.ru","origin":"https://tripadvisor.ru","robots":{"status":"ok","http":200,"url":"https://www.tripadvisor.ru/robots.txt","bytes":22767,"groups":9},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["partial","named",623,8],"OAI-SearchBot":["partial","named",623,8],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",621,8],"Claude-SearchBot":["partial","*",621,8],"anthropic-ai":["partial","*",621,8],"PerplexityBot":["partial","named",625,8],"Perplexity-User":["partial","*",621,8],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["blocked","named",1,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["blocked","named",1,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["partial","*",621,8],"ContextFetchBot":["partial","*",621,8]},"llms":{"status":"http_error","http":403,"bytes":774},"home":{"status":"http_error","http":403,"url":"https://www.tripadvisor.ru/"}},
{"rank":9966,"domain":"xxx.com","origin":"https://xxx.com","robots":{"status":"ok","http":200,"url":"https://www.xxx.com/robots.txt","bytes":0,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":4364},"home":{"status":"ok","http":200,"url":"https://www.xxx.com/","html_tokens":4801,"markdown_tokens":131,"pruned_tokens":131}},
{"rank":9967,"domain":"foodnavigator.com","origin":"https://foodnavigator.com","robots":{"status":"ok","http":200,"url":"https://www.foodnavigator.com/robots.txt","bytes":7987,"groups":5},"bots":{"GPTBot":["partial","named",3,0],"ChatGPT-User":["partial","*",3,0],"OAI-SearchBot":["partial","named",3,0],"ClaudeBot":["partial","named",3,0],"Claude-User":["partial","*",3,0],"Claude-SearchBot":["partial","*",3,0],"anthropic-ai":["partial","named",3,0],"PerplexityBot":["partial","*",3,0],"Perplexity-User":["partial","*",3,0],"Google-Extended":["partial","*",3,0],"Applebot-Extended":["partial","*",3,0],"CCBot":["blocked","named",1,0],"Bytespider":["partial","named",3,0],"Amazonbot":["partial","named",3,0],"meta-externalagent":["partial","*",3,0],"cohere-ai":["partial","*",3,0],"Diffbot":["blocked","named",1,0],"ContextFetchBot":["partial","*",3,0]},"llms":{"status":"missing","http":404,"bytes":391183},"home":{"status":"ok","http":200,"url":"https://www.foodnavigator.com/","html_tokens":608492,"markdown_tokens":7,"pruned_tokens":7}},
{"rank":9968,"domain":"cs.com.cn","origin":"https://cs.com.cn","robots":{"status":"missing","http":404,"url":"https://cs.com.cn/robots.txt","bytes":1911,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":1911},"home":{"status":"ok","http":200,"url":"https://cs.com.cn/","html_tokens":29433,"markdown_tokens":1071,"pruned_tokens":172}},
{"rank":9969,"domain":"gks.ru","origin":"http://gks.ru","robots":{"status":"refused","http":403,"url":"http://gks.ru/robots.txt","bytes":146,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":9970,"domain":"enorth.com.cn","origin":"https://enorth.com.cn","robots":{"status":"missing","http":404,"url":"https://enorth.com.cn/robots.txt","bytes":146,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"other_text","http":200,"bytes":43},"home":{"status":"ok","http":200,"url":"https://enorth.com.cn/","html_tokens":42068,"markdown_tokens":549,"pruned_tokens":63}},
{"rank":9971,"domain":"panotoday.com","origin":"https://panotoday.com","robots":{"status":"unreachable","http":null,"url":"https://panotoday.com/robots.txt","bytes":0,"groups":0,"error":"cannot resolve panotoday.com"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":9972,"domain":"qixintian.com","origin":"https://qixintian.com","robots":{"status":"timeout","http":null,"url":"https://qixintian.com/robots.txt","bytes":0,"groups":0,"error":"ConnectTimeout"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":9973,"domain":"cnews.fr","origin":"https://cnews.fr","robots":{"status":"ok","http":200,"url":"https://www.cnews.fr/robots.txt","bytes":1917,"groups":3},"bots":{"GPTBot":["partial","*",60,3],"ChatGPT-User":["partial","*",60,3],"OAI-SearchBot":["partial","*",60,3],"ClaudeBot":["partial","*",60,3],"Claude-User":["partial","*",60,3],"Claude-SearchBot":["partial","*",60,3],"anthropic-ai":["partial","*",60,3],"PerplexityBot":["partial","*",60,3],"Perplexity-User":["partial","*",60,3],"Google-Extended":["partial","*",60,3],"Applebot-Extended":["partial","*",60,3],"CCBot":["partial","*",60,3],"Bytespider":["partial","*",60,3],"Amazonbot":["partial","*",60,3],"meta-externalagent":["partial","*",60,3],"cohere-ai":["partial","*",60,3],"Diffbot":["partial","*",60,3],"ContextFetchBot":["partial","*",60,3]},"llms":{"status":"missing","http":404,"bytes":186},"home":{"status":"http_error","http":403,"url":"https://cnews.fr/"}},
{"rank":9974,"domain":"adguard.com","origin":"https://adguard.com","robots":{"status":"ok","http":200,"url":"https://adguard.com/robots.txt","bytes":4973,"groups":2},"bots":{"GPTBot":["partial","*",65,114],"ChatGPT-User":["partial","*",65,114],"OAI-SearchBot":["partial","*",65,114],"ClaudeBot":["partial","*",65,114],"Claude-User":["partial","*",65,114],"Claude-SearchBot":["partial","*",65,114],"anthropic-ai":["partial","*",65,114],"PerplexityBot":["partial","*",65,114],"Perplexity-User":["partial","*",65,114],"Google-Extended":["partial","*",65,114],"Applebot-Extended":["partial","*",65,114],"CCBot":["partial","*",65,114],"Bytespider":["partial","*",65,114],"Amazonbot":["partial","*",65,114],"meta-externalagent":["partial","*",65,114],"cohere-ai":["partial","*",65,114],"Diffbot":["partial","*",65,114],"ContextFetchBot":["partial","*",65,114]},"llms":{"status":"missing","http":404,"bytes":115923},"home":{"status":"ok","http":200,"url":"https://adguard.com/en/welcome.html","html_tokens":110116,"markdown_tokens":1659,"pruned_tokens":299}},
{"rank":9975,"domain":"swoogo.com","origin":"https://swoogo.com","robots":{"status":"missing","http":404,"url":"https://www.swoogo.com/robots.txt","bytes":2857,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":2857},"home":{"status":"http_error","http":403,"url":"https://swoogo.events/"}},
{"rank":9976,"domain":"xmind.com","origin":"https://xmind.com","robots":{"status":"ok","http":200,"url":"https://xmind.com/robots.txt","bytes":210,"groups":4},"bots":{"GPTBot":["allowed","*",0,1],"ChatGPT-User":["allowed","*",0,1],"OAI-SearchBot":["allowed","*",0,1],"ClaudeBot":["allowed","*",0,1],"Claude-User":["allowed","*",0,1],"Claude-SearchBot":["allowed","*",0,1],"anthropic-ai":["allowed","*",0,1],"PerplexityBot":["allowed","*",0,1],"Perplexity-User":["allowed","*",0,1],"Google-Extended":["allowed","*",0,1],"Applebot-Extended":["allowed","*",0,1],"CCBot":["allowed","*",0,1],"Bytespider":["allowed","*",0,1],"Amazonbot":["allowed","*",0,1],"meta-externalagent":["allowed","*",0,1],"cohere-ai":["allowed","*",0,1],"Diffbot":["allowed","*",0,1],"ContextFetchBot":["allowed","*",0,1]},"llms":{"status":"present","http":200,"bytes":23126},"home":{"status":"ok","http":200,"url":"https://xmind.com/","html_tokens":256665,"markdown_tokens":480,"pruned_tokens":80}},
{"rank":9977,"domain":"mosreg.ru","origin":"https://mosreg.ru","robots":{"status":"refused","http":403,"url":"https://mosreg.ru/robots.txt","bytes":1293,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":9978,"domain":"multiscreensite.com","origin":"https://www.multiscreensite.com","robots":{"status":"missing","http":404,"url":"https://www.multiscreensite.com/robots.txt","bytes":928,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":928},"home":{"status":"http_error","http":404,"url":"https://www.multiscreensite.com/"}},
{"rank":9979,"domain":"litespeedtech.com","origin":"https://litespeedtech.com","robots":{"status":"ok","http":200,"url":"https://litespeedtech.com/robots.txt","bytes":1587,"groups":4},"bots":{"GPTBot":["partial","*",22,0],"ChatGPT-User":["partial","*",22,0],"OAI-SearchBot":["partial","*",22,0],"ClaudeBot":["partial","*",22,0],"Claude-User":["partial","*",22,0],"Claude-SearchBot":["partial","*",22,0],"anthropic-ai":["partial","*",22,0],"PerplexityBot":["partial","*",22,0],"Perplexity-User":["partial","*",22,0],"Google-Extended":["partial","*",22,0],"Applebot-Extended":["partial","*",22,0],"CCBot":["partial","*",22,0],"Bytespider":["partial","*",22,0],"Amazonbot":["partial","*",22,0],"meta-externalagent":["partial","*",22,0],"cohere-ai":["partial","*",22,0],"Diffbot":["partial","*",22,0],"ContextFetchBot":["partial","*",22,0]},"llms":{"status":"missing","http":404,"bytes":2703},"home":{"status":"ok","http":200,"url":"https://litespeedtech.com/","html_tokens":14315,"markdown_tokens":403,"pruned_tokens":249}},
{"rank":9980,"domain":"armytimes.com","origin":"https://armytimes.com","robots":{"status":"ok","http":200,"url":"https://www.armytimes.com/robots.txt","bytes":250,"groups":1},"bots":{"GPTBot":["partial","*",3,0],"ChatGPT-User":["partial","*",3,0],"OAI-SearchBot":["partial","*",3,0],"ClaudeBot":["partial","*",3,0],"Claude-User":["partial","*",3,0],"Claude-SearchBot":["partial","*",3,0],"anthropic-ai":["partial","*",3,0],"PerplexityBot":["partial","*",3,0],"Perplexity-User":["partial","*",3,0],"Google-Extended":["partial","*",3,0],"Applebot-Extended":["partial","*",3,0],"CCBot":["partial","*",3,0],"Bytespider":["partial","*",3,0],"Amazonbot":["partial","*",3,0],"meta-externalagent":["partial","*",3,0],"cohere-ai":["partial","*",3,0],"Diffbot":["partial","*",3,0],"ContextFetchBot":["partial","*",3,0]},"llms":{"status":"missing","http":404,"bytes":65485},"home":{"status":"ok","http":200,"url":"https://www.armytimes.com/","html_tokens":139476,"markdown_tokens":274,"pruned_tokens":158}},
{"rank":9981,"domain":"jiuhuax.com","origin":"https://jiuhuax.com","robots":{"status":"unreachable","http":null,"url":"https://jiuhuax.com/robots.txt","bytes":0,"groups":0,"error":"cannot resolve jiuhuax.com"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":9982,"domain":"diyorganicz.com","origin":"https://diyorganicz.com","robots":{"status":"ok","http":200,"url":"https://diyorganicz.com/robots.txt","bytes":173,"groups":1},"bots":{"GPTBot":["allowed","*",0,0],"ChatGPT-User":["allowed","*",0,0],"OAI-SearchBot":["allowed","*",0,0],"ClaudeBot":["allowed","*",0,0],"Claude-User":["allowed","*",0,0],"Claude-SearchBot":["allowed","*",0,0],"anthropic-ai":["allowed","*",0,0],"PerplexityBot":["allowed","*",0,0],"Perplexity-User":["allowed","*",0,0],"Google-Extended":["allowed","*",0,0],"Applebot-Extended":["allowed","*",0,0],"CCBot":["allowed","*",0,0],"Bytespider":["allowed","*",0,0],"Amazonbot":["allowed","*",0,0],"meta-externalagent":["allowed","*",0,0],"cohere-ai":["allowed","*",0,0],"Diffbot":["allowed","*",0,0],"ContextFetchBot":["allowed","*",0,0]},"llms":{"status":"missing","http":404,"bytes":64333},"home":{"status":"ok","http":200,"url":"https://diyorganicz.com/","html_tokens":37519,"markdown_tokens":125,"pruned_tokens":125}},
{"rank":9983,"domain":"rg.org","origin":"https://rg.org","robots":{"status":"ok","http":200,"url":"https://rg.org/robots.txt","bytes":828,"groups":2},"bots":{"GPTBot":["partial","*",7,8],"ChatGPT-User":["partial","*",7,8],"OAI-SearchBot":["partial","*",7,8],"ClaudeBot":["partial","*",7,8],"Claude-User":["partial","*",7,8],"Claude-SearchBot":["partial","*",7,8],"anthropic-ai":["partial","*",7,8],"PerplexityBot":["partial","*",7,8],"Perplexity-User":["partial","*",7,8],"Google-Extended":["partial","*",7,8],"Applebot-Extended":["partial","*",7,8],"CCBot":["partial","*",7,8],"Bytespider":["partial","*",7,8],"Amazonbot":["partial","*",7,8],"meta-externalagent":["partial","*",7,8],"cohere-ai":["partial","*",7,8],"Diffbot":["partial","*",7,8],"ContextFetchBot":["partial","*",7,8]},"llms":{"status":"missing","http":404,"bytes":8861},"home":{"status":"ok","http":200,"url":"https://rg.org/","html_tokens":246166,"markdown_tokens":403,"pruned_tokens":57}},
{"rank":9984,"domain":"papermag.com","origin":"https://papermag.com","robots":{"status":"ok","http":200,"url":"https://www.papermag.com/robots.txt","bytes":318,"groups":1},"bots":{"GPTBot":["partial","*",3,0],"ChatGPT-User":["partial","*",3,0],"OAI-SearchBot":["partial","*",3,0],"ClaudeBot":["partial","*",3,0],"Claude-User":["partial","*",3,0],"Claude-SearchBot":["partial","*",3,0],"anthropic-ai":["partial","*",3,0],"PerplexityBot":["partial","*",3,0],"Perplexity-User":["partial","*",3,0],"Google-Extended":["partial","*",3,0],"Applebot-Extended":["partial","*",3,0],"CCBot":["partial","*",3,0],"Bytespider":["partial","*",3,0],"Amazonbot":["partial","*",3,0],"meta-externalagent":["partial","*",3,0],"cohere-ai":["partial","*",3,0],"Diffbot":["partial","*",3,0],"ContextFetchBot":["partial","*",3,0]},"llms":{"status":"missing","http":404,"bytes":182204},"home":{"status":"ok","http":200,"url":"https://www.papermag.com/","html_tokens":93780,"markdown_tokens":201,"pruned_tokens":118}},
{"rank":9985,"domain":"bamf.de","origin":"https://bamf.de","robots":{"status":"missing","http":404,"url":"https://www.bamf.de/robots.txt","bytes":51137,"groups":0},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":51548},"home":{"status":"ok","http":200,"url":"https://www.bamf.de/DE/Startseite/startseite_node.html","html_tokens":21351,"markdown_tokens":238,"pruned_tokens":81}},
{"rank":9986,"domain":"blogocial.com","origin":"https://blogocial.com","robots":{"status":"ok","http":200,"url":"https://blogocial.com/robots.txt","bytes":72,"groups":2},"bots":{"GPTBot":["partial","*",1,0],"ChatGPT-User":["partial","*",1,0],"OAI-SearchBot":["partial","*",1,0],"ClaudeBot":["partial","*",1,0],"Claude-User":["partial","*",1,0],"Claude-SearchBot":["partial","*",1,0],"anthropic-ai":["partial","*",1,0],"PerplexityBot":["partial","*",1,0],"Perplexity-User":["partial","*",1,0],"Google-Extended":["partial","*",1,0],"Applebot-Extended":["partial","*",1,0],"CCBot":["partial","*",1,0],"Bytespider":["partial","*",1,0],"Amazonbot":["partial","*",1,0],"meta-externalagent":["partial","*",1,0],"cohere-ai":["partial","*",1,0],"Diffbot":["partial","*",1,0],"ContextFetchBot":["partial","*",1,0]},"llms":{"status":"missing","http":404,"bytes":529},"home":{"status":"ok","http":200,"url":"https://blogocial.com/","html_tokens":1875,"markdown_tokens":36,"pruned_tokens":36}},
{"rank":9987,"domain":"univ-amu.fr","origin":"https://univ-amu.fr","robots":{"status":"ok","http":200,"url":"https://www.univ-amu.fr/robots.txt","bytes":2141,"groups":1},"bots":{"GPTBot":["partial","*",35,22],"ChatGPT-User":["partial","*",35,22],"OAI-SearchBot":["partial","*",35,22],"ClaudeBot":["partial","*",35,22],"Claude-User":["partial","*",35,22],"Claude-SearchBot":["partial","*",35,22],"anthropic-ai":["partial","*",35,22],"PerplexityBot":["partial","*",35,22],"Perplexity-User":["partial","*",35,22],"Google-Extended":["partial","*",35,22],"Applebot-Extended":["partial","*",35,22],"CCBot":["partial","*",35,22],"Bytespider":["partial","*",35,22],"Amazonbot":["partial","*",35,22],"meta-externalagent":["partial","*",35,22],"cohere-ai":["partial","*",35,22],"Diffbot":["partial","*",35,22],"ContextFetchBot":["partial","*",35,22]},"llms":{"status":"http_error","http":403,"bytes":146},"home":{"status":"ok","http":200,"url":"https://www.univ-amu.fr/","html_tokens":27171,"markdown_tokens":255,"pruned_tokens":168}},
{"rank":9988,"domain":"s.click.taobao.com","origin":"https://s.click.taobao.com","robots":{"status":"html","http":200,"url":"https://error.taobao.com/tbpc/error.html","bytes":1996,"groups":0},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":9989,"domain":"mastodon.online","origin":"https://mastodon.online","robots":{"status":"ok","http":200,"url":"https://mastodon.online/robots.txt","bytes":231,"groups":2},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["partial","*",3,0],"OAI-SearchBot":["partial","*",3,0],"ClaudeBot":["partial","*",3,0],"Claude-User":["partial","*",3,0],"Claude-SearchBot":["partial","*",3,0],"anthropic-ai":["partial","*",3,0],"PerplexityBot":["partial","*",3,0],"Perplexity-User":["partial","*",3,0],"Google-Extended":["partial","*",3,0],"Applebot-Extended":["partial","*",3,0],"CCBot":["partial","*",3,0],"Bytespider":["partial","*",3,0],"Amazonbot":["partial","*",3,0],"meta-externalagent":["partial","*",3,0],"cohere-ai":["partial","*",3,0],"Diffbot":["partial","*",3,0],"ContextFetchBot":["partial","*",3,0]},"llms":{"status":"missing","http":404,"bytes":1539},"home":{"status":"ok","http":200,"url":"https://mastodon.online/","html_tokens":12181,"markdown_tokens":34,"pruned_tokens":34}},
{"rank":9990,"domain":"iol.pt","origin":"https://iol.pt","robots":{"status":"ok","http":200,"url":"https://www.iol.pt/robots.txt","bytes":531,"groups":7},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["blocked","named",1,0],"OAI-SearchBot":["blocked","named",1,0],"ClaudeBot":["partial","*",4,0],"Claude-User":["partial","*",4,0],"Claude-SearchBot":["partial","*",4,0],"anthropic-ai":["partial","*",4,0],"PerplexityBot":["partial","*",4,0],"Perplexity-User":["partial","*",4,0],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["partial","*",4,0],"CCBot":["partial","*",4,0],"Bytespider":["partial","*",4,0],"Amazonbot":["partial","*",4,0],"meta-externalagent":["partial","*",4,0],"cohere-ai":["partial","*",4,0],"Diffbot":["partial","*",4,0],"ContextFetchBot":["partial","*",4,0]},"llms":{"status":"missing","http":404,"bytes":238418},"home":{"status":"ok","http":200,"url":"https://www.iol.pt/","html_tokens":91465,"markdown_tokens":1339,"pruned_tokens":37}},
{"rank":9991,"domain":"entrust.com","origin":"https://entrust.com","robots":{"status":"refused","http":403,"url":"https://entrust.com/robots.txt","bytes":5570,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":9992,"domain":"you.com","origin":"https://you.com","robots":{"status":"ok","http":200,"url":"https://you.com/robots.txt","bytes":59,"groups":1},"bots":{"GPTBot":["allowed","*",0,1],"ChatGPT-User":["allowed","*",0,1],"OAI-SearchBot":["allowed","*",0,1],"ClaudeBot":["allowed","*",0,1],"Claude-User":["allowed","*",0,1],"Claude-SearchBot":["allowed","*",0,1],"anthropic-ai":["allowed","*",0,1],"PerplexityBot":["allowed","*",0,1],"Perplexity-User":["allowed","*",0,1],"Google-Extended":["allowed","*",0,1],"Applebot-Extended":["allowed","*",0,1],"CCBot":["allowed","*",0,1],"Bytespider":["allowed","*",0,1],"Amazonbot":["allowed","*",0,1],"meta-externalagent":["allowed","*",0,1],"cohere-ai":["allowed","*",0,1],"Diffbot":["allowed","*",0,1],"ContextFetchBot":["allowed","*",0,1]},"llms":{"status":"present","http":200,"bytes":22649},"home":{"status":"ok","http":200,"url":"https://you.com/","html_tokens":93823,"markdown_tokens":237,"pruned_tokens":108}},
{"rank":9993,"domain":"givesmart.com","origin":"https://givesmart.com","robots":{"status":"ok","http":200,"url":"https://www.givesmart.com/robots.txt","bytes":102,"groups":1},"bots":{"GPTBot":["allowed","none",0,0],"ChatGPT-User":["allowed","none",0,0],"OAI-SearchBot":["allowed","none",0,0],"ClaudeBot":["allowed","none",0,0],"Claude-User":["allowed","none",0,0],"Claude-SearchBot":["allowed","none",0,0],"anthropic-ai":["allowed","none",0,0],"PerplexityBot":["allowed","none",0,0],"Perplexity-User":["allowed","none",0,0],"Google-Extended":["allowed","none",0,0],"Applebot-Extended":["allowed","none",0,0],"CCBot":["allowed","none",0,0],"Bytespider":["allowed","none",0,0],"Amazonbot":["allowed","none",0,0],"meta-externalagent":["allowed","none",0,0],"cohere-ai":["allowed","none",0,0],"Diffbot":["allowed","none",0,0],"ContextFetchBot":["allowed","none",0,0]},"llms":{"status":"missing","http":404,"bytes":60},"home":{"status":"ok","http":200,"url":"https://www.givesmart.com/","html_tokens":61163,"markdown_tokens":871,"pruned_tokens":33}},
{"rank":9994,"domain":"inpe.br","origin":"https://inpe.br","robots":{"status":"unreachable","http":null,"url":"https://inpe.br/robots.txt","bytes":0,"groups":0,"error":"cannot resolve inpe.br"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":9995,"domain":"czegoijaksieuczyc.pl","origin":"https://czegoijaksieuczyc.pl","robots":{"status":"ok","http":200,"url":"https://czegoijaksieuczyc.pl/robots.txt","bytes":44,"groups":1},"bots":{"GPTBot":["partial","*",2,0],"ChatGPT-User":["partial","*",2,0],"OAI-SearchBot":["partial","*",2,0],"ClaudeBot":["partial","*",2,0],"Claude-User":["partial","*",2,0],"Claude-SearchBot":["partial","*",2,0],"anthropic-ai":["partial","*",2,0],"PerplexityBot":["partial","*",2,0],"Perplexity-User":["partial","*",2,0],"Google-Extended":["partial","*",2,0],"Applebot-Extended":["partial","*",2,0],"CCBot":["partial","*",2,0],"Bytespider":["partial","*",2,0],"Amazonbot":["partial","*",2,0],"meta-externalagent":["partial","*",2,0],"cohere-ai":["partial","*",2,0],"Diffbot":["partial","*",2,0],"ContextFetchBot":["partial","*",2,0]},"llms":{"status":"missing","http":404,"bytes":513},"home":{"status":"ok","http":200,"url":"https://czegoijaksieuczyc.pl/","html_tokens":10937,"markdown_tokens":3475,"pruned_tokens":136}},
{"rank":9996,"domain":"ramsar.org","origin":"https://ramsar.org","robots":{"status":"refused","http":403,"url":"https://ramsar.org/robots.txt","bytes":5505,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":9997,"domain":"americastestkitchen.com","origin":"https://americastestkitchen.com","robots":{"status":"ok","http":200,"url":"https://www.americastestkitchen.com/robots.txt","bytes":1598,"groups":24},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["blocked","named",1,0],"OAI-SearchBot":["blocked","named",1,0],"ClaudeBot":["blocked","named",1,0],"Claude-User":["partial","*",17,0],"Claude-SearchBot":["partial","*",17,0],"anthropic-ai":["blocked","named",1,0],"PerplexityBot":["blocked","named",1,0],"Perplexity-User":["partial","*",17,0],"Google-Extended":["blocked","named",1,0],"Applebot-Extended":["partial","*",17,0],"CCBot":["blocked","named",1,0],"Bytespider":["blocked","named",1,0],"Amazonbot":["blocked","named",1,0],"meta-externalagent":["partial","*",17,0],"cohere-ai":["blocked","named",1,0],"Diffbot":["partial","*",17,0],"ContextFetchBot":["partial","*",17,0]},"llms":{"status":"missing","http":404,"bytes":231186},"home":{"status":"ok","http":200,"url":"https://www.americastestkitchen.com/","html_tokens":207684,"markdown_tokens":22,"pruned_tokens":22}},
{"rank":9998,"domain":"zacks.com","origin":"https://zacks.com","robots":{"status":"ok","http":200,"url":"https://www.zacks.com/robots.txt","bytes":2133,"groups":3},"bots":{"GPTBot":["blocked","named",1,0],"ChatGPT-User":["partial","*",56,0],"OAI-SearchBot":["partial","*",56,0],"ClaudeBot":["partial","*",56,0],"Claude-User":["partial","*",56,0],"Claude-SearchBot":["partial","*",56,0],"anthropic-ai":["partial","*",56,0],"PerplexityBot":["partial","*",56,0],"Perplexity-User":["partial","*",56,0],"Google-Extended":["partial","*",56,0],"Applebot-Extended":["partial","*",56,0],"CCBot":["partial","*",56,0],"Bytespider":["partial","*",56,0],"Amazonbot":["partial","*",56,0],"meta-externalagent":["partial","*",56,0],"cohere-ai":["partial","*",56,0],"Diffbot":["partial","*",56,0],"ContextFetchBot":["partial","*",56,0]},"llms":{"status":"missing","http":404,"bytes":144677},"home":{"status":"ok","http":200,"url":"https://www.zacks.com/","html_tokens":1546,"markdown_tokens":134,"pruned_tokens":134}},
{"rank":9999,"domain":"equinix.com","origin":"https://equinix.com","robots":{"status":"refused","http":403,"url":"https://www.equinix.com/robots.txt","bytes":383,"groups":0,"error":"HTTP 403"},"llms":{"status":"skipped"},"home":{"status":"skipped"}},
{"rank":10000,"domain":"fark.com","origin":"https://fark.com","robots":{"status":"ok","http":200,"url":"https://www.fark.com/robots.txt","bytes":3897,"groups":13},"bots":{"GPTBot":["partial","named",1,3],"ChatGPT-User":["partial","*",22,4],"OAI-SearchBot":["partial","*",22,4],"ClaudeBot":["partial","named",1,3],"Claude-User":["partial","*",22,4],"Claude-SearchBot":["partial","named",1,3],"anthropic-ai":["partial","*",22,4],"PerplexityBot":["partial","*",22,4],"Perplexity-User":["partial","*",22,4],"Google-Extended":["partial","named",1,3],"Applebot-Extended":["partial","named",1,3],"CCBot":["partial","named",1,3],"Bytespider":["partial","named",1,3],"Amazonbot":["partial","named",1,3],"meta-externalagent":["partial","named",1,3],"cohere-ai":["partial","*",22,4],"Diffbot":["partial","*",22,4],"ContextFetchBot":["partial","*",22,4]},"llms":{"status":"present","http":200,"bytes":5400},"home":{"status":"http_error","http":403,"url":"https://fark.com/"}}
]}